From 60f859503445b75c18adc3c7823e88e92ba1517c Mon Sep 17 00:00:00 2001 From: He Weiwei Date: Mon, 16 Mar 2020 10:20:17 +0800 Subject: [PATCH] refactor(quota): implement internal quota APIs by quota controller (#11058) 1. Use quota controller to implement the internal quota APIs. 2. The internal quota APIs can exceed the quota limitations. Signed-off-by: He Weiwei --- src/api/quota/controller.go | 30 +- src/api/quota/controller_test.go | 6 +- src/api/quota/driver/project/project.go | 10 + src/api/quota/driver/project/util.go | 5 +- src/api/quota/options.go | 38 ++ src/common/models/project.go | 11 +- src/core/api/harborapi_test.go | 5 - src/core/api/internal.go | 78 ++- src/core/api/project.go | 12 +- src/core/api/quota/chart/chart.go | 226 --------- src/core/api/quota/migrator.go | 199 -------- src/core/api/quota/registry/registry.go | 460 ------------------ src/core/main.go | 71 +-- src/pkg/quota/dao/dao.go | 28 +- src/pkg/quota/dao/dao_test.go | 69 ++- src/pkg/quota/manager.go | 15 +- src/pkg/quota/util.go | 4 +- src/pkg/quota/util_test.go | 23 +- .../project/summary/summary.component.html | 2 +- src/testing/api/quota/controller.go | 42 +- src/testing/pkg/quota/manager.go | 27 +- 21 files changed, 311 insertions(+), 1050 deletions(-) create mode 100644 src/api/quota/options.go delete mode 100644 src/core/api/quota/chart/chart.go delete mode 100644 src/core/api/quota/migrator.go delete mode 100644 src/core/api/quota/registry/registry.go diff --git a/src/api/quota/controller.go b/src/api/quota/controller.go index 8cd9c3e45..eaca3a2a2 100644 --- a/src/api/quota/controller.go +++ b/src/api/quota/controller.go @@ -53,11 +53,14 @@ type Controller interface { // Get returns quota by id Get(ctx context.Context, id int64) (*quota.Quota, error) + // GetByRef returns quota by reference object + GetByRef(ctx context.Context, reference, referenceID string) (*quota.Quota, error) + // IsEnabled returns true when quota enabled for reference object IsEnabled(ctx context.Context, reference, referenceID string) (bool, error) // Refresh refresh quota for the reference object - Refresh(ctx context.Context, reference, referenceID string) error + Refresh(ctx context.Context, reference, referenceID string, options ...Option) error // Request request resources to run f // Before run the function, it reserves the resources, @@ -94,8 +97,12 @@ func (c *controller) Get(ctx context.Context, id int64) (*quota.Quota, error) { return c.quotaMgr.Get(ctx, id) } +func (c *controller) GetByRef(ctx context.Context, reference, referenceID string) (*quota.Quota, error) { + return c.quotaMgr.GetByRef(ctx, reference, referenceID) +} + func (c *controller) IsEnabled(ctx context.Context, reference, referenceID string) (bool, error) { - d, err := quotaDriver(ctx, reference, referenceID) + d, err := Driver(ctx, reference) if err != nil { return false, err } @@ -139,7 +146,7 @@ func (c *controller) setReservedResources(ctx context.Context, reference, refere func (c *controller) reserveResources(ctx context.Context, reference, referenceID string, resources types.ResourceList) error { reserve := func(ctx context.Context) error { - q, err := c.quotaMgr.GetForUpdate(ctx, reference, referenceID) + q, err := c.quotaMgr.GetByRefForUpdate(ctx, reference, referenceID) if err != nil { return err } @@ -163,7 +170,7 @@ func (c *controller) reserveResources(ctx context.Context, reference, referenceI newReserved := types.Add(reserved, resources) newUsed := types.Add(used, newReserved) - if err := quota.IsSafe(hardLimits, used, newUsed); err != nil { + if err := quota.IsSafe(hardLimits, used, newUsed, false); err != nil { return ierror.DeniedError(nil).WithMessage("Quota exceeded when processing the request of %v", err) } @@ -180,7 +187,7 @@ func (c *controller) reserveResources(ctx context.Context, reference, referenceI func (c *controller) unreserveResources(ctx context.Context, reference, referenceID string, resources types.ResourceList) error { unreserve := func(ctx context.Context) error { - if _, err := c.quotaMgr.GetForUpdate(ctx, reference, referenceID); err != nil { + if _, err := c.quotaMgr.GetByRefForUpdate(ctx, reference, referenceID); err != nil { return err } @@ -207,14 +214,16 @@ func (c *controller) unreserveResources(ctx context.Context, reference, referenc return orm.WithTransaction(unreserve)(ctx) } -func (c *controller) Refresh(ctx context.Context, reference, referenceID string) error { - driver, err := quotaDriver(ctx, reference, referenceID) +func (c *controller) Refresh(ctx context.Context, reference, referenceID string, options ...Option) error { + driver, err := Driver(ctx, reference) if err != nil { return err } + opts := newOptions(options...) + refresh := func(ctx context.Context) error { - q, err := c.quotaMgr.GetForUpdate(ctx, reference, referenceID) + q, err := c.quotaMgr.GetByRefForUpdate(ctx, reference, referenceID) if err != nil { return err } @@ -240,7 +249,7 @@ func (c *controller) Refresh(ctx context.Context, reference, referenceID string) return fmt.Errorf("quota usage is negative for resource(s): %s", quota.PrettyPrintResourceNames(negativeUsed)) } - if err := quota.IsSafe(hardLimits, used, newUsed); err != nil { + if err := quota.IsSafe(hardLimits, used, newUsed, opts.IgnoreLimitation); err != nil { return err } @@ -277,7 +286,8 @@ func (c *controller) Request(ctx context.Context, reference, referenceID string, return c.Refresh(ctx, reference, referenceID) } -func quotaDriver(ctx context.Context, reference, referenceID string) (driver.Driver, error) { +// Driver returns quota driver for the reference +func Driver(ctx context.Context, reference string) (driver.Driver, error) { d, ok := driver.Get(reference) if !ok { return nil, fmt.Errorf("quota not support for %s", reference) diff --git a/src/api/quota/controller_test.go b/src/api/quota/controller_test.go index f74f91a81..b10bf1839 100644 --- a/src/api/quota/controller_test.go +++ b/src/api/quota/controller_test.go @@ -70,7 +70,7 @@ func (suite *ControllerTestSuite) TestReserveResources() { hardLimits := types.ResourceList{types.ResourceCount: 1} - mock.OnAnything(quotaMgr, "GetForUpdate").Return("a.Quota{Hard: hardLimits.String(), Used: types.Zero(hardLimits).String()}, nil) + mock.OnAnything(quotaMgr, "GetByRefForUpdate").Return("a.Quota{Hard: hardLimits.String(), Used: types.Zero(hardLimits).String()}, nil) ctl := &controller{quotaMgr: quotaMgr, reservedExpiration: defaultReservedExpiration} @@ -88,7 +88,7 @@ func (suite *ControllerTestSuite) TestUnreserveResources() { hardLimits := types.ResourceList{types.ResourceCount: 1} - mock.OnAnything(quotaMgr, "GetForUpdate").Return("a.Quota{Hard: hardLimits.String(), Used: types.Zero(hardLimits).String()}, nil) + mock.OnAnything(quotaMgr, "GetByRefForUpdate").Return("a.Quota{Hard: hardLimits.String(), Used: types.Zero(hardLimits).String()}, nil) ctl := &controller{quotaMgr: quotaMgr, reservedExpiration: defaultReservedExpiration} @@ -113,7 +113,7 @@ func (suite *ControllerTestSuite) TestRequest() { q := "a.Quota{Hard: hardLimits.String(), Used: types.Zero(hardLimits).String()} used := types.ResourceList{types.ResourceCount: 0} - mock.OnAnything(quotaMgr, "GetForUpdate").Return(q, nil) + mock.OnAnything(quotaMgr, "GetByRefForUpdate").Return(q, nil) mock.OnAnything(quotaMgr, "Update").Return(nil).Run(func(mock.Arguments) { q.SetUsed(used) diff --git a/src/api/quota/driver/project/project.go b/src/api/quota/driver/project/project.go index 3226b84f5..a287952cb 100644 --- a/src/api/quota/driver/project/project.go +++ b/src/api/quota/driver/project/project.go @@ -25,6 +25,7 @@ import ( "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/config" "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/pkg/q" dr "github.com/goharbor/harbor/src/pkg/quota/driver" "github.com/goharbor/harbor/src/pkg/types" @@ -45,10 +46,19 @@ type driver struct { } func (d *driver) Enabled(ctx context.Context, key string) (bool, error) { + // NOTE: every time load the new configurations from the db to get the latest configurations may have performance problem. + if err := d.cfg.Load(); err != nil { + return false, err + } return d.cfg.Get(common.QuotaPerProjectEnable).GetBool(), nil } func (d *driver) HardLimits(ctx context.Context) types.ResourceList { + // NOTE: every time load the new configurations from the db to get the latest configurations may have performance problem. + if err := d.cfg.Load(); err != nil { + log.Warningf("load configurations failed, error: %v", err) + } + return types.ResourceList{ types.ResourceCount: d.cfg.Get(common.CountPerProject).GetInt64(), types.ResourceStorage: d.cfg.Get(common.StoragePerProject).GetInt64(), diff --git a/src/api/quota/driver/project/util.go b/src/api/quota/driver/project/util.go index 78a7669e2..a3b101f78 100644 --- a/src/api/quota/driver/project/util.go +++ b/src/api/quota/driver/project/util.go @@ -16,11 +16,11 @@ package project import ( "context" - "fmt" "strconv" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" + ierror "github.com/goharbor/harbor/src/internal/error" "github.com/goharbor/harbor/src/pkg/project" "github.com/graph-gophers/dataloader" ) @@ -70,7 +70,8 @@ func getProjectsBatchFn(ctx context.Context, keys dataloader.Keys) []*dataloader for _, projectID := range projectIDs { project, ok := projectsMap[projectID] if !ok { - return handleError(fmt.Errorf("project not found, "+"project_id: %d", projectID)) + err := ierror.NotFoundError(nil).WithMessage("project %d not found", projectID) + return handleError(err) } owner, ok := ownersMap[project.OwnerID] diff --git a/src/api/quota/options.go b/src/api/quota/options.go new file mode 100644 index 000000000..fec096859 --- /dev/null +++ b/src/api/quota/options.go @@ -0,0 +1,38 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quota + +// Option option for `Refresh` method of `Controller` +type Option func(*Options) + +// Options options used by `Refresh` method of `Controller` +type Options struct { + IgnoreLimitation bool +} + +// IgnoreLimitation set IgnoreLimitation for the Options +func IgnoreLimitation(ignoreLimitation bool) func(*Options) { + return func(opts *Options) { + opts.IgnoreLimitation = ignoreLimitation + } +} + +func newOptions(options ...Option) *Options { + opts := &Options{} + for _, f := range options { + f(opts) + } + return opts +} diff --git a/src/common/models/project.go b/src/common/models/project.go index 7528a9bdc..fc324ac7e 100644 --- a/src/common/models/project.go +++ b/src/common/models/project.go @@ -192,6 +192,12 @@ func (p *Project) TableName() string { return ProjectTable } +// QuotaSummary ... +type QuotaSummary struct { + Hard types.ResourceList `json:"hard"` + Used types.ResourceList `json:"used"` +} + // ProjectSummary ... type ProjectSummary struct { RepoCount int64 `json:"repo_count"` @@ -203,8 +209,5 @@ type ProjectSummary struct { GuestCount int64 `json:"guest_count"` LimitedGuestCount int64 `json:"limited_guest_count"` - Quota struct { - Hard types.ResourceList `json:"hard"` - Used types.ResourceList `json:"used"` - } `json:"quota"` + Quota *QuotaSummary `json:"quota,omitempty"` } diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index 72071406e..90b983bfb 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -37,7 +37,6 @@ import ( testutils "github.com/goharbor/harbor/src/common/utils/test" api_models "github.com/goharbor/harbor/src/core/api/models" apimodels "github.com/goharbor/harbor/src/core/api/models" - quota "github.com/goharbor/harbor/src/core/api/quota" _ "github.com/goharbor/harbor/src/core/auth/db" _ "github.com/goharbor/harbor/src/core/auth/ldap" "github.com/goharbor/harbor/src/core/config" @@ -211,10 +210,6 @@ func init() { beego.Router("/api/projects/:pid([0-9]+)/scanner", proScannerAPI, "get:GetProjectScanner;put:SetProjectScanner") beego.Router("/api/projects/:pid([0-9]+)/scanner/candidates", proScannerAPI, "get:GetProScannerCandidates") - if err := quota.Sync(config.GlobalProjectMgr, false); err != nil { - log.Fatalf("failed to sync quota from backend: %v", err) - } - // Init user Info admin = &usrInfo{adminName, adminPwd} unknownUsr = &usrInfo{"unknown", "unknown"} diff --git a/src/core/api/internal.go b/src/core/api/internal.go index 33257d039..bdd33ea63 100644 --- a/src/core/api/internal.go +++ b/src/core/api/internal.go @@ -15,18 +15,18 @@ package api import ( - "fmt" + "context" + + o "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/api/quota" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" - common_quota "github.com/goharbor/harbor/src/common/quota" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/config" - "github.com/goharbor/harbor/src/jobservice/logger" + ierror "github.com/goharbor/harbor/src/internal/error" + "github.com/goharbor/harbor/src/internal/orm" "github.com/pkg/errors" - "strconv" - - quota "github.com/goharbor/harbor/src/core/api/quota" ) // InternalAPI handles request of harbor admin... @@ -86,7 +86,9 @@ func (ia *InternalAPI) SwitchQuota() { config.GetCfgManager().Set(common.ReadOnly, true) config.GetCfgManager().Save() } - if err := ia.ensureQuota(); err != nil { + + ctx := orm.NewContext(ia.Ctx.Request.Context(), o.NewOrm()) + if err := ia.refreshQuotas(ctx); err != nil { ia.SendInternalServerError(err) return } @@ -99,50 +101,33 @@ func (ia *InternalAPI) SwitchQuota() { return } -func (ia *InternalAPI) ensureQuota() error { +func (ia *InternalAPI) refreshQuotas(ctx context.Context) error { + driver, err := quota.Driver(ctx, quota.ProjectReference) + if err != nil { + return err + } + projects, err := dao.GetProjects(nil) if err != nil { return err } - for _, project := range projects { - pSize, err := dao.CountSizeOfProject(project.ProjectID) - if err != nil { - logger.Warningf("error happen on counting size of project:%d , error:%v, just skip it.", project.ProjectID, err) - continue - } - afQuery := &models.ArtifactQuery{ - PID: project.ProjectID, - } - afs, err := dao.ListArtifacts(afQuery) - if err != nil { - logger.Warningf("error happen on counting number of project:%d , error:%v, just skip it.", project.ProjectID, err) - continue - } - pCount := int64(len(afs)) - // it needs to append the chart count - if config.WithChartMuseum() { - count, err := chartController.GetCountOfCharts([]string{project.Name}) - if err != nil { - err = errors.Wrap(err, fmt.Sprintf("get chart count of project %d failed", project.ProjectID)) - logger.Error(err) + for _, project := range projects { + referenceID := quota.ReferenceID(project.ProjectID) + + _, err := quota.Ctl.GetByRef(ctx, quota.ProjectReference, referenceID) + if ierror.IsNotFoundErr(err) { + if _, err := quota.Ctl.Create(ctx, quota.ProjectReference, referenceID, driver.HardLimits(ctx)); err != nil { + log.Warningf("initialize quota for project %s failed, error: %v", project.Name, err) continue } - pCount = pCount + int64(count) + } else if err != nil { + log.Warningf("get quota of the project %s failed, error: %v", project.Name, err) + continue } - quotaMgr, err := common_quota.NewManager("project", strconv.FormatInt(project.ProjectID, 10)) - if err != nil { - logger.Errorf("Error occurred when to new quota manager %v, just skip it.", err) - continue - } - used := common_quota.ResourceList{ - common_quota.ResourceStorage: pSize, - common_quota.ResourceCount: pCount, - } - if err := quotaMgr.EnsureQuota(used); err != nil { - logger.Errorf("cannot ensure quota for the project: %d, err: %v, just skip it.", project.ProjectID, err) - continue + if err := quota.Ctl.Refresh(ctx, quota.ProjectReference, referenceID, quota.IgnoreLimitation(true)); err != nil { + log.Warningf("refresh quota usage for project %s failed, error: %v", project.Name, err) } } return nil @@ -150,6 +135,11 @@ func (ia *InternalAPI) ensureQuota() error { // SyncQuota ... func (ia *InternalAPI) SyncQuota() { + if !config.QuotaPerProjectEnable() { + ia.SendError(ierror.ForbiddenError(nil).WithMessage("quota per project is disabled")) + return + } + cur := config.ReadOnly() cfgMgr := config.GetCfgManager() if !cur { @@ -163,8 +153,8 @@ func (ia *InternalAPI) SyncQuota() { cfgMgr.Save() }() log.Info("start to sync quota(API), the system will be set to ReadOnly and back it normal once it done.") - // As the sync function ignores all of duplicate error, it's safe to enable persist DB. - err := quota.Sync(ia.ProjectMgr, true) + ctx := orm.NewContext(context.TODO(), o.NewOrm()) + err := ia.refreshQuotas(ctx) if err != nil { log.Errorf("fail to sync quota(API), but with error: %v, please try to do it again.", err) return diff --git a/src/core/api/project.go b/src/core/api/project.go index f8b2e25aa..830c9ba48 100644 --- a/src/core/api/project.go +++ b/src/core/api/project.go @@ -16,6 +16,12 @@ package api import ( "fmt" + "net/http" + "regexp" + "strconv" + "strings" + "sync" + "github.com/goharbor/harbor/src/api/event" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" @@ -31,11 +37,6 @@ import ( "github.com/goharbor/harbor/src/pkg/scan/vuln" "github.com/goharbor/harbor/src/pkg/types" "github.com/pkg/errors" - "net/http" - "regexp" - "strconv" - "strings" - "sync" ) type deletableResp struct { @@ -657,6 +658,7 @@ func getProjectQuotaSummary(projectID int64, summary *models.ProjectSummary) { quota := quotas[0] + summary.Quota = &models.QuotaSummary{} summary.Quota.Hard, _ = types.NewResourceList(quota.Hard) summary.Quota.Used, _ = types.NewResourceList(quota.Used) } diff --git a/src/core/api/quota/chart/chart.go b/src/core/api/quota/chart/chart.go deleted file mode 100644 index 23175480e..000000000 --- a/src/core/api/quota/chart/chart.go +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package chart - -import ( - "fmt" - "github.com/goharbor/harbor/src/chartserver" - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" - common_quota "github.com/goharbor/harbor/src/common/quota" - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/core/api" - quota "github.com/goharbor/harbor/src/core/api/quota" - "github.com/goharbor/harbor/src/core/config" - "github.com/goharbor/harbor/src/core/promgr" - "github.com/pkg/errors" - "net/url" - "strings" - "sync" -) - -// Migrator ... -type Migrator struct { - pm promgr.ProjectManager -} - -// NewChartMigrator returns a new RegistryMigrator. -func NewChartMigrator(pm promgr.ProjectManager) quota.QuotaMigrator { - migrator := Migrator{ - pm: pm, - } - return &migrator -} - -var ( - controller *chartserver.Controller - controllerErr error - controllerOnce sync.Once -) - -// Ping ... -func (rm *Migrator) Ping() error { - return quota.Check(api.HealthCheckerRegistry["chartmuseum"].Check) -} - -// Dump ... -// Depends on DB to dump chart data, as chart cannot get all of namespaces. -func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) { - var ( - projects []quota.ProjectInfo - wg sync.WaitGroup - err error - ) - - all, err := dao.GetProjects(nil) - if err != nil { - return nil, err - } - - wg.Add(len(all)) - errChan := make(chan error, 1) - infoChan := make(chan interface{}) - done := make(chan bool, 1) - - go func() { - defer func() { - done <- true - }() - - for { - select { - case result := <-infoChan: - if result == nil { - return - } - project, ok := result.(quota.ProjectInfo) - if ok { - projects = append(projects, project) - } - - case e := <-errChan: - if err == nil { - err = errors.Wrap(e, "quota sync error on getting info of project") - } else { - err = errors.Wrap(e, err.Error()) - } - } - } - }() - - for _, project := range all { - go func(project *models.Project) { - defer wg.Done() - - var repos []quota.RepoData - ctr, err := chartController() - if err != nil { - errChan <- err - return - } - - chartInfo, err := ctr.ListCharts(project.Name) - if err != nil { - errChan <- err - return - } - - // repo - for _, chart := range chartInfo { - var afs []*models.Artifact - chartVersions, err := ctr.GetChart(project.Name, chart.Name) - if err != nil { - errChan <- err - continue - } - for _, chart := range chartVersions { - af := &models.Artifact{ - PID: project.ProjectID, - Repo: chart.Name, - Tag: chart.Version, - Digest: chart.Digest, - Kind: "Chart", - } - afs = append(afs, af) - } - repoData := quota.RepoData{ - Name: project.Name, - Afs: afs, - } - repos = append(repos, repoData) - } - - projectInfo := quota.ProjectInfo{ - Name: project.Name, - Repos: repos, - } - - infoChan <- projectInfo - }(project) - } - - wg.Wait() - close(infoChan) - - <-done - - if err != nil { - return nil, err - } - - return projects, nil -} - -// Usage ... -// Chart will not cover size. -func (rm *Migrator) Usage(projects []quota.ProjectInfo) ([]quota.ProjectUsage, error) { - var pros []quota.ProjectUsage - for _, project := range projects { - var count int64 - // usage count - for _, repo := range project.Repos { - count = count + int64(len(repo.Afs)) - } - proUsage := quota.ProjectUsage{ - Project: project.Name, - Used: common_quota.ResourceList{ - common_quota.ResourceCount: count, - common_quota.ResourceStorage: 0, - }, - } - pros = append(pros, proUsage) - } - return pros, nil - -} - -// Persist ... -// Chart will not persist data into db. -func (rm *Migrator) Persist(projects []quota.ProjectInfo) error { - return nil -} - -func chartController() (*chartserver.Controller, error) { - controllerOnce.Do(func() { - addr, err := config.GetChartMuseumEndpoint() - if err != nil { - controllerErr = fmt.Errorf("failed to get the endpoint URL of chart storage server: %s", err.Error()) - return - } - - addr = strings.TrimSuffix(addr, "/") - url, err := url.Parse(addr) - if err != nil { - controllerErr = errors.New("endpoint URL of chart storage server is malformed") - return - } - - ctr, err := chartserver.NewController(url) - if err != nil { - controllerErr = errors.New("failed to initialize chart API controller") - } - - controller = ctr - - log.Debugf("Chart storage server is set to %s", url.String()) - log.Info("API controller for chart repository server is successfully initialized") - }) - - return controller, controllerErr -} - -func init() { - quota.Register("chart", NewChartMigrator) -} diff --git a/src/core/api/quota/migrator.go b/src/core/api/quota/migrator.go deleted file mode 100644 index 6d21a8c26..000000000 --- a/src/core/api/quota/migrator.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/quota" - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/core/config" - "github.com/goharbor/harbor/src/core/promgr" - "github.com/goharbor/harbor/src/pkg/types" - "strconv" - "time" -) - -// QuotaMigrator ... -type QuotaMigrator interface { - // Ping validates and wait for backend service ready. - Ping() error - - // Dump exports all data from backend service, registry, chartmuseum - Dump() ([]ProjectInfo, error) - - // Usage computes the quota usage of all the projects - Usage([]ProjectInfo) ([]ProjectUsage, error) - - // Persist record the data to DB, artifact, artifact_blob and blob table. - Persist([]ProjectInfo) error -} - -// ProjectInfo ... -type ProjectInfo struct { - Name string - Repos []RepoData -} - -// RepoData ... -type RepoData struct { - Name string - Afs []*models.Artifact - Afnbs []*models.ArtifactAndBlob - Blobs []*models.Blob -} - -// ProjectUsage ... -type ProjectUsage struct { - Project string - Used quota.ResourceList -} - -// Instance ... -type Instance func(promgr.ProjectManager) QuotaMigrator - -var adapters = make(map[string]Instance) - -// Register ... -func Register(name string, adapter Instance) { - if adapter == nil { - panic("quota: Register adapter is nil") - } - if _, ok := adapters[name]; ok { - panic("quota: Register called twice for adapter " + name) - } - adapters[name] = adapter -} - -// Sync ... -func Sync(pm promgr.ProjectManager, populate bool) error { - totalUsage := make(map[string][]ProjectUsage) - for name, instanceFunc := range adapters { - if !config.WithChartMuseum() { - if name == "chart" { - continue - } - } - adapter := instanceFunc(pm) - log.Infof("[Quota-Sync]:: start to ping server ... [%s]", name) - if err := adapter.Ping(); err != nil { - log.Infof("[Quota-Sync]:: fail to ping server ... [%s], quit sync ...", name) - return err - } - log.Infof("[Quota-Sync]:: success to ping server ... [%s]", name) - log.Infof("[Quota-Sync]:: start to dump data from server ... [%s]", name) - data, err := adapter.Dump() - if err != nil { - log.Infof("[Quota-Sync]:: fail to dump data from server ... [%s], quit sync ...", name) - return err - } - log.Infof("[Quota-Sync]:: success to dump data from server ... [%s]", name) - usage, err := adapter.Usage(data) - if err != nil { - return err - } - totalUsage[name] = usage - if populate { - log.Infof("[Quota-Sync]:: start to persist data for server ... [%s]", name) - if err := adapter.Persist(data); err != nil { - log.Infof("[Quota-Sync]:: fail to persist data from server ... [%s], quit sync ...", name) - return err - } - log.Infof("[Quota-Sync]:: success to persist data for server ... [%s]", name) - } - } - merged := mergeUsage(totalUsage) - if err := ensureQuota(merged); err != nil { - return err - } - return nil -} - -// Check ... -func Check(f func() error) error { - return retry(10, 2*time.Second, f) -} - -// mergeUsage merges the usage of adapters -func mergeUsage(total map[string][]ProjectUsage) []ProjectUsage { - if !config.WithChartMuseum() { - return total["registry"] - } - regUsgs := total["registry"] - chartUsgs := total["chart"] - - var mergedUsage []ProjectUsage - temp := make(map[string]quota.ResourceList) - - for _, regUsg := range regUsgs { - _, exist := temp[regUsg.Project] - if !exist { - temp[regUsg.Project] = regUsg.Used - mergedUsage = append(mergedUsage, ProjectUsage{ - Project: regUsg.Project, - Used: regUsg.Used, - }) - } - } - for _, chartUsg := range chartUsgs { - var usedTemp quota.ResourceList - _, exist := temp[chartUsg.Project] - if !exist { - usedTemp = chartUsg.Used - } else { - usedTemp = types.Add(temp[chartUsg.Project], chartUsg.Used) - } - temp[chartUsg.Project] = usedTemp - mergedUsage = append(mergedUsage, ProjectUsage{ - Project: chartUsg.Project, - Used: usedTemp, - }) - } - return mergedUsage -} - -// ensureQuota updates the quota and quota usage in the data base. -func ensureQuota(usages []ProjectUsage) error { - var pid int64 - for _, usage := range usages { - project, err := dao.GetProjectByName(usage.Project) - if err != nil { - log.Error(err) - return err - } - pid = project.ProjectID - quotaMgr, err := quota.NewManager("project", strconv.FormatInt(pid, 10)) - if err != nil { - log.Errorf("Error occurred when to new quota manager %v", err) - return err - } - if err := quotaMgr.EnsureQuota(usage.Used); err != nil { - log.Errorf("cannot ensure quota for the project: %d, err: %v", pid, err) - return err - } - } - return nil -} - -func retry(attempts int, sleep time.Duration, f func() error) error { - if err := f(); err != nil { - if attempts--; attempts > 0 { - time.Sleep(sleep) - return retry(attempts, sleep, f) - } - return err - } - return nil -} diff --git a/src/core/api/quota/registry/registry.go b/src/core/api/quota/registry/registry.go deleted file mode 100644 index 2dfc7c607..000000000 --- a/src/core/api/quota/registry/registry.go +++ /dev/null @@ -1,460 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package registry - -import ( - "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" - common_quota "github.com/goharbor/harbor/src/common/quota" - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/core/api" - quota "github.com/goharbor/harbor/src/core/api/quota" - "github.com/goharbor/harbor/src/core/promgr" - "github.com/goharbor/harbor/src/pkg/registry" - "github.com/pkg/errors" - "strings" - "sync" - "time" -) - -// Migrator ... -type Migrator struct { - pm promgr.ProjectManager -} - -// NewRegistryMigrator returns a new Migrator. -func NewRegistryMigrator(pm promgr.ProjectManager) quota.QuotaMigrator { - migrator := Migrator{ - pm: pm, - } - return &migrator -} - -// Ping ... -func (rm *Migrator) Ping() error { - return quota.Check(api.HealthCheckerRegistry["registry"].Check) -} - -// Dump ... -func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) { - var ( - projects []quota.ProjectInfo - wg sync.WaitGroup - err error - ) - - reposInRegistry, err := registry.Cli.Catalog() - if err != nil { - return nil, err - } - - // repoMap : map[project_name : []repo list] - repoMap := make(map[string][]string) - for _, item := range reposInRegistry { - projectName := strings.Split(item, "/")[0] - pro, err := rm.pm.Get(projectName) - if err != nil { - log.Errorf("failed to get project %s: %v", projectName, err) - continue - } - if pro == nil { - continue - } - _, exist := repoMap[pro.Name] - if !exist { - repoMap[pro.Name] = []string{item} - } else { - repos := repoMap[pro.Name] - repos = append(repos, item) - repoMap[pro.Name] = repos - } - } - repoMap, err = rm.appendEmptyProject(repoMap) - if err != nil { - log.Errorf("fail to add empty projects: %v", err) - return nil, err - } - - wg.Add(len(repoMap)) - errChan := make(chan error, 1) - infoChan := make(chan interface{}) - done := make(chan bool, 1) - - go func() { - defer func() { - done <- true - }() - - for { - select { - case result := <-infoChan: - if result == nil { - return - } - project, ok := result.(quota.ProjectInfo) - if ok { - projects = append(projects, project) - } - - case e := <-errChan: - if err == nil { - err = errors.Wrap(e, "quota sync error on getting info of project") - } else { - err = errors.Wrap(e, err.Error()) - } - } - } - }() - - for project, repos := range repoMap { - go func(project string, repos []string) { - defer wg.Done() - info, err := infoOfProject(project, repos) - if err != nil { - errChan <- err - return - } - infoChan <- info - }(project, repos) - } - - wg.Wait() - close(infoChan) - - // wait for all of project info - <-done - - if err != nil { - return nil, err - } - - return projects, nil -} - -// As catalog api cannot list the empty projects in harbor, here it needs to append the empty projects into repo infor -// so that quota syncer can add 0 usage into quota usage. -func (rm *Migrator) appendEmptyProject(repoMap map[string][]string) (map[string][]string, error) { - var withEmptyProjects map[string][]string - all, err := dao.GetProjects(nil) - if err != nil { - return withEmptyProjects, err - } - withEmptyProjects = repoMap - for _, pro := range all { - _, exist := repoMap[pro.Name] - if !exist { - withEmptyProjects[pro.Name] = []string{} - } - } - return withEmptyProjects, nil -} - -// Usage ... -// registry needs to merge the shard blobs of different repositories. -func (rm *Migrator) Usage(projects []quota.ProjectInfo) ([]quota.ProjectUsage, error) { - var pros []quota.ProjectUsage - - for _, project := range projects { - var size, count int64 - var blobs = make(map[string]int64) - - // usage count - for _, repo := range project.Repos { - count = count + int64(len(repo.Afs)) - // Because that there are some shared blobs between repositories, it needs to remove the duplicate items. - for _, blob := range repo.Blobs { - _, exist := blobs[blob.Digest] - // foreign blob won't be calculated - if !exist && blob.ContentType != common.ForeignLayer { - blobs[blob.Digest] = blob.Size - } - } - } - // size - for _, item := range blobs { - size = size + item - } - - proUsage := quota.ProjectUsage{ - Project: project.Name, - Used: common_quota.ResourceList{ - common_quota.ResourceCount: count, - common_quota.ResourceStorage: size, - }, - } - pros = append(pros, proUsage) - } - - return pros, nil -} - -// Persist ... -func (rm *Migrator) Persist(projects []quota.ProjectInfo) error { - total := len(projects) - for i, project := range projects { - log.Infof("[Quota-Sync]:: start to persist artifact&blob for project: %s, progress... [%d/%d]", project.Name, i, total) - for _, repo := range project.Repos { - if err := persistAf(repo.Afs); err != nil { - return err - } - if err := persistAfnbs(repo.Afnbs); err != nil { - return err - } - if err := persistBlob(repo.Blobs); err != nil { - return err - } - } - log.Infof("[Quota-Sync]:: success to persist artifact&blob for project: %s, progress... [%d/%d]", project.Name, i, total) - } - if err := persistPB(projects); err != nil { - return err - } - return nil -} - -func persistAf(afs []*models.Artifact) error { - if len(afs) != 0 { - for _, af := range afs { - _, err := dao.AddArtifact(af) - if err != nil { - if err == dao.ErrDupRows { - continue - } - log.Error(err) - return err - } - } - } - return nil -} - -func persistAfnbs(afnbs []*models.ArtifactAndBlob) error { - if len(afnbs) != 0 { - for _, afnb := range afnbs { - _, err := dao.AddArtifactNBlob(afnb) - if err != nil { - if err == dao.ErrDupRows { - continue - } - log.Error(err) - return err - } - } - } - return nil -} - -func persistBlob(blobs []*models.Blob) error { - if len(blobs) != 0 { - for _, blob := range blobs { - _, err := dao.AddBlob(blob) - if err != nil { - if err == dao.ErrDupRows { - continue - } - log.Error(err) - return err - } - } - } - return nil -} - -func persistPB(projects []quota.ProjectInfo) error { - total := len(projects) - for i, project := range projects { - log.Infof("[Quota-Sync]:: start to persist project&blob for project: %s, progress... [%d/%d]", project.Name, i, total) - var blobs = make(map[string]int64) - var blobsOfPro []*models.Blob - for _, repo := range project.Repos { - for _, blob := range repo.Blobs { - _, exist := blobs[blob.Digest] - if exist { - continue - } - blobs[blob.Digest] = blob.Size - blobInDB, err := dao.GetBlob(blob.Digest) - if err != nil { - log.Error(err) - return err - } - if blobInDB != nil { - blobsOfPro = append(blobsOfPro, blobInDB) - } - } - } - pro, err := dao.GetProjectByName(project.Name) - if err != nil { - log.Error(err) - return err - } - _, err = dao.AddBlobsToProject(pro.ProjectID, blobsOfPro...) - if err != nil { - if err == dao.ErrDupRows { - continue - } - log.Error(err) - return err - } - log.Infof("[Quota-Sync]:: success to persist project&blob for project: %s, progress... [%d/%d]", project.Name, i, total) - } - return nil -} - -func infoOfProject(project string, repoList []string) (quota.ProjectInfo, error) { - var ( - repos []quota.RepoData - wg sync.WaitGroup - err error - ) - wg.Add(len(repoList)) - - errChan := make(chan error, 1) - infoChan := make(chan interface{}) - done := make(chan bool, 1) - - pro, err := dao.GetProjectByName(project) - if err != nil { - log.Error(err) - return quota.ProjectInfo{}, err - } - - go func() { - defer func() { - done <- true - }() - - for { - select { - case result := <-infoChan: - if result == nil { - return - } - repoData, ok := result.(quota.RepoData) - if ok { - repos = append(repos, repoData) - } - - case e := <-errChan: - if err == nil { - err = errors.Wrap(e, "quota sync error on getting info of repo") - } else { - err = errors.Wrap(e, err.Error()) - } - } - } - }() - - for _, repo := range repoList { - go func(pid int64, repo string) { - defer func() { - wg.Done() - }() - info, err := infoOfRepo(pid, repo) - if err != nil { - errChan <- err - return - } - infoChan <- info - }(pro.ProjectID, repo) - } - - wg.Wait() - close(infoChan) - - <-done - - if err != nil { - return quota.ProjectInfo{}, err - } - - return quota.ProjectInfo{ - Name: project, - Repos: repos, - }, nil -} - -func infoOfRepo(pid int64, repo string) (quota.RepoData, error) { - tags, err := registry.Cli.ListTags(repo) - if err != nil { - return quota.RepoData{}, err - } - var afnbs []*models.ArtifactAndBlob - var afs []*models.Artifact - var blobs []*models.Blob - - for _, tag := range tags { - manifest, digest, err := registry.Cli.PullManifest(repo, tag) - if err != nil { - log.Error(err) - // To workaround issue: https://github.com/goharbor/harbor/issues/9299, just log the error and do not raise it. - // Let the sync process pass, but the 'Unknown manifest' will not be counted into size and count of quota usage. - // User still can view there images with size 0 in harbor. - continue - } - mediaType, payload, err := manifest.Payload() - if err != nil { - return quota.RepoData{}, err - } - // self - afnb := &models.ArtifactAndBlob{ - DigestAF: digest, - DigestBlob: digest, - } - afnbs = append(afnbs, afnb) - // add manifest as a blob. - blob := &models.Blob{ - Digest: digest, - ContentType: mediaType, - Size: int64(len(payload)), - CreationTime: time.Now(), - } - blobs = append(blobs, blob) - for _, layer := range manifest.References() { - afnb := &models.ArtifactAndBlob{ - DigestAF: digest, - DigestBlob: layer.Digest.String(), - } - afnbs = append(afnbs, afnb) - blob := &models.Blob{ - Digest: layer.Digest.String(), - ContentType: layer.MediaType, - Size: layer.Size, - CreationTime: time.Now(), - } - blobs = append(blobs, blob) - } - af := &models.Artifact{ - PID: pid, - Repo: repo, - Tag: tag, - Digest: digest, - Kind: "Docker-Image", - CreationTime: time.Now(), - } - afs = append(afs, af) - } - return quota.RepoData{ - Name: repo, - Afs: afs, - Afnbs: afnbs, - Blobs: blobs, - }, nil -} - -func init() { - quota.Register("registry", NewRegistryMigrator) -} diff --git a/src/core/main.go b/src/core/main.go index 5bb23d64e..c44fb05cf 100755 --- a/src/core/main.go +++ b/src/core/main.go @@ -17,10 +17,8 @@ package main import ( "encoding/gob" "fmt" - "github.com/goharbor/harbor/src/migration" "os" "os/signal" - "strconv" "syscall" "time" @@ -30,13 +28,9 @@ import ( "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/job" "github.com/goharbor/harbor/src/common/models" - common_quota "github.com/goharbor/harbor/src/common/quota" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/api" - quota "github.com/goharbor/harbor/src/core/api/quota" - _ "github.com/goharbor/harbor/src/core/api/quota/chart" - _ "github.com/goharbor/harbor/src/core/api/quota/registry" _ "github.com/goharbor/harbor/src/core/auth/authproxy" _ "github.com/goharbor/harbor/src/core/auth/db" _ "github.com/goharbor/harbor/src/core/auth/ldap" @@ -46,13 +40,13 @@ import ( "github.com/goharbor/harbor/src/core/filter" "github.com/goharbor/harbor/src/core/middlewares" "github.com/goharbor/harbor/src/core/service/token" + "github.com/goharbor/harbor/src/migration" "github.com/goharbor/harbor/src/pkg/notification" _ "github.com/goharbor/harbor/src/pkg/notifier/topic" "github.com/goharbor/harbor/src/pkg/scan" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" "github.com/goharbor/harbor/src/pkg/scan/event" "github.com/goharbor/harbor/src/pkg/scheduler" - "github.com/goharbor/harbor/src/pkg/types" "github.com/goharbor/harbor/src/pkg/version" "github.com/goharbor/harbor/src/replication" "github.com/goharbor/harbor/src/server" @@ -88,69 +82,6 @@ func updateInitPassword(userID int, password string) error { return nil } -// Quota migration -func quotaSync() error { - projects, err := dao.GetProjects(nil) - if err != nil { - log.Errorf("list project error, %v", err) - return err - } - - var pids []string - for _, project := range projects { - pids = append(pids, strconv.FormatInt(project.ProjectID, 10)) - } - usages, err := dao.ListQuotaUsages(&models.QuotaUsageQuery{Reference: "project", ReferenceIDs: pids}) - if err != nil { - log.Errorf("list quota usage error, %v", err) - return err - } - - // The condition handles these two cases: - // 1, len(project) > 1 && len(usages) == 1. existing projects without usage, as we do always has 'library' usage in DB. - // 2, migration fails at the phase of inserting usage into DB, and parts of them are inserted successfully. - if len(projects) != len(usages) { - log.Info("Start to sync quota data .....") - if err := quota.Sync(config.GlobalProjectMgr, true); err != nil { - log.Errorf("Fail to sync quota data, %v", err) - return err - } - log.Info("Success to sync quota data .....") - return nil - } - - // Only has one project without usage - zero := common_quota.ResourceList{ - common_quota.ResourceCount: 0, - common_quota.ResourceStorage: 0, - } - if len(projects) == 1 && len(usages) == 1 { - totalRepo, err := dao.GetTotalOfRepositories() - if totalRepo == 0 { - return nil - } - refID, err := strconv.ParseInt(usages[0].ReferenceID, 10, 64) - if err != nil { - log.Error(err) - return err - } - usedRes, err := types.NewResourceList(usages[0].Used) - if err != nil { - log.Error(err) - return err - } - if types.Equals(usedRes, zero) && refID == projects[0].ProjectID { - log.Info("Start to sync quota data .....") - if err := quota.Sync(config.GlobalProjectMgr, true); err != nil { - log.Errorf("Fail to sync quota data, %v", err) - return err - } - log.Info("Success to sync quota data .....") - } - } - return nil -} - func gracefulShutdown(closing, done chan struct{}) { signals := make(chan os.Signal, 1) signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) diff --git a/src/pkg/quota/dao/dao.go b/src/pkg/quota/dao/dao.go index 65741b65a..00794427b 100644 --- a/src/pkg/quota/dao/dao.go +++ b/src/pkg/quota/dao/dao.go @@ -34,8 +34,11 @@ type DAO interface { // Get returns quota by id Get(ctx context.Context, id int64) (*models.Quota, error) - // GetForUpdate get quota by reference object and lock it for update - GetForUpdate(ctx context.Context, reference, referenceID string) (*models.Quota, error) + // GetByRef returns quota by reference object + GetByRef(ctx context.Context, reference, referenceID string) (*models.Quota, error) + + // GetByRefForUpdate get quota by reference object and lock it for update + GetByRefForUpdate(ctx context.Context, reference, referenceID string) (*models.Quota, error) // Update update quota Update(ctx context.Context, quota *models.Quota) error @@ -124,7 +127,26 @@ func (d *dao) Get(ctx context.Context, id int64) (*models.Quota, error) { return toQuota(quota, usage), nil } -func (d *dao) GetForUpdate(ctx context.Context, reference, referenceID string) (*models.Quota, error) { +func (d *dao) GetByRef(ctx context.Context, reference, referenceID string) (*models.Quota, error) { + o, err := orm.FromContext(ctx) + if err != nil { + return nil, err + } + + quota := &Quota{Reference: reference, ReferenceID: referenceID} + if err := o.Read(quota, "reference", "reference_id"); err != nil { + return nil, orm.WrapNotFoundError(err, "quota not found for (%s, %s)", reference, referenceID) + } + + usage := &QuotaUsage{Reference: reference, ReferenceID: referenceID} + if err := o.Read(usage, "reference", "reference_id"); err != nil { + return nil, orm.WrapNotFoundError(err, "quota usage not found for (%s, %s)", reference, referenceID) + } + + return toQuota(quota, usage), nil +} + +func (d *dao) GetByRefForUpdate(ctx context.Context, reference, referenceID string) (*models.Quota, error) { o, err := orm.FromContext(ctx) if err != nil { return nil, err diff --git a/src/pkg/quota/dao/dao_test.go b/src/pkg/quota/dao/dao_test.go index d73bce9f1..525caae4a 100644 --- a/src/pkg/quota/dao/dao_test.go +++ b/src/pkg/quota/dao/dao_test.go @@ -15,8 +15,11 @@ package dao import ( + "context" + "sync" "testing" + "github.com/goharbor/harbor/src/internal/orm" "github.com/goharbor/harbor/src/pkg/types" htesting "github.com/goharbor/harbor/src/testing" "github.com/stretchr/testify/suite" @@ -73,11 +76,75 @@ func (suite *DaoTestSuite) TestDelete() { } } +func (suite *DaoTestSuite) TestGetByRef() { + hardLimits := types.ResourceList{types.ResourceCount: 1} + usage := types.ResourceList{types.ResourceCount: 0} + + reference, referenceID := "project", "4" + id, err := suite.dao.Create(suite.Context(), reference, referenceID, hardLimits, usage) + suite.Nil(err) + + { + q, err := suite.dao.GetByRef(suite.Context(), reference, referenceID) + suite.Nil(err) + suite.NotNil(q) + } + + suite.Nil(suite.dao.Delete(suite.Context(), id)) + + { + _, err := suite.dao.GetByRef(suite.Context(), reference, referenceID) + suite.Error(err) + } +} + +func (suite *DaoTestSuite) TestGetByRefForUpdate() { + hardLimits := types.ResourceList{types.ResourceCount: 1} + usage := types.ResourceList{types.ResourceCount: 0} + + reference, referenceID := "project", "5" + id, err := suite.dao.Create(suite.Context(), reference, referenceID, hardLimits, usage) + suite.Nil(err) + + var wg sync.WaitGroup + + count := int64(10) + + for i := int64(0); i < count; i++ { + wg.Add(1) + go func() { + defer wg.Done() + f := func(ctx context.Context) error { + q, err := suite.dao.GetByRefForUpdate(ctx, reference, referenceID) + suite.Nil(err) + + used, _ := q.GetUsed() + used[types.ResourceCount]++ + q.SetUsed(used) + + suite.dao.Update(ctx, q) + + return nil + } + + orm.WithTransaction(f)(suite.Context()) + }() + } + wg.Wait() + + { + q, err := suite.dao.Get(suite.Context(), id) + suite.Nil(err) + used, _ := q.GetUsed() + suite.Equal(count, used[types.ResourceCount]) + } +} + func (suite *DaoTestSuite) TestUpdate() { hardLimits := types.ResourceList{types.ResourceCount: 1} usage := types.ResourceList{types.ResourceCount: 0} - id, err := suite.dao.Create(suite.Context(), "project", "3", hardLimits, usage) + id, err := suite.dao.Create(suite.Context(), "project", "6", hardLimits, usage) suite.Nil(err) newHardLimits := types.ResourceList{types.ResourceCount: 2} diff --git a/src/pkg/quota/manager.go b/src/pkg/quota/manager.go index 52ae52e0b..1b71b4750 100644 --- a/src/pkg/quota/manager.go +++ b/src/pkg/quota/manager.go @@ -37,8 +37,11 @@ type Manager interface { // Get returns quota by id Get(ctx context.Context, id int64) (*Quota, error) - // GetForUpdate returns quota by reference and reference id for update - GetForUpdate(ctx context.Context, reference, referenceID string) (*Quota, error) + // GetByRef returns quota by reference object + GetByRef(ctx context.Context, reference, referenceID string) (*Quota, error) + + // GetByRefForUpdate returns quota by reference and reference id for update + GetByRefForUpdate(ctx context.Context, reference, referenceID string) (*Quota, error) // Update update quota Update(ctx context.Context, quota *Quota) error @@ -80,8 +83,12 @@ func (m *manager) Get(ctx context.Context, id int64) (*Quota, error) { return m.dao.Get(ctx, id) } -func (m *manager) GetForUpdate(ctx context.Context, reference, referenceID string) (*Quota, error) { - return m.dao.GetForUpdate(ctx, reference, referenceID) +func (m *manager) GetByRef(ctx context.Context, reference, referenceID string) (*Quota, error) { + return m.dao.GetByRef(ctx, reference, referenceID) +} + +func (m *manager) GetByRefForUpdate(ctx context.Context, reference, referenceID string) (*Quota, error) { + return m.dao.GetByRefForUpdate(ctx, reference, referenceID) } func (m *manager) Update(ctx context.Context, q *Quota) error { diff --git a/src/pkg/quota/util.go b/src/pkg/quota/util.go index e45230e29..8d085d093 100644 --- a/src/pkg/quota/util.go +++ b/src/pkg/quota/util.go @@ -22,7 +22,7 @@ import ( ) // IsSafe check new used is safe under the hard limits -func IsSafe(hardLimits types.ResourceList, currentUsed types.ResourceList, newUsed types.ResourceList) error { +func IsSafe(hardLimits types.ResourceList, currentUsed types.ResourceList, newUsed types.ResourceList, ignoreLimitation bool) error { var errs Errors for resource, value := range newUsed { @@ -36,7 +36,7 @@ func IsSafe(hardLimits types.ResourceList, currentUsed types.ResourceList, newUs continue } - if value > hardLimit { + if value > hardLimit && !ignoreLimitation { errs = errs.Add(NewResourceOverflowError(resource, hardLimit, currentUsed[resource], value)) } } diff --git a/src/pkg/quota/util_test.go b/src/pkg/quota/util_test.go index 14adb37f1..e41607d6b 100644 --- a/src/pkg/quota/util_test.go +++ b/src/pkg/quota/util_test.go @@ -22,9 +22,10 @@ import ( func TestIsSafe(t *testing.T) { type args struct { - hardLimits types.ResourceList - currentUsed types.ResourceList - newUsed types.ResourceList + hardLimits types.ResourceList + currentUsed types.ResourceList + newUsed types.ResourceList + ignoreLimitation bool } tests := []struct { name string @@ -37,6 +38,7 @@ func TestIsSafe(t *testing.T) { types.ResourceList{types.ResourceStorage: types.UNLIMITED}, types.ResourceList{types.ResourceStorage: 1000}, types.ResourceList{types.ResourceStorage: 1000}, + false, }, false, }, @@ -46,6 +48,7 @@ func TestIsSafe(t *testing.T) { types.ResourceList{types.ResourceStorage: 100}, types.ResourceList{types.ResourceStorage: 10}, types.ResourceList{types.ResourceStorage: 1}, + false, }, false, }, @@ -55,22 +58,34 @@ func TestIsSafe(t *testing.T) { types.ResourceList{types.ResourceStorage: 100}, types.ResourceList{types.ResourceStorage: 0}, types.ResourceList{types.ResourceStorage: 200}, + false, }, true, }, + { + "ignore limitation", + args{ + types.ResourceList{types.ResourceStorage: 100}, + types.ResourceList{types.ResourceStorage: 0}, + types.ResourceList{types.ResourceStorage: 200}, + true, + }, + false, + }, { "hard limit not found", args{ types.ResourceList{types.ResourceStorage: 100}, types.ResourceList{types.ResourceCount: 0}, types.ResourceList{types.ResourceCount: 1}, + false, }, true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if err := IsSafe(tt.args.hardLimits, tt.args.currentUsed, tt.args.newUsed); (err != nil) != tt.wantErr { + if err := IsSafe(tt.args.hardLimits, tt.args.currentUsed, tt.args.newUsed, tt.args.ignoreLimitation); (err != nil) != tt.wantErr { t.Errorf("IsSafe() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/src/portal/src/app/project/summary/summary.component.html b/src/portal/src/app/project/summary/summary.component.html index 244f70ae6..a07f45149 100644 --- a/src/portal/src/app/project/summary/summary.component.html +++ b/src/portal/src/app/project/summary/summary.component.html @@ -23,7 +23,7 @@ -
+
{{'SUMMARY.PROJECT_QUOTAS' | translate}}
diff --git a/src/testing/api/quota/controller.go b/src/testing/api/quota/controller.go index 13ce86f6d..9ebfd3a80 100644 --- a/src/testing/api/quota/controller.go +++ b/src/testing/api/quota/controller.go @@ -8,6 +8,8 @@ import ( models "github.com/goharbor/harbor/src/pkg/quota/models" mock "github.com/stretchr/testify/mock" + quota "github.com/goharbor/harbor/src/api/quota" + types "github.com/goharbor/harbor/src/pkg/types" ) @@ -81,6 +83,29 @@ func (_m *Controller) Get(ctx context.Context, id int64) (*models.Quota, error) return r0, r1 } +// GetByRef provides a mock function with given fields: ctx, reference, referenceID +func (_m *Controller) GetByRef(ctx context.Context, reference string, referenceID string) (*models.Quota, error) { + ret := _m.Called(ctx, reference, referenceID) + + var r0 *models.Quota + if rf, ok := ret.Get(0).(func(context.Context, string, string) *models.Quota); ok { + r0 = rf(ctx, reference, referenceID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*models.Quota) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, reference, referenceID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // IsEnabled provides a mock function with given fields: ctx, reference, referenceID func (_m *Controller) IsEnabled(ctx context.Context, reference string, referenceID string) (bool, error) { ret := _m.Called(ctx, reference, referenceID) @@ -102,13 +127,20 @@ func (_m *Controller) IsEnabled(ctx context.Context, reference string, reference return r0, r1 } -// Refresh provides a mock function with given fields: ctx, reference, referenceID -func (_m *Controller) Refresh(ctx context.Context, reference string, referenceID string) error { - ret := _m.Called(ctx, reference, referenceID) +// Refresh provides a mock function with given fields: ctx, reference, referenceID, options +func (_m *Controller) Refresh(ctx context.Context, reference string, referenceID string, options ...quota.Option) error { + _va := make([]interface{}, len(options)) + for _i := range options { + _va[_i] = options[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, reference, referenceID) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { - r0 = rf(ctx, reference, referenceID) + if rf, ok := ret.Get(0).(func(context.Context, string, string, ...quota.Option) error); ok { + r0 = rf(ctx, reference, referenceID, options...) } else { r0 = ret.Error(0) } diff --git a/src/testing/pkg/quota/manager.go b/src/testing/pkg/quota/manager.go index fd297982b..1b654fd75 100644 --- a/src/testing/pkg/quota/manager.go +++ b/src/testing/pkg/quota/manager.go @@ -81,8 +81,31 @@ func (_m *Manager) Get(ctx context.Context, id int64) (*models.Quota, error) { return r0, r1 } -// GetForUpdate provides a mock function with given fields: ctx, reference, referenceID -func (_m *Manager) GetForUpdate(ctx context.Context, reference string, referenceID string) (*models.Quota, error) { +// GetByRef provides a mock function with given fields: ctx, reference, referenceID +func (_m *Manager) GetByRef(ctx context.Context, reference string, referenceID string) (*models.Quota, error) { + ret := _m.Called(ctx, reference, referenceID) + + var r0 *models.Quota + if rf, ok := ret.Get(0).(func(context.Context, string, string) *models.Quota); ok { + r0 = rf(ctx, reference, referenceID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*models.Quota) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, reference, referenceID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetByRefForUpdate provides a mock function with given fields: ctx, reference, referenceID +func (_m *Manager) GetByRefForUpdate(ctx context.Context, reference string, referenceID string) (*models.Quota, error) { ret := _m.Called(ctx, reference, referenceID) var r0 *models.Quota