From a73742c0a76161a33656465a5cf33f1d383fcb0e Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Sun, 16 Aug 2020 21:22:22 +0800 Subject: [PATCH 1/6] Return 403 when trying to push artifacts into the proxy cache project to avoid the retrying in the docker client Return 403 when trying to push artifacts into the proxy cache project to avoid the retrying in the docker client fixes #12731 Signed-off-by: Wenkai Yin --- src/server/middleware/repoproxy/proxy.go | 2 +- src/server/registry/route.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/server/middleware/repoproxy/proxy.go b/src/server/middleware/repoproxy/proxy.go index fdef48ba8..002aab0f4 100644 --- a/src/server/middleware/repoproxy/proxy.go +++ b/src/server/middleware/repoproxy/proxy.go @@ -184,7 +184,7 @@ func DisableBlobAndManifestUploadMiddleware() func(http.Handler) http.Handler { } if isProxyProject(p) && !isProxySession(ctx) { httpLib.SendError(w, - errors.MethodNotAllowedError( + errors.DeniedError( errors.Errorf("can not push artifact to a proxy project: %v", p.Name))) return } diff --git a/src/server/registry/route.go b/src/server/registry/route.go index ec2a7c0be..d744f42e8 100644 --- a/src/server/registry/route.go +++ b/src/server/registry/route.go @@ -77,6 +77,7 @@ func RegisterRoutes() { root.NewRoute(). Method(http.MethodPost). Path("/*/blobs/uploads"). + Middleware(repoproxy.DisableBlobAndManifestUploadMiddleware()). Middleware(quota.PostInitiateBlobUploadMiddleware()). Middleware(blob.PostInitiateBlobUploadMiddleware()). Handler(proxy) @@ -84,13 +85,11 @@ func RegisterRoutes() { root.NewRoute(). Method(http.MethodPatch). Path("/*/blobs/uploads/:session_id"). - Middleware(repoproxy.DisableBlobAndManifestUploadMiddleware()). Middleware(blob.PatchBlobUploadMiddleware()). Handler(proxy) root.NewRoute(). Method(http.MethodPut). Path("/*/blobs/uploads/:session_id"). - Middleware(repoproxy.DisableBlobAndManifestUploadMiddleware()). Middleware(quota.PutBlobUploadMiddleware()). Middleware(blob.PutBlobUploadMiddleware()). Handler(proxy) From e9ce631aa3a4e1663beaa6cf79c33cd57d30fb3d Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Sun, 16 Aug 2020 22:02:30 +0800 Subject: [PATCH 2/6] Disable the tag creation for the artifact under a proxy cache project Disable the tag creation for the artifact under a proxy cache project Fixes #12713 Signed-off-by: Wenkai Yin --- api/v2.0/swagger.yaml | 2 ++ src/server/v2.0/handler/artifact.go | 24 ++++++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 8dfd3df76..92a60ff30 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -606,6 +606,8 @@ paths: $ref: '#/responses/403' '404': $ref: '#/responses/404' + '405': + $ref: '#/responses/405' '409': $ref: '#/responses/409' '500': diff --git a/src/server/v2.0/handler/artifact.go b/src/server/v2.0/handler/artifact.go index 7818b6fa3..53a56f279 100644 --- a/src/server/v2.0/handler/artifact.go +++ b/src/server/v2.0/handler/artifact.go @@ -155,14 +155,9 @@ func (a *artifactAPI) CopyArtifact(ctx context.Context, params operation.CopyArt return a.SendError(ctx, err) } - pro, err := a.proCtl.GetByName(ctx, params.ProjectName) - if err != nil { + if err := a.requireNonProxyCacheProject(ctx, params.ProjectName); err != nil { return a.SendError(ctx, err) } - if pro.RegistryID > 0 { - return a.SendError(ctx, errors.New(nil).WithCode(errors.MethodNotAllowedCode). - WithMessage("cannot copy the artifact to a proxy cache project")) - } srcRepo, ref, err := parse(params.From) if err != nil { @@ -212,6 +207,11 @@ func (a *artifactAPI) CreateTag(ctx context.Context, params operation.CreateTagP if err := a.RequireProjectAccess(ctx, params.ProjectName, rbac.ActionCreate, rbac.ResourceTag); err != nil { return a.SendError(ctx, err) } + + if err := a.requireNonProxyCacheProject(ctx, params.ProjectName); err != nil { + return a.SendError(ctx, err) + } + art, err := a.artCtl.GetByReference(ctx, fmt.Sprintf("%s/%s", params.ProjectName, params.RepositoryName), params.Reference, &artifact.Option{ WithTag: true, @@ -239,6 +239,18 @@ func (a *artifactAPI) CreateTag(ctx context.Context, params operation.CreateTagP return operation.NewCreateTagCreated() } +func (a *artifactAPI) requireNonProxyCacheProject(ctx context.Context, name string) error { + pro, err := a.proCtl.GetByName(ctx, name) + if err != nil { + return err + } + if pro.RegistryID > 0 { + return errors.New(nil).WithCode(errors.MethodNotAllowedCode). + WithMessage("the operation isn't supported for a proxy cache project") + } + return nil +} + func (a *artifactAPI) DeleteTag(ctx context.Context, params operation.DeleteTagParams) middleware.Responder { if err := a.RequireProjectAccess(ctx, params.ProjectName, rbac.ActionDelete, rbac.ResourceTag); err != nil { return a.SendError(ctx, err) From 60427e71876461e05370e1d139c71ec3b8db3860 Mon Sep 17 00:00:00 2001 From: Wang Yan Date: Tue, 18 Aug 2020 14:11:52 +0800 Subject: [PATCH 3/6] build base image in CI (#12750) In git action, use the local build base images instead of pulling from docker hub. Signed-off-by: wang yan --- Makefile | 5 ++++- tests/ci/api_common_install.sh | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 43a077e72..145ef223a 100644 --- a/Makefile +++ b/Makefile @@ -90,6 +90,7 @@ GEN_TLS= # for docker image tag VERSIONTAG=dev # for base docker image tag +PUSHBASEIMAGE= BASEIMAGETAG=dev BASEIMAGENAMESPACE=goharbor # for harbor package name @@ -422,7 +423,9 @@ build_base_docker: @for name in chartserver clair clair-adapter trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \ echo $$name ; \ $(DOCKERBUILD) --pull --no-cache -f $(MAKEFILEPATH_PHOTON)/$$name/Dockerfile.base -t $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) --label base-build-date=$(date +"%Y%m%d") . && \ - $(PUSHSCRIPTPATH)/$(PUSHSCRIPTNAME) $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) $(REGISTRYUSER) $(REGISTRYPASSWORD) || exit 1; \ + if [ -n "$(PUSHBASEIMAGE)" ] ; then \ + $(PUSHSCRIPTPATH)/$(PUSHSCRIPTNAME) $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) $(REGISTRYUSER) $(REGISTRYPASSWORD) || exit 1; \ + fi ; \ done pull_base_docker: diff --git a/tests/ci/api_common_install.sh b/tests/ci/api_common_install.sh index 63be2ed02..c398306f1 100755 --- a/tests/ci/api_common_install.sh +++ b/tests/ci/api_common_install.sh @@ -30,7 +30,7 @@ if [ $GITHUB_TOKEN ]; then sed "s/# github_token: xxx/github_token: $GITHUB_TOKEN/" -i make/harbor.yml fi -sudo make compile build prepare COMPILETAG=compile_golangimage GOBUILDTAGS="include_oss include_gcs" NOTARYFLAG=true CLAIRFLAG=true TRIVYFLAG=true CHARTFLAG=true GEN_TLS=true +sudo make build_base_docker compile build prepare COMPILETAG=compile_golangimage GOBUILDTAGS="include_oss include_gcs" NOTARYFLAG=true CLAIRFLAG=true TRIVYFLAG=true CHARTFLAG=true GEN_TLS=true # set the debugging env echo "GC_TIME_WINDOW_HOURS=0" | sudo tee -a ./make/common/config/core/env From ef37bd1afbb09b7453df4cf0189e76625ac31430 Mon Sep 17 00:00:00 2001 From: He Weiwei Date: Tue, 18 Aug 2020 06:33:17 +0000 Subject: [PATCH 4/6] refactor(scan): remove duplicate CVESet types Closes #9471 Signed-off-by: He Weiwei --- src/common/models/cve_allowlist.go | 18 +++++++++++++++--- src/common/models/cve_allowlist_test.go | 14 +++++++------- src/controller/p2p/preheat/enforcer.go | 2 +- src/pkg/scan/report/summary.go | 15 +++------------ src/pkg/scan/report/summary_test.go | 3 ++- src/server/middleware/vulnerable/vulnerable.go | 2 +- 6 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/common/models/cve_allowlist.go b/src/common/models/cve_allowlist.go index 9b1883324..928e69ade 100644 --- a/src/common/models/cve_allowlist.go +++ b/src/common/models/cve_allowlist.go @@ -14,7 +14,9 @@ package models -import "time" +import ( + "time" +) // CVEAllowlist defines the data model for a CVE allowlist type CVEAllowlist struct { @@ -38,8 +40,8 @@ func (c *CVEAllowlist) TableName() string { } // CVESet returns the set of CVE id of the items in the allowlist to help filter the vulnerability list -func (c *CVEAllowlist) CVESet() map[string]struct{} { - r := map[string]struct{}{} +func (c *CVEAllowlist) CVESet() CVESet { + r := CVESet{} for _, it := range c.Items { r[it.CVEID] = struct{}{} } @@ -53,3 +55,13 @@ func (c *CVEAllowlist) IsExpired() bool { } return time.Now().Unix() >= *c.ExpiresAt } + +// CVESet defines the CVE allowlist with a hash set way for easy query. +type CVESet map[string]struct{} + +// Contains checks whether the specified CVE is in the set or not. +func (cs CVESet) Contains(cve string) bool { + _, ok := cs[cve] + + return ok +} diff --git a/src/common/models/cve_allowlist_test.go b/src/common/models/cve_allowlist_test.go index 9d5c87b71..7d06c1884 100644 --- a/src/common/models/cve_allowlist_test.go +++ b/src/common/models/cve_allowlist_test.go @@ -15,10 +15,10 @@ package models import ( - "github.com/stretchr/testify/assert" - "reflect" "testing" "time" + + "github.com/stretchr/testify/assert" ) func TestCVEAllowlist_All(t *testing.T) { @@ -26,7 +26,7 @@ func TestCVEAllowlist_All(t *testing.T) { now := time.Now().Unix() cases := []struct { input CVEAllowlist - cveset map[string]struct{} + cveset CVESet expired bool }{ { @@ -35,7 +35,7 @@ func TestCVEAllowlist_All(t *testing.T) { ProjectID: 0, Items: []CVEAllowlistItem{}, }, - cveset: map[string]struct{}{}, + cveset: CVESet{}, expired: false, }, { @@ -45,7 +45,7 @@ func TestCVEAllowlist_All(t *testing.T) { Items: []CVEAllowlistItem{}, ExpiresAt: &now, }, - cveset: map[string]struct{}{}, + cveset: CVESet{}, expired: true, }, { @@ -58,7 +58,7 @@ func TestCVEAllowlist_All(t *testing.T) { }, ExpiresAt: &future, }, - cveset: map[string]struct{}{ + cveset: CVESet{ "CVE-1999-0067": {}, "CVE-2016-7654321": {}, }, @@ -67,6 +67,6 @@ func TestCVEAllowlist_All(t *testing.T) { } for _, c := range cases { assert.Equal(t, c.expired, c.input.IsExpired()) - assert.True(t, reflect.DeepEqual(c.cveset, c.input.CVESet())) + assert.Equal(t, c.cveset, c.input.CVESet()) } } diff --git a/src/controller/p2p/preheat/enforcer.go b/src/controller/p2p/preheat/enforcer.go index c9fd55dd3..4e75b178d 100644 --- a/src/controller/p2p/preheat/enforcer.go +++ b/src/controller/p2p/preheat/enforcer.go @@ -476,7 +476,7 @@ func (de *defaultEnforcer) startTask(ctx context.Context, executionID int64, can // getVulnerabilitySev gets the severity code value for the given artifact with allowlist option set func (de *defaultEnforcer) getVulnerabilitySev(ctx context.Context, p *models.Project, art *artifact.Artifact) (uint, error) { - al := report.CVESet(p.CVEAllowlist.CVESet()) + al := p.CVEAllowlist.CVESet() r, err := de.scanCtl.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport}, report.WithCVEAllowlist(&al)) if err != nil { if errors.IsNotFoundErr(err) { diff --git a/src/pkg/scan/report/summary.go b/src/pkg/scan/report/summary.go index 9f54d3f39..373c3791b 100644 --- a/src/pkg/scan/report/summary.go +++ b/src/pkg/scan/report/summary.go @@ -17,6 +17,7 @@ package report import ( "reflect" + "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/pkg/scan/dao/scan" @@ -24,29 +25,19 @@ import ( "github.com/goharbor/harbor/src/pkg/scan/vuln" ) -// CVESet defines the CVE allowlist with a hash set way for easy query. -type CVESet map[string]struct{} - -// Contains checks whether the specified CVE is in the set or not. -func (cs CVESet) Contains(cve string) bool { - _, ok := cs[cve] - - return ok -} - // Options provides options for getting the report w/ summary. type Options struct { // If it is set, the returned report will contains artifact digest for the vulnerabilities ArtifactDigest string // If it is set, the returned summary will not count the CVEs in the list in. - CVEAllowlist CVESet + CVEAllowlist models.CVESet } // Option for getting the report w/ summary with func template way. type Option func(options *Options) // WithCVEAllowlist is an option of setting CVE allowlist. -func WithCVEAllowlist(set *CVESet) Option { +func WithCVEAllowlist(set *models.CVESet) Option { return func(options *Options) { options.CVEAllowlist = *set } diff --git a/src/pkg/scan/report/summary_test.go b/src/pkg/scan/report/summary_test.go index dd40525a6..0f8489f4f 100644 --- a/src/pkg/scan/report/summary_test.go +++ b/src/pkg/scan/report/summary_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/pkg/scan/dao/scan" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" "github.com/goharbor/harbor/src/pkg/scan/vuln" @@ -108,7 +109,7 @@ func (suite *SummaryTestSuite) TestSummaryGenerateSummaryNoOptions() { // TestSummaryGenerateSummaryWithOptions ... func (suite *SummaryTestSuite) TestSummaryGenerateSummaryWithOptions() { - cveSet := make(CVESet) + cveSet := make(models.CVESet) cveSet["2019-0980-0909"] = struct{}{} summaries, err := GenerateSummary(suite.r, WithCVEAllowlist(&cveSet)) diff --git a/src/server/middleware/vulnerable/vulnerable.go b/src/server/middleware/vulnerable/vulnerable.go index d0f1278bd..c616d2e8f 100644 --- a/src/server/middleware/vulnerable/vulnerable.go +++ b/src/server/middleware/vulnerable/vulnerable.go @@ -91,7 +91,7 @@ func Middleware() func(http.Handler) http.Handler { return nil } - allowlist := report.CVESet(proj.CVEAllowlist.CVESet()) + allowlist := proj.CVEAllowlist.CVESet() summaries, err := scanController.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport}, report.WithCVEAllowlist(&allowlist)) if err != nil { logger.Errorf("get vulnerability summary of the artifact %s@%s failed, error: %v", art.RepositoryName, art.Digest, err) From af0f36a1530bdaecc3bfefdd7ae10a1b115a7d0a Mon Sep 17 00:00:00 2001 From: Wang Yan Date: Tue, 18 Aug 2020 17:49:30 +0800 Subject: [PATCH 5/6] add timeout for CI (#12803) Add 1 hour(60 minutes) as the timeout of gitactions. Signed-off-by: wang yan --- .github/workflows/CI.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5270404ac..6dcf39a57 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -27,6 +27,7 @@ jobs: runs-on: #- self-hosted - ubuntu-latest + timeout-minutes: 60 steps: - name: Set up Go 1.14 uses: actions/setup-go@v1 @@ -92,6 +93,7 @@ jobs: runs-on: #- self-hosted - ubuntu-latest + timeout-minutes: 60 steps: - name: Set up Go 1.14 uses: actions/setup-go@v1 @@ -195,6 +197,7 @@ jobs: runs-on: #- self-hosted - ubuntu-latest + timeout-minutes: 60 steps: - name: Set up Go 1.14 uses: actions/setup-go@v1 @@ -253,6 +256,7 @@ jobs: runs-on: #- self-hosted - ubuntu-latest + timeout-minutes: 60 steps: - name: Set up Go 1.14 uses: actions/setup-go@v1 @@ -309,6 +313,7 @@ jobs: runs-on: #- self-hosted - ubuntu-latest + timeout-minutes: 60 steps: - uses: actions/setup-node@v1 with: From eb317fb8cb6336054c5751c73c83f86e5145478c Mon Sep 17 00:00:00 2001 From: Ted Guan Date: Wed, 19 Aug 2020 00:07:45 +0800 Subject: [PATCH 6/6] tag retention webhook support (#12749) Signed-off-by: guanxiatao --- src/controller/event/handler/init.go | 1 + .../handler/webhook/artifact/replication.go | 18 +- .../handler/webhook/artifact/retention.go | 160 ++++++++++++++++++ .../webhook/artifact/retention_test.go | 82 +++++++++ src/controller/event/metadata/retention.go | 33 ++++ .../event/metadata/retention_test.go | 34 ++++ src/controller/event/model/event.go | 61 +++++++ src/controller/event/topic.go | 11 ++ src/core/api/notification_policy.go | 1 + src/core/api/retention.go | 3 +- .../service/notifications/jobs/handler.go | 26 ++- src/pkg/notifier/model/event.go | 44 +---- src/pkg/retention/controller.go | 7 + src/pkg/retention/faked_controller.go | 96 +++++++++++ src/pkg/retention/mocks/api_controller.go | 8 + .../app/project/webhook/webhook.service.ts | 1 + 16 files changed, 534 insertions(+), 52 deletions(-) create mode 100644 src/controller/event/handler/webhook/artifact/retention.go create mode 100644 src/controller/event/handler/webhook/artifact/retention_test.go create mode 100644 src/controller/event/metadata/retention.go create mode 100644 src/controller/event/metadata/retention_test.go create mode 100644 src/controller/event/model/event.go create mode 100644 src/pkg/retention/faked_controller.go diff --git a/src/controller/event/handler/init.go b/src/controller/event/handler/init.go index f016a5411..1d85b0719 100644 --- a/src/controller/event/handler/init.go +++ b/src/controller/event/handler/init.go @@ -28,6 +28,7 @@ func init() { notifier.Subscribe(event.TopicScanningCompleted, &scan.Handler{}) notifier.Subscribe(event.TopicDeleteArtifact, &scan.DelArtHandler{}) notifier.Subscribe(event.TopicReplication, &artifact.ReplicationHandler{}) + notifier.Subscribe(event.TopicTagRetention, &artifact.RetentionHandler{RetentionController: artifact.DefaultRetentionControllerFunc}) // replication notifier.Subscribe(event.TopicPushArtifact, &replication.Handler{}) diff --git a/src/controller/event/handler/webhook/artifact/replication.go b/src/controller/event/handler/webhook/artifact/replication.go index 730b4bd58..c811d8b79 100644 --- a/src/controller/event/handler/webhook/artifact/replication.go +++ b/src/controller/event/handler/webhook/artifact/replication.go @@ -8,6 +8,7 @@ import ( commonModels "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/controller/event" "github.com/goharbor/harbor/src/controller/event/handler/util" + ctlModel "github.com/goharbor/harbor/src/controller/event/model" "github.com/goharbor/harbor/src/controller/project" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/jobservice/job" @@ -15,7 +16,6 @@ import ( "github.com/goharbor/harbor/src/lib/orm" "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/notifier/model" - notifyModel "github.com/goharbor/harbor/src/pkg/notifier/model" "github.com/goharbor/harbor/src/replication" rpModel "github.com/goharbor/harbor/src/replication/model" ) @@ -120,7 +120,7 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload, } hostname := strings.Split(extURL, ":")[0] - remoteRes := &model.ReplicationResource{ + remoteRes := &ctlModel.ReplicationResource{ RegistryName: remoteRegistry.Name, RegistryType: string(remoteRegistry.Type), Endpoint: remoteRegistry.URL, @@ -131,18 +131,18 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload, if err != nil { log.Errorf("Error while reading external endpoint: %v", err) } - localRes := &model.ReplicationResource{ + localRes := &ctlModel.ReplicationResource{ RegistryType: string(rpModel.RegistryTypeHarbor), Endpoint: ext, Namespace: destNamespace, } - payload := ¬ifyModel.Payload{ + payload := &model.Payload{ Type: event.EventType, OccurAt: event.OccurAt.Unix(), Operator: string(execution.Trigger), EventData: &model.EventData{ - Replication: &model.Replication{ + Replication: &ctlModel.Replication{ HarborHostname: hostname, JobStatus: event.Status, Description: rpPolicy.Description, @@ -174,20 +174,20 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload, } if event.Status == string(job.SuccessStatus) { - succeedArtifact := &model.ArtifactInfo{ + succeedArtifact := &ctlModel.ArtifactInfo{ Type: task.ResourceType, Status: task.Status, NameAndTag: nameAndTag, } - payload.EventData.Replication.SuccessfulArtifact = []*model.ArtifactInfo{succeedArtifact} + payload.EventData.Replication.SuccessfulArtifact = []*ctlModel.ArtifactInfo{succeedArtifact} } if event.Status == string(job.ErrorStatus) { - failedArtifact := &model.ArtifactInfo{ + failedArtifact := &ctlModel.ArtifactInfo{ Type: task.ResourceType, Status: task.Status, NameAndTag: nameAndTag, } - payload.EventData.Replication.FailedArtifact = []*model.ArtifactInfo{failedArtifact} + payload.EventData.Replication.FailedArtifact = []*ctlModel.ArtifactInfo{failedArtifact} } prj, err := project.Ctl.GetByName(orm.Context(), prjName, project.Metadata(true)) diff --git a/src/controller/event/handler/webhook/artifact/retention.go b/src/controller/event/handler/webhook/artifact/retention.go new file mode 100644 index 000000000..11114e7e7 --- /dev/null +++ b/src/controller/event/handler/webhook/artifact/retention.go @@ -0,0 +1,160 @@ +package artifact + +import ( + "fmt" + "strings" + + "github.com/goharbor/harbor/src/controller/event" + "github.com/goharbor/harbor/src/controller/event/handler/util" + evtModel "github.com/goharbor/harbor/src/controller/event/model" + "github.com/goharbor/harbor/src/core/api" + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/notification" + "github.com/goharbor/harbor/src/pkg/notifier/model" + "github.com/goharbor/harbor/src/pkg/retention" +) + +// RetentionHandler preprocess tag retention event data +type RetentionHandler struct { + RetentionController func() retention.APIController +} + +// DefaultRetentionControllerFunc ... +var DefaultRetentionControllerFunc = NewRetentionController + +// NewRetentionController ... +func NewRetentionController() retention.APIController { + return api.GetRetentionController() +} + +// Handle ... +func (r *RetentionHandler) Handle(value interface{}) error { + if !config.NotificationEnable() { + log.Debug("notification feature is not enabled") + return nil + } + + trEvent, ok := value.(*event.RetentionEvent) + if !ok { + return errors.New("invalid tag retention event type") + } + if trEvent == nil { + return errors.New("nil tag retention event") + } + if len(trEvent.Deleted) == 0 { + log.Debugf("empty delete info of retention event") + return nil + } + + payload, dryRun, project, err := r.constructRetentionPayload(trEvent) + if err != nil { + return err + } + // if dry run, do not trigger webhook + if dryRun { + log.Debugf("retention task %v is dry run", trEvent.TaskID) + return nil + } + + policies, err := notification.PolicyMgr.GetRelatedPolices(project, trEvent.EventType) + if err != nil { + log.Errorf("failed to find policy for %s event: %v", trEvent.EventType, err) + return err + } + if len(policies) == 0 { + log.Debugf("cannot find policy for %s event: %v", trEvent.EventType, trEvent) + return nil + } + err = util.SendHookWithPolicies(policies, payload, trEvent.EventType) + if err != nil { + return err + } + return nil +} + +// IsStateful ... +func (r *RetentionHandler) IsStateful() bool { + return false +} + +func (r *RetentionHandler) constructRetentionPayload(event *event.RetentionEvent) (*model.Payload, bool, int64, error) { + task, err := r.RetentionController().GetRetentionExecTask(event.TaskID) + if err != nil { + log.Errorf("failed to get retention task %d: error: %v", event.TaskID, err) + return nil, false, 0, err + } + if task == nil { + return nil, false, 0, fmt.Errorf("task %d not found with retention event", event.TaskID) + } + + execution, err := r.RetentionController().GetRetentionExec(task.ExecutionID) + if err != nil { + log.Errorf("failed to get retention execution %d: error: %v", task.ExecutionID, err) + return nil, false, 0, err + } + if execution == nil { + return nil, false, 0, fmt.Errorf("execution %d not found with retention event", task.ExecutionID) + } + + if execution.DryRun { + return nil, true, 0, nil + } + + md, err := r.RetentionController().GetRetention(execution.PolicyID) + if err != nil { + log.Errorf("failed to get tag retention policy %d: error: %v", execution.PolicyID, err) + return nil, false, 0, err + } + if md == nil { + return nil, false, 0, fmt.Errorf("policy %d not found with tag retention event", execution.PolicyID) + } + + extURL, err := config.ExtURL() + if err != nil { + log.Errorf("Error while reading external endpoint URL: %v", err) + } + hostname := strings.Split(extURL, ":")[0] + + payload := &model.Payload{ + Type: event.EventType, + OccurAt: event.OccurAt.Unix(), + Operator: execution.Trigger, + EventData: &model.EventData{ + Retention: &evtModel.Retention{ + Total: task.Total, + Retained: task.Retained, + HarborHostname: hostname, + ProjectName: event.Deleted[0].Target.Namespace, + RetentionPolicyID: execution.PolicyID, + Status: event.Status, + RetentionRules: []*evtModel.RetentionRule{}, + }, + }, + } + + for _, v := range event.Deleted { + target := v.Target + deletedArtifact := &evtModel.ArtifactInfo{ + Type: target.Kind, + Status: event.Status, + } + if len(target.Tags) != 0 { + deletedArtifact.NameAndTag = target.Repository + ":" + target.Tags[0] + } + payload.EventData.Retention.DeletedArtifact = []*evtModel.ArtifactInfo{deletedArtifact} + } + + for _, v := range md.Rules { + retentionRule := &evtModel.RetentionRule{ + Template: v.Template, + Parameters: v.Parameters, + TagSelectors: v.TagSelectors, + ScopeSelectors: v.ScopeSelectors, + } + payload.EventData.Retention.RetentionRules = append(payload.EventData.Retention.RetentionRules, retentionRule) + } + + return payload, false, event.Deleted[0].Target.NamespaceID, nil +} diff --git a/src/controller/event/handler/webhook/artifact/retention_test.go b/src/controller/event/handler/webhook/artifact/retention_test.go new file mode 100644 index 000000000..05f171279 --- /dev/null +++ b/src/controller/event/handler/webhook/artifact/retention_test.go @@ -0,0 +1,82 @@ +package artifact + +import ( + "testing" + "time" + + "github.com/goharbor/harbor/src/pkg/retention" + + "github.com/goharbor/harbor/src/controller/event" + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/lib/selector" + "github.com/goharbor/harbor/src/pkg/notification" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRetentionHandler_Handle(t *testing.T) { + config.Init() + handler := &RetentionHandler{RetentionController: DefaultRetentionControllerFunc} + + policyMgr := notification.PolicyMgr + retentionCtlFunc := handler.RetentionController + + defer func() { + notification.PolicyMgr = policyMgr + handler.RetentionController = retentionCtlFunc + }() + notification.PolicyMgr = &fakedNotificationPolicyMgr{} + handler.RetentionController = retention.FakedRetentionControllerFunc + + type args struct { + data interface{} + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "RetentionHandler Want Error 1", + args: args{ + data: "", + }, + wantErr: true, + }, + { + name: "RetentionHandler 1", + args: args{ + data: &event.RetentionEvent{ + OccurAt: time.Now(), + Deleted: []*selector.Result{ + { + Target: &selector.Candidate{ + NamespaceID: 1, + Namespace: "project1", + Tags: []string{"v1"}, + Labels: nil, + }, + }, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := handler.Handle(tt.args.data) + if tt.wantErr { + require.NotNil(t, err, "Error: %s", err) + return + } + assert.Nil(t, err) + }) + } + +} + +func TestRetentionHandler_IsStateful(t *testing.T) { + handler := &RetentionHandler{} + assert.False(t, handler.IsStateful()) +} diff --git a/src/controller/event/metadata/retention.go b/src/controller/event/metadata/retention.go new file mode 100644 index 000000000..492985146 --- /dev/null +++ b/src/controller/event/metadata/retention.go @@ -0,0 +1,33 @@ +package metadata + +import ( + "time" + + event2 "github.com/goharbor/harbor/src/controller/event" + "github.com/goharbor/harbor/src/lib/selector" + "github.com/goharbor/harbor/src/pkg/notifier/event" +) + +// RetentionMetaData defines tag retention related event data +type RetentionMetaData struct { + Total int + Retained int + Deleted []*selector.Result + Status string + TaskID int64 +} + +// Resolve tag retention metadata into tag retention event +func (r *RetentionMetaData) Resolve(evt *event.Event) error { + data := &event2.RetentionEvent{ + EventType: event2.TopicTagRetention, + OccurAt: time.Now(), + Status: r.Status, + Deleted: r.Deleted, + TaskID: r.TaskID, + } + + evt.Topic = event2.TopicTagRetention + evt.Data = data + return nil +} diff --git a/src/controller/event/metadata/retention_test.go b/src/controller/event/metadata/retention_test.go new file mode 100644 index 000000000..51a354052 --- /dev/null +++ b/src/controller/event/metadata/retention_test.go @@ -0,0 +1,34 @@ +package metadata + +import ( + "testing" + + event2 "github.com/goharbor/harbor/src/controller/event" + "github.com/goharbor/harbor/src/pkg/notifier/event" + "github.com/stretchr/testify/suite" +) + +type retentionEventTestSuite struct { + suite.Suite +} + +func (r *retentionEventTestSuite) TestResolveOfDeleteRepositoryEventMetadata() { + e := &event.Event{} + metadata := &RetentionMetaData{ + Total: 0, + Retained: 0, + Deleted: nil, + Status: "", + TaskID: 0, + } + err := metadata.Resolve(e) + r.Require().Nil(err) + r.Equal(event2.TopicTagRetention, e.Topic) + r.Require().NotNil(e.Data) + _, ok := e.Data.(*event2.RetentionEvent) + r.Require().True(ok) +} + +func TestRetentionEventTestSuite(t *testing.T) { + suite.Run(t, &retentionEventTestSuite{}) +} diff --git a/src/controller/event/model/event.go b/src/controller/event/model/event.go new file mode 100644 index 000000000..33e0ab45a --- /dev/null +++ b/src/controller/event/model/event.go @@ -0,0 +1,61 @@ +package model + +import "github.com/goharbor/harbor/src/pkg/retention/policy/rule" + +// Replication describes replication infos +type Replication struct { + HarborHostname string `json:"harbor_hostname,omitempty"` + JobStatus string `json:"job_status,omitempty"` + Description string `json:"description,omitempty"` + ArtifactType string `json:"artifact_type,omitempty"` + AuthenticationType string `json:"authentication_type,omitempty"` + OverrideMode bool `json:"override_mode,omitempty"` + TriggerType string `json:"trigger_type,omitempty"` + PolicyCreator string `json:"policy_creator,omitempty"` + ExecutionTimestamp int64 `json:"execution_timestamp,omitempty"` + SrcResource *ReplicationResource `json:"src_resource,omitempty"` + DestResource *ReplicationResource `json:"dest_resource,omitempty"` + SuccessfulArtifact []*ArtifactInfo `json:"successful_artifact,omitempty"` + FailedArtifact []*ArtifactInfo `json:"failed_artifact,omitempty"` +} + +// ArtifactInfo describe info of artifact +type ArtifactInfo struct { + Type string `json:"type"` + Status string `json:"status"` + NameAndTag string `json:"name_tag"` + FailReason string `json:"fail_reason,omitempty"` +} + +// ReplicationResource describes replication resource info +type ReplicationResource struct { + RegistryName string `json:"registry_name,omitempty"` + RegistryType string `json:"registry_type"` + Endpoint string `json:"endpoint"` + Provider string `json:"provider,omitempty"` + Namespace string `json:"namespace,omitempty"` +} + +// Retention describes tag retention infos +type Retention struct { + Total int `json:"total"` + Retained int `json:"retained"` + HarborHostname string `json:"harbor_hostname,omitempty"` + ProjectName string `json:"project_name,omitempty"` + RetentionPolicyID int64 `json:"retention_policy_id,omitempty"` + RetentionRules []*RetentionRule `json:"retention_rule,omitempty"` + Status string `json:"result,omitempty"` + DeletedArtifact []*ArtifactInfo `json:"deleted_artifact,omitempty"` +} + +// RetentionRule describes tag retention rule +type RetentionRule struct { + // Template ID + Template string `json:"template,omitempty"` + // The parameters of this rule + Parameters map[string]rule.Parameter `json:"params,omitempty"` + // Selector attached to the rule for filtering tags + TagSelectors []*rule.Selector `json:"tag_selectors,omitempty" ` + // Selector attached to the rule for filtering scope (e.g: repositories or namespaces) + ScopeSelectors map[string][]*rule.Selector `json:"scope_selectors,omitempty"` +} diff --git a/src/controller/event/topic.go b/src/controller/event/topic.go index 6d142b09f..efd336920 100644 --- a/src/controller/event/topic.go +++ b/src/controller/event/topic.go @@ -19,6 +19,7 @@ import ( "time" "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/lib/selector" "github.com/goharbor/harbor/src/pkg/artifact" "github.com/goharbor/harbor/src/pkg/audit/model" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" @@ -46,6 +47,7 @@ const ( TopicDeleteChart = "DELETE_CHART" TopicReplication = "REPLICATION" TopicArtifactLabeled = "ARTIFACT_LABELED" + TopicTagRetention = "TAG_RETENTION" ) // CreateProjectEvent is the creating project event @@ -289,3 +291,12 @@ type ArtifactLabeledEvent struct { OccurAt time.Time Operator string } + +// RetentionEvent is tag retention related event data to publish +type RetentionEvent struct { + TaskID int64 + EventType string + OccurAt time.Time + Status string + Deleted []*selector.Result +} diff --git a/src/core/api/notification_policy.go b/src/core/api/notification_policy.go index 92b2f6e1e..9b688aaed 100755 --- a/src/core/api/notification_policy.go +++ b/src/core/api/notification_policy.go @@ -384,6 +384,7 @@ func initSupportedEvents() map[string]struct{} { event.TopicScanningFailed, event.TopicScanningCompleted, event.TopicReplication, + event.TopicTagRetention, } var supportedEventTypes = make(map[string]struct{}) diff --git a/src/core/api/retention.go b/src/core/api/retention.go index 699541a88..cad95939c 100644 --- a/src/core/api/retention.go +++ b/src/core/api/retention.go @@ -4,10 +4,11 @@ import ( "encoding/json" "errors" "fmt" - "github.com/goharbor/harbor/src/core/config" "net/http" "strconv" + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/core/promgr" "github.com/goharbor/harbor/src/pkg/retention" diff --git a/src/core/service/notifications/jobs/handler.go b/src/core/service/notifications/jobs/handler.go index 48f19667e..efe354ec9 100755 --- a/src/core/service/notifications/jobs/handler.go +++ b/src/core/service/notifications/jobs/handler.go @@ -18,15 +18,15 @@ import ( "encoding/json" "time" - "github.com/goharbor/harbor/src/core/service/notifications" - "github.com/goharbor/harbor/src/common/job" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/controller/event/metadata" "github.com/goharbor/harbor/src/controller/scan" + "github.com/goharbor/harbor/src/core/service/notifications" jjob "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/lib/selector" "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/notifier/event" "github.com/goharbor/harbor/src/pkg/retention" @@ -188,8 +188,9 @@ func (h *Handler) HandleRetentionTask() { // handle checkin if h.checkIn != "" { var retainObj struct { - Total int `json:"total"` - Retained int `json:"retained"` + Total int `json:"total"` + Retained int `json:"retained"` + Deleted []*selector.Result `json:"deleted"` } if err := json.Unmarshal([]byte(h.checkIn), &retainObj); err != nil { log.Errorf("failed to resolve checkin of retention task %d: %v", taskID, err) @@ -205,6 +206,23 @@ func (h *Handler) HandleRetentionTask() { h.SendInternalServerError(err) return } + + e := &event.Event{} + metaData := &metadata.RetentionMetaData{ + Total: retainObj.Total, + Retained: retainObj.Retained, + Deleted: retainObj.Deleted, + Status: "SUCCESS", + TaskID: taskID, + } + + if err := e.Build(metaData); err == nil { + if err := e.Publish(); err != nil { + log.Error(errors.Wrap(err, "tag retention job hook handler: event publish")) + } + } else { + log.Error(errors.Wrap(err, "tag retention job hook handler: event publish")) + } return } diff --git a/src/pkg/notifier/model/event.go b/src/pkg/notifier/model/event.go index a1d63ba1d..3f9dbeb1f 100755 --- a/src/pkg/notifier/model/event.go +++ b/src/pkg/notifier/model/event.go @@ -2,6 +2,7 @@ package model import ( "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/controller/event/model" ) // HookEvent is hook related event data to publish @@ -22,10 +23,11 @@ type Payload struct { // EventData of notification event payload type EventData struct { - Resources []*Resource `json:"resources,omitempty"` - Repository *Repository `json:"repository,omitempty"` - Replication *Replication `json:"replication,omitempty"` - Custom map[string]string `json:"custom_attributes,omitempty"` + Resources []*Resource `json:"resources,omitempty"` + Repository *Repository `json:"repository,omitempty"` + Replication *model.Replication `json:"replication,omitempty"` + Retention *model.Retention `json:"retention,omitempty"` + Custom map[string]string `json:"custom_attributes,omitempty"` } // Resource describe infos of resource triggered notification @@ -44,37 +46,3 @@ type Repository struct { RepoFullName string `json:"repo_full_name"` RepoType string `json:"repo_type"` } - -// Replication describes replication infos -type Replication struct { - HarborHostname string `json:"harbor_hostname,omitempty"` - JobStatus string `json:"job_status,omitempty"` - Description string `json:"description,omitempty"` - ArtifactType string `json:"artifact_type,omitempty"` - AuthenticationType string `json:"authentication_type,omitempty"` - OverrideMode bool `json:"override_mode,omitempty"` - TriggerType string `json:"trigger_type,omitempty"` - PolicyCreator string `json:"policy_creator,omitempty"` - ExecutionTimestamp int64 `json:"execution_timestamp,omitempty"` - SrcResource *ReplicationResource `json:"src_resource,omitempty"` - DestResource *ReplicationResource `json:"dest_resource,omitempty"` - SuccessfulArtifact []*ArtifactInfo `json:"successful_artifact,omitempty"` - FailedArtifact []*ArtifactInfo `json:"failed_artifact,omitempty"` -} - -// ArtifactInfo describe info of artifact replicated -type ArtifactInfo struct { - Type string `json:"type"` - Status string `json:"status"` - NameAndTag string `json:"name_tag"` - FailReason string `json:"fail_reason,omitempty"` -} - -// ReplicationResource describes replication resource info -type ReplicationResource struct { - RegistryName string `json:"registry_name,omitempty"` - RegistryType string `json:"registry_type"` - Endpoint string `json:"endpoint"` - Provider string `json:"provider,omitempty"` - Namespace string `json:"namespace,omitempty"` -} diff --git a/src/pkg/retention/controller.go b/src/pkg/retention/controller.go index 16adf2248..dd022328a 100644 --- a/src/pkg/retention/controller.go +++ b/src/pkg/retention/controller.go @@ -53,6 +53,8 @@ type APIController interface { GetTotalOfRetentionExecTasks(executionID int64) (int64, error) GetRetentionExecTaskLog(taskID int64) ([]byte, error) + + GetRetentionExecTask(taskID int64) (*Task, error) } // DefaultAPIController ... @@ -255,6 +257,11 @@ func (r *DefaultAPIController) GetRetentionExecTaskLog(taskID int64) ([]byte, er return r.manager.GetTaskLog(taskID) } +// GetRetentionExecTask Get Retention Execution Task +func (r *DefaultAPIController) GetRetentionExecTask(taskID int64) (*Task, error) { + return r.manager.GetTask(taskID) +} + // NewAPIController ... func NewAPIController(retentionMgr Manager, projectManager project.Manager, repositoryMgr repository.Manager, scheduler scheduler.Scheduler, retentionLauncher Launcher) APIController { return &DefaultAPIController{ diff --git a/src/pkg/retention/faked_controller.go b/src/pkg/retention/faked_controller.go new file mode 100644 index 000000000..b4cc9ed37 --- /dev/null +++ b/src/pkg/retention/faked_controller.go @@ -0,0 +1,96 @@ +package retention + +import ( + "github.com/goharbor/harbor/src/pkg/retention/policy" + "github.com/goharbor/harbor/src/pkg/retention/q" +) + +// FakedRetentionController ... +type FakedRetentionController struct { +} + +// FakedRetentionControllerFunc ... +var FakedRetentionControllerFunc = NewFakedRetentionController + +// NewFakedRetentionController ... +func NewFakedRetentionController() APIController { + return &FakedRetentionController{} +} + +// GetRetention ... +func (f *FakedRetentionController) GetRetention(id int64) (*policy.Metadata, error) { + return &policy.Metadata{ + ID: 1, + Algorithm: "", + Rules: nil, + Trigger: nil, + Scope: nil, + }, nil +} + +// CreateRetention ... +func (f *FakedRetentionController) CreateRetention(p *policy.Metadata) (int64, error) { + return 0, nil +} + +// UpdateRetention ... +func (f *FakedRetentionController) UpdateRetention(p *policy.Metadata) error { + return nil +} + +// DeleteRetention ... +func (f *FakedRetentionController) DeleteRetention(id int64) error { + return nil +} + +// TriggerRetentionExec ... +func (f *FakedRetentionController) TriggerRetentionExec(policyID int64, trigger string, dryRun bool) (int64, error) { + + return 0, nil +} + +// OperateRetentionExec ... +func (f *FakedRetentionController) OperateRetentionExec(eid int64, action string) error { + return nil +} + +// GetRetentionExec ... +func (f *FakedRetentionController) GetRetentionExec(eid int64) (*Execution, error) { + return &Execution{ + DryRun: false, + PolicyID: 1, + }, nil +} + +// ListRetentionExecs ... +func (f *FakedRetentionController) ListRetentionExecs(policyID int64, query *q.Query) ([]*Execution, error) { + return nil, nil +} + +// GetTotalOfRetentionExecs ... +func (f *FakedRetentionController) GetTotalOfRetentionExecs(policyID int64) (int64, error) { + return 0, nil +} + +// ListRetentionExecTasks ... +func (f *FakedRetentionController) ListRetentionExecTasks(executionID int64, query *q.Query) ([]*Task, error) { + return nil, nil +} + +// GetTotalOfRetentionExecTasks ... +func (f *FakedRetentionController) GetTotalOfRetentionExecTasks(executionID int64) (int64, error) { + return 0, nil +} + +// GetRetentionExecTaskLog ... +func (f *FakedRetentionController) GetRetentionExecTaskLog(taskID int64) ([]byte, error) { + return nil, nil +} + +// GetRetentionExecTask ... +func (f *FakedRetentionController) GetRetentionExecTask(taskID int64) (*Task, error) { + return &Task{ + ID: 1, + ExecutionID: 1, + }, nil +} diff --git a/src/pkg/retention/mocks/api_controller.go b/src/pkg/retention/mocks/api_controller.go index 9c0ec3942..70a2fca60 100644 --- a/src/pkg/retention/mocks/api_controller.go +++ b/src/pkg/retention/mocks/api_controller.go @@ -119,6 +119,14 @@ func (_m *APIController) GetRetentionExecTaskLog(taskID int64) ([]byte, error) { return r0, r1 } +// GetRetentionExecTask provides a mock function with given fields: taskID +func (_m *APIController) GetRetentionExecTask(taskID int64) (*retention.Task, error) { + return &retention.Task{ + ID: 1, + ExecutionID: 1, + }, nil +} + // GetTotalOfRetentionExecTasks provides a mock function with given fields: executionID func (_m *APIController) GetTotalOfRetentionExecTasks(executionID int64) (int64, error) { ret := _m.Called(executionID) diff --git a/src/portal/src/app/project/webhook/webhook.service.ts b/src/portal/src/app/project/webhook/webhook.service.ts index 18d09247b..c04e6a493 100644 --- a/src/portal/src/app/project/webhook/webhook.service.ts +++ b/src/portal/src/app/project/webhook/webhook.service.ts @@ -30,6 +30,7 @@ const EVENT_TYPES_TEXT_MAP = { 'QUOTA_WARNING': 'Quota near threshold', 'SCANNING_FAILED': 'Scanning failed', 'SCANNING_COMPLETED': 'Scanning finished', + 'TAG_RETENTION': 'Tag retention finished', }; @Injectable()