diff --git a/.travis.yml b/.travis.yml index 9b3429fe8..7ba2076f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,8 @@ matrix: - go: 1.12.5 env: - OFFLINE=true - - node_js: 10.16.2 + - language: node_js + node_js: 10.16.2 env: - UI_UT=true env: diff --git a/docs/permissions.md b/docs/permissions.md index f732f0de6..2543ef3c3 100644 --- a/docs/permissions.md +++ b/docs/permissions.md @@ -43,3 +43,11 @@ The following table depicts the various user permission levels in a project. | Add/Remove labels of helm chart version | | ✓ | ✓ | ✓ | | See a list of project robots | | | ✓ | ✓ | | Create/edit/delete project robots | | | | ✓ | +| See configured CVE whitelist | ✓ | ✓ | ✓ | ✓ | +| Create/edit/remove CVE whitelist | | | | ✓ | +| Enable/disable webhooks | | ✓ | ✓ | ✓ | +| Create/delete tag retention rules | | ✓ | ✓ | ✓ | +| Enable/disable tag retention rules | | ✓ | ✓ | ✓ | +| See project quotas | ✓ | ✓ | ✓ | ✓ | +| Edit project quotas | | | | | + diff --git a/make/migrations/postgresql/0011_1.10.0_schema.up.sql b/make/migrations/postgresql/0011_1.10.0_schema.up.sql new file mode 100644 index 000000000..e03f84b09 --- /dev/null +++ b/make/migrations/postgresql/0011_1.10.0_schema.up.sql @@ -0,0 +1,34 @@ +/*Table for keeping the plug scanner registration*/ +CREATE TABLE scanner_registration +( + id SERIAL PRIMARY KEY NOT NULL, + uuid VARCHAR(64) UNIQUE NOT NULL, + url VARCHAR(256) UNIQUE NOT NULL, + name VARCHAR(128) UNIQUE NOT NULL, + description VARCHAR(1024) NULL, + auth VARCHAR(16) NOT NULL, + access_cred VARCHAR(512) NULL, + adapter VARCHAR(128) NOT NULL, + vendor VARCHAR(128) NOT NULL, + version VARCHAR(32) NOT NULL, + disabled BOOLEAN NOT NULL DEFAULT FALSE, + is_default BOOLEAN NOT NULL DEFAULT FALSE, + skip_cert_verify BOOLEAN NOT NULL DEFAULT FALSE, + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +/*Table for keeping the scanner report. The report details are stored as JSONB*/ +CREATE TABLE scanner_report +( + id SERIAL PRIMARY KEY NOT NULL, + digest VARCHAR(256) NOT NULL, + registration_id VARCHAR(64) NOT NULL, + job_id VARCHAR(32), + status VARCHAR(16) NOT NULL, + status_code INTEGER DEFAULT 0, + report JSON, + start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + end_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(digest, registration_id) +) diff --git a/make/photon/portal/Dockerfile b/make/photon/portal/Dockerfile index 9f71410f7..004d0113a 100644 --- a/make/photon/portal/Dockerfile +++ b/make/photon/portal/Dockerfile @@ -6,6 +6,7 @@ COPY ./LICENSE /portal_src WORKDIR /build_dir + RUN cp -r /portal_src/* /build_dir \ && ls -la \ && apt-get update \ @@ -14,7 +15,7 @@ RUN cp -r /portal_src/* /build_dir \ && npm install \ && npm run build_lib \ && npm run link_lib \ - && npm run release + && node --max_old_space_size=8192 'node_modules/@angular/cli/bin/ng' build --prod FROM photon:2.0 diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index b6ed840b2..b13fb2f30 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -206,6 +206,13 @@ func init() { beego.Router("/api/internal/switchquota", &InternalAPI{}, "put:SwitchQuota") beego.Router("/api/internal/syncquota", &InternalAPI{}, "post:SyncQuota") + // Add routes for plugin scanner management + scannerAPI := &ScannerAPI{} + beego.Router("/api/scanners", scannerAPI, "post:Create;get:List") + beego.Router("/api/scanners/:uid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault") + // Add routes for project level scanner + beego.Router("/api/projects/:pid([0-9]+)/scanner", scannerAPI, "get:GetProjectScanner;put:SetProjectScanner") + // syncRegistry if err := SyncRegistry(config.GlobalProjectMgr); err != nil { log.Fatalf("failed to sync repositories from registry: %v", err) diff --git a/src/core/api/plug_scanners.go b/src/core/api/plug_scanners.go new file mode 100644 index 000000000..d14424675 --- /dev/null +++ b/src/core/api/plug_scanners.go @@ -0,0 +1,348 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "fmt" + "net/http" + + "github.com/goharbor/harbor/src/pkg/q" + "github.com/goharbor/harbor/src/pkg/scan/scanner/api" + "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner" + "github.com/pkg/errors" +) + +// ScannerAPI provides the API for managing the plugin scanners +type ScannerAPI struct { + // The base controller to provide common utilities + BaseController + + // Controller for the plug scanners + c api.Controller +} + +// Prepare sth. for the subsequent actions +func (sa *ScannerAPI) Prepare() { + // Call super prepare method + sa.BaseController.Prepare() + + // Check access permissions + if !sa.SecurityCtx.IsAuthenticated() { + sa.SendUnAuthorizedError(errors.New("UnAuthorized")) + return + } + + if !sa.SecurityCtx.IsSysAdmin() { + sa.SendForbiddenError(errors.New(sa.SecurityCtx.GetUsername())) + return + } + + // Use the default controller + sa.c = api.DefaultController +} + +// Get the specified scanner +func (sa *ScannerAPI) Get() { + if r := sa.get(); r != nil { + // Response to the client + sa.Data["json"] = r + sa.ServeJSON() + } +} + +// List all the scanners +func (sa *ScannerAPI) List() { + p, pz, err := sa.GetPaginationParams() + if err != nil { + sa.SendBadRequestError(errors.Wrap(err, "scanner API: list all")) + return + } + + query := &q.Query{ + PageSize: pz, + PageNumber: p, + } + + // Get query key words + kws := make(map[string]string) + properties := []string{"name", "description", "url"} + for _, k := range properties { + kw := sa.GetString(k) + if len(kw) > 0 { + kws[k] = kw + } + } + + if len(kws) > 0 { + query.Keywords = kws + } + + all, err := sa.c.ListRegistrations(query) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: list all")) + return + } + + // Response to the client + sa.Data["json"] = all + sa.ServeJSON() +} + +// Create a new scanner +func (sa *ScannerAPI) Create() { + r := &scanner.Registration{} + + if err := sa.DecodeJSONReq(r); err != nil { + sa.SendBadRequestError(errors.Wrap(err, "scanner API: create")) + return + } + + if err := r.Validate(false); err != nil { + sa.SendBadRequestError(errors.Wrap(err, "scanner API: create")) + return + } + + // Explicitly check if conflict + if !sa.checkDuplicated("name", r.Name) || + !sa.checkDuplicated("url", r.URL) { + return + } + + // All newly created should be non default one except the 1st one + r.IsDefault = false + + uuid, err := sa.c.CreateRegistration(r) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: create")) + return + } + + location := fmt.Sprintf("%s/%s", sa.Ctx.Request.RequestURI, uuid) + sa.Ctx.ResponseWriter.Header().Add("Location", location) + + resp := make(map[string]string, 1) + resp["uuid"] = uuid + + // Response to the client + sa.Ctx.ResponseWriter.WriteHeader(http.StatusCreated) + sa.Data["json"] = resp + sa.ServeJSON() +} + +// Update a scanner +func (sa *ScannerAPI) Update() { + r := sa.get() + if r == nil { + // meet error + return + } + + // full dose updated + rr := &scanner.Registration{} + if err := sa.DecodeJSONReq(rr); err != nil { + sa.SendBadRequestError(errors.Wrap(err, "scanner API: update")) + return + } + + if err := r.Validate(true); err != nil { + sa.SendBadRequestError(errors.Wrap(err, "scanner API: update")) + return + } + + // Name changed? + if r.Name != rr.Name { + if !sa.checkDuplicated("name", rr.Name) { + return + } + } + + // URL changed? + if r.URL != rr.URL { + if !sa.checkDuplicated("url", rr.URL) { + return + } + } + + getChanges(r, rr) + + if err := sa.c.UpdateRegistration(r); err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: update")) + return + } + + location := fmt.Sprintf("%s/%s", sa.Ctx.Request.RequestURI, r.UUID) + sa.Ctx.ResponseWriter.Header().Add("Location", location) + + // Response to the client + sa.Data["json"] = r + sa.ServeJSON() +} + +// Delete the scanner +func (sa *ScannerAPI) Delete() { + uid := sa.GetStringFromPath(":uid") + if len(uid) == 0 { + sa.SendBadRequestError(errors.New("missing uid")) + return + } + + deleted, err := sa.c.DeleteRegistration(uid) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: delete")) + return + } + + if deleted == nil { + // Not found + sa.SendNotFoundError(errors.Errorf("scanner registration: %s", uid)) + return + } + + sa.Data["json"] = deleted + sa.ServeJSON() +} + +// SetAsDefault sets the given registration as default one +func (sa *ScannerAPI) SetAsDefault() { + uid := sa.GetStringFromPath(":uid") + if len(uid) == 0 { + sa.SendBadRequestError(errors.New("missing uid")) + return + } + + m := make(map[string]interface{}) + if err := sa.DecodeJSONReq(&m); err != nil { + sa.SendBadRequestError(errors.Wrap(err, "scanner API: set as default")) + return + } + + if v, ok := m["is_default"]; ok { + if isDefault, y := v.(bool); y && isDefault { + if err := sa.c.SetDefaultRegistration(uid); err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: set as default")) + } + + return + } + } + + // Not supported + sa.SendForbiddenError(errors.Errorf("not supported: %#v", m)) +} + +// GetProjectScanner gets the project level scanner +func (sa *ScannerAPI) GetProjectScanner() { + pid, err := sa.GetInt64FromPath(":pid") + if err != nil { + sa.SendBadRequestError(errors.Wrap(err, "scanner API: get project scanners")) + return + } + + r, err := sa.c.GetRegistrationByProject(pid) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: get project scanners")) + return + } + + if r != nil { + sa.Data["json"] = r + } else { + sa.Data["json"] = make(map[string]interface{}) + } + + sa.ServeJSON() +} + +// SetProjectScanner sets the project level scanner +func (sa *ScannerAPI) SetProjectScanner() { + pid, err := sa.GetInt64FromPath(":pid") + if err != nil { + sa.SendBadRequestError(errors.Wrap(err, "scanner API: set project scanners")) + return + } + + body := make(map[string]string) + if err := sa.DecodeJSONReq(&body); err != nil { + sa.SendBadRequestError(errors.Wrap(err, "scanner API: set project scanners")) + return + } + + uuid, ok := body["uuid"] + if !ok || len(uuid) == 0 { + sa.SendBadRequestError(errors.New("missing scanner uuid when setting project scanner")) + return + } + + if err := sa.c.SetRegistrationByProject(pid, uuid); err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: set project scanners")) + return + } +} + +// get the specified scanner +func (sa *ScannerAPI) get() *scanner.Registration { + uid := sa.GetStringFromPath(":uid") + if len(uid) == 0 { + sa.SendBadRequestError(errors.New("missing uid")) + return nil + } + + r, err := sa.c.GetRegistration(uid) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: get")) + return nil + } + + if r == nil { + // NOT found + sa.SendNotFoundError(errors.Errorf("scanner: %s", uid)) + return nil + } + + return r +} + +func (sa *ScannerAPI) checkDuplicated(property, value string) bool { + // Explicitly check if conflict + kw := make(map[string]string) + kw[property] = value + + query := &q.Query{ + Keywords: kw, + } + + l, err := sa.c.ListRegistrations(query) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: check existence")) + return false + } + + if len(l) > 0 { + sa.SendConflictError(errors.Errorf("duplicated entries: %s:%s", property, value)) + return false + } + + return true +} + +func getChanges(e *scanner.Registration, eChange *scanner.Registration) { + e.Name = eChange.Name + e.Description = eChange.Description + e.URL = eChange.URL + e.Auth = eChange.Auth + e.AccessCredential = eChange.AccessCredential + e.Disabled = eChange.Disabled + e.SkipCertVerify = eChange.SkipCertVerify +} diff --git a/src/core/api/plug_scanners_test.go b/src/core/api/plug_scanners_test.go new file mode 100644 index 000000000..f6beb9530 --- /dev/null +++ b/src/core/api/plug_scanners_test.go @@ -0,0 +1,444 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "fmt" + "net/http" + "testing" + + "github.com/goharbor/harbor/src/pkg/q" + "github.com/goharbor/harbor/src/pkg/scan/scanner/api" + dscan "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scan" + "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner" + "github.com/goharbor/harbor/src/pkg/scan/scanner/scan" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +const ( + rootRoute = "/api/scanners" +) + +// ScannerAPITestSuite is test suite for testing the scanner API +type ScannerAPITestSuite struct { + suite.Suite + + originC api.Controller + mockC *MockScannerAPIController +} + +// TestScannerAPI is the entry of ScannerAPITestSuite +func TestScannerAPI(t *testing.T) { + suite.Run(t, new(ScannerAPITestSuite)) +} + +// SetupSuite prepares testing env +func (suite *ScannerAPITestSuite) SetupTest() { + suite.originC = api.DefaultController + m := &MockScannerAPIController{} + api.DefaultController = m + + suite.mockC = m +} + +// TearDownTest clears test case env +func (suite *ScannerAPITestSuite) TearDownTest() { + // Restore + api.DefaultController = suite.originC +} + +// TestScannerAPICreate tests the post request to create new one +func (suite *ScannerAPITestSuite) TestScannerAPIBase() { + // Including general cases + cases := []*codeCheckingCase{ + // 401 + { + request: &testingRequest{ + url: rootRoute, + method: http.MethodPost, + }, + code: http.StatusUnauthorized, + }, + // 403 + { + request: &testingRequest{ + url: rootRoute, + method: http.MethodPost, + credential: nonSysAdmin, + }, + code: http.StatusForbidden, + }, + // 400 + { + request: &testingRequest{ + url: rootRoute, + method: http.MethodPost, + credential: sysAdmin, + bodyJSON: &scanner.Registration{ + URL: "http://a.b.c", + }, + }, + code: http.StatusBadRequest, + }, + } + + runCodeCheckingCases(suite.T(), cases...) +} + +// TestScannerAPIGet tests api get +func (suite *ScannerAPITestSuite) TestScannerAPIGet() { + res := &scanner.Registration{ + ID: 1000, + UUID: "uuid", + Name: "TestScannerAPIGet", + Description: "JUST FOR TEST", + URL: "https://a.b.c", + Adapter: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + } + suite.mockC.On("GetRegistration", "uuid").Return(res, nil) + + // Get + rr := &scanner.Registration{} + err := handleAndParse(&testingRequest{ + url: fmt.Sprintf("%s/%s", rootRoute, "uuid"), + method: http.MethodGet, + credential: sysAdmin, + }, rr) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), rr) + assert.Equal(suite.T(), res.Name, rr.Name) + assert.Equal(suite.T(), res.UUID, rr.UUID) +} + +// TestScannerAPICreate tests create. +func (suite *ScannerAPITestSuite) TestScannerAPICreate() { + r := &scanner.Registration{ + Name: "TestScannerAPICreate", + Description: "JUST FOR TEST", + URL: "https://a.b.c", + Adapter: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + } + + suite.mockQuery(r) + suite.mockC.On("CreateRegistration", r).Return("uuid", nil) + + // Create + res := make(map[string]string, 1) + err := handleAndParse( + &testingRequest{ + url: rootRoute, + method: http.MethodPost, + credential: sysAdmin, + bodyJSON: r, + }, &res) + require.NoError(suite.T(), err) + require.Condition(suite.T(), func() (success bool) { + success = res["uuid"] == "uuid" + return + }) +} + +// TestScannerAPIList tests list +func (suite *ScannerAPITestSuite) TestScannerAPIList() { + query := &q.Query{ + PageNumber: 1, + PageSize: 500, + } + ll := []*scanner.Registration{ + { + ID: 1001, + UUID: "uuid", + Name: "TestScannerAPIList", + Description: "JUST FOR TEST", + URL: "https://a.b.c", + Adapter: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + }} + suite.mockC.On("ListRegistrations", query).Return(ll, nil) + + // List + l := make([]*scanner.Registration, 0) + err := handleAndParse(&testingRequest{ + url: rootRoute, + method: http.MethodGet, + credential: sysAdmin, + }, &l) + require.NoError(suite.T(), err) + assert.Condition(suite.T(), func() (success bool) { + success = len(l) > 0 && l[0].Name == ll[0].Name + return + }) +} + +// TestScannerAPIUpdate tests the update API +func (suite *ScannerAPITestSuite) TestScannerAPIUpdate() { + before := &scanner.Registration{ + ID: 1002, + UUID: "uuid", + Name: "TestScannerAPIUpdate_before", + Description: "JUST FOR TEST", + URL: "https://a.b.c", + Adapter: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + } + + updated := &scanner.Registration{ + ID: 1002, + UUID: "uuid", + Name: "TestScannerAPIUpdate", + Description: "JUST FOR TEST", + URL: "https://a.b.c", + Adapter: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + } + + suite.mockQuery(updated) + suite.mockC.On("UpdateRegistration", updated).Return(nil) + suite.mockC.On("GetRegistration", "uuid").Return(before, nil) + + rr := &scanner.Registration{} + err := handleAndParse(&testingRequest{ + url: fmt.Sprintf("%s/%s", rootRoute, "uuid"), + method: http.MethodPut, + credential: sysAdmin, + bodyJSON: updated, + }, rr) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), rr) + + assert.Equal(suite.T(), updated.Name, rr.Name) + assert.Equal(suite.T(), updated.UUID, rr.UUID) +} + +// +func (suite *ScannerAPITestSuite) TestScannerAPIDelete() { + r := &scanner.Registration{ + ID: 1003, + UUID: "uuid", + Name: "TestScannerAPIDelete", + Description: "JUST FOR TEST", + URL: "https://a.b.c", + Adapter: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + } + + suite.mockC.On("DeleteRegistration", "uuid").Return(r, nil) + + deleted := &scanner.Registration{} + err := handleAndParse(&testingRequest{ + url: fmt.Sprintf("%s/%s", rootRoute, "uuid"), + method: http.MethodDelete, + credential: sysAdmin, + }, deleted) + + require.NoError(suite.T(), err) + assert.Equal(suite.T(), r.UUID, deleted.UUID) + assert.Equal(suite.T(), r.Name, deleted.Name) +} + +// TestScannerAPISetDefault tests the set default +func (suite *ScannerAPITestSuite) TestScannerAPISetDefault() { + suite.mockC.On("SetDefaultRegistration", "uuid").Return(nil) + + body := make(map[string]interface{}, 1) + body["is_default"] = true + runCodeCheckingCases(suite.T(), &codeCheckingCase{ + request: &testingRequest{ + url: fmt.Sprintf("%s/%s", rootRoute, "uuid"), + method: http.MethodPatch, + credential: sysAdmin, + bodyJSON: body, + }, + code: http.StatusOK, + }) +} + +// TestScannerAPIProjectScanner tests the API of getting/setting project level scanner +func (suite *ScannerAPITestSuite) TestScannerAPIProjectScanner() { + suite.mockC.On("SetRegistrationByProject", int64(1), "uuid").Return(nil) + + // Set + body := make(map[string]interface{}, 1) + body["uuid"] = "uuid" + runCodeCheckingCases(suite.T(), &codeCheckingCase{ + request: &testingRequest{ + url: fmt.Sprintf("/api/projects/%d/scanner", 1), + method: http.MethodPut, + credential: sysAdmin, + bodyJSON: body, + }, + code: http.StatusOK, + }) + + r := &scanner.Registration{ + ID: 1004, + UUID: "uuid", + Name: "TestScannerAPIProjectScanner", + Description: "JUST FOR TEST", + URL: "https://a.b.c", + Adapter: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + } + suite.mockC.On("GetRegistrationByProject", int64(1)).Return(r, nil) + + // Get + rr := &scanner.Registration{} + err := handleAndParse(&testingRequest{ + url: fmt.Sprintf("/api/projects/%d/scanner", 1), + method: http.MethodGet, + credential: sysAdmin, + }, rr) + require.NoError(suite.T(), err) + + assert.Equal(suite.T(), r.Name, rr.Name) + assert.Equal(suite.T(), r.UUID, rr.UUID) +} + +func (suite *ScannerAPITestSuite) mockQuery(r *scanner.Registration) { + kw := make(map[string]string, 1) + kw["name"] = r.Name + query := &q.Query{ + Keywords: kw, + } + emptyL := make([]*scanner.Registration, 0) + suite.mockC.On("ListRegistrations", query).Return(emptyL, nil) + + kw2 := make(map[string]string, 1) + kw2["url"] = r.URL + query2 := &q.Query{ + Keywords: kw2, + } + suite.mockC.On("ListRegistrations", query2).Return(emptyL, nil) +} + +// MockScannerAPIController is mock of scanner API controller +type MockScannerAPIController struct { + mock.Mock +} + +// ListRegistrations ... +func (m *MockScannerAPIController) ListRegistrations(query *q.Query) ([]*scanner.Registration, error) { + args := m.Called(query) + return args.Get(0).([]*scanner.Registration), args.Error(1) +} + +// CreateRegistration ... +func (m *MockScannerAPIController) CreateRegistration(registration *scanner.Registration) (string, error) { + args := m.Called(registration) + return args.String(0), args.Error(1) +} + +// GetRegistration ... +func (m *MockScannerAPIController) GetRegistration(registrationUUID string) (*scanner.Registration, error) { + args := m.Called(registrationUUID) + s := args.Get(0) + if s == nil { + return nil, args.Error(1) + } + + return s.(*scanner.Registration), args.Error(1) +} + +// RegistrationExists ... +func (m *MockScannerAPIController) RegistrationExists(registrationUUID string) bool { + args := m.Called(registrationUUID) + return args.Bool(0) +} + +// UpdateRegistration ... +func (m *MockScannerAPIController) UpdateRegistration(registration *scanner.Registration) error { + args := m.Called(registration) + return args.Error(0) +} + +// DeleteRegistration ... +func (m *MockScannerAPIController) DeleteRegistration(registrationUUID string) (*scanner.Registration, error) { + args := m.Called(registrationUUID) + s := args.Get(0) + if s == nil { + return nil, args.Error(1) + } + + return s.(*scanner.Registration), args.Error(1) +} + +// SetDefaultRegistration ... +func (m *MockScannerAPIController) SetDefaultRegistration(registrationUUID string) error { + args := m.Called(registrationUUID) + return args.Error(0) +} + +// SetRegistrationByProject ... +func (m *MockScannerAPIController) SetRegistrationByProject(projectID int64, scannerID string) error { + args := m.Called(projectID, scannerID) + return args.Error(0) +} + +// GetRegistrationByProject ... +func (m *MockScannerAPIController) GetRegistrationByProject(projectID int64) (*scanner.Registration, error) { + args := m.Called(projectID) + s := args.Get(0) + if s == nil { + return nil, args.Error(1) + } + + return s.(*scanner.Registration), args.Error(1) +} + +// Ping ... +func (m *MockScannerAPIController) Ping(registration *scanner.Registration) error { + args := m.Called(registration) + return args.Error(0) +} + +// Scan ... +func (m *MockScannerAPIController) Scan(artifact *scan.Artifact) error { + args := m.Called(artifact) + return args.Error(0) +} + +// GetReport ... +func (m *MockScannerAPIController) GetReport(artifact *scan.Artifact) ([]*dscan.Report, error) { + args := m.Called(artifact) + r := args.Get(0) + if r == nil { + return nil, args.Error(1) + } + + return r.([]*dscan.Report), args.Error(1) +} + +// GetScanLog ... +func (m *MockScannerAPIController) GetScanLog(digest string) ([]byte, error) { + args := m.Called(digest) + l := args.Get(0) + if l == nil { + return nil, args.Error(1) + } + + return l.([]byte), args.Error(1) +} diff --git a/src/core/api/project.go b/src/core/api/project.go index 77285453c..40559991a 100644 --- a/src/core/api/project.go +++ b/src/core/api/project.go @@ -234,6 +234,12 @@ func (p *ProjectAPI) Post() { // Head ... func (p *ProjectAPI) Head() { + + if !p.SecurityCtx.IsAuthenticated() { + p.SendUnAuthorizedError(errors.New("Unauthorized")) + return + } + name := p.GetString("project_name") if len(name) == 0 { p.SendBadRequestError(errors.New("project_name is needed")) diff --git a/src/core/api/project_test.go b/src/core/api/project_test.go index 2c2d3d8fe..9944fbcbb 100644 --- a/src/core/api/project_test.go +++ b/src/core/api/project_test.go @@ -329,13 +329,13 @@ func TestDeleteProject(t *testing.T) { } func TestProHead(t *testing.T) { - fmt.Println("\nTest for Project HEAD API") + t.Log("\nTest for Project HEAD API") assert := assert.New(t) apiTest := newHarborAPI() // ----------------------------case 1 : Response Code=200----------------------------// - fmt.Println("case 1: response code:200") + t.Log("case 1: response code:200") httpStatusCode, err := apiTest.ProjectsHead(*admin, "library") if err != nil { t.Error("Error while search project by proName", err.Error()) @@ -345,7 +345,7 @@ func TestProHead(t *testing.T) { } // ----------------------------case 2 : Response Code=404:Project name does not exist.----------------------------// - fmt.Println("case 2: response code:404,Project name does not exist.") + t.Log("case 2: response code:404,Project name does not exist.") httpStatusCode, err = apiTest.ProjectsHead(*admin, "libra") if err != nil { t.Error("Error while search project by proName", err.Error()) @@ -354,6 +354,24 @@ func TestProHead(t *testing.T) { assert.Equal(int(404), httpStatusCode, "httpStatusCode should be 404") } + t.Log("case 3: response code:401. Project exist with unauthenticated user") + httpStatusCode, err = apiTest.ProjectsHead(*unknownUsr, "library") + if err != nil { + t.Error("Error while search project by proName", err.Error()) + t.Log(err) + } else { + assert.Equal(int(401), httpStatusCode, "httpStatusCode should be 404") + } + + t.Log("case 4: response code:401. Project name does not exist with unauthenticated user") + httpStatusCode, err = apiTest.ProjectsHead(*unknownUsr, "libra") + if err != nil { + t.Error("Error while search project by proName", err.Error()) + t.Log(err) + } else { + assert.Equal(int(401), httpStatusCode, "httpStatusCode should be 404") + } + fmt.Printf("\n") } diff --git a/src/core/controllers/base.go b/src/core/controllers/base.go index 301dba41f..6337973fb 100644 --- a/src/core/controllers/base.go +++ b/src/core/controllers/base.go @@ -25,8 +25,6 @@ import ( "strconv" "strings" - "github.com/goharbor/harbor/src/core/filter" - "github.com/astaxie/beego" "github.com/beego/i18n" "github.com/goharbor/harbor/src/common" @@ -37,6 +35,7 @@ import ( "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/auth" "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/core/filter" ) const userKey = "user" diff --git a/src/core/router.go b/src/core/router.go index 7e01b934e..deb862c6a 100755 --- a/src/core/router.go +++ b/src/core/router.go @@ -192,6 +192,13 @@ func initRouters() { beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels/:id([0-9]+)", chartLabelAPIType, "delete:RemoveLabel") } + // Add routes for plugin scanner management + scannerAPI := &api.ScannerAPI{} + beego.Router("/api/scanners", scannerAPI, "post:Create;get:List") + beego.Router("/api/scanners/:uid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault") + // Add routes for project level scanner + beego.Router("/api/projects/:pid([0-9]+)/scanner", scannerAPI, "get:GetProjectScanner;put:SetProjectScanner") + // Error pages beego.ErrorController(&controllers.ErrorController{}) diff --git a/src/go.mod b/src/go.mod index fdc8554c8..11b8d578a 100644 --- a/src/go.mod +++ b/src/go.mod @@ -45,6 +45,7 @@ require ( github.com/google/certificate-transparency-go v1.0.21 // indirect github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect + github.com/google/uuid v1.1.1 github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.6.2 github.com/graph-gophers/dataloader v5.0.0+incompatible diff --git a/src/go.sum b/src/go.sum index 5ac4284ff..aa24c1d75 100644 --- a/src/go.sum +++ b/src/go.sum @@ -140,6 +140,8 @@ github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeq github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= diff --git a/src/jobservice/job/impl/replication/replication_test.go b/src/jobservice/job/impl/replication/replication_test.go index 8abcd65dc..81c8d36bc 100644 --- a/src/jobservice/job/impl/replication/replication_test.go +++ b/src/jobservice/job/impl/replication/replication_test.go @@ -88,10 +88,10 @@ func (f *fakedTransfer) Transfer(src *model.Resource, dst *model.Resource) error } func TestRun(t *testing.T) { - err := transfer.RegisterFactory("res", fakedTransferFactory) + err := transfer.RegisterFactory("art", fakedTransferFactory) require.Nil(t, err) params := map[string]interface{}{ - "src_resource": `{"type":"res"}`, + "src_resource": `{"type":"art"}`, "dst_resource": `{}`, } rep := &Replication{} diff --git a/src/pkg/retention/res/candidate.go b/src/pkg/art/candidate.go similarity index 99% rename from src/pkg/retention/res/candidate.go rename to src/pkg/art/candidate.go index 15f5e8088..f44e22b99 100644 --- a/src/pkg/retention/res/candidate.go +++ b/src/pkg/art/candidate.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package res +package art import ( "encoding/base64" diff --git a/src/pkg/retention/res/result.go b/src/pkg/art/result.go similarity index 98% rename from src/pkg/retention/res/result.go rename to src/pkg/art/result.go index be91be04a..43d09b29d 100644 --- a/src/pkg/retention/res/result.go +++ b/src/pkg/art/result.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package res +package art // Result keeps the action result type Result struct { diff --git a/src/pkg/retention/res/selector.go b/src/pkg/art/selector.go similarity index 98% rename from src/pkg/retention/res/selector.go rename to src/pkg/art/selector.go index de0d34836..4e7bbcdb0 100644 --- a/src/pkg/retention/res/selector.go +++ b/src/pkg/art/selector.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package res +package art // Selector is used to filter the inputting list type Selector interface { diff --git a/src/pkg/retention/res/selectors/doublestar/selector.go b/src/pkg/art/selectors/doublestar/selector.go similarity index 92% rename from src/pkg/retention/res/selectors/doublestar/selector.go rename to src/pkg/art/selectors/doublestar/selector.go index fcbb628b9..274dae730 100644 --- a/src/pkg/retention/res/selectors/doublestar/selector.go +++ b/src/pkg/art/selectors/doublestar/selector.go @@ -16,7 +16,7 @@ package doublestar import ( "github.com/bmatcuk/doublestar" - "github.com/goharbor/harbor/src/pkg/retention/res" + "github.com/goharbor/harbor/src/pkg/art" ) const ( @@ -46,7 +46,7 @@ type selector struct { } // Select candidates by regular expressions -func (s *selector) Select(artifacts []*res.Candidate) (selected []*res.Candidate, err error) { +func (s *selector) Select(artifacts []*art.Candidate) (selected []*art.Candidate, err error) { value := "" excludes := false @@ -86,7 +86,7 @@ func (s *selector) Select(artifacts []*res.Candidate) (selected []*res.Candidate } // New is factory method for doublestar selector -func New(decoration string, pattern string) res.Selector { +func New(decoration string, pattern string) art.Selector { return &selector{ decoration: decoration, pattern: pattern, diff --git a/src/pkg/retention/res/selectors/doublestar/selector_test.go b/src/pkg/art/selectors/doublestar/selector_test.go similarity index 96% rename from src/pkg/retention/res/selectors/doublestar/selector_test.go rename to src/pkg/art/selectors/doublestar/selector_test.go index 23c8dd377..f511f8ebe 100644 --- a/src/pkg/retention/res/selectors/doublestar/selector_test.go +++ b/src/pkg/art/selectors/doublestar/selector_test.go @@ -16,7 +16,7 @@ package doublestar import ( "fmt" - "github.com/goharbor/harbor/src/pkg/retention/res" + "github.com/goharbor/harbor/src/pkg/art" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -28,7 +28,7 @@ import ( type RegExpSelectorTestSuite struct { suite.Suite - artifacts []*res.Candidate + artifacts []*art.Candidate } // TestRegExpSelector is entrance for RegExpSelectorTestSuite @@ -38,13 +38,13 @@ func TestRegExpSelector(t *testing.T) { // SetupSuite to do preparation work func (suite *RegExpSelectorTestSuite) SetupSuite() { - suite.artifacts = []*res.Candidate{ + suite.artifacts = []*art.Candidate{ { NamespaceID: 1, Namespace: "library", Repository: "harbor", Tag: "latest", - Kind: res.Image, + Kind: art.Image, PushedTime: time.Now().Unix() - 3600, PulledTime: time.Now().Unix(), CreationTime: time.Now().Unix() - 7200, @@ -55,7 +55,7 @@ func (suite *RegExpSelectorTestSuite) SetupSuite() { Namespace: "retention", Repository: "redis", Tag: "4.0", - Kind: res.Image, + Kind: art.Image, PushedTime: time.Now().Unix() - 3600, PulledTime: time.Now().Unix(), CreationTime: time.Now().Unix() - 7200, @@ -66,7 +66,7 @@ func (suite *RegExpSelectorTestSuite) SetupSuite() { Namespace: "retention", Repository: "redis", Tag: "4.1", - Kind: res.Image, + Kind: art.Image, PushedTime: time.Now().Unix() - 3600, PulledTime: time.Now().Unix(), CreationTime: time.Now().Unix() - 7200, @@ -235,7 +235,7 @@ func (suite *RegExpSelectorTestSuite) TestNSExcludes() { } // Check whether the returned result matched the expected ones (only check repo:tag) -func expect(expected []string, candidates []*res.Candidate) bool { +func expect(expected []string, candidates []*art.Candidate) bool { hash := make(map[string]bool) for _, art := range candidates { diff --git a/src/pkg/retention/res/selectors/index/index.go b/src/pkg/art/selectors/index/index.go similarity index 89% rename from src/pkg/retention/res/selectors/index/index.go rename to src/pkg/art/selectors/index/index.go index 690beef2d..8387de7cd 100644 --- a/src/pkg/retention/res/selectors/index/index.go +++ b/src/pkg/art/selectors/index/index.go @@ -17,8 +17,8 @@ package index import ( "sync" - "github.com/goharbor/harbor/src/pkg/retention/res" - "github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar" + "github.com/goharbor/harbor/src/pkg/art" + "github.com/goharbor/harbor/src/pkg/art/selectors/doublestar" "github.com/pkg/errors" ) @@ -49,11 +49,11 @@ type IndexedMeta struct { // indexedItem defined item kept in the index type indexedItem struct { Meta *IndexedMeta - Factory res.SelectorFactory + Factory art.SelectorFactory } // Register the selector with the corresponding selector kind and decoration -func Register(kind string, decorations []string, factory res.SelectorFactory) { +func Register(kind string, decorations []string, factory art.SelectorFactory) { if len(kind) == 0 || factory == nil { // do nothing return @@ -69,7 +69,7 @@ func Register(kind string, decorations []string, factory res.SelectorFactory) { } // Get selector with the provided kind and decoration -func Get(kind, decoration, pattern string) (res.Selector, error) { +func Get(kind, decoration, pattern string) (art.Selector, error) { if len(kind) == 0 || len(decoration) == 0 { return nil, errors.New("empty selector kind or decoration") } diff --git a/src/pkg/retention/res/selectors/label/selector.go b/src/pkg/art/selectors/label/selector.go similarity index 89% rename from src/pkg/retention/res/selectors/label/selector.go rename to src/pkg/art/selectors/label/selector.go index 2fa788a5a..c43616fd6 100644 --- a/src/pkg/retention/res/selectors/label/selector.go +++ b/src/pkg/art/selectors/label/selector.go @@ -17,7 +17,7 @@ package label import ( "strings" - "github.com/goharbor/harbor/src/pkg/retention/res" + "github.com/goharbor/harbor/src/pkg/art" ) const ( @@ -39,7 +39,7 @@ type selector struct { } // Select candidates by the labels -func (s *selector) Select(artifacts []*res.Candidate) (selected []*res.Candidate, err error) { +func (s *selector) Select(artifacts []*art.Candidate) (selected []*art.Candidate, err error) { for _, art := range artifacts { if isMatched(s.labels, art.Labels, s.decoration) { selected = append(selected, art) @@ -50,7 +50,7 @@ func (s *selector) Select(artifacts []*res.Candidate) (selected []*res.Candidate } // New is factory method for list selector -func New(decoration string, pattern string) res.Selector { +func New(decoration string, pattern string) art.Selector { labels := make([]string, 0) if len(pattern) > 0 { labels = append(labels, strings.Split(pattern, ",")...) diff --git a/src/pkg/retention/res/selectors/label/selector_test.go b/src/pkg/art/selectors/label/selector_test.go similarity index 94% rename from src/pkg/retention/res/selectors/label/selector_test.go rename to src/pkg/art/selectors/label/selector_test.go index 6bf58118a..6e028b62d 100644 --- a/src/pkg/retention/res/selectors/label/selector_test.go +++ b/src/pkg/art/selectors/label/selector_test.go @@ -16,7 +16,7 @@ package label import ( "fmt" - "github.com/goharbor/harbor/src/pkg/retention/res" + "github.com/goharbor/harbor/src/pkg/art" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -28,7 +28,7 @@ import ( type LabelSelectorTestSuite struct { suite.Suite - artifacts []*res.Candidate + artifacts []*art.Candidate } // TestLabelSelector is entrance for LabelSelectorTestSuite @@ -38,13 +38,13 @@ func TestLabelSelector(t *testing.T) { // SetupSuite to do preparation work func (suite *LabelSelectorTestSuite) SetupSuite() { - suite.artifacts = []*res.Candidate{ + suite.artifacts = []*art.Candidate{ { NamespaceID: 1, Namespace: "library", Repository: "harbor", Tag: "1.9", - Kind: res.Image, + Kind: art.Image, PushedTime: time.Now().Unix() - 3600, PulledTime: time.Now().Unix(), CreationTime: time.Now().Unix() - 7200, @@ -55,7 +55,7 @@ func (suite *LabelSelectorTestSuite) SetupSuite() { Namespace: "library", Repository: "harbor", Tag: "dev", - Kind: res.Image, + Kind: art.Image, PushedTime: time.Now().Unix() - 3600, PulledTime: time.Now().Unix(), CreationTime: time.Now().Unix() - 7200, @@ -131,7 +131,7 @@ func (suite *LabelSelectorTestSuite) TestWithoutNoneExistingLabels() { } // Check whether the returned result matched the expected ones (only check repo:tag) -func expect(expected []string, candidates []*res.Candidate) bool { +func expect(expected []string, candidates []*art.Candidate) bool { hash := make(map[string]bool) for _, art := range candidates { diff --git a/src/pkg/q/query.go b/src/pkg/q/query.go new file mode 100644 index 000000000..048a25298 --- /dev/null +++ b/src/pkg/q/query.go @@ -0,0 +1,25 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package q + +// Query parameters +type Query struct { + // Page number + PageNumber int64 + // Page size + PageSize int64 + // List of key words + Keywords map[string]string +} diff --git a/src/pkg/retention/dep/client.go b/src/pkg/retention/dep/client.go index c51d427e8..871b8a924 100644 --- a/src/pkg/retention/dep/client.go +++ b/src/pkg/retention/dep/client.go @@ -21,8 +21,8 @@ import ( "github.com/goharbor/harbor/src/common/http/modifier/auth" "github.com/goharbor/harbor/src/jobservice/config" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/clients/core" - "github.com/goharbor/harbor/src/pkg/retention/res" ) // DefaultClient for the retention @@ -33,30 +33,30 @@ type Client interface { // Get the tag candidates under the repository // // Arguments: - // repo *res.Repository : repository info + // repo *art.Repository : repository info // // Returns: - // []*res.Candidate : candidates returned + // []*art.Candidate : candidates returned // error : common error if any errors occurred - GetCandidates(repo *res.Repository) ([]*res.Candidate, error) + GetCandidates(repo *art.Repository) ([]*art.Candidate, error) // Delete the given repository // // Arguments: - // repo *res.Repository : repository info + // repo *art.Repository : repository info // // Returns: // error : common error if any errors occurred - DeleteRepository(repo *res.Repository) error + DeleteRepository(repo *art.Repository) error // Delete the specified candidate // // Arguments: - // candidate *res.Candidate : the deleting candidate + // candidate *art.Candidate : the deleting candidate // // Returns: // error : common error if any errors occurred - Delete(candidate *res.Candidate) error + Delete(candidate *art.Candidate) error } // NewClient new a basic client @@ -88,13 +88,13 @@ type basicClient struct { } // GetCandidates gets the tag candidates under the repository -func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candidate, error) { +func (bc *basicClient) GetCandidates(repository *art.Repository) ([]*art.Candidate, error) { if repository == nil { return nil, errors.New("repository is nil") } - candidates := make([]*res.Candidate, 0) + candidates := make([]*art.Candidate, 0) switch repository.Kind { - case res.Image: + case art.Image: images, err := bc.coreClient.ListAllImages(repository.Namespace, repository.Name) if err != nil { return nil, err @@ -104,8 +104,8 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida for _, label := range image.Labels { labels = append(labels, label.Name) } - candidate := &res.Candidate{ - Kind: res.Image, + candidate := &art.Candidate{ + Kind: art.Image, Namespace: repository.Namespace, Repository: repository.Name, Tag: image.Name, @@ -118,7 +118,7 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida candidates = append(candidates, candidate) } /* - case res.Chart: + case art.Chart: charts, err := bc.coreClient.ListAllCharts(repository.Namespace, repository.Name) if err != nil { return nil, err @@ -128,8 +128,8 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida for _, label := range chart.Labels { labels = append(labels, label.Name) } - candidate := &res.Candidate{ - Kind: res.Chart, + candidate := &art.Candidate{ + Kind: art.Chart, Namespace: repository.Namespace, Repository: repository.Name, Tag: chart.Name, @@ -148,15 +148,15 @@ func (bc *basicClient) GetCandidates(repository *res.Repository) ([]*res.Candida } // DeleteRepository deletes the specified repository -func (bc *basicClient) DeleteRepository(repo *res.Repository) error { +func (bc *basicClient) DeleteRepository(repo *art.Repository) error { if repo == nil { return errors.New("repository is nil") } switch repo.Kind { - case res.Image: + case art.Image: return bc.coreClient.DeleteImageRepository(repo.Namespace, repo.Name) /* - case res.Chart: + case art.Chart: return bc.coreClient.DeleteChartRepository(repo.Namespace, repo.Name) */ default: @@ -165,15 +165,15 @@ func (bc *basicClient) DeleteRepository(repo *res.Repository) error { } // Deletes the specified candidate -func (bc *basicClient) Delete(candidate *res.Candidate) error { +func (bc *basicClient) Delete(candidate *art.Candidate) error { if candidate == nil { return errors.New("candidate is nil") } switch candidate.Kind { - case res.Image: + case art.Image: return bc.coreClient.DeleteImage(candidate.Namespace, candidate.Repository, candidate.Tag) /* - case res.Chart: + case art.Chart: return bc.coreClient.DeleteChart(candidate.Namespace, candidate.Repository, candidate.Tag) */ default: diff --git a/src/pkg/retention/dep/client_test.go b/src/pkg/retention/dep/client_test.go index 071cc230c..90c0e38c8 100644 --- a/src/pkg/retention/dep/client_test.go +++ b/src/pkg/retention/dep/client_test.go @@ -21,7 +21,7 @@ import ( jmodels "github.com/goharbor/harbor/src/common/job/models" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/jobservice/job" - "github.com/goharbor/harbor/src/pkg/retention/res" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/testing/clients" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -73,33 +73,33 @@ type clientTestSuite struct { func (c *clientTestSuite) TestGetCandidates() { client := &basicClient{} client.coreClient = &fakeCoreClient{} - var repository *res.Repository + var repository *art.Repository // nil repository candidates, err := client.GetCandidates(repository) require.NotNil(c.T(), err) // image repository - repository = &res.Repository{} - repository.Kind = res.Image + repository = &art.Repository{} + repository.Kind = art.Image repository.Namespace = "library" repository.Name = "hello-world" candidates, err = client.GetCandidates(repository) require.Nil(c.T(), err) assert.Equal(c.T(), 1, len(candidates)) - assert.Equal(c.T(), res.Image, candidates[0].Kind) + assert.Equal(c.T(), art.Image, candidates[0].Kind) assert.Equal(c.T(), "library", candidates[0].Namespace) assert.Equal(c.T(), "hello-world", candidates[0].Repository) assert.Equal(c.T(), "latest", candidates[0].Tag) /* // chart repository - repository.Kind = res.Chart + repository.Kind = art.Chart repository.Namespace = "goharbor" repository.Name = "harbor" candidates, err = client.GetCandidates(repository) require.Nil(c.T(), err) assert.Equal(c.T(), 1, len(candidates)) - assert.Equal(c.T(), res.Chart, candidates[0].Kind) + assert.Equal(c.T(), art.Chart, candidates[0].Kind) assert.Equal(c.T(), "goharbor", candidates[0].Namespace) assert.Equal(c.T(), "1.0", candidates[0].Tag) */ @@ -109,20 +109,20 @@ func (c *clientTestSuite) TestDelete() { client := &basicClient{} client.coreClient = &fakeCoreClient{} - var candidate *res.Candidate + var candidate *art.Candidate // nil candidate err := client.Delete(candidate) require.NotNil(c.T(), err) // image - candidate = &res.Candidate{} - candidate.Kind = res.Image + candidate = &art.Candidate{} + candidate.Kind = art.Image err = client.Delete(candidate) require.Nil(c.T(), err) /* // chart - candidate.Kind = res.Chart + candidate.Kind = art.Chart err = client.Delete(candidate) require.Nil(c.T(), err) */ diff --git a/src/pkg/retention/job.go b/src/pkg/retention/job.go index 4839b7002..0944aa0c1 100644 --- a/src/pkg/retention/job.go +++ b/src/pkg/retention/job.go @@ -23,10 +23,10 @@ import ( "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/dep" "github.com/goharbor/harbor/src/pkg/retention/policy" "github.com/goharbor/harbor/src/pkg/retention/policy/lwp" - "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/olekukonko/tablewriter" "github.com/pkg/errors" ) @@ -116,7 +116,7 @@ func (pj *Job) Run(ctx job.Context, params job.Parameters) error { return saveRetainNum(ctx, results, allCandidates) } -func saveRetainNum(ctx job.Context, retained []*res.Result, allCandidates []*res.Candidate) error { +func saveRetainNum(ctx job.Context, retained []*art.Result, allCandidates []*art.Candidate) error { var delNum int for _, r := range retained { if r.Error == nil { @@ -138,7 +138,7 @@ func saveRetainNum(ctx job.Context, retained []*res.Result, allCandidates []*res return nil } -func logResults(logger logger.Interface, all []*res.Candidate, results []*res.Result) { +func logResults(logger logger.Interface, all []*art.Candidate, results []*art.Result) { hash := make(map[string]error, len(results)) for _, r := range results { if r.Target != nil { @@ -146,7 +146,7 @@ func logResults(logger logger.Interface, all []*res.Candidate, results []*res.Re } } - op := func(art *res.Candidate) string { + op := func(art *art.Candidate) string { if e, exists := hash[art.Hash()]; exists { if e != nil { return actionMarkError @@ -194,7 +194,7 @@ func logResults(logger logger.Interface, all []*res.Candidate, results []*res.Re } } -func arn(art *res.Candidate) string { +func arn(art *art.Candidate) string { return fmt.Sprintf("%s/%s:%s", art.Namespace, art.Repository, art.Tag) } @@ -237,7 +237,7 @@ func getParamDryRun(params job.Parameters) (bool, error) { return dryRun, nil } -func getParamRepo(params job.Parameters) (*res.Repository, error) { +func getParamRepo(params job.Parameters) (*art.Repository, error) { v, ok := params[ParamRepo] if !ok { return nil, errors.Errorf("missing parameter: %s", ParamRepo) @@ -248,7 +248,7 @@ func getParamRepo(params job.Parameters) (*res.Repository, error) { return nil, errors.Errorf("invalid parameter: %s", ParamRepo) } - repo := &res.Repository{} + repo := &art.Repository{} if err := repo.FromJSON(repoJSON); err != nil { return nil, errors.Wrap(err, "parse repository from JSON") } diff --git a/src/pkg/retention/job_test.go b/src/pkg/retention/job_test.go index 6d960ae64..27ab7410d 100644 --- a/src/pkg/retention/job_test.go +++ b/src/pkg/retention/job_test.go @@ -22,14 +22,14 @@ import ( "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/pkg/art" + "github.com/goharbor/harbor/src/pkg/art/selectors/doublestar" "github.com/goharbor/harbor/src/pkg/retention/dep" "github.com/goharbor/harbor/src/pkg/retention/policy" "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/lwp" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestps" - "github.com/goharbor/harbor/src/pkg/retention/res" - "github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -60,10 +60,10 @@ func (suite *JobTestSuite) TearDownSuite() { func (suite *JobTestSuite) TestRunSuccess() { params := make(job.Parameters) params[ParamDryRun] = false - repository := &res.Repository{ + repository := &art.Repository{ Namespace: "library", Name: "harbor", - Kind: res.Image, + Kind: art.Image, } repoJSON, err := repository.ToJSON() require.Nil(suite.T(), err) @@ -112,8 +112,8 @@ func (suite *JobTestSuite) TestRunSuccess() { type fakeRetentionClient struct{} // GetCandidates ... -func (frc *fakeRetentionClient) GetCandidates(repo *res.Repository) ([]*res.Candidate, error) { - return []*res.Candidate{ +func (frc *fakeRetentionClient) GetCandidates(repo *art.Repository) ([]*art.Candidate, error) { + return []*art.Candidate{ { Namespace: "library", Repository: "harbor", @@ -140,12 +140,12 @@ func (frc *fakeRetentionClient) GetCandidates(repo *res.Repository) ([]*res.Cand } // Delete ... -func (frc *fakeRetentionClient) Delete(candidate *res.Candidate) error { +func (frc *fakeRetentionClient) Delete(candidate *art.Candidate) error { return nil } // SubmitTask ... -func (frc *fakeRetentionClient) DeleteRepository(repo *res.Repository) error { +func (frc *fakeRetentionClient) DeleteRepository(repo *art.Repository) error { return nil } diff --git a/src/pkg/retention/launcher.go b/src/pkg/retention/launcher.go index c9f6d4655..09d264ee7 100644 --- a/src/pkg/retention/launcher.go +++ b/src/pkg/retention/launcher.go @@ -19,7 +19,7 @@ import ( "time" "github.com/goharbor/harbor/src/jobservice/job" - "github.com/goharbor/harbor/src/pkg/retention/res/selectors/index" + "github.com/goharbor/harbor/src/pkg/art/selectors/index" cjob "github.com/goharbor/harbor/src/common/job" "github.com/goharbor/harbor/src/common/job/models" @@ -27,12 +27,12 @@ import ( "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/project" "github.com/goharbor/harbor/src/pkg/repository" "github.com/goharbor/harbor/src/pkg/retention/policy" "github.com/goharbor/harbor/src/pkg/retention/policy/lwp" "github.com/goharbor/harbor/src/pkg/retention/q" - "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/pkg/errors" ) @@ -84,7 +84,7 @@ func NewLauncher(projectMgr project.Manager, repositoryMgr repository.Manager, type jobData struct { TaskID int64 - Repository res.Repository + Repository art.Repository JobName string JobParams map[string]interface{} } @@ -111,9 +111,9 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool if scope == nil { return 0, launcherError(fmt.Errorf("the scope of policy is nil")) } - repositoryRules := make(map[res.Repository]*lwp.Metadata, 0) + repositoryRules := make(map[art.Repository]*lwp.Metadata, 0) level := scope.Level - var allProjects []*res.Candidate + var allProjects []*art.Candidate var err error if level == "system" { // get projects @@ -144,12 +144,12 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool } } case "project": - projectCandidates = append(projectCandidates, &res.Candidate{ + projectCandidates = append(projectCandidates, &art.Candidate{ NamespaceID: scope.Reference, }) } - var repositoryCandidates []*res.Candidate + var repositoryCandidates []*art.Candidate // get repositories of projects for _, projectCandidate := range projectCandidates { repositories, err := getRepositories(l.projectMgr, l.repositoryMgr, projectCandidate.NamespaceID, l.chartServerEnabled) @@ -174,7 +174,7 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool } for _, repositoryCandidate := range repositoryCandidates { - reposit := res.Repository{ + reposit := art.Repository{ Namespace: repositoryCandidate.Namespace, Name: repositoryCandidate.Repository, Kind: repositoryCandidate.Kind, @@ -214,7 +214,7 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool return int64(len(jobDatas)), nil } -func createJobs(repositoryRules map[res.Repository]*lwp.Metadata, isDryRun bool) ([]*jobData, error) { +func createJobs(repositoryRules map[art.Repository]*lwp.Metadata, isDryRun bool) ([]*jobData, error) { jobDatas := []*jobData{} for repository, policy := range repositoryRules { jobData := &jobData{ @@ -320,14 +320,14 @@ func launcherError(err error) error { return errors.Wrap(err, "launcher") } -func getProjects(projectMgr project.Manager) ([]*res.Candidate, error) { +func getProjects(projectMgr project.Manager) ([]*art.Candidate, error) { projects, err := projectMgr.List() if err != nil { return nil, err } - var candidates []*res.Candidate + var candidates []*art.Candidate for _, pro := range projects { - candidates = append(candidates, &res.Candidate{ + candidates = append(candidates, &art.Candidate{ NamespaceID: pro.ProjectID, Namespace: pro.Name, }) @@ -336,8 +336,8 @@ func getProjects(projectMgr project.Manager) ([]*res.Candidate, error) { } func getRepositories(projectMgr project.Manager, repositoryMgr repository.Manager, - projectID int64, chartServerEnabled bool) ([]*res.Candidate, error) { - var candidates []*res.Candidate + projectID int64, chartServerEnabled bool) ([]*art.Candidate, error) { + var candidates []*art.Candidate /* pro, err := projectMgr.Get(projectID) if err != nil { @@ -351,7 +351,7 @@ func getRepositories(projectMgr project.Manager, repositoryMgr repository.Manage } for _, r := range imageRepositories { namespace, repo := utils.ParseRepository(r.Name) - candidates = append(candidates, &res.Candidate{ + candidates = append(candidates, &art.Candidate{ Namespace: namespace, Repository: repo, Kind: "image", @@ -366,7 +366,7 @@ func getRepositories(projectMgr project.Manager, repositoryMgr repository.Manage return nil, err } for _, r := range chartRepositories { - candidates = append(candidates, &res.Candidate{ + candidates = append(candidates, &art.Candidate{ Namespace: pro.Name, Repository: r.Name, Kind: "chart", diff --git a/src/pkg/retention/launcher_test.go b/src/pkg/retention/launcher_test.go index c63b7bf28..3048b15a9 100644 --- a/src/pkg/retention/launcher_test.go +++ b/src/pkg/retention/launcher_test.go @@ -21,12 +21,12 @@ import ( "github.com/goharbor/harbor/src/chartserver" "github.com/goharbor/harbor/src/common/job" "github.com/goharbor/harbor/src/common/models" + _ "github.com/goharbor/harbor/src/pkg/art/selectors/doublestar" "github.com/goharbor/harbor/src/pkg/project" "github.com/goharbor/harbor/src/pkg/repository" "github.com/goharbor/harbor/src/pkg/retention/policy" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" "github.com/goharbor/harbor/src/pkg/retention/q" - _ "github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar" hjob "github.com/goharbor/harbor/src/testing/job" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/src/pkg/retention/policy/action/index/index_test.go b/src/pkg/retention/policy/action/index/index_test.go index f9d4f57e5..a873fed3c 100644 --- a/src/pkg/retention/policy/action/index/index_test.go +++ b/src/pkg/retention/policy/action/index/index_test.go @@ -18,8 +18,8 @@ import ( "testing" "time" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/action" - "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -29,7 +29,7 @@ import ( type IndexTestSuite struct { suite.Suite - candidates []*res.Candidate + candidates []*art.Candidate } // TestIndexEntry is entry of IndexTestSuite @@ -41,7 +41,7 @@ func TestIndexEntry(t *testing.T) { func (suite *IndexTestSuite) SetupSuite() { Register("fakeAction", newFakePerformer) - suite.candidates = []*res.Candidate{{ + suite.candidates = []*art.Candidate{{ Namespace: "library", Repository: "harbor", Kind: "image", @@ -77,9 +77,9 @@ type fakePerformer struct { } // Perform the artifacts -func (p *fakePerformer) Perform(candidates []*res.Candidate) (results []*res.Result, err error) { +func (p *fakePerformer) Perform(candidates []*art.Candidate) (results []*art.Result, err error) { for _, c := range candidates { - results = append(results, &res.Result{ + results = append(results, &art.Result{ Target: c, }) } diff --git a/src/pkg/retention/policy/action/performer.go b/src/pkg/retention/policy/action/performer.go index 72d34d612..2461d0945 100644 --- a/src/pkg/retention/policy/action/performer.go +++ b/src/pkg/retention/policy/action/performer.go @@ -15,8 +15,8 @@ package action import ( + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/dep" - "github.com/goharbor/harbor/src/pkg/retention/res" ) const ( @@ -29,12 +29,12 @@ type Performer interface { // Perform the action // // Arguments: - // candidates []*res.Candidate : the targets to perform + // candidates []*art.Candidate : the targets to perform // // Returns: - // []*res.Result : result infos + // []*art.Result : result infos // error : common error if any errors occurred - Perform(candidates []*res.Candidate) ([]*res.Result, error) + Perform(candidates []*art.Candidate) ([]*art.Result, error) } // PerformerFactory is factory method for creating Performer @@ -42,13 +42,13 @@ type PerformerFactory func(params interface{}, isDryRun bool) Performer // retainAction make sure all the candidates will be retained and others will be cleared type retainAction struct { - all []*res.Candidate + all []*art.Candidate // Indicate if it is a dry run isDryRun bool } // Perform the action -func (ra *retainAction) Perform(candidates []*res.Candidate) (results []*res.Result, err error) { +func (ra *retainAction) Perform(candidates []*art.Candidate) (results []*art.Result, err error) { retained := make(map[string]bool) for _, c := range candidates { retained[c.Hash()] = true @@ -56,14 +56,14 @@ func (ra *retainAction) Perform(candidates []*res.Candidate) (results []*res.Res // start to delete if len(ra.all) > 0 { - for _, art := range ra.all { - if _, ok := retained[art.Hash()]; !ok { - result := &res.Result{ - Target: art, + for _, c := range ra.all { + if _, ok := retained[c.Hash()]; !ok { + result := &art.Result{ + Target: c, } if !ra.isDryRun { - if err := dep.DefaultClient.Delete(art); err != nil { + if err := dep.DefaultClient.Delete(c); err != nil { result.Error = err } } @@ -79,7 +79,7 @@ func (ra *retainAction) Perform(candidates []*res.Candidate) (results []*res.Res // NewRetainAction is factory method for RetainAction func NewRetainAction(params interface{}, isDryRun bool) Performer { if params != nil { - if all, ok := params.([]*res.Candidate); ok { + if all, ok := params.([]*art.Candidate); ok { return &retainAction{ all: all, isDryRun: isDryRun, @@ -88,7 +88,7 @@ func NewRetainAction(params interface{}, isDryRun bool) Performer { } return &retainAction{ - all: make([]*res.Candidate, 0), + all: make([]*art.Candidate, 0), isDryRun: isDryRun, } } diff --git a/src/pkg/retention/policy/action/performer_test.go b/src/pkg/retention/policy/action/performer_test.go index 0f07c0433..868bb4c93 100644 --- a/src/pkg/retention/policy/action/performer_test.go +++ b/src/pkg/retention/policy/action/performer_test.go @@ -18,8 +18,8 @@ import ( "testing" "time" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/dep" - "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -31,7 +31,7 @@ type TestPerformerSuite struct { suite.Suite oldClient dep.Client - all []*res.Candidate + all []*art.Candidate } // TestPerformer is the entry of the TestPerformerSuite @@ -41,7 +41,7 @@ func TestPerformer(t *testing.T) { // SetupSuite ... func (suite *TestPerformerSuite) SetupSuite() { - suite.all = []*res.Candidate{ + suite.all = []*art.Candidate{ { Namespace: "library", Repository: "harbor", @@ -77,7 +77,7 @@ func (suite *TestPerformerSuite) TestPerform() { all: suite.all, } - candidates := []*res.Candidate{ + candidates := []*art.Candidate{ { Namespace: "library", Repository: "harbor", @@ -100,16 +100,16 @@ func (suite *TestPerformerSuite) TestPerform() { type fakeRetentionClient struct{} // GetCandidates ... -func (frc *fakeRetentionClient) GetCandidates(repo *res.Repository) ([]*res.Candidate, error) { +func (frc *fakeRetentionClient) GetCandidates(repo *art.Repository) ([]*art.Candidate, error) { return nil, errors.New("not implemented") } // Delete ... -func (frc *fakeRetentionClient) Delete(candidate *res.Candidate) error { +func (frc *fakeRetentionClient) Delete(candidate *art.Candidate) error { return nil } // DeleteRepository ... -func (frc *fakeRetentionClient) DeleteRepository(repo *res.Repository) error { +func (frc *fakeRetentionClient) DeleteRepository(repo *art.Repository) error { panic("implement me") } diff --git a/src/pkg/retention/policy/alg/or/processor.go b/src/pkg/retention/policy/alg/or/processor.go index 623e4f050..a940299dd 100644 --- a/src/pkg/retention/policy/alg/or/processor.go +++ b/src/pkg/retention/policy/alg/or/processor.go @@ -18,10 +18,10 @@ import ( "sync" "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/alg" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/pkg/errors" ) @@ -29,7 +29,7 @@ import ( type processor struct { // keep evaluator and its related selector if existing // attentions here, the selectors can be empty/nil, that means match all "**" - evaluators map[*rule.Evaluator][]res.Selector + evaluators map[*rule.Evaluator][]art.Selector // action performer performers map[string]action.Performer } @@ -37,7 +37,7 @@ type processor struct { // New processor func New(parameters []*alg.Parameter) alg.Processor { p := &processor{ - evaluators: make(map[*rule.Evaluator][]res.Selector), + evaluators: make(map[*rule.Evaluator][]art.Selector), performers: make(map[string]action.Performer), } @@ -59,10 +59,10 @@ func New(parameters []*alg.Parameter) alg.Processor { } // Process the candidates with the rules -func (p *processor) Process(artifacts []*res.Candidate) ([]*res.Result, error) { +func (p *processor) Process(artifacts []*art.Candidate) ([]*art.Result, error) { if len(artifacts) == 0 { log.Debug("no artifacts to retention") - return make([]*res.Result, 0), nil + return make([]*art.Result, 0), nil } var ( @@ -75,7 +75,7 @@ func (p *processor) Process(artifacts []*res.Candidate) ([]*res.Result, error) { // for sync type chanItem struct { action string - processed []*res.Candidate + processed []*art.Candidate } resChan := make(chan *chanItem, 1) @@ -124,9 +124,9 @@ func (p *processor) Process(artifacts []*res.Candidate) ([]*res.Result, error) { for eva, selectors := range p.evaluators { var evaluator = *eva - go func(evaluator rule.Evaluator, selectors []res.Selector) { + go func(evaluator rule.Evaluator, selectors []art.Selector) { var ( - processed []*res.Candidate + processed []*art.Candidate err error ) @@ -173,7 +173,7 @@ func (p *processor) Process(artifacts []*res.Candidate) ([]*res.Result, error) { return nil, err } - results := make([]*res.Result, 0) + results := make([]*art.Result, 0) // Perform actions for act, hash := range processedCandidates { var attachedErr error @@ -192,7 +192,7 @@ func (p *processor) Process(artifacts []*res.Candidate) ([]*res.Result, error) { if attachedErr != nil { for _, c := range cl { - results = append(results, &res.Result{ + results = append(results, &art.Result{ Target: c, Error: attachedErr, }) @@ -203,10 +203,10 @@ func (p *processor) Process(artifacts []*res.Candidate) ([]*res.Result, error) { return results, nil } -type cHash map[string]*res.Candidate +type cHash map[string]*art.Candidate -func (ch cHash) toList() []*res.Candidate { - l := make([]*res.Candidate, 0) +func (ch cHash) toList() []*art.Candidate { + l := make([]*art.Candidate, 0) for _, v := range ch { l = append(l, v) diff --git a/src/pkg/retention/policy/alg/or/processor_test.go b/src/pkg/retention/policy/alg/or/processor_test.go index 8d09966e5..54e5233f5 100644 --- a/src/pkg/retention/policy/alg/or/processor_test.go +++ b/src/pkg/retention/policy/alg/or/processor_test.go @@ -19,6 +19,9 @@ import ( "testing" "time" + "github.com/goharbor/harbor/src/pkg/art" + "github.com/goharbor/harbor/src/pkg/art/selectors/doublestar" + "github.com/goharbor/harbor/src/pkg/art/selectors/label" "github.com/goharbor/harbor/src/pkg/retention/dep" "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/alg" @@ -26,9 +29,6 @@ import ( "github.com/goharbor/harbor/src/pkg/retention/policy/rule/always" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/lastx" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestps" - "github.com/goharbor/harbor/src/pkg/retention/res" - "github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar" - "github.com/goharbor/harbor/src/pkg/retention/res/selectors/label" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -38,7 +38,7 @@ import ( type ProcessorTestSuite struct { suite.Suite - all []*res.Candidate + all []*art.Candidate oldClient dep.Client } @@ -50,7 +50,7 @@ func TestProcessor(t *testing.T) { // SetupSuite ... func (suite *ProcessorTestSuite) SetupSuite() { - suite.all = []*res.Candidate{ + suite.all = []*art.Candidate{ { Namespace: "library", Repository: "harbor", @@ -90,7 +90,7 @@ func (suite *ProcessorTestSuite) TestProcess() { lastxParams[lastx.ParameterX] = 10 params = append(params, &alg.Parameter{ Evaluator: lastx.New(lastxParams), - Selectors: []res.Selector{ + Selectors: []art.Selector{ doublestar.New(doublestar.Matches, "*dev*"), label.New(label.With, "L1,L2"), }, @@ -101,7 +101,7 @@ func (suite *ProcessorTestSuite) TestProcess() { latestKParams[latestps.ParameterK] = 10 params = append(params, &alg.Parameter{ Evaluator: latestps.New(latestKParams), - Selectors: []res.Selector{ + Selectors: []art.Selector{ label.New(label.With, "L3"), }, Performer: perf, @@ -131,7 +131,7 @@ func (suite *ProcessorTestSuite) TestProcess2() { alwaysParams := make(map[string]rule.Parameter) params = append(params, &alg.Parameter{ Evaluator: always.New(alwaysParams), - Selectors: []res.Selector{ + Selectors: []art.Selector{ doublestar.New(doublestar.Matches, "latest"), label.New(label.With, ""), }, @@ -163,16 +163,16 @@ func (suite *ProcessorTestSuite) TestProcess2() { type fakeRetentionClient struct{} // GetCandidates ... -func (frc *fakeRetentionClient) GetCandidates(repo *res.Repository) ([]*res.Candidate, error) { +func (frc *fakeRetentionClient) GetCandidates(repo *art.Repository) ([]*art.Candidate, error) { return nil, errors.New("not implemented") } // Delete ... -func (frc *fakeRetentionClient) Delete(candidate *res.Candidate) error { +func (frc *fakeRetentionClient) Delete(candidate *art.Candidate) error { return nil } // DeleteRepository ... -func (frc *fakeRetentionClient) DeleteRepository(repo *res.Repository) error { +func (frc *fakeRetentionClient) DeleteRepository(repo *art.Repository) error { panic("implement me") } diff --git a/src/pkg/retention/policy/alg/processor.go b/src/pkg/retention/policy/alg/processor.go index 4f7103a5f..a057b6fee 100644 --- a/src/pkg/retention/policy/alg/processor.go +++ b/src/pkg/retention/policy/alg/processor.go @@ -15,9 +15,9 @@ package alg import ( + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" ) // Processor processing the whole policy targeting a repository. @@ -27,12 +27,12 @@ type Processor interface { // Process the artifact candidates // // Arguments: - // artifacts []*res.Candidate : process the retention candidates + // artifacts []*art.Candidate : process the retention candidates // // Returns: - // []*res.Result : the processed results + // []*art.Result : the processed results // error : common error object if any errors occurred - Process(artifacts []*res.Candidate) ([]*res.Result, error) + Process(artifacts []*art.Candidate) ([]*art.Result, error) } // Parameter for constructing a processor @@ -42,7 +42,7 @@ type Parameter struct { Evaluator rule.Evaluator // Selectors for the rule - Selectors []res.Selector + Selectors []art.Selector // Performer for the rule evaluator Performer action.Performer diff --git a/src/pkg/retention/policy/builder.go b/src/pkg/retention/policy/builder.go index 88443fb6b..59884c86c 100644 --- a/src/pkg/retention/policy/builder.go +++ b/src/pkg/retention/policy/builder.go @@ -21,13 +21,13 @@ import ( index3 "github.com/goharbor/harbor/src/pkg/retention/policy/alg/index" - index2 "github.com/goharbor/harbor/src/pkg/retention/res/selectors/index" + index2 "github.com/goharbor/harbor/src/pkg/art/selectors/index" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/index" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/alg" "github.com/goharbor/harbor/src/pkg/retention/policy/lwp" - "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/pkg/errors" ) @@ -46,7 +46,7 @@ type Builder interface { } // NewBuilder news a basic builder -func NewBuilder(all []*res.Candidate) Builder { +func NewBuilder(all []*art.Candidate) Builder { return &basicBuilder{ allCandidates: all, } @@ -54,7 +54,7 @@ func NewBuilder(all []*res.Candidate) Builder { // basicBuilder is default implementation of Builder interface type basicBuilder struct { - allCandidates []*res.Candidate + allCandidates []*art.Candidate } // Build policy processor from the raw policy @@ -76,7 +76,7 @@ func (bb *basicBuilder) Build(policy *lwp.Metadata, isDryRun bool) (alg.Processo return nil, errors.Wrap(err, "get action performer by metadata") } - sl := make([]res.Selector, 0) + sl := make([]art.Selector, 0) for _, s := range r.TagSelectors { sel, err := index2.Get(s.Kind, s.Decoration, s.Pattern) if err != nil { diff --git a/src/pkg/retention/policy/builder_test.go b/src/pkg/retention/policy/builder_test.go index cd12b9494..60ba74e0e 100644 --- a/src/pkg/retention/policy/builder_test.go +++ b/src/pkg/retention/policy/builder_test.go @@ -22,7 +22,7 @@ import ( index2 "github.com/goharbor/harbor/src/pkg/retention/policy/alg/index" - "github.com/goharbor/harbor/src/pkg/retention/res/selectors/index" + "github.com/goharbor/harbor/src/pkg/art/selectors/index" "github.com/goharbor/harbor/src/pkg/retention/dep" @@ -30,9 +30,9 @@ import ( "github.com/goharbor/harbor/src/pkg/retention/policy/alg/or" - "github.com/goharbor/harbor/src/pkg/retention/res/selectors/label" + "github.com/goharbor/harbor/src/pkg/art/selectors/label" - "github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar" + "github.com/goharbor/harbor/src/pkg/art/selectors/doublestar" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestps" @@ -46,7 +46,7 @@ import ( "github.com/goharbor/harbor/src/pkg/retention/policy/lwp" - "github.com/goharbor/harbor/src/pkg/retention/res" + "github.com/goharbor/harbor/src/pkg/art" "github.com/stretchr/testify/suite" ) @@ -55,7 +55,7 @@ import ( type TestBuilderSuite struct { suite.Suite - all []*res.Candidate + all []*art.Candidate oldClient dep.Client } @@ -66,7 +66,7 @@ func TestBuilder(t *testing.T) { // SetupSuite prepares the testing content if needed func (suite *TestBuilderSuite) SetupSuite() { - suite.all = []*res.Candidate{ + suite.all = []*art.Candidate{ { NamespaceID: 1, Namespace: "library", @@ -163,21 +163,21 @@ func (suite *TestBuilderSuite) TestBuild() { type fakeRetentionClient struct{} -func (frc *fakeRetentionClient) DeleteRepository(repo *res.Repository) error { +func (frc *fakeRetentionClient) DeleteRepository(repo *art.Repository) error { panic("implement me") } // GetCandidates ... -func (frc *fakeRetentionClient) GetCandidates(repo *res.Repository) ([]*res.Candidate, error) { +func (frc *fakeRetentionClient) GetCandidates(repo *art.Repository) ([]*art.Candidate, error) { return nil, errors.New("not implemented") } // Delete ... -func (frc *fakeRetentionClient) Delete(candidate *res.Candidate) error { +func (frc *fakeRetentionClient) Delete(candidate *art.Candidate) error { return nil } // SubmitTask ... -func (frc *fakeRetentionClient) SubmitTask(taskID int64, repository *res.Repository, meta *lwp.Metadata) (string, error) { +func (frc *fakeRetentionClient) SubmitTask(taskID int64, repository *art.Repository, meta *lwp.Metadata) (string, error) { return "", errors.New("not implemented") } diff --git a/src/pkg/retention/policy/rule/always/evaluator.go b/src/pkg/retention/policy/rule/always/evaluator.go index 1cd4f4eb4..7155b5a99 100644 --- a/src/pkg/retention/policy/rule/always/evaluator.go +++ b/src/pkg/retention/policy/rule/always/evaluator.go @@ -15,9 +15,9 @@ package always import ( + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" ) const ( @@ -28,7 +28,7 @@ const ( type evaluator struct{} // Process for the "always" Evaluator simply returns the input with no error -func (e *evaluator) Process(artifacts []*res.Candidate) ([]*res.Candidate, error) { +func (e *evaluator) Process(artifacts []*art.Candidate) ([]*art.Candidate, error) { return artifacts, nil } diff --git a/src/pkg/retention/policy/rule/always/evaluator_test.go b/src/pkg/retention/policy/rule/always/evaluator_test.go index 9e7c53b77..52bb142b9 100644 --- a/src/pkg/retention/policy/rule/always/evaluator_test.go +++ b/src/pkg/retention/policy/rule/always/evaluator_test.go @@ -17,8 +17,8 @@ package always import ( "testing" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -36,7 +36,7 @@ func (e *EvaluatorTestSuite) TestNew() { func (e *EvaluatorTestSuite) TestProcess() { sut := New(rule.Parameters{}) - input := []*res.Candidate{{PushedTime: 0}, {PushedTime: 1}, {PushedTime: 2}, {PushedTime: 3}} + input := []*art.Candidate{{PushedTime: 0}, {PushedTime: 1}, {PushedTime: 2}, {PushedTime: 3}} result, err := sut.Process(input) diff --git a/src/pkg/retention/policy/rule/dayspl/evaluator.go b/src/pkg/retention/policy/rule/dayspl/evaluator.go index 7face2b73..962555257 100644 --- a/src/pkg/retention/policy/rule/dayspl/evaluator.go +++ b/src/pkg/retention/policy/rule/dayspl/evaluator.go @@ -20,9 +20,9 @@ import ( "time" "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" ) const ( @@ -41,7 +41,7 @@ type evaluator struct { n int } -func (e *evaluator) Process(artifacts []*res.Candidate) (result []*res.Candidate, err error) { +func (e *evaluator) Process(artifacts []*art.Candidate) (result []*art.Candidate, err error) { minPullTime := time.Now().UTC().Add(time.Duration(-1*24*e.n) * time.Hour).Unix() for _, a := range artifacts { if a.PulledTime >= minPullTime { diff --git a/src/pkg/retention/policy/rule/dayspl/evaluator_test.go b/src/pkg/retention/policy/rule/dayspl/evaluator_test.go index 49a98d2cb..1d4c19282 100644 --- a/src/pkg/retention/policy/rule/dayspl/evaluator_test.go +++ b/src/pkg/retention/policy/rule/dayspl/evaluator_test.go @@ -20,8 +20,8 @@ import ( "testing" "time" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -54,7 +54,7 @@ func (e *EvaluatorTestSuite) TestNew() { func (e *EvaluatorTestSuite) TestProcess() { now := time.Now().UTC() - data := []*res.Candidate{ + data := []*art.Candidate{ {PulledTime: daysAgo(now, 1)}, {PulledTime: daysAgo(now, 2)}, {PulledTime: daysAgo(now, 3)}, diff --git a/src/pkg/retention/policy/rule/daysps/evaluator.go b/src/pkg/retention/policy/rule/daysps/evaluator.go index 58cf73d2b..58ee79a57 100644 --- a/src/pkg/retention/policy/rule/daysps/evaluator.go +++ b/src/pkg/retention/policy/rule/daysps/evaluator.go @@ -20,9 +20,9 @@ import ( "time" "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" ) const ( @@ -41,7 +41,7 @@ type evaluator struct { n int } -func (e *evaluator) Process(artifacts []*res.Candidate) (result []*res.Candidate, err error) { +func (e *evaluator) Process(artifacts []*art.Candidate) (result []*art.Candidate, err error) { minPushTime := time.Now().UTC().Add(time.Duration(-1*24*e.n) * time.Hour).Unix() for _, a := range artifacts { if a.PushedTime >= minPushTime { diff --git a/src/pkg/retention/policy/rule/daysps/evaluator_test.go b/src/pkg/retention/policy/rule/daysps/evaluator_test.go index a40c2c5a2..7d4e6995c 100644 --- a/src/pkg/retention/policy/rule/daysps/evaluator_test.go +++ b/src/pkg/retention/policy/rule/daysps/evaluator_test.go @@ -20,8 +20,8 @@ import ( "testing" "time" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -54,7 +54,7 @@ func (e *EvaluatorTestSuite) TestNew() { func (e *EvaluatorTestSuite) TestProcess() { now := time.Now().UTC() - data := []*res.Candidate{ + data := []*art.Candidate{ {PushedTime: daysAgo(now, 1)}, {PushedTime: daysAgo(now, 2)}, {PushedTime: daysAgo(now, 3)}, diff --git a/src/pkg/retention/policy/rule/evaluator.go b/src/pkg/retention/policy/rule/evaluator.go index 18e641986..f36d0f344 100644 --- a/src/pkg/retention/policy/rule/evaluator.go +++ b/src/pkg/retention/policy/rule/evaluator.go @@ -14,19 +14,19 @@ package rule -import "github.com/goharbor/harbor/src/pkg/retention/res" +import "github.com/goharbor/harbor/src/pkg/art" // Evaluator defines method of executing rule type Evaluator interface { // Filter the inputs and return the filtered outputs // // Arguments: - // artifacts []*res.Candidate : candidates for processing + // artifacts []*art.Candidate : candidates for processing // // Returns: - // []*res.Candidate : matched candidates for next stage + // []*art.Candidate : matched candidates for next stage // error : common error object if any errors occurred - Process(artifacts []*res.Candidate) ([]*res.Candidate, error) + Process(artifacts []*art.Candidate) ([]*art.Candidate, error) // Specify what action is performed to the candidates processed by this evaluator Action() string diff --git a/src/pkg/retention/policy/rule/index/index_test.go b/src/pkg/retention/policy/rule/index/index_test.go index fd8268f18..9f2f1f3ef 100644 --- a/src/pkg/retention/policy/rule/index/index_test.go +++ b/src/pkg/retention/policy/rule/index/index_test.go @@ -22,8 +22,8 @@ import ( "github.com/stretchr/testify/require" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/stretchr/testify/suite" ) @@ -63,7 +63,7 @@ func (suite *IndexTestSuite) TestGet() { require.NoError(suite.T(), err) require.NotNil(suite.T(), evaluator) - candidates := []*res.Candidate{{ + candidates := []*art.Candidate{{ Namespace: "library", Repository: "harbor", Kind: "image", @@ -102,7 +102,7 @@ type fakeEvaluator struct { } // Process rule -func (e *fakeEvaluator) Process(artifacts []*res.Candidate) ([]*res.Candidate, error) { +func (e *fakeEvaluator) Process(artifacts []*art.Candidate) ([]*art.Candidate, error) { return artifacts, nil } diff --git a/src/pkg/retention/policy/rule/lastx/evaluator.go b/src/pkg/retention/policy/rule/lastx/evaluator.go index 7457c0db3..ad0447f65 100644 --- a/src/pkg/retention/policy/rule/lastx/evaluator.go +++ b/src/pkg/retention/policy/rule/lastx/evaluator.go @@ -19,9 +19,9 @@ import ( "time" "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" ) const ( @@ -40,7 +40,7 @@ type evaluator struct { } // Process the candidates based on the rule definition -func (e *evaluator) Process(artifacts []*res.Candidate) (retain []*res.Candidate, err error) { +func (e *evaluator) Process(artifacts []*art.Candidate) (retain []*art.Candidate, err error) { cutoff := time.Now().Add(time.Duration(e.x*-24) * time.Hour) for _, a := range artifacts { if time.Unix(a.PushedTime, 0).UTC().After(cutoff) { diff --git a/src/pkg/retention/policy/rule/lastx/evaluator_test.go b/src/pkg/retention/policy/rule/lastx/evaluator_test.go index becd79234..ee42425e8 100644 --- a/src/pkg/retention/policy/rule/lastx/evaluator_test.go +++ b/src/pkg/retention/policy/rule/lastx/evaluator_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -38,7 +38,7 @@ func (e *EvaluatorTestSuite) TestNew() { func (e *EvaluatorTestSuite) TestProcess() { now := time.Now().UTC() - data := []*res.Candidate{ + data := []*art.Candidate{ {PushedTime: now.Add(time.Duration(1*-24) * time.Hour).Unix()}, {PushedTime: now.Add(time.Duration(2*-24) * time.Hour).Unix()}, {PushedTime: now.Add(time.Duration(3*-24) * time.Hour).Unix()}, diff --git a/src/pkg/retention/policy/rule/latestk/evaluator.go b/src/pkg/retention/policy/rule/latestk/evaluator.go index 9f9610f55..3405841cc 100644 --- a/src/pkg/retention/policy/rule/latestk/evaluator.go +++ b/src/pkg/retention/policy/rule/latestk/evaluator.go @@ -19,9 +19,9 @@ import ( "sort" "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" ) const ( @@ -40,7 +40,7 @@ type evaluator struct { } // Process the candidates based on the rule definition -func (e *evaluator) Process(artifacts []*res.Candidate) ([]*res.Candidate, error) { +func (e *evaluator) Process(artifacts []*art.Candidate) ([]*art.Candidate, error) { // Sort artifacts by their "active time" // // Active time is defined as the selection of c.PulledTime or c.PushedTime, @@ -81,7 +81,7 @@ func New(params rule.Parameters) rule.Evaluator { } } -func activeTime(c *res.Candidate) int64 { +func activeTime(c *art.Candidate) int64 { if c.PulledTime > c.PushedTime { return c.PulledTime } diff --git a/src/pkg/retention/policy/rule/latestk/evaluator_test.go b/src/pkg/retention/policy/rule/latestk/evaluator_test.go index 24b04fb9e..2fb09a5b9 100644 --- a/src/pkg/retention/policy/rule/latestk/evaluator_test.go +++ b/src/pkg/retention/policy/rule/latestk/evaluator_test.go @@ -22,18 +22,18 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/goharbor/harbor/src/pkg/retention/res" + "github.com/goharbor/harbor/src/pkg/art" "github.com/stretchr/testify/suite" ) type EvaluatorTestSuite struct { suite.Suite - artifacts []*res.Candidate + artifacts []*art.Candidate } func (e *EvaluatorTestSuite) SetupSuite() { - e.artifacts = []*res.Candidate{ + e.artifacts = []*art.Candidate{ {PulledTime: 1, PushedTime: 2}, {PulledTime: 3, PushedTime: 4}, {PulledTime: 6, PushedTime: 5}, diff --git a/src/pkg/retention/policy/rule/latestpl/evaluator.go b/src/pkg/retention/policy/rule/latestpl/evaluator.go index 21381759c..7f63a3896 100644 --- a/src/pkg/retention/policy/rule/latestpl/evaluator.go +++ b/src/pkg/retention/policy/rule/latestpl/evaluator.go @@ -21,9 +21,9 @@ import ( "sort" "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" ) const ( @@ -41,7 +41,7 @@ type evaluator struct { n int } -func (e *evaluator) Process(artifacts []*res.Candidate) ([]*res.Candidate, error) { +func (e *evaluator) Process(artifacts []*art.Candidate) ([]*art.Candidate, error) { sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].PulledTime > artifacts[j].PulledTime }) diff --git a/src/pkg/retention/policy/rule/latestpl/evaluator_test.go b/src/pkg/retention/policy/rule/latestpl/evaluator_test.go index ebd1679ae..1c33b94ea 100644 --- a/src/pkg/retention/policy/rule/latestpl/evaluator_test.go +++ b/src/pkg/retention/policy/rule/latestpl/evaluator_test.go @@ -20,8 +20,8 @@ import ( "math/rand" "testing" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -52,7 +52,7 @@ func (e *EvaluatorTestSuite) TestNew() { } func (e *EvaluatorTestSuite) TestProcess() { - data := []*res.Candidate{{PulledTime: 0}, {PulledTime: 1}, {PulledTime: 2}, {PulledTime: 3}, {PulledTime: 4}} + data := []*art.Candidate{{PulledTime: 0}, {PulledTime: 1}, {PulledTime: 2}, {PulledTime: 3}, {PulledTime: 4}} rand.Shuffle(len(data), func(i, j int) { data[i], data[j] = data[j], data[i] }) diff --git a/src/pkg/retention/policy/rule/latestps/evaluator.go b/src/pkg/retention/policy/rule/latestps/evaluator.go index f672aa1c6..96a6bf19a 100644 --- a/src/pkg/retention/policy/rule/latestps/evaluator.go +++ b/src/pkg/retention/policy/rule/latestps/evaluator.go @@ -21,9 +21,9 @@ import ( "sort" "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" ) const ( @@ -42,7 +42,7 @@ type evaluator struct { } // Process the candidates based on the rule definition -func (e *evaluator) Process(artifacts []*res.Candidate) ([]*res.Candidate, error) { +func (e *evaluator) Process(artifacts []*art.Candidate) ([]*art.Candidate, error) { // The updated proposal does not guarantee the order artifacts are provided, so we have to sort them first sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].PushedTime > artifacts[j].PushedTime diff --git a/src/pkg/retention/policy/rule/latestps/evaluator_test.go b/src/pkg/retention/policy/rule/latestps/evaluator_test.go index 38fa64570..a0e727c08 100644 --- a/src/pkg/retention/policy/rule/latestps/evaluator_test.go +++ b/src/pkg/retention/policy/rule/latestps/evaluator_test.go @@ -8,8 +8,8 @@ import ( "github.com/stretchr/testify/suite" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/stretchr/testify/require" ) @@ -39,7 +39,7 @@ func (e *EvaluatorTestSuite) TestNew() { } func (e *EvaluatorTestSuite) TestProcess() { - data := []*res.Candidate{{PushedTime: 0}, {PushedTime: 1}, {PushedTime: 2}, {PushedTime: 3}, {PushedTime: 4}} + data := []*art.Candidate{{PushedTime: 0}, {PushedTime: 1}, {PushedTime: 2}, {PushedTime: 3}, {PushedTime: 4}} rand.Shuffle(len(data), func(i, j int) { data[i], data[j] = data[j], data[i] }) diff --git a/src/pkg/retention/policy/rule/nothing/evaluator.go b/src/pkg/retention/policy/rule/nothing/evaluator.go index 8bc4b9063..f926c20c7 100644 --- a/src/pkg/retention/policy/rule/nothing/evaluator.go +++ b/src/pkg/retention/policy/rule/nothing/evaluator.go @@ -15,9 +15,9 @@ package nothing import ( + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/action" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" ) const ( @@ -28,7 +28,7 @@ const ( type evaluator struct{} // Process for the "nothing" Evaluator simply returns the input with no error -func (e *evaluator) Process(artifacts []*res.Candidate) (processed []*res.Candidate, err error) { +func (e *evaluator) Process(artifacts []*art.Candidate) (processed []*art.Candidate, err error) { return processed, err } diff --git a/src/pkg/retention/policy/rule/nothing/evaluator_test.go b/src/pkg/retention/policy/rule/nothing/evaluator_test.go index 1432db651..db4cea68a 100644 --- a/src/pkg/retention/policy/rule/nothing/evaluator_test.go +++ b/src/pkg/retention/policy/rule/nothing/evaluator_test.go @@ -17,8 +17,8 @@ package nothing import ( "testing" + "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" - "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -36,7 +36,7 @@ func (e *EvaluatorTestSuite) TestNew() { func (e *EvaluatorTestSuite) TestProcess() { sut := New(rule.Parameters{}) - input := []*res.Candidate{{PushedTime: 0}, {PushedTime: 1}, {PushedTime: 2}, {PushedTime: 3}} + input := []*art.Candidate{{PushedTime: 0}, {PushedTime: 1}, {PushedTime: 2}, {PushedTime: 3}} result, err := sut.Process(input) diff --git a/src/pkg/scan/scanner/api/controller.go b/src/pkg/scan/scanner/api/controller.go new file mode 100644 index 000000000..090dcde5f --- /dev/null +++ b/src/pkg/scan/scanner/api/controller.go @@ -0,0 +1,157 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "github.com/goharbor/harbor/src/pkg/q" + dscan "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scan" + "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner" + "github.com/goharbor/harbor/src/pkg/scan/scanner/scan" +) + +// Controller provides the related operations of scanner for the upper API. +// All the capabilities of the scanner are defined here. +type Controller interface { + // ListRegistrations returns a list of currently configured scanner registrations. + // Query parameters are optional + // + // Arguments: + // query *q.Query : query parameters + // + // Returns: + // []*scanner.Registration : scanner list of all the matched ones + // error : non nil error if any errors occurred + ListRegistrations(query *q.Query) ([]*scanner.Registration, error) + + // CreateRegistration creates a new scanner registration with the given data. + // Returns the scanner registration identifier. + // + // Arguments: + // registration *scanner.Registration : scanner registration to create + // + // Returns: + // string : the generated UUID of the new scanner + // error : non nil error if any errors occurred + CreateRegistration(registration *scanner.Registration) (string, error) + + // GetRegistration returns the details of the specified scanner registration. + // + // Arguments: + // registrationUUID string : the UUID of the given scanner + // + // Returns: + // *scanner.Registration : the required scanner + // error : non nil error if any errors occurred + GetRegistration(registrationUUID string) (*scanner.Registration, error) + + // RegistrationExists checks if the provided registration is there. + // + // Arguments: + // registrationUUID string : the UUID of the given scanner + // + // Returns: + // true for existing or false for not existing + RegistrationExists(registrationUUID string) bool + + // UpdateRegistration updates the specified scanner registration. + // + // Arguments: + // registration *scanner.Registration : scanner registration to update + // + // Returns: + // error : non nil error if any errors occurred + UpdateRegistration(registration *scanner.Registration) error + + // DeleteRegistration deletes the specified scanner registration. + // + // Arguments: + // registrationUUID string : the UUID of the given scanner which is going to be deleted + // + // Returns: + // *scanner.Registration : the deleted scanner + // error : non nil error if any errors occurred + DeleteRegistration(registrationUUID string) (*scanner.Registration, error) + + // SetDefaultRegistration marks the specified scanner registration as default. + // The implementation is supposed to unset any registration previously set as default. + // + // Arguments: + // registrationUUID string : the UUID of the given scanner which is marked as default + // + // Returns: + // error : non nil error if any errors occurred + SetDefaultRegistration(registrationUUID string) error + + // SetRegistrationByProject sets scanner for the given project. + // + // Arguments: + // projectID int64 : the ID of the given project + // scannerID string : the UUID of the the scanner + // + // Returns: + // error : non nil error if any errors occurred + SetRegistrationByProject(projectID int64, scannerID string) error + + // GetRegistrationByProject returns the configured scanner registration of the given project or + // the system default registration if exists or `nil` if no system registrations set. + // + // Arguments: + // projectID int64 : the ID of the given project + // + // Returns: + // *scanner.Registration : the default scanner registration + // error : non nil error if any errors occurred + GetRegistrationByProject(projectID int64) (*scanner.Registration, error) + + // Ping pings Scanner Adapter to test EndpointURL and Authorization settings. + // The implementation is supposed to call the GetMetadata method on scanner.Client. + // Returns `nil` if connection succeeded, a non `nil` error otherwise. + // + // Arguments: + // registration *scanner.Registration : scanner registration to ping + // + // Returns: + // error : non nil error if any errors occurred + Ping(registration *scanner.Registration) error + + // Scan the given artifact + // + // Arguments: + // artifact *res.Artifact : artifact to be scanned + // + // Returns: + // error : non nil error if any errors occurred + Scan(artifact *scan.Artifact) error + + // GetReport gets the reports for the given artifact identified by the digest + // + // Arguments: + // artifact *res.Artifact : the scanned artifact + // + // Returns: + // []*scan.Report : scan results by different scanner vendors + // error : non nil error if any errors occurred + GetReport(artifact *scan.Artifact) ([]*dscan.Report, error) + + // Get the scan log for the specified artifact with the given digest + // + // Arguments: + // digest string : the digest of the artifact + // + // Returns: + // []byte : the log text stream + // error : non nil error if any errors occurred + GetScanLog(digest string) ([]byte, error) +} diff --git a/src/pkg/scan/scanner/api/controller_test.go b/src/pkg/scan/scanner/api/controller_test.go new file mode 100644 index 000000000..0ea4c5ad2 --- /dev/null +++ b/src/pkg/scan/scanner/api/controller_test.go @@ -0,0 +1,287 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "testing" + + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/pkg/q" + "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// ControllerTestSuite is test suite to test the basic api controller. +type ControllerTestSuite struct { + suite.Suite + + c *basicController + mMgr *MockScannerManager + mMeta *MockProMetaManager + + sample *scanner.Registration +} + +// TestController is the entry of controller test suite +func TestController(t *testing.T) { + suite.Run(t, new(ControllerTestSuite)) +} + +// SetupSuite prepares env for the controller test suite +func (suite *ControllerTestSuite) SetupSuite() { + suite.mMgr = new(MockScannerManager) + suite.mMeta = new(MockProMetaManager) + + suite.c = &basicController{ + manager: suite.mMgr, + proMetaMgr: suite.mMeta, + } + + suite.sample = &scanner.Registration{ + Name: "forUT", + Description: "sample registration", + URL: "https://sample.scanner.com", + Adapter: "Clair", + Version: "0.1.0", + Vendor: "Harbor", + } +} + +// Clear test case +func (suite *ControllerTestSuite) TearDownTest() { + suite.sample.UUID = "" +} + +// TestListRegistrations tests ListRegistrations +func (suite *ControllerTestSuite) TestListRegistrations() { + query := &q.Query{ + PageSize: 10, + PageNumber: 1, + } + + suite.sample.UUID = "uuid" + l := []*scanner.Registration{suite.sample} + + suite.mMgr.On("List", query).Return(l, nil) + + rl, err := suite.c.ListRegistrations(query) + require.NoError(suite.T(), err) + assert.Equal(suite.T(), 1, len(rl)) +} + +// TestCreateRegistration tests CreateRegistration +func (suite *ControllerTestSuite) TestCreateRegistration() { + suite.mMgr.On("Create", suite.sample).Return("uuid", nil) + + uid, err := suite.mMgr.Create(suite.sample) + + require.NoError(suite.T(), err) + assert.Equal(suite.T(), uid, "uuid") +} + +// TestGetRegistration tests GetRegistration +func (suite *ControllerTestSuite) TestGetRegistration() { + suite.sample.UUID = "uuid" + suite.mMgr.On("Get", "uuid").Return(suite.sample, nil) + + rr, err := suite.c.GetRegistration("uuid") + require.NoError(suite.T(), err) + assert.NotNil(suite.T(), rr) + assert.Equal(suite.T(), "forUT", rr.Name) +} + +// TestRegistrationExists tests RegistrationExists +func (suite *ControllerTestSuite) TestRegistrationExists() { + suite.sample.UUID = "uuid" + suite.mMgr.On("Get", "uuid").Return(suite.sample, nil) + + exists := suite.c.RegistrationExists("uuid") + assert.Equal(suite.T(), true, exists) + + suite.mMgr.On("Get", "uuid2").Return(nil, nil) + + exists = suite.c.RegistrationExists("uuid2") + assert.Equal(suite.T(), false, exists) +} + +// TestUpdateRegistration tests UpdateRegistration +func (suite *ControllerTestSuite) TestUpdateRegistration() { + suite.sample.UUID = "uuid" + suite.mMgr.On("Update", suite.sample).Return(nil) + + err := suite.c.UpdateRegistration(suite.sample) + require.NoError(suite.T(), err) +} + +// TestDeleteRegistration tests DeleteRegistration +func (suite *ControllerTestSuite) TestDeleteRegistration() { + suite.sample.UUID = "uuid" + suite.mMgr.On("Get", "uuid").Return(suite.sample, nil) + suite.mMgr.On("Delete", "uuid").Return(nil) + + r, err := suite.c.DeleteRegistration("uuid") + require.NoError(suite.T(), err) + require.NotNil(suite.T(), r) + assert.Equal(suite.T(), "forUT", r.Name) +} + +// TestSetDefaultRegistration tests SetDefaultRegistration +func (suite *ControllerTestSuite) TestSetDefaultRegistration() { + suite.mMgr.On("SetAsDefault", "uuid").Return(nil) + + err := suite.c.SetDefaultRegistration("uuid") + require.NoError(suite.T(), err) +} + +// TestSetRegistrationByProject tests SetRegistrationByProject +func (suite *ControllerTestSuite) TestSetRegistrationByProject() { + m := make(map[string]string, 1) + mm := make(map[string]string, 1) + mmm := make(map[string]string, 1) + mm[proScannerMetaKey] = "uuid" + mmm[proScannerMetaKey] = "uuid2" + + var pid, pid2 int64 = 1, 2 + + // not set before + suite.mMeta.On("Get", pid, []string{proScannerMetaKey}).Return(m, nil) + suite.mMeta.On("Add", pid, mm).Return(nil) + + err := suite.c.SetRegistrationByProject(pid, "uuid") + require.NoError(suite.T(), err) + + // Set before + suite.mMeta.On("Get", pid2, []string{proScannerMetaKey}).Return(mm, nil) + suite.mMeta.On("Update", pid2, mmm).Return(nil) + + err = suite.c.SetRegistrationByProject(pid2, "uuid2") + require.NoError(suite.T(), err) +} + +// TestGetRegistrationByProject tests GetRegistrationByProject +func (suite *ControllerTestSuite) TestGetRegistrationByProject() { + m := make(map[string]string, 1) + m[proScannerMetaKey] = "uuid" + + // Configured at project level + var pid int64 = 1 + suite.sample.UUID = "uuid" + + suite.mMeta.On("Get", pid, []string{proScannerMetaKey}).Return(m, nil) + suite.mMgr.On("Get", "uuid").Return(suite.sample, nil) + + r, err := suite.c.GetRegistrationByProject(pid) + require.NoError(suite.T(), err) + require.Equal(suite.T(), "forUT", r.Name) + + // Not configured at project level, return system default + suite.mMeta.On("Get", pid, []string{proScannerMetaKey}).Return(nil, nil) + suite.mMgr.On("GetDefault").Return(suite.sample, nil) + + r, err = suite.c.GetRegistrationByProject(pid) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), r) + assert.Equal(suite.T(), "forUT", r.Name) +} + +// MockScannerManager is mock of the scanner manager +type MockScannerManager struct { + mock.Mock +} + +// List ... +func (m *MockScannerManager) List(query *q.Query) ([]*scanner.Registration, error) { + args := m.Called(query) + return args.Get(0).([]*scanner.Registration), args.Error(1) +} + +// Create ... +func (m *MockScannerManager) Create(registration *scanner.Registration) (string, error) { + args := m.Called(registration) + return args.String(0), args.Error(1) +} + +// Get ... +func (m *MockScannerManager) Get(registrationUUID string) (*scanner.Registration, error) { + args := m.Called(registrationUUID) + r := args.Get(0) + if r == nil { + return nil, args.Error(1) + } + + return r.(*scanner.Registration), args.Error(1) +} + +// Update ... +func (m *MockScannerManager) Update(registration *scanner.Registration) error { + args := m.Called(registration) + return args.Error(0) +} + +// Delete ... +func (m *MockScannerManager) Delete(registrationUUID string) error { + args := m.Called(registrationUUID) + return args.Error(0) +} + +// SetAsDefault ... +func (m *MockScannerManager) SetAsDefault(registrationUUID string) error { + args := m.Called(registrationUUID) + return args.Error(0) +} + +// GetDefault ... +func (m *MockScannerManager) GetDefault() (*scanner.Registration, error) { + args := m.Called() + return args.Get(0).(*scanner.Registration), args.Error(1) +} + +// MockProMetaManager is the mock of the ProjectMetadataManager +type MockProMetaManager struct { + mock.Mock +} + +// Add ... +func (m *MockProMetaManager) Add(projectID int64, meta map[string]string) error { + args := m.Called(projectID, meta) + return args.Error(0) +} + +// Delete ... +func (m *MockProMetaManager) Delete(projecdtID int64, meta ...string) error { + args := m.Called(projecdtID, meta) + return args.Error(0) +} + +// Update ... +func (m *MockProMetaManager) Update(projectID int64, meta map[string]string) error { + args := m.Called(projectID, meta) + return args.Error(0) +} + +// Get ... +func (m *MockProMetaManager) Get(projectID int64, meta ...string) (map[string]string, error) { + args := m.Called(projectID, meta) + return args.Get(0).(map[string]string), args.Error(1) +} + +// List ... +func (m *MockProMetaManager) List(name, value string) ([]*models.ProjectMetadata, error) { + args := m.Called(name, value) + return args.Get(0).([]*models.ProjectMetadata), args.Error(1) +} diff --git a/src/pkg/scan/scanner/api/registration.go b/src/pkg/scan/scanner/api/registration.go new file mode 100644 index 000000000..879498915 --- /dev/null +++ b/src/pkg/scan/scanner/api/registration.go @@ -0,0 +1,194 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "github.com/goharbor/harbor/src/core/promgr/metamgr" + "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/pkg/q" + rscanner "github.com/goharbor/harbor/src/pkg/scan/scanner" + "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner" + "github.com/goharbor/harbor/src/pkg/scan/scanner/scan" + "github.com/pkg/errors" +) + +const ( + proScannerMetaKey = "projectScanner" +) + +// DefaultController is a singleton api controller for plug scanners +var DefaultController = New() + +// New a basic controller +func New() Controller { + return &basicController{ + manager: rscanner.New(), + proMetaMgr: metamgr.NewDefaultProjectMetadataManager(), + } +} + +// basicController is default implementation of api.Controller interface +type basicController struct { + // managers for managing the scanner registrations + manager rscanner.Manager + // for operating the project level configured scanner + proMetaMgr metamgr.ProjectMetadataManager + // controller for scan actions + c scan.Controller + // Client +} + +// ListRegistrations ... +func (bc *basicController) ListRegistrations(query *q.Query) ([]*scanner.Registration, error) { + return bc.manager.List(query) +} + +// CreateRegistration ... +func (bc *basicController) CreateRegistration(registration *scanner.Registration) (string, error) { + // TODO: Get metadata from the adapter service first + l, err := bc.manager.List(nil) + if err != nil { + return "", err + } + + if len(l) == 0 && !registration.IsDefault { + // Mark the 1st as default automatically + registration.IsDefault = true + } + + return bc.manager.Create(registration) +} + +// GetRegistration ... +func (bc *basicController) GetRegistration(registrationUUID string) (*scanner.Registration, error) { + return bc.manager.Get(registrationUUID) +} + +// RegistrationExists ... +func (bc *basicController) RegistrationExists(registrationUUID string) bool { + registration, err := bc.manager.Get(registrationUUID) + + // Just logged when an error occurred + if err != nil { + logger.Errorf("Check existence of registration error: %s", err) + } + + return !(err == nil && registration == nil) +} + +// UpdateRegistration ... +func (bc *basicController) UpdateRegistration(registration *scanner.Registration) error { + return bc.manager.Update(registration) +} + +// SetDefaultRegistration ... +func (bc *basicController) DeleteRegistration(registrationUUID string) (*scanner.Registration, error) { + registration, err := bc.manager.Get(registrationUUID) + if registration == nil && err == nil { + // Not found + return nil, nil + } + + if err := bc.manager.Delete(registrationUUID); err != nil { + return nil, errors.Wrap(err, "delete registration") + } + + return registration, nil +} + +// SetDefaultRegistration ... +func (bc *basicController) SetDefaultRegistration(registrationUUID string) error { + return bc.manager.SetAsDefault(registrationUUID) +} + +// SetRegistrationByProject ... +func (bc *basicController) SetRegistrationByProject(projectID int64, registrationID string) error { + if projectID == 0 { + return errors.New("invalid project ID") + } + + if len(registrationID) == 0 { + return errors.New("missing scanner UUID") + } + + // Only keep the UUID in the metadata of the given project + // Scanner metadata existing? + m, err := bc.proMetaMgr.Get(projectID, proScannerMetaKey) + if err != nil { + return errors.Wrap(err, "set project scanner") + } + + // Update if exists + if len(m) > 0 { + // Compare and set new + if registrationID != m[proScannerMetaKey] { + m[proScannerMetaKey] = registrationID + if err := bc.proMetaMgr.Update(projectID, m); err != nil { + return errors.Wrap(err, "set project scanner") + } + } + } else { + meta := make(map[string]string, 1) + meta[proScannerMetaKey] = registrationID + if err := bc.proMetaMgr.Add(projectID, meta); err != nil { + return errors.Wrap(err, "set project scanner") + } + } + + return nil +} + +// GetRegistrationByProject ... +func (bc *basicController) GetRegistrationByProject(projectID int64) (*scanner.Registration, error) { + if projectID == 0 { + return nil, errors.New("invalid project ID") + } + + // First, get it from the project metadata + m, err := bc.proMetaMgr.Get(projectID, proScannerMetaKey) + if err != nil { + return nil, errors.Wrap(err, "get project scanner") + } + + if len(m) > 0 { + if registrationID, ok := m[proScannerMetaKey]; ok && len(registrationID) > 0 { + registration, err := bc.manager.Get(registrationID) + if err != nil { + return nil, errors.Wrap(err, "get project scanner") + } + + if registration == nil { + // Not found + // Might be deleted by the admin, the project scanner ID reference should be cleared + if err := bc.proMetaMgr.Delete(projectID, proScannerMetaKey); err != nil { + return nil, errors.Wrap(err, "get project scanner") + } + } else { + return registration, nil + } + } + } + + // Second, get the default one + registration, err := bc.manager.GetDefault() + + // TODO: Check status by the client later + return registration, err +} + +// Ping ... +func (bc *basicController) Ping(registration *scanner.Registration) error { + return nil +} diff --git a/src/pkg/scan/scanner/api/scan.go b/src/pkg/scan/scanner/api/scan.go new file mode 100644 index 000000000..a8b1a47d4 --- /dev/null +++ b/src/pkg/scan/scanner/api/scan.go @@ -0,0 +1,35 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + dscan "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scan" + "github.com/goharbor/harbor/src/pkg/scan/scanner/scan" +) + +// Scan ... +func (bc *basicController) Scan(artifact *scan.Artifact) error { + return nil +} + +// GetReport ... +func (bc *basicController) GetReport(artifact *scan.Artifact) ([]*dscan.Report, error) { + return nil, nil +} + +// GetScanLog ... +func (bc *basicController) GetScanLog(digest string) ([]byte, error) { + return nil, nil +} diff --git a/src/pkg/scan/scanner/dao/scan/report.go b/src/pkg/scan/scanner/dao/scan/report.go new file mode 100644 index 000000000..13d3f6cc9 --- /dev/null +++ b/src/pkg/scan/scanner/dao/scan/report.go @@ -0,0 +1,43 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import "time" + +// Report of the scan +// Identified by the `digest` and `endpoint_id` +type Report struct { + ID int64 `orm:"pk;auto;column(id)"` + Digest string `orm:"column(digest)"` + ReregistrationID string `orm:"column(registration_id)"` + JobID string `orm:"column(job_id)"` + Status string `orm:"column(status)"` + StatusCode int `orm:"column(status_code)"` + Report string `orm:"column(report);type(json)"` + StartTime time.Time `orm:"column(start_time);auto_now_add;type(datetime)"` + EndTime time.Time `orm:"column(end_time);type(datetime)"` +} + +// TableName for Report +func (r *Report) TableName() string { + return "scanner_report" +} + +// TableUnique for Report +func (r *Report) TableUnique() [][]string { + return [][]string{ + {"digest", "registration_id"}, + } +} diff --git a/src/pkg/scan/scanner/dao/scanner/model.go b/src/pkg/scan/scanner/dao/scanner/model.go new file mode 100644 index 000000000..fa28b87df --- /dev/null +++ b/src/pkg/scan/scanner/dao/scanner/model.go @@ -0,0 +1,120 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scanner + +import ( + "encoding/json" + "net/url" + "strings" + "time" + + "github.com/pkg/errors" +) + +// Registration represents a named configuration for invoking a scanner via its adapter. +// UUID will be used to track the scanner.Endpoint as unique ID +type Registration struct { + // Basic information + // int64 ID is kept for being aligned with previous DB schema + ID int64 `orm:"pk;auto;column(id)" json:"-"` + UUID string `orm:"unique;column(uuid)" json:"uuid"` + Name string `orm:"unique;column(name);size(128)" json:"name"` + Description string `orm:"column(description);null;size(1024)" json:"description"` + URL string `orm:"column(url);unique;size(512)" json:"url"` + Disabled bool `orm:"column(disabled);default(true)" json:"disabled"` + IsDefault bool `orm:"column(is_default);default(false)" json:"is_default"` + Health bool `orm:"-" json:"health"` + + // Authentication settings + // "None","Basic" and "Bearer" can be supported + Auth string `orm:"column(auth);size(16)" json:"auth"` + AccessCredential string `orm:"column(access_cred);null;size(512)" json:"access_credential,omitempty"` + + // Http connection settings + SkipCertVerify bool `orm:"column(skip_cert_verify);default(false)" json:"skip_certVerify"` + + // Adapter settings + Adapter string `orm:"column(adapter);size(128)" json:"adapter"` + Vendor string `orm:"column(vendor);size(128)" json:"vendor"` + Version string `orm:"column(version);size(32)" json:"version"` + + // Timestamps + CreateTime time.Time `orm:"column(create_time);auto_now_add;type(datetime)" json:"create_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now;type(datetime)" json:"update_time"` +} + +// TableName for Endpoint +func (r *Registration) TableName() string { + return "scanner_registration" +} + +// FromJSON parses json data +func (r *Registration) FromJSON(jsonData string) error { + if len(jsonData) == 0 { + return errors.New("empty json data to parse") + } + + return json.Unmarshal([]byte(jsonData), r) +} + +// ToJSON marshals endpoint to JSON data +func (r *Registration) ToJSON() (string, error) { + data, err := json.Marshal(r) + if err != nil { + return "", err + } + + return string(data), nil +} + +// Validate endpoint +func (r *Registration) Validate(checkUUID bool) error { + if checkUUID && len(r.UUID) == 0 { + return errors.New("malformed endpoint") + } + + if len(r.Name) == 0 { + return errors.New("missing registration name") + } + + err := checkURL(r.URL) + if err != nil { + return errors.Wrap(err, "scanner registration validate") + } + + if len(r.Adapter) == 0 || + len(r.Vendor) == 0 || + len(r.Version) == 0 { + return errors.Errorf("missing adapter settings in registration %s:%s", r.Name, r.URL) + } + + return nil +} + +// Check the registration URL with url package +func checkURL(u string) error { + if len(strings.TrimSpace(u)) == 0 { + return errors.New("empty url") + } + + uri, err := url.Parse(u) + if err == nil { + if uri.Scheme != "http" && uri.Scheme != "https" { + err = errors.New("invalid scheme") + } + } + + return err +} diff --git a/src/pkg/scan/scanner/dao/scanner/model_test.go b/src/pkg/scan/scanner/dao/scanner/model_test.go new file mode 100644 index 000000000..25d837f92 --- /dev/null +++ b/src/pkg/scan/scanner/dao/scanner/model_test.go @@ -0,0 +1,87 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scanner + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// ModelTestSuite tests the utility functions of the model +type ModelTestSuite struct { + suite.Suite +} + +// TestModel is the entry of the model test suite +func TestModel(t *testing.T) { + suite.Run(t, new(ModelTestSuite)) +} + +// TestJSON tests the marshal and unmarshal functions +func (suite *ModelTestSuite) TestJSON() { + r := &Registration{ + Name: "forUT", + Description: "sample registration", + URL: "https://sample.scanner.com", + Adapter: "Clair", + Version: "0.1.0", + Vendor: "Harbor", + } + + json, err := r.ToJSON() + require.NoError(suite.T(), err) + assert.Condition(suite.T(), func() (success bool) { + success = len(json) > 0 + return + }) + + r2 := &Registration{} + err = r2.FromJSON(json) + require.NoError(suite.T(), err) + assert.Equal(suite.T(), "forUT", r2.Name) +} + +// TestValidate tests the validate function +func (suite *ModelTestSuite) TestValidate() { + r := &Registration{} + + err := r.Validate(true) + require.Error(suite.T(), err) + + r.UUID = "uuid" + err = r.Validate(true) + require.Error(suite.T(), err) + + r.Name = "forUT" + err = r.Validate(true) + require.Error(suite.T(), err) + + r.URL = "a.b.c" + err = r.Validate(true) + require.Error(suite.T(), err) + + r.URL = "http://a.b.c" + err = r.Validate(true) + require.Error(suite.T(), err) + + r.Adapter = "Clair" + r.Vendor = "Harbor" + r.Version = "0.1.0" + err = r.Validate(true) + require.NoError(suite.T(), err) +} diff --git a/src/pkg/scan/scanner/dao/scanner/registration.go b/src/pkg/scan/scanner/dao/scanner/registration.go new file mode 100644 index 000000000..da2912dea --- /dev/null +++ b/src/pkg/scan/scanner/dao/scanner/registration.go @@ -0,0 +1,147 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scanner + +import ( + "fmt" + + "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/pkg/q" + "github.com/pkg/errors" +) + +func init() { + orm.RegisterModel(new(Registration)) +} + +// AddRegistration adds a new registration +func AddRegistration(r *Registration) (int64, error) { + o := dao.GetOrmer() + return o.Insert(r) +} + +// GetRegistration gets the specified registration +func GetRegistration(UUID string) (*Registration, error) { + e := &Registration{} + + o := dao.GetOrmer() + qs := o.QueryTable(new(Registration)) + + if err := qs.Filter("uuid", UUID).One(e); err != nil { + if err == orm.ErrNoRows { + // Not existing case + return nil, nil + } + return nil, err + } + + return e, nil +} + +// UpdateRegistration update the specified registration +func UpdateRegistration(r *Registration, cols ...string) error { + o := dao.GetOrmer() + count, err := o.Update(r, cols...) + if err != nil { + return err + } + + if count == 0 { + return errors.Errorf("no item with UUID %s is updated", r.UUID) + } + + return nil +} + +// DeleteRegistration deletes the registration with the specified UUID +func DeleteRegistration(UUID string) error { + o := dao.GetOrmer() + qt := o.QueryTable(new(Registration)) + + // delete with query way + count, err := qt.Filter("uuid", UUID).Delete() + + if err != nil { + return err + } + + if count == 0 { + return errors.Errorf("no item with UUID %s is deleted", UUID) + } + + return nil +} + +// ListRegistrations lists all the existing registrations +func ListRegistrations(query *q.Query) ([]*Registration, error) { + o := dao.GetOrmer() + qt := o.QueryTable(new(Registration)) + + if query != nil { + if len(query.Keywords) > 0 { + for k, v := range query.Keywords { + qt = qt.Filter(fmt.Sprintf("%s__icontains", k), v) + } + } + + if query.PageNumber > 0 && query.PageSize > 0 { + qt = qt.Limit(query.PageSize, (query.PageNumber-1)*query.PageSize) + } + } + + l := make([]*Registration, 0) + _, err := qt.All(&l) + + return l, err +} + +// SetDefaultRegistration sets the specified registration as default one +func SetDefaultRegistration(UUID string) error { + o := dao.GetOrmer() + qt := o.QueryTable(new(Registration)) + + _, err := qt.Filter("is_default", true).Update(orm.Params{ + "is_default": false, + }) + + if err != nil { + return err + } + + qt2 := o.QueryTable(new(Registration)) + _, err = qt2.Filter("uuid", UUID).Update(orm.Params{ + "is_default": true, + }) + + return err +} + +// GetDefaultRegistration gets the default registration +func GetDefaultRegistration() (*Registration, error) { + o := dao.GetOrmer() + qt := o.QueryTable(new(Registration)) + + e := &Registration{} + if err := qt.Filter("is_default", true).One(e); err != nil { + if err == orm.ErrNoRows { + return nil, nil + } + + return nil, err + } + + return e, nil +} diff --git a/src/pkg/scan/scanner/dao/scanner/registration_test.go b/src/pkg/scan/scanner/dao/scanner/registration_test.go new file mode 100644 index 000000000..1575bf67f --- /dev/null +++ b/src/pkg/scan/scanner/dao/scanner/registration_test.go @@ -0,0 +1,144 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scanner + +import ( + "testing" + + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/pkg/q" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// RegistrationDAOTestSuite is test suite of testing registration DAO +type RegistrationDAOTestSuite struct { + suite.Suite + + registrationID string +} + +// TestRegistrationDAO is entry of test cases +func TestRegistrationDAO(t *testing.T) { + suite.Run(t, new(RegistrationDAOTestSuite)) +} + +// SetupSuite prepare testing env for the suite +func (suite *RegistrationDAOTestSuite) SetupSuite() { + dao.PrepareTestForPostgresSQL() +} + +// SetupTest prepare stuff for test cases +func (suite *RegistrationDAOTestSuite) SetupTest() { + suite.registrationID = uuid.New().String() + r := &Registration{ + UUID: suite.registrationID, + Name: "forUT", + Description: "sample registration", + URL: "https://sample.scanner.com", + Adapter: "Clair", + Version: "0.1.0", + Vendor: "Harbor", + } + + _, err := AddRegistration(r) + require.NoError(suite.T(), err, "add new registration") + +} + +// TearDownTest clears all the stuff of test cases +func (suite *RegistrationDAOTestSuite) TearDownTest() { + err := DeleteRegistration(suite.registrationID) + require.NoError(suite.T(), err, "clear registration") +} + +// TestGet tests get registration +func (suite *RegistrationDAOTestSuite) TestGet() { + // Found + r, err := GetRegistration(suite.registrationID) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), r) + assert.Equal(suite.T(), r.Name, "forUT") + + // Not found + re, err := GetRegistration("not_found") + require.NoError(suite.T(), err) + require.Nil(suite.T(), re) +} + +// TestUpdate tests update registration +func (suite *RegistrationDAOTestSuite) TestUpdate() { + r, err := GetRegistration(suite.registrationID) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), r) + + r.Disabled = true + r.IsDefault = true + r.URL = "http://updated.registration.com" + + err = UpdateRegistration(r) + require.NoError(suite.T(), err, "update registration") + + r, err = GetRegistration(suite.registrationID) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), r) + + assert.Equal(suite.T(), true, r.Disabled) + assert.Equal(suite.T(), true, r.IsDefault) + assert.Equal(suite.T(), "http://updated.registration.com", r.URL) +} + +// TestList tests list registrations +func (suite *RegistrationDAOTestSuite) TestList() { + // no query + l, err := ListRegistrations(nil) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) + + // with query and found items + keywords := make(map[string]string) + keywords["adapter"] = "Clair" + l, err = ListRegistrations(&q.Query{ + PageSize: 5, + PageNumber: 1, + Keywords: keywords, + }) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) + + // With query and not found items + keywords["adapter"] = "Micro scanner" + l, err = ListRegistrations(&q.Query{ + Keywords: keywords, + }) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 0, len(l)) +} + +// TestDefault tests set/get default +func (suite *RegistrationDAOTestSuite) TestDefault() { + dr, err := GetDefaultRegistration() + require.NoError(suite.T(), err, "not found") + require.Nil(suite.T(), dr) + + err = SetDefaultRegistration(suite.registrationID) + require.NoError(suite.T(), err) + + dr, err = GetDefaultRegistration() + require.NoError(suite.T(), err) + require.NotNil(suite.T(), dr) +} diff --git a/src/pkg/scan/scanner/manager.go b/src/pkg/scan/scanner/manager.go new file mode 100644 index 000000000..3387a8fb7 --- /dev/null +++ b/src/pkg/scan/scanner/manager.go @@ -0,0 +1,131 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scanner + +import ( + "github.com/goharbor/harbor/src/pkg/q" + "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner" + "github.com/google/uuid" + "github.com/pkg/errors" +) + +// Manager defines the related scanner API endpoints +type Manager interface { + // List returns a list of currently configured scanner registrations. + // Query parameters are optional + List(query *q.Query) ([]*scanner.Registration, error) + + // Create creates a new scanner registration with the given data. + // Returns the scanner registration identifier. + Create(registration *scanner.Registration) (string, error) + + // Get returns the details of the specified scanner registration. + Get(registrationUUID string) (*scanner.Registration, error) + + // Update updates the specified scanner registration. + Update(registration *scanner.Registration) error + + // Delete deletes the specified scanner registration. + Delete(registrationUUID string) error + + // SetAsDefault marks the specified scanner registration as default. + // The implementation is supposed to unset any registration previously set as default. + SetAsDefault(registrationUUID string) error + + // GetDefault returns the default scanner registration or `nil` if there are no registrations configured. + GetDefault() (*scanner.Registration, error) +} + +// basicManager is the default implementation of Manager +type basicManager struct{} + +// New a basic manager +func New() Manager { + return &basicManager{} +} + +// Create ... +func (bm *basicManager) Create(registration *scanner.Registration) (string, error) { + if registration == nil { + return "", errors.New("nil endpoint to create") + } + + // Inject new UUID + uid, err := uuid.NewUUID() + if err != nil { + return "", errors.Wrap(err, "new UUID: create registration") + } + registration.UUID = uid.String() + + if err := registration.Validate(true); err != nil { + return "", errors.Wrap(err, "create registration") + } + + if _, err := scanner.AddRegistration(registration); err != nil { + return "", errors.Wrap(err, "dao: create registration") + } + + return uid.String(), nil +} + +// Get ... +func (bm *basicManager) Get(registrationUUID string) (*scanner.Registration, error) { + if len(registrationUUID) == 0 { + return nil, errors.New("empty uuid of registration") + } + + return scanner.GetRegistration(registrationUUID) +} + +// Update ... +func (bm *basicManager) Update(registration *scanner.Registration) error { + if registration == nil { + return errors.New("nil endpoint to update") + } + + if err := registration.Validate(true); err != nil { + return errors.Wrap(err, "update endpoint") + } + + return scanner.UpdateRegistration(registration) +} + +// Delete ... +func (bm *basicManager) Delete(registrationUUID string) error { + if len(registrationUUID) == 0 { + return errors.New("empty UUID to delete") + } + + return scanner.DeleteRegistration(registrationUUID) +} + +// List ... +func (bm *basicManager) List(query *q.Query) ([]*scanner.Registration, error) { + return scanner.ListRegistrations(query) +} + +// SetAsDefault ... +func (bm *basicManager) SetAsDefault(registrationUUID string) error { + if len(registrationUUID) == 0 { + return errors.New("empty UUID to set default") + } + + return scanner.SetDefaultRegistration(registrationUUID) +} + +// GetDefault ... +func (bm *basicManager) GetDefault() (*scanner.Registration, error) { + return scanner.GetDefaultRegistration() +} diff --git a/src/pkg/scan/scanner/manager_test.go b/src/pkg/scan/scanner/manager_test.go new file mode 100644 index 000000000..0cee77117 --- /dev/null +++ b/src/pkg/scan/scanner/manager_test.go @@ -0,0 +1,115 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scanner + +import ( + "testing" + + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/pkg/q" + "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// BasicManagerTestSuite tests the basic manager +type BasicManagerTestSuite struct { + suite.Suite + + mgr Manager + sampleUUID string +} + +// TestBasicManager is the entry of BasicManagerTestSuite +func TestBasicManager(t *testing.T) { + suite.Run(t, new(BasicManagerTestSuite)) +} + +// SetupSuite prepares env for test suite +func (suite *BasicManagerTestSuite) SetupSuite() { + dao.PrepareTestForPostgresSQL() + + suite.mgr = New() + + r := &scanner.Registration{ + Name: "forUT", + Description: "sample registration", + URL: "https://sample.scanner.com", + Adapter: "Clair", + Version: "0.1.0", + Vendor: "Harbor", + } + + uid, err := suite.mgr.Create(r) + require.NoError(suite.T(), err) + suite.sampleUUID = uid +} + +// TearDownSuite clears env for test suite +func (suite *BasicManagerTestSuite) TearDownSuite() { + err := suite.mgr.Delete(suite.sampleUUID) + require.NoError(suite.T(), err, "delete registration") +} + +// TestList tests list registrations +func (suite *BasicManagerTestSuite) TestList() { + m := make(map[string]string, 1) + m["name"] = "forUT" + + l, err := suite.mgr.List(&q.Query{ + PageNumber: 1, + PageSize: 10, + Keywords: m, + }) + + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) +} + +// TestGet tests get registration +func (suite *BasicManagerTestSuite) TestGet() { + r, err := suite.mgr.Get(suite.sampleUUID) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), r) + assert.Equal(suite.T(), "forUT", r.Name) +} + +// TestUpdate tests update registration +func (suite *BasicManagerTestSuite) TestUpdate() { + r, err := suite.mgr.Get(suite.sampleUUID) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), r) + + r.URL = "https://updated.com" + err = suite.mgr.Update(r) + require.NoError(suite.T(), err) + + r, err = suite.mgr.Get(suite.sampleUUID) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), r) + assert.Equal(suite.T(), "https://updated.com", r.URL) +} + +// TestDefault tests get/set default registration +func (suite *BasicManagerTestSuite) TestDefault() { + err := suite.mgr.SetAsDefault(suite.sampleUUID) + require.NoError(suite.T(), err) + + dr, err := suite.mgr.GetDefault() + require.NoError(suite.T(), err) + require.NotNil(suite.T(), dr) + assert.Equal(suite.T(), true, dr.IsDefault) +} diff --git a/src/pkg/scan/scanner/scan/controller.go b/src/pkg/scan/scanner/scan/controller.go new file mode 100644 index 000000000..e7feb622f --- /dev/null +++ b/src/pkg/scan/scanner/scan/controller.go @@ -0,0 +1,48 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scan" + +// Options object for the scan action +type Options struct{} + +// Option for scan action +type Option interface { + // Apply option to the passing in options + Apply(options *Options) error +} + +// Controller defines operations for scan controlling +type Controller interface { + // Scan the given artifact + // + // Arguments: + // artifact *res.Artifact : artifact to be scanned + // + // Returns: + // error : non nil error if any errors occurred + Scan(artifact *Artifact, options ...Option) error + + // GetReport gets the reports for the given artifact identified by the digest + // + // Arguments: + // artifact *res.Artifact : the scanned artifact + // + // Returns: + // []*scan.Report : scan results by different scanner vendors + // error : non nil error if any errors occurred + GetReport(artifact *Artifact) ([]*scan.Report, error) +} diff --git a/src/pkg/scan/scanner/scan/models.go b/src/pkg/scan/scanner/scan/models.go new file mode 100644 index 000000000..7bb67b909 --- /dev/null +++ b/src/pkg/scan/scanner/scan/models.go @@ -0,0 +1,46 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +// Artifact represents an artifact stored in Registry. +type Artifact struct { + // The full name of a Harbor repository containing the artifact, including the namespace. + // For example, `library/oracle/nosql`. + Repository string + // The artifact's digest, consisting of an algorithm and hex portion. + // For example, `sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b`, + // represents sha256 based digest. + Digest string + // The mime type of the scanned artifact + MimeType string +} + +// Registry represents Registry connection settings. +type Registry struct { + // A base URL of the Docker Registry v2 API exposed by Harbor. + URL string + // An optional value of the HTTP Authorization header sent with each request to the Docker Registry v2 API. + // For example, `Bearer: JWTTOKENGOESHERE`. + Authorization string +} + +// Request represents a structure that is sent to a Scanner Adapter to initiate artifact scanning. +// Conducts all the details required to pull the artifact from a Harbor registry. +type Request struct { + // Connection settings for the Docker Registry v2 API exposed by Harbor. + Registry *Registry + // Artifact to be scanned. + Artifact *Artifact +} diff --git a/src/portal/angular.json b/src/portal/angular.json index 157657038..b7bf203d3 100644 --- a/src/portal/angular.json +++ b/src/portal/angular.json @@ -26,6 +26,7 @@ "node_modules/@clr/ui/clr-ui.min.css", "node_modules/swagger-ui/dist/swagger-ui.css", "node_modules/prismjs/themes/prism-solarizedlight.css", + "src/global.scss", "src/styles.css" ], "scripts": [ diff --git a/src/portal/lib/ng-package.json b/src/portal/lib/ng-package.json index 89852ea86..921319790 100644 --- a/src/portal/lib/ng-package.json +++ b/src/portal/lib/ng-package.json @@ -4,11 +4,6 @@ "deleteDestPath": false, "lib": { "entryFile": "index.ts", - "externals": { - "@ngx-translate/core": "ngx-translate-core", - "@ngx-translate/core/index": "ngx-translate-core", - "ngx-markdown": "ngx-markdown" - }, "umdModuleIds": { "@clr/angular" : "angular", "ngx-markdown" : "ngxMarkdown", diff --git a/src/portal/lib/ng-package.prod.json b/src/portal/lib/ng-package.prod.json index 85a87a50d..4bf1dc101 100644 --- a/src/portal/lib/ng-package.prod.json +++ b/src/portal/lib/ng-package.prod.json @@ -3,11 +3,6 @@ "dest": "./dist", "lib": { "entryFile": "index.ts", - "externals": { - "@ngx-translate/core": "ngx-translate-core", - "@ngx-translate/core/index": "ngx-translate-core", - "ngx-markdown": "ngx-markdown" - }, "umdModuleIds": { "@clr/angular" : "angular", "ngx-markdown" : "ngxMarkdown", diff --git a/src/portal/lib/package-lock.json b/src/portal/lib/package-lock.json new file mode 100644 index 000000000..e4655fb88 --- /dev/null +++ b/src/portal/lib/package-lock.json @@ -0,0 +1,5 @@ +{ + "name": "@harbor/ui", + "version": "1.10.0", + "lockfileVersion": 1 +} diff --git a/src/portal/lib/package.json b/src/portal/lib/package.json index 9c49c4207..db2c23758 100644 --- a/src/portal/lib/package.json +++ b/src/portal/lib/package.json @@ -1,7 +1,7 @@ { "name": "@harbor/ui", - "version": "1.9.0", - "description": "Harbor shared UI components based on Clarity and Angular7", + "version": "1.10.0", + "description": "Harbor shared UI components based on Clarity and Angular8", "author": "CNCF", "module": "index.js", "main": "bundles/harborui.umd.min.js", @@ -19,26 +19,26 @@ }, "homepage": "https://github.com/vmware/harbor#readme", "peerDependencies": { - "@angular/animations": "^7.1.3", - "@angular/common": "^7.1.3", - "@angular/compiler": "^7.1.3", - "@angular/core": "^7.1.3", - "@angular/forms": "^7.1.3", - "@angular/http": "^7.1.3", - "@angular/platform-browser": "^7.1.3", - "@angular/platform-browser-dynamic": "^7.1.3", - "@angular/router": "^7.1.3", + "@angular/animations": "^8.2.0", + "@angular/common": "^8.2.0", + "@angular/compiler": "^8.2.0", + "@angular/core": "^8.2.0", + "@angular/forms": "^8.2.0", + "@angular/http": "^8.2.0", + "@angular/platform-browser": "^8.2.0", + "@angular/platform-browser-dynamic": "^8.2.0", + "@angular/router": "^8.2.0", "@ngx-translate/core": "^10.0.2", "@ngx-translate/http-loader": "^3.0.1", "@webcomponents/custom-elements": "^1.1.3", - "@clr/angular": "^1.0.0", - "@clr/ui": "^1.0.0", - "@clr/icons": "^1.0.0", + "@clr/angular": "^2.1.0", + "@clr/icons": "^2.1.0", + "@clr/ui": "^2.1.0", "core-js": "^2.5.4", "intl": "^1.2.5", "mutationobserver-shim": "^0.3.2", "ngx-cookie": "^1.0.0", - "ngx-markdown": "^6.2.0", + "ngx-markdown": "^8.1.0", "rxjs": "^6.3.3", "ts-helpers": "^1.1.1", "web-animations-js": "^2.2.1", diff --git a/src/portal/lib/src/config/config.ts b/src/portal/lib/src/config/config.ts index 9505b11c5..7d11b5d0a 100644 --- a/src/portal/lib/src/config/config.ts +++ b/src/portal/lib/src/config/config.ts @@ -100,6 +100,7 @@ export class Configuration { oidc_scope?: StringValueItem; count_per_project: NumberValueItem; storage_per_project: NumberValueItem; + cfg_expiration: NumberValueItem; public constructor() { this.auth_mode = new StringValueItem("db_auth", true); this.project_creation_restriction = new StringValueItem("everyone", true); diff --git a/src/portal/lib/src/config/gc/gc.component.spec.ts b/src/portal/lib/src/config/gc/gc.component.spec.ts index c9b442682..1f4941b8c 100644 --- a/src/portal/lib/src/config/gc/gc.component.spec.ts +++ b/src/portal/lib/src/config/gc/gc.component.spec.ts @@ -9,6 +9,7 @@ import { GcViewModelFactory } from './gc.viewmodel.factory'; import { CronScheduleComponent } from '../../cron-schedule/cron-schedule.component'; import { CronTooltipComponent } from "../../cron-schedule/cron-tooltip/cron-tooltip.component"; import { of } from 'rxjs'; +import { GcJobData } from './gcLog'; describe('GcComponent', () => { let component: GcComponent; @@ -18,13 +19,17 @@ describe('GcComponent', () => { systemInfoEndpoint: "/api/system/gc" }; let mockSchedule = []; - let mockJobs = [ + let mockJobs: GcJobData[] = [ { id: 22222, schedule: null, job_status: 'string', - creation_time: new Date(), - update_time: new Date(), + creation_time: new Date().toDateString(), + update_time: new Date().toDateString(), + job_name: 'string', + job_kind: 'string', + job_uuid: 'string', + delete: false } ]; let spySchedule: jasmine.Spy; diff --git a/src/portal/lib/src/config/gc/gc.component.ts b/src/portal/lib/src/config/gc/gc.component.ts index 44d805d92..ccd4196aa 100644 --- a/src/portal/lib/src/config/gc/gc.component.ts +++ b/src/portal/lib/src/config/gc/gc.component.ts @@ -32,7 +32,7 @@ export class GcComponent implements OnInit { getText = 'CONFIG.GC'; getLabelCurrent = 'GC.CURRENT_SCHEDULE'; @Output() loadingGcStatus = new EventEmitter(); - @ViewChild(CronScheduleComponent) + @ViewChild(CronScheduleComponent, {static: false}) CronScheduleComponent: CronScheduleComponent; constructor( private gcRepoService: GcRepoService, diff --git a/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.html b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.html index c9bd48440..89ab0c585 100644 --- a/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.html +++ b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.html @@ -4,79 +4,73 @@ -
-
- -
- -
-
- -
+ + + + + + + + + + + + {{ 'DESTINATION.NAME_IS_REQUIRED' | translate }} + + + + + + + + +
+ +
+
+ +
-
- -
- - -
- -
- - -
- -
- - +
+ + {{ 'DESTINATION.URL_IS_REQUIRED' | translate }} +
- -
- - +
+ + + + + + +
+ +
+
+ + + + +
- -
- - - - - -
- -
- - +
+ + +
-
- - -
-
+ + + + + +
+ + +
\ No newline at end of file diff --git a/src/portal/lib/src/create-edit-endpoint/create-edit-endpoint.component.ts b/src/portal/lib/src/create-edit-endpoint/create-edit-endpoint.component.ts index 28d7624e4..b02b6feef 100644 --- a/src/portal/lib/src/create-edit-endpoint/create-edit-endpoint.component.ts +++ b/src/portal/lib/src/create-edit-endpoint/create-edit-endpoint.component.ts @@ -63,13 +63,13 @@ export class CreateEditEndpointComponent selectedType: string; initVal: Endpoint; targetForm: NgForm; - @ViewChild("targetForm") currentForm: NgForm; + @ViewChild("targetForm", {static: false}) currentForm: NgForm; targetEndpoint; testOngoing: boolean; onGoing: boolean; endpointId: number | string; - @ViewChild(InlineAlertComponent) inlineAlert: InlineAlertComponent; + @ViewChild(InlineAlertComponent, {static: false}) inlineAlert: InlineAlertComponent; @Output() reload = new EventEmitter(); diff --git a/src/portal/lib/src/create-edit-label/create-edit-label.component.html b/src/portal/lib/src/create-edit-label/create-edit-label.component.html index ab1f83b9d..92f040130 100644 --- a/src/portal/lib/src/create-edit-label/create-edit-label.component.html +++ b/src/portal/lib/src/create-edit-label/create-edit-label.component.html @@ -3,13 +3,13 @@
- +
diff --git a/src/portal/lib/src/project-policy-config/project-policy-config.component.scss b/src/portal/lib/src/project-policy-config/project-policy-config.component.scss index ddda99c37..53759f80c 100644 --- a/src/portal/lib/src/project-policy-config/project-policy-config.component.scss +++ b/src/portal/lib/src/project-policy-config/project-policy-config.component.scss @@ -5,6 +5,7 @@ .select { width: 120px; } + .margin-top-4 { margin-top: 4px; } @@ -17,6 +18,7 @@ width: 270px; color: #0079bb; overflow-y: auto; + li { height: 24px; line-height: 24px; @@ -43,16 +45,16 @@ .padding-top-8 { padding-top: 8px; } - -.padding-left-80 { - padding-left: 80px; +.position-relative { + position: relative; } - .add-modal { position: absolute; padding: 0 8px; background-color: rgb(238, 238, 238); - + .flex-direction-column { + flex-direction: column; + } input { width: 100%; border: 1px solid; @@ -63,8 +65,34 @@ } } -.hand{ +.hand { cursor: pointer; margin: 0; } +.config-subtext { + font-size: 0.55rem; + line-height: 1.2rem; + color: rgb(86, 86, 86); + font-weight: 300; +} + +.mt-05 { + margin-bottom: 0.5rem; +} + +.col-flex-grow-0 { + flex-grow: 0; +} + +.expire-data { + min-width: 12.5rem; + margin-top: -1rem; +} + +.bottom-line { + display: flex; + flex-direction: column-reverse; + font-size: 13px; + color: #000; +} diff --git a/src/portal/lib/src/project-policy-config/project-policy-config.component.ts b/src/portal/lib/src/project-policy-config/project-policy-config.component.ts index a6ab59495..1727ad04e 100644 --- a/src/portal/lib/src/project-policy-config/project-policy-config.component.ts +++ b/src/portal/lib/src/project-policy-config/project-policy-config.component.ts @@ -61,9 +61,9 @@ export class ProjectPolicyConfigComponent implements OnInit { @Input() hasSignedIn: boolean; @Input() hasProjectAdminRole: boolean; - @ViewChild('cfgConfirmationDialog') confirmationDlg: ConfirmationDialogComponent; - @ViewChild('dateInput') dateInput: ElementRef; - @ViewChild('dateSystemInput') dateSystemInput: ElementRef; + @ViewChild('cfgConfirmationDialog', {static: false}) confirmationDlg: ConfirmationDialogComponent; + @ViewChild('dateInput', {static: false}) dateInput: ElementRef; + @ViewChild('dateSystemInput', {static: false}) dateSystemInput: ElementRef; systemInfo: SystemInfo; orgProjectPolicy = new ProjectPolicy(); diff --git a/src/portal/lib/src/push-image/push-image.component.spec.ts b/src/portal/lib/src/push-image/push-image.component.spec.ts index a9178514b..4605fcb0d 100644 --- a/src/portal/lib/src/push-image/push-image.component.spec.ts +++ b/src/portal/lib/src/push-image/push-image.component.spec.ts @@ -39,7 +39,7 @@ describe('PushImageButtonComponent (inline template)', () => { expect(component).toBeTruthy(); }); - it('should open the drop-down panel', fakeAsync(() => { + it('should open the drop-down panel', () => { fixture.detectChanges(); fixture.whenStable().then(() => { fixture.detectChanges(); @@ -57,6 +57,6 @@ describe('PushImageButtonComponent (inline template)', () => { expect(copyInputs[1].value.trim()).toEqual(`docker push ${component.registryUrl}/${component.projectName}/IMAGE[:TAG]`); }); }); - })); + }); }); diff --git a/src/portal/lib/src/push-image/push-image.component.ts b/src/portal/lib/src/push-image/push-image.component.ts index 0fd21a60a..3c35ae2a8 100644 --- a/src/portal/lib/src/push-image/push-image.component.ts +++ b/src/portal/lib/src/push-image/push-image.component.ts @@ -13,9 +13,9 @@ export class PushImageButtonComponent { @Input() registryUrl: string = "unknown"; @Input() projectName: string = "unknown"; - @ViewChild("tagCopy") tagCopyInput: CopyInputComponent; - @ViewChild("pushCopy") pushCopyInput: CopyInputComponent; - @ViewChild("copyAlert") copyAlert: InlineAlertComponent; + @ViewChild("tagCopy", {static: false}) tagCopyInput: CopyInputComponent; + @ViewChild("pushCopy", {static: false}) pushCopyInput: CopyInputComponent; + @ViewChild("copyAlert", {static: false}) copyAlert: InlineAlertComponent; public get tagCommand(): string { return `docker tag SOURCE_IMAGE[:TAG] ${this.registryUrl}/${ diff --git a/src/portal/lib/src/push-image/push-image.scss b/src/portal/lib/src/push-image/push-image.scss index d5707a3d3..74835d8ae 100644 --- a/src/portal/lib/src/push-image/push-image.scss +++ b/src/portal/lib/src/push-image/push-image.scss @@ -32,6 +32,7 @@ .command-input { font-size: 14px; font-weight: 500; + border: 0; } :host>>>.dropdown-menu { diff --git a/src/portal/lib/src/replication/replication.component.ts b/src/portal/lib/src/replication/replication.component.ts index 1bdbfe200..8079ac3bb 100644 --- a/src/portal/lib/src/replication/replication.component.ts +++ b/src/portal/lib/src/replication/replication.component.ts @@ -120,16 +120,16 @@ export class ReplicationComponent implements OnInit, OnDestroy { jobs: ReplicationJobItem[]; - @ViewChild(ListReplicationRuleComponent) + @ViewChild(ListReplicationRuleComponent, {static: false}) listReplicationRule: ListReplicationRuleComponent; - @ViewChild(CreateEditRuleComponent) + @ViewChild(CreateEditRuleComponent, {static: false}) createEditPolicyComponent: CreateEditRuleComponent; - @ViewChild("replicationConfirmDialog") + @ViewChild("replicationConfirmDialog", {static: false}) replicationConfirmDialog: ConfirmationDialogComponent; - @ViewChild("StopConfirmDialog") + @ViewChild("StopConfirmDialog", {static: false}) StopConfirmDialog: ConfirmationDialogComponent; creationTimeComparator: Comparator = new CustomComparator< diff --git a/src/portal/lib/src/repository-gridview/repository-gridview.component.ts b/src/portal/lib/src/repository-gridview/repository-gridview.component.ts index fe533195b..62bd594ba 100644 --- a/src/portal/lib/src/repository-gridview/repository-gridview.component.ts +++ b/src/portal/lib/src/repository-gridview/repository-gridview.component.ts @@ -80,10 +80,10 @@ export class RepositoryGridviewComponent implements OnChanges, OnInit { totalCount = 0; currentState: State; - @ViewChild("confirmationDialog") + @ViewChild("confirmationDialog", {static: false}) confirmationDialog: ConfirmationDialogComponent; - @ViewChild("gridView") gridView: GridViewComponent; + @ViewChild("gridView", {static: false}) gridView: GridViewComponent; hasCreateRepositoryPermission: boolean; hasDeleteRepositoryPermission: boolean; constructor(@Inject(SERVICE_CONFIG) private configInfo: IServiceConfig, diff --git a/src/portal/lib/src/repository/repository.component.html b/src/portal/lib/src/repository/repository.component.html index a5216ed33..c2ef10c67 100644 --- a/src/portal/lib/src/repository/repository.component.html +++ b/src/portal/lib/src/repository/repository.component.html @@ -33,7 +33,7 @@ {{ 'REPOSITORY.MARKDOWN' | translate }} -
+

{{'REPOSITORY.NO_INFO' | translate }}

@@ -42,11 +42,12 @@
- +
- - + +
diff --git a/src/portal/lib/src/repository/repository.component.ts b/src/portal/lib/src/repository/repository.component.ts index 74204ec1e..1ae176e48 100644 --- a/src/portal/lib/src/repository/repository.component.ts +++ b/src/portal/lib/src/repository/repository.component.ts @@ -56,7 +56,7 @@ export class RepositoryComponent implements OnInit { timerHandler: any; - @ViewChild('confirmationDialog') + @ViewChild('confirmationDialog', {static: false}) confirmationDlg: ConfirmationDialogComponent; constructor( diff --git a/src/portal/lib/src/tag/tag-detail.component.html b/src/portal/lib/src/tag/tag-detail.component.html index 83c053a2f..bee923073 100644 --- a/src/portal/lib/src/tag/tag-detail.component.html +++ b/src/portal/lib/src/tag/tag-detail.component.html @@ -89,8 +89,8 @@ - - + + {{ 'REPOSITORY.BUILD_HISTORY' | translate }} diff --git a/src/portal/lib/src/tag/tag.component.html b/src/portal/lib/src/tag/tag.component.html index 9df2ddad1..445d6d9ef 100644 --- a/src/portal/lib/src/tag/tag.component.html +++ b/src/portal/lib/src/tag/tag.component.html @@ -3,7 +3,7 @@