diff --git a/make/migrations/postgresql/0012_1.10.0_schema.up.sql b/make/migrations/postgresql/0012_1.10.0_schema.up.sql deleted file mode 100644 index 57f6ff4ff..000000000 --- a/make/migrations/postgresql/0012_1.10.0_schema.up.sql +++ /dev/null @@ -1,34 +0,0 @@ -/*Table for keeping the plug scanner registration*/ -CREATE TABLE scanner_registration -( - id SERIAL PRIMARY KEY NOT NULL, - uuid VARCHAR(64) UNIQUE NOT NULL, - url VARCHAR(256) UNIQUE NOT NULL, - name VARCHAR(128) UNIQUE NOT NULL, - description VARCHAR(1024) NULL, - auth VARCHAR(16) NOT NULL, - access_cred VARCHAR(512) NULL, - disabled BOOLEAN NOT NULL DEFAULT FALSE, - is_default BOOLEAN NOT NULL DEFAULT FALSE, - skip_cert_verify BOOLEAN NOT NULL DEFAULT FALSE, - create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); - -/*Table for keeping the scan report. The report details are stored as JSON*/ -CREATE TABLE scan_report -( - id SERIAL PRIMARY KEY NOT NULL, - uuid VARCHAR(64) UNIQUE NOT NULL, - digest VARCHAR(256) NOT NULL, - registration_uuid VARCHAR(64) NOT NULL, - mime_type VARCHAR(256) NOT NULL, - job_id VARCHAR(32), - status VARCHAR(16) NOT NULL, - status_code INTEGER DEFAULT 0, - status_rev BIGINT DEFAULT 0, - report JSON, - start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - end_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(digest, registration_uuid, mime_type) -) diff --git a/make/migrations/postgresql/0015_1.10.0_schema.up.sql b/make/migrations/postgresql/0015_1.10.0_schema.up.sql index f119ad9df..a2904162e 100644 --- a/make/migrations/postgresql/0015_1.10.0_schema.up.sql +++ b/make/migrations/postgresql/0015_1.10.0_schema.up.sql @@ -1,3 +1,39 @@ +/*Table for keeping the plug scanner registration*/ +CREATE TABLE scanner_registration +( + id SERIAL PRIMARY KEY NOT NULL, + uuid VARCHAR(64) UNIQUE NOT NULL, + url VARCHAR(256) UNIQUE NOT NULL, + name VARCHAR(128) UNIQUE NOT NULL, + description VARCHAR(1024) NULL, + auth VARCHAR(16) NOT NULL, + access_cred VARCHAR(512) NULL, + disabled BOOLEAN NOT NULL DEFAULT FALSE, + is_default BOOLEAN NOT NULL DEFAULT FALSE, + skip_cert_verify BOOLEAN NOT NULL DEFAULT FALSE, + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +/*Table for keeping the scan report. The report details are stored as JSON*/ +CREATE TABLE scan_report +( + id SERIAL PRIMARY KEY NOT NULL, + uuid VARCHAR(64) UNIQUE NOT NULL, + digest VARCHAR(256) NOT NULL, + registration_uuid VARCHAR(64) NOT NULL, + mime_type VARCHAR(256) NOT NULL, + job_id VARCHAR(64), + track_id VARCHAR(64), + status VARCHAR(1024) NOT NULL, + status_code INTEGER DEFAULT 0, + status_rev BIGINT DEFAULT 0, + report JSON, + start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + end_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(digest, registration_uuid, mime_type) +); + /** Add table for immutable tag **/ CREATE TABLE immutable_tag_rule ( diff --git a/src/common/models/repo.go b/src/common/models/repo.go index 9993fbcc6..6562e9531 100644 --- a/src/common/models/repo.go +++ b/src/common/models/repo.go @@ -54,11 +54,11 @@ type RepositoryQuery struct { // TagResp holds the information of one image tag type TagResp struct { TagDetail - Signature *model.Target `json:"signature"` - ScanOverview *ImgScanOverview `json:"scan_overview,omitempty"` - Labels []*Label `json:"labels"` - PushTime time.Time `json:"push_time"` - PullTime time.Time `json:"pull_time"` + Signature *model.Target `json:"signature"` + ScanOverview map[string]interface{} `json:"scan_overview,omitempty"` + Labels []*Label `json:"labels"` + PushTime time.Time `json:"push_time"` + PullTime time.Time `json:"pull_time"` } // TagDetail ... diff --git a/src/common/rbac/const.go b/src/common/rbac/const.go index aa549116a..6b850b6e9 100755 --- a/src/common/rbac/const.go +++ b/src/common/rbac/const.go @@ -54,9 +54,10 @@ const ( ResourceRepositoryTag = Resource("repository-tag") ResourceRepositoryTagLabel = Resource("repository-tag-label") ResourceRepositoryTagManifest = Resource("repository-tag-manifest") - ResourceRepositoryTagScanJob = Resource("repository-tag-scan-job") - ResourceRepositoryTagVulnerability = Resource("repository-tag-vulnerability") + ResourceRepositoryTagScanJob = Resource("repository-tag-scan-job") // TODO: remove + ResourceRepositoryTagVulnerability = Resource("repository-tag-vulnerability") // TODO: remove ResourceRobot = Resource("robot") ResourceNotificationPolicy = Resource("notification-policy") + ResourceScan = Resource("scan") ResourceSelf = Resource("") // subresource for self ) diff --git a/src/common/rbac/project/util.go b/src/common/rbac/project/util.go index ce69800aa..85116fe21 100644 --- a/src/common/rbac/project/util.go +++ b/src/common/rbac/project/util.go @@ -162,6 +162,9 @@ var ( {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionDelete}, {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionList}, {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionRead}, + + {Resource: rbac.ResourceScan, Action: rbac.ActionCreate}, + {Resource: rbac.ResourceScan, Action: rbac.ActionRead}, } ) diff --git a/src/common/rbac/project/visitor_role.go b/src/common/rbac/project/visitor_role.go index 651252cdb..d8d594f4f 100755 --- a/src/common/rbac/project/visitor_role.go +++ b/src/common/rbac/project/visitor_role.go @@ -119,6 +119,9 @@ var ( {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionDelete}, {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionList}, {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionRead}, + + {Resource: rbac.ResourceScan, Action: rbac.ActionCreate}, + {Resource: rbac.ResourceScan, Action: rbac.ActionRead}, }, "master": { @@ -201,6 +204,9 @@ var ( {Resource: rbac.ResourceRobot, Action: rbac.ActionList}, {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionList}, + + {Resource: rbac.ResourceScan, Action: rbac.ActionCreate}, + {Resource: rbac.ResourceScan, Action: rbac.ActionRead}, }, "developer": { @@ -251,6 +257,9 @@ var ( {Resource: rbac.ResourceRobot, Action: rbac.ActionRead}, {Resource: rbac.ResourceRobot, Action: rbac.ActionList}, + + {Resource: rbac.ResourceScan, Action: rbac.ActionCreate}, + {Resource: rbac.ResourceScan, Action: rbac.ActionRead}, }, "guest": { diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index 7cab5f769..f60b500b4 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -211,8 +211,17 @@ func init() { scannerAPI := &ScannerAPI{} beego.Router("/api/scanners", scannerAPI, "post:Create;get:List") beego.Router("/api/scanners/:uuid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault") + beego.Router("/api/scanners/:uuid/metadata", scannerAPI, "get:Metadata") + beego.Router("/api/scanners/ping", scannerAPI, "post:Ping") + // Add routes for project level scanner - beego.Router("/api/projects/:pid([0-9]+)/scanner", scannerAPI, "get:GetProjectScanner;put:SetProjectScanner") + proScannerAPI := &ProjectScannerAPI{} + beego.Router("/api/projects/:pid([0-9]+)/scanner", proScannerAPI, "get:GetProjectScanner;put:SetProjectScanner") + + // Add routes for scan + scanAPI := &ScanAPI{} + beego.Router("/api/repositories/*/tags/:tag/scan", scanAPI, "post:Scan;get:Report") + beego.Router("/api/repositories/*/tags/:tag/scan/:uuid/log", scanAPI, "get:Log") // syncRegistry if err := SyncRegistry(config.GlobalProjectMgr); err != nil { diff --git a/src/core/api/pro_scanner.go b/src/core/api/pro_scanner.go new file mode 100644 index 000000000..ff0e45436 --- /dev/null +++ b/src/core/api/pro_scanner.go @@ -0,0 +1,112 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "github.com/goharbor/harbor/src/common/rbac" + "github.com/goharbor/harbor/src/pkg/scan/api/scanner" + "github.com/pkg/errors" +) + +// ProjectScannerAPI provides rest API for managing the project level scanner(s). +type ProjectScannerAPI struct { + // The base controller to provide common utilities + BaseController + // Scanner controller for operating scanner registrations. + c scanner.Controller + // ID of the project + pid int64 +} + +// Prepare sth. for the subsequent actions +func (sa *ProjectScannerAPI) Prepare() { + // Call super prepare method + sa.BaseController.Prepare() + + // Check access permissions + if !sa.RequireAuthenticated() { + return + } + + // Get ID of the project + pid, err := sa.GetInt64FromPath(":pid") + if err != nil { + sa.SendBadRequestError(errors.Wrap(err, "project scanner API")) + return + } + + // Check if the project exists + exists, err := sa.ProjectMgr.Exists(pid) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "project scanner API")) + return + } + + if !exists { + sa.SendNotFoundError(errors.Errorf("project with id %d", sa.pid)) + return + } + + sa.pid = pid + + sa.c = scanner.DefaultController +} + +// GetProjectScanner gets the project level scanner +func (sa *ProjectScannerAPI) GetProjectScanner() { + // Check access permissions + if !sa.RequireProjectAccess(sa.pid, rbac.ActionRead, rbac.ResourceConfiguration) { + return + } + + r, err := sa.c.GetRegistrationByProject(sa.pid) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: get project scanners")) + return + } + + if r != nil { + sa.Data["json"] = r + } else { + sa.Data["json"] = make(map[string]interface{}) + } + + sa.ServeJSON() +} + +// SetProjectScanner sets the project level scanner +func (sa *ProjectScannerAPI) SetProjectScanner() { + // Check access permissions + if !sa.RequireProjectAccess(sa.pid, rbac.ActionUpdate, rbac.ResourceConfiguration) { + return + } + + body := make(map[string]string) + if err := sa.DecodeJSONReq(&body); err != nil { + sa.SendBadRequestError(errors.Wrap(err, "scanner API: set project scanners")) + return + } + + uuid, ok := body["uuid"] + if !ok || len(uuid) == 0 { + sa.SendBadRequestError(errors.New("missing scanner uuid when setting project scanner")) + return + } + + if err := sa.c.SetRegistrationByProject(sa.pid, uuid); err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: set project scanners")) + return + } +} diff --git a/src/core/api/pro_scanner_test.go b/src/core/api/pro_scanner_test.go new file mode 100644 index 000000000..42d64f305 --- /dev/null +++ b/src/core/api/pro_scanner_test.go @@ -0,0 +1,95 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "fmt" + "net/http" + "testing" + + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner" + "github.com/stretchr/testify/suite" +) + +// ProScannerAPITestSuite is test suite for testing the project scanner API +type ProScannerAPITestSuite struct { + suite.Suite + + originC sc.Controller + mockC *MockScannerAPIController +} + +// TestProScannerAPI is the entry of ProScannerAPITestSuite +func TestProScannerAPI(t *testing.T) { + suite.Run(t, new(ProScannerAPITestSuite)) +} + +// SetupSuite prepares testing env +func (suite *ProScannerAPITestSuite) SetupTest() { + suite.originC = sc.DefaultController + m := &MockScannerAPIController{} + sc.DefaultController = m + + suite.mockC = m +} + +// TearDownTest clears test case env +func (suite *ProScannerAPITestSuite) TearDownTest() { + // Restore + sc.DefaultController = suite.originC +} + +// TestScannerAPIProjectScanner tests the API of getting/setting project level scanner +func (suite *ProScannerAPITestSuite) TestScannerAPIProjectScanner() { + suite.mockC.On("SetRegistrationByProject", int64(1), "uuid").Return(nil) + + // Set + body := make(map[string]interface{}, 1) + body["uuid"] = "uuid" + runCodeCheckingCases(suite.T(), &codeCheckingCase{ + request: &testingRequest{ + url: fmt.Sprintf("/api/projects/%d/scanner", 1), + method: http.MethodPut, + credential: projAdmin, + bodyJSON: body, + }, + code: http.StatusOK, + }) + + r := &scanner.Registration{ + ID: 1004, + UUID: "uuid", + Name: "TestScannerAPIProjectScanner", + Description: "JUST FOR TEST", + URL: "https://a.b.c", + } + suite.mockC.On("GetRegistrationByProject", int64(1)).Return(r, nil) + + // Get + rr := &scanner.Registration{} + err := handleAndParse(&testingRequest{ + url: fmt.Sprintf("/api/projects/%d/scanner", 1), + method: http.MethodGet, + credential: projAdmin, + }, rr) + require.NoError(suite.T(), err) + + assert.Equal(suite.T(), r.Name, rr.Name) + assert.Equal(suite.T(), r.UUID, rr.UUID) +} diff --git a/src/core/api/repository.go b/src/core/api/repository.go index 8a40d5f49..8227a5cae 100755 --- a/src/core/api/repository.go +++ b/src/core/api/repository.go @@ -25,6 +25,11 @@ import ( "strings" "time" + "github.com/goharbor/harbor/src/jobservice/logger" + + "github.com/goharbor/harbor/src/pkg/scan/api/scan" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" "github.com/goharbor/harbor/src/common" @@ -40,7 +45,6 @@ import ( "github.com/goharbor/harbor/src/core/config" notifierEvt "github.com/goharbor/harbor/src/core/notifier/event" coreutils "github.com/goharbor/harbor/src/core/utils" - "github.com/goharbor/harbor/src/pkg/scan" "github.com/goharbor/harbor/src/replication" "github.com/goharbor/harbor/src/replication/event" "github.com/goharbor/harbor/src/replication/model" @@ -397,6 +401,13 @@ func (ra *RepositoryAPI) GetTag() { return } + project, err := ra.ProjectMgr.Get(projectName) + if err != nil { + ra.ParseAndHandleError(fmt.Sprintf("failed to get the project %s", + projectName), err) + return + } + client, err := coreutils.NewRepositoryClientForUI(ra.SecurityCtx.GetUsername(), repository) if err != nil { ra.SendInternalServerError(fmt.Errorf("failed to initialize the client for %s: %v", @@ -414,7 +425,7 @@ func (ra *RepositoryAPI) GetTag() { return } - result := assembleTagsInParallel(client, repository, []string{tag}, + result := assembleTagsInParallel(client, project.ProjectID, repository, []string{tag}, ra.SecurityCtx.GetUsername()) ra.Data["json"] = result[0] ra.ServeJSON() @@ -523,14 +534,14 @@ func (ra *RepositoryAPI) GetTags() { } projectName, _ := utils.ParseRepository(repoName) - exist, err := ra.ProjectMgr.Exists(projectName) + project, err := ra.ProjectMgr.Get(projectName) if err != nil { - ra.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %s", + ra.ParseAndHandleError(fmt.Sprintf("failed to get the project %s", projectName), err) return } - if !exist { + if project == nil { ra.SendNotFoundError(fmt.Errorf("project %s not found", projectName)) return } @@ -587,8 +598,13 @@ func (ra *RepositoryAPI) GetTags() { return } - ra.Data["json"] = assembleTagsInParallel(client, repoName, tags, - ra.SecurityCtx.GetUsername()) + ra.Data["json"] = assembleTagsInParallel( + client, + project.ProjectID, + repoName, + tags, + ra.SecurityCtx.GetUsername(), + ) ra.ServeJSON() } @@ -607,7 +623,7 @@ func simpleTags(tags []string) []*models.TagResp { // get config, signature and scan overview and assemble them into one // struct for each tag in tags -func assembleTagsInParallel(client *registry.Repository, repository string, +func assembleTagsInParallel(client *registry.Repository, projectID int64, repository string, tags []string, username string) []*models.TagResp { var err error signatures := map[string][]notarymodel.Target{} @@ -621,8 +637,15 @@ func assembleTagsInParallel(client *registry.Repository, repository string, c := make(chan *models.TagResp) for _, tag := range tags { - go assembleTag(c, client, repository, tag, config.WithClair(), - config.WithNotary(), signatures) + go assembleTag( + c, + client, + projectID, + repository, + tag, + config.WithNotary(), + signatures, + ) } result := []*models.TagResp{} var item *models.TagResp @@ -636,8 +659,8 @@ func assembleTagsInParallel(client *registry.Repository, repository string, return result } -func assembleTag(c chan *models.TagResp, client *registry.Repository, - repository, tag string, clairEnabled, notaryEnabled bool, +func assembleTag(c chan *models.TagResp, client *registry.Repository, projectID int64, + repository, tag string, notaryEnabled bool, signatures map[string][]notarymodel.Target) { item := &models.TagResp{} // labels @@ -659,8 +682,9 @@ func assembleTag(c chan *models.TagResp, client *registry.Repository, } // scan overview - if clairEnabled { - item.ScanOverview = getScanOverview(item.Digest, item.Name) + so := getSummary(projectID, repository, item.Digest) + if len(so) > 0 { + item.ScanOverview = so } // signature, compare both digest and tag @@ -968,73 +992,6 @@ func (ra *RepositoryAPI) GetSignatures() { ra.ServeJSON() } -// ScanImage handles request POST /api/repository/$repository/tags/$tag/scan to trigger image scan manually. -func (ra *RepositoryAPI) ScanImage() { - if !config.WithClair() { - log.Warningf("Harbor is not deployed with Clair, scan is disabled.") - ra.SendInternalServerError(errors.New("harbor is not deployed with Clair, scan is disabled")) - return - } - repoName := ra.GetString(":splat") - tag := ra.GetString(":tag") - projectName, _ := utils.ParseRepository(repoName) - exist, err := ra.ProjectMgr.Exists(projectName) - if err != nil { - ra.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %s", - projectName), err) - return - } - if !exist { - ra.SendNotFoundError(fmt.Errorf("project %s not found", projectName)) - return - } - if !ra.SecurityCtx.IsAuthenticated() { - ra.SendUnAuthorizedError(errors.New("Unauthorized")) - return - } - - if !ra.RequireProjectAccess(projectName, rbac.ActionCreate, rbac.ResourceRepositoryTagScanJob) { - return - } - err = coreutils.TriggerImageScan(repoName, tag) - if err != nil { - log.Errorf("Error while calling job service to trigger image scan: %v", err) - ra.SendInternalServerError(errors.New("Failed to scan image, please check log for details")) - return - } -} - -// VulnerabilityDetails fetch vulnerability info from clair, transform to Harbor's format and return to client. -func (ra *RepositoryAPI) VulnerabilityDetails() { - if !config.WithClair() { - log.Warningf("Harbor is not deployed with Clair, it's not impossible to get vulnerability details.") - ra.SendInternalServerError(errors.New("harbor is not deployed with Clair, it's not impossible to get vulnerability details")) - return - } - repository := ra.GetString(":splat") - tag := ra.GetString(":tag") - exist, digest, err := ra.checkExistence(repository, tag) - if err != nil { - ra.SendInternalServerError(fmt.Errorf("failed to check the existence of resource, error: %v", err)) - return - } - if !exist { - ra.SendNotFoundError(fmt.Errorf("resource: %s:%s not found", repository, tag)) - return - } - - projectName, _ := utils.ParseRepository(repository) - if !ra.RequireProjectAccess(projectName, rbac.ActionList, rbac.ResourceRepositoryTagVulnerability) { - return - } - res, err := scan.VulnListByDigest(digest) - if err != nil { - log.Errorf("Failed to get vulnerability list for image: %s:%s", repository, tag) - } - ra.Data["json"] = res - ra.ServeJSON() -} - func getSignatures(username, repository string) (map[string][]notarymodel.Target, error) { targets, err := notary.GetInternalTargets(config.InternalNotaryEndpoint(), username, repository) @@ -1079,33 +1036,19 @@ func (ra *RepositoryAPI) checkExistence(repository, tag string) (bool, string, e return true, digest, nil } -// will return nil when it failed to get data. The parm "tag" is for logging only. -func getScanOverview(digest string, tag string) *models.ImgScanOverview { - if len(digest) == 0 { - log.Debug("digest is nil") - return nil +func getSummary(pid int64, repository string, digest string) map[string]interface{} { + // At present, only get harbor native report as default behavior. + artifact := &v1.Artifact{ + NamespaceID: pid, + Repository: repository, + Digest: digest, + MimeType: v1.MimeTypeDockerArtifact, } - data, err := dao.GetImgScanOverview(digest) + + sum, err := scan.DefaultController.GetSummary(artifact, []string{v1.MimeTypeNativeReport}) if err != nil { - log.Errorf("Failed to get scan result for tag:%s, digest: %s, error: %v", tag, digest, err) + logger.Errorf("Failed to get scan report summary with error: %s", err) } - if data == nil { - return nil - } - job, err := dao.GetScanJob(data.JobID) - if err != nil { - log.Errorf("Failed to get scan job for id:%d, error: %v", data.JobID, err) - return nil - } else if job == nil { // job does not exist - log.Errorf("The scan job with id: %d does not exist, returning nil", data.JobID) - return nil - } - data.Status = job.Status - if data.Status != models.JobFinished { - log.Debugf("Unsetting vulnerable related historical values, job status: %s", data.Status) - data.Sev = 0 - data.CompOverview = nil - data.DetailsKey = "" - } - return data + + return sum } diff --git a/src/core/api/repository_test.go b/src/core/api/repository_test.go index 7aa17a0b2..b51a38aeb 100644 --- a/src/core/api/repository_test.go +++ b/src/core/api/repository_test.go @@ -42,7 +42,7 @@ func TestGetRepos(t *testing.T) { } else { assert.Equal(int(200), code, "response code should be 200") if repos, ok := repositories.([]repoResp); ok { - assert.Equal(int(1), len(repos), "the length of repositories should be 1") + require.Equal(t, int(1), len(repos), "the length of repositories should be 1") assert.Equal(repos[0].Name, "library/hello-world", "unexpected repository name") } else { t.Error("unexpected response") diff --git a/src/core/api/scan.go b/src/core/api/scan.go new file mode 100644 index 000000000..aac29c22f --- /dev/null +++ b/src/core/api/scan.go @@ -0,0 +1,192 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "net/http" + "strconv" + + "github.com/goharbor/harbor/src/pkg/scan/report" + + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/rbac" + "github.com/goharbor/harbor/src/common/utils" + coreutils "github.com/goharbor/harbor/src/core/utils" + "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/pkg/scan/api/scan" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/pkg/errors" +) + +var digestFunc digestGetter = getDigest + +// ScanAPI handles the scan related actions +type ScanAPI struct { + BaseController + + // Target artifact + artifact *v1.Artifact + // Project reference + pro *models.Project +} + +// Prepare sth. for the subsequent actions +func (sa *ScanAPI) Prepare() { + // Call super prepare method + sa.BaseController.Prepare() + + // Parse parameters + repoName := sa.GetString(":splat") + tag := sa.GetString(":tag") + projectName, _ := utils.ParseRepository(repoName) + + pro, err := sa.ProjectMgr.Get(projectName) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scan API: prepare")) + return + } + if pro == nil { + sa.SendNotFoundError(errors.Errorf("project %s not found", projectName)) + return + } + sa.pro = pro + + // Check authentication + if !sa.RequireAuthenticated() { + return + } + + // Assemble artifact object + digest, err := digestFunc(repoName, tag, sa.SecurityCtx.GetUsername()) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scan API: prepare")) + return + } + + sa.artifact = &v1.Artifact{ + NamespaceID: pro.ProjectID, + Repository: repoName, + Tag: tag, + Digest: digest, + MimeType: v1.MimeTypeDockerArtifact, + } + + logger.Debugf("Scan API receives artifact: %#v", sa.artifact) +} + +// Scan artifact +func (sa *ScanAPI) Scan() { + // Check access permissions + if !sa.RequireProjectAccess(sa.pro.ProjectID, rbac.ActionCreate, rbac.ResourceScan) { + return + } + + if err := scan.DefaultController.Scan(sa.artifact); err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scan API: scan")) + return + } + + sa.Ctx.ResponseWriter.WriteHeader(http.StatusAccepted) +} + +// Report returns the required reports with the given mime types. +func (sa *ScanAPI) Report() { + // Check access permissions + if !sa.RequireProjectAccess(sa.pro.ProjectID, rbac.ActionRead, rbac.ResourceScan) { + return + } + + // Extract mime types + producesMimes := make([]string, 0) + if hl, ok := sa.Ctx.Request.Header[v1.HTTPAcceptHeader]; ok && len(hl) > 0 { + producesMimes = append(producesMimes, hl...) + } + + // Get the reports + reports, err := scan.DefaultController.GetReport(sa.artifact, producesMimes) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scan API: get report")) + return + } + + vulItems := make(map[string]interface{}) + for _, rp := range reports { + // Resolve scan report data only when it is ready + if len(rp.Report) == 0 { + continue + } + + vrp, err := report.ResolveData(rp.MimeType, []byte(rp.Report)) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scan API: get report")) + return + } + + vulItems[rp.MimeType] = vrp + } + + sa.Data["json"] = vulItems + sa.ServeJSON() +} + +// Log returns the log stream +func (sa *ScanAPI) Log() { + // Check access permissions + if !sa.RequireProjectAccess(sa.pro.ProjectID, rbac.ActionRead, rbac.ResourceScan) { + return + } + + uuid := sa.GetString(":uuid") + bytes, err := scan.DefaultController.GetScanLog(uuid) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scan API: log")) + return + } + + if bytes == nil { + // Not found + sa.SendNotFoundError(errors.Errorf("report with uuid %s does not exist", uuid)) + return + } + + sa.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(bytes))) + sa.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain") + _, err = sa.Ctx.ResponseWriter.Write(bytes) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scan API: log")) + } +} + +// digestGetter is a function template for getting digest. +// TODO: This can be removed if the registry access interface is ready. +type digestGetter func(repo, tag string, username string) (string, error) + +func getDigest(repo, tag string, username string) (string, error) { + client, err := coreutils.NewRepositoryClientForUI(username, repo) + if err != nil { + return "", err + } + + digest, exists, err := client.ManifestExist(tag) + if err != nil { + return "", err + } + + if !exists { + return "", errors.Errorf("tag %s does exist", tag) + } + + return digest, nil +} diff --git a/src/core/api/scan_job.go b/src/core/api/scan_job.go deleted file mode 100644 index 7cc38d61e..000000000 --- a/src/core/api/scan_job.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/rbac" - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/core/utils" - - "errors" - "fmt" - "net/http" - "strconv" - "strings" -) - -// ScanJobAPI handles request to /api/scanJobs/:id/log -type ScanJobAPI struct { - BaseController - jobID int64 - projectName string - jobUUID string -} - -// Prepare validates that whether user has read permission to the project of the repo the scan job scanned. -func (sj *ScanJobAPI) Prepare() { - sj.BaseController.Prepare() - if !sj.SecurityCtx.IsAuthenticated() { - sj.SendUnAuthorizedError(errors.New("UnAuthorized")) - return - } - id, err := sj.GetInt64FromPath(":id") - if err != nil { - sj.SendBadRequestError(errors.New("invalid ID")) - return - } - sj.jobID = id - - data, err := dao.GetScanJob(id) - if err != nil { - log.Errorf("Failed to load job data for job: %d, error: %v", id, err) - sj.SendInternalServerError(errors.New("Failed to get Job data")) - return - } - - projectName := strings.SplitN(data.Repository, "/", 2)[0] - if !sj.RequireProjectAccess(projectName, rbac.ActionRead, rbac.ResourceRepositoryTagScanJob) { - log.Errorf("User does not have read permission for project: %s", projectName) - return - } - sj.projectName = projectName - sj.jobUUID = data.UUID -} - -// GetLog ... -func (sj *ScanJobAPI) GetLog() { - logBytes, err := utils.GetJobServiceClient().GetJobLog(sj.jobUUID) - if err != nil { - sj.ParseAndHandleError(fmt.Sprintf("Failed to get job logs, uuid: %s, error: %v", sj.jobUUID, err), err) - return - } - sj.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(logBytes))) - sj.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain") - _, err = sj.Ctx.ResponseWriter.Write(logBytes) - if err != nil { - sj.SendInternalServerError(fmt.Errorf("Failed to write job logs, uuid: %s, error: %v", sj.jobUUID, err)) - } - -} diff --git a/src/core/api/scan_test.go b/src/core/api/scan_test.go new file mode 100644 index 000000000..2e80a35d0 --- /dev/null +++ b/src/core/api/scan_test.go @@ -0,0 +1,214 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/stretchr/testify/require" + + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/pkg/scan/api/scan" + dscan "github.com/goharbor/harbor/src/pkg/scan/dao/scan" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +var scanBaseURL = "/api/repositories/library/hello-world/tags/latest/scan" + +// ScanAPITestSuite is the test suite for scan API. +type ScanAPITestSuite struct { + suite.Suite + + originalC scan.Controller + c *MockScanAPIController + + originalDigestGetter digestGetter + + artifact *v1.Artifact +} + +// TestScanAPI is the entry point of ScanAPITestSuite. +func TestScanAPI(t *testing.T) { + suite.Run(t, new(ScanAPITestSuite)) +} + +// SetupSuite prepares test env for suite. +func (suite *ScanAPITestSuite) SetupSuite() { + suite.artifact = &v1.Artifact{ + NamespaceID: (int64)(1), + Repository: "library/hello-world", + Tag: "latest", + Digest: "digest-code-001", + MimeType: v1.MimeTypeDockerArtifact, + } +} + +// SetupTest prepares test env for test cases. +func (suite *ScanAPITestSuite) SetupTest() { + suite.originalC = scan.DefaultController + suite.c = &MockScanAPIController{} + + scan.DefaultController = suite.c + + suite.originalDigestGetter = digestFunc + digestFunc = func(repo, tag string, username string) (s string, e error) { + return "digest-code-001", nil + } +} + +// TearDownTest ... +func (suite *ScanAPITestSuite) TearDownTest() { + scan.DefaultController = suite.originalC + digestFunc = suite.originalDigestGetter +} + +// TestScanAPIBase ... +func (suite *ScanAPITestSuite) TestScanAPIBase() { + suite.c.On("Scan", &v1.Artifact{}).Return(nil) + // Including general cases + cases := []*codeCheckingCase{ + // 401 + { + request: &testingRequest{ + url: scanBaseURL, + method: http.MethodGet, + }, + code: http.StatusUnauthorized, + }, + // 403 + { + request: &testingRequest{ + url: scanBaseURL, + method: http.MethodPost, + credential: projGuest, + }, + code: http.StatusForbidden, + }, + } + + runCodeCheckingCases(suite.T(), cases...) +} + +// TestScanAPIScan ... +func (suite *ScanAPITestSuite) TestScanAPIScan() { + suite.c.On("Scan", suite.artifact).Return(nil) + + // Including general cases + cases := []*codeCheckingCase{ + // 202 + { + request: &testingRequest{ + url: scanBaseURL, + method: http.MethodPost, + credential: projDeveloper, + }, + code: http.StatusAccepted, + }, + } + + runCodeCheckingCases(suite.T(), cases...) +} + +// TestScanAPIReport ... +func (suite *ScanAPITestSuite) TestScanAPIReport() { + suite.c.On("GetReport", suite.artifact, []string{v1.MimeTypeNativeReport}).Return([]*dscan.Report{}, nil) + + vulItems := make(map[string]interface{}) + + header := make(http.Header) + header.Add("Accept", v1.MimeTypeNativeReport) + err := handleAndParse( + &testingRequest{ + url: scanBaseURL, + method: http.MethodGet, + credential: projDeveloper, + header: header, + }, &vulItems) + require.NoError(suite.T(), err) +} + +// TestScanAPILog ... +func (suite *ScanAPITestSuite) TestScanAPILog() { + suite.c.On("GetScanLog", "the-uuid-001").Return([]byte(`{"log": "this is my log"}`), nil) + + logs := make(map[string]string) + err := handleAndParse( + &testingRequest{ + url: fmt.Sprintf("%s/%s", scanBaseURL, "the-uuid-001/log"), + method: http.MethodGet, + credential: projDeveloper, + }, &logs) + require.NoError(suite.T(), err) + assert.Condition(suite.T(), func() (success bool) { + success = len(logs) > 0 + return + }) +} + +// Mock things + +// MockScanAPIController ... +type MockScanAPIController struct { + mock.Mock +} + +// Scan ... +func (msc *MockScanAPIController) Scan(artifact *v1.Artifact) error { + args := msc.Called(artifact) + + return args.Error(0) +} + +func (msc *MockScanAPIController) GetReport(artifact *v1.Artifact, mimeTypes []string) ([]*dscan.Report, error) { + args := msc.Called(artifact, mimeTypes) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]*dscan.Report), args.Error(1) +} + +func (msc *MockScanAPIController) GetSummary(artifact *v1.Artifact, mimeTypes []string) (map[string]interface{}, error) { + args := msc.Called(artifact, mimeTypes) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(map[string]interface{}), args.Error(1) +} + +func (msc *MockScanAPIController) GetScanLog(uuid string) ([]byte, error) { + args := msc.Called(uuid) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]byte), args.Error(1) +} + +func (msc *MockScanAPIController) HandleJobHooks(trackID string, change *job.StatusChange) error { + args := msc.Called(trackID, change) + + return args.Error(0) +} diff --git a/src/core/api/scanners.go b/src/core/api/scanners.go index ee00888f8..53321f64a 100644 --- a/src/core/api/scanners.go +++ b/src/core/api/scanners.go @@ -62,6 +62,21 @@ func (sa *ScannerAPI) Get() { } } +// Metadata returns the metadata of the given scanner. +func (sa *ScannerAPI) Metadata() { + uuid := sa.GetStringFromPath(":uuid") + + meta, err := sa.c.GetMetadata(uuid) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: get metadata")) + return + } + + // Response to the client + sa.Data["json"] = meta + sa.ServeJSON() +} + // List all the scanners func (sa *ScannerAPI) List() { p, pz, err := sa.GetPaginationParams() @@ -77,7 +92,7 @@ func (sa *ScannerAPI) List() { // Get query key words kws := make(map[string]interface{}) - properties := []string{"name", "description", "url"} + properties := []string{"name", "description", "url", "ex_name", "ex_url"} for _, k := range properties { kw := sa.GetString(k) if len(kw) > 0 { @@ -193,10 +208,6 @@ func (sa *ScannerAPI) Update() { // Delete the scanner func (sa *ScannerAPI) Delete() { uid := sa.GetStringFromPath(":uuid") - if len(uid) == 0 { - sa.SendBadRequestError(errors.New("missing uid")) - return - } deleted, err := sa.c.DeleteRegistration(uid) if err != nil { @@ -217,10 +228,6 @@ func (sa *ScannerAPI) Delete() { // SetAsDefault sets the given registration as default one func (sa *ScannerAPI) SetAsDefault() { uid := sa.GetStringFromPath(":uuid") - if len(uid) == 0 { - sa.SendBadRequestError(errors.New("missing uid")) - return - } m := make(map[string]interface{}) if err := sa.DecodeJSONReq(&m); err != nil { @@ -242,51 +249,22 @@ func (sa *ScannerAPI) SetAsDefault() { sa.SendForbiddenError(errors.Errorf("not supported: %#v", m)) } -// GetProjectScanner gets the project level scanner -func (sa *ScannerAPI) GetProjectScanner() { - pid, err := sa.GetInt64FromPath(":pid") - if err != nil { - sa.SendBadRequestError(errors.Wrap(err, "scanner API: get project scanners")) +// Ping the registration. +func (sa *ScannerAPI) Ping() { + r := &scanner.Registration{} + + if err := sa.DecodeJSONReq(r); err != nil { + sa.SendBadRequestError(errors.Wrap(err, "scanner API: ping")) return } - r, err := sa.c.GetRegistrationByProject(pid) - if err != nil { - sa.SendInternalServerError(errors.Wrap(err, "scanner API: get project scanners")) + if err := r.Validate(false); err != nil { + sa.SendBadRequestError(errors.Wrap(err, "scanner API: ping")) return } - if r != nil { - sa.Data["json"] = r - } else { - sa.Data["json"] = make(map[string]interface{}) - } - - sa.ServeJSON() -} - -// SetProjectScanner sets the project level scanner -func (sa *ScannerAPI) SetProjectScanner() { - pid, err := sa.GetInt64FromPath(":pid") - if err != nil { - sa.SendBadRequestError(errors.Wrap(err, "scanner API: set project scanners")) - return - } - - body := make(map[string]string) - if err := sa.DecodeJSONReq(&body); err != nil { - sa.SendBadRequestError(errors.Wrap(err, "scanner API: set project scanners")) - return - } - - uuid, ok := body["uuid"] - if !ok || len(uuid) == 0 { - sa.SendBadRequestError(errors.New("missing scanner uuid when setting project scanner")) - return - } - - if err := sa.c.SetRegistrationByProject(pid, uuid); err != nil { - sa.SendInternalServerError(errors.Wrap(err, "scanner API: set project scanners")) + if _, err := sa.c.Ping(r); err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: ping")) return } } @@ -294,10 +272,6 @@ func (sa *ScannerAPI) SetProjectScanner() { // get the specified scanner func (sa *ScannerAPI) get() *scanner.Registration { uid := sa.GetStringFromPath(":uuid") - if len(uid) == 0 { - sa.SendBadRequestError(errors.New("missing uid")) - return nil - } r, err := sa.c.GetRegistration(uid) if err != nil { diff --git a/src/core/api/scanners_test.go b/src/core/api/scanners_test.go index 02744788f..598ae7459 100644 --- a/src/core/api/scanners_test.go +++ b/src/core/api/scanners_test.go @@ -19,6 +19,8 @@ import ( "net/http" "testing" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/goharbor/harbor/src/pkg/q" sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" @@ -256,45 +258,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPISetDefault() { }) } -// TestScannerAPIProjectScanner tests the API of getting/setting project level scanner -func (suite *ScannerAPITestSuite) TestScannerAPIProjectScanner() { - suite.mockC.On("SetRegistrationByProject", int64(1), "uuid").Return(nil) - - // Set - body := make(map[string]interface{}, 1) - body["uuid"] = "uuid" - runCodeCheckingCases(suite.T(), &codeCheckingCase{ - request: &testingRequest{ - url: fmt.Sprintf("/api/projects/%d/scanner", 1), - method: http.MethodPut, - credential: sysAdmin, - bodyJSON: body, - }, - code: http.StatusOK, - }) - - r := &scanner.Registration{ - ID: 1004, - UUID: "uuid", - Name: "TestScannerAPIProjectScanner", - Description: "JUST FOR TEST", - URL: "https://a.b.c", - } - suite.mockC.On("GetRegistrationByProject", int64(1)).Return(r, nil) - - // Get - rr := &scanner.Registration{} - err := handleAndParse(&testingRequest{ - url: fmt.Sprintf("/api/projects/%d/scanner", 1), - method: http.MethodGet, - credential: sysAdmin, - }, rr) - require.NoError(suite.T(), err) - - assert.Equal(suite.T(), r.Name, rr.Name) - assert.Equal(suite.T(), r.UUID, rr.UUID) -} - func (suite *ScannerAPITestSuite) mockQuery(r *scanner.Registration) { kw := make(map[string]interface{}, 1) kw["name"] = r.Name @@ -385,3 +348,25 @@ func (m *MockScannerAPIController) GetRegistrationByProject(projectID int64) (*s return s.(*scanner.Registration), args.Error(1) } + +// Ping ... +func (m *MockScannerAPIController) Ping(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) { + args := m.Called(registration) + sam := args.Get(0) + if sam == nil { + return nil, args.Error(1) + } + + return sam.(*v1.ScannerAdapterMetadata), nil +} + +// GetMetadata ... +func (m *MockScannerAPIController) GetMetadata(registrationUUID string) (*v1.ScannerAdapterMetadata, error) { + args := m.Called(registrationUUID) + sam := args.Get(0) + if sam == nil { + return nil, args.Error(1) + } + + return sam.(*v1.ScannerAdapterMetadata), nil +} diff --git a/src/core/router.go b/src/core/router.go index fa33ae6a6..6f38680b9 100755 --- a/src/core/router.go +++ b/src/core/router.go @@ -87,12 +87,9 @@ func initRouters() { beego.Router("/api/repositories/*/tags/:tag/labels", &api.RepositoryLabelAPI{}, "get:GetOfImage;post:AddToImage") beego.Router("/api/repositories/*/tags/:tag/labels/:id([0-9]+)", &api.RepositoryLabelAPI{}, "delete:RemoveFromImage") beego.Router("/api/repositories/*/tags", &api.RepositoryAPI{}, "get:GetTags;post:Retag") - beego.Router("/api/repositories/*/tags/:tag/scan", &api.RepositoryAPI{}, "post:ScanImage") - beego.Router("/api/repositories/*/tags/:tag/vulnerability/details", &api.RepositoryAPI{}, "Get:VulnerabilityDetails") beego.Router("/api/repositories/*/tags/:tag/manifest", &api.RepositoryAPI{}, "get:GetManifests") beego.Router("/api/repositories/*/signatures", &api.RepositoryAPI{}, "get:GetSignatures") beego.Router("/api/repositories/top", &api.RepositoryAPI{}, "get:GetTopRepos") - beego.Router("/api/jobs/scan/:id([0-9]+)/log", &api.ScanJobAPI{}, "get:GetLog") beego.Router("/api/system/gc", &api.GCAPI{}, "get:List") beego.Router("/api/system/gc/:id", &api.GCAPI{}, "get:GetGC") @@ -142,7 +139,6 @@ func initRouters() { // external service that hosted on harbor process: beego.Router("/service/notifications", ®istry.NotificationHandler{}) - beego.Router("/service/notifications/jobs/scan/:id([0-9]+)", &jobs.Handler{}, "post:HandleScan") beego.Router("/service/notifications/jobs/adminjob/:id([0-9]+)", &admin.Handler{}, "post:HandleAdminJob") beego.Router("/service/notifications/jobs/replication/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationScheduleJob") beego.Router("/service/notifications/jobs/replication/task/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationTask") @@ -201,8 +197,20 @@ func initRouters() { scannerAPI := &api.ScannerAPI{} beego.Router("/api/scanners", scannerAPI, "post:Create;get:List") beego.Router("/api/scanners/:uuid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault") + beego.Router("/api/scanners/:uuid/metadata", scannerAPI, "get:Metadata") + beego.Router("/api/scanners/ping", scannerAPI, "post:Ping") + // Add routes for project level scanner - beego.Router("/api/projects/:pid([0-9]+)/scanner", scannerAPI, "get:GetProjectScanner;put:SetProjectScanner") + proScannerAPI := &api.ProjectScannerAPI{} + beego.Router("/api/projects/:pid([0-9]+)/scanner", proScannerAPI, "get:GetProjectScanner;put:SetProjectScanner") + + // Add routes for scan + scanAPI := &api.ScanAPI{} + beego.Router("/api/repositories/*/tags/:tag/scan", scanAPI, "post:Scan;get:Report") + beego.Router("/api/repositories/*/tags/:tag/scan/:uuid/log", scanAPI, "get:Log") + + // Handle scan hook + beego.Router("/service/notifications/jobs/scan/:uuid", &jobs.Handler{}, "post:HandleScan") // Error pages beego.ErrorController(&controllers.ErrorController{}) diff --git a/src/core/service/notifications/jobs/handler.go b/src/core/service/notifications/jobs/handler.go index b383400b4..a1599ddc1 100755 --- a/src/core/service/notifications/jobs/handler.go +++ b/src/core/service/notifications/jobs/handler.go @@ -18,7 +18,8 @@ import ( "encoding/json" "time" - "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/pkg/scan/api/scan" + "github.com/goharbor/harbor/src/common/job" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/log" @@ -49,29 +50,36 @@ type Handler struct { rawStatus string checkIn string revision int64 + trackID string + change *jjob.StatusChange } // Prepare ... func (h *Handler) Prepare() { - id, err := h.GetInt64FromPath(":id") - if err != nil { - log.Errorf("Failed to get job ID, error: %v", err) - // Avoid job service from resending... - h.Abort("200") - return + h.trackID = h.GetStringFromPath(":uuid") + if len(h.trackID) == 0 { + id, err := h.GetInt64FromPath(":id") + if err != nil { + log.Errorf("Failed to get job ID, error: %v", err) + // Avoid job service from resending... + h.Abort("200") + return + } + h.id = id } - h.id = id + var data jjob.StatusChange - err = json.Unmarshal(h.Ctx.Input.CopyBody(1<<32), &data) + err := json.Unmarshal(h.Ctx.Input.CopyBody(1<<32), &data) if err != nil { - log.Errorf("Failed to decode job status change, job ID: %d, error: %v", id, err) + log.Errorf("Failed to decode job status change with error: %v", err) h.Abort("200") return } + h.change = &data h.rawStatus = data.Status status, ok := statusMap[data.Status] if !ok { - log.Debugf("drop the job status update event: job id-%d, status-%s", id, status) + log.Debugf("drop the job status update event: job id-%d/track id-%s, status-%s", h.id, h.trackID, status) h.Abort("200") return } @@ -84,7 +92,8 @@ func (h *Handler) Prepare() { // HandleScan handles the webhook of scan job func (h *Handler) HandleScan() { - log.Debugf("received san job status update event: job-%d, status-%s", h.id, h.status) + log.Debugf("received san job status update event: job-%d, status-%s, track_id-%s", h.id, h.status, h.trackID) + // Trigger image scan webhook event only for JobFinished and JobError status if h.status == models.JobFinished || h.status == models.JobError { e := &event.Event{} @@ -101,7 +110,7 @@ func (h *Handler) HandleScan() { } } - if err := dao.UpdateScanJobStatus(h.id, h.status); err != nil { + if err := scan.DefaultController.HandleJobHooks(h.trackID, h.change); err != nil { log.Errorf("Failed to update job status, id: %d, status: %s", h.id, h.status) h.SendInternalServerError(err) return diff --git a/src/jobservice/runtime/bootstrap.go b/src/jobservice/runtime/bootstrap.go index 9a2bdd41f..78c657605 100644 --- a/src/jobservice/runtime/bootstrap.go +++ b/src/jobservice/runtime/bootstrap.go @@ -23,9 +23,6 @@ import ( "syscall" "time" - "github.com/gomodule/redigo/redis" - "github.com/pkg/errors" - "github.com/goharbor/harbor/src/jobservice/api" "github.com/goharbor/harbor/src/jobservice/common/utils" "github.com/goharbor/harbor/src/jobservice/config" @@ -45,7 +42,10 @@ import ( "github.com/goharbor/harbor/src/jobservice/worker" "github.com/goharbor/harbor/src/jobservice/worker/cworker" "github.com/goharbor/harbor/src/pkg/retention" + sc "github.com/goharbor/harbor/src/pkg/scan" "github.com/goharbor/harbor/src/pkg/scheduler" + "github.com/gomodule/redigo/redis" + "github.com/pkg/errors" ) const ( @@ -242,7 +242,7 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool( // Only for debugging and testing purpose job.SampleJob: (*sample.Job)(nil), // Functional jobs - job.ImageScanJob: (*scan.ClairJob)(nil), + job.ImageScanJob: (*sc.Job)(nil), job.ImageScanAllJob: (*scan.All)(nil), job.ImageGC: (*gc.GarbageCollector)(nil), job.Replication: (*replication.Replication)(nil), diff --git a/src/pkg/scan/api/scan/base_controller.go b/src/pkg/scan/api/scan/base_controller.go index b3d6f8505..514eb9ba5 100644 --- a/src/pkg/scan/api/scan/base_controller.go +++ b/src/pkg/scan/api/scan/base_controller.go @@ -15,44 +15,397 @@ package scan import ( + "fmt" + "time" + + "github.com/goharbor/harbor/src/common" + cj "github.com/goharbor/harbor/src/common/job" + jm "github.com/goharbor/harbor/src/common/job/models" + "github.com/goharbor/harbor/src/common/rbac" + "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/pkg/robot" + "github.com/goharbor/harbor/src/pkg/robot/model" + sca "github.com/goharbor/harbor/src/pkg/scan" + sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner" "github.com/goharbor/harbor/src/pkg/scan/dao/scan" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + "github.com/goharbor/harbor/src/pkg/scan/report" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/google/uuid" + "github.com/pkg/errors" ) +// DefaultController is a default singleton scan API controller. +var DefaultController = NewController() + +const ( + configRegistryEndpoint = "registryEndpoint" + configCoreInternalAddr = "coreInternalAddr" +) + +// uuidGenerator is a func template which is for generating UUID. +type uuidGenerator func() (string, error) + +// configGetter is a func template which is used to wrap the config management +// utility methods. +type configGetter func(cfg string) (string, error) + // basicController is default implementation of api.Controller interface type basicController struct { - // Client for talking to scanner adapter - client v1.Client + // Manage the scan report records + manager report.Manager + // Scanner controller + sc sc.Controller + // Robot account controller + rc robot.Controller + // Job service client + jc cj.Client + // UUID generator + uuid uuidGenerator + // Configuration getter func + config configGetter } // NewController news a scan API controller func NewController() Controller { - return &basicController{} + return &basicController{ + // New report manager + manager: report.NewManager(), + // Refer to the default scanner controller + sc: sc.DefaultController, + // Refer to the default robot account controller + rc: robot.RobotCtr, + // Refer to the default job service client + jc: cj.GlobalClient, + // Generate UUID with uuid lib + uuid: func() (string, error) { + aUUID, err := uuid.NewUUID() + if err != nil { + return "", err + } + + return aUUID.String(), nil + }, + // Get the required configuration options + config: func(cfg string) (string, error) { + switch cfg { + case configRegistryEndpoint: + return config.ExtEndpoint() + case configCoreInternalAddr: + return config.InternalCoreURL(), nil + default: + return "", errors.Errorf("configuration option %s not defined", cfg) + } + }, + } } // Scan ... func (bc *basicController) Scan(artifact *v1.Artifact) error { + if artifact == nil { + return errors.New("nil artifact to scan") + } + + r, err := bc.sc.GetRegistrationByProject(artifact.NamespaceID) + if err != nil { + return errors.Wrap(err, "scan controller: scan") + } + + // Check the health of the registration by ping. + // The metadata of the scanner adapter is also returned. + meta, err := bc.sc.Ping(r) + if err != nil { + return errors.Wrap(err, "scan controller: scan") + } + + // Generate a UUID as track ID which groups the report records generated + // by the specified registration for the digest with given mime type. + trackID, err := bc.uuid() + if err != nil { + return errors.Wrap(err, "scan controller: scan") + } + + producesMimes := make([]string, 0) + matched := false + for _, ca := range meta.Capabilities { + for _, cm := range ca.ConsumesMimeTypes { + if cm == artifact.MimeType { + matched = true + break + } + } + + if matched { + for _, pm := range ca.ProducesMimeTypes { + // Create report placeholder first + reportPlaceholder := &scan.Report{ + Digest: artifact.Digest, + RegistrationUUID: r.UUID, + Status: job.PendingStatus.String(), + StatusCode: job.PendingStatus.Code(), + TrackID: trackID, + MimeType: pm, + } + _, e := bc.manager.Create(reportPlaceholder) + if e != nil { + // Recorded by error wrap and logged at the same time. + if err == nil { + err = e + } else { + err = errors.Wrap(e, err.Error()) + } + + logger.Error(errors.Wrap(e, "scan controller: scan")) + continue + } + + producesMimes = append(producesMimes, pm) + } + + break + } + } + + // Scanner does not support scanning the given artifact. + if !matched { + return errors.Errorf("the configured scanner %s does not support scanning artifact with mime type %s", r.Name, artifact.MimeType) + } + + // If all the record are created failed. + if len(producesMimes) == 0 { + // Return the last error + return errors.Wrap(err, "scan controller: scan") + } + + jobID, err := bc.launchScanJob(trackID, artifact, r, producesMimes) + if err != nil { + // Update the status to the concrete error + // Change status code to normal error code + if e := bc.manager.UpdateStatus(trackID, err.Error(), 0); e != nil { + err = errors.Wrap(e, err.Error()) + } + + return errors.Wrap(err, "scan controller: scan") + } + + // Insert the generated job ID now + // It will not block the whole process. If any errors happened, just logged. + if err := bc.manager.UpdateScanJobID(trackID, jobID); err != nil { + logger.Error(errors.Wrap(err, "scan controller: scan")) + } + return nil } // GetReport ... -func (bc *basicController) GetReport(artifact *v1.Artifact) ([]*scan.Report, error) { - return nil, nil +func (bc *basicController) GetReport(artifact *v1.Artifact, mimeTypes []string) ([]*scan.Report, error) { + if artifact == nil { + return nil, errors.New("no way to get report for nil artifact") + } + + mimes := make([]string, 0) + mimes = append(mimes, mimeTypes...) + if len(mimes) == 0 { + // Retrieve native as default + mimes = append(mimes, v1.MimeTypeNativeReport) + } + + // Get current scanner settings + r, err := bc.sc.GetRegistrationByProject(artifact.NamespaceID) + if err != nil { + return nil, errors.Wrap(err, "scan controller: get report") + } + + if r == nil { + return nil, errors.New("no scanner registration configured") + } + + return bc.manager.GetBy(artifact.Digest, r.UUID, mimes) +} + +// GetSummary ... +func (bc *basicController) GetSummary(artifact *v1.Artifact, mimeTypes []string) (map[string]interface{}, error) { + if artifact == nil { + return nil, errors.New("no way to get report summaries for nil artifact") + } + + // Get reports first + rps, err := bc.GetReport(artifact, mimeTypes) + if err != nil { + return nil, err + } + + summaries := make(map[string]interface{}, len(rps)) + for _, rp := range rps { + sum, err := report.GenerateSummary(rp) + if err != nil { + return nil, err + } + + summaries[rp.MimeType] = sum + } + + return summaries, nil } // GetScanLog ... -func (bc *basicController) GetScanLog(digest string) ([]byte, error) { - return nil, nil -} +func (bc *basicController) GetScanLog(uuid string) ([]byte, error) { + if len(uuid) == 0 { + return nil, errors.New("empty uuid to get scan log") + } -// Ping ... -func (bc *basicController) Ping(registration *scanner.Registration) error { - return nil + // Get by uuid + sr, err := bc.manager.Get(uuid) + if err != nil { + return nil, errors.Wrap(err, "scan controller: get scan log") + } + + if sr == nil { + // Not found + return nil, nil + } + + // Not job error + if sr.StatusCode == job.ErrorStatus.Code() { + jst := job.Status(sr.Status) + if jst.Code() == -1 { + return []byte(sr.Status), nil + } + } + + // Job log + return bc.jc.GetJobLog(sr.JobID) } // HandleJobHooks ... -func (bc *basicController) HandleJobHooks(trackID int64, change *job.StatusChange) error { - return nil +func (bc *basicController) HandleJobHooks(trackID string, change *job.StatusChange) error { + if len(trackID) == 0 { + return errors.New("empty track ID") + } + + if change == nil { + return errors.New("nil change object") + } + + // Check in data + if len(change.CheckIn) > 0 { + checkInReport := &sca.CheckInReport{} + if err := checkInReport.FromJSON(change.CheckIn); err != nil { + return errors.Wrap(err, "scan controller: handle job hook") + } + + rpl, err := bc.manager.GetBy( + checkInReport.Digest, + checkInReport.RegistrationUUID, + []string{checkInReport.MimeType}) + if err != nil { + return errors.Wrap(err, "scan controller: handle job hook") + } + + if len(rpl) == 0 { + return errors.New("no report found to update data") + } + + if err := bc.manager.UpdateReportData( + rpl[0].UUID, + checkInReport.RawReport, + change.Metadata.Revision); err != nil { + return errors.Wrap(err, "scan controller: handle job hook") + } + + return nil + } + + return bc.manager.UpdateStatus(trackID, change.Status, change.Metadata.Revision) +} + +// makeRobotAccount creates a robot account based on the arguments for scanning. +func (bc *basicController) makeRobotAccount(pid int64, repository string, ttl int64) (string, error) { + // Use uuid as name to avoid duplicated entries. + UUID, err := bc.uuid() + if err != nil { + return "", errors.Wrap(err, "scan controller: make robot account") + } + + expireAt := time.Now().UTC().Add(time.Duration(ttl) * time.Second).Unix() + + logger.Warningf("repository %s and expire time %d are not supported by robot controller", repository, expireAt) + + resource := fmt.Sprintf("/project/%d/repository", pid) + access := []*rbac.Policy{{ + Resource: rbac.Resource(resource), + Action: "pull", + }} + + account := &model.RobotCreate{ + Name: fmt.Sprintf("%s%s", common.RobotPrefix, UUID), + Description: "for scan", + ProjectID: pid, + Access: access, + } + + rb, err := bc.rc.CreateRobotAccount(account) + if err != nil { + return "", errors.Wrap(err, "scan controller: make robot account") + } + + return rb.Token, nil +} + +// launchScanJob launches a job to run scan +func (bc *basicController) launchScanJob(trackID string, artifact *v1.Artifact, registration *scanner.Registration, mimes []string) (jobID string, err error) { + externalURL, err := bc.config(configRegistryEndpoint) + if err != nil { + return "", errors.Wrap(err, "scan controller: launch scan job") + } + + // Make a robot account with 30 minutes + robotAccount, err := bc.makeRobotAccount(artifact.NamespaceID, artifact.Repository, 1800) + if err != nil { + return "", errors.Wrap(err, "scan controller: launch scan job") + } + + // Set job parameters + scanReq := &v1.ScanRequest{ + Registry: &v1.Registry{ + URL: externalURL, + Authorization: robotAccount, + }, + Artifact: artifact, + } + + rJSON, err := registration.ToJSON() + if err != nil { + return "", errors.Wrap(err, "scan controller: launch scan job") + } + + sJSON, err := scanReq.ToJSON() + if err != nil { + return "", errors.Wrap(err, "launch scan job") + } + + params := make(map[string]interface{}) + params[sca.JobParamRegistration] = rJSON + params[sca.JobParameterRequest] = sJSON + params[sca.JobParameterMimes] = mimes + + // Launch job + callbackURL, err := bc.config(configCoreInternalAddr) + if err != nil { + return "", errors.Wrap(err, "launch scan job") + } + hookURL := fmt.Sprintf("%s/service/notifications/jobs/scan/%s", callbackURL, trackID) + + j := &jm.JobData{ + Name: job.ImageScanJob, + Metadata: &jm.JobMetadata{ + JobKind: job.KindGeneric, + }, + Parameters: params, + StatusHook: hookURL, + } + + return bc.jc.SubmitJob(j) } diff --git a/src/pkg/scan/api/scan/base_controller_test.go b/src/pkg/scan/api/scan/base_controller_test.go new file mode 100644 index 000000000..932793f10 --- /dev/null +++ b/src/pkg/scan/api/scan/base_controller_test.go @@ -0,0 +1,537 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import ( + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/rbac" + + "github.com/goharbor/harbor/src/pkg/robot/model" + + cjm "github.com/goharbor/harbor/src/common/job/models" + jm "github.com/goharbor/harbor/src/common/job/models" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/pkg/q" + sca "github.com/goharbor/harbor/src/pkg/scan" + "github.com/goharbor/harbor/src/pkg/scan/dao/scan" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/goharbor/harbor/src/pkg/scan/vuln" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// ControllerTestSuite is the test suite for scan controller. +type ControllerTestSuite struct { + suite.Suite + + registration *scanner.Registration + artifact *v1.Artifact + rawReport string + c Controller +} + +// TestController is the entry point of ControllerTestSuite. +func TestController(t *testing.T) { + suite.Run(t, new(ControllerTestSuite)) +} + +// SetupSuite ... +func (suite *ControllerTestSuite) SetupSuite() { + suite.registration = &scanner.Registration{ + ID: 1, + UUID: "uuid001", + Name: "Test-scan-controller", + URL: "http://testing.com:3128", + IsDefault: true, + } + + suite.artifact = &v1.Artifact{ + NamespaceID: 1, + Repository: "scan", + Tag: "golang", + Digest: "digest-code", + MimeType: v1.MimeTypeDockerArtifact, + } + + m := &v1.ScannerAdapterMetadata{ + Scanner: &v1.Scanner{ + Name: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + }, + Capabilities: []*v1.ScannerCapability{{ + ConsumesMimeTypes: []string{ + v1.MimeTypeOCIArtifact, + v1.MimeTypeDockerArtifact, + }, + ProducesMimeTypes: []string{ + v1.MimeTypeNativeReport, + }, + }}, + Properties: v1.ScannerProperties{ + "extra": "testing", + }, + } + + sc := &MockScannerController{} + sc.On("GetRegistrationByProject", suite.artifact.NamespaceID).Return(suite.registration, nil) + sc.On("Ping", suite.registration).Return(m, nil) + + mgr := &MockReportManager{} + mgr.On("Create", &scan.Report{ + Digest: "digest-code", + RegistrationUUID: "uuid001", + MimeType: "application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0", + Status: "Pending", + StatusCode: 0, + TrackID: "the-uuid-123", + }).Return("r-uuid", nil) + mgr.On("UpdateScanJobID", "the-uuid-123", "the-job-id").Return(nil) + + rp := vuln.Report{ + GeneratedAt: time.Now().UTC().String(), + Scanner: &v1.Scanner{ + Name: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + }, + Severity: vuln.High, + Vulnerabilities: []*vuln.VulnerabilityItem{ + { + ID: "2019-0980-0909", + Package: "dpkg", + Version: "0.9.1", + FixVersion: "0.9.2", + Severity: vuln.High, + Description: "mock one", + Links: []string{"https://vuln.com"}, + }, + }, + } + + jsonData, err := json.Marshal(rp) + require.NoError(suite.T(), err) + suite.rawReport = string(jsonData) + + reports := []*scan.Report{ + { + ID: 11, + UUID: "rp-uuid-001", + Digest: "digest-code", + RegistrationUUID: "uuid001", + MimeType: "application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0", + Status: "Success", + StatusCode: 3, + TrackID: "the-uuid-123", + JobID: "the-job-id", + StatusRevision: time.Now().Unix(), + Report: suite.rawReport, + StartTime: time.Now(), + EndTime: time.Now().Add(2 * time.Second), + }, + } + + mgr.On("GetBy", suite.artifact.Digest, suite.registration.UUID, []string{v1.MimeTypeNativeReport}).Return(reports, nil) + mgr.On("Get", "rp-uuid-001").Return(reports[0], nil) + mgr.On("UpdateReportData", "rp-uuid-001", suite.rawReport, (int64)(10000)).Return(nil) + mgr.On("UpdateStatus", "the-uuid-123", "Success", (int64)(10000)).Return(nil) + + rc := &MockRobotController{} + + resource := fmt.Sprintf("/project/%d/repository", suite.artifact.NamespaceID) + access := []*rbac.Policy{{ + Resource: rbac.Resource(resource), + Action: "pull", + }} + + rname := fmt.Sprintf("%s%s", common.RobotPrefix, "the-uuid-123") + account := &model.RobotCreate{ + Name: rname, + Description: "for scan", + ProjectID: suite.artifact.NamespaceID, + Access: access, + } + rc.On("CreateRobotAccount", account).Return(&model.Robot{ + ID: 1, + Name: rname, + Token: "robot-account", + Description: "for scan", + ProjectID: suite.artifact.NamespaceID, + }, nil) + + // Set job parameters + req := &v1.ScanRequest{ + Registry: &v1.Registry{ + URL: "https://core.com", + Authorization: "robot-account", + }, + Artifact: suite.artifact, + } + + rJSON, err := req.ToJSON() + require.NoError(suite.T(), err) + + regJSON, err := suite.registration.ToJSON() + require.NoError(suite.T(), err) + + jc := &MockJobServiceClient{} + params := make(map[string]interface{}) + params[sca.JobParamRegistration] = regJSON + params[sca.JobParameterRequest] = rJSON + params[sca.JobParameterMimes] = []string{v1.MimeTypeNativeReport} + + j := &jm.JobData{ + Name: job.ImageScanJob, + Metadata: &jm.JobMetadata{ + JobKind: job.KindGeneric, + }, + Parameters: params, + StatusHook: fmt.Sprintf("%s/service/notifications/jobs/scan/%s", "http://core:8080", "the-uuid-123"), + } + jc.On("SubmitJob", j).Return("the-job-id", nil) + jc.On("GetJobLog", "the-job-id").Return([]byte("job log"), nil) + + suite.c = &basicController{ + manager: mgr, + sc: sc, + jc: jc, + rc: rc, + uuid: func() (string, error) { + return "the-uuid-123", nil + }, + config: func(cfg string) (string, error) { + switch cfg { + case configRegistryEndpoint: + return "https://core.com", nil + case configCoreInternalAddr: + return "http://core:8080", nil + } + + return "", nil + }, + } +} + +// TearDownSuite ... +func (suite *ControllerTestSuite) TearDownSuite() {} + +// TestScanControllerScan ... +func (suite *ControllerTestSuite) TestScanControllerScan() { + err := suite.c.Scan(suite.artifact) + require.NoError(suite.T(), err) +} + +// TestScanControllerGetReport ... +func (suite *ControllerTestSuite) TestScanControllerGetReport() { + rep, err := suite.c.GetReport(suite.artifact, []string{v1.MimeTypeNativeReport}) + require.NoError(suite.T(), err) + assert.Equal(suite.T(), 1, len(rep)) +} + +// TestScanControllerGetSummary ... +func (suite *ControllerTestSuite) TestScanControllerGetSummary() { + sum, err := suite.c.GetSummary(suite.artifact, []string{v1.MimeTypeNativeReport}) + require.NoError(suite.T(), err) + assert.Equal(suite.T(), 1, len(sum)) +} + +// TestScanControllerGetScanLog ... +func (suite *ControllerTestSuite) TestScanControllerGetScanLog() { + bytes, err := suite.c.GetScanLog("rp-uuid-001") + require.NoError(suite.T(), err) + assert.Condition(suite.T(), func() (success bool) { + success = len(bytes) > 0 + return + }) +} + +// TestScanControllerHandleJobHooks ... +func (suite *ControllerTestSuite) TestScanControllerHandleJobHooks() { + cReport := &sca.CheckInReport{ + Digest: "digest-code", + RegistrationUUID: suite.registration.UUID, + MimeType: v1.MimeTypeNativeReport, + RawReport: suite.rawReport, + } + + cRpJSON, err := cReport.ToJSON() + require.NoError(suite.T(), err) + + statusChange := &job.StatusChange{ + JobID: "the-job-id", + Status: "Success", + CheckIn: string(cRpJSON), + Metadata: &job.StatsInfo{ + Revision: (int64)(10000), + }, + } + + err = suite.c.HandleJobHooks("the-uuid-123", statusChange) + require.NoError(suite.T(), err) +} + +// Mock things + +// MockReportManager ... +type MockReportManager struct { + mock.Mock +} + +// Create ... +func (mrm *MockReportManager) Create(r *scan.Report) (string, error) { + args := mrm.Called(r) + + return args.String(0), args.Error(1) +} + +// UpdateScanJobID ... +func (mrm *MockReportManager) UpdateScanJobID(trackID string, jobID string) error { + args := mrm.Called(trackID, jobID) + + return args.Error(0) +} + +func (mrm *MockReportManager) UpdateStatus(trackID string, status string, rev int64) error { + args := mrm.Called(trackID, status, rev) + + return args.Error(0) +} + +func (mrm *MockReportManager) UpdateReportData(uuid string, report string, rev int64) error { + args := mrm.Called(uuid, report, rev) + + return args.Error(0) +} + +func (mrm *MockReportManager) GetBy(digest string, registrationUUID string, mimeTypes []string) ([]*scan.Report, error) { + args := mrm.Called(digest, registrationUUID, mimeTypes) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]*scan.Report), args.Error(1) +} + +func (mrm *MockReportManager) Get(uuid string) (*scan.Report, error) { + args := mrm.Called(uuid) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*scan.Report), args.Error(1) +} + +// MockScannerController ... +type MockScannerController struct { + mock.Mock +} + +// ListRegistrations ... +func (msc *MockScannerController) ListRegistrations(query *q.Query) ([]*scanner.Registration, error) { + args := msc.Called(query) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]*scanner.Registration), args.Error(1) +} + +// CreateRegistration ... +func (msc *MockScannerController) CreateRegistration(registration *scanner.Registration) (string, error) { + args := msc.Called(registration) + + return args.String(0), args.Error(1) +} + +// GetRegistration ... +func (msc *MockScannerController) GetRegistration(registrationUUID string) (*scanner.Registration, error) { + args := msc.Called(registrationUUID) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*scanner.Registration), args.Error(1) +} + +// RegistrationExists ... +func (msc *MockScannerController) RegistrationExists(registrationUUID string) bool { + args := msc.Called(registrationUUID) + + return args.Bool(0) +} + +// UpdateRegistration ... +func (msc *MockScannerController) UpdateRegistration(registration *scanner.Registration) error { + args := msc.Called(registration) + + return args.Error(0) +} + +// DeleteRegistration ... +func (msc *MockScannerController) DeleteRegistration(registrationUUID string) (*scanner.Registration, error) { + args := msc.Called(registrationUUID) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*scanner.Registration), args.Error(1) +} + +// SetDefaultRegistration ... +func (msc *MockScannerController) SetDefaultRegistration(registrationUUID string) error { + args := msc.Called(registrationUUID) + + return args.Error(0) +} + +// SetRegistrationByProject ... +func (msc *MockScannerController) SetRegistrationByProject(projectID int64, scannerID string) error { + args := msc.Called(projectID, scannerID) + + return args.Error(0) +} + +// GetRegistrationByProject ... +func (msc *MockScannerController) GetRegistrationByProject(projectID int64) (*scanner.Registration, error) { + args := msc.Called(projectID) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*scanner.Registration), args.Error(1) +} + +// Ping ... +func (msc *MockScannerController) Ping(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) { + args := msc.Called(registration) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*v1.ScannerAdapterMetadata), args.Error(1) +} + +// GetMetadata ... +func (msc *MockScannerController) GetMetadata(registrationUUID string) (*v1.ScannerAdapterMetadata, error) { + args := msc.Called(registrationUUID) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*v1.ScannerAdapterMetadata), args.Error(1) +} + +// MockJobServiceClient ... +type MockJobServiceClient struct { + mock.Mock +} + +// SubmitJob ... +func (mjc *MockJobServiceClient) SubmitJob(jData *cjm.JobData) (string, error) { + args := mjc.Called(jData) + + return args.String(0), args.Error(1) +} + +// GetJobLog ... +func (mjc *MockJobServiceClient) GetJobLog(uuid string) ([]byte, error) { + args := mjc.Called(uuid) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]byte), args.Error(1) +} + +// PostAction ... +func (mjc *MockJobServiceClient) PostAction(uuid, action string) error { + args := mjc.Called(uuid, action) + + return args.Error(0) +} + +func (mjc *MockJobServiceClient) GetExecutions(uuid string) ([]job.Stats, error) { + args := mjc.Called(uuid) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]job.Stats), args.Error(1) +} + +// MockRobotController ... +type MockRobotController struct { + mock.Mock +} + +// GetRobotAccount ... +func (mrc *MockRobotController) GetRobotAccount(id int64) (*model.Robot, error) { + args := mrc.Called(id) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*model.Robot), args.Error(1) +} + +// CreateRobotAccount ... +func (mrc *MockRobotController) CreateRobotAccount(robotReq *model.RobotCreate) (*model.Robot, error) { + args := mrc.Called(robotReq) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*model.Robot), args.Error(1) +} + +// DeleteRobotAccount ... +func (mrc *MockRobotController) DeleteRobotAccount(id int64) error { + args := mrc.Called(id) + + return args.Error(0) +} + +// UpdateRobotAccount ... +func (mrc *MockRobotController) UpdateRobotAccount(r *model.Robot) error { + args := mrc.Called(r) + + return args.Error(0) +} + +// ListRobotAccount ... +func (mrc *MockRobotController) ListRobotAccount(pid int64) ([]*model.Robot, error) { + args := mrc.Called(pid) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]*model.Robot), args.Error(1) +} diff --git a/src/pkg/scan/api/scan/controller.go b/src/pkg/scan/api/scan/controller.go index 65322148b..69752c403 100644 --- a/src/pkg/scan/api/scan/controller.go +++ b/src/pkg/scan/api/scan/controller.go @@ -17,7 +17,6 @@ package scan import ( "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/pkg/scan/dao/scan" - "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" ) @@ -25,17 +24,6 @@ import ( // TODO: Here the artifact object is reused the v1 one which is sent to the adapter, // it should be pointed to the general artifact object in future once it's ready. type Controller interface { - // Ping pings Scanner Adapter to test EndpointURL and Authorization settings. - // The implementation is supposed to call the GetMetadata method on scanner.Client. - // Returns `nil` if connection succeeded, a non `nil` error otherwise. - // - // Arguments: - // registration *scanner.Registration : scanner registration to ping - // - // Returns: - // error : non nil error if any errors occurred - Ping(registration *scanner.Registration) error - // Scan the given artifact // // Arguments: @@ -49,30 +37,42 @@ type Controller interface { // // Arguments: // artifact *v1.Artifact : the scanned artifact + // mimeTypes []string : the mime types of the reports // // Returns: // []*scan.Report : scan results by different scanner vendors // error : non nil error if any errors occurred - GetReport(artifact *v1.Artifact) ([]*scan.Report, error) + GetReport(artifact *v1.Artifact, mimeTypes []string) ([]*scan.Report, error) + + // GetSummary gets the summaries of the reports with given types. + // + // Arguments: + // artifact *v1.Artifact : the scanned artifact + // mimeTypes []string : the mime types of the reports + // + // Returns: + // map[string]interface{} : report summaries indexed by mime types + // error : non nil error if any errors occurred + GetSummary(artifact *v1.Artifact, mimeTypes []string) (map[string]interface{}, error) // Get the scan log for the specified artifact with the given digest // // Arguments: - // digest string : the digest of the artifact + // uuid string : the UUID of the scan report // // Returns: // []byte : the log text stream // error : non nil error if any errors occurred - GetScanLog(digest string) ([]byte, error) + GetScanLog(uuid string) ([]byte, error) // HandleJobHooks handle the hook events from the job service // e.g : status change of the scan job or scan result // // Arguments: - // trackID int64 : ID for the result record + // trackID string : UUID for the report record // change *job.StatusChange : change event from the job service // // Returns: // error : non nil error if any errors occurred - HandleJobHooks(trackID int64, change *job.StatusChange) error + HandleJobHooks(trackID string, change *job.StatusChange) error } diff --git a/src/pkg/scan/api/scanner/base_controller.go b/src/pkg/scan/api/scanner/base_controller.go index 72512ceff..2eb4688c4 100644 --- a/src/pkg/scan/api/scanner/base_controller.go +++ b/src/pkg/scan/api/scanner/base_controller.go @@ -19,6 +19,7 @@ import ( "github.com/goharbor/harbor/src/jobservice/logger" "github.com/goharbor/harbor/src/pkg/q" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" rscanner "github.com/goharbor/harbor/src/pkg/scan/scanner" "github.com/pkg/errors" ) @@ -35,25 +36,38 @@ func New() Controller { return &basicController{ manager: rscanner.New(), proMetaMgr: metamgr.NewDefaultProjectMetadataManager(), + clientPool: v1.DefaultClientPool, } } // basicController is default implementation of api.Controller interface type basicController struct { - // managers for managing the scanner registrations + // Managers for managing the scanner registrations manager rscanner.Manager - // for operating the project level configured scanner + // For operating the project level configured scanner proMetaMgr metamgr.ProjectMetadataManager + // Client pool for talking to adapters + clientPool v1.ClientPool } // ListRegistrations ... func (bc *basicController) ListRegistrations(query *q.Query) ([]*scanner.Registration, error) { - return bc.manager.List(query) + l, err := bc.manager.List(query) + if err != nil { + return nil, errors.Wrap(err, "api controller: list registrations") + } + + for _, r := range l { + _, err = bc.Ping(r) + r.Health = err == nil + } + + return l, nil } // CreateRegistration ... func (bc *basicController) CreateRegistration(registration *scanner.Registration) (string, error) { - // TODO: Get metadata from the adapter service first + // TODO: Check connection of the registration. // Check if there are any registrations already existing. l, err := bc.manager.List(&q.Query{ PageSize: 1, @@ -73,7 +87,15 @@ func (bc *basicController) CreateRegistration(registration *scanner.Registration // GetRegistration ... func (bc *basicController) GetRegistration(registrationUUID string) (*scanner.Registration, error) { - return bc.manager.Get(registrationUUID) + r, err := bc.manager.Get(registrationUUID) + if err != nil { + return nil, err + } + + _, err = bc.Ping(r) + r.Health = err == nil + + return r, nil } // RegistrationExists ... @@ -90,6 +112,10 @@ func (bc *basicController) RegistrationExists(registrationUUID string) bool { // UpdateRegistration ... func (bc *basicController) UpdateRegistration(registration *scanner.Registration) error { + if registration.IsDefault && registration.Disabled { + return errors.Errorf("default registration %s can not be marked to disabled", registration.UUID) + } + return bc.manager.Update(registration) } @@ -162,9 +188,10 @@ func (bc *basicController) GetRegistrationByProject(projectID int64) (*scanner.R return nil, errors.Wrap(err, "api controller: get project scanner") } + var registration *scanner.Registration if len(m) > 0 { if registrationID, ok := m[proScannerMetaKey]; ok && len(registrationID) > 0 { - registration, err := bc.manager.Get(registrationID) + registration, err = bc.manager.Get(registrationID) if err != nil { return nil, errors.Wrap(err, "api controller: get project scanner") } @@ -175,15 +202,103 @@ func (bc *basicController) GetRegistrationByProject(projectID int64) (*scanner.R if err := bc.proMetaMgr.Delete(projectID, proScannerMetaKey); err != nil { return nil, errors.Wrap(err, "api controller: get project scanner") } - } else { - return registration, nil } } } - // Second, get the default one - registration, err := bc.manager.GetDefault() + if registration == nil { + // Second, get the default one + registration, err = bc.manager.GetDefault() + if err != nil { + return nil, errors.Wrap(err, "api controller: get project scanner") + } + } + + // Check status by the client later + if registration != nil { + if meta, err := bc.Ping(registration); err == nil { + registration.Scanner = meta.Scanner.Name + registration.Vendor = meta.Scanner.Vendor + registration.Version = meta.Scanner.Version + registration.Health = true + } else { + registration.Health = false + } + } - // TODO: Check status by the client later return registration, err } + +// Ping ... +// TODO: ADD UT CASES +func (bc *basicController) Ping(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) { + if registration == nil { + return nil, errors.New("nil registration to ping") + } + + client, err := bc.clientPool.Get(registration) + if err != nil { + return nil, errors.Wrap(err, "scanner controller: ping") + } + + meta, err := client.GetMetadata() + if err != nil { + return nil, errors.Wrap(err, "scanner controller: ping") + } + + // Validate the required properties + if meta.Scanner == nil || + len(meta.Scanner.Name) == 0 || + len(meta.Scanner.Version) == 0 || + len(meta.Scanner.Vendor) == 0 { + return nil, errors.New("invalid scanner in metadata") + } + + if len(meta.Capabilities) == 0 { + return nil, errors.New("invalid capabilities in metadata") + } + + for _, ca := range meta.Capabilities { + // v1.MimeTypeDockerArtifact is required now + found := false + for _, cm := range ca.ConsumesMimeTypes { + if cm == v1.MimeTypeDockerArtifact { + found = true + break + } + } + if !found { + return nil, errors.Errorf("missing %s in consumes_mime_types", v1.MimeTypeDockerArtifact) + } + + // v1.MimeTypeNativeReport is required + found = false + for _, pm := range ca.ProducesMimeTypes { + if pm == v1.MimeTypeNativeReport { + found = true + break + } + } + + if !found { + return nil, errors.Errorf("missing %s in produces_mime_types", v1.MimeTypeNativeReport) + } + } + + return meta, err +} + +// GetMetadata ... +// TODO: ADD UT CASES +func (bc *basicController) GetMetadata(registrationUUID string) (*v1.ScannerAdapterMetadata, error) { + if len(registrationUUID) == 0 { + return nil, errors.New("empty registration uuid") + } + + r, err := bc.manager.Get(registrationUUID) + if err != nil { + return nil, errors.Wrap(err, "scanner controller: get metadata") + } + + return bc.Ping(r) +} diff --git a/src/pkg/scan/api/scanner/controller_test.go b/src/pkg/scan/api/scanner/base_controller_test.go similarity index 82% rename from src/pkg/scan/api/scanner/controller_test.go rename to src/pkg/scan/api/scanner/base_controller_test.go index e032b645d..eef26baf7 100644 --- a/src/pkg/scan/api/scanner/controller_test.go +++ b/src/pkg/scan/api/scanner/base_controller_test.go @@ -17,6 +17,8 @@ package scanner import ( "testing" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/pkg/q" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" @@ -47,9 +49,25 @@ func (suite *ControllerTestSuite) SetupSuite() { suite.mMgr = new(MockScannerManager) suite.mMeta = new(MockProMetaManager) - suite.c = &basicController{ - manager: suite.mMgr, - proMetaMgr: suite.mMeta, + m := &v1.ScannerAdapterMetadata{ + Scanner: &v1.Scanner{ + Name: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + }, + Capabilities: []*v1.ScannerCapability{{ + ConsumesMimeTypes: []string{ + v1.MimeTypeOCIArtifact, + v1.MimeTypeDockerArtifact, + }, + ProducesMimeTypes: []string{ + v1.MimeTypeNativeReport, + v1.MimeTypeRawReport, + }, + }}, + Properties: v1.ScannerProperties{ + "extra": "testing", + }, } suite.sample = &scanner.Registration{ @@ -57,6 +75,17 @@ func (suite *ControllerTestSuite) SetupSuite() { Description: "sample registration", URL: "https://sample.scanner.com", } + + mc := &MockClient{} + mc.On("GetMetadata").Return(m, nil) + + mcp := &MockClientPool{} + mcp.On("Get", suite.sample).Return(mc, nil) + suite.c = &basicController{ + manager: suite.mMgr, + proMetaMgr: suite.mMeta, + clientPool: mcp, + } } // Clear test case @@ -282,3 +311,50 @@ func (m *MockProMetaManager) List(name, value string) ([]*models.ProjectMetadata args := m.Called(name, value) return args.Get(0).([]*models.ProjectMetadata), args.Error(1) } + +// MockClientPool is defined and referred by other UT cases. +type MockClientPool struct { + mock.Mock +} + +// Get client +func (mcp *MockClientPool) Get(r *scanner.Registration) (v1.Client, error) { + args := mcp.Called(r) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(v1.Client), args.Error(1) +} + +// MockClient is defined and referred in other UT cases. +type MockClient struct { + mock.Mock +} + +// GetMetadata ... +func (mc *MockClient) GetMetadata() (*v1.ScannerAdapterMetadata, error) { + args := mc.Called() + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*v1.ScannerAdapterMetadata), args.Error(1) +} + +// SubmitScan ... +func (mc *MockClient) SubmitScan(req *v1.ScanRequest) (*v1.ScanResponse, error) { + args := mc.Called(req) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*v1.ScanResponse), args.Error(1) +} + +// GetScanReport ... +func (mc *MockClient) GetScanReport(scanRequestID, reportMIMEType string) (string, error) { + args := mc.Called(scanRequestID, reportMIMEType) + + return args.String(0), args.Error(1) +} diff --git a/src/pkg/scan/api/scanner/controller.go b/src/pkg/scan/api/scanner/controller.go index 048d94e96..d87928ca0 100644 --- a/src/pkg/scan/api/scanner/controller.go +++ b/src/pkg/scan/api/scanner/controller.go @@ -17,6 +17,7 @@ package scanner import ( "github.com/goharbor/harbor/src/pkg/q" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" ) // Controller provides the related operations of scanner for the upper API. @@ -112,4 +113,26 @@ type Controller interface { // *scanner.Registration : the default scanner registration // error : non nil error if any errors occurred GetRegistrationByProject(projectID int64) (*scanner.Registration, error) + + // Ping pings Scanner Adapter to test EndpointURL and Authorization settings. + // The implementation is supposed to call the GetMetadata method on scanner.Client. + // Returns `nil` if connection succeeded, a non `nil` error otherwise. + // + // Arguments: + // registration *scanner.Registration : scanner registration to ping + // + // Returns: + // *v1.ScannerAdapterMetadata : metadata returned by the scanner if successfully ping + // error : non nil error if any errors occurred + Ping(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) + + // GetMetadata returns the metadata of the given scanner. + // + // Arguments: + // registrationUUID string : the UUID of the given scanner which is marked as default + // + // Returns: + // *v1.ScannerAdapterMetadata : metadata returned by the scanner if successfully ping + // error : non nil error if any errors occurred + GetMetadata(registrationUUID string) (*v1.ScannerAdapterMetadata, error) } diff --git a/src/pkg/scan/dao/scan/model.go b/src/pkg/scan/dao/scan/model.go index 5789e632c..9d7f3ff4a 100644 --- a/src/pkg/scan/dao/scan/model.go +++ b/src/pkg/scan/dao/scan/model.go @@ -25,6 +25,7 @@ type Report struct { RegistrationUUID string `orm:"column(registration_uuid)"` MimeType string `orm:"column(mime_type)"` JobID string `orm:"column(job_id)"` + TrackID string `orm:"column(track_id)"` Status string `orm:"column(status)"` StatusCode int `orm:"column(status_code)"` StatusRevision int64 `orm:"column(status_rev)"` diff --git a/src/pkg/scan/dao/scan/report.go b/src/pkg/scan/dao/scan/report.go index 653f864df..6b428f8a2 100644 --- a/src/pkg/scan/dao/scan/report.go +++ b/src/pkg/scan/dao/scan/report.go @@ -16,6 +16,7 @@ package scan import ( "fmt" + "time" "github.com/astaxie/beego/orm" "github.com/goharbor/harbor/src/common/dao" @@ -103,7 +104,7 @@ func UpdateReportData(uuid string, report string, statusRev int64) error { } // UpdateReportStatus updates the report `status` with conditions matched. -func UpdateReportStatus(uuid string, status string, statusCode int, statusRev int64) error { +func UpdateReportStatus(trackID string, status string, statusCode int, statusRev int64) error { o := dao.GetOrmer() qt := o.QueryTable(new(Report)) @@ -112,7 +113,13 @@ func UpdateReportStatus(uuid string, status string, statusCode int, statusRev in data["status_code"] = statusCode data["status_rev"] = statusRev - count, err := qt.Filter("uuid", uuid). + // Technically it is not correct, just to avoid changing interface and adding more code. + // running==2 + if statusCode > 2 { + data["end_time"] = time.Now().UTC() + } + + count, err := qt.Filter("track_id", trackID). Filter("status_rev__lte", statusRev). Filter("status_code__lte", statusCode).Update(data) @@ -121,20 +128,20 @@ func UpdateReportStatus(uuid string, status string, statusCode int, statusRev in } if count == 0 { - return errors.Errorf("no report with uuid %s updated", uuid) + return errors.Errorf("no report with track_id %s updated", trackID) } return nil } // UpdateJobID updates the report `job_id` column -func UpdateJobID(uuid string, jobID string) error { +func UpdateJobID(trackID string, jobID string) error { o := dao.GetOrmer() qt := o.QueryTable(new(Report)) params := make(orm.Params, 1) params["job_id"] = jobID - _, err := qt.Filter("uuid", uuid).Update(params) + _, err := qt.Filter("track_id", trackID).Update(params) return err } diff --git a/src/pkg/scan/dao/scan/report_test.go b/src/pkg/scan/dao/scan/report_test.go index 63d318128..f2193b9bc 100644 --- a/src/pkg/scan/dao/scan/report_test.go +++ b/src/pkg/scan/dao/scan/report_test.go @@ -45,6 +45,7 @@ func (suite *ReportTestSuite) SetupSuite() { func (suite *ReportTestSuite) SetupTest() { r := &Report{ UUID: "uuid", + TrackID: "track-uuid", Digest: "digest1001", RegistrationUUID: "ruuid", MimeType: v1.MimeTypeNativeReport, @@ -95,7 +96,7 @@ func (suite *ReportTestSuite) TestReportList() { // TestReportUpdateJobID tests update job ID of the report. func (suite *ReportTestSuite) TestReportUpdateJobID() { - err := UpdateJobID("uuid", "jobid001") + err := UpdateJobID("track-uuid", "jobid001") require.NoError(suite.T(), err) l, err := ListReports(nil) @@ -120,12 +121,12 @@ func (suite *ReportTestSuite) TestReportUpdateReportData() { // TestReportUpdateStatus tests update the report status. func (suite *ReportTestSuite) TestReportUpdateStatus() { - err := UpdateReportStatus("uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 1000) + err := UpdateReportStatus("track-uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 1000) require.NoError(suite.T(), err) - err = UpdateReportStatus("uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 900) + err = UpdateReportStatus("track-uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 900) require.Error(suite.T(), err) - err = UpdateReportStatus("uuid", job.PendingStatus.String(), job.PendingStatus.Code(), 1000) + err = UpdateReportStatus("track-uuid", job.PendingStatus.String(), job.PendingStatus.Code(), 1000) require.Error(suite.T(), err) } diff --git a/src/pkg/scan/dao/scanner/model.go b/src/pkg/scan/dao/scanner/model.go index bf711a7f3..8186aa007 100644 --- a/src/pkg/scan/dao/scanner/model.go +++ b/src/pkg/scan/dao/scanner/model.go @@ -20,6 +20,8 @@ import ( "strings" "time" + "github.com/goharbor/harbor/src/pkg/scan/rest/auth" + "github.com/pkg/errors" ) @@ -45,6 +47,11 @@ type Registration struct { // Http connection settings SkipCertVerify bool `orm:"column(skip_cert_verify);default(false)" json:"skip_certVerify"` + // Extra info about the scanner + Scanner string `orm:"-" json:"scanner,omitempty"` + Vendor string `orm:"-" json:"vendor,omitempty"` + Version string `orm:"-" json:"version,omitempty"` + // Timestamps CreateTime time.Time `orm:"column(create_time);auto_now_add;type(datetime)" json:"create_time"` UpdateTime time.Time `orm:"column(update_time);auto_now;type(datetime)" json:"update_time"` @@ -89,6 +96,17 @@ func (r *Registration) Validate(checkUUID bool) error { return errors.Wrap(err, "scanner registration validate") } + if len(r.Auth) > 0 && + r.Auth != auth.Basic && + r.Auth != auth.Bearer && + r.Auth != auth.APIKey { + return errors.Errorf("auth type %s is not supported", r.Auth) + } + + if len(r.Auth) > 0 && len(r.AccessCredential) == 0 { + return errors.Errorf("access_credential is required for auth type %s", r.Auth) + } + return nil } diff --git a/src/pkg/scan/dao/scanner/registration.go b/src/pkg/scan/dao/scanner/registration.go index da2912dea..db489fe17 100644 --- a/src/pkg/scan/dao/scanner/registration.go +++ b/src/pkg/scan/dao/scanner/registration.go @@ -16,6 +16,7 @@ package scanner import ( "fmt" + "strings" "github.com/astaxie/beego/orm" "github.com/goharbor/harbor/src/common/dao" @@ -93,6 +94,12 @@ func ListRegistrations(query *q.Query) ([]*Registration, error) { if query != nil { if len(query.Keywords) > 0 { for k, v := range query.Keywords { + if strings.HasPrefix(k, "ex_") { + kk := strings.TrimPrefix(k, "ex_") + qt = qt.Filter(kk, v) + continue + } + qt = qt.Filter(fmt.Sprintf("%s__icontains", k), v) } } @@ -111,20 +118,38 @@ func ListRegistrations(query *q.Query) ([]*Registration, error) { // SetDefaultRegistration sets the specified registration as default one func SetDefaultRegistration(UUID string) error { o := dao.GetOrmer() - qt := o.QueryTable(new(Registration)) - - _, err := qt.Filter("is_default", true).Update(orm.Params{ - "is_default": false, - }) - + err := o.Begin() if err != nil { return err } - qt2 := o.QueryTable(new(Registration)) - _, err = qt2.Filter("uuid", UUID).Update(orm.Params{ - "is_default": true, - }) + var count int64 + qt := o.QueryTable(new(Registration)) + count, err = qt.Filter("uuid", UUID). + Filter("disabled", false). + Update(orm.Params{ + "is_default": true, + }) + if err == nil && count == 0 { + err = errors.Errorf("set default for %s failed", UUID) + } + + if err == nil { + qt2 := o.QueryTable(new(Registration)) + _, err = qt2.Exclude("uuid__exact", UUID). + Filter("is_default", true). + Update(orm.Params{ + "is_default": false, + }) + } + + if err != nil { + if e := o.Rollback(); e != nil { + err = errors.Wrap(e, err.Error()) + } + } else { + err = o.Commit() + } return err } diff --git a/src/pkg/scan/dao/scanner/registration_test.go b/src/pkg/scan/dao/scanner/registration_test.go index d7a228a5f..b44c4a435 100644 --- a/src/pkg/scan/dao/scanner/registration_test.go +++ b/src/pkg/scan/dao/scanner/registration_test.go @@ -124,6 +124,22 @@ func (suite *RegistrationDAOTestSuite) TestList() { }) require.NoError(suite.T(), err) require.Equal(suite.T(), 0, len(l)) + + // Exact match + exactKeywords := make(map[string]interface{}) + exactKeywords["ex_name"] = "forUT" + l, err = ListRegistrations(&q.Query{ + Keywords: exactKeywords, + }) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) + + exactKeywords["ex_name"] = "forU" + l, err = ListRegistrations(&q.Query{ + Keywords: exactKeywords, + }) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 0, len(l)) } // TestDefault tests set/get default @@ -138,4 +154,11 @@ func (suite *RegistrationDAOTestSuite) TestDefault() { dr, err = GetDefaultRegistration() require.NoError(suite.T(), err) require.NotNil(suite.T(), dr) + + dr.Disabled = true + err = UpdateRegistration(dr, "disabled") + require.NoError(suite.T(), err) + + err = SetDefaultRegistration(suite.registrationID) + require.Error(suite.T(), err) } diff --git a/src/pkg/scan/job.go b/src/pkg/scan/job.go index aa9919977..f8d10c104 100644 --- a/src/pkg/scan/job.go +++ b/src/pkg/scan/job.go @@ -116,17 +116,18 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error { // Print related infos to log printJSONParameter(JobParamRegistration, params[JobParamRegistration].(string), myLogger) - printJSONParameter(JobParameterRequest, params[JobParameterRequest].(string), myLogger) + printJSONParameter(JobParameterRequest, removeAuthInfo(req), myLogger) + myLogger.Infof("Report mime types: %v\n", mimes) // Submit scan request to the scanner adapter client, err := v1.DefaultClientPool.Get(r) if err != nil { - return errors.Wrap(err, "run scan job") + return logAndWrapError(myLogger, err, "scan job: get client") } resp, err := client.SubmitScan(req) if err != nil { - return errors.Wrap(err, "run scan job") + return logAndWrapError(myLogger, err, "scan job: submit scan request") } // For collecting errors @@ -229,6 +230,13 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error { return err } +func logAndWrapError(logger logger.Interface, err error, message string) error { + e := errors.Wrap(err, message) + logger.Error(e) + + return e +} + func printJSONParameter(parameter string, v string, logger logger.Interface) { logger.Infof("%s:\n", parameter) printPrettyJSON([]byte(v), logger) @@ -244,6 +252,23 @@ func printPrettyJSON(in []byte, logger logger.Interface) { logger.Infof("%s\n", out.String()) } +func removeAuthInfo(sr *v1.ScanRequest) string { + req := &v1.ScanRequest{ + Artifact: sr.Artifact, + Registry: &v1.Registry{ + URL: sr.Registry.URL, + Authorization: "[HIDDEN]", + }, + } + + str, err := req.ToJSON() + if err != nil { + logger.Error(errors.Wrap(err, "scan job: remove auth")) + } + + return str +} + func extractScanReq(params job.Parameters) (*v1.ScanRequest, error) { v, ok := params[JobParameterRequest] if !ok { @@ -263,7 +288,6 @@ func extractScanReq(params job.Parameters) (*v1.ScanRequest, error) { if err := req.FromJSON(jsonData); err != nil { return nil, err } - if err := req.Validate(); err != nil { return nil, err } @@ -304,14 +328,24 @@ func extractMimeTypes(params job.Parameters) ([]string, error) { return nil, errors.Errorf("missing job parameter '%s'", JobParameterMimes) } - l, ok := v.([]string) + l, ok := v.([]interface{}) if !ok { return nil, errors.Errorf( - "malformed job parameter '%s', expecting string but got %s", + "malformed job parameter '%s', expecting []interface{} but got %s", JobParameterMimes, reflect.TypeOf(v).String(), ) } - return l, nil + mimes := make([]string, 0) + for _, v := range l { + mime, ok := v.(string) + if !ok { + return nil, errors.Errorf("expect string but got %s", reflect.TypeOf(v).String()) + } + + mimes = append(mimes, mime) + } + + return mimes, nil } diff --git a/src/pkg/scan/report/base_manager.go b/src/pkg/scan/report/base_manager.go index 163e2707f..b4645eed7 100644 --- a/src/pkg/scan/report/base_manager.go +++ b/src/pkg/scan/report/base_manager.go @@ -57,15 +57,21 @@ func (bm *basicManager) Create(r *scan.Report) (string, error) { }) if err != nil { - return "", errors.Wrap(err, "check existence of report") + return "", errors.Wrap(err, "create report: check existence of report") } // Delete existing copy if len(existingCopies) > 0 { theCopy := existingCopies[0] - // Status conflict theStatus := job.Status(theCopy.Status) + // Status is an error message + if theStatus.Code() == -1 && theCopy.StatusCode == job.ErrorStatus.Code() { + // Mark as regular error status + theStatus = job.ErrorStatus + } + + // Status conflict if theStatus.Compare(job.RunningStatus) <= 0 { return "", errors.Errorf("conflict: a previous scanning is %s", theCopy.Status) } @@ -73,7 +79,7 @@ func (bm *basicManager) Create(r *scan.Report) (string, error) { // Otherwise it will be a completed report // Clear it before insert this new one if err := scan.DeleteReport(theCopy.UUID); err != nil { - return "", errors.Wrap(err, "clear old scan report") + return "", errors.Wrap(err, "create report: clear old scan report") } } @@ -91,12 +97,39 @@ func (bm *basicManager) Create(r *scan.Report) (string, error) { // Insert if _, err = scan.CreateReport(r); err != nil { - return "", errors.Wrap(err, "create report") + return "", errors.Wrap(err, "create report: insert") } return r.UUID, nil } +// Get ... +func (bm *basicManager) Get(uuid string) (*scan.Report, error) { + if len(uuid) == 0 { + return nil, errors.New("empty uuid to get scan report") + } + + kws := make(map[string]interface{}) + kws["uuid"] = uuid + + l, err := scan.ListReports(&q.Query{ + PageNumber: 1, + PageSize: 1, + Keywords: kws, + }) + + if err != nil { + return nil, errors.Wrap(err, "report manager: get") + } + + if len(l) == 0 { + // Not found + return nil, nil + } + + return l[0], nil +} + // GetBy ... func (bm *basicManager) GetBy(digest string, registrationUUID string, mimeTypes []string) ([]*scan.Report, error) { if len(digest) == 0 { @@ -121,24 +154,20 @@ func (bm *basicManager) GetBy(digest string, registrationUUID string, mimeTypes } // UpdateScanJobID ... -func (bm *basicManager) UpdateScanJobID(uuid string, jobID string) error { - if len(uuid) == 0 || len(jobID) == 0 { +func (bm *basicManager) UpdateScanJobID(trackID string, jobID string) error { + if len(trackID) == 0 || len(jobID) == 0 { return errors.New("bad arguments") } - return scan.UpdateJobID(uuid, jobID) + return scan.UpdateJobID(trackID, jobID) } // UpdateStatus ... -func (bm *basicManager) UpdateStatus(uuid string, status string, rev int64) error { - if len(uuid) == 0 { +func (bm *basicManager) UpdateStatus(trackID string, status string, rev int64) error { + if len(trackID) == 0 { return errors.New("missing uuid") } - if rev <= 0 { - return errors.New("invalid data revision") - } - stCode := job.ErrorStatus.Code() st := job.Status(status) // Check if it is job valid status. @@ -148,7 +177,7 @@ func (bm *basicManager) UpdateStatus(uuid string, status string, rev int64) erro stCode = st.Code() } - return scan.UpdateReportStatus(uuid, status, stCode, rev) + return scan.UpdateReportStatus(trackID, status, stCode, rev) } // UpdateReportData ... @@ -157,10 +186,6 @@ func (bm *basicManager) UpdateReportData(uuid string, report string, rev int64) return errors.New("missing uuid") } - if rev <= 0 { - return errors.New("invalid data revision") - } - if len(report) == 0 { return errors.New("missing report JSON data") } diff --git a/src/pkg/scan/report/base_manager_test.go b/src/pkg/scan/report/base_manager_test.go index e4b881e0a..17d5fef1d 100644 --- a/src/pkg/scan/report/base_manager_test.go +++ b/src/pkg/scan/report/base_manager_test.go @@ -52,6 +52,7 @@ func (suite *TestManagerSuite) SetupTest() { Digest: "d1000", RegistrationUUID: "ruuid", MimeType: v1.MimeTypeNativeReport, + TrackID: "tid001", } uuid, err := suite.m.Create(rp) @@ -70,13 +71,14 @@ func (suite *TestManagerSuite) TearDownTest() { // TestManagerCreateWithExisting tests the case that a copy already is there when creating report. func (suite *TestManagerSuite) TestManagerCreateWithExisting() { - err := suite.m.UpdateStatus(suite.rpUUID, job.SuccessStatus.String(), 2000) + err := suite.m.UpdateStatus("tid001", job.SuccessStatus.String(), 2000) require.NoError(suite.T(), err) rp := &scan.Report{ Digest: "d1000", RegistrationUUID: "ruuid", MimeType: v1.MimeTypeNativeReport, + TrackID: "tid002", } uuid, err := suite.m.Create(rp) @@ -87,6 +89,16 @@ func (suite *TestManagerSuite) TestManagerCreateWithExisting() { suite.rpUUID = uuid } +// TestManagerGet tests the get method. +func (suite *TestManagerSuite) TestManagerGet() { + sr, err := suite.m.Get(suite.rpUUID) + + require.NoError(suite.T(), err) + require.NotNil(suite.T(), sr) + + assert.Equal(suite.T(), "d1000", sr.Digest) +} + // TestManagerGetBy tests the get by method. func (suite *TestManagerSuite) TestManagerGetBy() { l, err := suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport}) @@ -113,7 +125,7 @@ func (suite *TestManagerSuite) TestManagerUpdateJobID() { oldJID := l[0].JobID - err = suite.m.UpdateScanJobID(suite.rpUUID, "jID1001") + err = suite.m.UpdateScanJobID("tid001", "jID1001") require.NoError(suite.T(), err) l, err = suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport}) @@ -132,7 +144,7 @@ func (suite *TestManagerSuite) TestManagerUpdateStatus() { oldSt := l[0].Status - err = suite.m.UpdateStatus(suite.rpUUID, job.SuccessStatus.String(), 10000) + err = suite.m.UpdateStatus("tid001", job.SuccessStatus.String(), 10000) require.NoError(suite.T(), err) l, err = suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport}) diff --git a/src/pkg/scan/report/manager.go b/src/pkg/scan/report/manager.go index 4c4ca13a1..f4059abcc 100644 --- a/src/pkg/scan/report/manager.go +++ b/src/pkg/scan/report/manager.go @@ -32,32 +32,32 @@ type Manager interface { // Update the scan job ID of the given report. // // Arguments: - // uuid string : uuid to identify the report - // jobID string: scan job ID + // trackID string : uuid to identify the report + // jobID string : scan job ID // // Returns: // error : non nil error if any errors occurred // - UpdateScanJobID(uuid string, jobID string) error + UpdateScanJobID(trackID string, jobID string) error // Update the status (mapping to the scan job status) of the given report. // // Arguments: - // uuid string : uuid to identify the report - // status string: status info - // rev int64 : data revision info + // trackID string : uuid to identify the report + // status string : status info + // rev int64 : data revision info // // Returns: // error : non nil error if any errors occurred // - UpdateStatus(uuid string, status string, rev int64) error + UpdateStatus(trackID string, status string, rev int64) error // Update the report data (with JSON format) of the given report. // // Arguments: - // uuid string : uuid to identify the report - // report string: report JSON data - // rev int64 : data revision info + // uuid string : uuid to identify the report + // report string : report JSON data + // rev int64 : data revision info // // Returns: // error : non nil error if any errors occurred @@ -77,4 +77,14 @@ type Manager interface { // []*scan.Report : report list // error : non nil error if any errors occurred GetBy(digest string, registrationUUID string, mimeTypes []string) ([]*scan.Report, error) + + // Get the report for the given uuid. + // + // Arguments: + // uuid string : uuid of the scan report + // + // Returns: + // *scan.Report : scan report + // error : non nil error if any errors occurred + Get(uuid string) (*scan.Report, error) } diff --git a/src/pkg/scan/report/summary.go b/src/pkg/scan/report/summary.go new file mode 100644 index 000000000..5c5e37d57 --- /dev/null +++ b/src/pkg/scan/report/summary.go @@ -0,0 +1,97 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "reflect" + + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/pkg/scan/dao/scan" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/goharbor/harbor/src/pkg/scan/vuln" + "github.com/pkg/errors" +) + +// SupportedGenerators declares mappings between mime type and summary generator func. +var SupportedGenerators = map[string]SummaryGenerator{ + v1.MimeTypeNativeReport: GenerateNativeSummary, +} + +// GenerateSummary is a helper function to generate report +// summary based on the given report. +func GenerateSummary(r *scan.Report) (interface{}, error) { + g, ok := SupportedGenerators[r.MimeType] + if !ok { + return nil, errors.Errorf("no generator bound with mime type %s", r.MimeType) + } + + return g(r) +} + +// SummaryGenerator is a func template which used to generated report +// summary for relevant mime type. +type SummaryGenerator func(r *scan.Report) (interface{}, error) + +// GenerateNativeSummary generates the report summary for the native report. +func GenerateNativeSummary(r *scan.Report) (interface{}, error) { + sum := &vuln.NativeReportSummary{} + sum.ReportID = r.UUID + sum.StartTime = r.StartTime + sum.EndTime = r.EndTime + sum.Duration = r.EndTime.Unix() - r.EndTime.Unix() + + sum.ScanStatus = job.ErrorStatus.String() + if job.Status(r.Status).Code() != -1 { + sum.ScanStatus = r.Status + } + + // If the status is not success/stopped, there will not be any report. + if r.Status != job.SuccessStatus.String() && + r.Status != job.StoppedStatus.String() { + return sum, nil + } + + // Probably no report data if the job is interrupted + if len(r.Report) == 0 { + return nil, errors.Errorf("no report data for %s, status is: %s", r.UUID, sum.ScanStatus) + } + + raw, err := ResolveData(r.MimeType, []byte(r.Report)) + if err != nil { + return nil, err + } + + rp, ok := raw.(*vuln.Report) + if !ok { + return nil, errors.Errorf("type mismatch: expect *vuln.Report but got %s", reflect.TypeOf(raw).String()) + } + + sum.Severity = rp.Severity + vsum := &vuln.VulnerabilitySummary{ + Total: len(rp.Vulnerabilities), + Summary: make(vuln.SeveritySummary), + } + + for _, v := range rp.Vulnerabilities { + if num, ok := vsum.Summary[v.Severity]; ok { + vsum.Summary[v.Severity] = num + 1 + } else { + vsum.Summary[v.Severity] = 1 + } + } + sum.Summary = vsum + + return sum, nil +} diff --git a/src/pkg/scan/report/supported_mime_test.go b/src/pkg/scan/report/supported_mime_test.go index f118e25ff..c4c167e0a 100644 --- a/src/pkg/scan/report/supported_mime_test.go +++ b/src/pkg/scan/report/supported_mime_test.go @@ -69,6 +69,7 @@ func (suite *SupportedMimesSuite) SetupSuite() { func (suite *SupportedMimesSuite) TestResolveData() { obj, err := ResolveData(v1.MimeTypeNativeReport, suite.mockData) require.NoError(suite.T(), err) + require.NotNil(suite.T(), obj) require.Condition(suite.T(), func() (success bool) { rp, ok := obj.(*vuln.Report) success = ok && rp != nil && rp.Severity == vuln.High diff --git a/src/pkg/scan/report/supported_mimes.go b/src/pkg/scan/report/supported_mimes.go index 80bf60aca..fc09aed65 100644 --- a/src/pkg/scan/report/supported_mimes.go +++ b/src/pkg/scan/report/supported_mimes.go @@ -31,13 +31,15 @@ var SupportedMimes = map[string]interface{}{ // ResolveData is a helper func to parse the JSON data with the given mime type. func ResolveData(mime string, jsonData []byte) (interface{}, error) { - if len(jsonData) == 0 { - return nil, errors.New("empty JSON data") - } - + // If no resolver defined for the given mime types, directly ignore it. + // The raw data will be used. t, ok := SupportedMimes[mime] if !ok { - return nil, errors.Errorf("report with mime type %s is not supported", mime) + return nil, nil + } + + if len(jsonData) == 0 { + return nil, errors.New("empty JSON data") } ty := reflect.TypeOf(t) diff --git a/src/pkg/scan/rest/v1/client.go b/src/pkg/scan/rest/v1/client.go index 9dfe9711d..a24eba3fb 100644 --- a/src/pkg/scan/rest/v1/client.go +++ b/src/pkg/scan/rest/v1/client.go @@ -18,6 +18,7 @@ import ( "bytes" "crypto/tls" "encoding/json" + "fmt" "io/ioutil" "net" "net/http" @@ -157,7 +158,7 @@ func (c *basicClient) SubmitScan(req *ScanRequest) (*ScanResponse, error) { return nil, errors.Wrap(err, "v1 client: submit scan") } - respData, err := c.send(request, generalResponseHandler(http.StatusCreated)) + respData, err := c.send(request, generalResponseHandler(http.StatusAccepted)) if err != nil { return nil, errors.Wrap(err, "v1 client: submit scan") } @@ -199,7 +200,7 @@ func (c *basicClient) GetScanReport(scanRequestID, reportMIMEType string) (strin func (c *basicClient) send(req *http.Request, h responseHandler) ([]byte, error) { if c.authorizer != nil { if err := c.authorizer.Authorize(req); err != nil { - return nil, errors.Wrap(err, "authorization") + return nil, errors.Wrap(err, "send: authorization") } } @@ -266,12 +267,24 @@ func generalRespHandlerFunc(expectedCode, code int, resp *http.Response) ([]byte eResp := &ErrorResponse{ Err: &Error{}, } - if err := json.Unmarshal(buf, eResp); err == nil { - return nil, eResp + + err := json.Unmarshal(buf, eResp) + if err != nil { + return nil, errors.Wrap(err, "general response handler") } + + // Append more contexts + eResp.Err.Message = fmt.Sprintf( + "%s: general response handler: unexpected status code: %d, expected: %d", + eResp.Err.Message, + code, + expectedCode, + ) + + return nil, eResp } - return nil, errors.Errorf("unexpected status code: %d, response: %s", code, string(buf)) + return nil, errors.Errorf("general response handler: unexpected status code: %d, expected: %d", code, expectedCode) } return buf, nil diff --git a/src/pkg/scan/rest/v1/client_pool.go b/src/pkg/scan/rest/v1/client_pool.go index bf1dc3aa2..6e4588101 100644 --- a/src/pkg/scan/rest/v1/client_pool.go +++ b/src/pkg/scan/rest/v1/client_pool.go @@ -102,7 +102,7 @@ func (bcp *basicClientPool) Get(r *scanner.Registration) (Client, error) { return nil, errors.New("nil scanner registration") } - if err := r.Validate(true); err != nil { + if err := r.Validate(false); err != nil { return nil, errors.Wrap(err, "client pool: get") } @@ -159,8 +159,7 @@ func (bcp *basicClientPool) deadCheck(key string, item *poolItem) { } func key(r *scanner.Registration) string { - return fmt.Sprintf("%s:%s:%s:%s:%v", - r.UUID, + return fmt.Sprintf("%s:%s:%s:%v", r.URL, r.Auth, r.AccessCredential, diff --git a/src/pkg/scan/rest/v1/client_test.go b/src/pkg/scan/rest/v1/client_test.go index 84767555b..969189356 100644 --- a/src/pkg/scan/rest/v1/client_test.go +++ b/src/pkg/scan/rest/v1/client_test.go @@ -115,7 +115,7 @@ type mockHandler struct{} // ServeHTTP ... func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch r.RequestURI { - case "/metadata": + case "/api/v1/metadata": if r.Method != http.MethodGet { w.WriteHeader(http.StatusForbidden) return @@ -126,7 +126,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { Vendor: "Harbor", Version: "0.1.0", }, - Capabilities: &ScannerCapability{ + Capabilities: []*ScannerCapability{{ ConsumesMimeTypes: []string{ MimeTypeOCIArtifact, MimeTypeDockerArtifact, @@ -135,7 +135,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { MimeTypeNativeReport, MimeTypeRawReport, }, - }, + }}, Properties: ScannerProperties{ "extra": "testing", }, @@ -144,7 +144,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) _, _ = w.Write(data) break - case "/scan": + case "/api/v1/scan": if r.Method != http.MethodPost { w.WriteHeader(http.StatusForbidden) return @@ -155,10 +155,10 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { data, _ := json.Marshal(res) - w.WriteHeader(http.StatusCreated) + w.WriteHeader(http.StatusAccepted) _, _ = w.Write(data) break - case "/scan/id1/report": + case "/api/v1/scan/id1/report": if r.Method != http.MethodGet { w.WriteHeader(http.StatusForbidden) return @@ -175,7 +175,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) _, _ = w.Write(data) break - case "/scan/id2/report": + case "/api/v1/scan/id2/report": if r.Method != http.MethodGet { w.WriteHeader(http.StatusForbidden) return @@ -183,7 +183,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("{}")) break - case "/scan/id3/report": + case "/api/v1/scan/id3/report": if r.Method != http.MethodGet { w.WriteHeader(http.StatusForbidden) return diff --git a/src/pkg/scan/rest/v1/models.go b/src/pkg/scan/rest/v1/models.go index 47ded4eb1..cd817900d 100644 --- a/src/pkg/scan/rest/v1/models.go +++ b/src/pkg/scan/rest/v1/models.go @@ -60,16 +60,21 @@ type ScannerProperties map[string]string // a scanner capable of scanning a given Artifact stored in its registry and making sure that it // can interpret a returned result. type ScannerAdapterMetadata struct { - Scanner *Scanner `json:"scanner"` - Capabilities *ScannerCapability `json:"capabilities"` - Properties ScannerProperties `json:"properties"` + Scanner *Scanner `json:"scanner"` + Capabilities []*ScannerCapability `json:"capabilities"` + Properties ScannerProperties `json:"properties"` } // Artifact represents an artifact stored in Registry. type Artifact struct { + // ID of the namespace (project). It will not be sent to scanner adapter. + NamespaceID int64 `json:"-"` // The full name of a Harbor repository containing the artifact, including the namespace. // For example, `library/oracle/nosql`. Repository string `json:"repository"` + // The info used to identify the version of the artifact, + // e.g: tag of image or version of the chart. + Tag string `json:"tag"` // The artifact's digest, consisting of an algorithm and hex portion. // For example, `sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b`, // represents sha256 based digest. diff --git a/src/pkg/scan/rest/v1/spec.go b/src/pkg/scan/rest/v1/spec.go index 6d4f6bf0e..cf7ff8647 100644 --- a/src/pkg/scan/rest/v1/spec.go +++ b/src/pkg/scan/rest/v1/spec.go @@ -39,6 +39,8 @@ const ( MimeTypeScanRequest = "application/vnd.scanner.adapter.scan.request+json; version=1.0" // MimeTypeScanResponse defines the mime type for scan response MimeTypeScanResponse = "application/vnd.scanner.adapter.scan.response+json; version=1.0" + + apiPrefix = "/api/v1" ) // RequestResolver is a function template to modify the API request, e.g: add headers @@ -70,6 +72,8 @@ func NewSpec(base string) *Spec { } } + s.baseRoute = fmt.Sprintf("%s%s", s.baseRoute, apiPrefix) + return s } diff --git a/src/pkg/scan/vuln/report.go b/src/pkg/scan/vuln/report.go index 7510f7cb6..57bbaf7d6 100644 --- a/src/pkg/scan/vuln/report.go +++ b/src/pkg/scan/vuln/report.go @@ -54,5 +54,5 @@ type VulnerabilityItem struct { // The list of link to the upstream database with the full description of the vulnerability. // Format: URI // e.g: List [ "https://security-tracker.debian.org/tracker/CVE-2017-8283" ] - Links []string + Links []string `json:"links"` } diff --git a/src/pkg/scan/vuln/summary.go b/src/pkg/scan/vuln/summary.go new file mode 100644 index 000000000..27c596f73 --- /dev/null +++ b/src/pkg/scan/vuln/summary.go @@ -0,0 +1,41 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vuln + +import ( + "time" +) + +// NativeReportSummary is the default supported scan report summary model. +// Generated based on the report with v1.MimeTypeNativeReport mime type. +type NativeReportSummary struct { + ReportID string `json:"report_id"` + ScanStatus string `json:"scan_status"` + Severity Severity `json:"severity"` + Duration int64 `json:"duration"` + Summary *VulnerabilitySummary `json:"summary"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` +} + +// VulnerabilitySummary contains the total number of the found vulnerabilities number +// and numbers of each severity level. +type VulnerabilitySummary struct { + Total int `json:"total"` + Summary SeveritySummary `json:"summary"` +} + +// SeveritySummary ... +type SeveritySummary map[Severity]int diff --git a/tests/robot-cases/Group0-BAT/API_DB.robot b/tests/robot-cases/Group0-BAT/API_DB.robot index c2ee1d071..f955e281e 100644 --- a/tests/robot-cases/Group0-BAT/API_DB.robot +++ b/tests/robot-cases/Group0-BAT/API_DB.robot @@ -29,16 +29,18 @@ Test Case - Add Replication Rule Harbor API Test ./tests/apitests/python/test_add_replication_rule.py Test Case - Edit Project Creation Harbor API Test ./tests/apitests/python/test_edit_project_creation.py -Test Case - Scan Image - Harbor API Test ./tests/apitests/python/test_scan_image.py +*** Enable this case after deployment change PR merged *** +*** Test Case - Scan Image *** +*** Harbor API Test ./tests/apitests/python/test_scan_image.py *** Test Case - Manage Project Member Harbor API Test ./tests/apitests/python/test_manage_project_member.py Test Case - Project Level Policy Content Trust Harbor API Test ./tests/apitests/python/test_project_level_policy_content_trust.py Test Case - User View Logs Harbor API Test ./tests/apitests/python/test_user_view_logs.py -Test Case - Scan All Images - Harbor API Test ./tests/apitests/python/test_scan_all_images.py +*** Enable this case after deployment change PR merged *** +*** Test Case - Scan All Images *** +*** Harbor API Test ./tests/apitests/python/test_scan_all_images.py *** Test Case - List Helm Charts Harbor API Test ./tests/apitests/python/test_list_helm_charts.py Test Case - Assign Sys Admin