upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor synced 2025-04-14 20:11:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add vulnerability search API (#18924)

Browse Source 
use q.Query to pass all query conditions

Signed-off-by: stonezdj <daojunz@vmware.com>
This commit is contained in:
stonezdj(Daojun Zhang) 2023-07-19 10:17:14 +08:00 committed by GitHub
parent 82ee5295ea
commit d4aa9b13c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 737 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
api/v2.0/swagger.yaml
View File

@ -6063,13 +6063,13 @@ paths:
- $ref: '#/parameters/requestId'
- name: with_dangerous_cve
in: query
description: Specify whether the dangerous CVE is include in the security summary
description: Specify whether the dangerous CVEs are included inside summary information
type: boolean
required: false
default: false
- name: with_dangerous_artifact
in: query
description: Specify whether the dangerous artifacts is include in the security summary
description: Specify whether the dangerous Artifact are included inside summary information
type: boolean
required: false
default: false
@ -6086,6 +6086,61 @@ paths:
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
/security/vul:
get:
summary: Get the vulnerability list.
description: |
Get the vulnerability list. use q to pass the query condition,
supported conditions:
cve_id(exact match)
cvss_score_v3(range condition)
severity(exact match)
repository_name(exact match)
project_id(exact match)
package(exact match)
and tag(exact match)
tags:
- securityhub
operationId: ListVulnerabilities
parameters:
- $ref: '#/parameters/requestId'
- $ref: '#/parameters/query'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- name: tune_count
in: query
description: Enable to ignore X-Total-Count when the total count > 1000, if the total count is less than 1000, the real total count is returned, else -1.
type: boolean
required: false
default: false
- name: with_tag
in: query
description: Specify whether the tag information is included inside vulnerability information
type: boolean
required: false
default: false
responses:
'200':
description: The vulnerability list.
schema:
type: array
items:
$ref: '#/definitions/VulnerabilityItem'
headers:
X-Total-Count:
description: The total count of vulnerabilities
type: integer
Link:
description: Link refers to the previous page and next page
type: string
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'500':
$ref: '#/responses/500'
parameters:
query:
name: q
@ -9760,3 +9815,50 @@ definitions:
type: integer
x-omitempty: false
description: the count of medium vulnerabilities
VulnerabilityItem:
type: object
description: the vulnerability item info
properties:
project_id:
type: integer
format: int64
description: the project ID of the artifact
repository_name:
type: string
description: the repository name of the artifact
digest:
type: string
description: the digest of the artifact
tags:
type: array
items:
type: string
description: the tags of the artifact
cve_id:
type: string
description: the CVE id of the vulnerability.
severity:
type: string
description: the severity of the vulnerability
cvss_v3_score:
type: number
format: float
description: the nvd cvss v3 score of the vulnerability
package:
type: string
description: the package of the vulnerability
version:
type: string
description: the version of the package
fixed_version:
type: string
description: the fixed version of the package
desc:
type: string
description: The description of the vulnerability
links:
type: array
items:
type: string
description: Links of the vulnerability

2
make/migrations/postgresql/0120_2.9.0_schema.up.sql
View File

@ -11,6 +11,8 @@ UPDATE vulnerability_record
SET cvss_score_v3 = (vendor_attributes->'CVSS'->'nvd'->>'V3Score')::double precision
WHERE jsonb_path_exists(vendor_attributes::jsonb, '$.CVSS.nvd.V3Score');
CREATE INDEX IF NOT EXISTS idx_vulnerability_record_cvss_score_v3 ON vulnerability_record (cvss_score_v3);
/* add summary information in scan_report */
ALTER TABLE scan_report ADD COLUMN IF NOT EXISTS critical_cnt BIGINT;
ALTER TABLE scan_report ADD COLUMN IF NOT EXISTS high_cnt BIGINT;

1
src/common/rbac/system/policies.go
View File

@ -86,5 +86,6 @@ var (
{Resource: rbac.ResourceJobServiceMonitor, Action: rbac.ActionStop},
{Resource: rbac.ResourceSecurityHub, Action: rbac.ActionRead},
{Resource: rbac.ResourceSecurityHub, Action: rbac.ActionList},
}
)

61
src/controller/securityhub/controller.go
View File

@ -23,6 +23,7 @@ import (
"github.com/goharbor/harbor/src/pkg/scan/scanner"
"github.com/goharbor/harbor/src/pkg/securityhub"
secHubModel "github.com/goharbor/harbor/src/pkg/securityhub/model"
"github.com/goharbor/harbor/src/pkg/tag"
)
// Ctl is the global controller for security hub
@ -63,26 +64,32 @@ func WithArtifact(enable bool) Option {
type Controller interface {
// SecuritySummary returns the security summary of the specified project.
SecuritySummary(ctx context.Context, projectID int64, options ...Option) (*secHubModel.Summary, error)
// ListVuls list vulnerabilities by query
ListVuls(ctx context.Context, scannerUUID string, projectID int64, withTag bool, query *q.Query) ([]*secHubModel.VulnerabilityItem, error)
// CountVuls get all vulnerability count by query
CountVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error)
}
type controller struct {
artifactMgr artifact.Manager
scannerMgr scanner.Manager
secHubMgr securityhub.Manager
tagMgr tag.Manager
}
// NewController ...
func NewController() Controller {
return &controller{
artifactMgr: pkg.ArtifactMgr,
scannerMgr: scanner.New(),
scannerMgr: scanner.Mgr,
secHubMgr: securityhub.Mgr,
tagMgr: tag.Mgr,
}
}
func (c *controller) SecuritySummary(ctx context.Context, projectID int64, options ...Option) (*secHubModel.Summary, error) {
opts := newOptions(options...)
scannerUUID, err := c.defaultScannerUUID(ctx)
scannerUUID, err := c.scannerMgr.DefaultScannerUUID(ctx)
if err != nil {
return nil, err
}
@ -114,7 +121,7 @@ func (c *controller) SecuritySummary(ctx context.Context, projectID int64, optio
}
func (c *controller) scannedArtifactCount(ctx context.Context, projectID int64) (int64, error) {
scannerUUID, err := c.defaultScannerUUID(ctx)
scannerUUID, err := c.scannerMgr.DefaultScannerUUID(ctx)
if err != nil {
return 0, err
}
@ -128,11 +135,49 @@ func (c *controller) totalArtifactCount(ctx context.Context, projectID int64) (i
return c.artifactMgr.Count(ctx, q.New(q.KeyWords{"project_id": projectID}))
}
// defaultScannerUUID returns the default scanner uuid.
func (c *controller) defaultScannerUUID(ctx context.Context) (string, error) {
reg, err := c.scannerMgr.GetDefault(ctx)
func (c *controller) ListVuls(ctx context.Context, scannerUUID string, projectID int64, withTag bool, query *q.Query) ([]*secHubModel.VulnerabilityItem, error) {
vuls, err := c.secHubMgr.ListVuls(ctx, scannerUUID, projectID, query)
if err != nil {
return "", err
return nil, err
}
return reg.UUID, nil
if withTag {
return c.attachTags(ctx, vuls)
}
return vuls, nil
}
func (c *controller) attachTags(ctx context.Context, vuls []*secHubModel.VulnerabilityItem) ([]*secHubModel.VulnerabilityItem, error) {
// get all artifact_ids
artifactTagMap := make(map[int64][]string, 0)
for _, v := range vuls {
artifactTagMap[v.ArtifactID] = make([]string, 0)
}
// get tags in the artifact list
var artifactIds []interface{}
for k := range artifactTagMap {
artifactIds = append(artifactIds, k)
}
query := q.New(q.KeyWords{"artifact_id": q.NewOrList(artifactIds)})
tags, err := c.tagMgr.List(ctx, query)
if err != nil {
return vuls, err
}
for _, tag := range tags {
artifactTagMap[tag.ArtifactID] = append(artifactTagMap[tag.ArtifactID], tag.Name)
}
// attach tags, only show 10 tags
for _, v := range vuls {
if len(artifactTagMap[v.ArtifactID]) > 10 {
v.Tags = artifactTagMap[v.ArtifactID][:10]
continue
}
v.Tags = artifactTagMap[v.ArtifactID]
}
return vuls, nil
}
func (c *controller) CountVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error) {
return c.secHubMgr.TotalVuls(ctx, scannerUUID, projectID, tuneCount, query)
}

79
src/controller/securityhub/controller_test.go
View File

@ -21,13 +21,14 @@ import (
"github.com/stretchr/testify/suite"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/goharbor/harbor/src/pkg/securityhub/model"
"github.com/goharbor/harbor/src/pkg/tag/model/tag"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/goharbor/harbor/src/testing/mock"
artifactMock "github.com/goharbor/harbor/src/testing/pkg/artifact"
scannerMock "github.com/goharbor/harbor/src/testing/pkg/scan/scanner"
securityMock "github.com/goharbor/harbor/src/testing/pkg/securityhub"
tagMock "github.com/goharbor/harbor/src/testing/pkg/tag"
)
var sum = &model.Summary{
@ -45,6 +46,7 @@ type ControllerTestSuite struct {
artifactMgr *artifactMock.Manager
scannerMgr *scannerMock.Manager
secHubMgr *securityMock.Manager
tagMgr *tagMock.FakeManager
}
// TestController is the entry of controller test suite
@ -57,10 +59,13 @@ func (suite *ControllerTestSuite) SetupTest() {
suite.artifactMgr = &artifactMock.Manager{}
suite.secHubMgr = &securityMock.Manager{}
suite.scannerMgr = &scannerMock.Manager{}
suite.tagMgr = &tagMock.FakeManager{}
suite.c = &controller{
artifactMgr: suite.artifactMgr,
secHubMgr: suite.secHubMgr,
scannerMgr: suite.scannerMgr,
tagMgr: suite.tagMgr,
}
}
@ -74,7 +79,7 @@ func (suite *ControllerTestSuite) TestSecuritySummary() {
mock.OnAnything(suite.artifactMgr, "Count").Return(int64(1234), nil)
mock.OnAnything(suite.secHubMgr, "ScannedArtifactsCount").Return(int64(1000), nil)
mock.OnAnything(suite.secHubMgr, "Summary").Return(sum, nil).Twice()
mock.OnAnything(suite.scannerMgr, "GetDefault").Return(&scanner.Registration{UUID: "ruuid"}, nil)
mock.OnAnything(suite.scannerMgr, "DefaultScannerUUID").Return("ruuid", nil)
summary, err := suite.c.SecuritySummary(ctx, 0, WithArtifact(false), WithCVE(false))
suite.NoError(err)
suite.NotNil(summary)
@ -119,7 +124,7 @@ func (suite *ControllerTestSuite) TestSecuritySummary() {
// TestSecuritySummaryError tests the security summary with error
func (suite *ControllerTestSuite) TestSecuritySummaryError() {
ctx := suite.Context()
mock.OnAnything(suite.scannerMgr, "GetDefault").Return(&scanner.Registration{UUID: "ruuid"}, nil)
mock.OnAnything(suite.scannerMgr, "DefaultScannerUUID").Return("ruuid", nil)
mock.OnAnything(suite.secHubMgr, "ScannedArtifactsCount").Return(int64(1000), nil)
mock.OnAnything(suite.secHubMgr, "Summary").Return(nil, errors.New("invalid project")).Once()
summary, err := suite.c.SecuritySummary(ctx, 0, WithCVE(false), WithArtifact(false))
@ -133,25 +138,63 @@ func (suite *ControllerTestSuite) TestSecuritySummaryError() {
}
// TestGetDefaultScanner tests the get default scanner
func (suite *ControllerTestSuite) TestGetDefaultScanner() {
ctx := suite.Context()
mock.OnAnything(suite.scannerMgr, "GetDefault").Return(&scanner.Registration{UUID: ""}, nil).Once()
scanner, err := suite.c.defaultScannerUUID(ctx)
suite.NoError(err)
suite.Equal("", scanner)
mock.OnAnything(suite.scannerMgr, "GetDefault").Return(nil, errors.New("failed to get scanner")).Once()
scanner, err = suite.c.defaultScannerUUID(ctx)
suite.Error(err)
suite.Equal("", scanner)
}
func (suite *ControllerTestSuite) TestScannedArtifact() {
ctx := suite.Context()
mock.OnAnything(suite.scannerMgr, "GetDefault").Return(&scanner.Registration{UUID: "ruuid"}, nil)
mock.OnAnything(suite.scannerMgr, "DefaultScannerUUID").Return("ruuid", nil)
mock.OnAnything(suite.secHubMgr, "ScannedArtifactsCount").Return(int64(1000), nil)
scanned, err := suite.c.scannedArtifactCount(ctx, 0)
suite.NoError(err)
suite.Equal(int64(1000), scanned)
}
// TestAttachTags test the attachTags
func (suite *ControllerTestSuite) TestAttachTags() {
ctx := suite.Context()
tagList := []*tag.Tag{
{ArtifactID: int64(1), Name: "latest"},
{ArtifactID: int64(1), Name: "tag1"},
{ArtifactID: int64(1), Name: "tag2"},
{ArtifactID: int64(1), Name: "tag3"},
{ArtifactID: int64(1), Name: "tag4"},
{ArtifactID: int64(1), Name: "tag5"},
{ArtifactID: int64(1), Name: "tag6"},
{ArtifactID: int64(1), Name: "tag7"},
{ArtifactID: int64(1), Name: "tag8"},
{ArtifactID: int64(1), Name: "tag9"},
{ArtifactID: int64(1), Name: "tag10"},
}
vulItems := []*model.VulnerabilityItem{
{ArtifactID: int64(1)},
}
mock.OnAnything(suite.c.tagMgr, "List").Return(tagList, nil).Once()
resultItems, err := suite.c.attachTags(ctx, vulItems)
suite.NoError(err)
suite.Equal(len(vulItems), len(resultItems))
suite.Equal([]string{"latest"}, resultItems[0].Tags[:1])
suite.Equal(10, len(resultItems[0].Tags))
}
// TestListVuls tests the list vulnerabilities
func (suite *ControllerTestSuite) TestListVuls() {
ctx := suite.Context()
vulItems := []*model.VulnerabilityItem{
{ArtifactID: int64(1)},
}
tagList := []*tag.Tag{
{ArtifactID: int64(1), Name: "latest"},
}
mock.OnAnything(suite.c.secHubMgr, "ListVuls").Return(vulItems, nil)
mock.OnAnything(suite.c.tagMgr, "List").Return(tagList, nil).Once()
vulResult, err := suite.c.ListVuls(ctx, "", 0, true, nil)
suite.NoError(err)
suite.Equal(1, len(vulResult))
suite.Equal(int64(1), vulResult[0].ArtifactID)
}
func (suite *ControllerTestSuite) TestCountVuls() {
ctx := suite.Context()
mock.OnAnything(suite.c.secHubMgr, "TotalVuls").Return(int64(10), nil)
count, err := suite.c.CountVuls(ctx, "", 0, true, nil)
suite.NoError(err)
suite.Equal(int64(10), count)
}

15
src/pkg/scan/scanner/manager.go
View File

@ -24,6 +24,9 @@ import (
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
)
// Mgr is the global manager for scanner
var Mgr = New()
// Manager defines the related scanner API endpoints
type Manager interface {
// Count returns the total count of scanner registrations according to the query.
@ -52,6 +55,9 @@ type Manager interface {
// GetDefault returns the default scanner registration or `nil` if there are no registrations configured.
GetDefault(ctx context.Context) (*scanner.Registration, error)
// DefaultScannerUUID get default scanner UUID
DefaultScannerUUID(ctx context.Context) (string, error)
}
// basicManager is the default implementation of Manager
@ -139,3 +145,12 @@ func (bm *basicManager) SetAsDefault(ctx context.Context, registrationUUID strin
func (bm *basicManager) GetDefault(ctx context.Context) (*scanner.Registration, error) {
return scanner.GetDefaultRegistration(ctx)
}
// DefaultScannerUUID returns the default scanner uuid.
func (bm *basicManager) DefaultScannerUUID(ctx context.Context) (string, error) {
reg, err := bm.GetDefault(ctx)
if err != nil {
return "", err
}
return reg.UUID, nil
}

9
src/pkg/scan/scanner/manager_test.go
View File

@ -111,3 +111,12 @@ func (suite *BasicManagerTestSuite) TestDefault() {
require.NotNil(suite.T(), dr)
assert.Equal(suite.T(), true, dr.IsDefault)
}
// TestGetDefaultScanner tests the get default scanner
func (suite *BasicManagerTestSuite) TestGetDefaultScanner() {
ctx := suite.Context()
suite.mgr.SetAsDefault(ctx, suite.sampleUUID)
scanner, err := suite.mgr.DefaultScannerUUID(ctx)
suite.NoError(err)
suite.Equal(suite.sampleUUID, scanner)
}

207
src/pkg/securityhub/dao/security.go
View File

@ -16,7 +16,10 @@ package dao
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
@ -24,6 +27,7 @@ import (
)
const (
// sql to query the security summary
summarySQL = `select sum(s.critical_cnt) critical_cnt,
sum(s.high_cnt) high_cnt,
sum(s.medium_cnt) medium_cnt,
@ -34,7 +38,7 @@ const (
from artifact a
left join scan_report s on a.digest = s.digest
where s.registration_uuid = ?`
// sql to query the dangerous artifact
dangerousArtifactSQL = `select a.project_id project, a.repository_name repository, a.digest, s.critical_cnt, s.high_cnt, s.medium_cnt, s.low_cnt
from artifact a,
scan_report s
@ -43,19 +47,95 @@ where a.digest = s.digest
order by s.critical_cnt desc, s.high_cnt desc, s.medium_cnt desc, s.low_cnt desc
limit 5`
// sql to query the scanned artifact count
scannedArtifactCountSQL = `select count(1)
from artifact a
left join scan_report s on a.digest = s.digest
where s.registration_uuid= ? and s.uuid is not null`
// sql to query the dangerous CVEs
dangerousCVESQL = `select vr.*
from vulnerability_record vr
where vr.cvss_score_v3 is not null
and vr.registration_uuid = ?
order by vr.cvss_score_v3 desc
limit 5`
// sql to query vulnerabilities
vulnerabilitySQL = `select vr.cve_id, vr.cvss_score_v3, vr.package, a.repository_name, a.id artifact_id, a.digest, vr.package, vr.package_version, vr.severity, vr.fixed_version, vr.description, vr.urls, a.project_id
from artifact a,
scan_report s,
report_vulnerability_record rvr,
vulnerability_record vr
where a.digest = s.digest
and s.uuid = rvr.report_uuid
and rvr.vuln_record_id = vr.id
and rvr.report_uuid is not null
and vr.registration_uuid = ? `
stringType = "string"
intType = "int"
rangeType = "range"
)
type filterMetaData struct {
DataType string
FilterFunc func(ctx context.Context, key string, query *q.Query) (sqlStr string, params []interface{})
}
var filterMap = map[string]*filterMetaData{
"cve_id": &filterMetaData{DataType: stringType, FilterFunc: exactMatchFilter},
"severity": &filterMetaData{DataType: stringType, FilterFunc: exactMatchFilter},
"cvss_score_v3": &filterMetaData{DataType: rangeType, FilterFunc: rangeFilter},
"project_id": &filterMetaData{DataType: stringType, FilterFunc: exactMatchFilter},
"repository_name": &filterMetaData{DataType: stringType, FilterFunc: exactMatchFilter},
"package": &filterMetaData{DataType: stringType, FilterFunc: exactMatchFilter},
"tag": &filterMetaData{DataType: stringType, FilterFunc: tagFilter},
}
var applyFilterFunc func(ctx context.Context, key string, query *q.Query) (sqlStr string, params []interface{})
func exactMatchFilter(ctx context.Context, key string, query *q.Query) (sqlStr string, params []interface{}) {
if query == nil {
return
}
if val, ok := query.Keywords[key]; ok {
sqlStr = fmt.Sprintf(" and %v = ?", key)
params = append(params, val)
return
}
return
}
func rangeFilter(ctx context.Context, key string, query *q.Query) (sqlStr string, params []interface{}) {
if query == nil {
return
}
if val, ok := query.Keywords[key]; ok {
if r, ok := val.(*q.Range); ok {
sqlStr = fmt.Sprintf(" and %v between ? and ?", key)
params = append(params, r.Min, r.Max)
}
}
return
}
func tagFilter(ctx context.Context, key string, query *q.Query) (sqlStr string, params []interface{}) {
if query == nil {
return
}
if val, ok := query.Keywords["tag"]; ok {
inClause, err := orm.CreateInClause(ctx, `SELECT artifact_id FROM tag
WHERE tag.name = ?`, val)
if err != nil {
log.Errorf("failed to create in clause: %v, skip this condition", err)
} else {
sqlStr = " and a.id " + inClause
}
}
return
}
// SecurityHubDao defines the interface to access security hub data.
type SecurityHubDao interface {
// Summary returns the summary of the scan cve reports.
@ -66,6 +146,10 @@ type SecurityHubDao interface {
DangerousArtifacts(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) ([]*model.DangerousArtifact, error)
// ScannedArtifactsCount return the count of scanned artifacts.
ScannedArtifactsCount(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) (int64, error)
// ListVulnerabilities search vulnerability record by cveID
ListVulnerabilities(ctx context.Context, registrationUUID string, projectID int64, query *q.Query) ([]*model.VulnerabilityItem, error)
// CountVulnerabilities count the total vulnerabilities
CountVulnerabilities(ctx context.Context, registrationUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error)
}
// New creates a new SecurityHubDao instance.
@ -131,3 +215,124 @@ func (d *dao) DangerousCVEs(ctx context.Context, scannerUUID string, projectID i
_, err = o.Raw(dangerousCVESQL, scannerUUID).QueryRows(&cves)
return cves, err
}
func countSQL(strSQL string) string {
return fmt.Sprintf(`select count(1) cnt from (%v) as t`, strSQL)
}
func (d *dao) CountVulnerabilities(ctx context.Context, registrationUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error) {
o, err := orm.FromContext(ctx)
if err != nil {
return 0, err
}
sqlStr := vulnerabilitySQL
params := []interface{}{registrationUUID}
if err := checkQFilter(query, filterMap); err != nil {
return 0, err
}
sqlStr, params = applyVulFilter(ctx, sqlStr, query, params)
if tuneCount {
exceedLimit, err := d.countExceedLimit(ctx, sqlStr, params)
if err != nil {
return 0, err
}
if exceedLimit {
log.Warning("the count is exceed to limit 1000 due to the tuneCount is enabled, return count with -1 instead")
return -1, nil
}
}
var cnt int64
err = o.Raw(countSQL(sqlStr), params).QueryRow(&cnt)
return cnt, err
}
// countExceedLimit check if the count is exceed to limit 1000, avoid count all record for large table
func (d *dao) countExceedLimit(ctx context.Context, sqlStr string, params []interface{}) (bool, error) {
o, err := orm.FromContext(ctx)
if err != nil {
return false, err
}
queryExceed := fmt.Sprintf(`SELECT EXISTS (%s LIMIT 1 OFFSET 1000)`, sqlStr)
var exceed bool
err = o.Raw(queryExceed, params).QueryRow(&exceed)
if err != nil {
return false, err
}
return exceed, nil
}
func (d *dao) ListVulnerabilities(ctx context.Context, registrationUUID string, projectID int64, query *q.Query) ([]*model.VulnerabilityItem, error) {
o, err := orm.FromContext(ctx)
if err != nil {
return nil, err
}
sqlStr := vulnerabilitySQL
params := []interface{}{registrationUUID}
if err := checkQFilter(query, filterMap); err != nil {
return nil, err
}
sqlStr, params = applyVulFilter(ctx, sqlStr, query, params)
sqlStr, params = applyVulPagination(sqlStr, query, params)
vulnRecs := make([]*model.VulnerabilityItem, 0)
_, err = o.Raw(sqlStr, params).QueryRows(&vulnRecs)
return vulnRecs, err
}
func applyVulFilter(ctx context.Context, sqlStr string, query *q.Query, params []interface{}) (queryStr string, newParam []interface{}) {
if query == nil {
return sqlStr, params
}
queryStr = sqlStr
newParam = params
for k, m := range filterMap {
s, p := m.FilterFunc(ctx, k, query)
queryStr = queryStr + s
newParam = append(newParam, p...)
}
return queryStr, newParam
}
// applyVulPagination apply pagination to the query and sort by cvss_score_v3 desc
func applyVulPagination(sqlStr string, query *q.Query, params []interface{}) (string, []interface{}) {
offSet := int64(0)
pageSize := int64(15)
if query != nil && query.PageNumber > 1 {
offSet = (query.PageNumber - 1) * query.PageSize
}
if query != nil && query.PageSize > 0 {
pageSize = query.PageSize
}
params = append(params, pageSize, offSet)
return fmt.Sprintf("%v order by cvss_score_v3 desc nulls last limit ? offset ? ", sqlStr), params
}
func checkQFilter(query *q.Query, filterMap map[string]*filterMetaData) error {
if query == nil {
return nil
}
if len(query.Keywords) == 0 {
return nil
}
for k := range query.Keywords {
if metadata, exist := filterMap[k]; exist {
typeName := metadata.DataType
switch typeName {
case rangeType:
if _, ok := query.Keywords[k].(*q.Range); !ok {
return errors.BadRequestError(fmt.Errorf("keyword: %v, the query type is not allowed", k))
}
case stringType:
if _, ok := query.Keywords[k].(string); !ok {
return errors.BadRequestError(fmt.Errorf("keyword: %v, the query type is not allowed", k))
}
case intType:
if _, ok := query.Keywords[k].(int); !ok {
return errors.BadRequestError(fmt.Errorf("keyword: %v, the query type is not allowed", k))
}
}
} else {
return errors.BadRequestError(fmt.Errorf("keyword: %v is not allowed", k))
}
}
return nil
}

88
src/pkg/securityhub/dao/security_test.go
View File

@ -15,12 +15,14 @@
package dao
import (
"context"
"testing"
"github.com/stretchr/testify/suite"
testDao "github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
htesting "github.com/goharbor/harbor/src/testing"
)
@ -61,7 +63,6 @@ values (1, 'library/hello-world', 'digest1001', 'IMAGE', '2023-06-02 09:16:47.8
})
}
// TearDownTest clears enf for test case.
func (suite *SecurityDaoTestSuite) TearDownTest() {
testDao.ExecuteBatchSQL([]string{
`delete from scan_report where uuid = 'uuid'`,
@ -102,3 +103,88 @@ func (suite *SecurityDaoTestSuite) TestGetDangerousCVEs() {
suite.NoError(err, "Error when fetching most dangerous artifact")
suite.Equal(5, len(records))
}
func Test_checkQFilter(t *testing.T) {
type args struct {
query *q.Query
filterMap map[string]*filterMetaData
}
tests := []struct {
name string
args args
wantErr bool
}{
{"happy_path", args{q.New(q.KeyWords{"sample": 1}), map[string]*filterMetaData{"sample": &filterMetaData{intType, exactMatchFilter}}}, false},
{"happy_path_cve_id", args{q.New(q.KeyWords{"cve_id": "CVE-2023-2345"}), map[string]*filterMetaData{"cve_id": &filterMetaData{stringType, exactMatchFilter}}}, false},
{"happy_path_severity", args{q.New(q.KeyWords{"severity": "Critical"}), map[string]*filterMetaData{"severity": &filterMetaData{stringType, exactMatchFilter}}}, false},
{"happy_path_cvss_score_v3", args{q.New(q.KeyWords{"cvss_score_v3": &q.Range{Min: 2.0, Max: 3.0}}), map[string]*filterMetaData{"cvss_score_v3": &filterMetaData{rangeType, rangeFilter}}}, false},
{"unhappy_path", args{q.New(q.KeyWords{"sample": 1}), map[string]*filterMetaData{"a": &filterMetaData{DataType: intType}}}, true},
{"unhappy_path2", args{q.New(q.KeyWords{"cve_id": 1}), map[string]*filterMetaData{"cve_id": &filterMetaData{stringType, exactMatchFilter}}}, true},
{"unhappy_path3", args{q.New(q.KeyWords{"severity": &q.Range{Min: 2.0, Max: 10.0}}), map[string]*filterMetaData{"severity": &filterMetaData{stringType, exactMatchFilter}}}, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := checkQFilter(tt.args.query, tt.args.filterMap); (err != nil) != tt.wantErr {
t.Errorf("checkQFilter() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func (suite *SecurityDaoTestSuite) TestExacthMatchFilter() {
type args struct {
ctx context.Context
key string
query *q.Query
}
tests := []struct {
name string
args args
wantSQLStr string
wantParams []interface{}
}{
{"normal", args{suite.Context(), "cve_id", q.New(q.KeyWords{"cve_id": "CVE-2023-2345"})}, " and cve_id = ?", []interface{}{"CVE-2023-2345"}},
}
for _, tt := range tests {
suite.Run(tt.name, func() {
gotSQLStr, gotParams := exactMatchFilter(tt.args.ctx, tt.args.key, tt.args.query)
suite.Equal(gotSQLStr, tt.wantSQLStr, "exactMatchFilter() gotSqlStr = %v, want %v", gotSQLStr, tt.wantSQLStr)
suite.Equal(gotParams, tt.wantParams, "exactMatchFilter() gotParams = %v, want %v", gotParams, tt.wantParams)
})
}
}
func (suite *SecurityDaoTestSuite) TestRangeFilter() {
type args struct {
ctx context.Context
key string
query *q.Query
}
tests := []struct {
name string
args args
wantSQLStr string
wantParams []interface{}
}{
{"normal", args{suite.Context(), "cvss_score_v3", q.New(q.KeyWords{"cvss_score_v3": &q.Range{1.0, 2.0}})}, " and cvss_score_v3 between ? and ?", []interface{}{1.0, 2.0}},
}
for _, tt := range tests {
suite.Run(tt.name, func() {
gotSQLStr, gotParams := rangeFilter(tt.args.ctx, tt.args.key, tt.args.query)
suite.Equal(tt.wantSQLStr, gotSQLStr, "exactMatchFilter() gotSqlStr = %v, want %v", gotSQLStr, tt.wantSQLStr)
suite.Equal(tt.wantParams, gotParams, "exactMatchFilter() gotParams = %v, want %v", gotParams, tt.wantParams)
})
}
}
func (suite *SecurityDaoTestSuite) TestCountVul() {
count, err := suite.dao.CountVulnerabilities(suite.Context(), "ruuid", 0, true, nil)
suite.NoError(err)
suite.Equal(int64(1), count)
}
func (suite *SecurityDaoTestSuite) TestListVul() {
vuls, err := suite.dao.ListVulnerabilities(suite.Context(), "ruuid", 0, nil)
suite.NoError(err)
suite.Equal(1, len(vuls))
}

12
src/pkg/securityhub/manager.go
View File

@ -38,6 +38,10 @@ type Manager interface {
ScannedArtifactsCount(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) (int64, error)
// DangerousCVEs returns the most dangerous CVEs for the given scanner.
DangerousCVEs(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) ([]*scan.VulnerabilityRecord, error)
// TotalVuls return the count of vulnerabilities
TotalVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error)
// ListVuls returns vulnerabilities list
ListVuls(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) ([]*model.VulnerabilityItem, error)
}
// NewManager news security manager.
@ -67,3 +71,11 @@ func (s *securityManager) ScannedArtifactsCount(ctx context.Context, scannerUUID
func (s *securityManager) DangerousCVEs(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) ([]*scan.VulnerabilityRecord, error) {
return s.dao.DangerousCVEs(ctx, scannerUUID, projectID, query)
}
func (s *securityManager) TotalVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error) {
return s.dao.CountVulnerabilities(ctx, scannerUUID, projectID, tuneCount, query)
}
func (s *securityManager) ListVuls(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) ([]*model.VulnerabilityItem, error) {
return s.dao.ListVulnerabilities(ctx, scannerUUID, projectID, query)
}

10
src/pkg/securityhub/model/model.go
View File

@ -42,3 +42,13 @@ type DangerousArtifact struct {
MediumCnt int64 `json:"medium_cnt" orm:"column(medium_cnt)"`
LowCnt int64 `json:"low_cnt" orm:"column(low_cnt)"`
}
// VulnerabilityItem is the item of vulnerability
type VulnerabilityItem struct {
scan.VulnerabilityRecord
ArtifactID int64 `orm:"column(artifact_id)"`
RepositoryName string `orm:"column(repository_name)"`
Digest string `orm:"column(digest)"`
Tags []string `orm:"-"`
ProjectID int64 `orm:"column(project_id)"`
}

51
src/server/v2.0/handler/security.go
View File

@ -16,10 +16,12 @@ package handler
import (
"context"
"strings"
"github.com/go-openapi/runtime/middleware"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/pkg/scan/scanner"
"github.com/goharbor/harbor/src/server/v2.0/models"
securityModel "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/securityhub"
@ -96,3 +98,52 @@ func toDangerousCves(cves []*scan.VulnerabilityRecord) []*models.DangerousCVE {
}
return result
}
func (s *securityAPI) ListVulnerabilities(ctx context.Context, params securityModel.ListVulnerabilitiesParams) middleware.Responder {
if err := s.RequireSystemAccess(ctx, rbac.ActionList, rbac.ResourceSecurityHub); err != nil {
return s.SendError(ctx, err)
}
query, err := s.BuildQuery(ctx, params.Q, nil, params.Page, params.PageSize)
if err != nil {
return s.SendError(ctx, err)
}
scannerUUID, err := scanner.Mgr.DefaultScannerUUID(ctx)
if err != nil {
return s.SendError(ctx, err)
}
cnt, err := s.controller.CountVuls(ctx, scannerUUID, 0, *params.TuneCount, query)
if err != nil {
return s.SendError(ctx, err)
}
vuls, err := s.controller.ListVuls(ctx, scannerUUID, 0, *params.WithTag, query)
if err != nil {
return s.SendError(ctx, err)
}
link := s.Links(ctx, params.HTTPRequest.URL, cnt, query.PageNumber, query.PageSize).String()
return securityModel.NewListVulnerabilitiesOK().WithPayload(toVulnerabilities(vuls)).WithLink(link).WithXTotalCount(cnt)
}
func toVulnerabilities(vuls []*secHubModel.VulnerabilityItem) []*models.VulnerabilityItem {
result := make([]*models.VulnerabilityItem, 0)
for _, item := range vuls {
score := float32(0)
if item.CVE3Score != nil {
score = float32(*item.CVE3Score)
}
result = append(result, &models.VulnerabilityItem{
ProjectID: item.ProjectID,
RepositoryName: item.RepositoryName,
Digest: item.Digest,
CVEID: item.CVEID,
Severity: item.Severity,
Package: item.Package,
Tags: item.Tags,
Version: item.PackageVersion,
FixedVersion: item.Fix,
Desc: item.Description,
CvssV3Score: score,
Links: strings.Split(item.URLs, "|"),
})
}
return result
}

52
src/testing/controller/securityhub/controller.go
View File

@ -8,6 +8,8 @@ import (
model "github.com/goharbor/harbor/src/pkg/securityhub/model"
mock "github.com/stretchr/testify/mock"
q "github.com/goharbor/harbor/src/lib/q"
securityhub "github.com/goharbor/harbor/src/controller/securityhub"
)
@ -16,6 +18,56 @@ type Controller struct {
mock.Mock
}
// CountVuls provides a mock function with given fields: ctx, scannerUUID, projectID, tuneCount, query
func (_m *Controller) CountVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error) {
ret := _m.Called(ctx, scannerUUID, projectID, tuneCount, query)
var r0 int64
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string, int64, bool, *q.Query) (int64, error)); ok {
return rf(ctx, scannerUUID, projectID, tuneCount, query)
}
if rf, ok := ret.Get(0).(func(context.Context, string, int64, bool, *q.Query) int64); ok {
r0 = rf(ctx, scannerUUID, projectID, tuneCount, query)
} else {
r0 = ret.Get(0).(int64)
}
if rf, ok := ret.Get(1).(func(context.Context, string, int64, bool, *q.Query) error); ok {
r1 = rf(ctx, scannerUUID, projectID, tuneCount, query)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ListVuls provides a mock function with given fields: ctx, scannerUUID, projectID, withTag, query
func (_m *Controller) ListVuls(ctx context.Context, scannerUUID string, projectID int64, withTag bool, query *q.Query) ([]*model.VulnerabilityItem, error) {
ret := _m.Called(ctx, scannerUUID, projectID, withTag, query)
var r0 []*model.VulnerabilityItem
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string, int64, bool, *q.Query) ([]*model.VulnerabilityItem, error)); ok {
return rf(ctx, scannerUUID, projectID, withTag, query)
}
if rf, ok := ret.Get(0).(func(context.Context, string, int64, bool, *q.Query) []*model.VulnerabilityItem); ok {
r0 = rf(ctx, scannerUUID, projectID, withTag, query)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.VulnerabilityItem)
}
}
if rf, ok := ret.Get(1).(func(context.Context, string, int64, bool, *q.Query) error); ok {
r1 = rf(ctx, scannerUUID, projectID, withTag, query)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// SecuritySummary provides a mock function with given fields: ctx, projectID, options
func (_m *Controller) SecuritySummary(ctx context.Context, projectID int64, options ...securityhub.Option) (*model.Summary, error) {
_va := make([]interface{}, len(options))

24
src/testing/pkg/scan/scanner/manager.go
View File

@ -64,6 +64,30 @@ func (_m *Manager) Create(ctx context.Context, registration *daoscanner.Registra
return r0, r1
}
// DefaultScannerUUID provides a mock function with given fields: ctx
func (_m *Manager) DefaultScannerUUID(ctx context.Context) (string, error) {
ret := _m.Called(ctx)
var r0 string
var r1 error
if rf, ok := ret.Get(0).(func(context.Context) (string, error)); ok {
return rf(ctx)
}
if rf, ok := ret.Get(0).(func(context.Context) string); ok {
r0 = rf(ctx)
} else {
r0 = ret.Get(0).(string)
}
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
r1 = rf(ctx)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Delete provides a mock function with given fields: ctx, registrationUUID
func (_m *Manager) Delete(ctx context.Context, registrationUUID string) error {
ret := _m.Called(ctx, registrationUUID)

50
src/testing/pkg/securityhub/manager.go
View File

@ -70,6 +70,32 @@ func (_m *Manager) DangerousCVEs(ctx context.Context, scannerUUID string, projec
return r0, r1
}
// ListVuls provides a mock function with given fields: ctx, scannerUUID, projectID, query
func (_m *Manager) ListVuls(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) ([]*model.VulnerabilityItem, error) {
ret := _m.Called(ctx, scannerUUID, projectID, query)
var r0 []*model.VulnerabilityItem
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string, int64, *q.Query) ([]*model.VulnerabilityItem, error)); ok {
return rf(ctx, scannerUUID, projectID, query)
}
if rf, ok := ret.Get(0).(func(context.Context, string, int64, *q.Query) []*model.VulnerabilityItem); ok {
r0 = rf(ctx, scannerUUID, projectID, query)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.VulnerabilityItem)
}
}
if rf, ok := ret.Get(1).(func(context.Context, string, int64, *q.Query) error); ok {
r1 = rf(ctx, scannerUUID, projectID, query)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ScannedArtifactsCount provides a mock function with given fields: ctx, scannerUUID, projectID, query
func (_m *Manager) ScannedArtifactsCount(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) (int64, error) {
ret := _m.Called(ctx, scannerUUID, projectID, query)
@ -120,6 +146,30 @@ func (_m *Manager) Summary(ctx context.Context, scannerUUID string, projectID in
return r0, r1
}
// TotalVuls provides a mock function with given fields: ctx, scannerUUID, projectID, tuneCount, query
func (_m *Manager) TotalVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error) {
ret := _m.Called(ctx, scannerUUID, projectID, tuneCount, query)
var r0 int64
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string, int64, bool, *q.Query) (int64, error)); ok {
return rf(ctx, scannerUUID, projectID, tuneCount, query)
}
if rf, ok := ret.Get(0).(func(context.Context, string, int64, bool, *q.Query) int64); ok {
r0 = rf(ctx, scannerUUID, projectID, tuneCount, query)
} else {
r0 = ret.Get(0).(int64)
}
if rf, ok := ret.Get(1).(func(context.Context, string, int64, bool, *q.Query) error); ok {
r1 = rf(ctx, scannerUUID, projectID, tuneCount, query)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
type mockConstructorTestingTNewManager interface {
mock.TestingT
Cleanup(func())