upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor synced 2024-09-20 16:47:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

do changes to let the vul policy check compatiable with new framework

Browse Source 
- update the scan/scanner controller
- enhance the report summary generation
- do changes to the vulnerable handler
- remove the unused clair related code
- add more UT cases
- update the scan web hook event
- drop the unsed tables/index/triggers in sql schema

Signed-off-by: Steven Zou <szou@vmware.com>
This commit is contained in:
Steven Zou 2019-10-15 13:45:53 +08:00
parent 9d37e9472c
commit f18afc0a3f
50 changed files with 729 additions and 2268 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
make/migrations/postgresql/0015_1.10.0_schema.up.sql
View File

@ -44,4 +44,17 @@ CREATE TABLE immutable_tag_rule
creation_time timestamp default CURRENT_TIMESTAMP
);
ALTER TABLE robot ADD COLUMN visible boolean DEFAULT true NOT NULL;
ALTER TABLE robot ADD COLUMN visible boolean DEFAULT true NOT NULL;
/** Drop the unused vul related tables **/
DROP INDEX IF EXISTS idx_status;
DROP INDEX IF EXISTS idx_digest;
DROP INDEX IF EXISTS idx_uuid;
DROP INDEX IF EXISTS idx_repository_tag;
DROP TRIGGER IF EXISTS img_scan_job_update_time_at_modtime ON img_scan_job;
DROP TABLE IF EXISTS img_scan_job;
DROP TRIGGER IF EXISTS TRIGGER ON img_scan_overview;
DROP TABLE IF EXISTS img_scan_overview;
DROP TABLE IF EXISTS clair_vuln_timestamp

52
src/common/dao/clair.go
View File

@ -1,52 +0,0 @@
// Copyright Project Harbor Authors
//
// licensed under the apache license, version 2.0 (the "license");
// you may not use this file except in compliance with the license.
// you may obtain a copy of the license at
//
// http://www.apache.org/licenses/license-2.0
//
// unless required by applicable law or agreed to in writing, software
// distributed under the license is distributed on an "as is" basis,
// without warranties or conditions of any kind, either express or implied.
// see the license for the specific language governing permissions and
// limitations under the license.
package dao
import (
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"time"
)
// SetClairVulnTimestamp update the last_update of a namespace. If there's no record for this namespace, one will be created.
func SetClairVulnTimestamp(namespace string, timestamp time.Time) error {
o := GetOrmer()
rec := &models.ClairVulnTimestamp{
Namespace: namespace,
LastUpdate: timestamp,
}
created, _, err := o.ReadOrCreate(rec, "Namespace")
if err != nil {
return err
}
if !created {
rec.LastUpdate = timestamp
n, err := o.Update(rec)
if n == 0 {
log.Warningf("no records are updated for %v", *rec)
}
return err
}
return nil
}
// ListClairVulnTimestamps return a list of all records in vuln timestamp table.
func ListClairVulnTimestamps() ([]*models.ClairVulnTimestamp, error) {
var res []*models.ClairVulnTimestamp
o := GetOrmer()
_, err := o.QueryTable(models.ClairVulnTimestampTable).Limit(-1).All(&res)
return res, err
}

73
src/common/dao/clair/base.go
View File

@ -1,73 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package clair
import (
"fmt"
"strconv"
"sync"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/dao"
)
const (
updaterLast = "updater/last"
)
var (
ormer orm.Ormer
once sync.Once
)
// GetOrmer return the singleton of Ormer for clair DB.
func GetOrmer() orm.Ormer {
once.Do(func() {
dbInstance, err := orm.GetDB(dao.ClairDBAlias)
if err != nil {
panic(err)
}
ormer, err = orm.NewOrmWithDB("postgres", dao.ClairDBAlias, dbInstance)
if err != nil {
panic(err)
}
})
return ormer
}
// GetLastUpdate query the table `keyvalue` in clair's DB return the value of `updater/last`
func GetLastUpdate() (int64, error) {
var list orm.ParamsList
num, err := GetOrmer().Raw("SELECT value from keyvalue where key=?", updaterLast).ValuesFlat(&list)
if err != nil {
return 0, err
}
if num == 1 {
s, ok := list[0].(string)
if !ok { // shouldn't be here.
return 0, fmt.Errorf("The value: %v, is non-string", list[0])
}
res, err := strconv.ParseInt(s, 0, 64)
if err != nil { // shouldn't be here.
return 0, err
}
return res, nil
}
if num > 1 {
return 0, fmt.Errorf("Multiple entries for %s in Clair DB", updaterLast)
}
// num is zero, it's not updated yet.
return 0, nil
}

175
src/common/dao/dao_test.go
View File

@ -759,181 +759,6 @@ func TestDeleteRepository(t *testing.T) {
}
}
var sj1 = models.ScanJob{
Status: models.JobPending,
Repository: "library/ubuntu",
Tag: "14.04",
}
var sj2 = models.ScanJob{
Status: models.JobPending,
Repository: "library/ubuntu",
Tag: "15.10",
Digest: "sha256:0204dc6e09fa57ab99ac40e415eb637d62c8b2571ecbbc9ca0eb5e2ad2b5c56f",
}
func TestAddScanJob(t *testing.T) {
assert := assert.New(t)
id, err := AddScanJob(sj1)
assert.Nil(err)
r1, err := GetScanJob(id)
assert.Nil(err)
assert.Equal(sj1.Tag, r1.Tag)
assert.Equal(sj1.Status, r1.Status)
assert.Equal(sj1.Repository, r1.Repository)
err = ClearTable(models.ScanJobTable)
assert.Nil(err)
}
func TestGetScanJobs(t *testing.T) {
assert := assert.New(t)
_, err := AddScanJob(sj1)
assert.Nil(err)
id2, err := AddScanJob(sj1)
assert.Nil(err)
_, err = AddScanJob(sj2)
assert.Nil(err)
r, err := GetScanJobsByImage("library/ubuntu", "14.04")
assert.Nil(err)
assert.Equal(2, len(r))
assert.Equal(id2, r[0].ID)
r, err = GetScanJobsByImage("library/ubuntu", "14.04", 1)
assert.Nil(err)
assert.Equal(1, len(r))
r, err = GetScanJobsByDigest("sha256:nono")
assert.Nil(err)
assert.Equal(0, len(r))
r, err = GetScanJobsByDigest(sj2.Digest)
assert.Equal(1, len(r))
assert.Equal(sj2.Tag, r[0].Tag)
assert.Nil(err)
err = ClearTable(models.ScanJobTable)
assert.Nil(err)
}
func TestSetScanJobUUID(t *testing.T) {
uuid := "u-scan-job-uuid"
assert := assert.New(t)
id, err := AddScanJob(sj1)
assert.Nil(err)
err = SetScanJobUUID(id, uuid)
assert.Nil(err)
j, err := GetScanJob(id)
assert.Nil(err)
assert.Equal(uuid, j.UUID)
err = ClearTable(models.ScanJobTable)
assert.Nil(err)
}
func TestUpdateScanJobStatus(t *testing.T) {
assert := assert.New(t)
id, err := AddScanJob(sj1)
assert.Nil(err)
err = UpdateScanJobStatus(id, "newstatus")
assert.Nil(err)
j, err := GetScanJob(id)
assert.Nil(err)
assert.Equal("newstatus", j.Status)
err = ClearTable(models.ScanJobTable)
assert.Nil(err)
}
func TestImgScanOverview(t *testing.T) {
assert := assert.New(t)
err := ClearTable(models.ScanOverviewTable)
assert.Nil(err)
digest := "sha256:0204dc6e09fa57ab99ac40e415eb637d62c8b2571ecbbc9ca0eb5e2ad2b5c56f"
res, err := GetImgScanOverview(digest)
assert.Nil(err)
assert.Nil(res)
err = SetScanJobForImg(digest, 33)
assert.Nil(err)
res, err = GetImgScanOverview(digest)
assert.Nil(err)
assert.Equal(int64(33), res.JobID)
err = SetScanJobForImg(digest, 22)
assert.Nil(err)
res, err = GetImgScanOverview(digest)
assert.Nil(err)
assert.Equal(int64(22), res.JobID)
pk := "22-sha256:sdfsdfarfwefwr23r43t34ggregergerger"
comp := &models.ComponentsOverview{
Total: 2,
Summary: []*models.ComponentsOverviewEntry{
{
Sev: int(models.SevMedium),
Count: 2,
},
},
}
err = UpdateImgScanOverview(digest, pk, models.SevMedium, comp)
assert.Nil(err)
res, err = GetImgScanOverview(digest)
assert.Nil(err)
assert.Equal(pk, res.DetailsKey)
assert.Equal(int(models.SevMedium), res.Sev)
assert.Equal(2, res.CompOverview.Summary[0].Count)
}
func TestVulnTimestamp(t *testing.T) {
assert := assert.New(t)
err := ClearTable(models.ClairVulnTimestampTable)
assert.Nil(err)
ns := "ubuntu:14"
res, err := ListClairVulnTimestamps()
assert.Nil(err)
assert.Equal(0, len(res))
err = SetClairVulnTimestamp(ns, time.Now())
assert.Nil(err)
res, err = ListClairVulnTimestamps()
assert.Nil(err)
assert.Equal(1, len(res))
assert.Equal(ns, res[0].Namespace)
old := time.Now()
t.Logf("Sleep 3 seconds")
time.Sleep(3 * time.Second)
err = SetClairVulnTimestamp(ns, time.Now())
assert.Nil(err)
res, err = ListClairVulnTimestamps()
assert.Nil(err)
assert.Equal(1, len(res))
d := res[0].LastUpdate.Sub(old)
if d < 2*time.Second {
t.Errorf("Delta should be larger than 2 seconds! old: %v, lastupdate: %v", old, res[0].LastUpdate)
}
}
func TestListScanOverviews(t *testing.T) {
assert := assert.New(t)
err := ClearTable(models.ScanOverviewTable)
assert.Nil(err)
l, err := ListImgScanOverviews()
assert.Nil(err)
assert.Equal(0, len(l))
err = ClearTable(models.ScanOverviewTable)
assert.Nil(err)
}
func TestGetScanJobsByStatus(t *testing.T) {
assert := assert.New(t)
err := ClearTable(models.ScanOverviewTable)
assert.Nil(err)
id, err := AddScanJob(sj1)
assert.Nil(err)
err = UpdateScanJobStatus(id, models.JobRunning)
assert.Nil(err)
r1, err := GetScanJobsByStatus(models.JobPending, models.JobCanceled)
assert.Nil(err)
assert.Equal(0, len(r1))
r2, err := GetScanJobsByStatus(models.JobPending, models.JobRunning)
assert.Nil(err)
assert.Equal(1, len(r2))
assert.Equal(sj1.Repository, r2[0].Repository)
}
func TestIsSuperUser(t *testing.T) {
assert := assert.New(t)
assert.True(IsSuperUser("admin"))

195
src/common/dao/scan_job.go
View File

@ -1,195 +0,0 @@
// Copyright Project Harbor Authors
//
// licensed under the apache license, version 2.0 (the "license");
// you may not use this file except in compliance with the license.
// you may obtain a copy of the license at
//
// http://www.apache.org/licenses/license-2.0
//
// unless required by applicable law or agreed to in writing, software
// distributed under the license is distributed on an "as is" basis,
// without warranties or conditions of any kind, either express or implied.
// see the license for the specific language governing permissions and
// limitations under the license.
package dao
import (
"encoding/json"
"fmt"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"time"
)
// AddScanJob ...
func AddScanJob(job models.ScanJob) (int64, error) {
o := GetOrmer()
if len(job.Status) == 0 {
job.Status = models.JobPending
}
return o.Insert(&job)
}
// GetScanJob ...
func GetScanJob(id int64) (*models.ScanJob, error) {
o := GetOrmer()
j := models.ScanJob{ID: id}
err := o.Read(&j)
if err == orm.ErrNoRows {
return nil, nil
}
return &j, nil
}
// GetScanJobsByImage returns a list of scan jobs with given repository and tag
func GetScanJobsByImage(repository, tag string, limit ...int) ([]*models.ScanJob, error) {
var res []*models.ScanJob
_, err := scanJobQs(limit...).Filter("repository", repository).Filter("tag", tag).OrderBy("-id").All(&res)
return res, err
}
// GetScanJobsByDigest returns a list of scan jobs with given digest
func GetScanJobsByDigest(digest string, limit ...int) ([]*models.ScanJob, error) {
var res []*models.ScanJob
_, err := scanJobQs(limit...).Filter("digest", digest).OrderBy("-id").All(&res)
return res, err
}
// GetScanJobsByStatus return a list of scan jobs with any of the given statuses in param
func GetScanJobsByStatus(status ...string) ([]*models.ScanJob, error) {
var res []*models.ScanJob
var t []interface{}
for _, s := range status {
t = append(t, interface{}(s))
}
_, err := scanJobQs().Filter("status__in", t...).All(&res)
return res, err
}
// UpdateScanJobStatus updates the status of a scan job.
func UpdateScanJobStatus(id int64, status string) error {
o := GetOrmer()
sj := models.ScanJob{
ID: id,
Status: status,
UpdateTime: time.Now(),
}
n, err := o.Update(&sj, "Status", "UpdateTime")
if n == 0 {
log.Warningf("no records are updated when updating scan job %d", id)
}
return err
}
// SetScanJobUUID set UUID to the record so it associates with the job in job service.
func SetScanJobUUID(id int64, uuid string) error {
o := GetOrmer()
sj := models.ScanJob{
ID: id,
UUID: uuid,
}
n, err := o.Update(&sj, "UUID")
if n == 0 {
log.Warningf("no records are updated when updating scan job %d", id)
}
return err
}
func scanJobQs(limit ...int) orm.QuerySeter {
o := GetOrmer()
l := -1
if len(limit) == 1 {
l = limit[0]
}
return o.QueryTable(models.ScanJobTable).Limit(l)
}
// SetScanJobForImg updates the scan_job_id based on the digest of image, if there's no data, it created one record.
func SetScanJobForImg(digest string, jobID int64) error {
o := GetOrmer()
rec := &models.ImgScanOverview{
Digest: digest,
JobID: jobID,
UpdateTime: time.Now(),
}
created, _, err := o.ReadOrCreate(rec, "Digest")
if err != nil {
return err
}
if !created {
rec.JobID = jobID
rec.UpdateTime = time.Now()
n, err := o.Update(rec, "JobID", "UpdateTime")
if n == 0 {
log.Warningf("no records are updated when setting scan job for image with digest %s", digest)
}
return err
}
return nil
}
// GetImgScanOverview returns the ImgScanOverview based on the digest.
func GetImgScanOverview(digest string) (*models.ImgScanOverview, error) {
res := []*models.ImgScanOverview{}
_, err := scanOverviewQs().Filter("image_digest", digest).All(&res)
if err != nil {
return nil, err
}
if len(res) == 0 {
return nil, nil
}
if len(res) > 1 {
return nil, fmt.Errorf("Found multiple scan_overview entries for digest: %s", digest)
}
rec := res[0]
if len(rec.CompOverviewStr) > 0 {
co := &models.ComponentsOverview{}
if err := json.Unmarshal([]byte(rec.CompOverviewStr), co); err != nil {
return nil, err
}
rec.CompOverview = co
}
return rec, nil
}
// UpdateImgScanOverview updates the serverity and components status of a record in img_scan_overview
func UpdateImgScanOverview(digest, detailsKey string, sev models.Severity, compOverview *models.ComponentsOverview) error {
o := GetOrmer()
rec, err := GetImgScanOverview(digest)
if err != nil {
return fmt.Errorf("Failed to getting scan_overview record for update: %v", err)
}
if rec == nil {
return fmt.Errorf("No scan_overview record for digest: %s", digest)
}
b, err := json.Marshal(compOverview)
if err != nil {
return err
}
rec.Sev = int(sev)
rec.CompOverviewStr = string(b)
rec.DetailsKey = detailsKey
rec.UpdateTime = time.Now()
_, err = o.Update(rec, "Sev", "CompOverviewStr", "DetailsKey", "UpdateTime")
if err != nil {
return fmt.Errorf("Failed to update scan overview record with digest: %s, error: %v", digest, err)
}
return nil
}
// ListImgScanOverviews list all records in table img_scan_overview, it is called in notification handler when it needs to refresh the severity of all images.
func ListImgScanOverviews() ([]*models.ImgScanOverview, error) {
var res []*models.ImgScanOverview
o := GetOrmer()
_, err := o.QueryTable(models.ScanOverviewTable).All(&res)
return res, err
}
func scanOverviewQs() orm.QuerySeter {
o := GetOrmer()
return o.QueryTable(models.ScanOverviewTable)
}

2
src/common/models/base.go
View File

@ -24,9 +24,7 @@ func init() {
new(Project),
new(Role),
new(AccessLog),
new(ScanJob),
new(RepoRecord),
new(ImgScanOverview),
new(ClairVulnTimestamp),
new(ProjectMetadata),
new(ConfigEntry),

99
src/common/models/scan_job.go
View File

@ -1,99 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import "time"
// ScanJobTable is the name of the table whose data is mapped by ScanJob struct.
const ScanJobTable = "img_scan_job"
// ScanOverviewTable is the name of the table whose data is mapped by ImgScanOverview struct.
const ScanOverviewTable = "img_scan_overview"
// ScanJob is the model to represent a job for image scan in DB.
type ScanJob struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
Status string `orm:"column(status)" json:"status"`
Repository string `orm:"column(repository)" json:"repository"`
Tag string `orm:"column(tag)" json:"tag"`
Digest string `orm:"column(digest)" json:"digest"`
UUID string `orm:"column(job_uuid)" json:"-"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
// TableName is required by by beego orm to map ScanJob to table img_scan_job
func (s *ScanJob) TableName() string {
return ScanJobTable
}
// ImgScanOverview mapped to a record of image scan overview.
type ImgScanOverview struct {
ID int64 `orm:"pk;auto;column(id)" json:"-"`
Digest string `orm:"column(image_digest)" json:"image_digest"`
Status string `orm:"-" json:"scan_status"`
JobID int64 `orm:"column(scan_job_id)" json:"job_id"`
Sev int `orm:"column(severity)" json:"severity"`
CompOverviewStr string `orm:"column(components_overview)" json:"-"`
CompOverview *ComponentsOverview `orm:"-" json:"components,omitempty"`
DetailsKey string `orm:"column(details_key)" json:"details_key"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time,omitempty"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time,omitempty"`
}
// TableName ...
func (iso *ImgScanOverview) TableName() string {
return ScanOverviewTable
}
// ComponentsOverview has the total number and a list of components number of different serverity level.
type ComponentsOverview struct {
Total int `json:"total"`
Summary []*ComponentsOverviewEntry `json:"summary"`
}
// ComponentsOverviewEntry ...
type ComponentsOverviewEntry struct {
Sev int `json:"severity"`
Count int `json:"count"`
}
// ImageScanReq represents the request body to send to job service for image scan
type ImageScanReq struct {
Repo string `json:"repository"`
Tag string `json:"tag"`
}
// ScanAllPolicy is represent the json request and object for scan all policy, the parm is het
type ScanAllPolicy struct {
Type string `json:"type"`
Parm map[string]interface{} `json:"parameter,omitempty"`
}
const (
// ScanAllNone "none" for not doing any scan all
ScanAllNone = "none"
// ScanAllDaily for doing scan all daily
ScanAllDaily = "daily"
// ScanAllOnRefresh for doing scan all when the Clair DB is refreshed.
ScanAllOnRefresh = "on_refresh"
// ScanAllDailyTime the key for parm of daily scan all policy.
ScanAllDailyTime = "daily_time"
)
// DefaultScanAllPolicy ...
var DefaultScanAllPolicy = ScanAllPolicy{
Type: ScanAllNone,
}

26
src/common/models/sev.go
View File

@ -1,26 +0,0 @@
package models
// Severity represents the severity of a image/component in terms of vulnerability.
type Severity int64
// Sevxxx is the list of severity of image after scanning.
const (
_ Severity = iota
SevNone
SevUnknown
SevLow
SevMedium
SevHigh
)
// String is the output function for severity variable
func (sev Severity) String() string {
name := []string{"negligible", "unknown", "low", "medium", "high"}
i := int64(sev)
switch {
case i >= 1 && i <= int64(SevHigh):
return name[i-1]
default:
return "unknown"
}
}

162
src/common/utils/clair/client.go
View File

@ -1,162 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package clair
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strings"
// "path"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
)
// Client communicates with clair endpoint to scan image and get detailed scan result
type Client struct {
endpoint string
// need to customize the logger to write output to job log.
logger *log.Logger
client *http.Client
}
// NewClient creates a new instance of client, set the logger as the job's logger if it's used in a job handler.
func NewClient(endpoint string, logger *log.Logger) *Client {
if logger == nil {
logger = log.DefaultLogger()
}
return &Client{
endpoint: strings.TrimSuffix(endpoint, "/"),
logger: logger,
client: &http.Client{},
}
}
func (c *Client) send(req *http.Request, expectedStatus int) ([]byte, error) {
resp, err := c.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != expectedStatus {
return nil, fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
}
return b, nil
}
// ScanLayer calls Clair's API to scan a layer.
func (c *Client) ScanLayer(l models.ClairLayer) error {
layer := models.ClairLayerEnvelope{
Layer: &l,
Error: nil,
}
data, err := json.Marshal(layer)
if err != nil {
return err
}
req, err := http.NewRequest(http.MethodPost, c.endpoint+"/v1/layers", bytes.NewReader(data))
if err != nil {
return err
}
req.Header.Set(http.CanonicalHeaderKey("Content-Type"), "application/json")
_, err = c.send(req, http.StatusCreated)
if err != nil {
log.Errorf("Unexpected error: %v", err)
}
return err
}
// GetResult calls Clair's API to get layers with detailed vulnerability list
func (c *Client) GetResult(layerName string) (*models.ClairLayerEnvelope, error) {
req, err := http.NewRequest(http.MethodGet, c.endpoint+"/v1/layers/"+layerName+"?features&vulnerabilities", nil)
if err != nil {
return nil, err
}
b, err := c.send(req, http.StatusOK)
if err != nil {
return nil, err
}
var res models.ClairLayerEnvelope
err = json.Unmarshal(b, &res)
if err != nil {
return nil, err
}
return &res, nil
}
// GetNotification calls Clair's API to get details of notification
func (c *Client) GetNotification(id string) (*models.ClairNotification, error) {
req, err := http.NewRequest(http.MethodGet, c.endpoint+"/v1/notifications/"+id+"?limit=2", nil)
if err != nil {
return nil, err
}
b, err := c.send(req, http.StatusOK)
if err != nil {
return nil, err
}
var ne models.ClairNotificationEnvelope
err = json.Unmarshal(b, &ne)
if err != nil {
return nil, err
}
if ne.Error != nil {
return nil, fmt.Errorf("Clair error: %s", ne.Error.Message)
}
log.Debugf("Retrieved notification %s from Clair.", id)
return ne.Notification, nil
}
// DeleteNotification deletes a notification record from Clair
func (c *Client) DeleteNotification(id string) error {
req, err := http.NewRequest(http.MethodDelete, c.endpoint+"/v1/notifications/"+id, nil)
if err != nil {
return err
}
_, err = c.send(req, http.StatusOK)
if err != nil {
return err
}
log.Debugf("Deleted notification %s from Clair.", id)
return nil
}
// ListNamespaces list the namespaces in Clair
func (c *Client) ListNamespaces() ([]string, error) {
req, err := http.NewRequest(http.MethodGet, c.endpoint+"/v1/namespaces", nil)
if err != nil {
return nil, err
}
b, err := c.send(req, http.StatusOK)
if err != nil {
return nil, err
}
var nse models.ClairNamespaceEnvelope
err = json.Unmarshal(b, &nse)
if err != nil {
return nil, err
}
res := []string{}
for _, ns := range *nse.Namespaces {
res = append(res, ns.Name)
}
return res, nil
}

73
src/common/utils/clair/client_test.go
View File

@ -1,73 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package clair
import (
"os"
"testing"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/clair/test"
"github.com/stretchr/testify/assert"
)
var (
notificationID = "ec45ec87-bfc8-4129-a1c3-d2b82622175a"
layerName = "03adedf41d4e0ea1b2458546a5b4717bf5f24b23489b25589e20c692aaf84d19"
client *Client
)
func TestMain(m *testing.M) {
mockClairServer := test.NewMockServer()
defer mockClairServer.Close()
client = NewClient(mockClairServer.URL, nil)
rc := m.Run()
if rc != 0 {
os.Exit(rc)
}
}
func TestListNamespaces(t *testing.T) {
assert := assert.New(t)
ns, err := client.ListNamespaces()
assert.Nil(err)
assert.Equal(25, len(ns))
}
func TestNotifications(t *testing.T) {
assert := assert.New(t)
n, err := client.GetNotification(notificationID)
assert.Nil(err)
assert.Equal(notificationID, n.Name)
_, err = client.GetNotification("noexist")
assert.NotNil(err)
err = client.DeleteNotification(notificationID)
assert.Nil(err)
}
func TestLaysers(t *testing.T) {
assert := assert.New(t)
layer := models.ClairLayer{
Name: "fakelayer",
ParentName: "parent",
Path: "http://registry:5000/layers/xxx",
}
err := client.ScanLayer(layer)
assert.Nil(err)
data, err := client.GetResult(layerName)
assert.Nil(err)
assert.Equal(layerName, data.Layer.Name)
_, err = client.GetResult("notexist")
assert.NotNil(err)
}

62
src/common/utils/clair/test/notification.json
View File

@ -1,62 +0,0 @@
{
"Notification": {
"Name": "ec45ec87-bfc8-4129-a1c3-d2b82622175a",
"Created": "1456247389",
"Notified": "1456246708",
"Limit": 2,
"Page": "gAAAAABWzJaC2JCH6Apr_R1f2EkjGdibnrKOobTcYXBWl6t0Cw6Q04ENGIymB6XlZ3Zi0bYt2c-2cXe43fvsJ7ECZhZz4P8C8F9efr_SR0HPiejzQTuG0qAzeO8klogFfFjSz2peBvgP",
"NextPage": "gAAAAABWzJaCTyr6QXP2aYsCwEZfWIkU2GkNplSMlTOhLJfiR3LorBv8QYgEIgyOvZRmHQEzJKvkI6TP2PkRczBkcD17GE89btaaKMqEX14yHDgyfQvdasW1tj3-5bBRt0esKi9ym5En",
"New": {
"Vulnerability": {
"Name": "CVE-TEST",
"NamespaceName": "debian:8",
"Description": "New CVE",
"Severity": "Low",
"FixedIn": [
{
"Name": "grep",
"NamespaceName": "debian:8",
"Version": "2.25"
}
]
},
"OrderedLayersIntroducingVulnerability": [
{
"Index": 1,
"LayerName": "523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6"
},
{
"Index": 2,
"LayerName": "3b59c795b34670618fbcace4dac7a27c5ecec156812c9e2c90d3f4be1916b12d"
}
],
"LayersIntroducingVulnerability": [
"523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6",
"3b59c795b34670618fbcace4dac7a27c5ecec156812c9e2c90d182371916b12d"
]
},
"Old": {
"Vulnerability": {
"Name": "CVE-TEST",
"NamespaceName": "debian:8",
"Description": "New CVE",
"Severity": "Low",
"FixedIn": []
},
"OrderedLayersIntroducingVulnerability": [
{
"Index": 1,
"LayerName": "523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6"
},
{
"Index": 2,
"LayerName": "3b59c795b34670618fbcace4dac7a27c5ecec156812c9e2c90d3f4be1916b12d"
}
],
"LayersIntroducingVulnerability": [
"3b59c795b34670618fbcace4dac7a27c5ecec156812c9e2c90d3f4be1916b12d",
"523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6"
]
}
}
}

1
src/common/utils/clair/test/ns.json
View File

@ -1 +0,0 @@
{"Namespaces":[{"Name":"debian:7","VersionFormat":"dpkg"},{"Name":"debian:unstable","VersionFormat":"dpkg"},{"Name":"debian:9","VersionFormat":"dpkg"},{"Name":"debian:10","VersionFormat":"dpkg"},{"Name":"debian:8","VersionFormat":"dpkg"},{"Name":"alpine:v3.6","VersionFormat":"dpkg"},{"Name":"alpine:v3.5","VersionFormat":"dpkg"},{"Name":"alpine:v3.4","VersionFormat":"dpkg"},{"Name":"alpine:v3.3","VersionFormat":"dpkg"},{"Name":"oracle:6","VersionFormat":"rpm"},{"Name":"oracle:7","VersionFormat":"rpm"},{"Name":"oracle:5","VersionFormat":"rpm"},{"Name":"ubuntu:14.04","VersionFormat":"dpkg"},{"Name":"ubuntu:15.10","VersionFormat":"dpkg"},{"Name":"ubuntu:17.04","VersionFormat":"dpkg"},{"Name":"ubuntu:16.04","VersionFormat":"dpkg"},{"Name":"ubuntu:12.04","VersionFormat":"dpkg"},{"Name":"ubuntu:13.04","VersionFormat":"dpkg"},{"Name":"ubuntu:14.10","VersionFormat":"dpkg"},{"Name":"ubuntu:12.10","VersionFormat":"dpkg"},{"Name":"ubuntu:16.10","VersionFormat":"dpkg"},{"Name":"ubuntu:15.04","VersionFormat":"dpkg"},{"Name":"centos:6","VersionFormat":"rpm"},{"Name":"centos:7","VersionFormat":"rpm"},{"Name":"centos:5","VersionFormat":"rpm"}]}

117
src/common/utils/clair/test/server.go
View File

@ -1,117 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"path"
"runtime"
"strings"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
)
func currPath() string {
_, f, _, ok := runtime.Caller(0)
if !ok {
panic("Failed to get current directory")
}
return path.Dir(f)
}
func serveFile(rw http.ResponseWriter, p string) {
data, err := ioutil.ReadFile(p)
if err != nil {
http.Error(rw, err.Error(), 500)
}
_, err2 := rw.Write(data)
if err2 != nil {
http.Error(rw, err2.Error(), 500)
}
}
type notificationHandler struct {
id string
}
func (n *notificationHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
suffix := strings.TrimPrefix(req.URL.Path, "/v1/notifications/")
if req.Method == http.MethodDelete {
rw.WriteHeader(200)
} else if req.Method == http.MethodGet {
if strings.HasPrefix(suffix, n.id) {
serveFile(rw, path.Join(currPath(), "notification.json"))
} else {
rw.WriteHeader(404)
}
} else {
rw.WriteHeader(http.StatusMethodNotAllowed)
}
}
type layerHandler struct {
name string
}
func (l *layerHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if req.Method == http.MethodPost {
data, err := ioutil.ReadAll(req.Body)
defer req.Body.Close()
if err != nil {
http.Error(rw, err.Error(), 500)
}
layer := &models.ClairLayerEnvelope{}
if err := json.Unmarshal(data, layer); err != nil {
http.Error(rw, err.Error(), http.StatusBadRequest)
}
rw.WriteHeader(http.StatusCreated)
} else if req.Method == http.MethodGet {
name := strings.TrimPrefix(req.URL.Path, "/v1/layers/")
if name == l.name {
serveFile(rw, path.Join(currPath(), "total-12.json"))
} else {
http.Error(rw, fmt.Sprintf("Invalid layer name: %s", name), http.StatusNotFound)
}
} else {
http.Error(rw, "", http.StatusMethodNotAllowed)
}
}
// NewMockServer ...
func NewMockServer() *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc("/v1/namespaces", func(rw http.ResponseWriter, req *http.Request) {
if req.Method == http.MethodGet {
serveFile(rw, path.Join(currPath(), "ns.json"))
} else {
rw.WriteHeader(http.StatusMethodNotAllowed)
}
})
mux.Handle("/v1/notifications/", &notificationHandler{id: "ec45ec87-bfc8-4129-a1c3-d2b82622175a"})
mux.Handle("/v1/layers", &layerHandler{name: "03adedf41d4e0ea1b2458546a5b4717bf5f24b23489b25589e20c692aaf84d19"})
mux.Handle("/v1/layers/", &layerHandler{name: "03adedf41d4e0ea1b2458546a5b4717bf5f24b23489b25589e20c692aaf84d19"})
mux.HandleFunc("/", func(rw http.ResponseWriter, req *http.Request) {
log.Infof("method: %s, path: %s", req.Method, req.URL.Path)
rw.WriteHeader(http.StatusNotFound)
},
)
return httptest.NewServer(mux)
}

1
src/common/utils/clair/test/total-12.json
View File

@ -1 +0,0 @@
{"Layer":{"Name":"03adedf41d4e0ea1b2458546a5b4717bf5f24b23489b25589e20c692aaf84d19","NamespaceName":"alpine:v3.4","ParentName":"61171f6a2863a80d24d842e11d277d6a8b216502456d90833d87759fb6a30516","IndexedByVersion":3,"Features":[{"Name":"musl","NamespaceName":"alpine:v3.4","VersionFormat":"dpkg","Version":"1.1.14-r11","Vulnerabilities":[{"Name":"CVE-2016-8859","NamespaceName":"alpine:v3.4","Link":"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8859","Severity":"High","Metadata":{"NVD":{"CVSSv2":{"Score":7.5,"Vectors":"AV:N/AC:L/Au:N/C:P/I:P"}}},"FixedBy":"1.1.14-r13"}],"AddedBy":"63e57014ab640f34a397c174f7cd085729aec27eca7715832786412ac7ffbd71"},{"Name":"libssl1.0","NamespaceName":"alpine:v3.4","VersionFormat":"dpkg","Version":"1.0.2h-r4","AddedBy":"63e57014ab640f34a397c174f7cd085729aec27eca7715832786412ac7ffbd71"},{"Name":"busybox","NamespaceName":"alpine:v3.4","VersionFormat":"dpkg","Version":"1.24.2-r11","Vulnerabilities":[{"Name":"CVE-2016-6301","NamespaceName":"alpine:v3.4","Link":"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-6301","Severity":"High","Metadata":{"NVD":{"CVSSv2":{"Score":7.8,"Vectors":"AV:N/AC:L/Au:N/C:N/I:N"}}},"FixedBy":"1.24.2-r12"}],"AddedBy":"63e57014ab640f34a397c174f7cd085729aec27eca7715832786412ac7ffbd71"},{"Name":"alpine-keys","NamespaceName":"alpine:v3.4","VersionFormat":"dpkg","Version":"1.1-r0","AddedBy":"63e57014ab640f34a397c174f7cd085729aec27eca7715832786412ac7ffbd71"},{"Name":"alpine-baselayout","NamespaceName":"alpine:v3.4","VersionFormat":"dpkg","Version":"3.0.3-r0","AddedBy":"63e57014ab640f34a397c174f7cd085729aec27eca7715832786412ac7ffbd71"},{"Name":"libcrypto1.0","NamespaceName":"alpine:v3.4","VersionFormat":"dpkg","Version":"1.0.2h-r4","AddedBy":"63e57014ab640f34a397c174f7cd085729aec27eca7715832786412ac7ffbd71"},{"Name":"musl-utils","NamespaceName":"alpine:v3.4","VersionFormat":"dpkg","Version":"1.1.14-r11","AddedBy":"63e57014ab640f34a397c174f7cd085729aec27eca7715832786412ac7ffbd71"},{"Name":"apk-tools","NamespaceName":"alpine:v3.4","VersionFormat":"dpkg","Version":"2.6.7-r0","AddedBy":"63e57014ab640f34a397c174f7cd085729aec27eca7715832786412ac7ffbd71"},{"Name":"zlib","NamespaceName":"alpine:v3.4","VersionFormat":"dpkg","Version":"1.2.8-r2","Vulnerabilities":[{"Name":"CVE-2016-9841","NamespaceName":"alpine:v3.4","Link":"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-9841","Severity":"High","Metadata":{"NVD":{"CVSSv2":{"Score":7.5,"Vectors":"AV:N/AC:L/Au:N/C:P/I:P"}}},"FixedBy":"1.2.11-r0"},{"Name":"CVE-2016-9843","NamespaceName":"alpine:v3.4","Link":"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-9843","Severity":"High","Metadata":{"NVD":{"CVSSv2":{"Score":7.5,"Vectors":"AV:N/AC:L/Au:N/C:P/I:P"}}},"FixedBy":"1.2.11-r0"},{"Name":"CVE-2016-9840","NamespaceName":"alpine:v3.4","Link":"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-9840","Severity":"Medium","Metadata":{"NVD":{"CVSSv2":{"Score":6.8,"Vectors":"AV:N/AC:M/Au:N/C:P/I:P"}}},"FixedBy":"1.2.11-r0"},{"Name":"CVE-2016-9842","NamespaceName":"alpine:v3.4","Link":"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-9842","Severity":"Medium","Metadata":{"NVD":{"CVSSv2":{"Score":6.8,"Vectors":"AV:N/AC:M/Au:N/C:P/I:P"}}},"FixedBy":"1.2.11-r0"}],"AddedBy":"63e57014ab640f34a397c174f7cd085729aec27eca7715832786412ac7ffbd71"},{"Name":"libc-utils","NamespaceName":"alpine:v3.4","VersionFormat":"dpkg","Version":"0.7-r0","AddedBy":"63e57014ab640f34a397c174f7cd085729aec27eca7715832786412ac7ffbd71"},{"Name":"scanelf","NamespaceName":"alpine:v3.4","VersionFormat":"dpkg","Version":"1.1.6-r0","AddedBy":"63e57014ab640f34a397c174f7cd085729aec27eca7715832786412ac7ffbd71"},{"Name":"dnsmasq","NamespaceName":"alpine:v3.4","VersionFormat":"dpkg","Version":"2.76-r0","AddedBy":"73cda0d278577331d47943734385a88b3de2c69355d9f9edfe87fb48d7af7528"}]}}

77
src/common/utils/clair/utils.go
View File

@ -1,77 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package clair
import (
"github.com/goharbor/harbor/src/common/models"
"strings"
)
// var client = NewClient()
// ParseClairSev parse the severity of clair to Harbor's Severity type if the string is not recognized the value will be set to unknown.
func ParseClairSev(clairSev string) models.Severity {
sev := strings.ToLower(clairSev)
switch sev {
case models.SeverityNone:
return models.SevNone
case models.SeverityLow:
return models.SevLow
case models.SeverityMedium:
return models.SevMedium
case models.SeverityHigh, models.SeverityCritical:
return models.SevHigh
default:
return models.SevUnknown
}
}
func transformVuln(clairVuln *models.ClairLayerEnvelope) (*models.ComponentsOverview, models.Severity) {
vulnMap := make(map[models.Severity]int)
features := clairVuln.Layer.Features
totalComponents := len(features)
var temp models.Severity
for _, f := range features {
sev := models.SevNone
for _, v := range f.Vulnerabilities {
temp = ParseClairSev(v.Severity)
if temp > sev {
sev = temp
}
}
vulnMap[sev]++
}
overallSev := models.SevNone
compSummary := []*models.ComponentsOverviewEntry{}
for k, v := range vulnMap {
if k > overallSev {
overallSev = k
}
entry := &models.ComponentsOverviewEntry{
Sev: int(k),
Count: v,
}
compSummary = append(compSummary, entry)
}
return &models.ComponentsOverview{
Total: totalComponents,
Summary: compSummary,
}, overallSev
}
// TransformVuln is for running scanning job in both job service V1 and V2.
func TransformVuln(clairVuln *models.ClairLayerEnvelope) (*models.ComponentsOverview, models.Severity) {
return transformVuln(clairVuln)
}

78
src/common/utils/clair/utils_test.go
View File

@ -1,78 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package clair
import (
"encoding/json"
"io/ioutil"
"path"
"runtime"
"testing"
"github.com/goharbor/harbor/src/common/models"
"github.com/stretchr/testify/assert"
)
func TestParseServerity(t *testing.T) {
newAssert := assert.New(t)
in := map[string]models.Severity{
"negligible": models.SevNone,
"whatever": models.SevUnknown,
"LOW": models.SevLow,
"Medium": models.SevMedium,
"high": models.SevHigh,
"Critical": models.SevHigh,
}
for k, v := range in {
newAssert.Equal(v, ParseClairSev(k))
}
}
func TestTransformVuln(t *testing.T) {
var clairVuln = &models.ClairLayerEnvelope{}
newAssert := assert.New(t)
empty := []byte(`{"Layer":{"Features":[]}}`)
loadVuln(empty, clairVuln)
output, o := transformVuln(clairVuln)
newAssert.Equal(0, output.Total)
newAssert.Equal(models.SevNone, o)
_, f, _, ok := runtime.Caller(0)
if !ok {
panic("Failed to get current directory")
}
curDir := path.Dir(f)
fileData, err := ioutil.ReadFile(path.Join(curDir, "test/total-12.json"))
if err != nil {
panic(err)
}
loadVuln(fileData, clairVuln)
output, o = transformVuln(clairVuln)
newAssert.Equal(12, output.Total)
newAssert.Equal(models.SevHigh, o)
hit := false
for _, s := range output.Summary {
if s.Sev == int(models.SevHigh) {
newAssert.Equal(3, s.Count, "There should be 3 components with High severity")
hit = true
}
}
newAssert.True(hit, "Not found entry for high severity in summary list")
}
func loadVuln(input []byte, data *models.ClairLayerEnvelope) {
err := json.Unmarshal(input, data)
if err != nil {
panic(err)
}
}

8
src/core/api/config.go
View File

@ -15,17 +15,17 @@
package api
import (
"errors"
"fmt"
"strings"
"errors"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/config"
"github.com/goharbor/harbor/src/common/config/metadata"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/security/secret"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/api/models"
corecfg "github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/filter"
)
@ -151,7 +151,9 @@ func convertForGet(cfg map[string]interface{}) (map[string]*value, error) {
}
if _, ok := cfg[common.ScanAllPolicy]; !ok {
cfg[common.ScanAllPolicy] = models.DefaultScanAllPolicy
cfg[common.ScanAllPolicy] = models.ScanAllPolicy{
Type: "none", // For legacy compatible
}
}
for k, v := range cfg {
result[k] = &value{

11
src/core/api/models/admin_job.go
View File

@ -156,11 +156,11 @@ func ConvertSchedule(cronStr string) (ScheduleParam, error) {
convertedSchedule.Type = "custom"
if strings.Contains(cronStr, "parameter") {
scheduleModel := common_models.ScanAllPolicy{}
scheduleModel := ScanAllPolicy{}
if err := json.Unmarshal([]byte(cronStr), &scheduleModel); err != nil {
return ScheduleParam{}, err
}
h, m, s := common_utils.ParseOfftime(int64(scheduleModel.Parm["daily_time"].(float64)))
h, m, s := common_utils.ParseOfftime(int64(scheduleModel.Param["daily_time"].(float64)))
cron := fmt.Sprintf("%d %d %d * * *", s, m, h)
convertedSchedule.Cron = cron
return convertedSchedule, nil
@ -181,3 +181,10 @@ func ConvertSchedule(cronStr string) (ScheduleParam, error) {
return ScheduleParam{}, fmt.Errorf("unsupported cron format, %s", cronStr)
}
// ScanAllPolicy is represent the json request and object for scan all policy
// Only for migrating from the legacy schedule.
type ScanAllPolicy struct {
Type string `json:"type"`
Param map[string]interface{} `json:"parameter,omitempty"`
}

11
src/core/api/scan_test.go
View File

@ -19,15 +19,14 @@ import (
"net/http"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
dscan "github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/report"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
@ -187,8 +186,8 @@ func (msc *MockScanAPIController) GetReport(artifact *v1.Artifact, mimeTypes []s
return args.Get(0).([]*dscan.Report), args.Error(1)
}
func (msc *MockScanAPIController) GetSummary(artifact *v1.Artifact, mimeTypes []string) (map[string]interface{}, error) {
args := msc.Called(artifact, mimeTypes)
func (msc *MockScanAPIController) GetSummary(artifact *v1.Artifact, mimeTypes []string, options ...report.Option) (map[string]interface{}, error) {
args := msc.Called(artifact, mimeTypes, options)
if args.Get(0) == nil {
return nil, args.Error(1)

8
src/core/api/scanners_test.go
View File

@ -370,3 +370,11 @@ func (m *MockScannerAPIController) GetMetadata(registrationUUID string) (*v1.Sca
return sam.(*v1.ScannerAdapterMetadata), nil
}
// IsScannerAvailable ...
// TODO: Remove it when the interface is changed
func (m *MockScannerAPIController) IsScannerAvailable(projectID int64) (bool, error) {
args := m.Called(projectID)
return args.Bool(0), args.Error(1)
}

96
src/core/api/systeminfo.go
View File

@ -21,14 +21,10 @@ import (
"net/http"
"os"
"strings"
"sync"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
clairdao "github.com/goharbor/harbor/src/common/dao/clair"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/clair"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/systeminfo"
@ -54,44 +50,9 @@ type Storage struct {
Free uint64 `json:"free"`
}
// namespaces stores all name spaces on Clair, it should be initialised only once.
type clairNamespaces struct {
sync.RWMutex
l []string
clair *clair.Client
}
func (n *clairNamespaces) get() ([]string, error) {
n.Lock()
defer n.Unlock()
if len(n.l) == 0 {
m := make(map[string]struct{})
if n.clair == nil {
n.clair = clair.NewClient(config.ClairEndpoint(), nil)
}
list, err := n.clair.ListNamespaces()
if err != nil {
return n.l, err
}
for _, n := range list {
ns := strings.Split(n, ":")[0]
m[ns] = struct{}{}
}
for k := range m {
n.l = append(n.l, k)
}
}
return n.l, nil
}
var (
namespaces = &clairNamespaces{}
)
// GeneralInfo wraps common systeminfo for anonymous request
type GeneralInfo struct {
WithNotary bool `json:"with_notary"`
WithClair bool `json:"with_clair"`
WithAdmiral bool `json:"with_admiral"`
AdmiralEndpoint string `json:"admiral_endpoint"`
AuthMode string `json:"auth_mode"`
@ -151,7 +112,7 @@ func (sia *SystemInfoAPI) GetCert() {
return
} else {
log.Errorf("Unexpected error: %v", err)
sia.SendInternalServerError(fmt.Errorf("Unexpected error: %v", err))
sia.SendInternalServerError(fmt.Errorf("unexpected error: %v", err))
return
}
}
@ -161,7 +122,7 @@ func (sia *SystemInfoAPI) GetGeneralInfo() {
cfg, err := config.GetSystemCfg()
if err != nil {
log.Errorf("Error occurred getting config: %v", err)
sia.SendInternalServerError(fmt.Errorf("Unexpected error: %v", err))
sia.SendInternalServerError(fmt.Errorf("unexpected error: %v", err))
return
}
extURL := cfg[common.ExtEndpoint].(string)
@ -178,7 +139,6 @@ func (sia *SystemInfoAPI) GetGeneralInfo() {
AdmiralEndpoint: utils.SafeCastString(cfg[common.AdmiralEndpoint]),
WithAdmiral: config.WithAdmiral(),
WithNotary: config.WithNotary(),
WithClair: config.WithClair(),
AuthMode: utils.SafeCastString(cfg[common.AUTHMode]),
ProjectCreationRestrict: utils.SafeCastString(cfg[common.ProjectCreationRestriction]),
SelfRegistration: utils.SafeCastBool(cfg[common.SelfRegistration]),
@ -191,9 +151,7 @@ func (sia *SystemInfoAPI) GetGeneralInfo() {
WithChartMuseum: config.WithChartMuseum(),
NotificationEnable: utils.SafeCastBool(cfg[common.NotificationEnable]),
}
if info.WithClair {
info.ClairVulnStatus = getClairVulnStatus()
}
if info.AuthMode == common.HTTPAuth {
if s, err := config.HTTPAuthProxySetting(); err == nil {
info.AuthProxySettings = s
@ -215,54 +173,6 @@ func (sia *SystemInfoAPI) getVersion() string {
return string(version[:])
}
func getClairVulnStatus() *models.ClairVulnerabilityStatus {
res := &models.ClairVulnerabilityStatus{}
last, err := clairdao.GetLastUpdate()
if err != nil {
log.Errorf("Failed to get last update from Clair DB, error: %v", err)
res.OverallUTC = 0
} else {
res.OverallUTC = last
log.Debugf("Clair vuln DB last update: %d", last)
}
details := []models.ClairNamespaceTimestamp{}
if res.OverallUTC > 0 {
l, err := dao.ListClairVulnTimestamps()
if err != nil {
log.Errorf("Failed to list Clair vulnerability timestamps, error:%v", err)
return res
}
m := make(map[string]int64)
for _, e := range l {
ns := strings.Split(e.Namespace, ":")
// only returns the latest time of one distro, i.e. unbuntu:14.04 and ubuntu:15.4 shares one timestamp
el := e.LastUpdate.UTC().Unix()
if ts, ok := m[ns[0]]; !ok || ts < el {
m[ns[0]] = el
}
}
list, err := namespaces.get()
if err != nil {
log.Errorf("Failed to get namespace list from Clair, error: %v", err)
}
// For namespaces not reported by notifier, the timestamp will be the overall db timestamp.
for _, n := range list {
if _, ok := m[n]; !ok {
m[n] = res.OverallUTC
}
}
for k, v := range m {
e := models.ClairNamespaceTimestamp{
Namespace: k,
Timestamp: v,
}
details = append(details, e)
}
}
res.Details = details
return res
}
// Ping ping the harbor core service.
func (sia *SystemInfoAPI) Ping() {
sia.Data["json"] = "Pong"

12
src/core/config/config.go
View File

@ -20,7 +20,6 @@ package config
import (
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
@ -394,17 +393,6 @@ func AdmiralEndpoint() string {
return cfgMgr.Get(common.AdmiralEndpoint).GetString()
}
// ScanAllPolicy returns the policy which controls the scan all.
func ScanAllPolicy() models.ScanAllPolicy {
var res models.ScanAllPolicy
log.Infof("Scan all policy %v", cfgMgr.Get(common.ScanAllPolicy).GetString())
if err := json.Unmarshal([]byte(cfgMgr.Get(common.ScanAllPolicy).GetString()), &res); err != nil {
log.Errorf("Failed to unmarshal the value in configuration for Scan All policy, error: %v, returning the default policy", err)
return models.DefaultScanAllPolicy
}
return res
}
// WithAdmiral returns a bool to indicate if Harbor's deployed with admiral.
func WithAdmiral() bool {
return len(AdmiralEndpoint()) > 0

16
src/core/config/config_test.go
View File

@ -14,14 +14,11 @@
package config
import (
"encoding/json"
"os"
"path"
"runtime"
"testing"
"fmt"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
@ -189,10 +186,6 @@ func TestConfig(t *testing.T) {
t.Errorf("unexpected mode: %s != %s", mode, "db_auth")
}
if s := ScanAllPolicy(); s.Type != "none" {
t.Errorf("unexpected scan all policy %v", s)
}
if tokenKeyPath := TokenPrivateKeyPath(); tokenKeyPath != "/etc/core/private_key.pem" {
t.Errorf("Unexpected token private key path: %s, expected: %s", tokenKeyPath, "/etc/core/private_key.pem")
}
@ -221,15 +214,6 @@ func currPath() string {
}
return path.Dir(f)
}
func TestConfigureValue_GetMap(t *testing.T) {
var policy models.ScanAllPolicy
value2 := `{"parameter":{"daily_time":0},"type":"daily"}`
err := json.Unmarshal([]byte(value2), &policy)
if err != nil {
t.Errorf("Failed with error %v", err)
}
fmt.Printf("%+v\n", policy)
}
func TestHTTPAuthProxySetting(t *testing.T) {
m := map[string]interface{}{

22
src/core/middlewares/util/util.go
View File

@ -35,12 +35,13 @@ import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/clair"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/promgr"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
"github.com/goharbor/harbor/src/pkg/scan/whitelist"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
type contextKey string
@ -327,11 +328,13 @@ func CopyResp(rec *httptest.ResponseRecorder, rw http.ResponseWriter) {
}
// PolicyChecker checks the policy of a project by project name, to determine if it's needed to check the image's status under this project.
// TODO: The policy check related things should be moved to pkg package later
// and refactored to include the `check` capabilities, not just property getters.
type PolicyChecker interface {
// contentTrustEnabled returns whether a project has enabled content trust.
ContentTrustEnabled(name string) bool
// vulnerablePolicy returns whether a project has enabled vulnerable, and the project's severity.
VulnerablePolicy(name string) (bool, models.Severity, models.CVEWhitelist)
VulnerablePolicy(name string) (bool, vuln.Severity, models.CVEWhitelist)
}
// PmsPolicyChecker ...
@ -354,28 +357,33 @@ func (pc PmsPolicyChecker) ContentTrustEnabled(name string) bool {
}
// VulnerablePolicy ...
func (pc PmsPolicyChecker) VulnerablePolicy(name string) (bool, models.Severity, models.CVEWhitelist) {
func (pc PmsPolicyChecker) VulnerablePolicy(name string) (bool, vuln.Severity, models.CVEWhitelist) {
project, err := pc.pm.Get(name)
wl := models.CVEWhitelist{}
if err != nil {
log.Errorf("Unexpected error when getting the project, error: %v", err)
return true, models.SevUnknown, wl
return true, vuln.Unknown, wl
}
mgr := whitelist.NewDefaultManager()
if project.ReuseSysCVEWhitelist() {
w, err := mgr.GetSys()
if err != nil {
return project.VulPrevented(), clair.ParseClairSev(project.Severity()), wl
log.Error(errors.Wrap(err, "policy checker: vulnerable policy"))
return project.VulPrevented(), vuln.Severity(project.Severity()), wl
}
wl = *w
// Use the real project ID
wl.ProjectID = project.ProjectID
} else {
w, err := mgr.Get(project.ProjectID)
if err != nil {
return project.VulPrevented(), clair.ParseClairSev(project.Severity()), wl
log.Error(errors.Wrap(err, "policy checker: vulnerable policy"))
return project.VulPrevented(), vuln.Severity(project.Severity()), wl
}
wl = *w
}
return project.VulPrevented(), clair.ParseClairSev(project.Severity()), wl
return project.VulPrevented(), vuln.Severity(project.Severity()), wl
}

5
src/core/middlewares/util/util_test.go
View File

@ -32,6 +32,7 @@ import (
notarytest "github.com/goharbor/harbor/src/common/utils/notary/test"
testutils "github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
digest "github.com/opencontainers/go-digest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -170,7 +171,7 @@ func TestPMSPolicyChecker(t *testing.T) {
Metadata: map[string]string{
models.ProMetaEnableContentTrust: "true",
models.ProMetaPreventVul: "true",
models.ProMetaSeverity: "low",
models.ProMetaSeverity: "Low",
models.ProMetaReuseSysCVEWhitelist: "false",
},
})
@ -185,7 +186,7 @@ func TestPMSPolicyChecker(t *testing.T) {
assert.True(t, contentTrustFlag)
projectVulnerableEnabled, projectVulnerableSeverity, wl := GetPolicyChecker().VulnerablePolicy("project_for_test_get_sev_low")
assert.True(t, projectVulnerableEnabled)
assert.Equal(t, projectVulnerableSeverity, models.SevLow)
assert.Equal(t, projectVulnerableSeverity, vuln.Low)
assert.Empty(t, wl.Items)
contentTrustFlag = GetPolicyChecker().ContentTrustEnabled("non_exist_project")

97
src/core/middlewares/vulnerable/handler.go
View File

@ -15,12 +15,15 @@
package vulnerable
import (
"fmt"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/util"
"github.com/goharbor/harbor/src/pkg/scan"
"net/http"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/middlewares/util"
sc "github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/goharbor/harbor/src/pkg/scan/report"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
"github.com/pkg/errors"
)
type vulnerableHandler struct {
@ -37,44 +40,86 @@ func New(next http.Handler) http.Handler {
// ServeHTTP ...
func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
imgRaw := req.Context().Value(util.ImageInfoCtxKey)
if imgRaw == nil || !config.WithClair() {
if imgRaw == nil {
vh.next.ServeHTTP(rw, req)
return
}
img, _ := req.Context().Value(util.ImageInfoCtxKey).(util.ImageInfo)
if img.Digest == "" {
// Expected artifact specified?
img, ok := imgRaw.(util.ImageInfo)
if !ok || len(img.Digest) == 0 {
vh.next.ServeHTTP(rw, req)
return
}
// Is vulnerable policy set?
projectVulnerableEnabled, projectVulnerableSeverity, wl := util.GetPolicyChecker().VulnerablePolicy(img.ProjectName)
if !projectVulnerableEnabled {
vh.next.ServeHTTP(rw, req)
return
}
vl, err := scan.VulnListByDigest(img.Digest)
// Invalid project ID
if wl.ProjectID == 0 {
err := errors.Errorf("project verification error: project %s", img.ProjectName)
vh.sendError(err, rw)
return
}
// Get the vulnerability summary
artifact := &v1.Artifact{
NamespaceID: wl.ProjectID,
Repository: img.Repository,
Tag: img.Reference,
Digest: img.Digest,
MimeType: v1.MimeTypeDockerArtifact,
}
cve := report.CVESet(wl.CVESet())
summaries, err := sc.DefaultController.GetSummary(
artifact,
[]string{v1.MimeTypeNativeReport},
report.WithCVEWhitelist(&cve),
)
if err != nil {
log.Errorf("Failed to get the vulnerability list, error: %v", err)
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", "Failed to get vulnerabilities."), http.StatusPreconditionFailed)
err = errors.Wrap(err, "middleware: vulnerable handler")
vh.sendError(err, rw)
return
}
filtered := vl.ApplyWhitelist(wl)
msg := vh.filterMsg(img, filtered)
log.Info(msg)
if int(vl.Severity()) >= int(projectVulnerableSeverity) {
log.Debugf("the image severity: %q is higher then project setting: %q, failing the response.", vl.Severity(), projectVulnerableSeverity)
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", vl.Severity(), projectVulnerableSeverity)), http.StatusPreconditionFailed)
rawSummary, ok := summaries[v1.MimeTypeNativeReport]
// No report yet?
if !ok {
err = errors.Errorf("no scan report existing for the artifact: %s:%s@%s", img.Repository, img.Reference, img.Digest)
vh.sendError(err, rw)
return
}
summary := rawSummary.(*vuln.NativeReportSummary)
// Do judgement
if summary.Severity.Code() >= projectVulnerableSeverity.Code() {
err = errors.Errorf("the pulling image severity %q is higher than or equal with the project setting %q, reject the response.", summary.Severity, projectVulnerableSeverity)
vh.sendError(err, rw)
return
}
// Print bypass CVE list
if len(summary.CVEBypassed) > 0 {
for _, cve := range summary.CVEBypassed {
log.Infof("Vulnerable policy check: bypass CVE %s", cve)
}
}
vh.next.ServeHTTP(rw, req)
}
func (vh vulnerableHandler) filterMsg(img util.ImageInfo, filtered scan.VulnerabilityList) string {
filterMsg := fmt.Sprintf("Image: %s/%s:%s, digest: %s, vulnerabilities fitered by whitelist:", img.ProjectName, img.Repository, img.Reference, img.Digest)
if len(filtered) == 0 {
filterMsg = fmt.Sprintf("%s none.", filterMsg)
}
for _, v := range filtered {
filterMsg = fmt.Sprintf("%s ID: %s, severity: %s;", filterMsg, v.ID, v.Severity)
}
return filterMsg
func (vh vulnerableHandler) sendError(err error, rw http.ResponseWriter) {
log.Error(err)
http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", err.Error()), http.StatusPreconditionFailed)
}

7
src/core/notifier/event/event.go
View File

@ -8,6 +8,7 @@ import (
"github.com/goharbor/harbor/src/core/notifier"
"github.com/goharbor/harbor/src/core/notifier/model"
notifyModel "github.com/goharbor/harbor/src/pkg/notification/model"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/pkg/errors"
)
@ -185,8 +186,8 @@ func (cd *ChartDeleteMetaData) Resolve(evt *Event) error {
// ScanImageMetaData defines meta data of image scanning event
type ScanImageMetaData struct {
JobID int64
Status string
Artifact *v1.Artifact
Status string
}
// Resolve image scanning metadata into common chart event
@ -204,7 +205,7 @@ func (si *ScanImageMetaData) Resolve(evt *Event) error {
}
data := &model.ScanImageEvent{
EventType: eventType,
JobID: si.JobID,
Artifact: si.Artifact,
OccurAt: time.Now(),
Operator: autoTriggeredOperator,
}

5
src/core/notifier/handler/notification/chart_handler_test.go
View File

@ -1,6 +1,8 @@
package notification
import (
"testing"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/notifier/model"
@ -8,7 +10,6 @@ import (
notificationModel "github.com/goharbor/harbor/src/pkg/notification/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"testing"
)
type fakedPolicyMgr struct {
@ -80,7 +81,7 @@ func TestChartPreprocessHandler_Handle(t *testing.T) {
Metadata: map[string]string{
models.ProMetaEnableContentTrust: "true",
models.ProMetaPreventVul: "true",
models.ProMetaSeverity: "low",
models.ProMetaSeverity: "Low",
models.ProMetaReuseSysCVEWhitelist: "false",
},
})

29
src/core/notifier/handler/notification/processor.go
View File

@ -12,7 +12,6 @@ import (
"github.com/goharbor/harbor/src/core/notifier/event"
notifyModel "github.com/goharbor/harbor/src/core/notifier/model"
"github.com/goharbor/harbor/src/pkg/notification"
pkgNotifyModel "github.com/goharbor/harbor/src/pkg/notification/model"
)
// getNameFromImgRepoFullName gets image name from repo full name with format `repoName/imageName`
@ -176,32 +175,4 @@ func preprocessAndSendImageHook(value interface{}) error {
}
return nil
}
// will return nil when it failed to get data
func getScanOverview(digest string, tag string, eventType string) *models.ImgScanOverview {
if len(digest) == 0 {
log.Debug("digest is nil")
return nil
}
data, err := dao.GetImgScanOverview(digest)
if err != nil {
log.Errorf("Failed to get scan result for tag:%s, digest: %s, error: %v", tag, digest, err)
}
if data == nil {
return nil
}
// Status should set by the eventType but the status from jobData in DB
if eventType == pkgNotifyModel.EventTypeScanningCompleted {
data.Status = models.JobFinished
} else {
log.Debugf("Unsetting vulnerable related historical values, job status: %s", data.Status)
data.Status = models.JobError
data.Sev = 0
data.CompOverview = nil
data.DetailsKey = ""
}
return data
}

93
src/core/notifier/handler/notification/scan_image_handler.go
View File

@ -1,15 +1,14 @@
package notification
import (
"errors"
"fmt"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/notifier/model"
"github.com/goharbor/harbor/src/pkg/notification"
"strings"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/pkg/errors"
)
// ScanImagePreprocessHandler preprocess chart event data
@ -24,56 +23,40 @@ func (si *ScanImagePreprocessHandler) Handle(value interface{}) error {
return nil
}
if value == nil {
return errors.New("empty scan image event")
}
e, ok := value.(*model.ScanImageEvent)
if !ok {
return errors.New("invalid scan image event type")
}
if e == nil {
return errors.New("empty scan image event")
policies, err := notification.PolicyMgr.GetRelatedPolices(e.Artifact.NamespaceID, e.EventType)
if err != nil {
return errors.Wrap(err, "scan preprocess handler")
}
job, err := dao.GetScanJob(e.JobID)
if err != nil {
log.Errorf("failed to find scan job[%d] for scanning webhook: %v", e.JobID, err)
return err
}
if job == nil {
log.Errorf("can't find scan job[%d] for scanning webhook", e.JobID)
return fmt.Errorf("scan job for scanning webhook not found: %d", e.JobID)
}
rs := strings.SplitN(job.Repository, "/", 2)
projectName := rs[0]
repoName := rs[1]
project, err := config.GlobalProjectMgr.Get(projectName)
if err != nil {
log.Errorf("failed to find project[%s] for scan image event: %v", projectName, err)
return err
}
if project == nil {
return fmt.Errorf("project[%s] not found", projectName)
}
policies, err := notification.PolicyMgr.GetRelatedPolices(project.ProjectID, e.EventType)
if err != nil {
log.Errorf("failed to find policy for %s event: %v", e.EventType, err)
return err
}
// if cannot find policy including event type in project, return directly
// If we cannot find policy including event type in project, return directly
if len(policies) == 0 {
log.Debugf("cannot find policy for %s event: %v", e.EventType, e)
log.Debugf("Cannot find policy for %s event: %v", e.EventType, e)
return nil
}
payload, err := constructScanImagePayload(e, job, project, projectName, repoName)
// Get project
project, err := config.GlobalProjectMgr.Get(e.Artifact.NamespaceID)
if err != nil {
return err
return errors.Wrap(err, "scan preprocess handler")
}
payload, err := constructScanImagePayload(e, project)
if err != nil {
return errors.Wrap(err, "scan preprocess handler")
}
err = sendHookWithPolicies(policies, payload, e.EventType)
if err != nil {
return err
return errors.Wrap(err, "scan preprocess handler")
}
return nil
@ -84,20 +67,22 @@ func (si *ScanImagePreprocessHandler) IsStateful() bool {
return false
}
func constructScanImagePayload(event *model.ScanImageEvent, job *models.ScanJob, project *models.Project, projectName, repoName string) (*model.Payload, error) {
func constructScanImagePayload(event *model.ScanImageEvent, project *models.Project) (*model.Payload, error) {
repoType := models.ProjectPrivate
if project.IsPublic() {
repoType = models.ProjectPublic
}
repoName := getNameFromImgRepoFullName(event.Artifact.Repository)
payload := &model.Payload{
Type: event.EventType,
OccurAt: event.OccurAt.Unix(),
EventData: &model.EventData{
Repository: &model.Repository{
Name: repoName,
Namespace: projectName,
RepoFullName: job.Repository,
Namespace: project.Name,
RepoFullName: event.Artifact.Repository,
RepoType: repoType,
},
},
@ -106,25 +91,27 @@ func constructScanImagePayload(event *model.ScanImageEvent, job *models.ScanJob,
extURL, err := config.ExtURL()
if err != nil {
return nil, fmt.Errorf("get external endpoint failed: %v", err)
return nil, errors.Wrap(err, "construct scan payload")
}
resURL, err := buildImageResourceURL(extURL, event.Artifact.Repository, event.Artifact.Tag)
if err != nil {
return nil, errors.Wrap(err, "construct scan payload")
}
resURL, _ := buildImageResourceURL(extURL, job.Repository, job.Tag)
// Add scan overview
scanOverview := getScanOverview(job.Digest, job.Tag, event.EventType)
if scanOverview == nil {
scanOverview = &models.ImgScanOverview{
JobID: job.ID,
Status: job.Status,
CreationTime: job.CreationTime,
}
summaries, err := scan.DefaultController.GetSummary(event.Artifact, []string{v1.MimeTypeNativeReport})
if err != nil {
return nil, errors.Wrap(err, "construct scan payload")
}
resource := &model.Resource{
Tag: job.Tag,
Digest: job.Digest,
Tag: event.Artifact.Tag,
Digest: event.Artifact.Digest,
ResourceURL: resURL,
ScanOverview: scanOverview,
ScanOverview: summaries,
}
payload.EventData.Resources = append(payload.EventData.Resources, resource)
return payload, nil
}

240
src/core/notifier/handler/notification/scan_image_handler_test.go
View File

@ -1,108 +1,154 @@
package notification
import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/notifier/model"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"testing"
"time"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/notifier"
"github.com/goharbor/harbor/src/core/notifier/model"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/notification"
nm "github.com/goharbor/harbor/src/pkg/notification/model"
"github.com/goharbor/harbor/src/pkg/notification/policy"
sc "github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/report"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
func TestScanImagePreprocessHandler_Handle(t *testing.T) {
PolicyMgr := notification.PolicyMgr
defer func() {
notification.PolicyMgr = PolicyMgr
}()
notification.PolicyMgr = &fakedPolicyMgr{}
// ScanImagePreprocessHandlerSuite is a test suite to test scan image preprocess handler.
type ScanImagePreprocessHandlerSuite struct {
suite.Suite
handler := &ScanImagePreprocessHandler{}
config.Init()
name := "project_for_test_scanning_event_preprocess"
id, _ := config.GlobalProjectMgr.Create(&models.Project{
Name: name,
OwnerID: 1,
Metadata: map[string]string{
models.ProMetaEnableContentTrust: "true",
models.ProMetaPreventVul: "true",
models.ProMetaSeverity: "low",
models.ProMetaReuseSysCVEWhitelist: "false",
},
})
defer func(id int64) {
if err := config.GlobalProjectMgr.Delete(id); err != nil {
t.Logf("failed to delete project %d: %v", id, err)
}
}(id)
jID, _ := dao.AddScanJob(models.ScanJob{
Status: "finished",
Repository: "project_for_test_scanning_event_preprocess/testrepo",
Tag: "v1.0.0",
Digest: "sha256:5a539a2c733ca9efcd62d4561b36ea93d55436c5a86825b8e43ce8303a7a0752",
CreationTime: time.Now(),
UpdateTime: time.Now(),
})
type args struct {
data interface{}
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "ScanImagePreprocessHandler Want Error 1",
args: args{
data: nil,
},
wantErr: true,
},
{
name: "ScanImagePreprocessHandler Want Error 2",
args: args{
data: &model.ScanImageEvent{},
},
wantErr: true,
},
{
name: "ScanImagePreprocessHandler Want Error 3",
args: args{
data: &model.ScanImageEvent{
JobID: jID + 1000,
},
},
wantErr: true,
},
{
name: "ScanImagePreprocessHandler Want Error 4",
args: args{
data: &model.ScanImageEvent{
JobID: jID,
},
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := handler.Handle(tt.args.data)
if tt.wantErr {
require.NotNil(t, err, "Error: %v", err)
return
}
assert.Nil(t, err)
})
}
om policy.Manager
pid int64
evt *model.ScanImageEvent
c sc.Controller
}
func TestScanImagePreprocessHandler_IsStateful(t *testing.T) {
handler := &ScanImagePreprocessHandler{}
assert.False(t, handler.IsStateful())
// TestScanImagePreprocessHandler is the entry point of ScanImagePreprocessHandlerSuite.
func TestScanImagePreprocessHandler(t *testing.T) {
suite.Run(t, &ScanImagePreprocessHandlerSuite{})
}
// SetupSuite prepares env for test suite.
func (suite *ScanImagePreprocessHandlerSuite) SetupSuite() {
cfg := map[string]interface{}{
common.NotificationEnable: true,
}
config.InitWithSettings(cfg)
a := &v1.Artifact{
NamespaceID: int64(1),
Repository: "library/redis",
Tag: "latest",
Digest: "digest-code",
MimeType: v1.MimeTypeDockerArtifact,
}
suite.evt = &model.ScanImageEvent{
EventType: nm.EventTypeScanningCompleted,
OccurAt: time.Now().UTC(),
Operator: "admin",
Artifact: a,
}
suite.c = sc.DefaultController
mc := &MockScanAPIController{}
var options []report.Option
s := make(map[string]interface{})
mc.On("GetSummary", a, []string{v1.MimeTypeNativeReport}, options).Return(s, nil)
sc.DefaultController = mc
suite.om = notification.PolicyMgr
mp := &fakedPolicyMgr{}
notification.PolicyMgr = mp
h := &MockHTTPHandler{}
err := notifier.Subscribe(model.WebhookTopic, h)
require.NoError(suite.T(), err)
}
// TearDownSuite clears the env for test suite.
func (suite *ScanImagePreprocessHandlerSuite) TearDownSuite() {
notification.PolicyMgr = suite.om
sc.DefaultController = suite.c
}
// TestHandle ...
func (suite *ScanImagePreprocessHandlerSuite) TestHandle() {
handler := &ScanImagePreprocessHandler{}
err := handler.Handle(suite.evt)
suite.NoError(err)
}
// Mock things
// MockScanAPIController ...
type MockScanAPIController struct {
mock.Mock
}
// Scan ...
func (msc *MockScanAPIController) Scan(artifact *v1.Artifact) error {
args := msc.Called(artifact)
return args.Error(0)
}
func (msc *MockScanAPIController) GetReport(artifact *v1.Artifact, mimeTypes []string) ([]*scan.Report, error) {
args := msc.Called(artifact, mimeTypes)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).([]*scan.Report), args.Error(1)
}
func (msc *MockScanAPIController) GetSummary(artifact *v1.Artifact, mimeTypes []string, options ...report.Option) (map[string]interface{}, error) {
args := msc.Called(artifact, mimeTypes, options)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(map[string]interface{}), args.Error(1)
}
func (msc *MockScanAPIController) GetScanLog(uuid string) ([]byte, error) {
args := msc.Called(uuid)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).([]byte), args.Error(1)
}
func (msc *MockScanAPIController) HandleJobHooks(trackID string, change *job.StatusChange) error {
args := msc.Called(trackID, change)
return args.Error(0)
}
// MockHTTPHandler ...
type MockHTTPHandler struct{}
// Handle ...
func (m *MockHTTPHandler) Handle(value interface{}) error {
return nil
}
// IsStateful ...
func (m *MockHTTPHandler) IsStateful() bool {
return false
}

11
src/core/notifier/model/event.go
View File

@ -4,6 +4,7 @@ import (
"time"
"github.com/goharbor/harbor/src/common/models"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
)
// ImageEvent is image related event data to publish
@ -35,7 +36,7 @@ type ChartEvent struct {
// ScanImageEvent is scanning image related event data to publish
type ScanImageEvent struct {
EventType string
JobID int64
Artifact *v1.Artifact
OccurAt time.Time
Operator string
}
@ -64,10 +65,10 @@ type EventData struct {
// Resource describe infos of resource triggered notification
type Resource struct {
Digest string `json:"digest,omitempty"`
Tag string `json:"tag"`
ResourceURL string `json:"resource_url,omitempty"`
ScanOverview *models.ImgScanOverview `json:"scan_overview,omitempty"`
Digest string `json:"digest,omitempty"`
Tag string `json:"tag"`
ResourceURL string `json:"resource_url,omitempty"`
ScanOverview map[string]interface{} `json:"scan_overview,omitempty"`
}
// Repository info of notification event

38
src/core/service/notifications/jobs/handler.go
View File

@ -18,8 +18,6 @@ import (
"encoding/json"
"time"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
@ -28,9 +26,12 @@ import (
jjob "github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/pkg/retention"
sc "github.com/goharbor/harbor/src/pkg/scan"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/operation/hook"
"github.com/goharbor/harbor/src/replication/policy/scheduler"
"github.com/pkg/errors"
)
var statusMap = map[string]string{
@ -92,27 +93,36 @@ func (h *Handler) Prepare() {
// HandleScan handles the webhook of scan job
func (h *Handler) HandleScan() {
log.Debugf("received san job status update event: job-%d, status-%s, track_id-%s", h.id, h.status, h.trackID)
log.Debugf("received san job status update event: job UUID: %s, status-%s, track id-%s", h.change.JobID, h.status, h.trackID)
// Trigger image scan webhook event only for JobFinished and JobError status
if h.status == models.JobFinished || h.status == models.JobError {
e := &event.Event{}
metaData := &event.ScanImageMetaData{
JobID: h.id,
Status: h.status,
}
if err := e.Build(metaData); err == nil {
if err := e.Publish(); err != nil {
log.Errorf("failed to publish image scanning event: %v", err)
}
// Get the required info from the job parameters
req, err := sc.ExtractScanReq(h.change.Metadata.Parameters)
if err != nil {
log.Error(errors.Wrap(err, "scan job hook handler: event publish"))
} else {
log.Errorf("failed to build image scanning event metadata: %v", err)
e := &event.Event{}
metaData := &event.ScanImageMetaData{
Artifact: req.Artifact,
Status: h.status,
}
if err := e.Build(metaData); err == nil {
if err := e.Publish(); err != nil {
log.Error(errors.Wrap(err, "scan job hook handler: event publish"))
}
} else {
log.Error(errors.Wrap(err, "scan job hook handler: event publish"))
}
}
}
if err := scan.DefaultController.HandleJobHooks(h.trackID, h.change); err != nil {
log.Errorf("Failed to update job status, id: %d, status: %s", h.id, h.status)
err = errors.Wrap(err, "scan job hook handler")
log.Error(err)
h.SendInternalServerError(err)
return
}
}

29
src/core/service/notifications/registry/handler.go
View File

@ -21,7 +21,6 @@ import (
"time"
"github.com/goharbor/harbor/src/common/dao"
clairdao "github.com/goharbor/harbor/src/common/dao/clair"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
@ -29,10 +28,14 @@ import (
"github.com/goharbor/harbor/src/core/config"
notifierEvt "github.com/goharbor/harbor/src/core/notifier/event"
coreutils "github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/goharbor/harbor/src/pkg/scan/api/scanner"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/adapter"
rep_event "github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication/model"
"github.com/pkg/errors"
)
// NotificationHandler handles request on /service/notifications/, which listens to registry's events.
@ -158,13 +161,16 @@ func (n *NotificationHandler) Post() {
}()
if autoScanEnabled(pro) {
last, err := clairdao.GetLastUpdate()
if err != nil {
log.Errorf("Failed to get last update from Clair DB, error: %v, the auto scan will be skipped.", err)
} else if last == 0 {
log.Infof("The Vulnerability data is not ready in Clair DB, the auto scan will be skipped, error %v", err)
} else if err := coreutils.TriggerImageScan(repository, tag); err != nil {
log.Warningf("Failed to scan image, repository: %s, tag: %s, error: %v", repository, tag, err)
artifact := &v1.Artifact{
NamespaceID: pro.ProjectID,
Repository: repository,
Tag: tag,
MimeType: v1.MimeTypeDockerArtifact,
Digest: event.Target.Digest,
}
if err := scan.DefaultController.Scan(artifact); err != nil {
log.Error(errors.Wrap(err, "registry notification: trigger scan when pushing automatically"))
}
}
}
@ -279,12 +285,13 @@ func checkEvent(event *models.Event) bool {
}
func autoScanEnabled(project *models.Project) bool {
if !config.WithClair() {
log.Debugf("Auto Scan disabled because Harbor is not deployed with Clair")
available, err := scanner.DefaultController.IsScannerAvailable(project.ProjectID)
if err != nil {
log.Error(errors.Wrap(err, "check auto scan enable"))
return false
}
return project.AutoScan()
return available && project.AutoScan()
}
// Render returns nil as it won't render any template.

83
src/core/utils/job.go
View File

@ -16,15 +16,9 @@
package utils
import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/job"
jobmodels "github.com/goharbor/harbor/src/common/job/models"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"encoding/json"
"fmt"
"sync"
)
@ -42,80 +36,3 @@ func GetJobServiceClient() job.Client {
}
return jobServiceClient
}
// TriggerImageScan triggers an image scan job on jobservice.
func TriggerImageScan(repository string, tag string) error {
repoClient, err := NewRepositoryClientForUI("harbor-core", repository)
if err != nil {
return err
}
digest, exist, err := repoClient.ManifestExist(tag)
if !exist {
return fmt.Errorf("unable to perform scan: the manifest of image %s:%s does not exist", repository, tag)
}
if err != nil {
log.Errorf("Failed to get Manifest for %s:%s", repository, tag)
return err
}
return triggerImageScan(repository, tag, digest, GetJobServiceClient())
}
func triggerImageScan(repository, tag, digest string, client job.Client) error {
id, err := dao.AddScanJob(models.ScanJob{
Repository: repository,
Digest: digest,
Tag: tag,
Status: models.JobPending,
})
if err != nil {
return err
}
err = dao.SetScanJobForImg(digest, id)
if err != nil {
return err
}
data, err := buildScanJobData(id, repository, tag, digest)
if err != nil {
return err
}
uuid, err := client.SubmitJob(data)
if err != nil {
return err
}
err = dao.SetScanJobUUID(id, uuid)
if err != nil {
log.Warningf("Failed to set UUID for scan job, ID: %d, UUID: %v, repository: %s, tag: %s", id, uuid, repository, tag)
}
return nil
}
func buildScanJobData(jobID int64, repository, tag, digest string) (*jobmodels.JobData, error) {
parms := job.ScanJobParms{
JobID: jobID,
Repository: repository,
Digest: digest,
Tag: tag,
}
parmsMap := make(map[string]interface{})
b, err := json.Marshal(parms)
if err != nil {
return nil, err
}
err = json.Unmarshal(b, &parmsMap)
if err != nil {
return nil, err
}
meta := jobmodels.JobMetadata{
JobKind: job.JobKindGeneric,
IsUnique: false,
}
data := &jobmodels.JobData{
Name: job.ImageScanJob,
Parameters: jobmodels.Parameters(parmsMap),
Metadata: &meta,
StatusHook: fmt.Sprintf("%s/service/notifications/jobs/scan/%d", config.InternalCoreURL(), jobID),
}
return data, nil
}

50
src/core/utils/job_test.go
View File

@ -1,50 +0,0 @@
package utils
import (
"fmt"
"testing"
"github.com/goharbor/harbor/src/common/job"
jobmodels "github.com/goharbor/harbor/src/common/job/models"
"github.com/goharbor/harbor/src/core/config"
"github.com/stretchr/testify/assert"
)
type jobDataTestEntry struct {
input job.ScanJobParms
expect jobmodels.JobData
}
func TestBuildScanJobData(t *testing.T) {
assert := assert.New(t)
testData := []jobDataTestEntry{
{input: job.ScanJobParms{
JobID: 123,
Digest: "sha256:abcde",
Repository: "library/ubuntu",
Tag: "latest",
},
expect: jobmodels.JobData{
Name: job.ImageScanJob,
Parameters: map[string]interface{}{
"job_int_id": 123,
"repository": "library/ubuntu",
"tag": "latest",
"digest": "sha256:abcde",
},
Metadata: &jobmodels.JobMetadata{
JobKind: job.JobKindGeneric,
IsUnique: false,
},
StatusHook: fmt.Sprintf("%s/service/notifications/jobs/scan/%d", config.InternalCoreURL(), 123),
},
},
}
for _, d := range testData {
r, err := buildScanJobData(d.input.JobID, d.input.Repository, d.input.Tag, d.input.Digest)
assert.Nil(err)
assert.Equal(d.expect.Name, r.Name)
// assert.Equal(d.expect.Parameters, r.Parameters)
assert.Equal(d.expect.StatusHook, r.StatusHook)
}
}

179
src/jobservice/job/impl/scan/clair_job.go
View File

@ -1,179 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"crypto/sha256"
"encoding/json"
"fmt"
"os"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
cjob "github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/clair"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/jobservice/job/impl/utils"
)
// ClairJob is the struct to scan Harbor's Image with Clair
type ClairJob struct {
registryURL string
secret string
tokenEndpoint string
clairEndpoint string
}
// MaxFails implements the interface in job/Interface
func (cj *ClairJob) MaxFails() uint {
return 1
}
// ShouldRetry implements the interface in job/Interface
func (cj *ClairJob) ShouldRetry() bool {
return false
}
// Validate implements the interface in job/Interface
func (cj *ClairJob) Validate(params job.Parameters) error {
return nil
}
// Run implements the interface in job/Interface
func (cj *ClairJob) Run(ctx job.Context, params job.Parameters) error {
logger := ctx.GetLogger()
if err := cj.init(ctx); err != nil {
logger.Errorf("Failed to initialize the job, error: %v", err)
return err
}
jobParms, err := transformParam(params)
if err != nil {
logger.Errorf("Failed to prepare parms for scan job, error: %v", err)
return err
}
repoClient, err := utils.NewRepositoryClientForJobservice(jobParms.Repository, cj.registryURL, cj.secret, cj.tokenEndpoint)
if err != nil {
logger.Errorf("Failed create repository client for repo: %s, error: %v", jobParms.Repository, err)
return err
}
_, _, payload, err := repoClient.PullManifest(jobParms.Tag, []string{schema2.MediaTypeManifest})
if err != nil {
logger.Errorf("Error pulling manifest for image %s:%s :%v", jobParms.Repository, jobParms.Tag, err)
return err
}
token, err := utils.GetTokenForRepo(jobParms.Repository, cj.secret, cj.tokenEndpoint)
if err != nil {
logger.Errorf("Failed to get token, error: %v", err)
return err
}
layers, err := prepareLayers(payload, cj.registryURL, jobParms.Repository, token)
if err != nil {
logger.Errorf("Failed to prepare layers, error: %v", err)
return err
}
loggerImpl, ok := logger.(*log.Logger)
if !ok {
loggerImpl = log.DefaultLogger()
}
clairClient := clair.NewClient(cj.clairEndpoint, loggerImpl)
for _, l := range layers {
logger.Infof("Scanning Layer: %s, path: %s", l.Name, l.Path)
if err := clairClient.ScanLayer(l); err != nil {
logger.Errorf("Failed to scan layer: %s, error: %v", l.Name, err)
return err
}
}
layerName := layers[len(layers)-1].Name
res, err := clairClient.GetResult(layerName)
if err != nil {
logger.Errorf("Failed to get result from Clair, error: %v", err)
return err
}
compOverview, sev := clair.TransformVuln(res)
err = dao.UpdateImgScanOverview(jobParms.Digest, layerName, sev, compOverview)
return err
}
func (cj *ClairJob) init(ctx job.Context) error {
errTpl := "failed to get required property: %s"
if v, ok := ctx.Get(common.RegistryURL); ok && len(v.(string)) > 0 {
cj.registryURL = v.(string)
} else {
return fmt.Errorf(errTpl, common.RegistryURL)
}
if v := os.Getenv("JOBSERVICE_SECRET"); len(v) > 0 {
cj.secret = v
} else {
return fmt.Errorf(errTpl, "JOBSERVICE_SECRET")
}
if v, ok := ctx.Get(common.TokenServiceURL); ok && len(v.(string)) > 0 {
cj.tokenEndpoint = v.(string)
} else {
return fmt.Errorf(errTpl, common.TokenServiceURL)
}
if v, ok := ctx.Get(common.ClairURL); ok && len(v.(string)) > 0 {
cj.clairEndpoint = v.(string)
} else {
return fmt.Errorf(errTpl, common.ClairURL)
}
return nil
}
func transformParam(params job.Parameters) (*cjob.ScanJobParms, error) {
res := cjob.ScanJobParms{}
parmsBytes, err := json.Marshal(params)
if err != nil {
return nil, err
}
err = json.Unmarshal(parmsBytes, &res)
return &res, err
}
func prepareLayers(payload []byte, registryURL, repo, tk string) ([]models.ClairLayer, error) {
layers := make([]models.ClairLayer, 0)
manifest, _, err := distribution.UnmarshalManifest(schema2.MediaTypeManifest, payload)
if err != nil {
return layers, err
}
tokenHeader := map[string]string{"Connection": "close", "Authorization": fmt.Sprintf("Bearer %s", tk)}
// form the chain by using the digests of all parent layers in the image, such that if another image is built on top of this image the layer name can be re-used.
shaChain := ""
for _, d := range manifest.References() {
if d.MediaType == schema2.MediaTypeImageConfig {
continue
}
shaChain += string(d.Digest) + "-"
l := models.ClairLayer{
Name: fmt.Sprintf("%x", sha256.Sum256([]byte(shaChain))),
Headers: tokenHeader,
Format: "Docker",
Path: utils.BuildBlobURL(registryURL, repo, string(d.Digest)),
}
if len(layers) > 0 {
l.ParentName = layers[len(layers)-1].Name
}
layers = append(layers, l)
}
return layers, nil
}

32
src/pkg/scan/api/scan/base_controller.go
View File

@ -15,10 +15,10 @@
package scan
import (
"encoding/base64"
"fmt"
"time"
"github.com/goharbor/harbor/src/common"
cj "github.com/goharbor/harbor/src/common/job"
jm "github.com/goharbor/harbor/src/common/job/models"
"github.com/goharbor/harbor/src/common/rbac"
@ -52,6 +52,9 @@ type uuidGenerator func() (string, error)
// utility methods.
type configGetter func(cfg string) (string, error)
// jcGetter is a func template which is used to get the job service client.
type jcGetter func() cj.Client
// basicController is default implementation of api.Controller interface
type basicController struct {
// Manage the scan report records
@ -61,7 +64,7 @@ type basicController struct {
// Robot account controller
rc robot.Controller
// Job service client
jc cj.Client
jc jcGetter
// UUID generator
uuid uuidGenerator
// Configuration getter func
@ -78,7 +81,9 @@ func NewController() Controller {
// Refer to the default robot account controller
rc: robot.RobotCtr,
// Refer to the default job service client
jc: cj.GlobalClient,
jc: func() cj.Client {
return cj.GlobalClient
},
// Generate UUID with uuid lib
uuid: func() (string, error) {
aUUID, err := uuid.NewUUID()
@ -226,7 +231,7 @@ func (bc *basicController) GetReport(artifact *v1.Artifact, mimeTypes []string)
}
// GetSummary ...
func (bc *basicController) GetSummary(artifact *v1.Artifact, mimeTypes []string) (map[string]interface{}, error) {
func (bc *basicController) GetSummary(artifact *v1.Artifact, mimeTypes []string, options ...report.Option) (map[string]interface{}, error) {
if artifact == nil {
return nil, errors.New("no way to get report summaries for nil artifact")
}
@ -239,7 +244,7 @@ func (bc *basicController) GetSummary(artifact *v1.Artifact, mimeTypes []string)
summaries := make(map[string]interface{}, len(rps))
for _, rp := range rps {
sum, err := report.GenerateSummary(rp)
sum, err := report.GenerateSummary(rp, options...)
if err != nil {
return nil, err
}
@ -276,7 +281,7 @@ func (bc *basicController) GetScanLog(uuid string) ([]byte, error) {
}
// Job log
return bc.jc.GetJobLog(sr.JobID)
return bc.jc().GetJobLog(sr.JobID)
}
// HandleJobHooks ...
@ -333,14 +338,14 @@ func (bc *basicController) makeRobotAccount(pid int64, repository string, ttl in
logger.Warningf("repository %s and expire time %d are not supported by robot controller", repository, expireAt)
resource := fmt.Sprintf("/project/%d/repository", pid)
resource := rbac.NewProjectNamespace(pid).Resource(rbac.ResourceRepository)
access := []*rbac.Policy{{
Resource: rbac.Resource(resource),
Action: "pull",
Resource: resource,
Action: rbac.ActionPull,
}}
account := &model.RobotCreate{
Name: fmt.Sprintf("%s%s", common.RobotPrefix, UUID),
Name: UUID,
Description: "for scan",
ProjectID: pid,
Access: access,
@ -351,7 +356,10 @@ func (bc *basicController) makeRobotAccount(pid int64, repository string, ttl in
return "", errors.Wrap(err, "scan controller: make robot account")
}
return rb.Token, nil
basic := fmt.Sprintf("%s:%s", rb.Name, rb.Token)
encoded := base64.StdEncoding.EncodeToString([]byte(basic))
return fmt.Sprintf("Basic %s", encoded), nil
}
// launchScanJob launches a job to run scan
@ -407,5 +415,5 @@ func (bc *basicController) launchScanJob(trackID string, artifact *v1.Artifact,
StatusHook: hookURL,
}
return bc.jc.SubmitJob(j)
return bc.jc().SubmitJob(j)
}

20
src/pkg/scan/api/scan/base_controller_test.go
View File

@ -21,14 +21,13 @@ import (
"time"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/pkg/robot/model"
cj "github.com/goharbor/harbor/src/common/job"
cjm "github.com/goharbor/harbor/src/common/job/models"
jm "github.com/goharbor/harbor/src/common/job/models"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/robot/model"
sca "github.com/goharbor/harbor/src/pkg/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
@ -214,8 +213,10 @@ func (suite *ControllerTestSuite) SetupSuite() {
suite.c = &basicController{
manager: mgr,
sc: sc,
jc: jc,
rc: rc,
jc: func() cj.Client {
return jc
},
rc: rc,
uuid: func() (string, error) {
return "the-uuid-123", nil
},
@ -449,6 +450,13 @@ func (msc *MockScannerController) GetMetadata(registrationUUID string) (*v1.Scan
return args.Get(0).(*v1.ScannerAdapterMetadata), args.Error(1)
}
// IsScannerAvailable ...
func (msc *MockScannerController) IsScannerAvailable(projectID int64) (bool, error) {
args := msc.Called(projectID)
return args.Bool(0), args.Error(1)
}
// MockJobServiceClient ...
type MockJobServiceClient struct {
mock.Mock

8
src/pkg/scan/api/scan/controller.go
View File

@ -17,6 +17,7 @@ package scan
import (
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/report"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
)
@ -47,13 +48,14 @@ type Controller interface {
// GetSummary gets the summaries of the reports with given types.
//
// Arguments:
// artifact *v1.Artifact : the scanned artifact
// mimeTypes []string : the mime types of the reports
// artifact *v1.Artifact : the scanned artifact
// mimeTypes []string : the mime types of the reports
// options ...report.Option : optional report options, specify if needed
//
// Returns:
// map[string]interface{} : report summaries indexed by mime types
// error : non nil error if any errors occurred
GetSummary(artifact *v1.Artifact, mimeTypes []string) (map[string]interface{}, error)
GetSummary(artifact *v1.Artifact, mimeTypes []string, options ...report.Option) (map[string]interface{}, error)
// Get the scan log for the specified artifact with the given digest
//

37
src/pkg/scan/api/scanner/base_controller.go
View File

@ -230,7 +230,6 @@ func (bc *basicController) GetRegistrationByProject(projectID int64) (*scanner.R
}
// Ping ...
// TODO: ADD UT CASES
func (bc *basicController) Ping(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) {
if registration == nil {
return nil, errors.New("nil registration to ping")
@ -289,7 +288,6 @@ func (bc *basicController) Ping(registration *scanner.Registration) (*v1.Scanner
}
// GetMetadata ...
// TODO: ADD UT CASES
func (bc *basicController) GetMetadata(registrationUUID string) (*v1.ScannerAdapterMetadata, error) {
if len(registrationUUID) == 0 {
return nil, errors.New("empty registration uuid")
@ -302,3 +300,38 @@ func (bc *basicController) GetMetadata(registrationUUID string) (*v1.ScannerAdap
return bc.Ping(r)
}
// IsScannerAvailable ...
// TODO: This method will be removed if we change the method of getting project
// registration without ping later.
func (bc *basicController) IsScannerAvailable(projectID int64) (bool, error) {
if projectID == 0 {
return false, errors.New("invalid project ID")
}
// First, get it from the project metadata
m, err := bc.proMetaMgr.Get(projectID, proScannerMetaKey)
if err != nil {
return false, errors.Wrap(err, "api controller: check scanner availability")
}
var registration *scanner.Registration
if len(m) > 0 {
if registrationID, ok := m[proScannerMetaKey]; ok && len(registrationID) > 0 {
registration, err = bc.manager.Get(registrationID)
if err != nil {
return false, errors.Wrap(err, "api controller: check scanner availability")
}
}
}
if registration == nil {
// Second, get the default one
registration, err = bc.manager.GetDefault()
if err != nil {
return false, errors.Wrap(err, "api controller: check scanner availability")
}
}
return registration != nil && !registration.Disabled, nil
}

21
src/pkg/scan/api/scanner/base_controller_test.go
View File

@ -17,11 +17,10 @@ package scanner
import (
"testing"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
@ -225,6 +224,24 @@ func (suite *ControllerTestSuite) TestGetRegistrationByProject() {
assert.Equal(suite.T(), "forUT", r.Name)
}
// TestPing ...
func (suite *ControllerTestSuite) TestPing() {
meta, err := suite.c.Ping(suite.sample)
require.NoError(suite.T(), err)
suite.NotNil(meta)
}
// TestGetMetadata ...
func (suite *ControllerTestSuite) TestGetMetadata() {
suite.sample.UUID = "uuid"
suite.mMgr.On("Get", "uuid").Return(suite.sample, nil)
meta, err := suite.c.GetMetadata(suite.sample.UUID)
require.NoError(suite.T(), err)
suite.NotNil(meta)
suite.Equal(1, len(meta.Capabilities))
}
// MockScannerManager is mock of the scanner manager
type MockScannerManager struct {
mock.Mock

10
src/pkg/scan/api/scanner/controller.go
View File

@ -135,4 +135,14 @@ type Controller interface {
// *v1.ScannerAdapterMetadata : metadata returned by the scanner if successfully ping
// error : non nil error if any errors occurred
GetMetadata(registrationUUID string) (*v1.ScannerAdapterMetadata, error)
// IsScannerAvailable checks if the scanner is available for the specified project.
//
// Arguments:
// projectID int64 : the ID of the given project
//
// Returns:
// bool : the scanner if configured for the specified project
// error : non nil error if any errors occurred
IsScannerAvailable(projectID int64) (bool, error)
}

57
src/pkg/scan/job.go
View File

@ -93,7 +93,7 @@ func (j *Job) Validate(params job.Parameters) error {
return errors.Wrap(err, "job validate")
}
if _, err := extractScanReq(params); err != nil {
if _, err := ExtractScanReq(params); err != nil {
return errors.Wrap(err, "job validate")
}
@ -111,7 +111,7 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error {
// Ignore errors as they have been validated already
r, _ := extractRegistration(params)
req, _ := extractScanReq(params)
req, _ := ExtractScanReq(params)
mimes, _ := extractMimeTypes(params)
// Print related infos to log
@ -230,6 +230,33 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error {
return err
}
// ExtractScanReq extracts the scan request from the job parameters.
func ExtractScanReq(params job.Parameters) (*v1.ScanRequest, error) {
v, ok := params[JobParameterRequest]
if !ok {
return nil, errors.Errorf("missing job parameter '%s'", JobParameterRequest)
}
jsonData, ok := v.(string)
if !ok {
return nil, errors.Errorf(
"malformed job parameter '%s', expecting string but got %s",
JobParameterRequest,
reflect.TypeOf(v).String(),
)
}
req := &v1.ScanRequest{}
if err := req.FromJSON(jsonData); err != nil {
return nil, err
}
if err := req.Validate(); err != nil {
return nil, err
}
return req, nil
}
func logAndWrapError(logger logger.Interface, err error, message string) error {
e := errors.Wrap(err, message)
logger.Error(e)
@ -269,32 +296,6 @@ func removeAuthInfo(sr *v1.ScanRequest) string {
return str
}
func extractScanReq(params job.Parameters) (*v1.ScanRequest, error) {
v, ok := params[JobParameterRequest]
if !ok {
return nil, errors.Errorf("missing job parameter '%s'", JobParameterRequest)
}
jsonData, ok := v.(string)
if !ok {
return nil, errors.Errorf(
"malformed job parameter '%s', expecting string but got %s",
JobParameterRequest,
reflect.TypeOf(v).String(),
)
}
req := &v1.ScanRequest{}
if err := req.FromJSON(jsonData); err != nil {
return nil, err
}
if err := req.Validate(); err != nil {
return nil, err
}
return req, nil
}
func extractRegistration(params job.Parameters) (*scanner.Registration, error) {
v, ok := params[JobParamRegistration]
if !ok {

63
src/pkg/scan/report/summary.go
View File

@ -24,6 +24,32 @@ import (
"github.com/pkg/errors"
)
// CVESet defines the CVE whitelist with a hash set way for easy query.
type CVESet map[string]struct{}
// Contains checks whether the specified CVE is in the set or not.
func (cs CVESet) Contains(cve string) bool {
_, ok := cs[cve]
return ok
}
// Options provides options for getting the report w/ summary.
type Options struct {
// If it is set, the returned summary will not count the CVEs in the list in.
CVEWhitelist CVESet
}
// Option for getting the report w/ summary with func template way.
type Option func(options *Options)
// WithCVEWhitelist is an option of setting CVE whitelist.
func WithCVEWhitelist(set *CVESet) Option {
return func(options *Options) {
options.CVEWhitelist = *set
}
}
// SupportedGenerators declares mappings between mime type and summary generator func.
var SupportedGenerators = map[string]SummaryGenerator{
v1.MimeTypeNativeReport: GenerateNativeSummary,
@ -31,26 +57,34 @@ var SupportedGenerators = map[string]SummaryGenerator{
// GenerateSummary is a helper function to generate report
// summary based on the given report.
func GenerateSummary(r *scan.Report) (interface{}, error) {
func GenerateSummary(r *scan.Report, options ...Option) (interface{}, error) {
g, ok := SupportedGenerators[r.MimeType]
if !ok {
return nil, errors.Errorf("no generator bound with mime type %s", r.MimeType)
}
return g(r)
return g(r, options...)
}
// SummaryGenerator is a func template which used to generated report
// summary for relevant mime type.
type SummaryGenerator func(r *scan.Report) (interface{}, error)
type SummaryGenerator func(r *scan.Report, options ...Option) (interface{}, error)
// GenerateNativeSummary generates the report summary for the native report.
func GenerateNativeSummary(r *scan.Report) (interface{}, error) {
func GenerateNativeSummary(r *scan.Report, options ...Option) (interface{}, error) {
ops := &Options{}
for _, op := range options {
op(ops)
}
sum := &vuln.NativeReportSummary{}
sum.ReportID = r.UUID
sum.StartTime = r.StartTime
sum.EndTime = r.EndTime
sum.Duration = r.EndTime.Unix() - r.EndTime.Unix()
if len(ops.CVEWhitelist) > 0 {
sum.CVEBypassed = make([]string, 0)
}
sum.ScanStatus = job.ErrorStatus.String()
if job.Status(r.Status).Code() != -1 {
@ -84,14 +118,35 @@ func GenerateNativeSummary(r *scan.Report) (interface{}, error) {
Summary: make(vuln.SeveritySummary),
}
overallSev := vuln.None
for _, v := range rp.Vulnerabilities {
if len(ops.CVEWhitelist) > 0 && ops.CVEWhitelist.Contains(v.ID) {
// If whitelist is set, then check if we need to bypass it
// Reduce the total
vsum.Total--
// Append the by passed CVEs specified in the whitelist
sum.CVEBypassed = append(sum.CVEBypassed, v.ID)
continue
}
if num, ok := vsum.Summary[v.Severity]; ok {
vsum.Summary[v.Severity] = num + 1
} else {
vsum.Summary[v.Severity] = 1
}
// Update the overall severity if necessary
if v.Severity.Code() > overallSev.Code() {
overallSev = v.Severity
}
}
sum.Summary = vsum
// Override the overall severity of the filtered list if needed.
if len(ops.CVEWhitelist) > 0 {
sum.Severity = overallSev
}
return sum, nil
}

131
src/pkg/scan/report/summary_test.go Normal file
View File

@ -0,0 +1,131 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package report
import (
"encoding/json"
"testing"
"time"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
// SummaryTestSuite is test suite for testing report summary.
type SummaryTestSuite struct {
suite.Suite
r *scan.Report
}
// TestSummary is the entry point of SummaryTestSuite.
func TestSummary(t *testing.T) {
suite.Run(t, &SummaryTestSuite{})
}
// SetupSuite prepares testing env for the testing suite.
func (suite *SummaryTestSuite) SetupSuite() {
rp := vuln.Report{
GeneratedAt: time.Now().UTC().String(),
Scanner: &v1.Scanner{
Name: "Clair",
Vendor: "Harbor",
Version: "0.1.0",
},
Severity: vuln.High,
Vulnerabilities: []*vuln.VulnerabilityItem{
{
ID: "2019-0980-0909",
Package: "dpkg",
Version: "0.9.1",
FixVersion: "0.9.2",
Severity: vuln.High,
Description: "mock one",
Links: []string{"https://vuln1.com"},
},
{
ID: "2019-0980-1010",
Package: "dpkg",
Version: "5.0.1",
FixVersion: "5.0.2",
Severity: vuln.Medium,
Description: "mock two",
Links: []string{"https://vuln2.com"},
},
},
}
jsonData, err := json.Marshal(rp)
require.NoError(suite.T(), err)
suite.r = &scan.Report{
ID: 1,
UUID: "r-uuid-001",
Digest: "digest-code",
RegistrationUUID: "reg-uuid-001",
MimeType: v1.MimeTypeNativeReport,
JobID: "job-uuid-001",
TrackID: "track-uuid-001",
Status: "Success",
StatusCode: 3,
StatusRevision: 10000,
Report: string(jsonData),
}
}
// TestSummaryGenerateSummaryNoOptions ...
func (suite *SummaryTestSuite) TestSummaryGenerateSummaryNoOptions() {
summaries, err := GenerateSummary(suite.r)
require.NoError(suite.T(), err)
require.NotNil(suite.T(), summaries)
nativeSummary, ok := summaries.(*vuln.NativeReportSummary)
require.Equal(suite.T(), true, ok)
suite.Equal(vuln.High, nativeSummary.Severity)
suite.Nil(nativeSummary.CVEBypassed)
suite.Equal(2, nativeSummary.Summary.Total)
}
// TestSummaryGenerateSummaryWithOptions ...
func (suite *SummaryTestSuite) TestSummaryGenerateSummaryWithOptions() {
cveSet := make(CVESet)
cveSet["2019-0980-0909"] = struct{}{}
summaries, err := GenerateSummary(suite.r, WithCVEWhitelist(&cveSet))
require.NoError(suite.T(), err)
require.NotNil(suite.T(), summaries)
nativeSummary, ok := summaries.(*vuln.NativeReportSummary)
require.Equal(suite.T(), true, ok)
suite.Equal(vuln.Medium, nativeSummary.Severity)
suite.Equal(1, len(nativeSummary.CVEBypassed))
suite.Equal(1, nativeSummary.Summary.Total)
}
// TestSummaryGenerateSummaryWrongMime ...
func (suite *SummaryTestSuite) TestSummaryGenerateSummaryWrongMime() {
suite.r.MimeType = "wrong-mime"
defer func() {
suite.r.MimeType = v1.MimeTypeNativeReport
}()
_, err := GenerateSummary(suite.r)
require.Error(suite.T(), err)
}

136
src/pkg/scan/vuln.go
View File

@ -1,136 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"fmt"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/clair"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"reflect"
)
// VulnerabilityItem represents a vulnerability reported by scanner
type VulnerabilityItem struct {
ID string `json:"id"`
Severity models.Severity `json:"severity"`
Pkg string `json:"package"`
Version string `json:"version"`
Description string `json:"description"`
Link string `json:"link"`
Fixed string `json:"fixedVersion,omitempty"`
}
// VulnerabilityList is a list of vulnerabilities, which should be scanner-agnostic
type VulnerabilityList []VulnerabilityItem
// ApplyWhitelist filters out the CVE defined in the whitelist in the parm.
// It returns the items that are filtered for the caller to track or log.
func (vl *VulnerabilityList) ApplyWhitelist(whitelist models.CVEWhitelist) VulnerabilityList {
filtered := VulnerabilityList{}
if whitelist.IsExpired() {
log.Info("The input whitelist is expired, skip filtering")
return filtered
}
s := whitelist.CVESet()
r := (*vl)[:0]
for _, v := range *vl {
if _, ok := s[v.ID]; ok {
log.Debugf("Filtered Vulnerability in whitelist, CVE ID: %s, severity: %s", v.ID, v.Severity)
filtered = append(filtered, v)
} else {
r = append(r, v)
}
}
val := reflect.ValueOf(vl)
val.Elem().SetLen(len(r))
return filtered
}
// Severity returns the highest severity of the vulnerabilities in the list
func (vl *VulnerabilityList) Severity() models.Severity {
s := models.SevNone
for _, v := range *vl {
if v.Severity > s {
s = v.Severity
}
}
return s
}
// HasCVE returns whether the vulnerability list has the vulnerability with CVE ID in the parm
func (vl *VulnerabilityList) HasCVE(id string) bool {
for _, v := range *vl {
if v.ID == id {
return true
}
}
return false
}
// VulnListFromClairResult transforms the returned value of Clair API to a VulnerabilityList
func VulnListFromClairResult(layerWithVuln *models.ClairLayerEnvelope) VulnerabilityList {
res := VulnerabilityList{}
if layerWithVuln == nil {
return res
}
l := layerWithVuln.Layer
if l == nil {
return res
}
features := l.Features
if features == nil {
return res
}
for _, f := range features {
vulnerabilities := f.Vulnerabilities
if vulnerabilities == nil {
continue
}
for _, v := range vulnerabilities {
vItem := VulnerabilityItem{
ID: v.Name,
Pkg: f.Name,
Version: f.Version,
Severity: clair.ParseClairSev(v.Severity),
Fixed: v.FixedBy,
Link: v.Link,
Description: v.Description,
}
res = append(res, vItem)
}
}
return res
}
// VulnListByDigest returns the VulnerabilityList based on the scan result of artifact with the digest in the parm
func VulnListByDigest(digest string) (VulnerabilityList, error) {
var res VulnerabilityList
overview, err := dao.GetImgScanOverview(digest)
if err != nil {
return res, err
}
if overview == nil || len(overview.DetailsKey) == 0 {
return res, fmt.Errorf("unable to get the scan result for digest: %s, the artifact is not scanned", digest)
}
c := clair.NewClient(config.ClairEndpoint(), nil)
clairRes, err := c.GetResult(overview.DetailsKey)
if err != nil {
return res, fmt.Errorf("failed to get scan result from Clair, error: %v", err)
}
return VulnListFromClairResult(clairRes), nil
}

31
src/pkg/scan/vuln/severity.go
View File

@ -15,15 +15,19 @@
package vuln
const (
// None - only used to mark the overall severity of the scanned artifacts,
// means no vulnerabilities attached with the artifacts,
// (might be bypassed by the CVE whitelist).
None Severity = "None"
// Unknown - either a security problem that has not been assigned to a priority yet or
// a priority that the scanner did not recognize.
Unknown Severity = "Unknown"
// Low - a security problem, but is hard to exploit due to environment, requires a
// user-assisted attack, a small install base, or does very little damage.
Low Severity = "Low"
// Negligible - technically a security problem, but is only theoretical in nature, requires
// a very special situation, has almost no install base, or does no real damage.
Negligible Severity = "Negligible"
// Low - a security problem, but is hard to exploit due to environment, requires a
// user-assisted attack, a small install base, or does very little damage.
Low Severity = "Low"
// Medium - a real security problem, and is exploitable for many people. Includes network
// daemon denial of service attacks, cross-site scripting, and gaining user privileges.
Medium Severity = "Medium"
@ -37,3 +41,24 @@ const (
// Severity is a standard scale for measuring the severity of a vulnerability.
type Severity string
// Code returns the int code of the severity for comparing.
func (s Severity) Code() int {
switch s {
case None:
return 0
case Negligible:
return 1
case Low:
return 2
case Medium:
return 3
case High:
return 4
case Critical:
return 5
default:
// Assign the highest code to the unknown severity to provide more secure protection.
return 99
}
}

15
src/pkg/scan/vuln/summary.go
View File

@ -21,13 +21,14 @@ import (
// NativeReportSummary is the default supported scan report summary model.
// Generated based on the report with v1.MimeTypeNativeReport mime type.
type NativeReportSummary struct {
ReportID string `json:"report_id"`
ScanStatus string `json:"scan_status"`
Severity Severity `json:"severity"`
Duration int64 `json:"duration"`
Summary *VulnerabilitySummary `json:"summary"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
ReportID string `json:"report_id"`
ScanStatus string `json:"scan_status"`
Severity Severity `json:"severity"`
Duration int64 `json:"duration"`
Summary *VulnerabilitySummary `json:"summary"`
CVEBypassed []string `json:"-"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
}
// VulnerabilitySummary contains the total number of the found vulnerabilities number

178
src/pkg/scan/vuln_test.go
View File

@ -1,178 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/stretchr/testify/assert"
"os"
"testing"
)
var (
past = int64(1561967574)
vulnList1 = VulnerabilityList{}
vulnList2 = VulnerabilityList{
{ID: "CVE-2018-10754",
Severity: models.SevLow,
Pkg: "ncurses",
Version: "6.0+20161126-1+deb9u2",
},
{
ID: "CVE-2018-6485",
Severity: models.SevHigh,
Pkg: "glibc",
Version: "2.24-11+deb9u4",
},
}
whiteList1 = models.CVEWhitelist{
ExpiresAt: &past,
Items: []models.CVEWhitelistItem{
{CVEID: "CVE-2018-6485"},
},
}
whiteList2 = models.CVEWhitelist{
Items: []models.CVEWhitelistItem{
{CVEID: "CVE-2018-6485"},
},
}
whiteList3 = models.CVEWhitelist{
Items: []models.CVEWhitelistItem{
{CVEID: "CVE-2018-6485"},
{CVEID: "CVE-2018-10754"},
{CVEID: "CVE-2019-12817"},
},
}
)
func TestMain(m *testing.M) {
dao.PrepareTestForPostgresSQL()
os.Exit(m.Run())
}
func TestVulnerabilityList_HasCVE(t *testing.T) {
cases := []struct {
input VulnerabilityList
cve string
result bool
}{
{
input: vulnList1,
cve: "CVE-2018-10754",
result: false,
},
{
input: vulnList2,
cve: "CVE-2018-10754",
result: true,
},
}
for _, c := range cases {
assert.Equal(t, c.result, c.input.HasCVE(c.cve))
}
}
func TestVulnerabilityList_Severity(t *testing.T) {
cases := []struct {
input VulnerabilityList
expect models.Severity
}{
{
input: vulnList1,
expect: models.SevNone,
}, {
input: vulnList2,
expect: models.SevHigh,
},
}
for _, c := range cases {
assert.Equal(t, c.expect, c.input.Severity())
}
}
func TestVulnerabilityList_ApplyWhitelist(t *testing.T) {
cases := []struct {
vl VulnerabilityList
wl models.CVEWhitelist
expectFiltered VulnerabilityList
expectSev models.Severity
}{
{
vl: vulnList2,
wl: whiteList1,
expectFiltered: VulnerabilityList{},
expectSev: models.SevHigh,
},
{
vl: vulnList2,
wl: whiteList2,
expectFiltered: VulnerabilityList{
{
ID: "CVE-2018-6485",
Severity: models.SevHigh,
Pkg: "glibc",
Version: "2.24-11+deb9u4",
},
},
expectSev: models.SevLow,
},
{
vl: vulnList1,
wl: whiteList3,
expectFiltered: VulnerabilityList{},
expectSev: models.SevNone,
},
{
vl: vulnList2,
wl: whiteList3,
expectFiltered: VulnerabilityList{
{ID: "CVE-2018-10754",
Severity: models.SevLow,
Pkg: "ncurses",
Version: "6.0+20161126-1+deb9u2",
},
{
ID: "CVE-2018-6485",
Severity: models.SevHigh,
Pkg: "glibc",
Version: "2.24-11+deb9u4",
},
},
expectSev: models.SevNone,
},
}
for _, c := range cases {
filtered := c.vl.ApplyWhitelist(c.wl)
assert.Equal(t, c.expectFiltered, filtered)
assert.Equal(t, c.vl.Severity(), c.expectSev)
}
}
func TestVulnListByDigest(t *testing.T) {
_, err := VulnListByDigest("notexist")
assert.NotNil(t, err)
}
func TestVulnListFromClairResult(t *testing.T) {
l := VulnListFromClairResult(nil)
assert.Equal(t, VulnerabilityList{}, l)
lv := &models.ClairLayerEnvelope{
Layer: nil,
Error: nil,
}
l2 := VulnListFromClairResult(lv)
assert.Equal(t, VulnerabilityList{}, l2)
}