add list project arifacts API (#20803)

* add list project arifacts API

This API supports listing all artifacts belonging to a specified project. It also allows fetching the latest artifact
in each repositry, with the option to filter by either media_type or artifact_type.

Signed-off-by: wang yan <wangyan@vmware.com>

* resolve the comments

Signed-off-by: wang yan <wangyan@vmware.com>

---------

Signed-off-by: wang yan <wangyan@vmware.com>
This commit is contained in:
Wang Yan 2024-08-06 18:29:13 +08:00 committed by GitHub
parent 5deedf4c7c
commit b7f1b59495
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 462 additions and 10 deletions

View File

@ -1548,6 +1548,88 @@ paths:
$ref: '#/responses/409'
'500':
$ref: '#/responses/500'
/projects/{project_name_or_id}/artifacts:
get:
summary: List artifacts
description: List artifacts of the specified project
tags:
- project
operationId: listArtifactsOfProject
parameters:
- $ref: '#/parameters/requestId'
- $ref: '#/parameters/isResourceName'
- $ref: '#/parameters/projectNameOrId'
- $ref: '#/parameters/query'
- $ref: '#/parameters/sort'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- $ref: '#/parameters/acceptVulnerabilities'
- name: with_tag
in: query
description: Specify whether the tags are included inside the returning artifacts
type: boolean
required: false
default: true
- name: with_label
in: query
description: Specify whether the labels are included inside the returning artifacts
type: boolean
required: false
default: false
- name: with_scan_overview
in: query
description: Specify whether the scan overview is included inside the returning artifacts
type: boolean
required: false
default: false
- name: with_sbom_overview
in: query
description: Specify whether the SBOM overview is included in returning artifacts, when this option is true, the SBOM overview will be included in the response
type: boolean
required: false
default: false
- name: with_immutable_status
in: query
description: Specify whether the immutable status is included inside the tags of the returning artifacts. Only works when setting "with_immutable_status=true"
type: boolean
required: false
default: false
- name: with_accessory
in: query
description: Specify whether the accessories are included of the returning artifacts. Only works when setting "with_accessory=true"
type: boolean
required: false
default: false
- name: latest_in_repository
in: query
description: Specify whether only the latest pushed artifact of each repository is included inside the returning artifacts. Only works when either artifact_type or media_type is included in the query.
type: boolean
required: false
default: false
responses:
'200':
description: Success
headers:
X-Total-Count:
description: The total count of artifacts
type: integer
Link:
description: Link refers to the previous page and next page
type: string
schema:
type: array
items:
$ref: '#/definitions/Artifact'
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
'/projects/{project_name_or_id}/scanner':
get:
summary: Get project level scanner
@ -6586,6 +6668,9 @@ definitions:
manifest_media_type:
type: string
description: The manifest media type of the artifact
artifact_type:
type: string
description: The artifact_type in the manifest of the artifact
project_id:
type: integer
format: int64
@ -6594,6 +6679,9 @@ definitions:
type: integer
format: int64
description: The ID of the repository that the artifact belongs to
repository_name:
type: string
description: The name of the repository that the artifact belongs to
digest:
type: string
description: The digest of the artifact

View File

@ -118,6 +118,8 @@ type Controller interface {
Walk(ctx context.Context, root *Artifact, walkFn func(*Artifact) error, option *Option) error
// HasUnscannableLayer check artifact with digest if has unscannable layer
HasUnscannableLayer(ctx context.Context, dgst string) (bool, error)
// ListWithLatest list the artifacts when the latest_in_repository in the query was set
ListWithLatest(ctx context.Context, query *q.Query, option *Option) (artifacts []*Artifact, err error)
}
// NewController creates an instance of the default artifact controller
@ -782,3 +784,16 @@ func (c *controller) HasUnscannableLayer(ctx context.Context, dgst string) (bool
}
return false, nil
}
// ListWithLatest ...
func (c *controller) ListWithLatest(ctx context.Context, query *q.Query, option *Option) (artifacts []*Artifact, err error) {
arts, err := c.artMgr.ListWithLatest(ctx, query)
if err != nil {
return nil, err
}
var res []*Artifact
for _, art := range arts {
res = append(res, c.assembleArtifact(ctx, art, option))
}
return res, nil
}

View File

@ -323,6 +323,44 @@ func (c *controllerTestSuite) TestList() {
c.Equal(0, len(artifacts[0].Accessories))
}
func (c *controllerTestSuite) TestListWithLatest() {
query := &q.Query{}
option := &Option{
WithTag: true,
WithAccessory: true,
}
c.artMgr.On("ListWithLatest", mock.Anything, mock.Anything).Return([]*artifact.Artifact{
{
ID: 1,
RepositoryID: 1,
},
}, nil)
c.tagCtl.On("List").Return([]*tag.Tag{
{
Tag: model_tag.Tag{
ID: 1,
RepositoryID: 1,
ArtifactID: 1,
Name: "latest",
},
},
}, nil)
c.repoMgr.On("Get", mock.Anything, mock.Anything).Return(&repomodel.RepoRecord{
Name: "library/hello-world",
}, nil)
c.repoMgr.On("List", mock.Anything, mock.Anything).Return([]*repomodel.RepoRecord{
{RepositoryID: 1, Name: "library/hello-world"},
}, nil)
c.accMgr.On("List", mock.Anything, mock.Anything).Return([]accessorymodel.Accessory{}, nil)
artifacts, err := c.ctl.ListWithLatest(nil, query, option)
c.Require().Nil(err)
c.Require().Len(artifacts, 1)
c.Equal(int64(1), artifacts[0].ID)
c.Require().Len(artifacts[0].Tags, 1)
c.Equal(int64(1), artifacts[0].Tags[0].ID)
c.Equal(0, len(artifacts[0].Accessories))
}
func (c *controllerTestSuite) TestGet() {
c.artMgr.On("Get", mock.Anything, mock.Anything, mock.Anything).Return(&artifact.Artifact{
ID: 1,

View File

@ -102,8 +102,9 @@ type AdditionLink struct {
// Option is used to specify the properties returned when listing/getting artifacts
type Option struct {
WithTag bool
TagOption *tag.Option // only works when WithTag is set to true
WithLabel bool
WithAccessory bool
WithTag bool
TagOption *tag.Option // only works when WithTag is set to true
WithLabel bool
WithAccessory bool
LatestInRepository bool
}

View File

@ -54,6 +54,8 @@ type DAO interface {
DeleteReference(ctx context.Context, id int64) (err error)
// DeleteReferences deletes the references referenced by the artifact specified by parent ID
DeleteReferences(ctx context.Context, parentID int64) (err error)
// ListWithLatest ...
ListWithLatest(ctx context.Context, query *q.Query) (artifacts []*Artifact, err error)
}
const (
@ -282,6 +284,53 @@ func (d *dao) DeleteReferences(ctx context.Context, parentID int64) error {
return err
}
func (d *dao) ListWithLatest(ctx context.Context, query *q.Query) (artifacts []*Artifact, err error) {
ormer, err := orm.FromContext(ctx)
if err != nil {
return nil, err
}
sql := `SELECT a.*
FROM artifact a
JOIN (
SELECT repository_name, MAX(push_time) AS latest_push_time
FROM artifact
WHERE project_id = ? and %s = ?
GROUP BY repository_name
) latest ON a.repository_name = latest.repository_name AND a.push_time = latest.latest_push_time`
queryParam := make([]interface{}, 0)
var ok bool
var pid interface{}
if pid, ok = query.Keywords["ProjectID"]; !ok {
return nil, errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage(`the value of "ProjectID" must be set`)
}
queryParam = append(queryParam, pid)
var attributionValue interface{}
if attributionValue, ok = query.Keywords["media_type"]; ok {
sql = fmt.Sprintf(sql, "media_type")
} else if attributionValue, ok = query.Keywords["artifact_type"]; ok {
sql = fmt.Sprintf(sql, "artifact_type")
}
if attributionValue == "" {
return nil, errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage(`the value of "media_type" or "artifact_type" must be set`)
}
queryParam = append(queryParam, attributionValue)
sql, queryParam = orm.PaginationOnRawSQL(query, sql, queryParam)
arts := []*Artifact{}
_, err = ormer.Raw(sql, queryParam...).QueryRows(&arts)
if err != nil {
return nil, err
}
return arts, nil
}
func querySetter(ctx context.Context, query *q.Query, options ...orm.Option) (beegoorm.QuerySeter, error) {
qs, err := orm.QuerySetter(ctx, &Artifact{}, query, options...)
if err != nil {

View File

@ -472,6 +472,75 @@ func (d *daoTestSuite) TestDeleteReferences() {
d.True(errors.IsErr(err, errors.NotFoundCode))
}
func (d *daoTestSuite) TestListWithLatest() {
now := time.Now()
art := &Artifact{
Type: "IMAGE",
MediaType: v1.MediaTypeImageConfig,
ManifestMediaType: v1.MediaTypeImageIndex,
ProjectID: 1234,
RepositoryID: 1234,
RepositoryName: "library2/hello-world1",
Digest: "digest",
PushTime: now,
PullTime: now,
Annotations: `{"anno1":"value1"}`,
}
id, err := d.dao.Create(d.ctx, art)
d.Require().Nil(err)
time.Sleep(1 * time.Second)
now = time.Now()
art2 := &Artifact{
Type: "IMAGE",
MediaType: v1.MediaTypeImageConfig,
ManifestMediaType: v1.MediaTypeImageIndex,
ProjectID: 1234,
RepositoryID: 1235,
RepositoryName: "library2/hello-world2",
Digest: "digest",
PushTime: now,
PullTime: now,
Annotations: `{"anno1":"value1"}`,
}
id1, err := d.dao.Create(d.ctx, art2)
d.Require().Nil(err)
time.Sleep(1 * time.Second)
now = time.Now()
art3 := &Artifact{
Type: "IMAGE",
MediaType: v1.MediaTypeImageConfig,
ManifestMediaType: v1.MediaTypeImageIndex,
ProjectID: 1234,
RepositoryID: 1235,
RepositoryName: "library2/hello-world2",
Digest: "digest2",
PushTime: now,
PullTime: now,
Annotations: `{"anno1":"value1"}`,
}
id2, err := d.dao.Create(d.ctx, art3)
d.Require().Nil(err)
latest, err := d.dao.ListWithLatest(d.ctx, &q.Query{
Keywords: map[string]interface{}{
"ProjectID": 1234,
"media_type": v1.MediaTypeImageConfig,
},
})
d.Require().Nil(err)
d.Require().Equal(2, len(latest))
d.Equal("library2/hello-world1", latest[0].RepositoryName)
defer d.dao.Delete(d.ctx, id)
defer d.dao.Delete(d.ctx, id1)
defer d.dao.Delete(d.ctx, id2)
}
func TestDaoTestSuite(t *testing.T) {
suite.Run(t, &daoTestSuite{})
}

View File

@ -48,6 +48,8 @@ type Manager interface {
ListReferences(ctx context.Context, query *q.Query) (references []*Reference, err error)
// DeleteReference specified by ID
DeleteReference(ctx context.Context, id int64) (err error)
// ListWithLatest list the artifacts when the latest_in_repository in the query was set
ListWithLatest(ctx context.Context, query *q.Query) (artifacts []*Artifact, err error)
}
// NewManager returns an instance of the default manager
@ -147,6 +149,22 @@ func (m *manager) DeleteReference(ctx context.Context, id int64) error {
return m.dao.DeleteReference(ctx, id)
}
func (m *manager) ListWithLatest(ctx context.Context, query *q.Query) ([]*Artifact, error) {
arts, err := m.dao.ListWithLatest(ctx, query)
if err != nil {
return nil, err
}
var artifacts []*Artifact
for _, art := range arts {
artifact, err := m.assemble(ctx, art)
if err != nil {
return nil, err
}
artifacts = append(artifacts, artifact)
}
return artifacts, nil
}
// assemble the artifact with references populated
func (m *manager) assemble(ctx context.Context, art *dao.Artifact) (*Artifact, error) {
artifact := &Artifact{}

View File

@ -80,6 +80,11 @@ func (f *fakeDao) DeleteReferences(ctx context.Context, parentID int64) error {
return args.Error(0)
}
func (f *fakeDao) ListWithLatest(ctx context.Context, query *q.Query) ([]*dao.Artifact, error) {
args := f.Called()
return args.Get(0).([]*dao.Artifact), args.Error(1)
}
type managerTestSuite struct {
suite.Suite
mgr *manager
@ -135,6 +140,28 @@ func (m *managerTestSuite) TestAssemble() {
m.Equal(2, len(artifact.References))
}
func (m *managerTestSuite) TestListWithLatest() {
art := &dao.Artifact{
ID: 1,
Type: "IMAGE",
MediaType: "application/vnd.oci.image.config.v1+json",
ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
ProjectID: 1,
RepositoryID: 1,
Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
Size: 1024,
PushTime: time.Now(),
PullTime: time.Now(),
ExtraAttrs: `{"attr1":"value1"}`,
Annotations: `{"anno1":"value1"}`,
}
m.dao.On("ListWithLatest", mock.Anything).Return([]*dao.Artifact{art}, nil)
artifacts, err := m.mgr.ListWithLatest(nil, nil)
m.Require().Nil(err)
m.Equal(1, len(artifacts))
m.Equal(art.ID, artifacts[0].ID)
}
func (m *managerTestSuite) TestList() {
art := &dao.Artifact{
ID: 1,

View File

@ -65,6 +65,10 @@ func (m *Manager) List(ctx context.Context, query *q.Query) ([]*artifact.Artifac
return m.delegator.List(ctx, query)
}
func (m *Manager) ListWithLatest(ctx context.Context, query *q.Query) ([]*artifact.Artifact, error) {
return m.delegator.ListWithLatest(ctx, query)
}
func (m *Manager) Create(ctx context.Context, artifact *artifact.Artifact) (int64, error) {
return m.delegator.Create(ctx, artifact)
}

View File

@ -95,7 +95,7 @@ func (a *artifactAPI) ListArtifacts(ctx context.Context, params operation.ListAr
// set option
option := option(params.WithTag, params.WithImmutableStatus,
params.WithLabel, params.WithAccessory)
params.WithLabel, params.WithAccessory, nil)
// get the total count of artifacts
total, err := a.artCtl.Count(ctx, query)
@ -129,7 +129,7 @@ func (a *artifactAPI) GetArtifact(ctx context.Context, params operation.GetArtif
}
// set option
option := option(params.WithTag, params.WithImmutableStatus,
params.WithLabel, params.WithAccessory)
params.WithLabel, params.WithAccessory, nil)
// get the artifact
artifact, err := a.artCtl.GetByReference(ctx, fmt.Sprintf("%s/%s", params.ProjectName, params.RepositoryName), params.Reference, option)
@ -501,11 +501,12 @@ func (a *artifactAPI) RequireLabelInProject(ctx context.Context, projectID, labe
return nil
}
func option(withTag, withImmutableStatus, withLabel, withAccessory *bool) *artifact.Option {
func option(withTag, withImmutableStatus, withLabel, withAccessory *bool, latestInRepository *bool) *artifact.Option {
option := &artifact.Option{
WithTag: true, // return the tag by default
WithLabel: lib.BoolValue(withLabel),
WithAccessory: true, // return the accessory by default
WithTag: true, // return the tag by default
WithLabel: lib.BoolValue(withLabel),
WithAccessory: true, // return the accessory by default
LatestInRepository: lib.BoolValue(latestInRepository),
}
if withTag != nil {

View File

@ -49,6 +49,8 @@ func (a *Artifact) ToSwagger() *models.Artifact {
PushTime: strfmt.DateTime(a.PushTime),
ExtraAttrs: a.ExtraAttrs,
Annotations: a.Annotations,
ArtifactType: a.ArtifactType,
RepositoryName: a.RepositoryName,
}
for _, reference := range a.References {

View File

@ -29,6 +29,7 @@ import (
"github.com/goharbor/harbor/src/common/security"
"github.com/goharbor/harbor/src/common/security/local"
robotSec "github.com/goharbor/harbor/src/common/security/robot"
"github.com/goharbor/harbor/src/controller/artifact"
"github.com/goharbor/harbor/src/controller/p2p/preheat"
"github.com/goharbor/harbor/src/controller/project"
"github.com/goharbor/harbor/src/controller/quota"
@ -52,6 +53,7 @@ import (
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/robot"
userModels "github.com/goharbor/harbor/src/pkg/user/models"
"github.com/goharbor/harbor/src/server/v2.0/handler/assembler"
"github.com/goharbor/harbor/src/server/v2.0/handler/model"
"github.com/goharbor/harbor/src/server/v2.0/models"
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/project"
@ -63,6 +65,7 @@ const defaultDaysToRetentionForProxyCacheProject = 7
func newProjectAPI() *projectAPI {
return &projectAPI{
auditMgr: audit.Mgr,
artCtl: artifact.Ctl,
metadataMgr: pkg.ProjectMetaMgr,
userCtl: user.Ctl,
repositoryCtl: repository.Ctl,
@ -79,6 +82,7 @@ func newProjectAPI() *projectAPI {
type projectAPI struct {
BaseAPI
auditMgr audit.Manager
artCtl artifact.Controller
metadataMgr metadata.Manager
userCtl user.Controller
repositoryCtl repository.Controller
@ -660,6 +664,82 @@ func (a *projectAPI) SetScannerOfProject(ctx context.Context, params operation.S
return operation.NewSetScannerOfProjectOK()
}
func (a *projectAPI) ListArtifactsOfProject(ctx context.Context, params operation.ListArtifactsOfProjectParams) middleware.Responder {
if err := a.RequireAuthenticated(ctx); err != nil {
return a.SendError(ctx, err)
}
projectNameOrID := parseProjectNameOrID(params.ProjectNameOrID, params.XIsResourceName)
if err := a.RequireProjectAccess(ctx, projectNameOrID, rbac.ActionList, rbac.ResourceArtifact); err != nil {
return a.SendError(ctx, err)
}
// set query
pro, err := a.projectCtl.Get(ctx, projectNameOrID)
if err != nil {
return a.SendError(ctx, err)
}
query, err := a.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
if err != nil {
return a.SendError(ctx, err)
}
query.Keywords["ProjectID"] = pro.ProjectID
// set option
option := option(params.WithTag, params.WithImmutableStatus,
params.WithLabel, params.WithAccessory, params.LatestInRepository)
var total int64
// list artifacts according to the query and option
var arts []*artifact.Artifact
if option.LatestInRepository {
// ignore page & page_size
_, hasMediaType := query.Keywords["media_type"]
_, hasArtifactType := query.Keywords["artifact_type"]
if hasMediaType == hasArtifactType {
return a.SendError(ctx, errors.BadRequestError(fmt.Errorf("either 'media_type' or 'artifact_type' must be specified, but not both, when querying with latest_in_repository")))
}
getCount := func() (int64, error) {
var countQ *q.Query
if query != nil {
countQ = q.New(query.Keywords)
}
allArts, err := a.artCtl.ListWithLatest(ctx, countQ, nil)
if err != nil {
return int64(0), err
}
return int64(len(allArts)), nil
}
total, err = getCount()
if err != nil {
return a.SendError(ctx, err)
}
arts, err = a.artCtl.ListWithLatest(ctx, query, option)
} else {
total, err = a.artCtl.Count(ctx, query)
if err != nil {
return a.SendError(ctx, err)
}
arts, err = a.artCtl.List(ctx, query, option)
}
if err != nil {
return a.SendError(ctx, err)
}
overviewOpts := model.NewOverviewOptions(model.WithSBOM(lib.BoolValue(params.WithSbomOverview)), model.WithVuln(lib.BoolValue(params.WithScanOverview)))
assembler := assembler.NewScanReportAssembler(overviewOpts, parseScanReportMimeTypes(params.XAcceptVulnerabilities))
var artifacts []*models.Artifact
for _, art := range arts {
artifact := &model.Artifact{}
artifact.Artifact = *art
_ = assembler.WithArtifacts(artifact).Assemble(ctx)
artifacts = append(artifacts, artifact.ToSwagger())
}
return operation.NewListArtifactsOfProjectOK().
WithXTotalCount(total).
WithLink(a.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
WithPayload(artifacts)
}
func (a *projectAPI) deletable(ctx context.Context, projectNameOrID interface{}) (*project.Project, *models.ProjectDeletable, error) {
p, err := a.getProject(ctx, projectNameOrID)
if err != nil {

View File

@ -296,6 +296,36 @@ func (_m *Controller) List(ctx context.Context, query *q.Query, option *artifact
return r0, r1
}
// ListWithLatest provides a mock function with given fields: ctx, query, option
func (_m *Controller) ListWithLatest(ctx context.Context, query *q.Query, option *artifact.Option) ([]*artifact.Artifact, error) {
ret := _m.Called(ctx, query, option)
if len(ret) == 0 {
panic("no return value specified for ListWithLatest")
}
var r0 []*artifact.Artifact
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *q.Query, *artifact.Option) ([]*artifact.Artifact, error)); ok {
return rf(ctx, query, option)
}
if rf, ok := ret.Get(0).(func(context.Context, *q.Query, *artifact.Option) []*artifact.Artifact); ok {
r0 = rf(ctx, query, option)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*artifact.Artifact)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *q.Query, *artifact.Option) error); ok {
r1 = rf(ctx, query, option)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// RemoveLabel provides a mock function with given fields: ctx, artifactID, labelID
func (_m *Controller) RemoveLabel(ctx context.Context, artifactID int64, labelID int64) error {
ret := _m.Called(ctx, artifactID, labelID)

View File

@ -231,6 +231,36 @@ func (_m *Manager) ListReferences(ctx context.Context, query *q.Query) ([]*artif
return r0, r1
}
// ListWithLatest provides a mock function with given fields: ctx, query
func (_m *Manager) ListWithLatest(ctx context.Context, query *q.Query) ([]*artifact.Artifact, error) {
ret := _m.Called(ctx, query)
if len(ret) == 0 {
panic("no return value specified for ListWithLatest")
}
var r0 []*artifact.Artifact
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *q.Query) ([]*artifact.Artifact, error)); ok {
return rf(ctx, query)
}
if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*artifact.Artifact); ok {
r0 = rf(ctx, query)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*artifact.Artifact)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
r1 = rf(ctx, query)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Update provides a mock function with given fields: ctx, _a1, props
func (_m *Manager) Update(ctx context.Context, _a1 *artifact.Artifact, props ...string) error {
_va := make([]interface{}, len(props))