diff --git a/make/migrations/postgresql/0050_2.2.0_schema.up.sql b/make/migrations/postgresql/0050_2.2.0_schema.up.sql index 54f824029..e5d562d86 100644 --- a/make/migrations/postgresql/0050_2.2.0_schema.up.sql +++ b/make/migrations/postgresql/0050_2.2.0_schema.up.sql @@ -1 +1,13 @@ -ALTER TABLE schedule ADD COLUMN IF NOT EXISTS cron_type varchar(64); \ No newline at end of file +ALTER TABLE schedule ADD COLUMN IF NOT EXISTS cron_type varchar(64); + +DO $$ +DECLARE + art RECORD; + art_size integer; +BEGIN + FOR art IN SELECT * FROM artifact WHERE size = 0 + LOOP + SELECT sum(size) INTO art_size FROM blob WHERE digest IN (SELECT digest_blob FROM artifact_blob WHERE digest_af=art.digest); + UPDATE artifact SET size=art_size WHERE id = art.id; + END LOOP; +END $$; diff --git a/src/controller/artifact/abstractor.go b/src/controller/artifact/abstractor.go index 4a1e1cac5..67c75a35b 100644 --- a/src/controller/artifact/abstractor.go +++ b/src/controller/artifact/abstractor.go @@ -18,11 +18,14 @@ import ( "context" "encoding/json" "fmt" + "github.com/docker/distribution/manifest/manifestlist" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" "github.com/goharbor/harbor/src/controller/artifact/processor" + "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/goharbor/harbor/src/pkg/blob" "github.com/goharbor/harbor/src/pkg/registry" v1 "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -36,14 +39,16 @@ type Abstractor interface { // NewAbstractor creates a new abstractor func NewAbstractor() Abstractor { return &abstractor{ - artMgr: artifact.Mgr, - regCli: registry.Cli, + artMgr: artifact.Mgr, + blobMgr: blob.Mgr, + regCli: registry.Cli, } } type abstractor struct { - artMgr artifact.Manager - regCli registry.Client + artMgr artifact.Manager + blobMgr blob.Manager + regCli registry.Client } func (a *abstractor) AbstractMetadata(ctx context.Context, artifact *artifact.Artifact) error { @@ -60,7 +65,9 @@ func (a *abstractor) AbstractMetadata(ctx context.Context, artifact *artifact.Ar switch artifact.ManifestMediaType { case "", "application/json", schema1.MediaTypeSignedManifest: - a.abstractManifestV1Metadata(artifact) + if err := a.abstractManifestV1Metadata(ctx, artifact, content); err != nil { + return err + } case v1.MediaTypeImageManifest, schema2.MediaTypeManifest: if err = a.abstractManifestV2Metadata(artifact, content); err != nil { return err @@ -76,13 +83,36 @@ func (a *abstractor) AbstractMetadata(ctx context.Context, artifact *artifact.Ar } // the artifact is enveloped by docker manifest v1 -func (a *abstractor) abstractManifestV1Metadata(artifact *artifact.Artifact) { +func (a *abstractor) abstractManifestV1Metadata(ctx context.Context, artifact *artifact.Artifact, content []byte) error { // unify the media type of v1 manifest to "schema1.MediaTypeSignedManifest" artifact.ManifestMediaType = schema1.MediaTypeSignedManifest // as no config layer in the docker v1 manifest, use the "schema1.MediaTypeSignedManifest" // as the media type of artifact artifact.MediaType = schema1.MediaTypeSignedManifest - // there is no layer size in v1 manifest, doesn't set the artifact size + + manifest := &schema1.Manifest{} + if err := json.Unmarshal(content, manifest); err != nil { + return err + } + + digests := make([]string, len(manifest.FSLayers)) + for i, fsLayer := range manifest.FSLayers { + digests[i] = fsLayer.BlobSum.String() + } + + // there is no layer size in v1 manifest, compute the artifact size from the blobs + blobs, err := a.blobMgr.List(ctx, blob.ListParams{BlobDigests: digests}) + if err != nil { + log.G(ctx).Errorf("failed to get blobs of the artifact %s, error %v", artifact.Digest, err) + return err + } + + artifact.Size = int64(len(content)) + for _, blob := range blobs { + artifact.Size += blob.Size + } + + return nil } // the artifact is enveloped by OCI manifest or docker manifest v2 diff --git a/src/controller/artifact/abstractor_test.go b/src/controller/artifact/abstractor_test.go index 3c164547b..adb05605f 100644 --- a/src/controller/artifact/abstractor_test.go +++ b/src/controller/artifact/abstractor_test.go @@ -19,8 +19,10 @@ import ( "github.com/goharbor/harbor/src/controller/artifact/processor" "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/goharbor/harbor/src/pkg/blob" "github.com/goharbor/harbor/src/testing/mock" tart "github.com/goharbor/harbor/src/testing/pkg/artifact" + tblob "github.com/goharbor/harbor/src/testing/pkg/blob" tpro "github.com/goharbor/harbor/src/testing/pkg/processor" "github.com/goharbor/harbor/src/testing/pkg/registry" @@ -205,6 +207,7 @@ var ( type abstractorTestSuite struct { suite.Suite argMgr *tart.FakeManager + blobMgr *tblob.Manager regCli *registry.FakeClient abstractor *abstractor processor *tpro.Processor @@ -213,9 +216,11 @@ type abstractorTestSuite struct { func (a *abstractorTestSuite) SetupTest() { a.regCli = ®istry.FakeClient{} a.argMgr = &tart.FakeManager{} + a.blobMgr = &tblob.Manager{} a.abstractor = &abstractor{ - artMgr: a.argMgr, - regCli: a.regCli, + artMgr: a.argMgr, + blobMgr: a.blobMgr, + regCli: a.regCli, } a.processor = &tpro.Processor{} // clear all registered processors @@ -227,6 +232,10 @@ func (a *abstractorTestSuite) SetupTest() { func (a *abstractorTestSuite) TestAbstractMetadataOfV1Manifest() { manifest, _, err := distribution.UnmarshalManifest(schema1.MediaTypeSignedManifest, []byte(v1Manifest)) a.Require().Nil(err) + mock.OnAnything(a.blobMgr, "List").Return([]*blob.Blob{ + {Size: 10}, + {Size: 20}, + }, nil) a.regCli.On("PullManifest").Return(manifest, "", nil) artifact := &artifact.Artifact{ ID: 1, @@ -236,7 +245,7 @@ func (a *abstractorTestSuite) TestAbstractMetadataOfV1Manifest() { a.Assert().Equal(int64(1), artifact.ID) a.Assert().Equal(schema1.MediaTypeSignedManifest, artifact.ManifestMediaType) a.Assert().Equal(schema1.MediaTypeSignedManifest, artifact.MediaType) - a.Assert().Equal(int64(0), artifact.Size) + a.Assert().Equal(int64(30+len([]byte(v1Manifest))), artifact.Size) } // docker manifest v2