diff --git a/make/migrations/postgresql/0030_1.11.0_schema.up.sql b/make/migrations/postgresql/0030_1.11.0_schema.up.sql new file mode 100644 index 000000000..6ee271417 --- /dev/null +++ b/make/migrations/postgresql/0030_1.11.0_schema.up.sql @@ -0,0 +1,42 @@ +/* TODO remove the table artifact_2 and use the artifact instead after finishing the upgrade work */ +CREATE TABLE artifact_2 +( + id SERIAL PRIMARY KEY NOT NULL, + /* image, chart, etc */ + type varchar(255), + media_type varchar(255), + manifest_media_type varchar(255), + project_id int NOT NULL, + repository_id int NOT NULL, + digest varchar(255) NOT NULL, + size bigint, + push_time timestamp default CURRENT_TIMESTAMP, + platform varchar(255), + extra_attrs text, + annotations jsonb, + /* when updating the data the revision MUST be checked and updated */ + revision varchar(64) NOT NULL, + CONSTRAINT unique_artifact_2 UNIQUE (repository_id, digest) +); + +CREATE TABLE tag +( + id SERIAL PRIMARY KEY NOT NULL, + repository_id int NOT NULL, + artifact_id int NOT NULL, + name varchar(255), + push_time timestamp default CURRENT_TIMESTAMP, + pull_time timestamp, + /* when updating the data the revision MUST be checked and updated */ + revision varchar(64) NOT NULL, + CONSTRAINT unique_tag UNIQUE (repository_id, name) +); + +/* artifact_reference records the child artifact referenced by parent artifact */ +CREATE TABLE artifact_reference +( + id SERIAL PRIMARY KEY NOT NULL, + parent_id int NOT NULL, + child_id int NOT NULL, + CONSTRAINT unique_reference UNIQUE (parent_id, child_id) +); \ No newline at end of file diff --git a/src/go.mod b/src/go.mod index 11b8d578a..cd54f8151 100644 --- a/src/go.mod +++ b/src/go.mod @@ -59,7 +59,7 @@ require ( github.com/miekg/pkcs11 v0.0.0-20170220202408-7283ca79f35e // indirect github.com/olekukonko/tablewriter v0.0.1 github.com/opencontainers/go-digest v1.0.0-rc0 - github.com/opencontainers/image-spec v1.0.1 // indirect + github.com/opencontainers/image-spec v1.0.1 github.com/opentracing/opentracing-go v1.1.0 // indirect github.com/pkg/errors v0.8.1 github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect diff --git a/src/pkg/artifact/controller.go b/src/pkg/artifact/controller.go new file mode 100644 index 000000000..a76d5c7d6 --- /dev/null +++ b/src/pkg/artifact/controller.go @@ -0,0 +1,15 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package artifact diff --git a/src/pkg/artifact/manager/dao/model.go b/src/pkg/artifact/manager/dao/model.go new file mode 100644 index 000000000..cd1b423d5 --- /dev/null +++ b/src/pkg/artifact/manager/dao/model.go @@ -0,0 +1,78 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "time" + + "github.com/astaxie/beego/orm" +) + +func init() { + orm.RegisterModel(&Artifact{}) + orm.RegisterModel(&Tag{}) + orm.RegisterModel(&ArtifactReference{}) +} + +// Artifact model in database +type Artifact struct { + ID int64 `orm:"pk;auto;column(id)"` + Type string `orm:"column(type)"` // image or chart + MediaType string `orm:"column(media_type)"` // the media type of artifact + ManifestMediaType string `orm:"column(manifest_media_type)"` // the media type of manifest/index + ProjectID int64 `orm:"column(project_id)"` // needed for quota + RepositoryID int64 `orm:"column(repository_id)"` + Digest string `orm:"column(digest)"` + Size int64 `orm:"column(size)"` + PushTime time.Time `orm:"column(push_time)"` + Platform string `orm:"column(platform)"` // json string + ExtraAttrs string `orm:"column(extra_attrs)"` // json string + Annotations string `orm:"column(annotations);type(jsonb)"` // json string + Revision string `orm:"column(revision)"` // record data revision, when updating the data the revision MUST be checked and updated +} + +// TableName for artifact +func (a *Artifact) TableName() string { + // TODO use "artifact" after finishing the upgrade/migration work + return "artifact_2" +} + +// Tag model in database +type Tag struct { + ID int64 `orm:"pk;auto;column(id)"` + RepositoryID int64 `orm:"column(repository_id)"` // tags are the resources of repository, one repository only contains one same name tag + ArtifactID int64 `orm:"column(artifact_id)"` // the artifact ID that the tag attaches to, it changes when pushing a same name but different digest artifact + Name string `orm:"column(name)"` + PushTime time.Time `orm:"column(push_time)"` + PullTime time.Time `orm:"column(pull_time)"` + Revision string `orm:"column(revision)"` // record data revision, when updating the data the revision MUST be checked and updated +} + +// TableName for tag +func (t *Tag) TableName() string { + return "tag" +} + +// ArtifactReference records the child artifact referenced by parent artifact +type ArtifactReference struct { + ID int64 `orm:"pk;auto;column(id)"` + ParentID int64 `orm:"column(parent_id)"` + ChildID int64 `orm:"column(child_id)"` +} + +// TableName for artifact reference +func (a *ArtifactReference) TableName() string { + return "artifact_reference" +} diff --git a/src/pkg/artifact/manager/manager.go b/src/pkg/artifact/manager/manager.go new file mode 100644 index 000000000..f3f313aa9 --- /dev/null +++ b/src/pkg/artifact/manager/manager.go @@ -0,0 +1,15 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager diff --git a/src/pkg/artifact/model/model.go b/src/pkg/artifact/model/model.go new file mode 100644 index 000000000..a74f60352 --- /dev/null +++ b/src/pkg/artifact/model/model.go @@ -0,0 +1,165 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "encoding/json" + "time" + + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/artifact/manager/dao" + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +// Artifact is the abstract object managed by Harbor. It hides the +// underlying concrete detail and provides an unified artifact view +// for all users. +type Artifact struct { + ID int64 + Type string // image, chart, etc + MediaType string // the media type of artifact. Mostly, it's the value of `manifest.config.mediatype` + ManifestMediaType string // the media type of manifest/index + Repository *models.RepoRecord + Tags []*Tag // the list of tags that attached to the artifact + Digest string + Size int64 + PushTime time.Time + Platform *v1.Platform // when the parent of the artifact is an index, populate the platform information here + ExtraAttrs map[string]interface{} // only contains the simple attributes specific for the different artifact type, most of them should come from the config layer + SubResourceLinks map[string][]*ResourceLink // the resource link for build history(image), values.yaml(chart), dependency(chart), etc + Annotations map[string]string + References []int64 // child artifacts referenced by the parent artifact if the artifact is an index + Revision string // record data revision + // TODO: As the labels and signature aren't handled inside the artifact module, + // we should move it to the API level artifact model rather than + // keeping it here. The same to scan information + // Labels []*models.Label + // Signature *Signature // add the signature in the artifact level rather than tag level as we cannot make sure the signature always apply to tag +} + +// From converts the database level artifact to the business level object +func (a *Artifact) From(art *dao.Artifact) { + a.ID = art.ID + a.Type = art.Type + a.MediaType = art.MediaType + a.ManifestMediaType = art.ManifestMediaType + a.Repository = &models.RepoRecord{ + ProjectID: art.ProjectID, + RepositoryID: art.RepositoryID, + } + a.Digest = art.Digest + a.Size = art.Size + a.PushTime = art.PushTime + a.ExtraAttrs = map[string]interface{}{} + a.Annotations = map[string]string{} + a.Revision = art.Revision + if len(art.Platform) > 0 { + if err := json.Unmarshal([]byte(art.Platform), &a.Platform); err != nil { + log.Errorf("failed to unmarshal the platform of artifact %d: %v", art.ID, err) + } + } + if len(art.ExtraAttrs) > 0 { + if err := json.Unmarshal([]byte(art.ExtraAttrs), &a.ExtraAttrs); err != nil { + log.Errorf("failed to unmarshal the extra attrs of artifact %d: %v", art.ID, err) + } + } + if len(art.Annotations) > 0 { + if err := json.Unmarshal([]byte(art.Annotations), &a.Annotations); err != nil { + log.Errorf("failed to unmarshal the annotations of artifact %d: %v", art.ID, err) + } + } +} + +// To converts the artifact to the database level object +func (a *Artifact) To() *dao.Artifact { + art := &dao.Artifact{ + ID: a.ID, + Type: a.Type, + MediaType: a.MediaType, + ManifestMediaType: a.ManifestMediaType, + ProjectID: a.Repository.ProjectID, + RepositoryID: a.Repository.RepositoryID, + Digest: a.Digest, + Size: a.Size, + PushTime: a.PushTime, + Revision: a.Revision, + } + + if a.Platform != nil { + platform, err := json.Marshal(a.Platform) + if err != nil { + log.Errorf("failed to marshal the platform of artifact %d: %v", a.ID, err) + } + art.Platform = string(platform) + } + if len(a.ExtraAttrs) > 0 { + attrs, err := json.Marshal(a.ExtraAttrs) + if err != nil { + log.Errorf("failed to marshal the extra attrs of artifact %d: %v", a.ID, err) + } + art.ExtraAttrs = string(attrs) + } + if len(a.Annotations) > 0 { + annotations, err := json.Marshal(a.Annotations) + if err != nil { + log.Errorf("failed to marshal the annotations of artifact %d: %v", a.ID, err) + } + art.Annotations = string(annotations) + } + return art +} + +// ResourceLink is a link via that a resource can be fetched +type ResourceLink struct { + HREF string + Absolute bool // specify the href is an absolute URL or not +} + +// TODO: move it to the API level artifact model +// Signature information +// type Signature struct { +// Signatures map[string]bool // tag: signed or not +// } + +// Tag belongs to one repository and can only be attached to a single +// one artifact under the repository +type Tag struct { + ID int64 + Name string + PushTime time.Time + PullTime time.Time + Revision string // record data revision +} + +// From converts the database level tag to the business level object +func (t *Tag) From(tag *dao.Tag) { + t.ID = tag.ID + t.Name = tag.Name + t.PushTime = tag.PushTime + t.PullTime = tag.PullTime + t.Revision = tag.Revision +} + +// To converts the tag to the database level model +func (t *Tag) To() *dao.Tag { + return &dao.Tag{ + ID: t.ID, + Name: t.Name, + PushTime: t.PushTime, + PullTime: t.PullTime, + Revision: t.Revision, + } +} diff --git a/src/pkg/artifact/model/model_test.go b/src/pkg/artifact/model/model_test.go new file mode 100644 index 000000000..49652379f --- /dev/null +++ b/src/pkg/artifact/model/model_test.go @@ -0,0 +1,144 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "testing" + "time" + + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/pkg/artifact/manager/dao" + v1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type modelTestSuite struct { + suite.Suite +} + +func (m *modelTestSuite) TestArtifactFrom() { + t := m.T() + dbArt := &dao.Artifact{ + ID: 1, + Type: "IMAGE", + MediaType: "application/vnd.oci.image.config.v1+json", + ManifestMediaType: "application/vnd.oci.image.manifest.v1+json", + ProjectID: 1, + RepositoryID: 1, + Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180", + Size: 1024, + PushTime: time.Now(), + Platform: `{"architecture":"amd64"}`, + ExtraAttrs: `{"attr1":"value1"}`, + Annotations: `{"anno1":"value1"}`, + Revision: "1", + } + art := &Artifact{} + art.From(dbArt) + assert.Equal(t, dbArt.ID, art.ID) + assert.Equal(t, dbArt.Type, art.Type) + assert.Equal(t, dbArt.MediaType, art.MediaType) + assert.Equal(t, dbArt.ManifestMediaType, art.ManifestMediaType) + assert.Equal(t, dbArt.ProjectID, art.Repository.ProjectID) + assert.Equal(t, dbArt.RepositoryID, art.Repository.RepositoryID) + assert.Equal(t, dbArt.Digest, art.Digest) + assert.Equal(t, dbArt.Size, art.Size) + assert.Equal(t, dbArt.PushTime, art.PushTime) + assert.Equal(t, "amd64", art.Platform.Architecture) + assert.Equal(t, "value1", art.ExtraAttrs["attr1"].(string)) + assert.Equal(t, "value1", art.Annotations["anno1"]) + assert.Equal(t, dbArt.Revision, art.Revision) +} + +func (m *modelTestSuite) TestArtifactTo() { + t := m.T() + art := &Artifact{ + ID: 1, + Type: "IMAGE", + Repository: &models.RepoRecord{ + ProjectID: 1, + RepositoryID: 1, + }, + MediaType: "application/vnd.oci.image.config.v1+json", + ManifestMediaType: "application/vnd.oci.image.manifest.v1+json", + Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180", + Size: 1024, + PushTime: time.Now(), + Platform: &v1.Platform{ + Architecture: "amd64", + }, + ExtraAttrs: map[string]interface{}{ + "attr1": "value1", + }, + Annotations: map[string]string{ + "anno1": "value1", + }, + Revision: "1", + } + dbArt := art.To() + assert.Equal(t, art.ID, dbArt.ID) + assert.Equal(t, art.Type, dbArt.Type) + assert.Equal(t, art.MediaType, dbArt.MediaType) + assert.Equal(t, art.ManifestMediaType, dbArt.ManifestMediaType) + assert.Equal(t, art.Repository.ProjectID, dbArt.ProjectID) + assert.Equal(t, art.Repository.RepositoryID, dbArt.RepositoryID) + assert.Equal(t, art.Digest, dbArt.Digest) + assert.Equal(t, art.Size, dbArt.Size) + assert.Equal(t, art.PushTime, dbArt.PushTime) + assert.Equal(t, `{"architecture":"amd64","os":""}`, dbArt.Platform) + assert.Equal(t, `{"attr1":"value1"}`, dbArt.ExtraAttrs) + assert.Equal(t, `{"anno1":"value1"}`, dbArt.Annotations) + assert.Equal(t, art.Revision, dbArt.Revision) +} + +func (m *modelTestSuite) TestTagFrom() { + t := m.T() + dbTag := &dao.Tag{ + ID: 1, + Name: "1.0", + PushTime: time.Now(), + PullTime: time.Now(), + Revision: "1", + } + tag := &Tag{} + tag.From(dbTag) + assert.Equal(t, dbTag.ID, tag.ID) + assert.Equal(t, dbTag.Name, tag.Name) + assert.Equal(t, dbTag.PushTime, tag.PushTime) + assert.Equal(t, dbTag.PullTime, tag.PullTime) + assert.Equal(t, dbTag.Revision, tag.Revision) +} + +func (m *modelTestSuite) TestTagTo() { + t := m.T() + tag := &Tag{ + ID: 1, + Name: "1.0", + PushTime: time.Now(), + PullTime: time.Now(), + Revision: "1", + } + dbTag := tag.To() + assert.Equal(t, tag.ID, dbTag.ID) + assert.Equal(t, tag.Name, dbTag.Name) + assert.Equal(t, tag.PushTime, dbTag.PushTime) + assert.Equal(t, tag.PullTime, dbTag.PullTime) + assert.Equal(t, tag.Revision, dbTag.Revision) +} + +func TestModel(t *testing.T) { + suite.Run(t, &modelTestSuite{}) +}