diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 9c42f73e8..65abd39e1 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -1449,7 +1449,14 @@ paths: in: path description: The type of addition. type: string - enum: [build_history, values.yaml, readme.md, dependencies, sbom] + enum: + - build_history + - values.yaml + - readme.md + - dependencies + - sbom + - license + - files required: true responses: '200': @@ -1795,7 +1802,7 @@ paths: items: $ref: '#/definitions/AuditLogEventType' '401': - $ref: '#/responses/401' + $ref: '#/responses/401' /projects/{project_name}/logs: get: summary: Get recent logs of the projects (deprecated) @@ -1863,7 +1870,7 @@ paths: '401': $ref: '#/responses/401' '500': - $ref: '#/responses/500' + $ref: '#/responses/500' /p2p/preheat/providers: get: summary: List P2P providers @@ -6949,8 +6956,8 @@ definitions: description: The time when this operation is triggered. AuditLogEventType: type: object - properties: - event_type: + properties: + event_type: type: string description: the event type, such as create_user. example: create_user diff --git a/icons/cnai.png b/icons/cnai.png new file mode 100644 index 000000000..08865029a Binary files /dev/null and b/icons/cnai.png differ diff --git a/src/controller/artifact/abstractor.go b/src/controller/artifact/abstractor.go index bbf75a1fa..de3c7a7d4 100644 --- a/src/controller/artifact/abstractor.go +++ b/src/controller/artifact/abstractor.go @@ -83,7 +83,7 @@ func (a *abstractor) AbstractMetadata(ctx context.Context, artifact *artifact.Ar default: return fmt.Errorf("unsupported manifest media type: %s", artifact.ManifestMediaType) } - return processor.Get(artifact.MediaType).AbstractMetadata(ctx, artifact, content) + return processor.Get(artifact.ResolveArtifactType()).AbstractMetadata(ctx, artifact, content) } // the artifact is enveloped by docker manifest v1 diff --git a/src/controller/artifact/controller.go b/src/controller/artifact/controller.go index 943b9313f..e84824963 100644 --- a/src/controller/artifact/controller.go +++ b/src/controller/artifact/controller.go @@ -28,6 +28,7 @@ import ( "github.com/goharbor/harbor/src/controller/artifact/processor" "github.com/goharbor/harbor/src/controller/artifact/processor/chart" "github.com/goharbor/harbor/src/controller/artifact/processor/cnab" + "github.com/goharbor/harbor/src/controller/artifact/processor/cnai" "github.com/goharbor/harbor/src/controller/artifact/processor/image" "github.com/goharbor/harbor/src/controller/artifact/processor/sbom" "github.com/goharbor/harbor/src/controller/artifact/processor/wasm" @@ -44,7 +45,7 @@ import ( accessorymodel "github.com/goharbor/harbor/src/pkg/accessory/model" "github.com/goharbor/harbor/src/pkg/artifact" "github.com/goharbor/harbor/src/pkg/artifactrash" - "github.com/goharbor/harbor/src/pkg/artifactrash/model" + trashmodel "github.com/goharbor/harbor/src/pkg/artifactrash/model" "github.com/goharbor/harbor/src/pkg/blob" "github.com/goharbor/harbor/src/pkg/immutable/match" "github.com/goharbor/harbor/src/pkg/immutable/match/rule" @@ -78,6 +79,7 @@ var ( cnab.ArtifactTypeCNAB: icon.DigestOfIconCNAB, wasm.ArtifactTypeWASM: icon.DigestOfIconWASM, sbom.ArtifactTypeSBOM: icon.DigestOfIconAccSBOM, + cnai.ArtifactTypeCNAI: icon.DigestOfIconCNAI, } ) @@ -219,7 +221,7 @@ func (c *controller) ensureArtifact(ctx context.Context, repository, digest stri } // populate the artifact type - artifact.Type = processor.Get(artifact.MediaType).GetArtifactType(ctx, artifact) + artifact.Type = processor.Get(artifact.ResolveArtifactType()).GetArtifactType(ctx, artifact) // create it // use orm.WithTransaction here to avoid the issue: @@ -437,7 +439,7 @@ func (c *controller) deleteDeeply(ctx context.Context, id int64, isRoot, isAcces // use orm.WithTransaction here to avoid the issue: // https://www.postgresql.org/message-id/002e01c04da9%24a8f95c20%2425efe6c1%40lasting.ro if err = orm.WithTransaction(func(ctx context.Context) error { - _, err = c.artrashMgr.Create(ctx, &model.ArtifactTrash{ + _, err = c.artrashMgr.Create(ctx, &trashmodel.ArtifactTrash{ MediaType: art.MediaType, ManifestMediaType: art.ManifestMediaType, RepositoryName: art.RepositoryName, @@ -599,7 +601,7 @@ func (c *controller) GetAddition(ctx context.Context, artifactID int64, addition if err != nil { return nil, err } - return processor.Get(artifact.MediaType).AbstractAddition(ctx, artifact, addition) + return processor.Get(artifact.ResolveArtifactType()).AbstractAddition(ctx, artifact, addition) } func (c *controller) AddLabel(ctx context.Context, artifactID int64, labelID int64) (err error) { @@ -757,7 +759,7 @@ func (c *controller) populateLabels(ctx context.Context, art *Artifact) { } func (c *controller) populateAdditionLinks(ctx context.Context, artifact *Artifact) { - types := processor.Get(artifact.MediaType).ListAdditionTypes(ctx, &artifact.Artifact) + types := processor.Get(artifact.ResolveArtifactType()).ListAdditionTypes(ctx, &artifact.Artifact) if len(types) > 0 { version := lib.GetAPIVersion(ctx) for _, t := range types { diff --git a/src/controller/artifact/processor/cnai/cnai.go b/src/controller/artifact/processor/cnai/cnai.go new file mode 100644 index 000000000..1e5d90f50 --- /dev/null +++ b/src/controller/artifact/processor/cnai/cnai.go @@ -0,0 +1,106 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cnai + +import ( + "context" + "encoding/json" + + modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + ps "github.com/goharbor/harbor/src/controller/artifact/processor" + "github.com/goharbor/harbor/src/controller/artifact/processor/base" + "github.com/goharbor/harbor/src/controller/artifact/processor/cnai/parser" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/artifact" +) + +// const definitions +const ( + // ArtifactTypeCNAI defines the artifact type for CNAI model. + ArtifactTypeCNAI = "CNAI" + + // AdditionTypeReadme defines the addition type readme for API. + AdditionTypeReadme = "README.MD" + // AdditionTypeLicense defines the addition type license for API. + AdditionTypeLicense = "LICENSE" + // AdditionTypeFiles defines the addition type files for API. + AdditionTypeFiles = "FILES" +) + +func init() { + pc := &processor{ + ManifestProcessor: base.NewManifestProcessor(), + } + + if err := ps.Register(pc, modelspec.ArtifactTypeModelManifest); err != nil { + log.Errorf("failed to register processor for artifact type %s: %v", modelspec.ArtifactTypeModelManifest, err) + return + } +} + +type processor struct { + *base.ManifestProcessor +} + +func (p *processor) AbstractAddition(ctx context.Context, artifact *artifact.Artifact, addition string) (*ps.Addition, error) { + var additionParser parser.Parser + switch addition { + case AdditionTypeReadme: + additionParser = parser.NewReadme(p.RegCli) + case AdditionTypeLicense: + additionParser = parser.NewLicense(p.RegCli) + case AdditionTypeFiles: + additionParser = parser.NewFiles(p.RegCli) + default: + return nil, errors.New(nil).WithCode(errors.BadRequestCode). + WithMessagef("addition %s isn't supported for %s", addition, ArtifactTypeCNAI) + } + + mf, _, err := p.RegCli.PullManifest(artifact.RepositoryName, artifact.Digest) + if err != nil { + return nil, err + } + + _, payload, err := mf.Payload() + if err != nil { + return nil, err + } + + manifest := &ocispec.Manifest{} + if err := json.Unmarshal(payload, manifest); err != nil { + return nil, err + } + + contentType, content, err := additionParser.Parse(ctx, artifact, manifest) + if err != nil { + return nil, err + } + + return &ps.Addition{ + ContentType: contentType, + Content: content, + }, nil +} + +func (p *processor) GetArtifactType(_ context.Context, _ *artifact.Artifact) string { + return ArtifactTypeCNAI +} + +func (p *processor) ListAdditionTypes(_ context.Context, _ *artifact.Artifact) []string { + return []string{AdditionTypeReadme, AdditionTypeLicense, AdditionTypeFiles} +} diff --git a/src/controller/artifact/processor/cnai/cnai_test.go b/src/controller/artifact/processor/cnai/cnai_test.go new file mode 100644 index 000000000..2e47cc2eb --- /dev/null +++ b/src/controller/artifact/processor/cnai/cnai_test.go @@ -0,0 +1,265 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package cnai + +import ( + "archive/tar" + "bytes" + "context" + "encoding/json" + "io" + "testing" + + modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + v1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/suite" + + "github.com/goharbor/harbor/src/controller/artifact/processor/base" + "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/goharbor/harbor/src/pkg/distribution" + "github.com/goharbor/harbor/src/testing/mock" + "github.com/goharbor/harbor/src/testing/pkg/registry" +) + +type ProcessorTestSuite struct { + suite.Suite + processor *processor + regCli *registry.Client +} + +func (p *ProcessorTestSuite) SetupTest() { + p.regCli = ®istry.Client{} + p.processor = &processor{} + p.processor.ManifestProcessor = &base.ManifestProcessor{ + RegCli: p.regCli, + } +} + +func createTarContent(filename, content string) ([]byte, error) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + + hdr := &tar.Header{ + Name: filename, + Mode: 0600, + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + return nil, err + } + if _, err := tw.Write([]byte(content)); err != nil { + return nil, err + } + if err := tw.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (p *ProcessorTestSuite) TestAbstractAddition() { + cases := []struct { + name string + addition string + manifest *ocispec.Manifest + setupMockReg func(*registry.Client, *ocispec.Manifest) + expectErr string + expectContent string + expectType string + }{ + { + name: "invalid addition type", + addition: "invalid", + manifest: &ocispec.Manifest{}, + setupMockReg: func(r *registry.Client, m *ocispec.Manifest) { + manifestJSON, err := json.Marshal(m) + p.Require().NoError(err) + manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, manifestJSON) + p.Require().NoError(err) + r.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil) + }, + expectErr: "addition invalid isn't supported for CNAI", + }, + { + name: "readme not found", + addition: AdditionTypeReadme, + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "other.txt", + }, + }, + }, + }, + setupMockReg: func(r *registry.Client, m *ocispec.Manifest) { + manifestJSON, err := json.Marshal(m) + p.Require().NoError(err) + manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, manifestJSON) + p.Require().NoError(err) + r.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil) + }, + expectErr: "readme layer not found", + }, + { + name: "valid readme", + addition: AdditionTypeReadme, + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "README.md", + }, + Digest: "sha256:abc", + }, + }, + }, + setupMockReg: func(r *registry.Client, m *ocispec.Manifest) { + manifestJSON, err := json.Marshal(m) + p.Require().NoError(err) + manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, manifestJSON) + p.Require().NoError(err) + r.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil) + + content := "# Test Model" + tarContent, err := createTarContent("README.md", content) + p.Require().NoError(err) + r.On("PullBlob", mock.Anything, "sha256:abc").Return( + int64(len(tarContent)), + io.NopCloser(bytes.NewReader(tarContent)), + nil, + ) + }, + expectContent: "# Test Model", + expectType: "text/markdown; charset=utf-8", + }, + { + name: "valid license", + addition: AdditionTypeLicense, + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "LICENSE", + }, + Digest: "sha256:def", + }, + }, + }, + setupMockReg: func(r *registry.Client, m *ocispec.Manifest) { + manifestJSON, err := json.Marshal(m) + p.Require().NoError(err) + manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, manifestJSON) + p.Require().NoError(err) + r.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil) + + content := "MIT License" + tarContent, err := createTarContent("LICENSE", content) + p.Require().NoError(err) + r.On("PullBlob", mock.Anything, "sha256:def").Return( + int64(len(tarContent)), + io.NopCloser(bytes.NewReader(tarContent)), + nil, + ) + }, + expectContent: "MIT License", + expectType: "text/plain; charset=utf-8", + }, + { + name: "valid files list", + addition: AdditionTypeFiles, + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Size: 100, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "model/weights.bin", + }, + }, + { + MediaType: modelspec.MediaTypeModelDoc, + Size: 50, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "config.json", + }, + }, + }, + }, + setupMockReg: func(r *registry.Client, m *ocispec.Manifest) { + manifestJSON, err := json.Marshal(m) + p.Require().NoError(err) + manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, manifestJSON) + p.Require().NoError(err) + r.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil) + }, + expectContent: `[{"name":"config.json","type":"file","size":50},{"name":"model","type":"directory","children":[{"name":"weights.bin","type":"file","size":100}]}]`, + expectType: "application/json; charset=utf-8", + }, + } + + for _, tc := range cases { + p.Run(tc.name, func() { + // Reset mock + p.SetupTest() + + if tc.setupMockReg != nil { + tc.setupMockReg(p.regCli, tc.manifest) + } + + addition, err := p.processor.AbstractAddition( + context.Background(), + &artifact.Artifact{}, + tc.addition, + ) + + if tc.expectErr != "" { + p.Error(err) + p.Contains(err.Error(), tc.expectErr) + return + } + + p.NoError(err) + if tc.expectContent != "" { + p.Equal(tc.expectContent, string(addition.Content)) + } + if tc.expectType != "" { + p.Equal(tc.expectType, addition.ContentType) + } + }) + } +} + +func (p *ProcessorTestSuite) TestGetArtifactType() { + p.Equal(ArtifactTypeCNAI, p.processor.GetArtifactType(nil, nil)) +} + +func (p *ProcessorTestSuite) TestListAdditionTypes() { + additions := p.processor.ListAdditionTypes(nil, nil) + p.ElementsMatch( + []string{ + AdditionTypeReadme, + AdditionTypeLicense, + AdditionTypeFiles, + }, + additions, + ) +} + +func TestProcessorTestSuite(t *testing.T) { + suite.Run(t, &ProcessorTestSuite{}) +} diff --git a/src/controller/artifact/processor/cnai/parser/base.go b/src/controller/artifact/processor/cnai/parser/base.go new file mode 100644 index 000000000..7575be7f5 --- /dev/null +++ b/src/controller/artifact/processor/cnai/parser/base.go @@ -0,0 +1,66 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "context" + "fmt" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/goharbor/harbor/src/pkg/registry" +) + +const ( + // contentTypeTextPlain is the content type of text/plain. + contentTypeTextPlain = "text/plain; charset=utf-8" + // contentTypeTextMarkdown is the content type of text/markdown. + contentTypeMarkdown = "text/markdown; charset=utf-8" + // contentTypeJSON is the content type of application/json. + contentTypeJSON = "application/json; charset=utf-8" +) + +// newBase creates a new base parser. +func newBase(cli registry.Client) *base { + return &base{ + regCli: cli, + } +} + +// base provides a default implementation for other parsers to build upon. +type base struct { + regCli registry.Client +} + +// Parse is the common implementation for parsing layer. +func (b *base) Parse(_ context.Context, artifact *artifact.Artifact, layer *ocispec.Descriptor) (string, []byte, error) { + if artifact == nil || layer == nil { + return "", nil, fmt.Errorf("artifact or manifest cannot be nil") + } + + _, stream, err := b.regCli.PullBlob(artifact.RepositoryName, layer.Digest.String()) + if err != nil { + return "", nil, fmt.Errorf("failed to pull blob from registry: %w", err) + } + + defer stream.Close() + content, err := untar(stream) + if err != nil { + return "", nil, fmt.Errorf("failed to untar the content: %w", err) + } + + return contentTypeTextPlain, content, nil +} diff --git a/src/controller/artifact/processor/cnai/parser/base_test.go b/src/controller/artifact/processor/cnai/parser/base_test.go new file mode 100644 index 000000000..cd351fbad --- /dev/null +++ b/src/controller/artifact/processor/cnai/parser/base_test.go @@ -0,0 +1,113 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "archive/tar" + "bytes" + "context" + "fmt" + "io" + "testing" + + "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/goharbor/harbor/src/pkg/registry" + mock "github.com/goharbor/harbor/src/testing/pkg/registry" + + v1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" +) + +func TestBaseParse(t *testing.T) { + tests := []struct { + name string + artifact *artifact.Artifact + layer *v1.Descriptor + mockSetup func(*mock.Client) + expectedType string + expectedError string + }{ + { + name: "nil artifact", + artifact: nil, + layer: &v1.Descriptor{}, + expectedError: "artifact or manifest cannot be nil", + }, + { + name: "nil layer", + artifact: &artifact.Artifact{}, + layer: nil, + expectedError: "artifact or manifest cannot be nil", + }, + { + name: "registry client error", + artifact: &artifact.Artifact{RepositoryName: "test/repo"}, + layer: &v1.Descriptor{ + Digest: "sha256:1234", + }, + mockSetup: func(m *mock.Client) { + m.On("PullBlob", "test/repo", "sha256:1234").Return(int64(0), nil, fmt.Errorf("registry error")) + }, + expectedError: "failed to pull blob from registry: registry error", + }, + { + name: "successful parse", + artifact: &artifact.Artifact{RepositoryName: "test/repo"}, + layer: &v1.Descriptor{ + Digest: "sha256:1234", + }, + mockSetup: func(m *mock.Client) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + + tw.WriteHeader(&tar.Header{ + Name: "test.txt", + Size: 12, + }) + tw.Write([]byte("test content")) + tw.Close() + m.On("PullBlob", "test/repo", "sha256:1234").Return(int64(0), io.NopCloser(bytes.NewReader(buf.Bytes())), nil) + }, + expectedType: contentTypeTextPlain, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockClient := &mock.Client{} + if tt.mockSetup != nil { + tt.mockSetup(mockClient) + } + + b := &base{regCli: mockClient} + contentType, _, err := b.Parse(context.Background(), tt.artifact, tt.layer) + + if tt.expectedError != "" { + assert.EqualError(t, err, tt.expectedError) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedType, contentType) + } + + mockClient.AssertExpectations(t) + }) + } +} + +func TestNewBase(t *testing.T) { + b := newBase(registry.Cli) + assert.NotNil(t, b) + assert.Equal(t, registry.Cli, b.regCli) +} diff --git a/src/controller/artifact/processor/cnai/parser/files.go b/src/controller/artifact/processor/cnai/parser/files.go new file mode 100644 index 000000000..bc3ed05b6 --- /dev/null +++ b/src/controller/artifact/processor/cnai/parser/files.go @@ -0,0 +1,109 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "context" + "encoding/json" + "fmt" + "sort" + + modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/goharbor/harbor/src/pkg/registry" +) + +// NewFiles creates a new files parser. +func NewFiles(cli registry.Client) Parser { + return &files{ + base: newBase(cli), + } +} + +// files is the parser for listing files in the model artifact. +type files struct { + *base +} + +type FileList struct { + Name string `json:"name"` + Type string `json:"type"` + Size int64 `json:"size,omitempty"` + Children []FileList `json:"children,omitempty"` +} + +// Parse parses the files list. +func (f *files) Parse(_ context.Context, _ *artifact.Artifact, manifest *ocispec.Manifest) (string, []byte, error) { + if manifest == nil { + return "", nil, fmt.Errorf("manifest cannot be nil") + } + + rootNode, err := walkManifest(*manifest) + if err != nil { + return "", nil, fmt.Errorf("failed to walk manifest: %w", err) + } + + fileLists := traverseFileNode(rootNode) + data, err := json.Marshal(fileLists) + if err != nil { + return "", nil, err + } + + return contentTypeJSON, data, nil +} + +// walkManifest walks the manifest and returns the root file node. +func walkManifest(manifest ocispec.Manifest) (*FileNode, error) { + root := NewDirectory("/") + for _, layer := range manifest.Layers { + if layer.Annotations != nil && layer.Annotations[modelspec.AnnotationFilepath] != "" { + filepath := layer.Annotations[modelspec.AnnotationFilepath] + // mark it to directory if the file path ends with "/". + isDir := filepath[len(filepath)-1] == '/' + _, err := root.AddNode(filepath, layer.Size, isDir) + if err != nil { + return nil, err + } + } + } + + return root, nil +} + +// traverseFileNode traverses the file node and returns the file list. +func traverseFileNode(node *FileNode) []FileList { + if node == nil { + return nil + } + + var children []FileList + for _, child := range node.Children { + children = append(children, FileList{ + Name: child.Name, + Type: child.Type, + Size: child.Size, + Children: traverseFileNode(child), + }) + } + + // sort the children by name. + sort.Slice(children, func(i, j int) bool { + return children[i].Name < children[j].Name + }) + + return children +} diff --git a/src/controller/artifact/processor/cnai/parser/files_test.go b/src/controller/artifact/processor/cnai/parser/files_test.go new file mode 100644 index 000000000..1c610931a --- /dev/null +++ b/src/controller/artifact/processor/cnai/parser/files_test.go @@ -0,0 +1,229 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "context" + "encoding/json" + "testing" + + modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + + "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/goharbor/harbor/src/pkg/registry" + mockregistry "github.com/goharbor/harbor/src/testing/pkg/registry" +) + +func TestFilesParser(t *testing.T) { + tests := []struct { + name string + manifest *ocispec.Manifest + expectedType string + expectedOutput []FileList + expectedError string + }{ + { + name: "nil manifest", + manifest: nil, + expectedError: "manifest cannot be nil", + }, + { + name: "empty manifest layers", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{}, + }, + expectedType: contentTypeJSON, + expectedOutput: nil, + }, + { + name: "single file", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Size: 100, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "model.bin", + }, + }, + }, + }, + expectedType: contentTypeJSON, + expectedOutput: []FileList{ + { + Name: "model.bin", + Type: TypeFile, + Size: 100, + }, + }, + }, + { + name: "file in directory", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Size: 200, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "models/v1/model.bin", + }, + }, + }, + }, + expectedType: contentTypeJSON, + expectedOutput: []FileList{ + { + Name: "models", + Type: TypeDirectory, + Children: []FileList{ + { + Name: "v1", + Type: TypeDirectory, + Children: []FileList{ + { + Name: "model.bin", + Type: TypeFile, + Size: 200, + }, + }, + }, + }, + }, + }, + }, + { + name: "multiple files and directories", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Size: 100, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "README.md", + }, + }, + { + MediaType: modelspec.MediaTypeModelDoc, + Size: 200, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "models/v1/model.bin", + }, + }, + { + MediaType: modelspec.MediaTypeModelDoc, + Size: 300, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "models/v2/", + }, + }, + { + MediaType: modelspec.MediaTypeModelDoc, + Size: 150, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "models/v2/model.bin", + }, + }, + }, + }, + expectedType: contentTypeJSON, + expectedOutput: []FileList{ + { + Name: "README.md", + Type: TypeFile, + Size: 100, + }, + { + Name: "models", + Type: TypeDirectory, + Children: []FileList{ + { + Name: "v1", + Type: TypeDirectory, + Children: []FileList{ + { + Name: "model.bin", + Type: TypeFile, + Size: 200, + }, + }, + }, + { + Name: "v2", + Type: TypeDirectory, + Children: []FileList{ + { + Name: "model.bin", + Type: TypeFile, + Size: 150, + }, + }, + }, + }, + }, + }, + }, + { + name: "layer without filepath annotation", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Size: 100, + Annotations: map[string]string{}, + }, + }, + }, + expectedType: contentTypeJSON, + expectedOutput: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockRegClient := &mockregistry.Client{} + parser := &files{ + base: &base{ + regCli: mockRegClient, + }, + } + + contentType, content, err := parser.Parse(context.Background(), &artifact.Artifact{}, tt.manifest) + + if tt.expectedError != "" { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedType, contentType) + + var fileList []FileList + err = json.Unmarshal(content, &fileList) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, fileList) + } + }) + } +} + +func TestNewFiles(t *testing.T) { + parser := NewFiles(registry.Cli) + assert.NotNil(t, parser) + + filesParser, ok := parser.(*files) + assert.True(t, ok, "Parser should be of type *files") + assert.Equal(t, registry.Cli, filesParser.base.regCli) +} diff --git a/src/controller/artifact/processor/cnai/parser/license.go b/src/controller/artifact/processor/cnai/parser/license.go new file mode 100644 index 000000000..f4e325301 --- /dev/null +++ b/src/controller/artifact/processor/cnai/parser/license.go @@ -0,0 +1,66 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "context" + "fmt" + + modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/goharbor/harbor/src/pkg/registry" +) + +// NewLicense creates a new license parser. +func NewLicense(cli registry.Client) Parser { + return &license{ + base: newBase(cli), + } +} + +// license is the parser for License file. +type license struct { + *base +} + +// Parse parses the License file. +func (l *license) Parse(ctx context.Context, artifact *artifact.Artifact, manifest *ocispec.Manifest) (string, []byte, error) { + if manifest == nil { + return "", nil, errors.New("manifest cannot be nil") + } + + // lookup the license file layer + var layer *ocispec.Descriptor + for _, desc := range manifest.Layers { + if desc.MediaType == modelspec.MediaTypeModelDoc { + if desc.Annotations != nil { + filepath := desc.Annotations[modelspec.AnnotationFilepath] + if filepath == "LICENSE" || filepath == "LICENSE.txt" { + layer = &desc + break + } + } + } + } + + if layer == nil { + return "", nil, errors.NotFoundError(fmt.Errorf("license layer not found")) + } + + return l.base.Parse(ctx, artifact, layer) +} diff --git a/src/controller/artifact/processor/cnai/parser/license_test.go b/src/controller/artifact/processor/cnai/parser/license_test.go new file mode 100644 index 000000000..6dee2d852 --- /dev/null +++ b/src/controller/artifact/processor/cnai/parser/license_test.go @@ -0,0 +1,237 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "archive/tar" + "bytes" + "context" + "fmt" + "io" + "testing" + + modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + + "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/goharbor/harbor/src/pkg/registry" + "github.com/goharbor/harbor/src/testing/mock" + mockregistry "github.com/goharbor/harbor/src/testing/pkg/registry" +) + +func TestLicenseParser(t *testing.T) { + tests := []struct { + name string + manifest *ocispec.Manifest + setupMockReg func(*mockregistry.Client) + expectedType string + expectedOutput []byte + expectedError string + }{ + { + name: "nil manifest", + manifest: nil, + expectedError: "manifest cannot be nil", + }, + { + name: "empty manifest layers", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{}, + }, + expectedError: "license layer not found", + }, + { + name: "LICENSE parse success", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "LICENSE", + }, + Digest: "sha256:abc123", + }, + }, + }, + setupMockReg: func(mc *mockregistry.Client) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + content := []byte("MIT License") + _ = tw.WriteHeader(&tar.Header{ + Name: "LICENSE", + Size: int64(len(content)), + }) + _, _ = tw.Write(content) + tw.Close() + + mc.On("PullBlob", mock.Anything, "sha256:abc123"). + Return(int64(buf.Len()), io.NopCloser(bytes.NewReader(buf.Bytes())), nil) + }, + expectedType: contentTypeTextPlain, + expectedOutput: []byte("MIT License"), + }, + { + name: "LICENSE.txt parse success", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "LICENSE.txt", + }, + Digest: "sha256:def456", + }, + }, + }, + setupMockReg: func(mc *mockregistry.Client) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + content := []byte("Apache License 2.0") + _ = tw.WriteHeader(&tar.Header{ + Name: "LICENSE.txt", + Size: int64(len(content)), + }) + _, _ = tw.Write(content) + tw.Close() + + mc.On("PullBlob", mock.Anything, "sha256:def456"). + Return(int64(buf.Len()), io.NopCloser(bytes.NewReader(buf.Bytes())), nil) + }, + expectedType: contentTypeTextPlain, + expectedOutput: []byte("Apache License 2.0"), + }, + { + name: "registry error", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "LICENSE", + }, + Digest: "sha256:ghi789", + }, + }, + }, + setupMockReg: func(mc *mockregistry.Client) { + mc.On("PullBlob", mock.Anything, "sha256:ghi789"). + Return(int64(0), nil, fmt.Errorf("registry error")) + }, + expectedError: "failed to pull blob from registry: registry error", + }, + { + name: "multiple layers with license", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "other.txt", + }, + }, + { + MediaType: modelspec.MediaTypeModelDoc, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "LICENSE", + }, + Digest: "sha256:jkl012", + }, + }, + }, + setupMockReg: func(mc *mockregistry.Client) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + content := []byte("BSD License") + _ = tw.WriteHeader(&tar.Header{ + Name: "LICENSE", + Size: int64(len(content)), + }) + _, _ = tw.Write(content) + tw.Close() + + mc.On("PullBlob", mock.Anything, "sha256:jkl012"). + Return(int64(buf.Len()), io.NopCloser(bytes.NewReader(buf.Bytes())), nil) + }, + expectedType: contentTypeTextPlain, + expectedOutput: []byte("BSD License"), + }, + { + name: "wrong media type", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: "wrong/type", + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "LICENSE", + }, + }, + }, + }, + expectedError: "license layer not found", + }, + { + name: "no matching license file", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "NOT_LICENSE", + }, + }, + }, + }, + expectedError: "license layer not found", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockRegClient := &mockregistry.Client{} + if tt.setupMockReg != nil { + tt.setupMockReg(mockRegClient) + } + + parser := &license{ + base: &base{ + regCli: mockRegClient, + }, + } + + contentType, content, err := parser.Parse(context.Background(), &artifact.Artifact{}, tt.manifest) + + if tt.expectedError != "" { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedType, contentType) + assert.Equal(t, tt.expectedOutput, content) + } + + mockRegClient.AssertExpectations(t) + }) + } +} + +func TestNewLicense(t *testing.T) { + parser := NewLicense(registry.Cli) + assert.NotNil(t, parser) + + licenseParser, ok := parser.(*license) + assert.True(t, ok, "Parser should be of type *license") + assert.Equal(t, registry.Cli, licenseParser.base.regCli) +} diff --git a/src/controller/artifact/processor/cnai/parser/parser.go b/src/controller/artifact/processor/cnai/parser/parser.go new file mode 100644 index 000000000..c9cc112ed --- /dev/null +++ b/src/controller/artifact/processor/cnai/parser/parser.go @@ -0,0 +1,29 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "context" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/goharbor/harbor/src/pkg/artifact" +) + +// Parser is the interface for parsing the content by different addition type. +type Parser interface { + // Parse returns the parsed content type and content. + Parse(ctx context.Context, artifact *artifact.Artifact, manifest *ocispec.Manifest) (contentType string, content []byte, err error) +} diff --git a/src/controller/artifact/processor/cnai/parser/readme.go b/src/controller/artifact/processor/cnai/parser/readme.go new file mode 100644 index 000000000..0e2c556e1 --- /dev/null +++ b/src/controller/artifact/processor/cnai/parser/readme.go @@ -0,0 +1,71 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "context" + "fmt" + + modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/goharbor/harbor/src/pkg/registry" +) + +// NewReadme creates a new readme parser. +func NewReadme(cli registry.Client) Parser { + return &readme{ + base: newBase(cli), + } +} + +// readme is the parser for README.md file. +type readme struct { + *base +} + +// Parse parses the README.md file. +func (r *readme) Parse(ctx context.Context, artifact *artifact.Artifact, manifest *ocispec.Manifest) (string, []byte, error) { + if manifest == nil { + return "", nil, errors.New("manifest cannot be nil") + } + + // lookup the readme file layer. + var layer *ocispec.Descriptor + for _, desc := range manifest.Layers { + if desc.MediaType == modelspec.MediaTypeModelDoc { + if desc.Annotations != nil { + filepath := desc.Annotations[modelspec.AnnotationFilepath] + if filepath == "README" || filepath == "README.md" { + layer = &desc + break + } + } + } + } + + if layer == nil { + return "", nil, errors.NotFoundError(fmt.Errorf("readme layer not found")) + } + + _, content, err := r.base.Parse(ctx, artifact, layer) + if err != nil { + return "", nil, err + } + + return contentTypeMarkdown, content, nil +} diff --git a/src/controller/artifact/processor/cnai/parser/readme_test.go b/src/controller/artifact/processor/cnai/parser/readme_test.go new file mode 100644 index 000000000..2745d0a82 --- /dev/null +++ b/src/controller/artifact/processor/cnai/parser/readme_test.go @@ -0,0 +1,209 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "archive/tar" + "bytes" + "context" + "fmt" + "io" + "testing" + + modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1" + "github.com/goharbor/harbor/src/testing/mock" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + + "github.com/goharbor/harbor/src/pkg/artifact" + "github.com/goharbor/harbor/src/pkg/registry" + mockregistry "github.com/goharbor/harbor/src/testing/pkg/registry" +) + +func TestReadmeParser(t *testing.T) { + tests := []struct { + name string + manifest *ocispec.Manifest + setupMockReg func(*mockregistry.Client) + expectedType string + expectedOutput []byte + expectedError string + }{ + { + name: "nil manifest", + manifest: nil, + expectedError: "manifest cannot be nil", + }, + { + name: "empty manifest layers", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{}, + }, + expectedError: "readme layer not found", + }, + { + name: "README.md parse success", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "README.md", + }, + Digest: "sha256:abc123", + }, + }, + }, + setupMockReg: func(mc *mockregistry.Client) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + content := []byte("# Test README") + _ = tw.WriteHeader(&tar.Header{ + Name: "README.md", + Size: int64(len(content)), + }) + _, _ = tw.Write(content) + tw.Close() + + mc.On("PullBlob", mock.Anything, "sha256:abc123"). + Return(int64(buf.Len()), io.NopCloser(bytes.NewReader(buf.Bytes())), nil) + }, + expectedType: contentTypeMarkdown, + expectedOutput: []byte("# Test README"), + }, + { + name: "README parse success", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "README", + }, + Digest: "sha256:def456", + }, + }, + }, + setupMockReg: func(mc *mockregistry.Client) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + content := []byte("# Test README") + _ = tw.WriteHeader(&tar.Header{ + Name: "README", + Size: int64(len(content)), + }) + _, _ = tw.Write(content) + tw.Close() + + mc.On("PullBlob", mock.Anything, "sha256:def456"). + Return(int64(buf.Len()), io.NopCloser(bytes.NewReader(buf.Bytes())), nil) + }, + expectedType: contentTypeMarkdown, + expectedOutput: []byte("# Test README"), + }, + { + name: "registry error", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "README.md", + }, + Digest: "sha256:ghi789", + }, + }, + }, + setupMockReg: func(mc *mockregistry.Client) { + mc.On("PullBlob", mock.Anything, "sha256:ghi789"). + Return(int64(0), nil, fmt.Errorf("registry error")) + }, + expectedError: "failed to pull blob from registry: registry error", + }, + { + name: "multiple layers with README", + manifest: &ocispec.Manifest{ + Layers: []ocispec.Descriptor{ + { + MediaType: modelspec.MediaTypeModelDoc, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "other.txt", + }, + }, + { + MediaType: modelspec.MediaTypeModelDoc, + Annotations: map[string]string{ + modelspec.AnnotationFilepath: "README.md", + }, + Digest: "sha256:jkl012", + }, + }, + }, + setupMockReg: func(mc *mockregistry.Client) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + content := []byte("# Second README") + _ = tw.WriteHeader(&tar.Header{ + Name: "README.md", + Size: int64(len(content)), + }) + _, _ = tw.Write(content) + tw.Close() + + mc.On("PullBlob", mock.Anything, "sha256:jkl012"). + Return(int64(buf.Len()), io.NopCloser(bytes.NewReader(buf.Bytes())), nil) + }, + expectedType: contentTypeMarkdown, + expectedOutput: []byte("# Second README"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockRegClient := &mockregistry.Client{} + if tt.setupMockReg != nil { + tt.setupMockReg(mockRegClient) + } + + parser := &readme{ + base: &base{ + regCli: mockRegClient, + }, + } + + contentType, content, err := parser.Parse(context.Background(), &artifact.Artifact{}, tt.manifest) + + if tt.expectedError != "" { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedType, contentType) + assert.Equal(t, tt.expectedOutput, content) + } + + mockRegClient.AssertExpectations(t) + }) + } +} + +func TestNewReadme(t *testing.T) { + parser := NewReadme(registry.Cli) + assert.NotNil(t, parser) + + readmeParser, ok := parser.(*readme) + assert.True(t, ok, "Parser should be of type *readme") + assert.Equal(t, registry.Cli, readmeParser.base.regCli) +} diff --git a/src/controller/artifact/processor/cnai/parser/util.go b/src/controller/artifact/processor/cnai/parser/util.go new file mode 100644 index 000000000..fb6583fe6 --- /dev/null +++ b/src/controller/artifact/processor/cnai/parser/util.go @@ -0,0 +1,150 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "archive/tar" + "bytes" + "fmt" + "io" + "path/filepath" + "strings" + "sync" +) + +func untar(reader io.Reader) ([]byte, error) { + tr := tar.NewReader(reader) + var buf bytes.Buffer + for { + header, err := tr.Next() + if err == io.EOF { + break + } + + if err != nil { + return nil, fmt.Errorf("failed to read tar header: %w", err) + } + + // skip the directory. + if header.Typeflag == tar.TypeDir { + continue + } + + if _, err := io.Copy(&buf, tr); err != nil { + return nil, fmt.Errorf("failed to copy content to buffer: %w", err) + } + } + + return buf.Bytes(), nil +} + +// FileType represents the type of a file. +type FileType = string + +const ( + TypeFile FileType = "file" + TypeDirectory FileType = "directory" +) + +type FileNode struct { + Name string + Type FileType + Size int64 + Children map[string]*FileNode + mu sync.RWMutex +} + +func NewFile(name string, size int64) *FileNode { + return &FileNode{ + Name: name, + Type: TypeFile, + Size: size, + } +} + +func NewDirectory(name string) *FileNode { + return &FileNode{ + Name: name, + Type: TypeDirectory, + Children: make(map[string]*FileNode), + } +} + +func (root *FileNode) AddChild(child *FileNode) error { + root.mu.Lock() + defer root.mu.Unlock() + + if root.Type != TypeDirectory { + return fmt.Errorf("cannot add child to non-directory node") + } + + root.Children[child.Name] = child + return nil +} + +func (root *FileNode) GetChild(name string) (*FileNode, bool) { + root.mu.RLock() + defer root.mu.RUnlock() + + child, ok := root.Children[name] + return child, ok +} + +func (root *FileNode) AddNode(path string, size int64, isDir bool) (*FileNode, error) { + path = filepath.Clean(path) + parts := strings.Split(path, string(filepath.Separator)) + + current := root + for i, part := range parts { + if part == "" { + continue + } + + isLastPart := i == len(parts)-1 + child, exists := current.GetChild(part) + if !exists { + var newNode *FileNode + if isLastPart { + if isDir { + newNode = NewDirectory(part) + } else { + newNode = NewFile(part, size) + } + } else { + newNode = NewDirectory(part) + } + + if err := current.AddChild(newNode); err != nil { + return nil, err + } + + current = newNode + } else { + child.mu.RLock() + nodeType := child.Type + child.mu.RUnlock() + + if isLastPart { + if (isDir && nodeType != TypeDirectory) || (!isDir && nodeType != TypeFile) { + return nil, fmt.Errorf("path conflicts: %s exists with different type", part) + } + } + + current = child + } + } + + return current, nil +} diff --git a/src/controller/artifact/processor/cnai/parser/util_test.go b/src/controller/artifact/processor/cnai/parser/util_test.go new file mode 100644 index 000000000..1ebc9ad30 --- /dev/null +++ b/src/controller/artifact/processor/cnai/parser/util_test.go @@ -0,0 +1,173 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "archive/tar" + "bytes" + "path/filepath" + "strings" + "testing" +) + +func TestUntar(t *testing.T) { + tests := []struct { + name string + content string + wantErr bool + expected string + }{ + { + name: "valid tar file with single file", + content: "test content", + wantErr: false, + expected: "test content", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + + hdr := &tar.Header{ + Name: "test.txt", + Mode: 0600, + Size: int64(len(tt.content)), + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(tt.content)); err != nil { + t.Fatal(err) + } + tw.Close() + + result, err := untar(&buf) + if (err != nil) != tt.wantErr { + t.Errorf("untar() error = %v, wantErr %v", err, tt.wantErr) + return + } + if string(result) != tt.expected { + t.Errorf("untar() = %v, want %v", string(result), tt.expected) + } + }) + } +} + +func TestFileNode(t *testing.T) { + t.Run("test file node operations", func(t *testing.T) { + // Test creating root directory. + root := NewDirectory("root") + if root.Type != TypeDirectory { + t.Errorf("Expected directory type, got %s", root.Type) + } + + // Test creating file. + file := NewFile("test.txt", 100) + if file.Type != TypeFile { + t.Errorf("Expected file type, got %s", file.Type) + } + + // Test adding child to directory. + err := root.AddChild(file) + if err != nil { + t.Errorf("Failed to add child: %v", err) + } + + // Test getting child. + child, exists := root.GetChild("test.txt") + if !exists { + t.Error("Expected child to exist") + } + if child.Name != "test.txt" { + t.Errorf("Expected name test.txt, got %s", child.Name) + } + + // Test adding child to file (should fail). + err = file.AddChild(NewFile("invalid.txt", 50)) + if err == nil { + t.Error("Expected error when adding child to file") + } + }) +} + +func TestAddNode(t *testing.T) { + tests := []struct { + name string + path string + size int64 + isDir bool + wantErr bool + setupFn func(*FileNode) + }{ + { + name: "add file", + path: "dir1/dir2/file.txt", + size: 100, + isDir: false, + wantErr: false, + }, + { + name: "add directory", + path: "dir1/dir2/dir3", + size: 0, + isDir: true, + wantErr: false, + }, + { + name: "add file with conflicting directory", + path: "dir1/dir2", + size: 100, + isDir: false, + wantErr: true, + setupFn: func(node *FileNode) { + node.AddNode("dir1/dir2", 0, true) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := NewDirectory("root") + if tt.setupFn != nil { + tt.setupFn(root) + } + + _, err := root.AddNode(tt.path, tt.size, tt.isDir) + if (err != nil) != tt.wantErr { + t.Errorf("AddNode() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !tt.wantErr { + // Verify the path exists. + current := root + parts := filepath.Clean(tt.path) + for _, part := range strings.Split(parts, string(filepath.Separator)) { + if part == "" { + continue + } + child, exists := current.GetChild(part) + if !exists { + t.Errorf("Expected path part %s to exist", part) + return + } + current = child + } + } + }) + } +} diff --git a/src/controller/icon/controller.go b/src/controller/icon/controller.go index f9787ce90..3bab2641f 100644 --- a/src/controller/icon/controller.go +++ b/src/controller/icon/controller.go @@ -77,6 +77,10 @@ var ( path: "./icons/default.png", resize: true, }, + icon.DigestOfIconCNAI: { + path: "./icons/cnai.png", + resize: true, + }, } // Ctl is a global icon controller instance Ctl = NewController() diff --git a/src/go.mod b/src/go.mod index 8222e4e28..16933e157 100644 --- a/src/go.mod +++ b/src/go.mod @@ -94,6 +94,7 @@ require ( github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect + github.com/CloudNativeAI/model-spec v0.0.1 // indirect github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect github.com/Masterminds/semver/v3 v3.3.0 // indirect github.com/Unknwon/goconfig v0.0.0-20160216183935-5f601ca6ef4d // indirect diff --git a/src/go.sum b/src/go.sum index eca6cc47d..b39db27ac 100644 --- a/src/go.sum +++ b/src/go.sum @@ -40,6 +40,8 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzS github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/CloudNativeAI/model-spec v0.0.1 h1:BgVIStKTLuL1DrLC5A/gmHcR8TEhFCDz9+fYdCUa/CY= +github.com/CloudNativeAI/model-spec v0.0.1/go.mod h1:3U/4zubBfbUkW59ATSg41HnkYyKrKUcKFH/cVdoPQnk= github.com/FZambia/sentinel v1.1.0 h1:qrCBfxc8SvJihYNjBWgwUI93ZCvFe/PJIPTHKmlp8a8= github.com/FZambia/sentinel v1.1.0/go.mod h1:ytL1Am/RLlAoAXG6Kj5LNuw/TRRQrv2rt2FT26vP5gI= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= diff --git a/src/lib/icon/const.go b/src/lib/icon/const.go index 3529cf46e..e76938242 100644 --- a/src/lib/icon/const.go +++ b/src/lib/icon/const.go @@ -21,6 +21,7 @@ const ( DigestOfIconCNAB = "sha256:089bdda265c14d8686111402c8ad629e8177a1ceb7dcd0f7f39b6480f623b3bd" DigestOfIconDefault = "sha256:da834479c923584f4cbcdecc0dac61f32bef1d51e8aae598cf16bd154efab49f" DigestOfIconWASM = "sha256:badd7693bcaf115be202748241dd0ea6ee3b0524bfab9ac22d1e1c43721afec6" + DigestOfIconCNAI = "sha256:1e1e5c5fdaf0931ec8655e835d1182f723a0c322a6760211622e1270f0193717" // ToDo add the accessories images DigestOfIconAccDefault = "" diff --git a/src/pkg/artifact/model.go b/src/pkg/artifact/model.go index 464a31924..cdf187e62 100644 --- a/src/pkg/artifact/model.go +++ b/src/pkg/artifact/model.go @@ -48,6 +48,15 @@ type Artifact struct { References []*Reference `json:"references"` // child artifacts referenced by the parent artifact if the artifact is an index } +// ResolveArtifactType returns the artifact type of the artifact, prefer ArtifactType, use MediaType if ArtifactType is empty. +func (a *Artifact) ResolveArtifactType() string { + if a.ArtifactType != "" { + return a.ArtifactType + } + + return a.MediaType +} + func (a *Artifact) String() string { return fmt.Sprintf("%s@%s", a.RepositoryName, a.Digest) }