From 2898117411667375867ec3f2bd60b0864c67230c Mon Sep 17 00:00:00 2001 From: fanjiankong Date: Wed, 2 Dec 2020 23:50:48 +0800 Subject: [PATCH] GHCR Provider Signed-off-by: fanjiankong --- .../job/impl/replication/replication.go | 2 + src/replication/adapter/githubcr/adapter.go | 199 +++++++++++++++ .../adapter/githubcr/adapter_test.go | 238 ++++++++++++++++++ src/replication/model/registry.go | 1 + src/replication/replication.go | 2 + 5 files changed, 442 insertions(+) create mode 100644 src/replication/adapter/githubcr/adapter.go create mode 100644 src/replication/adapter/githubcr/adapter_test.go diff --git a/src/jobservice/job/impl/replication/replication.go b/src/jobservice/job/impl/replication/replication.go index 164fbc8a3..634ac6126 100644 --- a/src/jobservice/job/impl/replication/replication.go +++ b/src/jobservice/job/impl/replication/replication.go @@ -56,6 +56,8 @@ import ( _ "github.com/goharbor/harbor/src/replication/adapter/artifacthub" // register the TencentCloud TCR adapter _ "github.com/goharbor/harbor/src/replication/adapter/tencentcr" + // register the Github Container Registry adapter + _ "github.com/goharbor/harbor/src/replication/adapter/githubcr" ) // Replication implements the job interface diff --git a/src/replication/adapter/githubcr/adapter.go b/src/replication/adapter/githubcr/adapter.go new file mode 100644 index 000000000..cbfc5ca08 --- /dev/null +++ b/src/replication/adapter/githubcr/adapter.go @@ -0,0 +1,199 @@ +package githubcr + +import ( + "errors" + "fmt" + "net/http" + + common_http "github.com/goharbor/harbor/src/common/http" + "github.com/goharbor/harbor/src/common/http/modifier" + "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/registry/auth/basic" + adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/adapter/native" + "github.com/goharbor/harbor/src/replication/filter" + "github.com/goharbor/harbor/src/replication/model" + "github.com/goharbor/harbor/src/replication/util" +) + +// !!!! Limits: +// - GHCR not support `/v2/_catalog`, access this API will return 404 status code. +// - NOT support DELETE manifest. + +func init() { + if err := adp.RegisterFactory(model.RegistryTypeGithubCR, new(factory)); err != nil { + log.Errorf("failed to register factory for %s: %v", model.RegistryTypeGithubCR, err) + return + } + log.Infof("the factory for adapter %s registered", model.RegistryTypeGithubCR) +} + +type factory struct{} + +// Create ... +func (f *factory) Create(r *model.Registry) (adp.Adapter, error) { + return newAdapter(r), nil +} + +// AdapterPattern ... +func (f *factory) AdapterPattern() *model.AdapterPattern { + return getAdapterPattern() +} + +func getAdapterPattern() *model.AdapterPattern { + return &model.AdapterPattern{ + EndpointPattern: &model.EndpointPattern{ + EndpointType: model.EndpointPatternTypeFix, + Endpoints: []*model.Endpoint{ + { + Key: "ghcr.io", + Value: "https://ghcr.io", + }, + }, + }, + } +} + +var ( + _ adp.Adapter = (*adapter)(nil) + _ adp.ArtifactRegistry = (*adapter)(nil) +) + +// adapter for to github container registry +type adapter struct { + client *common_http.Client + *native.Adapter + registry *model.Registry +} + +var _ adp.Adapter = &adapter{} + +func newAdapter(registry *model.Registry) *adapter { + var authorizer modifier.Modifier + if registry.Credential != nil { + authorizer = basic.NewAuthorizer( + registry.Credential.AccessKey, + registry.Credential.AccessSecret) + } + + var transport = util.GetHTTPTransport(registry.Insecure) + + return &adapter{ + Adapter: native.NewAdapter(registry), + registry: registry, + client: common_http.NewClient( + &http.Client{ + Transport: transport, + }, + authorizer, + ), + } +} + +// Info ... +func (a *adapter) Info() (info *model.RegistryInfo, err error) { + info = &model.RegistryInfo{ + Type: model.RegistryTypeGithubCR, + SupportedResourceTypes: []model.ResourceType{ + model.ResourceTypeImage, + }, + SupportedResourceFilters: []*model.FilterStyle{ + { + Type: model.FilterTypeName, + Style: model.FilterStyleTypeText, + }, + { + Type: model.FilterTypeTag, + Style: model.FilterStyleTypeText, + }, + }, + SupportedTriggers: []model.TriggerType{ + model.TriggerTypeManual, + model.TriggerTypeScheduled, + }, + } + return +} + +func (a *adapter) FetchArtifacts(filters []*model.Filter) (resources []*model.Resource, err error) { + pattern := "" + for _, filter := range filters { + if filter.Type == model.FilterTypeName { + pattern = filter.Value.(string) + break + } + } + var repositories []string + // if the pattern of repository name filter is a specific repository name, just returns + // the parsed repositories and will check the existence later when filtering the tags + if paths, ok := util.IsSpecificPath(pattern); ok { + repositories = paths + } else { + err = errors.New("Only support specific repository name") + return + } + + if len(repositories) == 0 { + return nil, nil + } + + var rawResources = make([]*model.Resource, len(repositories)) + runner := utils.NewLimitedConcurrentRunner(adp.MaxConcurrency) + + for i, r := range repositories { + index := i + repo := r + runner.AddTask(func() error { + + artifacts, err := a.listArtifacts(repo, filters) + if err != nil { + return fmt.Errorf("failed to list artifacts of repository %s: %v", repo, err) + } + if len(artifacts) == 0 { + return nil + } + rawResources[index] = &model.Resource{ + Type: model.ResourceTypeImage, + Registry: a.registry, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: repo, + }, + Artifacts: artifacts, + }, + } + + return nil + }) + } + if err = runner.Wait(); err != nil { + return nil, fmt.Errorf("failed to fetch artifacts: %v", err) + } + + for _, r := range rawResources { + if r != nil { + resources = append(resources, r) + } + } + + return resources, nil +} + +func (a *adapter) listArtifacts(repository string, filters []*model.Filter) ([]*model.Artifact, error) { + tags, err := a.ListTags(repository) + if err != nil { + return nil, err + } + var artifacts []*model.Artifact + for _, tag := range tags { + artifacts = append(artifacts, &model.Artifact{ + Tags: []string{tag}, + }) + } + return filter.DoFilterArtifacts(artifacts, filters) +} + +func (a *adapter) DeleteManifest(repository, reference string) error { + return nil +} diff --git a/src/replication/adapter/githubcr/adapter_test.go b/src/replication/adapter/githubcr/adapter_test.go new file mode 100644 index 000000000..38c41c626 --- /dev/null +++ b/src/replication/adapter/githubcr/adapter_test.go @@ -0,0 +1,238 @@ +package githubcr + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/replication/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_native_Info(t *testing.T) { + var registry = &model.Registry{URL: "abc"} + adapter := newAdapter(registry) + assert.NotNil(t, adapter) + + info, err := adapter.Info() + assert.Nil(t, err) + assert.NotNil(t, info) + assert.Equal(t, model.RegistryTypeGithubCR, info.Type) + assert.Equal(t, 1, len(info.SupportedResourceTypes)) + assert.Equal(t, 2, len(info.SupportedResourceFilters)) + assert.Equal(t, 2, len(info.SupportedTriggers)) + assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0]) +} + +func Test_getAdapterPattern(t *testing.T) { + var pattern = getAdapterPattern() + assert.NotNil(t, pattern) + assert.Equal(t, model.EndpointPatternTypeFix, pattern.EndpointPattern.EndpointType) +} + +func mockGHCR() (mock *httptest.Server) { + return test.NewServer( + &test.RequestHandlerMapping{ + Method: http.MethodGet, + Pattern: "/v2/_catalog", + Handler: func(w http.ResponseWriter, r *http.Request) { + r.Response.StatusCode = http.StatusNotFound + w.Write([]byte(``)) + }, + }, + &test.RequestHandlerMapping{ + Method: http.MethodGet, + Pattern: "/v2/kofj/a1/tags/list", + Handler: func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"name":"kofj/a1","tags":["tag11"]}`)) + }, + }, + &test.RequestHandlerMapping{ + Method: http.MethodGet, + Pattern: "/v2/kofj/b2/tags/list", + Handler: func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"name":"kofj/b2","tags":["tag11","tag2","tag13"]}`)) + }, + }, + &test.RequestHandlerMapping{ + Method: http.MethodGet, + Pattern: "/v2/kofj/c3/l3/tags/list", + Handler: func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"name":"kofj/c3/l3","tags":["tag4"]}`)) + }, + }, + ) +} + +func Test_native_FetchArtifacts(t *testing.T) { + var mock = mockGHCR() + defer mock.Close() + fmt.Println("mockGHCR URL: ", mock.URL) + + var registry = &model.Registry{ + Type: model.RegistryTypeDockerRegistry, + URL: mock.URL, + Insecure: true, + } + adapter := newAdapter(registry) + assert.NotNil(t, adapter) + + tests := []struct { + name string + filters []*model.Filter + want []*model.Resource + wantErr bool + }{ + { + name: "repository not exist", + filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "b1", + }, + }, + wantErr: false, + }, + { + name: "tag not exist", + filters: []*model.Filter{ + { + Type: model.FilterTypeTag, + Value: "this_tag_not_exist_in_the_mock_server", + }, + }, + wantErr: false, + }, + { + name: "no filters", + filters: []*model.Filter{}, + want: []*model.Resource{}, + wantErr: true, + }, + + { + name: "only special repository", + filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "kofj/a1", + }, + }, + want: []*model.Resource{ + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{Name: "kofj/a1"}, + Artifacts: []*model.Artifact{ + { + Tags: []string{"tag11"}, + }, + }, + }, + }, + }, + wantErr: false, + }, + + { + name: "only special tag", + filters: []*model.Filter{ + { + Type: model.FilterTypeTag, + Value: "tag11", + }, + }, + want: []*model.Resource{}, + wantErr: true, + }, + + { + name: "special repository and special tag", + filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "kofj/b2", + }, + { + Type: model.FilterTypeTag, + Value: "tag2", + }, + }, + want: []*model.Resource{ + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{Name: "kofj/b2"}, + Artifacts: []*model.Artifact{ + { + Tags: []string{"tag2"}, + }, + }, + }, + }, + }, + + wantErr: false, + }, + { + name: "only wildcard repository", + filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "kofj/b*", + }, + }, + want: []*model.Resource{}, + wantErr: true, + }, + { + name: "only wildcard tag", + filters: []*model.Filter{ + { + Type: model.FilterTypeTag, + Value: "tag1*", + }, + }, + want: []*model.Resource{}, + wantErr: true, + }, + { + name: "wildcard repository and wildcard tag", + filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "kofj/b*", + }, + { + Type: model.FilterTypeTag, + Value: "tag1*", + }, + }, + want: []*model.Resource{}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var resources, err = adapter.FetchArtifacts(tt.filters) + if tt.wantErr { + require.Len(t, resources, 0) + require.NotNil(t, err) + } else { + if err != nil { + t.Logf("Name=%s, error: %v", t.Name(), err) + } + require.Equal(t, len(tt.want), len(resources)) + for i, resource := range resources { + require.NotNil(t, resource.Metadata) + assert.Equal(t, tt.want[i].Metadata.Repository, resource.Metadata.Repository) + assert.ElementsMatch(t, tt.want[i].Metadata.Artifacts, resource.Metadata.Artifacts) + } + } + }) + } +} + +// *************************************** diff --git a/src/replication/model/registry.go b/src/replication/model/registry.go index 65c8e2eb9..005b9dee3 100644 --- a/src/replication/model/registry.go +++ b/src/replication/model/registry.go @@ -33,6 +33,7 @@ const ( RegistryTypeGitLab RegistryType = "gitlab" RegistryTypeDTR RegistryType = "dtr" RegistryTypeTencentTcr RegistryType = "tencent-tcr" + RegistryTypeGithubCR RegistryType = "github-ghcr" RegistryTypeHelmHub RegistryType = "helm-hub" RegistryTypeArtifactHub RegistryType = "artifact-hub" diff --git a/src/replication/replication.go b/src/replication/replication.go index 43af0db0e..c9d493b42 100644 --- a/src/replication/replication.go +++ b/src/replication/replication.go @@ -61,6 +61,8 @@ import ( _ "github.com/goharbor/harbor/src/replication/adapter/artifacthub" // register the TencentCloud TCR adapter _ "github.com/goharbor/harbor/src/replication/adapter/tencentcr" + // register the Github Container Registry adapter + _ "github.com/goharbor/harbor/src/replication/adapter/githubcr" ) var (