Merge pull request #7946 from bitsf/replication_gcr_1.9

gcr driver for replication
This commit is contained in:
Ziming 2019-06-27 14:16:07 +08:00 committed by GitHub
commit 50180f0d7a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 386 additions and 1 deletions

View File

@ -211,7 +211,7 @@ func (r *Repository) PushManifest(reference, mediaType string, payload []byte) (
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode == http.StatusCreated { if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusOK {
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")) digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return return
} }

View File

@ -34,6 +34,8 @@ import (
_ "github.com/goharbor/harbor/src/replication/adapter/native" _ "github.com/goharbor/harbor/src/replication/adapter/native"
// register the Huawei adapter // register the Huawei adapter
_ "github.com/goharbor/harbor/src/replication/adapter/huawei" _ "github.com/goharbor/harbor/src/replication/adapter/huawei"
// register the Google Gcr adapter
_ "github.com/goharbor/harbor/src/replication/adapter/googlegcr"
// register the AwsEcr adapter // register the AwsEcr adapter
_ "github.com/goharbor/harbor/src/replication/adapter/awsecr" _ "github.com/goharbor/harbor/src/replication/adapter/awsecr"
) )

View File

@ -0,0 +1,106 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package googlegcr
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
"net/http"
)
func init() {
if err := adp.RegisterFactory(model.RegistryTypeGoogleGcr, func(registry *model.Registry) (adp.Adapter, error) {
return newAdapter(registry)
}); err != nil {
log.Errorf("failed to register factory for %s: %v", model.RegistryTypeGoogleGcr, err)
return
}
log.Infof("the factory for adapter %s registered", model.RegistryTypeGoogleGcr)
}
func newAdapter(registry *model.Registry) (*adapter, error) {
var credential auth.Credential
if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 {
credential = auth.NewBasicAuthCredential(
registry.Credential.AccessKey,
registry.Credential.AccessSecret)
}
authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
}, credential)
reg, err := adp.NewDefaultImageRegistryWithCustomizedAuthorizer(registry, authorizer)
if err != nil {
return nil, err
}
return &adapter{
registry: registry,
DefaultImageRegistry: reg,
}, nil
}
type adapter struct {
*adp.DefaultImageRegistry
registry *model.Registry
}
var _ adp.Adapter = adapter{}
func (adapter) Info() (info *model.RegistryInfo, err error) {
return &model.RegistryInfo{
Type: model.RegistryTypeGoogleGcr,
SupportedResourceTypes: []model.ResourceType{
model.ResourceTypeImage,
},
SupportedResourceFilters: []*model.FilterStyle{
{
Type: model.FilterTypeName,
Style: model.FilterStyleTypeText,
},
{
Type: model.FilterTypeTag,
Style: model.FilterStyleTypeText,
},
},
SupportedTriggers: []model.TriggerType{
model.TriggerTypeManual,
model.TriggerTypeScheduled,
},
}, nil
}
// HealthCheck checks health status of a registry
func (a adapter) HealthCheck() (model.HealthStatus, error) {
var err error
if a.registry.Credential == nil ||
len(a.registry.Credential.AccessKey) == 0 || len(a.registry.Credential.AccessSecret) == 0 {
log.Errorf("no credential to ping registry %s", a.registry.URL)
return model.Unhealthy, nil
}
if err = a.PingGet(); err != nil {
log.Errorf("failed to ping registry %s: %v", a.registry.URL, err)
return model.Unhealthy, nil
}
return model.Healthy, nil
}
// PrepareForPush nothing need to do.
func (a adapter) PrepareForPush(resources []*model.Resource) error {
return nil
}

View File

@ -0,0 +1,161 @@
package googlegcr
import (
"fmt"
"github.com/goharbor/harbor/src/common/utils/test"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/stretchr/testify/assert"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
)
func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Server) {
server := test.NewServer(
&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/v2/_catalog",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte(`
{
"repositories": [
"test1"
]
}`))
},
},
&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/v2/{repo}/tags/list",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte(`
{
"name": "test1",
"tags": [
"latest"
]
}`))
},
},
&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/v2/",
Handler: func(w http.ResponseWriter, r *http.Request) {
fmt.Println(r.Method, r.URL)
if health {
w.WriteHeader(http.StatusOK)
} else {
w.WriteHeader(http.StatusBadRequest)
}
},
},
&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/",
Handler: func(w http.ResponseWriter, r *http.Request) {
fmt.Println(r.Method, r.URL)
w.WriteHeader(http.StatusOK)
},
},
&test.RequestHandlerMapping{
Method: http.MethodPost,
Pattern: "/",
Handler: func(w http.ResponseWriter, r *http.Request) {
fmt.Println(r.Method, r.URL)
if buf, e := ioutil.ReadAll(&io.LimitedReader{R: r.Body, N: 80}); e == nil {
fmt.Println("\t", string(buf))
}
w.WriteHeader(http.StatusOK)
},
},
)
registry := &model.Registry{
Type: model.RegistryTypeGoogleGcr,
URL: server.URL,
}
if hasCred {
registry.Credential = &model.Credential{
AccessKey: "_json_key",
AccessSecret: "ppp",
}
}
factory, err := adp.GetFactory(model.RegistryTypeGoogleGcr)
assert.Nil(t, err)
assert.NotNil(t, factory)
a, err := factory(registry)
assert.Nil(t, err)
return a.(*adapter), server
}
func TestAdapter_Info(t *testing.T) {
a, s := getMockAdapter(t, true, true)
defer s.Close()
info, err := a.Info()
assert.Nil(t, err)
assert.NotNil(t, info)
assert.EqualValues(t, 1, len(info.SupportedResourceTypes))
assert.EqualValues(t, model.ResourceTypeImage, info.SupportedResourceTypes[0])
}
func TestAdapter_HealthCheck(t *testing.T) {
a, s := getMockAdapter(t, false, true)
defer s.Close()
status, err := a.HealthCheck()
assert.Nil(t, err)
assert.NotNil(t, status)
assert.EqualValues(t, model.Unhealthy, status)
a, s = getMockAdapter(t, true, false)
defer s.Close()
status, err = a.HealthCheck()
assert.Nil(t, err)
assert.NotNil(t, status)
assert.EqualValues(t, model.Unhealthy, status)
a, s = getMockAdapter(t, true, true)
defer s.Close()
status, err = a.HealthCheck()
assert.Nil(t, err)
assert.NotNil(t, status)
assert.EqualValues(t, model.Healthy, status)
}
func TestAdapter_PrepareForPush(t *testing.T) {
a, s := getMockAdapter(t, true, true)
defer s.Close()
resources := []*model.Resource{
{
Type: model.ResourceTypeImage,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: "busybox",
},
},
},
}
err := a.PrepareForPush(resources)
assert.Nil(t, err)
}
func TestAdapter_FetchImages(t *testing.T) {
a, s := getMockAdapter(t, true, true)
defer s.Close()
resources, err := a.FetchImages([]*model.Filter{
{
Type: model.FilterTypeName,
Value: "*",
},
{
Type: model.FilterTypeTag,
Value: "*",
},
})
assert.Nil(t, err)
assert.NotNil(t, resources)
assert.Equal(t, 1, len(resources))
}

View File

@ -0,0 +1,113 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package googlegcr
import (
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
)
var _ adp.ImageRegistry = adapter{}
func (a adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
nameFilterPattern := ""
tagFilterPattern := ""
for _, filter := range filters {
switch filter.Type {
case model.FilterTypeName:
nameFilterPattern = filter.Value.(string)
case model.FilterTypeTag:
tagFilterPattern = filter.Value.(string)
}
}
repositories, err := a.filterRepositories(nameFilterPattern)
if err != nil {
return nil, err
}
var resources []*model.Resource
for _, repository := range repositories {
tags, err := a.filterTags(repository, tagFilterPattern)
if err != nil {
return nil, err
}
if len(tags) == 0 {
continue
}
resources = append(resources, &model.Resource{
Type: model.ResourceTypeImage,
Registry: a.registry,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: repository,
},
Vtags: tags,
},
})
}
return resources, nil
}
func (a adapter) filterRepositories(pattern string) ([]string, error) {
// if the pattern is a specific repository name, just returns the parsed repositories
// and will check the existence later when filtering the tags
if repositories, ok := util.IsSpecificPath(pattern); ok {
return repositories, nil
}
// search repositories from catalog api
repositories, err := a.Catalog()
if err != nil {
return nil, err
}
// if the pattern is null, just return the result of catalog API
if len(pattern) == 0 {
return repositories, nil
}
result := []string{}
for _, repository := range repositories {
match, err := util.Match(pattern, repository)
if err != nil {
return nil, err
}
if match {
result = append(result, repository)
}
}
return result, nil
}
func (a adapter) filterTags(repository, pattern string) ([]string, error) {
tags, err := a.ListTag(repository)
if err != nil {
return nil, err
}
if len(pattern) == 0 {
return tags, nil
}
var result []string
for _, tag := range tags {
match, err := util.Match(pattern, tag)
if err != nil {
return nil, err
}
if match {
result = append(result, tag)
}
}
return result, nil
}

View File

@ -26,6 +26,7 @@ const (
RegistryTypeDockerHub RegistryType = "docker-hub" RegistryTypeDockerHub RegistryType = "docker-hub"
RegistryTypeDockerRegistry RegistryType = "docker-registry" RegistryTypeDockerRegistry RegistryType = "docker-registry"
RegistryTypeHuawei RegistryType = "huawei-SWR" RegistryTypeHuawei RegistryType = "huawei-SWR"
RegistryTypeGoogleGcr RegistryType = "google-gcr"
RegistryTypeAwsEcr RegistryType = "aws-ecr" RegistryTypeAwsEcr RegistryType = "aws-ecr"
FilterStyleTypeText = "input" FilterStyleTypeText = "input"

View File

@ -35,6 +35,8 @@ import (
_ "github.com/goharbor/harbor/src/replication/adapter/native" _ "github.com/goharbor/harbor/src/replication/adapter/native"
// register the huawei adapter // register the huawei adapter
_ "github.com/goharbor/harbor/src/replication/adapter/huawei" _ "github.com/goharbor/harbor/src/replication/adapter/huawei"
// register the Google Gcr adapter
_ "github.com/goharbor/harbor/src/replication/adapter/googlegcr"
// register the AwsEcr adapter // register the AwsEcr adapter
_ "github.com/goharbor/harbor/src/replication/adapter/awsecr" _ "github.com/goharbor/harbor/src/replication/adapter/awsecr"
) )