diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 29fb4581e..eb874e131 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -641,6 +641,179 @@ paths: $ref: '#/responses/401' '500': $ref: '#/responses/500' + /p2p/preheat/providers: + get: + summary: List P2P providers + description: List P2P providers + tags: + - preheat + operationId: ListProviders + parameters: + - $ref: '#/parameters/requestId' + responses: + '200': + description: Success + schema: + type: array + items: + $ref: '#/definitions/Metadata' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' + /p2p/preheat/instances: + get: + summary: List P2P provider instances + description: List P2P provider instances + tags: + - preheat + operationId: ListInstances + parameters: + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/page' + - $ref: '#/parameters/pageSize' + - $ref: '#/parameters/query' + responses: + '200': + description: Success + headers: + X-Total-Count: + description: The total count of preheating provider instances + type: integer + Link: + description: Link refers to the previous page and next page + type: string + schema: + type: array + items: + $ref: '#/definitions/Instance' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' + post: + summary: Create p2p provider instances + description: Create p2p provider instances + tags: + - preheat + operationId: CreateInstance + parameters: + - $ref: '#/parameters/requestId' + - name: instance + in: body + description: The JSON object of instance. + required: true + schema: + $ref: '#/definitions/Instance' + responses: + '201': + description: Response to insatnce created + schema: + $ref: '#/definitions/InstanceCreatedResp' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '409': + $ref: '#/responses/409' + '500': + $ref: '#/responses/500' + /p2p/preheat/instances/{instance_id}: + get: + summary: Get a P2P provider instance + description: Get a P2P provider instance + tags: + - preheat + operationId: GetInstance + parameters: + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/instanceId' + responses: + '200': + description: Success + schema: + $ref: '#/definitions/Instance' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' + delete: + summary: Delete the specified P2P provider instance + description: Delete the specified P2P provider instance + tags: + - preheat + operationId: DeleteInstance + parameters: + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/instanceId' + responses: + '200': + description: Instance ID deleted + schema: + $ref: '#/definitions/InstanceDeletedResp' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' + put: + summary: Update the specified P2P provider instance + description: Update the specified P2P provider instance + tags: + - preheat + operationId: UpdateInstance + parameters: + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/instanceId' + - name: propertySet + in: body + description: The property set to update + required: true + schema: + type: object + additionalProperties: + type: object + additionalProperties: true + responses: + '200': + description: Success + schema: + $ref: '#/definitions/InstanceUpdateResp' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' parameters: query: name: q @@ -701,6 +874,12 @@ parameters: required: false description: The size of per page default: 10 + instanceId: + name: instance_id + in: path + description: Instance ID + required: true + type: integer responses: '200': description: Success @@ -1098,3 +1277,86 @@ definitions: op_time: type: string description: The time when this operation is triggered. + Metadata: + type: object + properties: + id: + type: string + description: id + name: + type: string + description: name + icon: + type: string + description: icon + maintainers: + type: array + description: maintainers + items: + type: string + version: + type: string + description: version + source: + type: string + description: source + Instance: + type: object + properties: + id: + type: integer + description: Unique ID + name: + type: string + description: Instance name + description: + type: string + description: Description of instance + vendor: + type: string + description: Based on which driver, identified by ID + endpoint: + type: string + description: The service endpoint of this instance + auth_mode: + type: string + description: The authentication way supported + auth_info: + type: object + description: The auth credential data if exists + additionalProperties: + type: string + status: + type: string + description: The health status + enabled: + type: boolean + description: Whether the instance is activated or not + default: + type: boolean + description: Whether the instance is default or not + insecure: + type: boolean + description: Whether the instance endpoint is insecure or not + setup_timestamp: + type: integer + format: int64 + description: The timestamp of instance setting up + InstanceUpdateResp: + type: object + properties: + updated: + type: integer + description: ID of instance updated + InstanceDeletedResp: + type: object + properties: + removed: + type: integer + description: ID of instance removed + InstanceCreatedResp: + type: object + properties: + id: + type: integer + description: ID of instance created diff --git a/codecov.yml b/codecov.yml index 62472e751..ad312c580 100644 --- a/codecov.yml +++ b/codecov.yml @@ -27,14 +27,6 @@ comment: require_changes: no ignore: - - "**/*.md" - - "**/*.yml" - - "docs" - - "api" - - "make" - - "contrib" - - "tests" - - "tools" - "src/vendor" - - "src/server/v2.0/models/**/*" - - "src/server/v2.0/restapi/**/*" + - "src/github.com/goharbor/harbor/src/server/v2.0/restapi/**/*" + - "src/github.com/goharbor/harbor/src/server/v2.0/models" diff --git a/make/migrations/postgresql/0040_2.1.0_schema.up.sql b/make/migrations/postgresql/0040_2.1.0_schema.up.sql index 7e66e5138..3f2ecc380 100644 --- a/make/migrations/postgresql/0040_2.1.0_schema.up.sql +++ b/make/migrations/postgresql/0040_2.1.0_schema.up.sql @@ -37,6 +37,20 @@ ALTER TABLE blob ADD COLUMN IF NOT EXISTS version BIGINT default 0; CREATE INDEX IF NOT EXISTS idx_status ON blob (status); CREATE INDEX IF NOT EXISTS idx_version ON blob (version); +CREATE TABLE p2p_preheat_instance ( + id SERIAL PRIMARY KEY NOT NULL, + name varchar(255) NOT NULL, + description varchar(255), + vendor varchar(255) NOT NULL, + endpoint varchar(255) NOT NULL, + auth_mode varchar(255), + auth_data text, + enabled boolean, + is_default boolean, + insecure boolean, + setup_timestamp int +); + CREATE TABLE IF NOT EXISTS p2p_preheat_policy ( id SERIAL PRIMARY KEY NOT NULL, name varchar(255) NOT NULL, diff --git a/src/controller/p2p/preheat/controller.go b/src/controller/p2p/preheat/controller.go new file mode 100644 index 000000000..ad4d9592a --- /dev/null +++ b/src/controller/p2p/preheat/controller.go @@ -0,0 +1,168 @@ +package preheat + +import ( + "context" + "errors" + "time" + + "github.com/goharbor/harbor/src/lib/q" + + "github.com/goharbor/harbor/src/pkg/p2p/preheat/instance" + providerModels "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider" + "github.com/goharbor/harbor/src/pkg/p2p/preheat/provider" +) + +var ( + // Ctl is a global preheat controller instance + Ctl = NewController() +) + +// ErrorConflict for handling conflicts +var ErrorConflict = errors.New("resource conflict") + +// ErrorUnhealthy for unhealthy +var ErrorUnhealthy = errors.New("instance unhealthy") + +// Controller defines related top interfaces to handle the workflow of +// the image distribution. +// TODO: Add health check API +type Controller interface { + // Get all the supported distribution providers + // + // If succeed, an metadata of provider list will be returned. + // Otherwise, a non nil error will be returned + // + GetAvailableProviders() ([]*provider.Metadata, error) + + // CountInstance all the setup instances of distribution providers + // + // params *q.Query : parameters for querying + // + // If succeed, matched provider instance count will be returned. + // Otherwise, a non nil error will be returned + // + CountInstance(ctx context.Context, query *q.Query) (int64, error) + + // ListInstance all the setup instances of distribution providers + // + // params *q.Query : parameters for querying + // + // If succeed, matched provider instance list will be returned. + // Otherwise, a non nil error will be returned + // + ListInstance(ctx context.Context, query *q.Query) ([]*providerModels.Instance, error) + + // GetInstance the metadata of the specified instance + // + // id string : ID of the instance being deleted + // + // If succeed, the metadata with nil error are returned + // Otherwise, a non nil error is returned + // + GetInstance(ctx context.Context, id int64) (*providerModels.Instance, error) + + // Create a new instance for the specified provider. + // + // If succeed, the ID of the instance will be returned. + // Any problems met, a non nil error will be returned. + // + CreateInstance(ctx context.Context, instance *providerModels.Instance) (int64, error) + + // Delete the specified provider instance. + // + // id string : ID of the instance being deleted + // + // Any problems met, a non nil error will be returned. + // + DeleteInstance(ctx context.Context, id int64) error + + // Update the instance with incremental way; + // Including update the enabled flag of the instance. + // + // id string : ID of the instance being updated + // properties ...string : The properties being updated + // + // Any problems met, a non nil error will be returned + // + UpdateInstance(ctx context.Context, instance *providerModels.Instance, properties ...string) error +} + +var _ Controller = (*controller)(nil) + +// controller is the default implementation of Controller interface. +// +type controller struct { + // For instance + iManager instance.Manager +} + +// NewController is constructor of controller +func NewController() Controller { + return &controller{ + iManager: instance.Mgr, + } +} + +// GetAvailableProviders implements @Controller.GetAvailableProviders +func (cc *controller) GetAvailableProviders() ([]*provider.Metadata, error) { + return provider.ListProviders() +} + +// CountInstance implements @Controller.CountInstance +func (cc *controller) CountInstance(ctx context.Context, query *q.Query) (int64, error) { + return cc.iManager.Count(ctx, query) +} + +// List implements @Controller.ListInstance +func (cc *controller) ListInstance(ctx context.Context, query *q.Query) ([]*providerModels.Instance, error) { + return cc.iManager.List(ctx, query) +} + +// CreateInstance implements @Controller.CreateInstance +func (cc *controller) CreateInstance(ctx context.Context, instance *providerModels.Instance) (int64, error) { + if instance == nil { + return 0, errors.New("nil instance object provided") + } + + // Avoid duplicated endpoint + var query = &q.Query{ + Keywords: map[string]interface{}{ + "endpoint": instance.Endpoint, + }, + } + num, err := cc.iManager.Count(ctx, query) + if err != nil { + return 0, err + } + if num > 0 { + return 0, ErrorConflict + } + + // !WARN: Check healthy status at fronted. + if instance.Status != "healthy" { + return 0, ErrorUnhealthy + } + + instance.SetupTimestamp = time.Now().Unix() + + return cc.iManager.Save(ctx, instance) +} + +// Delete implements @Controller.Delete +func (cc *controller) DeleteInstance(ctx context.Context, id int64) error { + return cc.iManager.Delete(ctx, id) +} + +// Update implements @Controller.Update +func (cc *controller) UpdateInstance(ctx context.Context, instance *providerModels.Instance, properties ...string) error { + if len(properties) == 0 { + return errors.New("no properties provided to update") + } + + return cc.iManager.Update(ctx, instance, properties...) +} + +// Get implements @Controller.Get +func (cc *controller) GetInstance(ctx context.Context, id int64) (*providerModels.Instance, error) { + return cc.iManager.Get(ctx, id) +} diff --git a/src/controller/p2p/preheat/controllor_test.go b/src/controller/p2p/preheat/controllor_test.go new file mode 100644 index 000000000..84d7a2bbe --- /dev/null +++ b/src/controller/p2p/preheat/controllor_test.go @@ -0,0 +1,155 @@ +package preheat + +import ( + "context" + "errors" + "testing" + + "github.com/goharbor/harbor/src/core/config" + imocks "github.com/goharbor/harbor/src/pkg/p2p/preheat/instance/mocks" + providerModel "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider" + "github.com/goharbor/harbor/src/pkg/p2p/preheat/provider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type preheatSuite struct { + suite.Suite + ctx context.Context + controller Controller + fackManager *imocks.Manager +} + +func TestPreheatSuite(t *testing.T) { + t.Log("Start TestPreheatSuite") + fackManager := &imocks.Manager{} + + var c = &controller{ + iManager: fackManager, + } + assert.NotNil(t, c) + + suite.Run(t, &preheatSuite{ + ctx: context.Background(), + controller: c, + fackManager: fackManager, + }) +} + +func TestNewController(t *testing.T) { + c := NewController() + assert.NotNil(t, c) +} + +func (s *preheatSuite) SetupSuite() { + config.Init() + + s.fackManager.On("List", mock.Anything, mock.Anything).Return([]*providerModel.Instance{ + { + ID: 1, + Vendor: "dragonfly", + Endpoint: "http://localhost", + Status: provider.DriverStatusHealthy, + Enabled: true, + }, + }, nil) + s.fackManager.On("Save", mock.Anything, mock.Anything).Return(int64(1), nil) + s.fackManager.On("Count", mock.Anything, &providerModel.Instance{Endpoint: "http://localhost"}).Return(int64(1), nil) + s.fackManager.On("Count", mock.Anything, mock.Anything).Return(int64(0), nil) + s.fackManager.On("Delete", mock.Anything, int64(1)).Return(nil) + s.fackManager.On("Delete", mock.Anything, int64(0)).Return(errors.New("not found")) + s.fackManager.On("Get", mock.Anything, int64(1)).Return(&providerModel.Instance{ + ID: 1, + Endpoint: "http://localhost", + }, nil) + s.fackManager.On("Get", mock.Anything, int64(0)).Return(nil, errors.New("not found")) +} + +func (s *preheatSuite) TestGetAvailableProviders() { + providers, err := s.controller.GetAvailableProviders() + s.Equal(2, len(providers)) + expectProviders := map[string]interface{}{} + expectProviders["dragonfly"] = nil + expectProviders["kraken"] = nil + _, ok := expectProviders[providers[0].ID] + s.True(ok) + _, ok = expectProviders[providers[1].ID] + s.True(ok) + s.NoError(err) +} + +func (s *preheatSuite) TestListInstance() { + instances, err := s.controller.ListInstance(s.ctx, nil) + s.NoError(err) + s.Equal(1, len(instances)) + s.Equal(int64(1), instances[0].ID) +} + +func (s *preheatSuite) TestCreateInstance() { + // Case: nil instance, expect error. + id, err := s.controller.CreateInstance(s.ctx, nil) + s.Empty(id) + s.Error(err) + + // Case: instance with already existed endpoint, expect conflict. + id, err = s.controller.CreateInstance(s.ctx, &providerModel.Instance{ + Endpoint: "http://localhost", + }) + s.Equal(ErrorUnhealthy, err) + s.Empty(id) + + // Case: instance with invalid provider, expect error. + id, err = s.controller.CreateInstance(s.ctx, &providerModel.Instance{ + Endpoint: "http://foo.bar", + Status: "healthy", + Vendor: "none", + }) + s.NoError(err) + s.Equal(int64(1), id) + + // Case: instance with valid provider, expect ok. + id, err = s.controller.CreateInstance(s.ctx, &providerModel.Instance{ + Endpoint: "http://foo.bar", + Status: "healthy", + Vendor: "dragonfly", + }) + s.NoError(err) + s.Equal(int64(1), id) + + id, err = s.controller.CreateInstance(s.ctx, &providerModel.Instance{ + Endpoint: "http://foo.bar2", + Status: "healthy", + Vendor: "kraken", + }) + s.NoError(err) + s.Equal(int64(1), id) +} + +func (s *preheatSuite) TestDeleteInstance() { + // err := s.controller.DeleteInstance(s.ctx, 0) + // s.Error(err) + + err := s.controller.DeleteInstance(s.ctx, int64(1)) + s.NoError(err) +} + +func (s *preheatSuite) TestUpdateInstance() { + // TODO: test update more + s.fackManager.On("Update", s.ctx, nil).Return(errors.New("no properties provided to update")) + err := s.controller.UpdateInstance(s.ctx, nil) + s.Error(err) + + err = s.controller.UpdateInstance(s.ctx, &providerModel.Instance{ID: 0}) + s.Error(err) + + s.fackManager.On("Update", mock.Anything, mock.Anything, mock.Anything).Return(nil) + err = s.controller.UpdateInstance(s.ctx, &providerModel.Instance{ID: 1}, "enabled") + s.NoError(err) +} + +func (s *preheatSuite) TestGetInstance() { + instance, err := s.controller.GetInstance(s.ctx, 1) + s.NoError(err) + s.NotNil(instance) +} diff --git a/src/go.mod b/src/go.mod index 0ab764d60..9d39735a7 100644 --- a/src/go.mod +++ b/src/go.mod @@ -57,6 +57,7 @@ require ( github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/image-spec v1.0.1 github.com/opentracing/opentracing-go v1.1.0 // indirect + github.com/pkg/errors v0.9.1 github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect github.com/robfig/cron v1.0.0 github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 // indirect diff --git a/src/pkg/p2p/preheat/dao/instance/dao.go b/src/pkg/p2p/preheat/dao/instance/dao.go new file mode 100644 index 000000000..27c96b1f9 --- /dev/null +++ b/src/pkg/p2p/preheat/dao/instance/dao.go @@ -0,0 +1,138 @@ +package instance + +import ( + "context" + + beego_orm "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider" +) + +// DAO for instance +type DAO interface { + Create(ctx context.Context, instance *provider.Instance) (int64, error) + Get(ctx context.Context, id int64) (*provider.Instance, error) + Update(ctx context.Context, instance *provider.Instance, props ...string) error + Delete(ctx context.Context, id int64) error + Count(ctx context.Context, query *q.Query) (total int64, err error) + List(ctx context.Context, query *q.Query) (ins []*provider.Instance, err error) +} + +// New instance dao +func New() DAO { + return &dao{} +} + +// ListInstanceQuery defines the query params of the instance record. +type ListInstanceQuery struct { + Page uint + PageSize uint + Keyword string +} + +type dao struct{} + +var _ DAO = (*dao)(nil) + +// Create adds a new distribution instance. +func (d *dao) Create(ctx context.Context, instance *provider.Instance) (int64, error) { + var o, err = orm.FromContext(ctx) + if err != nil { + return 0, err + } + return o.Insert(instance) +} + +// Get gets instance from db by id. +func (d *dao) Get(ctx context.Context, id int64) (*provider.Instance, error) { + var o, err = orm.FromContext(ctx) + if err != nil { + return nil, err + } + + di := provider.Instance{ID: id} + err = o.Read(&di, "ID") + if err == beego_orm.ErrNoRows { + return nil, nil + } + return &di, err +} + +// Update updates distribution instance. +func (d *dao) Update(ctx context.Context, instance *provider.Instance, props ...string) error { + var o, err = orm.FromContext(ctx) + if err != nil { + return err + } + err = o.Begin() + if err != nil { + return err + } + + // check default instances first + for _, prop := range props { + if prop == "default" && instance.Default { + + _, err = o.Raw("UPDATE ? SET default = false WHERE id != ?", instance.TableName(), instance.ID).Exec() + if err != nil { + if e := o.Rollback(); e != nil { + err = errors.Wrap(e, err.Error()) + } + return err + } + + break + } + } + + _, err = o.Update(instance, props...) + if err != nil { + if e := o.Rollback(); e != nil { + err = errors.Wrap(e, err.Error()) + } + } else { + err = o.Commit() + } + return err +} + +// Delete deletes one distribution instance by id. +func (d *dao) Delete(ctx context.Context, id int64) error { + var o, err = orm.FromContext(ctx) + if err != nil { + return err + } + + _, err = o.Delete(&provider.Instance{ID: id}) + return err +} + +// List count instances by query params. +func (d *dao) Count(ctx context.Context, query *q.Query) (total int64, err error) { + if query != nil { + // ignore the page number and size + query = &q.Query{ + Keywords: query.Keywords, + } + } + qs, err := orm.QuerySetter(ctx, &provider.Instance{}, query) + if err != nil { + return 0, err + } + return qs.Count() +} + +// List lists instances by query params. +func (d *dao) List(ctx context.Context, query *q.Query) (ins []*provider.Instance, err error) { + ins = []*provider.Instance{} + qs, err := orm.QuerySetter(ctx, &provider.Instance{}, query) + if err != nil { + return nil, err + } + if _, err = qs.All(&ins); err != nil { + return nil, err + } + return ins, nil +} diff --git a/src/pkg/p2p/preheat/dao/instance/dao_test.go b/src/pkg/p2p/preheat/dao/instance/dao_test.go new file mode 100644 index 000000000..ea4375deb --- /dev/null +++ b/src/pkg/p2p/preheat/dao/instance/dao_test.go @@ -0,0 +1,140 @@ +package instance + +import ( + "context" + "testing" + + beego_orm "github.com/astaxie/beego/orm" + common_dao "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" + models "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +var ( + defaultInstance = &models.Instance{ + ID: 1, + Name: "dragonfly-cn-1", + Description: "fake dragonfly server", + Vendor: "dragonfly", + Endpoint: "https://cn-1.dragonfly.com", + AuthMode: "basic", + AuthData: "{\"username\": \"admin\", \"password\": \"123456\"}", + Status: "healthy", + Enabled: true, + SetupTimestamp: 1582721396, + } +) + +type instanceSuite struct { + suite.Suite + ctx context.Context + dao DAO +} + +func (is *instanceSuite) SetupSuite() { + common_dao.PrepareTestForPostgresSQL() + is.ctx = orm.NewContext(nil, beego_orm.NewOrm()) + is.dao = New() +} + +func (is *instanceSuite) SetupTest() { + t := is.T() + _, err := is.dao.Create(is.ctx, defaultInstance) + assert.Nil(t, err) +} + +func (is *instanceSuite) TearDownTest() { + t := is.T() + err := is.dao.Delete(is.ctx, defaultInstance.ID) + assert.Nil(t, err) +} + +func (is *instanceSuite) TestGet() { + t := is.T() + i, err := is.dao.Get(is.ctx, defaultInstance.ID) + assert.Nil(t, err) + assert.Equal(t, defaultInstance.Name, i.Name) + + // not exist + i, err = is.dao.Get(is.ctx, 0) + assert.Nil(t, i) +} + +func (is *instanceSuite) TestUpdate() { + t := is.T() + i, err := is.dao.Get(is.ctx, defaultInstance.ID) + assert.Nil(t, err) + assert.NotNil(t, i) + + i.Enabled = false + err = is.dao.Update(is.ctx, i, "enabled") + assert.Nil(t, err) + + i.Default = true + err = is.dao.Update(is.ctx, i, "default") + assert.NotNil(t, err) + + i, err = is.dao.Get(is.ctx, defaultInstance.ID) + assert.Nil(t, err) + assert.NotNil(t, i) + assert.False(t, i.Enabled) +} + +func (is *instanceSuite) TestList() { + t := is.T() + // add more instances + testInstance1 := &models.Instance{ + ID: 2, + Name: "kraken-us-1", + Description: "fake kraken server", + Vendor: "kraken", + Endpoint: "https://us-1.kraken.com", + AuthMode: "none", + AuthData: "", + Status: "success", + Enabled: true, + SetupTimestamp: 0, + } + _, err := is.dao.Create(is.ctx, testInstance1) + assert.Nilf(t, err, "Create %d", testInstance1.ID) + defer func() { + // clean data + err = is.dao.Delete(is.ctx, testInstance1.ID) + assert.Nilf(t, err, "delete instance %d", testInstance1.ID) + }() + + total, err := is.dao.Count(is.ctx, nil) + assert.Nil(t, err) + assert.Equal(t, total, int64(2)) + // limit 1 + total, err = is.dao.Count(is.ctx, &q.Query{PageSize: 1, PageNumber: 1}) + assert.Nil(t, err) + assert.Equal(t, total, int64(2)) + + // without limit should return all instances + instances, err := is.dao.List(is.ctx, nil) + assert.Nil(t, err) + assert.Len(t, instances, 2) + + // limit 1 + instances, err = is.dao.List(is.ctx, &q.Query{PageSize: 1, PageNumber: 1}) + assert.Nil(t, err) + assert.Len(t, instances, 1, "instances number") + assert.Equal(t, defaultInstance.ID, instances[0].ID) + + // keyword search + keywords := make(map[string]interface{}) + keywords["name"] = "kraken-us-1" + instances, err = is.dao.List(is.ctx, &q.Query{Keywords: keywords}) + assert.Nil(t, err) + assert.Len(t, instances, 1) + assert.Equal(t, testInstance1.Name, instances[0].Name) + +} + +func TestInstance(t *testing.T) { + suite.Run(t, &instanceSuite{}) +} diff --git a/src/pkg/p2p/preheat/helper/helper.go b/src/pkg/p2p/preheat/helper/helper.go new file mode 100644 index 000000000..3e784675c --- /dev/null +++ b/src/pkg/p2p/preheat/helper/helper.go @@ -0,0 +1,52 @@ +package helper + +import ( + "strings" +) + +// ImageRepository represents the image repository name +// e.g: library/ubuntu:latest +type ImageRepository string + +// Valid checks if the repository name is valid +func (ir ImageRepository) Valid() bool { + if len(ir) == 0 { + return false + } + + trimName := strings.TrimSpace(string(ir)) + segments := strings.SplitN(trimName, "/", 2) + if len(segments) != 2 { + return false + } + + nameAndTag := segments[1] + subSegments := strings.SplitN(nameAndTag, ":", 2) + if len(subSegments) != 2 { + return false + } + + return true +} + +// Name returns the name of the image repository +func (ir ImageRepository) Name() string { + // No check here, should call Valid() before calling name + segments := strings.SplitN(string(ir), ":", 2) + if len(segments) == 0 { + return "" + } + + return segments[0] +} + +// Tag returns the tag of the image repository +func (ir ImageRepository) Tag() string { + // No check here, should call Valid() before calling name + segments := strings.SplitN(string(ir), ":", 2) + if len(segments) < 2 { + return "" + } + + return segments[1] +} diff --git a/src/pkg/p2p/preheat/helper/helper_test.go b/src/pkg/p2p/preheat/helper/helper_test.go new file mode 100644 index 000000000..560f81448 --- /dev/null +++ b/src/pkg/p2p/preheat/helper/helper_test.go @@ -0,0 +1,63 @@ +package helper + +import "testing" + +func TestImageRepository_Valid(t *testing.T) { + tests := []struct { + name string + ir ImageRepository + want bool + }{ + {"empty", "", false}, + {"invalid", "abc", false}, + {"invalid", "abc/def", false}, + {"valid", "abc/def:tag", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.ir.Valid(); got != tt.want { + t.Errorf("ImageRepository.Valid() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestImageRepository_Name(t *testing.T) { + tests := []struct { + name string + ir ImageRepository + want string + }{ + {"empty", "", ""}, + {"invalid", "abc", "abc"}, + {"invalid", "abc/def", "abc/def"}, + {"valid", "abc/def:tag", "abc/def"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.ir.Name(); got != tt.want { + t.Errorf("ImageRepository.Name() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestImageRepository_Tag(t *testing.T) { + tests := []struct { + name string + ir ImageRepository + want string + }{ + {"empty", "", ""}, + {"invalid", "abc", ""}, + {"invalid", "abc/def", ""}, + {"valid", "abc/def:tag", "tag"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.ir.Tag(); got != tt.want { + t.Errorf("ImageRepository.Tag() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/src/pkg/p2p/preheat/instance/manager.go b/src/pkg/p2p/preheat/instance/manager.go new file mode 100644 index 000000000..aef7ee597 --- /dev/null +++ b/src/pkg/p2p/preheat/instance/manager.go @@ -0,0 +1,111 @@ +package instance + +import ( + "context" + + "github.com/goharbor/harbor/src/lib/q" + + dao "github.com/goharbor/harbor/src/pkg/p2p/preheat/dao/instance" + "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider" +) + +// Mgr is the global instance manager instance +var Mgr = New() + +// Manager is responsible for storing the instances +type Manager interface { + // Save the instance metadata to the backend store + // + // inst *Instance : a ptr of instance + // + // If succeed, the uuid of the saved instance is returned; + // otherwise, a non nil error is returned + // + Save(ctx context.Context, inst *provider.Instance) (int64, error) + + // Delete the specified instance + // + // id int64 : the id of the instance + // + // If succeed, a nil error is returned; + // otherwise, a non nil error is returned + // + Delete(ctx context.Context, id int64) error + + // Update the specified instance + // + // inst *Instance : a ptr of instance + // + // If succeed, a nil error is returned; + // otherwise, a non nil error is returned + // + Update(ctx context.Context, inst *provider.Instance, props ...string) error + + // Get the instance with the ID + // + // id int64 : the id of the instance + // + // If succeed, a non nil Instance is returned; + // otherwise, a non nil error is returned + // + Get(ctx context.Context, id int64) (*provider.Instance, error) + + // Count the instances by the param + // + // query *q.Query : the query params + Count(ctx context.Context, query *q.Query) (int64, error) + + // Query the instances by the param + // + // query *q.Query : the query params + // + // If succeed, an instance list is returned; + // otherwise, a non nil error is returned + // + List(ctx context.Context, query *q.Query) ([]*provider.Instance, error) +} + +// manager implement the Manager interface +type manager struct { + dao dao.DAO +} + +// New returns an instance of DefaultManger +func New() Manager { + return &manager{ + dao: dao.New(), + } +} + +// Ensure *manager has implemented Manager interface. +var _ Manager = (*manager)(nil) + +// Save implements @Manager.Save +func (dm *manager) Save(ctx context.Context, inst *provider.Instance) (int64, error) { + return dm.dao.Create(ctx, inst) +} + +// Delete implements @Manager.Delete +func (dm *manager) Delete(ctx context.Context, id int64) error { + return dm.dao.Delete(ctx, id) +} + +// Update implements @Manager.Update +func (dm *manager) Update(ctx context.Context, inst *provider.Instance, props ...string) error { + return dm.dao.Update(ctx, inst, props...) +} + +// Get implements @Manager.Get +func (dm *manager) Get(ctx context.Context, id int64) (*provider.Instance, error) { + return dm.dao.Get(ctx, id) +} + +// Count implements @Manager.Count +func (dm *manager) Count(ctx context.Context, query *q.Query) (int64, error) { + return dm.dao.Count(ctx, query) +} + +// List implements @Manager.List +func (dm *manager) List(ctx context.Context, query *q.Query) ([]*provider.Instance, error) { + return dm.dao.List(ctx, query) +} diff --git a/src/pkg/p2p/preheat/instance/manager_test.go b/src/pkg/p2p/preheat/instance/manager_test.go new file mode 100644 index 000000000..834e549fc --- /dev/null +++ b/src/pkg/p2p/preheat/instance/manager_test.go @@ -0,0 +1,122 @@ +package instance + +import ( + "context" + "testing" + + "github.com/goharbor/harbor/src/lib/q" + dao "github.com/goharbor/harbor/src/pkg/p2p/preheat/dao/instance" + "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider" + providerModel "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type fakeDao struct { + mock.Mock +} + +var _ dao.DAO = (*fakeDao)(nil) + +func (d *fakeDao) Create(ctx context.Context, instance *provider.Instance) (int64, error) { + var args = d.Called() + return int64(args.Int(0)), args.Error(1) +} + +func (d *fakeDao) Get(ctx context.Context, id int64) (*provider.Instance, error) { + var args = d.Called() + var instance *provider.Instance + if args.Get(0) != nil { + instance = args.Get(0).(*provider.Instance) + } + return instance, args.Error(1) +} + +func (d *fakeDao) Update(ctx context.Context, instance *provider.Instance, props ...string) error { + var args = d.Called() + return args.Error(0) +} + +func (d *fakeDao) Delete(ctx context.Context, id int64) error { + var args = d.Called() + return args.Error(0) +} + +func (d *fakeDao) Count(ctx context.Context, query *q.Query) (total int64, err error) { + var args = d.Called() + + return int64(args.Int(0)), args.Error(1) +} + +func (d *fakeDao) List(ctx context.Context, query *q.Query) (ins []*provider.Instance, err error) { + var args = d.Called() + var instances []*provider.Instance + if args.Get(0) != nil { + instances = args.Get(0).([]*provider.Instance) + } + return instances, args.Error(1) +} + +type instanceManagerSuite struct { + suite.Suite + dao *fakeDao + ctx context.Context + manager Manager +} + +func (im *instanceManagerSuite) SetupSuite() { + im.dao = &fakeDao{} + im.manager = &manager{dao: im.dao} +} + +func (im *instanceManagerSuite) TestSave() { + im.dao.On("Create").Return(1, nil) + id, err := im.manager.Save(im.ctx, nil) + im.Require().Nil(err) + im.Require().Equal(int64(1), id) +} + +func (im *instanceManagerSuite) TestDelete() { + im.dao.On("Delete").Return(nil) + err := im.manager.Delete(im.ctx, 1) + im.Require().Nil(err) +} + +func (im *instanceManagerSuite) TestUpdate() { + im.dao.On("Update").Return(nil) + err := im.manager.Update(im.ctx, nil) + im.Require().Nil(err) +} + +func (im *instanceManagerSuite) TestGet() { + ins := &providerModel.Instance{Name: "abc"} + im.dao.On("Get").Return(ins, nil) + res, err := im.manager.Get(im.ctx, 1) + im.Require().Nil(err) + im.Require().Equal(ins, res) +} + +func (im *instanceManagerSuite) TestCount() { + im.dao.On("Count").Return(2, nil) + count, err := im.manager.Count(im.ctx, nil) + assert.Nil(im.T(), err) + assert.Equal(im.T(), int64(2), count) +} + +func (im *instanceManagerSuite) TestList() { + lists := []*providerModel.Instance{ + {Name: "abc"}, + {Name: "def"}, + } + im.dao.On("List").Return(lists, nil) + res, err := im.manager.List(im.ctx, nil) + assert.Nil(im.T(), err) + assert.Len(im.T(), res, 2) + assert.Equal(im.T(), lists, res) +} + +func TestInstanceManager(t *testing.T) { + suite.Run(t, &instanceManagerSuite{}) +} diff --git a/src/pkg/p2p/preheat/instance/mocks/Manager.go b/src/pkg/p2p/preheat/instance/mocks/Manager.go new file mode 100644 index 000000000..c82955a95 --- /dev/null +++ b/src/pkg/p2p/preheat/instance/mocks/Manager.go @@ -0,0 +1,138 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + q "github.com/goharbor/harbor/src/lib/q" + provider "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider" + mock "github.com/stretchr/testify/mock" +) + +// Manager is an autogenerated mock type for the Manager type +type Manager struct { + mock.Mock +} + +// Count provides a mock function with given fields: ctx, query +func (_m *Manager) Count(ctx context.Context, query *q.Query) (int64, error) { + ret := _m.Called(ctx, query) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok { + r0 = rf(ctx, query) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Delete provides a mock function with given fields: ctx, id +func (_m *Manager) Delete(ctx context.Context, id int64) error { + ret := _m.Called(ctx, id) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Get provides a mock function with given fields: ctx, id +func (_m *Manager) Get(ctx context.Context, id int64) (*provider.Instance, error) { + ret := _m.Called(ctx, id) + + var r0 *provider.Instance + if rf, ok := ret.Get(0).(func(context.Context, int64) *provider.Instance); ok { + r0 = rf(ctx, id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*provider.Instance) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// List provides a mock function with given fields: ctx, query +func (_m *Manager) List(ctx context.Context, query *q.Query) ([]*provider.Instance, error) { + ret := _m.Called(ctx, query) + + var r0 []*provider.Instance + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*provider.Instance); ok { + r0 = rf(ctx, query) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*provider.Instance) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Save provides a mock function with given fields: ctx, inst +func (_m *Manager) Save(ctx context.Context, inst *provider.Instance) (int64, error) { + ret := _m.Called(ctx, inst) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, *provider.Instance) int64); ok { + r0 = rf(ctx, inst) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *provider.Instance) error); ok { + r1 = rf(ctx, inst) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Update provides a mock function with given fields: ctx, inst, props +func (_m *Manager) Update(ctx context.Context, inst *provider.Instance, props ...string) error { + _va := make([]interface{}, len(props)) + for _i := range props { + _va[_i] = props[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, inst) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *provider.Instance, ...string) error); ok { + r0 = rf(ctx, inst, props...) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/src/pkg/p2p/preheat/models/provider/instance.go b/src/pkg/p2p/preheat/models/provider/instance.go index 7a67d7ec7..a3897f6a5 100644 --- a/src/pkg/p2p/preheat/models/provider/instance.go +++ b/src/pkg/p2p/preheat/models/provider/instance.go @@ -17,6 +17,7 @@ package provider import ( "encoding/json" + "github.com/astaxie/beego/orm" "github.com/goharbor/harbor/src/lib/errors" ) @@ -33,6 +34,10 @@ const ( PreheatingStatusFail = "FAIL" ) +func init() { + orm.RegisterModel(&Instance{}) +} + // Instance defines the properties of the preheating provider instance. type Instance struct { ID int64 `orm:"pk;auto;column(id)" json:"id"` @@ -42,11 +47,11 @@ type Instance struct { Endpoint string `orm:"column(endpoint)" json:"endpoint"` AuthMode string `orm:"column(auth_mode)" json:"auth_mode"` // The auth credential data if exists - AuthInfo map[string]string `orm:"column(-)" json:"auth_info,omitempty"` + AuthInfo map[string]string `orm:"-" json:"auth_info,omitempty"` // Data format for "AuthInfo" AuthData string `orm:"column(auth_data)" json:"-"` // Default 'Unknown', use separate API for client to retrieve - Status string `orm:"column(-)" json:"status"` + Status string `orm:"-" json:"status"` Enabled bool `orm:"column(enabled)" json:"enabled"` Default bool `orm:"column(is_default)" json:"default"` Insecure bool `orm:"column(insecure)" json:"insecure"` @@ -75,3 +80,8 @@ func (ins *Instance) ToJSON() (string, error) { return string(data), nil } + +// TableName ... +func (ins *Instance) TableName() string { + return "p2p_preheat_instance" +} diff --git a/src/server/v2.0/handler/handler.go b/src/server/v2.0/handler/handler.go index 429533954..b80d58aca 100644 --- a/src/server/v2.0/handler/handler.go +++ b/src/server/v2.0/handler/handler.go @@ -15,10 +15,11 @@ package handler import ( - lib_http "github.com/goharbor/harbor/src/lib/http" "log" "net/http" + lib_http "github.com/goharbor/harbor/src/lib/http" + "github.com/goharbor/harbor/src/server/middleware" "github.com/goharbor/harbor/src/server/middleware/blob" "github.com/goharbor/harbor/src/server/middleware/quota" @@ -33,6 +34,7 @@ func New() http.Handler { AuditlogAPI: newAuditLogAPI(), ScanAPI: newScanAPI(), ProjectAPI: newProjectAPI(), + PreheatAPI: newPreheatAPI(), }) if err != nil { log.Fatal(err) diff --git a/src/server/v2.0/handler/preheat.go b/src/server/v2.0/handler/preheat.go new file mode 100644 index 000000000..1f43753f4 --- /dev/null +++ b/src/server/v2.0/handler/preheat.go @@ -0,0 +1,82 @@ +package handler + +import ( + "context" + + "github.com/go-openapi/runtime/middleware" + preheatCtl "github.com/goharbor/harbor/src/controller/p2p/preheat" + "github.com/goharbor/harbor/src/pkg/p2p/preheat/provider" + "github.com/goharbor/harbor/src/server/v2.0/models" + "github.com/goharbor/harbor/src/server/v2.0/restapi" + operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/preheat" +) + +func newPreheatAPI() *preheatAPI { + return &preheatAPI{ + preheatCtl: preheatCtl.Ctl, + } +} + +var _ restapi.PreheatAPI = (*preheatAPI)(nil) + +type preheatAPI struct { + BaseAPI + preheatCtl preheatCtl.Controller +} + +func (api *preheatAPI) Prepare(ctx context.Context, operation string, params interface{}) middleware.Responder { + return nil +} + +func (api *preheatAPI) CreateInstance(ctx context.Context, params operation.CreateInstanceParams) middleware.Responder { + var payload *models.InstanceCreatedResp + return operation.NewCreateInstanceCreated().WithPayload(payload) +} + +func (api *preheatAPI) DeleteInstance(ctx context.Context, params operation.DeleteInstanceParams) middleware.Responder { + var payload *models.InstanceDeletedResp + return operation.NewDeleteInstanceOK().WithPayload(payload) +} + +func (api *preheatAPI) GetInstance(ctx context.Context, params operation.GetInstanceParams) middleware.Responder { + var payload *models.Instance + return operation.NewGetInstanceOK().WithPayload(payload) +} + +// ListInstances is List p2p instances +func (api *preheatAPI) ListInstances(ctx context.Context, params operation.ListInstancesParams) middleware.Responder { + var payload []*models.Instance + return operation.NewListInstancesOK().WithPayload(payload) +} + +func (api *preheatAPI) ListProviders(ctx context.Context, params operation.ListProvidersParams) middleware.Responder { + + var providers, err = preheatCtl.Ctl.GetAvailableProviders() + if err != nil { + return operation.NewListProvidersInternalServerError() + } + var payload = convertProvidersToFrontend(providers) + + return operation.NewListProvidersOK().WithPayload(payload) +} + +// UpdateInstance is Update instance +func (api *preheatAPI) UpdateInstance(ctx context.Context, params operation.UpdateInstanceParams) middleware.Responder { + var payload *models.InstanceUpdateResp + return operation.NewUpdateInstanceOK().WithPayload(payload) +} + +func convertProvidersToFrontend(backend []*provider.Metadata) (frontend []*models.Metadata) { + frontend = make([]*models.Metadata, 0) + for _, provider := range backend { + frontend = append(frontend, &models.Metadata{ + ID: provider.ID, + Icon: provider.Icon, + Name: provider.Name, + Source: provider.Source, + Version: provider.Version, + Maintainers: provider.Maintainers, + }) + } + return +} diff --git a/src/server/v2.0/handler/preheat_test.go b/src/server/v2.0/handler/preheat_test.go new file mode 100644 index 000000000..9e7a34cac --- /dev/null +++ b/src/server/v2.0/handler/preheat_test.go @@ -0,0 +1,33 @@ +package handler + +import ( + "reflect" + "testing" + + "github.com/goharbor/harbor/src/pkg/p2p/preheat/provider" + "github.com/goharbor/harbor/src/server/v2.0/models" +) + +func Test_convertProvidersToFrontend(t *testing.T) { + backend, _ := provider.ListProviders() + tests := []struct { + name string + backend []*provider.Metadata + wantFrontend []*models.Metadata + }{ + {"", + backend, + []*models.Metadata{ + {ID: "dragonfly", Icon: "https://raw.githubusercontent.com/alibaba/Dragonfly/master/docs/images/logo.png", Maintainers: []string{"Jin Zhang/taiyun.zj@alibaba-inc.com"}, Name: "Dragonfly", Source: "https://github.com/alibaba/Dragonfly", Version: "0.10.1"}, + {Icon: "https://github.com/uber/kraken/blob/master/assets/kraken-logo-color.svg", ID: "kraken", Maintainers: []string{"mmpei/peimingming@corp.netease.com"}, Name: "Kraken", Source: "https://github.com/uber/kraken", Version: "0.1.3"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotFrontend := convertProvidersToFrontend(tt.backend); !reflect.DeepEqual(gotFrontend, tt.wantFrontend) { + t.Errorf("convertProvidersToFrontend() = %#v, want %#v", gotFrontend, tt.wantFrontend) + } + }) + } +}