From e2242cffd230d96acb0b7b8fc5925e77a8f945bf Mon Sep 17 00:00:00 2001 From: Steven Zou Date: Fri, 27 Oct 2017 18:16:16 +0800 Subject: [PATCH 01/37] define filter interfaces for replication service including:interface Filter,Converter and FilterChain and other supporting models --- src/replication/consts.go | 10 ++++++++++ src/replication/models/filter_config.go | 7 +++++++ src/replication/models/filter_item.go | 19 +++++++++++++++++++ src/replication/source/convertor.go | 16 ++++++++++++++++ src/replication/source/filter.go | 21 +++++++++++++++++++++ src/replication/source/filter_chain.go | 21 +++++++++++++++++++++ 6 files changed, 94 insertions(+) create mode 100644 src/replication/consts.go create mode 100644 src/replication/models/filter_config.go create mode 100644 src/replication/models/filter_item.go create mode 100644 src/replication/source/convertor.go create mode 100644 src/replication/source/filter.go create mode 100644 src/replication/source/filter_chain.go diff --git a/src/replication/consts.go b/src/replication/consts.go new file mode 100644 index 000000000..10075bcb3 --- /dev/null +++ b/src/replication/consts.go @@ -0,0 +1,10 @@ +package replication + +const ( + //Kind of filter item is 'project' + filterItemKindProject = "project" + //Kind of filter item is 'repository' + filterItemKindRepository = "repository" + //Kind of filter item is 'tag' + filterItemKindTag = "tag" +) diff --git a/src/replication/models/filter_config.go b/src/replication/models/filter_config.go new file mode 100644 index 000000000..cdc8e60a9 --- /dev/null +++ b/src/replication/models/filter_config.go @@ -0,0 +1,7 @@ +package models + +//FilterConfig is data model to provide configurations to the filters. +type FilterConfig struct { + //The pattern for fuzzy matching + pattern string +} diff --git a/src/replication/models/filter_item.go b/src/replication/models/filter_item.go new file mode 100644 index 000000000..9565ac447 --- /dev/null +++ b/src/replication/models/filter_item.go @@ -0,0 +1,19 @@ +package models + +//FilterItem is the general data model represents the filtering resources which are used as input and output for the filters. +type FilterItem struct { + + //The kind of the filtering resources. Support 'project','repository' and 'tag' etc. + kind string + + //The key value of resource which can be used to filter out the resource matched with specified pattern. + //E.g: + //kind == 'project', value will be project name; + //kind == 'repository', value will be repository name + //kind == 'tag', value will be tag name. + value string + + //Extension placeholder. + //To append more additional information if required by the filter. + metadata map[string]interface{} +} diff --git a/src/replication/source/convertor.go b/src/replication/source/convertor.go new file mode 100644 index 000000000..d07aafd35 --- /dev/null +++ b/src/replication/source/convertor.go @@ -0,0 +1,16 @@ +package source + +import ( + "github.com/vmware/harbor/src/replication/models" +) + +//Convertor is designed to covert the format of output from upstream filter to the input format +//required by the downstream filter if needed. +//Each convertor covers only one specified conversion process between the two filters. +//E.g: +//If project filter connects to repository filter, then one convertor should be defined for this connection; +//If project filter connects to tag filter, then another one should be defined. The above one can not be reused. +type Convertor interface { + //Accept the items from upstream filter as input and then covert them to the required format and returned. + Convert(itemsOfUpstream []models.FilterItem) (itemsOfDownstream []models.FilterItem) +} diff --git a/src/replication/source/filter.go b/src/replication/source/filter.go new file mode 100644 index 000000000..831e4b017 --- /dev/null +++ b/src/replication/source/filter.go @@ -0,0 +1,21 @@ +package source + +import ( + "github.com/vmware/harbor/src/replication/models" +) + +//Filter define the operations of selecting the matched resources from the candidates +//according to the specified pattern. +type Filter interface { + //Initialize the filter with specified configurations like pattern definition + Init(config models.FilterConfig) + + //Set Convertor if necessary + SetConvertor(convertor Convertor) + + //Return the convertor if existing or nil if never set + GetConvertor() Convertor + + //Filter the items + DoFilter(filterItems []models.FilterItem) []models.FilterItem +} diff --git a/src/replication/source/filter_chain.go b/src/replication/source/filter_chain.go new file mode 100644 index 000000000..156ed6fc6 --- /dev/null +++ b/src/replication/source/filter_chain.go @@ -0,0 +1,21 @@ +package source + +import ( + "github.com/vmware/harbor/src/replication/models" +) + +//FilterChain is the interface to define the operations of coordinating multiple filters +//to work together as a whole pipeline. +//E.g: +//(original resources)---->[project filter]---->[repository filter]---->[tag filter]---->[......]---->(filter resources) +type FilterChain interface { + //Build the filter chain with the filters provided; + //if failed, an error will be returned. + Build(filter []Filter) error + + //Return all the filters in the chain. + Filters() []Filter + + //Filter the items and returned the filtered items via the appended filters in the chain. + DoFilter(filterItems []models.FilterItem) []models.FilterItem +} From cee0bcec22cef1958a2a283763192ca8ff116e56 Mon Sep 17 00:00:00 2001 From: Steven Zou Date: Tue, 31 Oct 2017 19:43:53 +0800 Subject: [PATCH 02/37] Update the alternate policy and corresponding task to support by weekly besides daily --- .../scan_policy_notitification_handler.go | 4 +- .../scheduler/policy/alternate_policy.go | 65 +++++++++++++++++-- .../scheduler/policy/alternate_policy_test.go | 41 ++++++++++-- src/common/scheduler/policy/policy.go | 1 + src/common/scheduler/policy/uuid.go | 22 +++++++ src/common/scheduler/task/replication_task.go | 24 +++++++ .../scheduler/task/replication_task_test.go | 14 ++++ .../scheduler/task/scan_all_task_test.go | 2 +- 8 files changed, 157 insertions(+), 16 deletions(-) create mode 100644 src/common/scheduler/policy/uuid.go create mode 100644 src/common/scheduler/task/replication_task.go create mode 100644 src/common/scheduler/task/replication_task_test.go diff --git a/src/common/notifier/scan_policy_notitification_handler.go b/src/common/notifier/scan_policy_notitification_handler.go index 076019f06..e3f7d3530 100644 --- a/src/common/notifier/scan_policy_notitification_handler.go +++ b/src/common/notifier/scan_policy_notitification_handler.go @@ -63,7 +63,7 @@ func (s *ScanPolicyNotificationHandler) Handle(value interface{}) error { //To check and compare if the related parameter is changed. if pl := scheduler.DefaultScheduler.GetPolicy(alternatePolicy); pl != nil { - policyCandidate := policy.NewAlternatePolicy(&policy.AlternatePolicyConfiguration{ + policyCandidate := policy.NewAlternatePolicy(alternatePolicy, &policy.AlternatePolicyConfiguration{ Duration: 24 * time.Hour, OffsetTime: notification.DailyTime, }) @@ -95,7 +95,7 @@ func (s *ScanPolicyNotificationHandler) Handle(value interface{}) error { //Schedule policy. func schedulePolicy(notification ScanPolicyNotification) error { - schedulePolicy := policy.NewAlternatePolicy(&policy.AlternatePolicyConfiguration{ + schedulePolicy := policy.NewAlternatePolicy(alternatePolicy, &policy.AlternatePolicyConfiguration{ Duration: 24 * time.Hour, OffsetTime: notification.DailyTime, }) diff --git a/src/common/scheduler/policy/alternate_policy.go b/src/common/scheduler/policy/alternate_policy.go index e37b466f4..dc1d3864f 100644 --- a/src/common/scheduler/policy/alternate_policy.go +++ b/src/common/scheduler/policy/alternate_policy.go @@ -10,11 +10,22 @@ import ( "github.com/vmware/harbor/src/common/utils/log" ) +const ( + oneDay = 24 * 3600 +) + //AlternatePolicyConfiguration store the related configurations for alternate policy. type AlternatePolicyConfiguration struct { //Duration is the interval of executing attached tasks. + //E.g: 24*3600 for daily + // 7*24*3600 for weekly Duration time.Duration + //An integer to indicate the the weekday of the week. Please be noted that Sunday is 7. + //Use default value 0 to indicate weekday is not set. + //To support by weekly function. + Weekday int8 + //OffsetTime is the execution time point of each turn //It's a number to indicate the seconds offset to the 00:00 of UTC time. OffsetTime int64 @@ -42,16 +53,21 @@ type AlternatePolicy struct { //Channel used to receive terminate signal. terminator chan bool + + //Unique name of this policy to support multiple instances + name string } //NewAlternatePolicy is constructor of creating AlternatePolicy. -func NewAlternatePolicy(config *AlternatePolicyConfiguration) *AlternatePolicy { +//Accept name and configuration as parameters. +func NewAlternatePolicy(name string, config *AlternatePolicyConfiguration) *AlternatePolicy { return &AlternatePolicy{ RWMutex: new(sync.RWMutex), tasks: task.NewDefaultStore(), config: config, isEnabled: false, terminator: make(chan bool), + name: name, } } @@ -62,7 +78,7 @@ func (alp *AlternatePolicy) GetConfig() *AlternatePolicyConfiguration { //Name is an implementation of same method in policy interface. func (alp *AlternatePolicy) Name() string { - return "Alternate Policy" + return alp.name } //Tasks is an implementation of same method in policy interface. @@ -110,6 +126,11 @@ func (alp *AlternatePolicy) Evaluate() (<-chan bool, error) { defer alp.Unlock() alp.Lock() + //Check if configuration is valid + if !alp.isValidConfig() { + return nil, errors.New("Policy configuration is not valid") + } + //Check if policy instance is still running if alp.isEnabled { return nil, fmt.Errorf("Instance of policy %s is still running", alp.Name()) @@ -124,19 +145,41 @@ func (alp *AlternatePolicy) Evaluate() (<-chan bool, error) { alp.evaluation = make(chan bool) go func() { + var ( + waitingTime int64 + ) timeNow := time.Now().UTC() //Reach the execution time point? + //Weekday is set + if alp.config.Weekday > 0 { + targetWeekday := (alp.config.Weekday + 7) % 7 + currentWeekday := timeNow.Weekday() + weekdayDiff := (int)(targetWeekday - (int8)(currentWeekday)) + if weekdayDiff < 0 { + weekdayDiff += 7 + } + waitingTime = (int64)(weekdayDiff * oneDay) + } + + //Time utcTime := (int64)(timeNow.Hour()*3600 + timeNow.Minute()*60) diff := alp.config.OffsetTime - utcTime - if diff < 0 { - diff += 24 * 3600 + if waitingTime > 0 { + waitingTime += diff + } else { + waitingTime = diff + if waitingTime < 0 { + waitingTime += oneDay + } } - if diff > 0 { + + //Let's wait for a while + if waitingTime > 0 { //Wait for a while. log.Infof("Waiting for %d seconds after comparing offset %d and utc time %d\n", diff, alp.config.OffsetTime, utcTime) select { - case <-time.After(time.Duration(diff) * time.Second): + case <-time.After(time.Duration(waitingTime) * time.Second): case <-alp.terminator: return } @@ -188,7 +231,10 @@ func (alp *AlternatePolicy) Equal(p Policy) bool { return false } - return cfg == nil || (cfg.Duration == cfg2.Duration && cfg.OffsetTime == cfg2.OffsetTime) + return cfg == nil || + (cfg.Duration == cfg2.Duration && + cfg.OffsetTime == cfg2.OffsetTime && + cfg.Weekday == cfg2.Weekday) } //IsEnabled is an implementation of same method in policy interface. @@ -198,3 +244,8 @@ func (alp *AlternatePolicy) IsEnabled() bool { return alp.isEnabled } + +//Check if the config is valid. At least it should have the configurations for supporting daily policy. +func (alp *AlternatePolicy) isValidConfig() bool { + return alp.config != nil && alp.config.Duration > 0 && alp.config.OffsetTime >= 0 +} diff --git a/src/common/scheduler/policy/alternate_policy_test.go b/src/common/scheduler/policy/alternate_policy_test.go index 777fe59de..5a3eda4e3 100644 --- a/src/common/scheduler/policy/alternate_policy_test.go +++ b/src/common/scheduler/policy/alternate_policy_test.go @@ -6,6 +6,10 @@ import ( "time" ) +const ( + testPolicyName = "TestingPolicy" +) + type fakeTask struct { number int32 } @@ -24,18 +28,18 @@ func (ft *fakeTask) Number() int32 { } func TestBasic(t *testing.T) { - tp := NewAlternatePolicy(&AlternatePolicyConfiguration{}) + tp := NewAlternatePolicy(testPolicyName, &AlternatePolicyConfiguration{}) err := tp.AttachTasks(&fakeTask{number: 100}) if err != nil { t.Fail() } if tp.GetConfig() == nil { - t.Fail() + t.Fatal("nil config") } - if tp.Name() != "Alternate Policy" { - t.Fail() + if tp.Name() != testPolicyName { + t.Fatalf("Wrong name %s", tp.Name()) } tks := tp.Tasks() @@ -48,7 +52,7 @@ func TestBasic(t *testing.T) { func TestEvaluatePolicy(t *testing.T) { now := time.Now().UTC() utcOffset := (int64)(now.Hour()*3600 + now.Minute()*60) - tp := NewAlternatePolicy(&AlternatePolicyConfiguration{ + tp := NewAlternatePolicy(testPolicyName, &AlternatePolicyConfiguration{ Duration: 1 * time.Second, OffsetTime: utcOffset + 1, }) @@ -78,7 +82,7 @@ func TestEvaluatePolicy(t *testing.T) { func TestDisablePolicy(t *testing.T) { now := time.Now().UTC() utcOffset := (int64)(now.Hour()*3600 + now.Minute()*60) - tp := NewAlternatePolicy(&AlternatePolicyConfiguration{ + tp := NewAlternatePolicy(testPolicyName, &AlternatePolicyConfiguration{ Duration: 1 * time.Second, OffsetTime: utcOffset + 1, }) @@ -118,3 +122,28 @@ func TestDisablePolicy(t *testing.T) { t.Fatalf("Policy is still running after calling Disable() %d=%d", atomic.LoadInt32(&copiedCounter), atomic.LoadInt32(&counter)) } } + +func TestPolicyEqual(t *testing.T) { + tp1 := NewAlternatePolicy(testPolicyName, &AlternatePolicyConfiguration{ + Duration: 1 * time.Second, + OffsetTime: 8000, + }) + + tp2 := NewAlternatePolicy(testPolicyName+"2", &AlternatePolicyConfiguration{ + Duration: 100 * time.Second, + OffsetTime: 8000, + }) + + if tp1.Equal(tp2) { + t.Fatal("tp1 should not equal tp2") + } + + tp3 := NewAlternatePolicy(testPolicyName, &AlternatePolicyConfiguration{ + Duration: 1 * time.Second, + OffsetTime: 8000, + }) + + if !tp1.Equal(tp3) { + t.Fatal("tp1 should equal tp3") + } +} diff --git a/src/common/scheduler/policy/policy.go b/src/common/scheduler/policy/policy.go index 732d56cca..4fc72c0d6 100644 --- a/src/common/scheduler/policy/policy.go +++ b/src/common/scheduler/policy/policy.go @@ -15,6 +15,7 @@ import ( // type Policy interface { //Name will return the name of the policy. + //If the policy supports multiple instances, please make sure the name is unique as an UUID. Name() string //Tasks will return the attached tasks with this policy. diff --git a/src/common/scheduler/policy/uuid.go b/src/common/scheduler/policy/uuid.go new file mode 100644 index 000000000..8bd1bd72c --- /dev/null +++ b/src/common/scheduler/policy/uuid.go @@ -0,0 +1,22 @@ +package policy + +import ( + "crypto/rand" + "fmt" + "io" +) + +//NewUUID will generate a new UUID. +//Code copied from https://play.golang.org/p/4FkNSiUDMg +func newUUID() (string, error) { + uuid := make([]byte, 16) + n, err := io.ReadFull(rand.Reader, uuid) + if n != len(uuid) || err != nil { + return "", err + } + + uuid[8] = uuid[8]&^0xc0 | 0x80 + uuid[6] = uuid[6]&^0xf0 | 0x40 + + return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil +} diff --git a/src/common/scheduler/task/replication_task.go b/src/common/scheduler/task/replication_task.go new file mode 100644 index 000000000..aaefcc571 --- /dev/null +++ b/src/common/scheduler/task/replication_task.go @@ -0,0 +1,24 @@ +package task + +import ( + "errors" +) + +//ReplicationTask is the task for triggering one replication +type ReplicationTask struct{} + +//NewReplicationTask is constructor of creating ReplicationTask +func NewReplicationTask() *ReplicationTask { + return &ReplicationTask{} +} + +//Name returns the name of this task +func (rt *ReplicationTask) Name() string { + return "replication" +} + +//Run the actions here +func (rt *ReplicationTask) Run() error { + //Trigger the replication here + return errors.New("Not implemented") +} diff --git a/src/common/scheduler/task/replication_task_test.go b/src/common/scheduler/task/replication_task_test.go new file mode 100644 index 000000000..cd0acd2c0 --- /dev/null +++ b/src/common/scheduler/task/replication_task_test.go @@ -0,0 +1,14 @@ +package task + +import "testing" + +func TestReplicationTask(t *testing.T) { + tk := NewReplicationTask() + if tk == nil { + t.Fail() + } + + if tk.Name() != "replication" { + t.Fail() + } +} diff --git a/src/common/scheduler/task/scan_all_task_test.go b/src/common/scheduler/task/scan_all_task_test.go index 18ac9202b..b7482fbfc 100644 --- a/src/common/scheduler/task/scan_all_task_test.go +++ b/src/common/scheduler/task/scan_all_task_test.go @@ -4,7 +4,7 @@ import ( "testing" ) -func TestTask(t *testing.T) { +func TestScanAllTask(t *testing.T) { tk := NewScanAllTask() if tk == nil { t.Fail() From 82b400c049d11b02f93bc43b1ee0600eb026da85 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 1 Nov 2017 14:13:34 +0800 Subject: [PATCH 03/37] Add replication filter implement --- src/replication/consts.go | 12 +-- src/replication/models/filter_config.go | 2 +- src/replication/models/filter_item.go | 6 +- src/replication/source/filter.go | 7 +- src/replication/source/pattern_filter.go | 84 +++++++++++++++++++ src/replication/source/pattern_filter_test.go | 63 ++++++++++++++ .../source/tag_combination_filter.go | 75 +++++++++++++++++ .../source/tag_combination_filter_test.go | 83 ++++++++++++++++++ 8 files changed, 317 insertions(+), 15 deletions(-) create mode 100644 src/replication/source/pattern_filter.go create mode 100644 src/replication/source/pattern_filter_test.go create mode 100644 src/replication/source/tag_combination_filter.go create mode 100644 src/replication/source/tag_combination_filter_test.go diff --git a/src/replication/consts.go b/src/replication/consts.go index 10075bcb3..76926f224 100644 --- a/src/replication/consts.go +++ b/src/replication/consts.go @@ -1,10 +1,10 @@ package replication const ( - //Kind of filter item is 'project' - filterItemKindProject = "project" - //Kind of filter item is 'repository' - filterItemKindRepository = "repository" - //Kind of filter item is 'tag' - filterItemKindTag = "tag" + //FilterItemKindProject : Kind of filter item is 'project' + FilterItemKindProject = "project" + //FilterItemKindRepository : Kind of filter item is 'repository' + FilterItemKindRepository = "repository" + //FilterItemKindTag : Kind of filter item is 'tag' + FilterItemKindTag = "tag" ) diff --git a/src/replication/models/filter_config.go b/src/replication/models/filter_config.go index cdc8e60a9..149780eaa 100644 --- a/src/replication/models/filter_config.go +++ b/src/replication/models/filter_config.go @@ -3,5 +3,5 @@ package models //FilterConfig is data model to provide configurations to the filters. type FilterConfig struct { //The pattern for fuzzy matching - pattern string + Pattern string } diff --git a/src/replication/models/filter_item.go b/src/replication/models/filter_item.go index 9565ac447..038287c78 100644 --- a/src/replication/models/filter_item.go +++ b/src/replication/models/filter_item.go @@ -4,16 +4,16 @@ package models type FilterItem struct { //The kind of the filtering resources. Support 'project','repository' and 'tag' etc. - kind string + Kind string //The key value of resource which can be used to filter out the resource matched with specified pattern. //E.g: //kind == 'project', value will be project name; //kind == 'repository', value will be repository name //kind == 'tag', value will be tag name. - value string + Value string //Extension placeholder. //To append more additional information if required by the filter. - metadata map[string]interface{} + Metadata map[string]interface{} } diff --git a/src/replication/source/filter.go b/src/replication/source/filter.go index 831e4b017..eb7b0623c 100644 --- a/src/replication/source/filter.go +++ b/src/replication/source/filter.go @@ -7,11 +7,8 @@ import ( //Filter define the operations of selecting the matched resources from the candidates //according to the specified pattern. type Filter interface { - //Initialize the filter with specified configurations like pattern definition - Init(config models.FilterConfig) - - //Set Convertor if necessary - SetConvertor(convertor Convertor) + //Initialize the filter + Init() error //Return the convertor if existing or nil if never set GetConvertor() Convertor diff --git a/src/replication/source/pattern_filter.go b/src/replication/source/pattern_filter.go new file mode 100644 index 000000000..6c895d0eb --- /dev/null +++ b/src/replication/source/pattern_filter.go @@ -0,0 +1,84 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "regexp" + + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/replication/models" +) + +// PatternFilter implements Filter interface for pattern filter +type PatternFilter struct { + kind string + pattern string + convertor Convertor +} + +// NewPatternFilter returns an instance of PatternFilter +func NewPatternFilter(kind, pattern string, convertor ...Convertor) *PatternFilter { + filer := &PatternFilter{ + kind: kind, + pattern: pattern, + } + + if len(convertor) > 0 { + filer.convertor = convertor[0] + } + + return filer +} + +// Init the filter. nil implement for now +func (p *PatternFilter) Init() error { + return nil +} + +// GetConvertor returns the convertor +func (p *PatternFilter) GetConvertor() Convertor { + return p.convertor +} + +// DoFilter filters resources +func (p *PatternFilter) DoFilter(filterItems []models.FilterItem) []models.FilterItem { + items := []models.FilterItem{} + for _, item := range filterItems { + if item.Kind != p.kind { + log.Warningf("unexpected filter item kind, expected: %s, got: %s, skip", + p.kind, item.Kind) + continue + } + + matched, err := regexp.MatchString(p.pattern, item.Value) + if err != nil { + log.Errorf("failed to match pattern %s, value %s: %v, skip", + p.pattern, item.Value, err) + continue + } + + if !matched { + log.Debugf("%s does not match to the %s filter %s, skip", + item.Value, p.kind, p.pattern) + continue + } + + log.Debugf("add %s to the result of %s filter %s", + item.Value, p.kind, p.pattern) + items = append(items, item) + } + + return items +} diff --git a/src/replication/source/pattern_filter_test.go b/src/replication/source/pattern_filter_test.go new file mode 100644 index 000000000..2f1ea372e --- /dev/null +++ b/src/replication/source/pattern_filter_test.go @@ -0,0 +1,63 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" + + "testing" +) + +var pfilter = NewPatternFilter(replication.FilterItemKindTag, "library/ubuntu:release-*", nil) + +func TestPatternFilterInit(t *testing.T) { + assert.Nil(t, pfilter.Init()) +} + +func TestPatternFilterGetConvertor(t *testing.T) { + assert.Nil(t, pfilter.GetConvertor()) +} + +func TestPatternFilterDoFilter(t *testing.T) { + items := []models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindProject, + }, + models.FilterItem{ + Kind: replication.FilterItemKindRepository, + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/ubuntu:release-14.04", + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/ubuntu:release-16.04", + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/ubuntu:test", + }, + } + result := pfilter.DoFilter(items) + assert.Equal(t, 2, len(result)) + assert.Equal(t, replication.FilterItemKindTag, result[0].Kind) + assert.Equal(t, "library/ubuntu:release-14.04", result[0].Value) + assert.Equal(t, replication.FilterItemKindTag, result[1].Kind) + assert.Equal(t, "library/ubuntu:release-16.04", result[1].Value) + +} diff --git a/src/replication/source/tag_combination_filter.go b/src/replication/source/tag_combination_filter.go new file mode 100644 index 000000000..ced506e44 --- /dev/null +++ b/src/replication/source/tag_combination_filter.go @@ -0,0 +1,75 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "strings" + + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" +) + +// TagCombinationFilter implements Filter interface for merging tag filter items +// whose repository are same into one repository filter item +type TagCombinationFilter struct{} + +// NewTagCombinationFilter returns an instance of TagCombinationFilter +func NewTagCombinationFilter() *TagCombinationFilter { + return &TagCombinationFilter{} +} + +// Init the filter. nil implement for now +func (t *TagCombinationFilter) Init() error { + return nil +} + +// GetConvertor returns the convertor +func (t *TagCombinationFilter) GetConvertor() Convertor { + return nil +} + +// DoFilter filters resources +func (t *TagCombinationFilter) DoFilter(filterItems []models.FilterItem) []models.FilterItem { + repos := map[string][]string{} + for _, item := range filterItems { + if item.Kind != replication.FilterItemKindTag { + log.Warningf("unexpected filter item kind, expected: %s, got: %s, skip", + replication.FilterItemKindTag, item.Kind) + continue + } + + strs := strings.Split(item.Value, ":") + if len(strs) != 2 { + log.Warningf("unexpected image format: %s, skip", item.Value) + continue + } + + repos[strs[0]] = append(repos[strs[0]], strs[1]) + } + + items := []models.FilterItem{} + for repo, tags := range repos { + items = append(items, models.FilterItem{ + Kind: replication.FilterItemKindRepository, + Value: repo, + Metadata: map[string]interface{}{ + "tags": tags, + }, + }) + } + + return items +} diff --git a/src/replication/source/tag_combination_filter_test.go b/src/replication/source/tag_combination_filter_test.go new file mode 100644 index 000000000..c3bfefff8 --- /dev/null +++ b/src/replication/source/tag_combination_filter_test.go @@ -0,0 +1,83 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" + + "testing" +) + +var tcfilter = NewTagCombinationFilter() + +func TestTagCombinationFilteInit(t *testing.T) { + assert.Nil(t, tcfilter.Init()) +} + +func TestTagCombinationFilteGetConvertor(t *testing.T) { + assert.Nil(t, tcfilter.GetConvertor()) +} + +func TestTagCombinationFilteDoFilter(t *testing.T) { + items := []models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindProject, + }, + models.FilterItem{ + Kind: replication.FilterItemKindRepository, + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/ubuntu:invalid_tag:latest", + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/ubuntu:14.04", + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/ubuntu:16.04", + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/centos:7", + }, + } + result := tcfilter.DoFilter(items) + assert.Equal(t, 2, len(result)) + + var ubuntu, centos models.FilterItem + if result[0].Value == "library/ubuntu" { + ubuntu = result[0] + centos = result[1] + } else { + centos = result[0] + ubuntu = result[1] + } + + assert.Equal(t, replication.FilterItemKindRepository, ubuntu.Kind) + assert.Equal(t, "library/ubuntu", ubuntu.Value) + metadata, ok := ubuntu.Metadata["tags"].([]string) + assert.True(t, ok) + assert.EqualValues(t, []string{"14.04", "16.04"}, metadata) + + assert.Equal(t, replication.FilterItemKindRepository, centos.Kind) + assert.Equal(t, "library/centos", centos.Value) + metadata, ok = centos.Metadata["tags"].([]string) + assert.True(t, ok) + assert.EqualValues(t, []string{"7"}, metadata) +} From 51d5df0849dec1207c73e5179b0836b3c82eaba3 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 2 Nov 2017 12:53:01 +0800 Subject: [PATCH 04/37] Update replication policy API to support trigger and filter --- docs/swagger.yaml | 44 ++++++++++- make/common/db/registry.sql | 2 + make/common/db/registry_sqlite.sql | 2 + src/common/dao/replication_job.go | 65 ++++++++++++----- src/common/models/replicate_test.go | 43 +++++++++++ src/common/models/replication_job.go | 93 +++++++++++++++++++----- src/ui/api/harborapi_test.go | 5 +- src/ui/api/replication_policy_test.go | 29 ++++++-- tests/apitests/apilib/rep_policy_post.go | 11 ++- tools/migration/changelog.md | 5 ++ 10 files changed, 252 insertions(+), 47 deletions(-) create mode 100644 src/common/models/replicate_test.go diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 1a4143473..f787b66df 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2397,9 +2397,18 @@ definitions: description: type: string description: The description of the policy. - cron_str: + trigger: type: string - description: The cron string for schedule job. + description: The trigger for schedule job. + filters: + type: array + description: >- + The replication policy filter array. + items: + $ref: '#/definitions/RepFilter' + replicate_deletion: + type: string + description: Whether replication deletion operation. start_time: type: string description: The start time of the policy. @@ -2428,6 +2437,18 @@ definitions: name: type: string description: The policy name. + trigger: + type: string + description: The trigger for schedule job. + filters: + type: array + description: >- + The replication policy filter array. + items: + $ref: '#/definitions/RepFilter' + replicate_deletion: + type: string + description: Whether replication deletion operation. enabled: type: integer format: int @@ -2449,9 +2470,26 @@ definitions: description: type: string description: The description of the policy. - cron_str: + trigger: type: string description: The cron string for schedule job. + filters: + type: array + description: The replication policy filter array. + items: + $ref: '#/definitions/RepFilter' + replicate_deletion: + type: string + description: Whether replication deletion operation. + RepFilter: + type: object + properties: + type: + type: string + description: The replication policy filter type. + value: + type: string + description: The replication policy filter value. RepPolicyEnablementReq: type: object properties: diff --git a/make/common/db/registry.sql b/make/common/db/registry.sql index 446627971..994b108d7 100644 --- a/make/common/db/registry.sql +++ b/make/common/db/registry.sql @@ -145,6 +145,8 @@ create table replication_policy ( description text, deleted tinyint (1) DEFAULT 0 NOT NULL, cron_str varchar(256), + filters varchar(1024), + replicate_deletion tinyint (1) DEFAULT 0 NOT NULL, start_time timestamp NULL, creation_time timestamp default CURRENT_TIMESTAMP, update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, diff --git a/make/common/db/registry_sqlite.sql b/make/common/db/registry_sqlite.sql index ce2cb9c20..2528a845d 100644 --- a/make/common/db/registry_sqlite.sql +++ b/make/common/db/registry_sqlite.sql @@ -141,6 +141,8 @@ create table replication_policy ( description text, deleted tinyint (1) DEFAULT 0 NOT NULL, cron_str varchar(256), + filters varchar(1024), + replicate_deletion tinyint (1) DEFAULT 0 NOT NULL, start_time timestamp NULL, creation_time timestamp default CURRENT_TIMESTAMP, update_time timestamp default CURRENT_TIMESTAMP diff --git a/src/common/dao/replication_job.go b/src/common/dao/replication_job.go index eb6593259..b9997838e 100644 --- a/src/common/dao/replication_job.go +++ b/src/common/dao/replication_job.go @@ -104,29 +104,18 @@ func FilterRepTargets(name string) ([]*models.RepTarget, error) { // AddRepPolicy ... func AddRepPolicy(policy models.RepPolicy) (int64, error) { - o := GetOrmer() - sql := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, start_time, creation_time, update_time ) values (?, ?, ?, ?, ?, ?, ?, ?, ?)` - p, err := o.Raw(sql).Prepare() - if err != nil { + if err := policy.MarshalFilter(); err != nil { return 0, err } - - params := []interface{}{} - params = append(params, policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.CronStr) now := time.Now() + policy.CreationTime = now + policy.UpdateTime = now if policy.Enabled == 1 { - params = append(params, now) - } else { - params = append(params, nil) + policy.StartTime = now } - params = append(params, now, now) + policy.Deleted = 0 - r, err := p.Exec(params...) - if err != nil { - return 0, err - } - id, err := r.LastInsertId() - return id, err + return GetOrmer().Insert(&policy) } // GetRepPolicy ... @@ -143,6 +132,10 @@ func GetRepPolicy(id int64) (*models.RepPolicy, error) { return nil, err } + if err := policy.UnmarshalFilter(); err != nil { + return nil, err + } + return &policy, nil } @@ -154,7 +147,8 @@ func FilterRepPolicies(name string, projectID int64) ([]*models.RepPolicy, error sql := `select rp.id, rp.project_id, rp.target_id, rt.name as target_name, rp.name, rp.enabled, rp.description, - rp.cron_str, rp.start_time, rp.creation_time, rp.update_time, + rp.cron_str, rp.filters, rp.replicate_deletion,rp.start_time, + rp.creation_time, rp.update_time, count(rj.status) as error_job_count from replication_policy rp left join replication_target rt on rp.target_id=rt.id @@ -180,6 +174,13 @@ func FilterRepPolicies(name string, projectID int64) ([]*models.RepPolicy, error if _, err := o.Raw(sql, args).QueryRows(&policies); err != nil { return nil, err } + + for _, policy := range policies { + if err := policy.UnmarshalFilter(); err != nil { + return nil, err + } + } + return policies, nil } @@ -197,6 +198,10 @@ func GetRepPolicyByName(name string) (*models.RepPolicy, error) { return nil, err } + if err := policy.UnmarshalFilter(); err != nil { + return nil, err + } + return &policy, nil } @@ -211,6 +216,12 @@ func GetRepPolicyByProject(projectID int64) ([]*models.RepPolicy, error) { return nil, err } + for _, policy := range policies { + if err := policy.UnmarshalFilter(); err != nil { + return nil, err + } + } + return policies, nil } @@ -225,6 +236,12 @@ func GetRepPolicyByTarget(targetID int64) ([]*models.RepPolicy, error) { return nil, err } + for _, policy := range policies { + if err := policy.UnmarshalFilter(); err != nil { + return nil, err + } + } + return policies, nil } @@ -239,14 +256,24 @@ func GetRepPolicyByProjectAndTarget(projectID, targetID int64) ([]*models.RepPol return nil, err } + for _, policy := range policies { + if err := policy.UnmarshalFilter(); err != nil { + return nil, err + } + } + return policies, nil } // UpdateRepPolicy ... func UpdateRepPolicy(policy *models.RepPolicy) error { + if err := policy.MarshalFilter(); err != nil { + return err + } o := GetOrmer() policy.UpdateTime = time.Now() - _, err := o.Update(policy, "TargetID", "Name", "Enabled", "Description", "CronStr", "UpdateTime") + _, err := o.Update(policy, "TargetID", "Name", "Enabled", "Description", + "Trigger", "FiltersInDB", "ReplicateDeletion", "UpdateTime") return err } diff --git a/src/common/models/replicate_test.go b/src/common/models/replicate_test.go new file mode 100644 index 000000000..142242aee --- /dev/null +++ b/src/common/models/replicate_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMarshalAndUnmarshalFilter(t *testing.T) { + filters := []*RepFilter{ + &RepFilter{ + Type: "repository", + Value: "library/ubuntu*", + }, + } + policy := &RepPolicy{ + Filters: filters, + } + + err := policy.MarshalFilter() + require.Nil(t, err) + + policy.Filters = nil + err = policy.UnmarshalFilter() + require.Nil(t, err) + + assert.EqualValues(t, filters, policy.Filters) +} diff --git a/src/common/models/replication_job.go b/src/common/models/replication_job.go index 255fc1d9a..f17ec5875 100644 --- a/src/common/models/replication_job.go +++ b/src/common/models/replication_job.go @@ -15,10 +15,13 @@ package models import ( + "encoding/json" + "fmt" "time" "github.com/astaxie/beego/validation" "github.com/vmware/harbor/src/common/utils" + "github.com/vmware/harbor/src/replication" ) const ( @@ -38,21 +41,23 @@ const ( // RepPolicy is the model for a replication policy, which associate to a project and a target (destination) type RepPolicy struct { - ID int64 `orm:"pk;auto;column(id)" json:"id"` - ProjectID int64 `orm:"column(project_id)" json:"project_id"` - ProjectName string `json:"project_name,omitempty"` - TargetID int64 `orm:"column(target_id)" json:"target_id"` - TargetName string `json:"target_name,omitempty"` - Name string `orm:"column(name)" json:"name"` - // Target RepTarget `orm:"-" json:"target"` - Enabled int `orm:"column(enabled)" json:"enabled"` - Description string `orm:"column(description)" json:"description"` - CronStr string `orm:"column(cron_str)" json:"cron_str"` - StartTime time.Time `orm:"column(start_time)" json:"start_time"` - CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` - UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` - ErrorJobCount int `json:"error_job_count"` - Deleted int `orm:"column(deleted)" json:"deleted"` + ID int64 `orm:"pk;auto;column(id)" json:"id"` + ProjectID int64 `orm:"column(project_id)" json:"project_id"` + ProjectName string `orm:"-" json:"project_name,omitempty"` + TargetID int64 `orm:"column(target_id)" json:"target_id"` + TargetName string `orm:"-" json:"target_name,omitempty"` + Name string `orm:"column(name)" json:"name"` + Enabled int `orm:"column(enabled)" json:"enabled"` + Description string `orm:"column(description)" json:"description"` + Trigger string `orm:"column(cron_str)" json:"trigger"` + Filters []*RepFilter `orm:"-" json:"filters"` + FiltersInDB string `orm:"column(filters)" json:"-"` + ReplicateDeletion bool `orm:"column(replicate_deletion)" json:"replicate_deletion"` + StartTime time.Time `orm:"column(start_time)" json:"start_time"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` + ErrorJobCount int `orm:"-" json:"error_job_count"` + Deleted int `orm:"column(deleted)" json:"deleted"` } // Valid ... @@ -77,8 +82,62 @@ func (r *RepPolicy) Valid(v *validation.Validation) { v.SetError("enabled", "must be 0 or 1") } - if len(r.CronStr) > 256 { - v.SetError("cron_str", "max length is 256") + if len(r.Trigger) > 256 { + v.SetError("trigger", "max length is 256") + } + + for _, filter := range r.Filters { + filter.Valid(v) + } + + if err := r.MarshalFilter(); err != nil { + v.SetError("filters", err.Error()) + } + if len(r.Filters) > 1024 { + v.SetError("filters", "max length is 1024") + } +} + +// MarshalFilter marshal RepFilter array to json string +func (r *RepPolicy) MarshalFilter() error { + if r.Filters != nil { + b, err := json.Marshal(r.Filters) + if err != nil { + return err + } + r.FiltersInDB = string(b) + } + return nil +} + +// UnmarshalFilter unmarshal json string to RepFilter array +func (r *RepPolicy) UnmarshalFilter() error { + if len(r.FiltersInDB) > 0 { + filter := []*RepFilter{} + if err := json.Unmarshal([]byte(r.FiltersInDB), &filter); err != nil { + return err + } + r.Filters = filter + } + return nil +} + +// RepFilter holds information for the replication policy filter +type RepFilter struct { + Type string `json:"type"` + Value string `json:"value"` +} + +// Valid ... +func (r *RepFilter) Valid(v *validation.Validation) { + if !(r.Type == replication.FilterItemKindProject || + r.Type == replication.FilterItemKindRepository || + r.Type == replication.FilterItemKindTag) { + v.SetError("filter.type", fmt.Sprintf("invalid filter type: %s", r.Type)) + } + + if len(r.Value) == 0 { + v.SetError("filter.value", "can not be empty") } } diff --git a/src/ui/api/harborapi_test.go b/src/ui/api/harborapi_test.go index ef754bad0..fa60ec3c9 100644 --- a/src/ui/api/harborapi_test.go +++ b/src/ui/api/harborapi_test.go @@ -711,7 +711,10 @@ func (a testapi) AddPolicy(authInfo usrInfo, repPolicy apilib.RepPolicyPost) (in _sling = _sling.Path(path) _sling = _sling.BodyJSON(repPolicy) - httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo) + httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo) + if httpStatusCode != http.StatusCreated { + log.Println(string(body)) + } return httpStatusCode, err } diff --git a/src/ui/api/replication_policy_test.go b/src/ui/api/replication_policy_test.go index f518537a2..622cbecb3 100644 --- a/src/ui/api/replication_policy_test.go +++ b/src/ui/api/replication_policy_test.go @@ -15,10 +15,14 @@ package api import ( "fmt" - "github.com/stretchr/testify/assert" - "github.com/vmware/harbor/tests/apitests/apilib" "strconv" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/tests/apitests/apilib" ) const ( @@ -37,7 +41,12 @@ func TestPoliciesPost(t *testing.T) { //add target CommonAddTarget() targetID := int64(CommonGetTarget()) - repPolicy := &apilib.RepPolicyPost{int64(1), targetID, addPolicyName} + repPolicy := &apilib.RepPolicyPost{int64(1), targetID, addPolicyName, []*models.RepFilter{ + &models.RepFilter{ + Type: replication.FilterItemKindRepository, + Value: "library/ubuntu*", + }, + }} fmt.Println("Testing Policies Post API") @@ -52,7 +61,7 @@ func TestPoliciesPost(t *testing.T) { } //-------------------case 2 : response code = 409------------------------// - fmt.Println("case 1 : response code = 409:policy already exists") + fmt.Println("case 2 : response code = 409:policy already exists") httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy) if err != nil { t.Error("Error while add policy", err.Error()) @@ -108,7 +117,7 @@ func TestPoliciesPost(t *testing.T) { } //-------------------case 7 : response code = 400------------------------// - fmt.Println("case 6 : response code = 400:target_id does not exist.") + fmt.Println("case 7 : response code = 400:target_id does not exist.") repPolicy.TargetId = int64(1111) httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy) @@ -119,6 +128,16 @@ func TestPoliciesPost(t *testing.T) { assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400") } + fmt.Println("case 8 : response code = 400: invalid filter") + repPolicy = &apilib.RepPolicyPost{int64(1), targetID, addPolicyName, []*models.RepFilter{ + &models.RepFilter{ + Type: "replication", + Value: "", + }, + }} + httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy) + require.Nil(t, err) + assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400") } func TestPoliciesList(t *testing.T) { diff --git a/tests/apitests/apilib/rep_policy_post.go b/tests/apitests/apilib/rep_policy_post.go index 6b8d42731..a78b01aa6 100644 --- a/tests/apitests/apilib/rep_policy_post.go +++ b/tests/apitests/apilib/rep_policy_post.go @@ -1,10 +1,10 @@ -/* +/* * Harbor API * * These APIs provide services for manipulating Harbor project. * * OpenAPI spec version: 0.3.0 - * + * * Generated by: https://github.com/swagger-api/swagger-codegen.git * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,6 +22,10 @@ package apilib +import ( + "github.com/vmware/harbor/src/common/models" +) + type RepPolicyPost struct { // The project ID. @@ -32,4 +36,7 @@ type RepPolicyPost struct { // The policy name. Name string `json:"name,omitempty"` + + // Filters + Filters []*models.RepFilter `json:"filters"` } diff --git a/tools/migration/changelog.md b/tools/migration/changelog.md index 8c40e5986..d2e9a0b70 100644 --- a/tools/migration/changelog.md +++ b/tools/migration/changelog.md @@ -56,3 +56,8 @@ Changelog for harbor database schema - insert data into table `project_metadata` - delete column `public` from table `project` - add column `insecure` to table `replication_target` + +## 1.3.1 + + - add column `filters` to table `replication_policy` + - add column `replicate_deletion` to table `replication_policy` From abe3d37a33bf46d6d5ae83328b98886cbf44525a Mon Sep 17 00:00:00 2001 From: Steven Zou Date: Thu, 2 Nov 2017 17:10:15 +0800 Subject: [PATCH 05/37] Add registry adaptor interface and sourcer definitions --- src/replication/consts.go | 3 ++ src/replication/models/registry_models.go | 34 +++++++++++++++++ src/replication/registry/adaptor.go | 34 +++++++++++++++++ src/replication/registry/harbor_adaptor.go | 44 ++++++++++++++++++++++ src/replication/source/sourcer.go | 44 ++++++++++++++++++++++ src/replication/source/sourcer_test.go | 24 ++++++++++++ 6 files changed, 183 insertions(+) create mode 100644 src/replication/models/registry_models.go create mode 100644 src/replication/registry/adaptor.go create mode 100644 src/replication/registry/harbor_adaptor.go create mode 100644 src/replication/source/sourcer.go create mode 100644 src/replication/source/sourcer_test.go diff --git a/src/replication/consts.go b/src/replication/consts.go index 76926f224..150937585 100644 --- a/src/replication/consts.go +++ b/src/replication/consts.go @@ -7,4 +7,7 @@ const ( FilterItemKindRepository = "repository" //FilterItemKindTag : Kind of filter item is 'tag' FilterItemKindTag = "tag" + + //AdaptorKindHarbor : Kind of adaptor of Harbor + AdaptorKindHarbor = "Harbor" ) diff --git a/src/replication/models/registry_models.go b/src/replication/models/registry_models.go new file mode 100644 index 000000000..69be2268c --- /dev/null +++ b/src/replication/models/registry_models.go @@ -0,0 +1,34 @@ +package models + +//Namespace is the resource group/scope like project in Harbor and organization in docker hub. +type Namespace struct { + //Name of the namespace + Name string + + //Extensions to provide flexibility + Metadata map[string]interface{} +} + +//Repository is to keep the info of image repository. +type Repository struct { + //Name of the repository + Name string + + //Project reference of this repository belongs to + Namespace Namespace + + //Extensions to provide flexibility + Metadata map[string]interface{} +} + +//Tag keeps the info of image with specified version +type Tag struct { + //Name of the tag + Name string + + //The repository reference of this tag belongs to + Repository Repository + + //Extensions to provide flexibility + Metadata map[string]interface{} +} diff --git a/src/replication/registry/adaptor.go b/src/replication/registry/adaptor.go new file mode 100644 index 000000000..29b3cf138 --- /dev/null +++ b/src/replication/registry/adaptor.go @@ -0,0 +1,34 @@ +package registry + +import ( + "github.com/vmware/harbor/src/replication/models" +) + +//Adaptor defines the unified operations for all the supported registries such as Harbor or DockerHub. +//It's used to adapt the different interfaces provied by the different registry providers. +//Use external registry with restful api providing as example, these intrefaces may depends on the +//related restful apis like: +// /api/vx/repositories/{namespace}/{repositoryName}/tags/{name} +// /api/v0/accounts/{namespace} +type Adaptor interface { + //Return the unique kind identifier of the adaptor + Kind() string + + //Get all the namespaces + GetNamespaces() []models.Namespace + + //Get the namespace with the specified name + GetNamespace(name string) models.Namespace + + //Get all the repositories under the specified namespace + GetRepositories(namespace string) []models.Repository + + //Get the repository with the specified name under the specified namespace + GetRepository(name string, namespace string) models.Repository + + //Get all the tags of the specified repository under the namespace + GetTags(repositoryName string, namespace string) []models.Tag + + //Get the tag with the specified name of the repository under the namespace + GetTag(name string, repositoryName string, namespace string) models.Tag +} diff --git a/src/replication/registry/harbor_adaptor.go b/src/replication/registry/harbor_adaptor.go new file mode 100644 index 000000000..6c7ffe09d --- /dev/null +++ b/src/replication/registry/harbor_adaptor.go @@ -0,0 +1,44 @@ +package registry + +import ( + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" +) + +//HarborAdaptor is defined to adapt the Harbor registry +type HarborAdaptor struct{} + +//Kind returns the unique kind identifier of the adaptor +func (ha *HarborAdaptor) Kind() string { + return replication.AdaptorKindHarbor +} + +//GetNamespaces is ued to get all the namespaces +func (ha *HarborAdaptor) GetNamespaces() []models.Namespace { + return nil +} + +//GetNamespace is used to get the namespace with the specified name +func (ha *HarborAdaptor) GetNamespace(name string) models.Namespace { + return models.Namespace{} +} + +//GetRepositories is used to get all the repositories under the specified namespace +func (ha *HarborAdaptor) GetRepositories(namespace string) []models.Repository { + return nil +} + +//GetRepository is used to get the repository with the specified name under the specified namespace +func (ha *HarborAdaptor) GetRepository(name string, namespace string) models.Repository { + return models.Repository{} +} + +//GetTags is used to get all the tags of the specified repository under the namespace +func (ha *HarborAdaptor) GetTags(repositoryName string, namespace string) []models.Tag { + return nil +} + +//GetTag is used to get the tag with the specified name of the repository under the namespace +func (ha *HarborAdaptor) GetTag(name string, repositoryName string, namespace string) models.Tag { + return models.Tag{} +} diff --git a/src/replication/source/sourcer.go b/src/replication/source/sourcer.go new file mode 100644 index 000000000..4bbfc2c44 --- /dev/null +++ b/src/replication/source/sourcer.go @@ -0,0 +1,44 @@ +package source + +import ( + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/registry" +) + +//Sourcer is used to manage and/or handle all the artifacts and information related with source registry. +//All the things with replication source should be covered in this object. +type Sourcer struct { + //Keep the adaptors we support now + adaptors map[string]registry.Adaptor +} + +//ReplicationSourcer is default sourcer for replication service. +var ReplicationSourcer = NewSourcer() + +//NewSourcer is the constructor of Sourcer +func NewSourcer() *Sourcer { + return &Sourcer{ + adaptors: make(map[string]registry.Adaptor), + } +} + +//Init will do some initialization work like registrying all the adaptors we support +func (sc *Sourcer) Init() { + //Register Harbor adaptor + sc.adaptors[replication.AdaptorKindHarbor] = ®istry.HarborAdaptor{} +} + +//GetAdaptor returns the required adaptor with the specified kind. +//If no adaptor with the specified kind existing, nil will be returned. +func (sc *Sourcer) GetAdaptor(kind string) registry.Adaptor { + if len(kind) == 0 { + return nil + } + + return sc.adaptors[kind] +} + +//Init the adaptors +func Init() { + ReplicationSourcer.Init() +} diff --git a/src/replication/source/sourcer_test.go b/src/replication/source/sourcer_test.go new file mode 100644 index 000000000..1cf9b0a90 --- /dev/null +++ b/src/replication/source/sourcer_test.go @@ -0,0 +1,24 @@ +package source + +import ( + "testing" + + "github.com/vmware/harbor/src/replication" +) + +func TestReplicationSourcer(t *testing.T) { + testingSourcer := NewSourcer() + if testingSourcer == nil { + t.Fatal("Failed to create sourcer") + } + + testingSourcer.Init() + + if testingSourcer.GetAdaptor("") != nil { + t.Fatal("Empty kind should not be supported") + } + + if testingSourcer.GetAdaptor(replication.AdaptorKindHarbor) == nil { + t.Fatalf("%s adaptor should be existing", replication.AdaptorKindHarbor) + } +} From f86a4be84d7b152aafdc9d10ef2b0c293f556978 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 3 Nov 2017 14:21:32 +0800 Subject: [PATCH 06/37] Implement Convert interface --- .../source/repository_convertor.go | 59 ++++++++++ .../source/repository_convertor_test.go | 101 ++++++++++++++++++ .../source/tag_combination_filter_test.go | 6 +- src/replication/source/tag_convertor.go | 59 ++++++++++ src/replication/source/tag_convertor_test.go | 57 ++++++++++ 5 files changed, 279 insertions(+), 3 deletions(-) create mode 100644 src/replication/source/repository_convertor.go create mode 100644 src/replication/source/repository_convertor_test.go create mode 100644 src/replication/source/tag_convertor.go create mode 100644 src/replication/source/tag_convertor_test.go diff --git a/src/replication/source/repository_convertor.go b/src/replication/source/repository_convertor.go new file mode 100644 index 000000000..12e5bda28 --- /dev/null +++ b/src/replication/source/repository_convertor.go @@ -0,0 +1,59 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" + "github.com/vmware/harbor/src/replication/registry" +) + +// RepositoryConvertor implement Convertor interface, convert projects to repositories +type RepositoryConvertor struct { + registry registry.Adaptor +} + +// NewRepositoryConvertor returns an instance of RepositoryConvertor +func NewRepositoryConvertor(registry registry.Adaptor) *RepositoryConvertor { + return &RepositoryConvertor{ + registry: registry, + } +} + +// Convert projects to repositories +func (r *RepositoryConvertor) Convert(items []models.FilterItem) []models.FilterItem { + result := []models.FilterItem{} + for _, item := range items { + if item.Kind != replication.FilterItemKindProject { + log.Warningf("unexpected filter item kind for repository convertor, expected %s got %s, skip", + replication.FilterItemKindProject, item.Kind) + continue + } + + repositories := r.registry.GetRepositories(item.Value) + for _, repository := range repositories { + result = append(result, models.FilterItem{ + Kind: replication.FilterItemKindRepository, + Value: repository.Name, + // public is used to create project if it does not exist when replicating + Metadata: map[string]interface{}{ + "public": item.Metadata["public"], + }, + }) + } + } + return result +} diff --git a/src/replication/source/repository_convertor_test.go b/src/replication/source/repository_convertor_test.go new file mode 100644 index 000000000..48e223232 --- /dev/null +++ b/src/replication/source/repository_convertor_test.go @@ -0,0 +1,101 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" +) + +func TestRepositoryConvert(t *testing.T) { + items := []models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindProject, + Value: "library", + Metadata: map[string]interface{}{ + "public": true, + }, + }, + models.FilterItem{ + Kind: replication.FilterItemKindRepository, + }, + } + expected := []models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindRepository, + Value: "library/ubuntu", + Metadata: map[string]interface{}{ + "public": true, + }, + }, + models.FilterItem{ + Kind: replication.FilterItemKindRepository, + Value: "library/centos", + Metadata: map[string]interface{}{ + "public": true, + }, + }, + } + + convertor := NewRepositoryConvertor(&fakeRegistryAdaptor{}) + assert.EqualValues(t, expected, convertor.Convert(items)) +} + +type fakeRegistryAdaptor struct{} + +func (f *fakeRegistryAdaptor) Kind() string { + return "fake" +} + +func (f *fakeRegistryAdaptor) GetNamespaces() []models.Namespace { + return nil +} + +func (f *fakeRegistryAdaptor) GetNamespace(name string) models.Namespace { + return models.Namespace{} +} + +func (f *fakeRegistryAdaptor) GetRepositories(namespace string) []models.Repository { + return []models.Repository{ + models.Repository{ + Name: "library/ubuntu", + }, + models.Repository{ + Name: "library/centos", + }, + } +} + +func (f *fakeRegistryAdaptor) GetRepository(name string, namespace string) models.Repository { + return models.Repository{} +} + +func (f *fakeRegistryAdaptor) GetTags(repositoryName string, namespace string) []models.Tag { + return []models.Tag{ + models.Tag{ + Name: "library/ubuntu:14.04", + }, + models.Tag{ + Name: "library/ubuntu:16.04", + }, + } +} + +func (f *fakeRegistryAdaptor) GetTag(name string, repositoryName string, namespace string) models.Tag { + return models.Tag{} +} diff --git a/src/replication/source/tag_combination_filter_test.go b/src/replication/source/tag_combination_filter_test.go index c3bfefff8..8a84218b6 100644 --- a/src/replication/source/tag_combination_filter_test.go +++ b/src/replication/source/tag_combination_filter_test.go @@ -24,15 +24,15 @@ import ( var tcfilter = NewTagCombinationFilter() -func TestTagCombinationFilteInit(t *testing.T) { +func TestTagCombinationFilterInit(t *testing.T) { assert.Nil(t, tcfilter.Init()) } -func TestTagCombinationFilteGetConvertor(t *testing.T) { +func TestTagCombinationFilterGetConvertor(t *testing.T) { assert.Nil(t, tcfilter.GetConvertor()) } -func TestTagCombinationFilteDoFilter(t *testing.T) { +func TestTagCombinationFilterDoFilter(t *testing.T) { items := []models.FilterItem{ models.FilterItem{ Kind: replication.FilterItemKindProject, diff --git a/src/replication/source/tag_convertor.go b/src/replication/source/tag_convertor.go new file mode 100644 index 000000000..236e04e3c --- /dev/null +++ b/src/replication/source/tag_convertor.go @@ -0,0 +1,59 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" + "github.com/vmware/harbor/src/replication/registry" +) + +// TagConvertor implement Convertor interface, convert repositories to tags +type TagConvertor struct { + registry registry.Adaptor +} + +// NewTagConvertor returns an instance of TagConvertor +func NewTagConvertor(registry registry.Adaptor) *TagConvertor { + return &TagConvertor{ + registry: registry, + } +} + +//Convert repositories to tags +func (t *TagConvertor) Convert(items []models.FilterItem) []models.FilterItem { + result := []models.FilterItem{} + for _, item := range items { + if item.Kind != replication.FilterItemKindRepository { + log.Warningf("unexpected filter item kind for tag convertor, expected %s got %s, skip", + replication.FilterItemKindRepository, item.Kind) + continue + } + + tags := t.registry.GetTags(item.Value, "") + for _, tag := range tags { + result = append(result, models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: tag.Name, + // public is used to create project if it does not exist when replicating + Metadata: map[string]interface{}{ + "public": item.Metadata["public"], + }, + }) + } + } + return result +} diff --git a/src/replication/source/tag_convertor_test.go b/src/replication/source/tag_convertor_test.go new file mode 100644 index 000000000..151192bbb --- /dev/null +++ b/src/replication/source/tag_convertor_test.go @@ -0,0 +1,57 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" +) + +func TestTagConvert(t *testing.T) { + items := []models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindRepository, + Value: "library/ubuntu", + Metadata: map[string]interface{}{ + "public": true, + }, + }, + models.FilterItem{ + Kind: replication.FilterItemKindProject, + }, + } + expected := []models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/ubuntu:14.04", + Metadata: map[string]interface{}{ + "public": true, + }, + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/ubuntu:16.04", + Metadata: map[string]interface{}{ + "public": true, + }, + }, + } + + convertor := NewTagConvertor(&fakeRegistryAdaptor{}) + assert.EqualValues(t, expected, convertor.Convert(items)) +} From 049642e894f93d8eda1ff533e79ded6feaaf4b49 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 2 Nov 2017 18:22:06 +0800 Subject: [PATCH 07/37] Implement filter chain --- .../source/default_filter_chain.go | 57 ++++++++++++++ .../source/default_filter_chain_test.go | 75 +++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 src/replication/source/default_filter_chain.go create mode 100644 src/replication/source/default_filter_chain_test.go diff --git a/src/replication/source/default_filter_chain.go b/src/replication/source/default_filter_chain.go new file mode 100644 index 000000000..1acd29bf8 --- /dev/null +++ b/src/replication/source/default_filter_chain.go @@ -0,0 +1,57 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "github.com/vmware/harbor/src/replication/models" +) + +// DefaultFilterChain provides a default implement for interface FilterChain +type DefaultFilterChain struct { + filters []Filter +} + +// NewDefaultFilterChain returns an instance of DefaultFilterChain +func NewDefaultFilterChain(filters []Filter) *DefaultFilterChain { + return &DefaultFilterChain{ + filters: filters, + } +} + +// Build nil implement now +func (d *DefaultFilterChain) Build(filters []Filter) error { + return nil +} + +// Filters returns the filter list +func (d *DefaultFilterChain) Filters() []Filter { + return d.filters +} + +// DoFilter does the filter works for filterItems +func (d *DefaultFilterChain) DoFilter(filterItems []models.FilterItem) []models.FilterItem { + if len(filterItems) == 0 { + return []models.FilterItem{} + } + + for _, filter := range d.filters { + convertor := filter.GetConvertor() + if convertor != nil { + filterItems = convertor.Convert(filterItems) + } + filterItems = filter.DoFilter(filterItems) + } + return filterItems +} diff --git a/src/replication/source/default_filter_chain_test.go b/src/replication/source/default_filter_chain_test.go new file mode 100644 index 000000000..ee1eec79a --- /dev/null +++ b/src/replication/source/default_filter_chain_test.go @@ -0,0 +1,75 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" +) + +func TestBuild(t *testing.T) { + chain := NewDefaultFilterChain(nil) + require.Nil(t, chain.Build(nil)) +} + +func TestFilters(t *testing.T) { + filters := []Filter{NewPatternFilter("project", "*")} + chain := NewDefaultFilterChain(filters) + assert.EqualValues(t, filters, chain.Filters()) +} + +func TestDoFilter(t *testing.T) { + projectFilter := NewPatternFilter(replication.FilterItemKindProject, "library*") + repositoryFilter := NewPatternFilter(replication.FilterItemKindRepository, + "library/ubuntu*", &fakeRepositoryConvertor{}) + filters := []Filter{projectFilter, repositoryFilter} + + items := []models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindProject, + Value: "library", + }, + models.FilterItem{ + Kind: replication.FilterItemKindProject, + Value: "test", + }, + } + chain := NewDefaultFilterChain(filters) + items = chain.DoFilter(items) + assert.EqualValues(t, []models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindRepository, + Value: "library/ubuntu", + }, + }, items) + +} + +type fakeRepositoryConvertor struct{} + +func (f *fakeRepositoryConvertor) Convert(items []models.FilterItem) []models.FilterItem { + result := []models.FilterItem{} + for _, item := range items { + result = append(result, models.FilterItem{ + Kind: replication.FilterItemKindRepository, + Value: item.Value + "/ubuntu", + }) + } + return result +} From 5cef58baa1c1390bf20099675b4f779c16ad35e4 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 8 Nov 2017 17:53:06 +0800 Subject: [PATCH 08/37] update according to the comments --- docs/swagger.yaml | 40 +++++++-- src/common/dao/replication_job.go | 18 ++-- src/common/models/replication_job.go | 90 +++++++++++++------ ...{replicate_test.go => replication_test.go} | 13 ++- src/replication/consts.go | 6 ++ src/ui/api/replication_policy_test.go | 31 ++++--- tests/apitests/apilib/rep_policy_post.go | 3 + 7 files changed, 144 insertions(+), 57 deletions(-) rename src/common/models/{replicate_test.go => replication_test.go} (77%) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index f787b66df..c6de6a1c1 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2398,17 +2398,21 @@ definitions: type: string description: The description of the policy. trigger: - type: string + type: object description: The trigger for schedule job. + items: + $ref: '#/definitions/RepTrigger' filters: type: array - description: >- - The replication policy filter array. + description: The replication policy filter array. items: $ref: '#/definitions/RepFilter' + replicate_existing_image_now: + type: string + description: Whether to replicate the existing images now. replicate_deletion: type: string - description: Whether replication deletion operation. + description: Whether to replicate the deletion operation. start_time: type: string description: The start time of the policy. @@ -2438,14 +2442,18 @@ definitions: type: string description: The policy name. trigger: - type: string + type: object description: The trigger for schedule job. + items: + $ref: '#/definitions/RepTrigger' filters: type: array - description: >- - The replication policy filter array. + description: The replication policy filter array. items: $ref: '#/definitions/RepFilter' + replicate_existing_image_now: + type: string + description: Whether to replicate the existing images now. replicate_deletion: type: string description: Whether replication deletion operation. @@ -2471,16 +2479,30 @@ definitions: type: string description: The description of the policy. trigger: - type: string - description: The cron string for schedule job. + type: object + description: The trigger for schedule job. + items: + $ref: '#/definitions/RepTrigger' filters: type: array description: The replication policy filter array. items: $ref: '#/definitions/RepFilter' + replicate_existing_image_now: + type: string + description: Whether to replicate the existing images now. replicate_deletion: type: string description: Whether replication deletion operation. + RepTrigger: + type: object + properties: + type: + type: string + description: The replication policy trigger type. + params: + type: object + description: The map is the replication policy trigger parameters. RepFilter: type: object properties: diff --git a/src/common/dao/replication_job.go b/src/common/dao/replication_job.go index b9997838e..8771b179c 100644 --- a/src/common/dao/replication_job.go +++ b/src/common/dao/replication_job.go @@ -104,7 +104,7 @@ func FilterRepTargets(name string) ([]*models.RepTarget, error) { // AddRepPolicy ... func AddRepPolicy(policy models.RepPolicy) (int64, error) { - if err := policy.MarshalFilter(); err != nil { + if err := policy.Marshal(); err != nil { return 0, err } now := time.Now() @@ -132,7 +132,7 @@ func GetRepPolicy(id int64) (*models.RepPolicy, error) { return nil, err } - if err := policy.UnmarshalFilter(); err != nil { + if err := policy.Unmarshal(); err != nil { return nil, err } @@ -176,7 +176,7 @@ func FilterRepPolicies(name string, projectID int64) ([]*models.RepPolicy, error } for _, policy := range policies { - if err := policy.UnmarshalFilter(); err != nil { + if err := policy.Unmarshal(); err != nil { return nil, err } } @@ -198,7 +198,7 @@ func GetRepPolicyByName(name string) (*models.RepPolicy, error) { return nil, err } - if err := policy.UnmarshalFilter(); err != nil { + if err := policy.Unmarshal(); err != nil { return nil, err } @@ -217,7 +217,7 @@ func GetRepPolicyByProject(projectID int64) ([]*models.RepPolicy, error) { } for _, policy := range policies { - if err := policy.UnmarshalFilter(); err != nil { + if err := policy.Unmarshal(); err != nil { return nil, err } } @@ -237,7 +237,7 @@ func GetRepPolicyByTarget(targetID int64) ([]*models.RepPolicy, error) { } for _, policy := range policies { - if err := policy.UnmarshalFilter(); err != nil { + if err := policy.Unmarshal(); err != nil { return nil, err } } @@ -257,7 +257,7 @@ func GetRepPolicyByProjectAndTarget(projectID, targetID int64) ([]*models.RepPol } for _, policy := range policies { - if err := policy.UnmarshalFilter(); err != nil { + if err := policy.Unmarshal(); err != nil { return nil, err } } @@ -267,13 +267,13 @@ func GetRepPolicyByProjectAndTarget(projectID, targetID int64) ([]*models.RepPol // UpdateRepPolicy ... func UpdateRepPolicy(policy *models.RepPolicy) error { - if err := policy.MarshalFilter(); err != nil { + if err := policy.Marshal(); err != nil { return err } o := GetOrmer() policy.UpdateTime = time.Now() _, err := o.Update(policy, "TargetID", "Name", "Enabled", "Description", - "Trigger", "FiltersInDB", "ReplicateDeletion", "UpdateTime") + "TriggerInDB", "FiltersInDB", "ReplicateDeletion", "UpdateTime") return err } diff --git a/src/common/models/replication_job.go b/src/common/models/replication_job.go index f17ec5875..55d726c99 100644 --- a/src/common/models/replication_job.go +++ b/src/common/models/replication_job.go @@ -41,23 +41,25 @@ const ( // RepPolicy is the model for a replication policy, which associate to a project and a target (destination) type RepPolicy struct { - ID int64 `orm:"pk;auto;column(id)" json:"id"` - ProjectID int64 `orm:"column(project_id)" json:"project_id"` - ProjectName string `orm:"-" json:"project_name,omitempty"` - TargetID int64 `orm:"column(target_id)" json:"target_id"` - TargetName string `orm:"-" json:"target_name,omitempty"` - Name string `orm:"column(name)" json:"name"` - Enabled int `orm:"column(enabled)" json:"enabled"` - Description string `orm:"column(description)" json:"description"` - Trigger string `orm:"column(cron_str)" json:"trigger"` - Filters []*RepFilter `orm:"-" json:"filters"` - FiltersInDB string `orm:"column(filters)" json:"-"` - ReplicateDeletion bool `orm:"column(replicate_deletion)" json:"replicate_deletion"` - StartTime time.Time `orm:"column(start_time)" json:"start_time"` - CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` - UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` - ErrorJobCount int `orm:"-" json:"error_job_count"` - Deleted int `orm:"column(deleted)" json:"deleted"` + ID int64 `orm:"pk;auto;column(id)" json:"id"` + ProjectID int64 `orm:"column(project_id)" json:"project_id"` + ProjectName string `orm:"-" json:"project_name,omitempty"` + TargetID int64 `orm:"column(target_id)" json:"target_id"` + TargetName string `orm:"-" json:"target_name,omitempty"` + Name string `orm:"column(name)" json:"name"` + Enabled int `orm:"column(enabled)" json:"enabled"` + Description string `orm:"column(description)" json:"description"` + Trigger *RepTrigger `orm:"-" json:"trigger"` + TriggerInDB string `orm:"column(cron_str)" json:"-"` + Filters []*RepFilter `orm:"-" json:"filters"` + FiltersInDB string `orm:"column(filters)" json:"-"` + ReplicateExistingImageNow bool `orm:"-" json:"replicate_existing_image_now"` + ReplicateDeletion bool `orm:"column(replicate_deletion)" json:"replicate_deletion"` + StartTime time.Time `orm:"column(start_time)" json:"start_time"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` + ErrorJobCount int `orm:"-" json:"error_job_count"` + Deleted int `orm:"column(deleted)" json:"deleted"` } // Valid ... @@ -82,24 +84,37 @@ func (r *RepPolicy) Valid(v *validation.Validation) { v.SetError("enabled", "must be 0 or 1") } - if len(r.Trigger) > 256 { - v.SetError("trigger", "max length is 256") + if r.Trigger != nil { + r.Trigger.Valid(v) } for _, filter := range r.Filters { filter.Valid(v) } - if err := r.MarshalFilter(); err != nil { - v.SetError("filters", err.Error()) + if err := r.Marshal(); err != nil { + v.SetError("trigger or filters", err.Error()) } - if len(r.Filters) > 1024 { + + if len(r.TriggerInDB) > 256 { + v.SetError("trigger", "max length is 256") + } + + if len(r.FiltersInDB) > 1024 { v.SetError("filters", "max length is 1024") } } -// MarshalFilter marshal RepFilter array to json string -func (r *RepPolicy) MarshalFilter() error { +// Marshal marshal RepTrigger and RepFilter array to json string +func (r *RepPolicy) Marshal() error { + if r.Trigger != nil { + b, err := json.Marshal(r.Trigger) + if err != nil { + return err + } + r.TriggerInDB = string(b) + } + if r.Filters != nil { b, err := json.Marshal(r.Filters) if err != nil { @@ -110,8 +125,16 @@ func (r *RepPolicy) MarshalFilter() error { return nil } -// UnmarshalFilter unmarshal json string to RepFilter array -func (r *RepPolicy) UnmarshalFilter() error { +// Unmarshal unmarshal json string to RepTrigger and RepFilter array +func (r *RepPolicy) Unmarshal() error { + if len(r.TriggerInDB) > 0 { + trigger := &RepTrigger{} + if err := json.Unmarshal([]byte(r.TriggerInDB), &trigger); err != nil { + return err + } + r.Trigger = trigger + } + if len(r.FiltersInDB) > 0 { filter := []*RepFilter{} if err := json.Unmarshal([]byte(r.FiltersInDB), &filter); err != nil { @@ -141,6 +164,21 @@ func (r *RepFilter) Valid(v *validation.Validation) { } } +// RepTrigger holds information for the replication policy trigger +type RepTrigger struct { + Type string `json:"type"` + Params map[string]interface{} `json:"params"` +} + +// Valid ... +func (r *RepTrigger) Valid(v *validation.Validation) { + if !(r.Type == replication.TriggerKindManually || + r.Type == replication.TriggerKindSchedule || + r.Type == replication.TriggerKindImmediately) { + v.SetError("trigger.type", fmt.Sprintf("invalid trigger type: %s", r.Type)) + } +} + // RepJob is the model for a replication job, which is the execution unit on job service, currently it is used to transfer/remove // a repository to/from a remote registry instance. type RepJob struct { diff --git a/src/common/models/replicate_test.go b/src/common/models/replication_test.go similarity index 77% rename from src/common/models/replicate_test.go rename to src/common/models/replication_test.go index 142242aee..52978f983 100644 --- a/src/common/models/replicate_test.go +++ b/src/common/models/replication_test.go @@ -21,7 +21,11 @@ import ( "github.com/stretchr/testify/require" ) -func TestMarshalAndUnmarshalFilter(t *testing.T) { +func TestMarshalAndUnmarshal(t *testing.T) { + trigger := &RepTrigger{ + Type: "schedule", + Params: map[string]interface{}{"date": "2:00"}, + } filters := []*RepFilter{ &RepFilter{ Type: "repository", @@ -29,15 +33,18 @@ func TestMarshalAndUnmarshalFilter(t *testing.T) { }, } policy := &RepPolicy{ + Trigger: trigger, Filters: filters, } - err := policy.MarshalFilter() + err := policy.Marshal() require.Nil(t, err) + policy.Trigger = nil policy.Filters = nil - err = policy.UnmarshalFilter() + err = policy.Unmarshal() require.Nil(t, err) assert.EqualValues(t, filters, policy.Filters) + assert.EqualValues(t, trigger, policy.Trigger) } diff --git a/src/replication/consts.go b/src/replication/consts.go index 76926f224..7b4be836c 100644 --- a/src/replication/consts.go +++ b/src/replication/consts.go @@ -7,4 +7,10 @@ const ( FilterItemKindRepository = "repository" //FilterItemKindTag : Kind of filter item is 'tag' FilterItemKindTag = "tag" + //TriggerKindManually : kind of trigger is 'manully' + TriggerKindManually = "manually" + //TriggerKindSchedule : kind of trigger is 'schedule' + TriggerKindSchedule = "schedule" + //TriggerKindImmediately : kind of trigger is 'immediately' + TriggerKindImmediately = "immediately" ) diff --git a/src/ui/api/replication_policy_test.go b/src/ui/api/replication_policy_test.go index 622cbecb3..f3db26711 100644 --- a/src/ui/api/replication_policy_test.go +++ b/src/ui/api/replication_policy_test.go @@ -41,12 +41,19 @@ func TestPoliciesPost(t *testing.T) { //add target CommonAddTarget() targetID := int64(CommonGetTarget()) - repPolicy := &apilib.RepPolicyPost{int64(1), targetID, addPolicyName, []*models.RepFilter{ - &models.RepFilter{ - Type: replication.FilterItemKindRepository, - Value: "library/ubuntu*", + repPolicy := &apilib.RepPolicyPost{int64(1), targetID, addPolicyName, + &models.RepTrigger{ + Type: replication.TriggerKindSchedule, + Params: map[string]interface{}{ + "date": "2:00", + }, }, - }} + []*models.RepFilter{ + &models.RepFilter{ + Type: replication.FilterItemKindRepository, + Value: "library/ubuntu*", + }, + }} fmt.Println("Testing Policies Post API") @@ -129,12 +136,16 @@ func TestPoliciesPost(t *testing.T) { } fmt.Println("case 8 : response code = 400: invalid filter") - repPolicy = &apilib.RepPolicyPost{int64(1), targetID, addPolicyName, []*models.RepFilter{ - &models.RepFilter{ - Type: "replication", - Value: "", + repPolicy = &apilib.RepPolicyPost{int64(1), targetID, addPolicyName, + &models.RepTrigger{ + Type: replication.TriggerKindManually, }, - }} + []*models.RepFilter{ + &models.RepFilter{ + Type: "replication", + Value: "", + }, + }} httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy) require.Nil(t, err) assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400") diff --git a/tests/apitests/apilib/rep_policy_post.go b/tests/apitests/apilib/rep_policy_post.go index a78b01aa6..365070539 100644 --- a/tests/apitests/apilib/rep_policy_post.go +++ b/tests/apitests/apilib/rep_policy_post.go @@ -37,6 +37,9 @@ type RepPolicyPost struct { // The policy name. Name string `json:"name,omitempty"` + // Trigger + Trigger *models.RepTrigger `json:"trigger"` + // Filters Filters []*models.RepFilter `json:"filters"` } From 149b6282920532727424c3bce088f603c429b3d9 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 9 Nov 2017 13:43:46 +0800 Subject: [PATCH 09/37] update --- src/common/dao/replication_job.go | 27 +++++++++++++++++++-------- src/common/models/replication_job.go | 2 +- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/common/dao/replication_job.go b/src/common/dao/replication_job.go index 8771b179c..6fd4e17c6 100644 --- a/src/common/dao/replication_job.go +++ b/src/common/dao/replication_job.go @@ -107,15 +107,26 @@ func AddRepPolicy(policy models.RepPolicy) (int64, error) { if err := policy.Marshal(); err != nil { return 0, err } - now := time.Now() - policy.CreationTime = now - policy.UpdateTime = now - if policy.Enabled == 1 { - policy.StartTime = now - } - policy.Deleted = 0 - return GetOrmer().Insert(&policy) + o := GetOrmer() + sql := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, start_time, creation_time, update_time, filters, replicate_deletion) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + params := []interface{}{} + params = append(params, policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.TriggerInDB) + now := time.Now() + if policy.Enabled == 1 { + params = append(params, now) + } else { + params = append(params, nil) + } + params = append(params, now, now, policy.FiltersInDB, policy.ReplicateDeletion) + + result, err := o.Raw(sql, params...).Exec() + if err != nil { + return 0, err + } + + return result.LastInsertId() } // GetRepPolicy ... diff --git a/src/common/models/replication_job.go b/src/common/models/replication_job.go index 55d726c99..09cfa2169 100644 --- a/src/common/models/replication_job.go +++ b/src/common/models/replication_job.go @@ -45,7 +45,7 @@ type RepPolicy struct { ProjectID int64 `orm:"column(project_id)" json:"project_id"` ProjectName string `orm:"-" json:"project_name,omitempty"` TargetID int64 `orm:"column(target_id)" json:"target_id"` - TargetName string `orm:"-" json:"target_name,omitempty"` + TargetName string `json:"target_name,omitempty"` Name string `orm:"column(name)" json:"name"` Enabled int `orm:"column(enabled)" json:"enabled"` Description string `orm:"column(description)" json:"description"` From c2e0c8d1f2a20be526c1782b5ea58c1268c91a6d Mon Sep 17 00:00:00 2001 From: Steven Zou Date: Thu, 9 Nov 2017 22:27:54 +0800 Subject: [PATCH 10/37] Define the related interfaces for triggers and core controllers of replication service --- src/common/notifier/init.go | 13 ++ src/common/notifier/notifier_test.go | 20 +- src/replication/consts.go | 15 +- src/replication/core/controller.go | 121 ++++++++++ src/replication/event/on_deletion_handler.go | 57 +++++ src/replication/event/on_push_handler.go | 56 +++++ .../event/start_replication_handler.go | 45 ++++ src/replication/event/topics.go | 12 + src/replication/models/policy.go | 28 +++ src/replication/models/trigger.go | 10 + src/replication/policy/manager.go | 42 ++++ src/replication/source/sourcer.go | 8 - src/replication/trigger/cache.go | 212 ++++++++++++++++++ src/replication/trigger/cache_test.go | 53 +++++ src/replication/trigger/immediate.go | 47 ++++ src/replication/trigger/interface.go | 13 ++ src/replication/trigger/manager.go | 136 +++++++++++ src/replication/trigger/param_immediate.go | 26 +++ src/replication/trigger/param_schedule.go | 30 +++ src/replication/trigger/schedule.go | 34 +++ src/replication/trigger/trigger_param.go | 17 ++ src/replication/trigger/watch_list.go | 43 ++++ 22 files changed, 1020 insertions(+), 18 deletions(-) create mode 100644 src/common/notifier/init.go create mode 100644 src/replication/core/controller.go create mode 100644 src/replication/event/on_deletion_handler.go create mode 100644 src/replication/event/on_push_handler.go create mode 100644 src/replication/event/start_replication_handler.go create mode 100644 src/replication/event/topics.go create mode 100644 src/replication/models/policy.go create mode 100644 src/replication/models/trigger.go create mode 100644 src/replication/policy/manager.go create mode 100644 src/replication/trigger/cache.go create mode 100644 src/replication/trigger/cache_test.go create mode 100644 src/replication/trigger/immediate.go create mode 100644 src/replication/trigger/interface.go create mode 100644 src/replication/trigger/manager.go create mode 100644 src/replication/trigger/param_immediate.go create mode 100644 src/replication/trigger/param_schedule.go create mode 100644 src/replication/trigger/schedule.go create mode 100644 src/replication/trigger/trigger_param.go create mode 100644 src/replication/trigger/watch_list.go diff --git a/src/common/notifier/init.go b/src/common/notifier/init.go new file mode 100644 index 000000000..b36d1f8de --- /dev/null +++ b/src/common/notifier/init.go @@ -0,0 +1,13 @@ +package notifier + +import ( + "github.com/vmware/harbor/src/replication/event" +) + +//Subscribe related topics +func init() { + //Listen the related event topics + Subscribe(event.StartReplicationTopic, &event.StartReplicationHandler{}) + Subscribe(event.ReplicationEventTopicOnPush, &event.OnPushHandler{}) + Subscribe(event.ReplicationEventTopicOnDeletion, &event.OnDeletionHandler{}) +} diff --git a/src/common/notifier/notifier_test.go b/src/common/notifier/notifier_test.go index 9f98679ce..8bb93c59d 100644 --- a/src/common/notifier/notifier_test.go +++ b/src/common/notifier/notifier_test.go @@ -39,6 +39,7 @@ func (fsh *fakeStatelessHandler) Handle(v interface{}) error { } func TestSubscribeAndUnSubscribe(t *testing.T) { + count := len(notificationWatcher.handlers) err := Subscribe("topic1", &fakeStatefulHandler{0}) if err != nil { t.Fatal(err) @@ -59,7 +60,7 @@ func TestSubscribeAndUnSubscribe(t *testing.T) { t.Fatal(err) } - if len(notificationWatcher.handlers) != 2 { + if len(notificationWatcher.handlers) != (count + 2) { t.Fail() } @@ -94,7 +95,7 @@ func TestSubscribeAndUnSubscribe(t *testing.T) { t.Fatal(err) } - if len(notificationWatcher.handlers) != 1 { + if len(notificationWatcher.handlers) != (count + 1) { t.Fail() } @@ -103,12 +104,13 @@ func TestSubscribeAndUnSubscribe(t *testing.T) { t.Fatal(err) } - if len(notificationWatcher.handlers) != 0 { + if len(notificationWatcher.handlers) != count { t.Fail() } } func TestPublish(t *testing.T) { + count := len(notificationWatcher.handlers) err := Subscribe("topic1", &fakeStatefulHandler{0}) if err != nil { t.Fatal(err) @@ -119,7 +121,7 @@ func TestPublish(t *testing.T) { t.Fatal(err) } - if len(notificationWatcher.handlers) != 2 { + if len(notificationWatcher.handlers) != (count + 2) { t.Fail() } @@ -149,12 +151,13 @@ func TestPublish(t *testing.T) { } func TestConcurrentPublish(t *testing.T) { + count := len(notificationWatcher.handlers) err := Subscribe("topic1", &fakeStatefulHandler{0}) if err != nil { t.Fatal(err) } - if len(notificationWatcher.handlers) != 1 { + if len(notificationWatcher.handlers) != (count + 1) { t.Fail() } @@ -186,11 +189,12 @@ func TestConcurrentPublishWithScanPolicyHandler(t *testing.T) { t.Fatal("Policy scheduler is not started") } + count := len(notificationWatcher.handlers) if err := Subscribe("testing_topic", &ScanPolicyNotificationHandler{}); err != nil { t.Fatal(err.Error()) } - if len(notificationWatcher.handlers) != 1 { - t.Fatal("Handler is not registered") + if len(notificationWatcher.handlers) != (count + 1) { + t.Fatalf("Handler is not registered") } utcTime := time.Now().UTC().Unix() @@ -209,7 +213,7 @@ func TestConcurrentPublishWithScanPolicyHandler(t *testing.T) { t.Fatal(err.Error()) } - if len(notificationWatcher.handlers) != 0 { + if len(notificationWatcher.handlers) != count { t.Fatal("Handler is not unregistered") } diff --git a/src/replication/consts.go b/src/replication/consts.go index c9407cbd9..64800b994 100644 --- a/src/replication/consts.go +++ b/src/replication/consts.go @@ -8,13 +8,24 @@ const ( //FilterItemKindTag : Kind of filter item is 'tag' FilterItemKindTag = "tag" + //TODO: Refactor constants + //TriggerKindManually : kind of trigger is 'manully' TriggerKindManually = "manually" - //TriggerKindSchedule : kind of trigger is 'schedule' - TriggerKindSchedule = "schedule" //TriggerKindImmediately : kind of trigger is 'immediately' TriggerKindImmediately = "immediately" //AdaptorKindHarbor : Kind of adaptor of Harbor AdaptorKindHarbor = "Harbor" + + //TriggerKindImmediate : Kind of trigger is 'Immediate' + TriggerKindImmediate = "Immediate" + //TriggerKindSchedule : Kind of trigger is 'Schedule' + TriggerKindSchedule = "Schedule" + //TriggerKindManual : Kind of trigger is 'Manual' + TriggerKindManual = "Manual" + //TriggerScheduleDaily : type of scheduling is 'daily' + TriggerScheduleDaily = "daily" + //TriggerScheduleWeekly : type of scheduling is 'weekly' + TriggerScheduleWeekly = "weekly" ) diff --git a/src/replication/core/controller.go b/src/replication/core/controller.go new file mode 100644 index 000000000..24430e7ef --- /dev/null +++ b/src/replication/core/controller.go @@ -0,0 +1,121 @@ +package core + +import ( + "fmt" + + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" + "github.com/vmware/harbor/src/replication/policy" + "github.com/vmware/harbor/src/replication/source" + "github.com/vmware/harbor/src/replication/trigger" +) + +//Controller is core module to cordinate and control the overall workflow of the +//replication modules. +type Controller struct { + //Indicate whether the controller has been initialized or not + initialized bool + + //Manage the policies + policyManager *policy.Manager + + //Handle the things related with source + sourcer *source.Sourcer + + //Manage the triggers of policies + triggerManager *trigger.Manager +} + +//Keep controller as singleton instance +var ( + DefaultController = NewController(ControllerConfig{}) //Use default data +) + +//ControllerConfig includes related configurations required by the controller +type ControllerConfig struct { + //The capacity of the cache storing enabled triggers + CacheCapacity int +} + +//NewController is the constructor of Controller. +func NewController(config ControllerConfig) *Controller { + //Controller refer the default instances + return &Controller{ + policyManager: policy.NewManager(), + sourcer: source.NewSourcer(), + triggerManager: trigger.NewManager(config.CacheCapacity), + } +} + +//Init will initialize the controller and the sub components +func (ctl *Controller) Init() error { + if ctl.initialized { + return nil + } + + //Build query parameters + triggerNames := []string{ + replication.TriggerKindImmediate, + replication.TriggerKindSchedule, + } + queryName := "" + for _, name := range triggerNames { + queryName = fmt.Sprintf("%s,%s", queryName, name) + } + //Enable the triggers + query := models.QueryParameter{ + TriggerName: queryName, + } + + policies := ctl.policyManager.GetPolicies(query) + if policies != nil && len(policies) > 0 { + for _, policy := range policies { + if err := ctl.triggerManager.SetupTrigger(policy.ID, policy.Trigger); err != nil { + //TODO: Log error + fmt.Printf("Error: %s", err) + //TODO:Update the status of policy + } + } + } + + //Initialize sourcer + ctl.sourcer.Init() + + ctl.initialized = true + + return nil +} + +//CreatePolicy is used to create a new policy and enable it if necessary +func (ctl *Controller) CreatePolicy(newPolicy models.ReplicationPolicy) error { + //Validate policy + //TODO: + return nil +} + +//UpdatePolicy will update the policy with new content. +//Parameter updatedPolicy must have the ID of the updated policy. +func (ctl *Controller) UpdatePolicy(updatedPolicy models.ReplicationPolicy) error { + return nil +} + +//RemovePolicy will remove the specified policy and clean the related settings +func (ctl *Controller) RemovePolicy(policyID int) error { + return nil +} + +//GetPolicy is delegation of GetPolicy of Policy.Manager +func (ctl *Controller) GetPolicy(policyID int) models.ReplicationPolicy { + return models.ReplicationPolicy{} +} + +//GetPolicies is delegation of GetPolicies of Policy.Manager +func (ctl *Controller) GetPolicies(query models.QueryParameter) []models.ReplicationPolicy { + return nil +} + +//Replicate starts one replication defined in the specified policy; +//Can be launched by the API layer and related triggers. +func (ctl *Controller) Replicate(policyID int) error { + return nil +} diff --git a/src/replication/event/on_deletion_handler.go b/src/replication/event/on_deletion_handler.go new file mode 100644 index 000000000..c3b76e299 --- /dev/null +++ b/src/replication/event/on_deletion_handler.go @@ -0,0 +1,57 @@ +package event + +import ( + "errors" + "fmt" + "reflect" + + "github.com/vmware/harbor/src/replication/core" + "github.com/vmware/harbor/src/replication/models" +) + +//OnDeletionHandler implements the notification handler interface to handle image on push event. +type OnDeletionHandler struct{} + +//OnDeletionNotification contains the data required by this handler +type OnDeletionNotification struct { + //The name of the project where the being pushed images are located + ProjectName string +} + +//Handle implements the same method of notification handler interface +func (oph *OnDeletionHandler) Handle(value interface{}) error { + if value == nil { + return errors.New("OnDeletionHandler can not handle nil value") + } + + vType := reflect.TypeOf(value) + if vType.Kind() != reflect.Struct || vType.String() != "event.OnDeletionNotification" { + return fmt.Errorf("Mismatch value type of OnDeletionHandler, expect %s but got %s", "event.OnDeletionNotification", vType.String()) + } + + notification := value.(OnDeletionNotification) + //TODO:Call projectManager to get the projectID + fmt.Println(notification.ProjectName) + query := models.QueryParameter{ + ProjectID: 0, + } + + policies := core.DefaultController.GetPolicies(query) + if policies != nil && len(policies) > 0 { + for _, p := range policies { + //Error accumulated and then return? + if err := core.DefaultController.Replicate(p.ID); err != nil { + //TODO:Log error + fmt.Println(err.Error()) + } + } + } + + return nil +} + +//IsStateful implements the same method of notification handler interface +func (oph *OnDeletionHandler) IsStateful() bool { + //Statless + return false +} diff --git a/src/replication/event/on_push_handler.go b/src/replication/event/on_push_handler.go new file mode 100644 index 000000000..ba6eae4df --- /dev/null +++ b/src/replication/event/on_push_handler.go @@ -0,0 +1,56 @@ +package event + +import ( + "errors" + "fmt" + "reflect" + + "github.com/vmware/harbor/src/replication/core" + "github.com/vmware/harbor/src/replication/models" +) + +//OnPushHandler implements the notification handler interface to handle image on push event. +type OnPushHandler struct{} + +//OnPushNotification contains the data required by this handler +type OnPushNotification struct { + //The ID of the project where the being pushed images are located + ProjectID int +} + +//Handle implements the same method of notification handler interface +func (oph *OnPushHandler) Handle(value interface{}) error { + if value == nil { + return errors.New("OnPushHandler can not handle nil value") + } + + vType := reflect.TypeOf(value) + if vType.Kind() != reflect.Struct || vType.String() != "event.OnPushNotification" { + return fmt.Errorf("Mismatch value type of OnPushHandler, expect %s but got %s", "event.OnPushNotification", vType.String()) + } + + notification := value.(OnDeletionNotification) + //TODO:Call projectManager to get the projectID + fmt.Println(notification.ProjectName) + query := models.QueryParameter{ + ProjectID: 0, + } + + policies := core.DefaultController.GetPolicies(query) + if policies != nil && len(policies) > 0 { + for _, p := range policies { + if err := core.DefaultController.Replicate(p.ID); err != nil { + //TODO:Log error + fmt.Println(err.Error()) + } + } + } + + return nil +} + +//IsStateful implements the same method of notification handler interface +func (oph *OnPushHandler) IsStateful() bool { + //Statless + return false +} diff --git a/src/replication/event/start_replication_handler.go b/src/replication/event/start_replication_handler.go new file mode 100644 index 000000000..aee83d576 --- /dev/null +++ b/src/replication/event/start_replication_handler.go @@ -0,0 +1,45 @@ +package event + +import ( + "errors" + "fmt" + "reflect" + + "github.com/vmware/harbor/src/replication/core" +) + +//StartReplicationHandler implements the notification handler interface to handle start replication requests. +type StartReplicationHandler struct{} + +//StartReplicationNotification contains data required by this handler +type StartReplicationNotification struct { + //ID of the policy + PolicyID int +} + +//Handle implements the same method of notification handler interface +func (srh *StartReplicationHandler) Handle(value interface{}) error { + if value == nil { + return errors.New("StartReplicationHandler can not handle nil value") + } + + vType := reflect.TypeOf(value) + if vType.Kind() != reflect.Struct || vType.String() != "core.StartReplicationNotification" { + return fmt.Errorf("Mismatch value type of StartReplicationHandler, expect %s but got %s", "core.StartReplicationNotification", vType.String()) + } + + notification := value.(StartReplicationNotification) + if notification.PolicyID <= 0 { + return errors.New("Invalid policy") + } + + //Start replication + //TODO: + return core.DefaultController.Replicate(notification.PolicyID) +} + +//IsStateful implements the same method of notification handler interface +func (srh *StartReplicationHandler) IsStateful() bool { + //Stateless + return false +} diff --git a/src/replication/event/topics.go b/src/replication/event/topics.go new file mode 100644 index 000000000..5a6aa954e --- /dev/null +++ b/src/replication/event/topics.go @@ -0,0 +1,12 @@ +package event + +const ( + //ReplicationEventTopicOnPush : OnPush event + ReplicationEventTopicOnPush = "OnPush" + + //ReplicationEventTopicOnDeletion : OnDeletion event + ReplicationEventTopicOnDeletion = "OnDeletion" + + //StartReplicationTopic : Start application request + StartReplicationTopic = "StartReplication" +) diff --git a/src/replication/models/policy.go b/src/replication/models/policy.go new file mode 100644 index 000000000..47f1867fb --- /dev/null +++ b/src/replication/models/policy.go @@ -0,0 +1,28 @@ +package models + +//ReplicationPolicy defines the structure of a replication policy. +type ReplicationPolicy struct { + //UUID of the policy + ID int + + //Projects attached to this policy + RelevantProjects []int + + //The trigger of the replication + Trigger Trigger +} + +//QueryParameter defines the parameters used to do query selection. +type QueryParameter struct { + //Query by page, couple with pageSize + Page int + + //Size of each page, couple with page + PageSize int + + //Query by the name of trigger + TriggerName string + + //Query by project ID + ProjectID int +} diff --git a/src/replication/models/trigger.go b/src/replication/models/trigger.go new file mode 100644 index 000000000..f8a06b4e5 --- /dev/null +++ b/src/replication/models/trigger.go @@ -0,0 +1,10 @@ +package models + +//Trigger is replication launching approach definition +type Trigger struct { + //The name of the trigger + Name string + + //The parameters with json text format required by the trigger + Param string +} diff --git a/src/replication/policy/manager.go b/src/replication/policy/manager.go new file mode 100644 index 000000000..22890ec73 --- /dev/null +++ b/src/replication/policy/manager.go @@ -0,0 +1,42 @@ +package policy + +import ( + "github.com/vmware/harbor/src/replication/models" +) + +//Manager provides replication policy CURD capabilities. +type Manager struct{} + +//NewManager is the constructor of Manager. +func NewManager() *Manager { + return &Manager{} +} + +//GetPolicies returns all the policies +func (m *Manager) GetPolicies(query models.QueryParameter) []models.ReplicationPolicy { + return []models.ReplicationPolicy{} +} + +//GetPolicy returns the policy with the specified ID +func (m *Manager) GetPolicy(policyID int) models.ReplicationPolicy { + return models.ReplicationPolicy{} +} + +//CreatePolicy creates a new policy with the provided data; +//If creating failed, error will be returned; +//If creating succeed, ID of the new created policy will be returned. +func (m *Manager) CreatePolicy(policy models.ReplicationPolicy) (int, error) { + return 0, nil +} + +//UpdatePolicy updates the policy; +//If updating failed, error will be returned. +func (m *Manager) UpdatePolicy(policy models.ReplicationPolicy) error { + return nil +} + +//RemovePolicy removes the specified policy; +//If removing failed, error will be returned. +func (m *Manager) RemovePolicy(policyID int) error { + return nil +} diff --git a/src/replication/source/sourcer.go b/src/replication/source/sourcer.go index 4bbfc2c44..9322bb50e 100644 --- a/src/replication/source/sourcer.go +++ b/src/replication/source/sourcer.go @@ -12,9 +12,6 @@ type Sourcer struct { adaptors map[string]registry.Adaptor } -//ReplicationSourcer is default sourcer for replication service. -var ReplicationSourcer = NewSourcer() - //NewSourcer is the constructor of Sourcer func NewSourcer() *Sourcer { return &Sourcer{ @@ -37,8 +34,3 @@ func (sc *Sourcer) GetAdaptor(kind string) registry.Adaptor { return sc.adaptors[kind] } - -//Init the adaptors -func Init() { - ReplicationSourcer.Init() -} diff --git a/src/replication/trigger/cache.go b/src/replication/trigger/cache.go new file mode 100644 index 000000000..0f74bc79d --- /dev/null +++ b/src/replication/trigger/cache.go @@ -0,0 +1,212 @@ +package trigger + +import ( + "container/heap" + "fmt" + "sync" + "time" +) + +const ( + //The max count of items the cache can keep + defaultCapacity = 1000 +) + +//Item keeps more metadata of the triggers which are stored in the heap. +type Item struct { + //Which policy the trigger belong to + policyID int + + //Frequency of cache querying + //First compration factor + frequency int + + //The timestamp of being put into heap + //Second compration factor + timestamp int64 + + //The index in the heap + index int +} + +//MetaQueue implements heap.Interface and holds items which are metadata of trigger +type MetaQueue []*Item + +//Len return the size of the queue +func (mq MetaQueue) Len() int { + return len(mq) +} + +//Less is a comparator of heap +func (mq MetaQueue) Less(i, j int) bool { + return mq[i].frequency < mq[j].frequency || + (mq[i].frequency == mq[j].frequency && + mq[i].timestamp < mq[j].timestamp) +} + +//Swap the items to rebuild heap +func (mq MetaQueue) Swap(i, j int) { + mq[i], mq[j] = mq[j], mq[i] + mq[i].index = i + mq[j].index = j +} + +//Push item into heap +func (mq *MetaQueue) Push(x interface{}) { + item := x.(*Item) + n := len(*mq) + item.index = n + item.timestamp = time.Now().UTC().UnixNano() + *mq = append(*mq, item) +} + +//Pop smallest item from heap +func (mq *MetaQueue) Pop() interface{} { + old := *mq + n := len(old) + item := old[n-1] //Smallest item + item.index = -1 //For safety + *mq = old[:n-1] + return item +} + +//Update the frequency of item +func (mq *MetaQueue) Update(item *Item) { + item.frequency++ + heap.Fix(mq, item.index) +} + +//CacheItem is the data stored in the cache. +//It contains trigger and heap item references. +type CacheItem struct { + //The trigger reference + trigger Interface + + //The heap item reference + item *Item +} + +//Cache is used to cache the enabled triggers with specified capacity. +//If exceed the capacity, cached items will be adjusted with the following rules: +// The item with least usage frequency will be replaced; +// If multiple items with same usage frequency, the oldest one will be replaced. +type Cache struct { + //The max count of items this cache can keep + capacity int + + //Lock to handle concurrent case + lock *sync.RWMutex + + //Hash map for quick locating cached item + hash map[string]CacheItem + + //Heap for quick locating the trigger with least usage + queue *MetaQueue +} + +//NewCache is constructor of cache +func NewCache(capacity int) *Cache { + cap := capacity + if cap <= 0 { + cap = defaultCapacity + } + + //Initialize heap + mq := make(MetaQueue, 0) + heap.Init(&mq) + + return &Cache{ + capacity: cap, + lock: new(sync.RWMutex), + hash: make(map[string]CacheItem), + queue: &mq, + } +} + +//Get the trigger interface with the specified policy ID +func (c *Cache) Get(policyID int) Interface { + if policyID <= 0 { + return nil + } + + c.lock.RLock() + defer c.lock.RUnlock() + + k := c.key(policyID) + + if cacheItem, ok := c.hash[k]; ok { + //Update frequency + c.queue.Update(cacheItem.item) + return cacheItem.trigger + } + + return nil +} + +//Put the item into cache with ID of ploicy as key +func (c *Cache) Put(policyID int, trigger Interface) { + if policyID <= 0 || trigger == nil { + return + } + + c.lock.Lock() + defer c.lock.Unlock() + + //Exceed the capacity? + if c.Size() >= c.capacity { + //Pop one for the new one + v := heap.Pop(c.queue) + item := v.(*Item) + //Remove from hash + delete(c.hash, c.key(item.policyID)) + } + + //Add to meta queue + item := &Item{ + policyID: policyID, + frequency: 1, + } + heap.Push(c.queue, item) + + //Cache + cacheItem := CacheItem{ + trigger: trigger, + item: item, + } + + k := c.key(policyID) + c.hash[k] = cacheItem +} + +//Remove the trigger attached to the specified policy +func (c *Cache) Remove(policyID int) Interface { + if policyID > 0 { + c.lock.Lock() + defer c.lock.Unlock() + + //If existing + k := c.key(policyID) + if cacheItem, ok := c.hash[k]; ok { + //Remove from heap + heap.Remove(c.queue, cacheItem.item.index) + + //Remove from hash + delete(c.hash, k) + + return cacheItem.trigger + } + + } + + return nil +} + +//Size return the count of triggers in the cache +func (c *Cache) Size() int { + return len(c.hash) +} + +//Generate a hash key with the policy ID +func (c *Cache) key(policyID int) string { + return fmt.Sprintf("trigger-%d", policyID) +} diff --git a/src/replication/trigger/cache_test.go b/src/replication/trigger/cache_test.go new file mode 100644 index 000000000..b7348cf26 --- /dev/null +++ b/src/replication/trigger/cache_test.go @@ -0,0 +1,53 @@ +package trigger + +import "testing" +import "time" + +func TestCache(t *testing.T) { + cache := NewCache(10) + trigger := NewImmediateTrigger(ImmediateParam{}) + + cache.Put(1, trigger) + if cache.Size() != 1 { + t.Fatalf("Invalid size, expect 1 but got %d", cache.Size()) + } + + tr := cache.Get(1) + if tr == nil { + t.Fatal("Should not get nil item") + } + + tri := cache.Remove(1) + if tri == nil || cache.Size() > 0 { + t.Fatal("Failed to remove") + } +} + +func TestCacheChange(t *testing.T) { + cache := NewCache(2) + trigger1 := NewImmediateTrigger(ImmediateParam{}) + trigger2 := NewImmediateTrigger(ImmediateParam{}) + cache.Put(1, trigger1) + cache.Put(2, trigger2) + + if cache.Size() != 2 { + t.Fatalf("Invalid size, expect 2 but got %d", cache.Size()) + } + + if tr := cache.Get(2); tr == nil { + t.Fatal("Should not get nil item") + } + + time.Sleep(100 * time.Microsecond) + + trigger3 := NewImmediateTrigger(ImmediateParam{}) + cache.Put(3, trigger3) + if cache.Size() != 2 { + t.Fatalf("Invalid size, expect 2 but got %d", cache.Size()) + } + + if tr := cache.Get(1); tr != nil { + t.Fatal("item1 should not exist") + } + +} diff --git a/src/replication/trigger/immediate.go b/src/replication/trigger/immediate.go new file mode 100644 index 000000000..af4ba3b2b --- /dev/null +++ b/src/replication/trigger/immediate.go @@ -0,0 +1,47 @@ +package trigger + +import ( + "errors" + + "github.com/vmware/harbor/src/replication" +) + +//ImmediateTrigger will setup watcher at the image pushing action to fire +//replication event at pushing happening time. +type ImmediateTrigger struct { + params ImmediateParam +} + +//NewImmediateTrigger is constructor of ImmediateTrigger +func NewImmediateTrigger(params ImmediateParam) *ImmediateTrigger { + return &ImmediateTrigger{ + params: params, + } +} + +//Kind is the implementation of same method defined in Trigger interface +func (st *ImmediateTrigger) Kind() string { + return replication.TriggerKindImmediate +} + +//Setup is the implementation of same method defined in Trigger interface +func (st *ImmediateTrigger) Setup() error { + if st.params.PolicyID <= 0 || len(st.params.Namespace) == 0 { + return errors.New("Invalid parameters for Immediate trigger") + } + + //TODO: Need more complicated logic here to handle partial updates + wt := WatchItem{ + PolicyID: st.params.PolicyID, + Namespace: st.params.Namespace, + OnDeletion: st.params.OnDeletion, + OnPush: true, + } + + return DefaultWatchList.Add(wt) +} + +//Unset is the implementation of same method defined in Trigger interface +func (st *ImmediateTrigger) Unset() error { + return errors.New("Not implemented") +} diff --git a/src/replication/trigger/interface.go b/src/replication/trigger/interface.go new file mode 100644 index 000000000..d08e75137 --- /dev/null +++ b/src/replication/trigger/interface.go @@ -0,0 +1,13 @@ +package trigger + +//Interface is certian mechanism to know when fire the replication operation. +type Interface interface { + //Kind indicates what type of the trigger is. + Kind() string + + //Setup/enable the trigger; if failed, an error would be returned. + Setup() error + + //Remove/disable the trigger; if failed, an error would be returned. + Unset() error +} diff --git a/src/replication/trigger/manager.go b/src/replication/trigger/manager.go new file mode 100644 index 000000000..daed480e0 --- /dev/null +++ b/src/replication/trigger/manager.go @@ -0,0 +1,136 @@ +package trigger + +import ( + "errors" + + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" +) + +//Manager provides unified methods to manage the triggers of policies; +//Cache the enabled triggers, setup/unset the trigger based on the parameters +//with json format. +type Manager struct { + //Cache for triggers + cache *Cache +} + +//NewManager is the constructor of trigger manager. +//capacity is the max number of trigger references manager can keep in memory +func NewManager(capacity int) *Manager { + return &Manager{ + cache: NewCache(capacity), + } +} + +//GetTrigger returns the enabled trigger reference if existing in the cache. +func (m *Manager) GetTrigger(policyID int) Interface { + return m.cache.Get(policyID) +} + +//RemoveTrigger will disable the trigger and remove it from the cache if existing. +func (m *Manager) RemoveTrigger(policyID int) error { + trigger := m.cache.Get(policyID) + if trigger == nil { + return errors.New("Trigger is not cached, please use UnsetTrigger to disable the trigger") + } + + //Unset trigger + if err := trigger.Unset(); err != nil { + return err + } + + //Remove from cache + //No need to check the return of remove because the dirty item cached in the cache + //will be removed out finally after a certain while + m.cache.Remove(policyID) + + return nil +} + +//SetupTrigger will create the new trigger based on the provided json parameters. +//If failed, an error will be returned. +func (m *Manager) SetupTrigger(policyID int, trigger models.Trigger) error { + if policyID <= 0 { + return errors.New("Invalid policy ID") + } + + if len(trigger.Name) == 0 { + return errors.New("Invalid replication trigger definition") + } + + switch trigger.Name { + case replication.TriggerKindSchedule: + param := ScheduleParam{} + if err := param.Parse(trigger.Param); err != nil { + return err + } + //Append policy ID info + param.PolicyID = policyID + + newTrigger := NewScheduleTrigger(param) + if err := newTrigger.Setup(); err != nil { + return err + } + case replication.TriggerKindImmediate: + param := ImmediateParam{} + if err := param.Parse(trigger.Param); err != nil { + return err + } + //Append policy ID info + param.PolicyID = policyID + + newTrigger := NewImmediateTrigger(param) + if err := newTrigger.Setup(); err != nil { + return err + } + default: + //Treat as manual trigger + break + } + + return nil +} + +//UnsetTrigger will disable the trigger which is not cached in the trigger cache. +func (m *Manager) UnsetTrigger(policyID int, trigger models.Trigger) error { + if policyID <= 0 { + return errors.New("Invalid policy ID") + } + + if len(trigger.Name) == 0 { + return errors.New("Invalid replication trigger definition") + } + + switch trigger.Name { + case replication.TriggerKindSchedule: + param := ScheduleParam{} + if err := param.Parse(trigger.Param); err != nil { + return err + } + //Append policy ID info + param.PolicyID = policyID + + newTrigger := NewScheduleTrigger(param) + if err := newTrigger.Unset(); err != nil { + return err + } + case replication.TriggerKindImmediate: + param := ImmediateParam{} + if err := param.Parse(trigger.Param); err != nil { + return err + } + //Append policy ID info + param.PolicyID = policyID + + newTrigger := NewImmediateTrigger(param) + if err := newTrigger.Unset(); err != nil { + return err + } + default: + //Treat as manual trigger + break + } + + return nil +} diff --git a/src/replication/trigger/param_immediate.go b/src/replication/trigger/param_immediate.go new file mode 100644 index 000000000..492c65872 --- /dev/null +++ b/src/replication/trigger/param_immediate.go @@ -0,0 +1,26 @@ +package trigger + +import ( + "errors" +) + +//NOTES: Whether replicate the existing images when the type of trigger is +//'Immediate' is a once-effective setting which will not be persisted +// and kept as one parameter of 'Immediate' trigger. It will only be +//covered by the UI logic. + +//ImmediateParam defines the parameter of immediate trigger +type ImmediateParam struct { + //Basic parameters + BasicParam + + //Namepace + Namespace string +} + +//Parse is the implementation of same method in TriggerParam interface +//NOTES: No need to implement this method for 'Immediate' trigger as +//it does not have any parameters with json format. +func (ip ImmediateParam) Parse(param string) error { + return errors.New("Should NOT be called as it's not implemented") +} diff --git a/src/replication/trigger/param_schedule.go b/src/replication/trigger/param_schedule.go new file mode 100644 index 000000000..84ca46f44 --- /dev/null +++ b/src/replication/trigger/param_schedule.go @@ -0,0 +1,30 @@ +package trigger + +import ( + "encoding/json" + "errors" +) + +//ScheduleParam defines the parameter of schedule trigger +type ScheduleParam struct { + //Basic parameters + BasicParam + + //Daily or weekly + Type string + + //Optional, only used when type is 'weekly' + Weekday int8 + + //The time offset with the UTC 00:00 in seconds + Offtime int64 +} + +//Parse is the implementation of same method in TriggerParam interface +func (stp ScheduleParam) Parse(param string) error { + if len(param) == 0 { + return errors.New("Parameter of schedule trigger should not be empty") + } + + return json.Unmarshal([]byte(param), &stp) +} diff --git a/src/replication/trigger/schedule.go b/src/replication/trigger/schedule.go new file mode 100644 index 000000000..d7c07cadd --- /dev/null +++ b/src/replication/trigger/schedule.go @@ -0,0 +1,34 @@ +package trigger + +import ( + "errors" + + "github.com/vmware/harbor/src/replication" +) + +//ScheduleTrigger will schedule a alternate policy to provide 'daily' and 'weekly' trigger ways. +type ScheduleTrigger struct { + params ScheduleParam +} + +//NewScheduleTrigger is constructor of ScheduleTrigger +func NewScheduleTrigger(params ScheduleParam) *ScheduleTrigger { + return &ScheduleTrigger{ + params: params, + } +} + +//Kind is the implementation of same method defined in Trigger interface +func (st *ScheduleTrigger) Kind() string { + return replication.TriggerKindSchedule +} + +//Setup is the implementation of same method defined in Trigger interface +func (st *ScheduleTrigger) Setup() error { + return errors.New("Not implemented") +} + +//Unset is the implementation of same method defined in Trigger interface +func (st *ScheduleTrigger) Unset() error { + return errors.New("Not implemented") +} diff --git a/src/replication/trigger/trigger_param.go b/src/replication/trigger/trigger_param.go new file mode 100644 index 000000000..10420ef5b --- /dev/null +++ b/src/replication/trigger/trigger_param.go @@ -0,0 +1,17 @@ +package trigger + +//BasicParam contains the general parameters for all triggers +type BasicParam struct { + //ID of the related policy + PolicyID int + + //Whether delete remote replicated images if local ones are deleted + OnDeletion bool +} + +//Parameter defines operation of doing initialization from parameter json text +type Parameter interface { + //Decode parameter with json style to the owner struct + //If failed, an error will be returned + Parse(param string) error +} diff --git a/src/replication/trigger/watch_list.go b/src/replication/trigger/watch_list.go new file mode 100644 index 000000000..6a2009f29 --- /dev/null +++ b/src/replication/trigger/watch_list.go @@ -0,0 +1,43 @@ +package trigger + +//DefaultWatchList is the default instance of WatchList +var DefaultWatchList = &WatchList{} + +//WatchList contains the items which should be evaluated for replication +//when image pushing or deleting happens. +type WatchList struct{} + +//WatchItem keeps the related data for evaluation in WatchList. +type WatchItem struct { + //ID of policy + PolicyID int + + //Corresponding namespace + Namespace string + + //For deletion event + OnDeletion bool + + //For pushing event + OnPush bool +} + +//Add item to the list and persist into DB +func (wl *WatchList) Add(item WatchItem) error { + return nil +} + +//Remove the specified watch item from list +func (wl *WatchList) Remove() WatchItem { + return WatchItem{} +} + +//Update the watch item in the list +func (wl *WatchList) Update(updatedItem WatchItem) error { + return nil +} + +//Get the specified watch item +func (wl *WatchList) Get(namespace string) WatchItem { + return WatchItem{} +} From 31cf6c078ed32fdf4d2abf81d43909b7926229d6 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 15 Nov 2017 14:41:26 +0800 Subject: [PATCH 11/37] Implement replication policy manager --- src/common/dao/replication_job.go | 47 +- src/common/models/replication_job.go | 152 +--- src/common/models/replication_test.go | 50 -- src/replication/core/controller.go | 34 +- src/replication/event/on_deletion_handler.go | 5 +- src/replication/event/on_push_handler.go | 5 +- .../event/start_replication_handler.go | 2 +- src/replication/models/filter_item.go | 26 +- src/replication/models/policy.go | 31 +- src/replication/models/trigger.go | 20 +- src/replication/policy/manager.go | 129 +++- src/replication/trigger/cache.go | 10 +- src/replication/trigger/manager.go | 16 +- src/replication/trigger/trigger_param.go | 2 +- src/replication/trigger/watch_list.go | 2 +- src/ui/api/api_test.go | 195 +++++ src/ui/api/harborapi_test.go | 1 - src/ui/api/models/replication_policy.go | 66 ++ src/ui/api/replication_policy.go | 414 ++++------ src/ui/api/replication_policy_test.go | 722 +++++++++++------- src/ui/router.go | 1 - tests/apitests/apilib/rep_policy_post.go | 10 - 22 files changed, 1111 insertions(+), 829 deletions(-) delete mode 100644 src/common/models/replication_test.go create mode 100644 src/ui/api/api_test.go create mode 100644 src/ui/api/models/replication_policy.go diff --git a/src/common/dao/replication_job.go b/src/common/dao/replication_job.go index 6fd4e17c6..a8f1ce512 100644 --- a/src/common/dao/replication_job.go +++ b/src/common/dao/replication_job.go @@ -104,22 +104,18 @@ func FilterRepTargets(name string) ([]*models.RepTarget, error) { // AddRepPolicy ... func AddRepPolicy(policy models.RepPolicy) (int64, error) { - if err := policy.Marshal(); err != nil { - return 0, err - } - o := GetOrmer() sql := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, start_time, creation_time, update_time, filters, replicate_deletion) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` params := []interface{}{} - params = append(params, policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.TriggerInDB) + params = append(params, policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.Trigger) now := time.Now() if policy.Enabled == 1 { params = append(params, now) } else { params = append(params, nil) } - params = append(params, now, now, policy.FiltersInDB, policy.ReplicateDeletion) + params = append(params, now, now, policy.Filters, policy.ReplicateDeletion) result, err := o.Raw(sql, params...).Exec() if err != nil { @@ -132,7 +128,7 @@ func AddRepPolicy(policy models.RepPolicy) (int64, error) { // GetRepPolicy ... func GetRepPolicy(id int64) (*models.RepPolicy, error) { o := GetOrmer() - sql := `select * from replication_policy where id = ?` + sql := `select * from replication_policy where id = ? and deleted = 0` var policy models.RepPolicy @@ -143,10 +139,6 @@ func GetRepPolicy(id int64) (*models.RepPolicy, error) { return nil, err } - if err := policy.Unmarshal(); err != nil { - return nil, err - } - return &policy, nil } @@ -186,12 +178,6 @@ func FilterRepPolicies(name string, projectID int64) ([]*models.RepPolicy, error return nil, err } - for _, policy := range policies { - if err := policy.Unmarshal(); err != nil { - return nil, err - } - } - return policies, nil } @@ -209,10 +195,6 @@ func GetRepPolicyByName(name string) (*models.RepPolicy, error) { return nil, err } - if err := policy.Unmarshal(); err != nil { - return nil, err - } - return &policy, nil } @@ -227,12 +209,6 @@ func GetRepPolicyByProject(projectID int64) ([]*models.RepPolicy, error) { return nil, err } - for _, policy := range policies { - if err := policy.Unmarshal(); err != nil { - return nil, err - } - } - return policies, nil } @@ -247,12 +223,6 @@ func GetRepPolicyByTarget(targetID int64) ([]*models.RepPolicy, error) { return nil, err } - for _, policy := range policies { - if err := policy.Unmarshal(); err != nil { - return nil, err - } - } - return policies, nil } @@ -267,24 +237,15 @@ func GetRepPolicyByProjectAndTarget(projectID, targetID int64) ([]*models.RepPol return nil, err } - for _, policy := range policies { - if err := policy.Unmarshal(); err != nil { - return nil, err - } - } - return policies, nil } // UpdateRepPolicy ... func UpdateRepPolicy(policy *models.RepPolicy) error { - if err := policy.Marshal(); err != nil { - return err - } o := GetOrmer() policy.UpdateTime = time.Now() _, err := o.Update(policy, "TargetID", "Name", "Enabled", "Description", - "TriggerInDB", "FiltersInDB", "ReplicateDeletion", "UpdateTime") + "Trigger", "Filters", "ReplicateDeletion", "UpdateTime") return err } diff --git a/src/common/models/replication_job.go b/src/common/models/replication_job.go index 09cfa2169..abf011a34 100644 --- a/src/common/models/replication_job.go +++ b/src/common/models/replication_job.go @@ -15,13 +15,10 @@ package models import ( - "encoding/json" - "fmt" "time" "github.com/astaxie/beego/validation" "github.com/vmware/harbor/src/common/utils" - "github.com/vmware/harbor/src/replication" ) const ( @@ -41,142 +38,19 @@ const ( // RepPolicy is the model for a replication policy, which associate to a project and a target (destination) type RepPolicy struct { - ID int64 `orm:"pk;auto;column(id)" json:"id"` - ProjectID int64 `orm:"column(project_id)" json:"project_id"` - ProjectName string `orm:"-" json:"project_name,omitempty"` - TargetID int64 `orm:"column(target_id)" json:"target_id"` - TargetName string `json:"target_name,omitempty"` - Name string `orm:"column(name)" json:"name"` - Enabled int `orm:"column(enabled)" json:"enabled"` - Description string `orm:"column(description)" json:"description"` - Trigger *RepTrigger `orm:"-" json:"trigger"` - TriggerInDB string `orm:"column(cron_str)" json:"-"` - Filters []*RepFilter `orm:"-" json:"filters"` - FiltersInDB string `orm:"column(filters)" json:"-"` - ReplicateExistingImageNow bool `orm:"-" json:"replicate_existing_image_now"` - ReplicateDeletion bool `orm:"column(replicate_deletion)" json:"replicate_deletion"` - StartTime time.Time `orm:"column(start_time)" json:"start_time"` - CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` - UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` - ErrorJobCount int `orm:"-" json:"error_job_count"` - Deleted int `orm:"column(deleted)" json:"deleted"` -} - -// Valid ... -func (r *RepPolicy) Valid(v *validation.Validation) { - if len(r.Name) == 0 { - v.SetError("name", "can not be empty") - } - - if len(r.Name) > 256 { - v.SetError("name", "max length is 256") - } - - if r.ProjectID <= 0 { - v.SetError("project_id", "invalid") - } - - if r.TargetID <= 0 { - v.SetError("target_id", "invalid") - } - - if r.Enabled != 0 && r.Enabled != 1 { - v.SetError("enabled", "must be 0 or 1") - } - - if r.Trigger != nil { - r.Trigger.Valid(v) - } - - for _, filter := range r.Filters { - filter.Valid(v) - } - - if err := r.Marshal(); err != nil { - v.SetError("trigger or filters", err.Error()) - } - - if len(r.TriggerInDB) > 256 { - v.SetError("trigger", "max length is 256") - } - - if len(r.FiltersInDB) > 1024 { - v.SetError("filters", "max length is 1024") - } -} - -// Marshal marshal RepTrigger and RepFilter array to json string -func (r *RepPolicy) Marshal() error { - if r.Trigger != nil { - b, err := json.Marshal(r.Trigger) - if err != nil { - return err - } - r.TriggerInDB = string(b) - } - - if r.Filters != nil { - b, err := json.Marshal(r.Filters) - if err != nil { - return err - } - r.FiltersInDB = string(b) - } - return nil -} - -// Unmarshal unmarshal json string to RepTrigger and RepFilter array -func (r *RepPolicy) Unmarshal() error { - if len(r.TriggerInDB) > 0 { - trigger := &RepTrigger{} - if err := json.Unmarshal([]byte(r.TriggerInDB), &trigger); err != nil { - return err - } - r.Trigger = trigger - } - - if len(r.FiltersInDB) > 0 { - filter := []*RepFilter{} - if err := json.Unmarshal([]byte(r.FiltersInDB), &filter); err != nil { - return err - } - r.Filters = filter - } - return nil -} - -// RepFilter holds information for the replication policy filter -type RepFilter struct { - Type string `json:"type"` - Value string `json:"value"` -} - -// Valid ... -func (r *RepFilter) Valid(v *validation.Validation) { - if !(r.Type == replication.FilterItemKindProject || - r.Type == replication.FilterItemKindRepository || - r.Type == replication.FilterItemKindTag) { - v.SetError("filter.type", fmt.Sprintf("invalid filter type: %s", r.Type)) - } - - if len(r.Value) == 0 { - v.SetError("filter.value", "can not be empty") - } -} - -// RepTrigger holds information for the replication policy trigger -type RepTrigger struct { - Type string `json:"type"` - Params map[string]interface{} `json:"params"` -} - -// Valid ... -func (r *RepTrigger) Valid(v *validation.Validation) { - if !(r.Type == replication.TriggerKindManually || - r.Type == replication.TriggerKindSchedule || - r.Type == replication.TriggerKindImmediately) { - v.SetError("trigger.type", fmt.Sprintf("invalid trigger type: %s", r.Type)) - } + ID int64 `orm:"pk;auto;column(id)"` + ProjectID int64 `orm:"column(project_id)" ` + TargetID int64 `orm:"column(target_id)"` + Name string `orm:"column(name)"` + Enabled int `orm:"column(enabled)"` + Description string `orm:"column(description)"` + Trigger string `orm:"column(cron_str)"` + Filters string `orm:"column(filters)"` + ReplicateDeletion bool `orm:"column(replicate_deletion)"` + StartTime time.Time `orm:"column(start_time)"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add"` + UpdateTime time.Time `orm:"column(update_time);auto_now"` + Deleted int `orm:"column(deleted)"` } // RepJob is the model for a replication job, which is the execution unit on job service, currently it is used to transfer/remove diff --git a/src/common/models/replication_test.go b/src/common/models/replication_test.go deleted file mode 100644 index 52978f983..000000000 --- a/src/common/models/replication_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2017 VMware, Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMarshalAndUnmarshal(t *testing.T) { - trigger := &RepTrigger{ - Type: "schedule", - Params: map[string]interface{}{"date": "2:00"}, - } - filters := []*RepFilter{ - &RepFilter{ - Type: "repository", - Value: "library/ubuntu*", - }, - } - policy := &RepPolicy{ - Trigger: trigger, - Filters: filters, - } - - err := policy.Marshal() - require.Nil(t, err) - - policy.Trigger = nil - policy.Filters = nil - err = policy.Unmarshal() - require.Nil(t, err) - - assert.EqualValues(t, filters, policy.Filters) - assert.EqualValues(t, trigger, policy.Trigger) -} diff --git a/src/replication/core/controller.go b/src/replication/core/controller.go index 24430e7ef..eb18f6ada 100644 --- a/src/replication/core/controller.go +++ b/src/replication/core/controller.go @@ -67,10 +67,13 @@ func (ctl *Controller) Init() error { TriggerName: queryName, } - policies := ctl.policyManager.GetPolicies(query) + policies, err := ctl.policyManager.GetPolicies(query) + if err != nil { + return err + } if policies != nil && len(policies) > 0 { for _, policy := range policies { - if err := ctl.triggerManager.SetupTrigger(policy.ID, policy.Trigger); err != nil { + if err := ctl.triggerManager.SetupTrigger(policy.ID, *policy.Trigger); err != nil { //TODO: Log error fmt.Printf("Error: %s", err) //TODO:Update the status of policy @@ -87,35 +90,38 @@ func (ctl *Controller) Init() error { } //CreatePolicy is used to create a new policy and enable it if necessary -func (ctl *Controller) CreatePolicy(newPolicy models.ReplicationPolicy) error { +func (ctl *Controller) CreatePolicy(newPolicy models.ReplicationPolicy) (int64, error) { //Validate policy - //TODO: - return nil + // TODO + + return ctl.policyManager.CreatePolicy(newPolicy) } //UpdatePolicy will update the policy with new content. //Parameter updatedPolicy must have the ID of the updated policy. func (ctl *Controller) UpdatePolicy(updatedPolicy models.ReplicationPolicy) error { - return nil + // TODO check pre-conditions + return ctl.policyManager.UpdatePolicy(updatedPolicy) } //RemovePolicy will remove the specified policy and clean the related settings -func (ctl *Controller) RemovePolicy(policyID int) error { - return nil +func (ctl *Controller) RemovePolicy(policyID int64) error { + // TODO check pre-conditions + return ctl.policyManager.RemovePolicy(policyID) } //GetPolicy is delegation of GetPolicy of Policy.Manager -func (ctl *Controller) GetPolicy(policyID int) models.ReplicationPolicy { - return models.ReplicationPolicy{} +func (ctl *Controller) GetPolicy(policyID int64) (models.ReplicationPolicy, error) { + return ctl.policyManager.GetPolicy(policyID) } -//GetPolicies is delegation of GetPolicies of Policy.Manager -func (ctl *Controller) GetPolicies(query models.QueryParameter) []models.ReplicationPolicy { - return nil +//GetPolicies is delegation of GetPoliciemodels.ReplicationPolicy{}s of Policy.Manager +func (ctl *Controller) GetPolicies(query models.QueryParameter) ([]models.ReplicationPolicy, error) { + return ctl.policyManager.GetPolicies(query) } //Replicate starts one replication defined in the specified policy; //Can be launched by the API layer and related triggers. -func (ctl *Controller) Replicate(policyID int) error { +func (ctl *Controller) Replicate(policyID int64) error { return nil } diff --git a/src/replication/event/on_deletion_handler.go b/src/replication/event/on_deletion_handler.go index c3b76e299..e16af893c 100644 --- a/src/replication/event/on_deletion_handler.go +++ b/src/replication/event/on_deletion_handler.go @@ -36,7 +36,10 @@ func (oph *OnDeletionHandler) Handle(value interface{}) error { ProjectID: 0, } - policies := core.DefaultController.GetPolicies(query) + policies, err := core.DefaultController.GetPolicies(query) + if err != nil { + return err + } if policies != nil && len(policies) > 0 { for _, p := range policies { //Error accumulated and then return? diff --git a/src/replication/event/on_push_handler.go b/src/replication/event/on_push_handler.go index ba6eae4df..d8c4d3aae 100644 --- a/src/replication/event/on_push_handler.go +++ b/src/replication/event/on_push_handler.go @@ -36,7 +36,10 @@ func (oph *OnPushHandler) Handle(value interface{}) error { ProjectID: 0, } - policies := core.DefaultController.GetPolicies(query) + policies, err := core.DefaultController.GetPolicies(query) + if err != nil { + return err + } if policies != nil && len(policies) > 0 { for _, p := range policies { if err := core.DefaultController.Replicate(p.ID); err != nil { diff --git a/src/replication/event/start_replication_handler.go b/src/replication/event/start_replication_handler.go index aee83d576..93658a593 100644 --- a/src/replication/event/start_replication_handler.go +++ b/src/replication/event/start_replication_handler.go @@ -14,7 +14,7 @@ type StartReplicationHandler struct{} //StartReplicationNotification contains data required by this handler type StartReplicationNotification struct { //ID of the policy - PolicyID int + PolicyID int64 } //Handle implements the same method of notification handler interface diff --git a/src/replication/models/filter_item.go b/src/replication/models/filter_item.go index 038287c78..d142fb70f 100644 --- a/src/replication/models/filter_item.go +++ b/src/replication/models/filter_item.go @@ -1,19 +1,39 @@ package models +import ( + "fmt" + + "github.com/astaxie/beego/validation" + "github.com/vmware/harbor/src/replication" +) + //FilterItem is the general data model represents the filtering resources which are used as input and output for the filters. type FilterItem struct { //The kind of the filtering resources. Support 'project','repository' and 'tag' etc. - Kind string + Kind string `json:"kind"` //The key value of resource which can be used to filter out the resource matched with specified pattern. //E.g: //kind == 'project', value will be project name; //kind == 'repository', value will be repository name //kind == 'tag', value will be tag name. - Value string + Value string `json:"value"` //Extension placeholder. //To append more additional information if required by the filter. - Metadata map[string]interface{} + Metadata map[string]interface{} `json:"metadata"` +} + +// Valid ... +func (f *FilterItem) Valid(v *validation.Validation) { + if !(f.Kind == replication.FilterItemKindProject || + f.Kind == replication.FilterItemKindRepository || + f.Kind == replication.FilterItemKindTag) { + v.SetError("kind", fmt.Sprintf("invalid filter kind: %s", f.Kind)) + } + + if len(f.Value) == 0 { + v.SetError("value", "filter value can not be empty") + } } diff --git a/src/replication/models/policy.go b/src/replication/models/policy.go index 47f1867fb..faa0295b1 100644 --- a/src/replication/models/policy.go +++ b/src/replication/models/policy.go @@ -1,28 +1,37 @@ package models +import ( + "time" +) + //ReplicationPolicy defines the structure of a replication policy. type ReplicationPolicy struct { - //UUID of the policy - ID int - - //Projects attached to this policy - RelevantProjects []int - - //The trigger of the replication - Trigger Trigger + ID int64 //UUID of the policy + Name string + Description string + Filters []FilterItem + ReplicateDeletion bool + Trigger *Trigger //The trigger of the replication + ProjectIDs []int64 //Projects attached to this policy + TargetIDs []int64 + CreationTime time.Time + UpdateTime time.Time } //QueryParameter defines the parameters used to do query selection. type QueryParameter struct { //Query by page, couple with pageSize - Page int + Page int64 //Size of each page, couple with page - PageSize int + PageSize int64 //Query by the name of trigger TriggerName string //Query by project ID - ProjectID int + ProjectID int64 + + //Query by name + Name string } diff --git a/src/replication/models/trigger.go b/src/replication/models/trigger.go index f8a06b4e5..4477f6363 100644 --- a/src/replication/models/trigger.go +++ b/src/replication/models/trigger.go @@ -1,10 +1,26 @@ package models +import ( + "fmt" + + "github.com/astaxie/beego/validation" + "github.com/vmware/harbor/src/replication" +) + //Trigger is replication launching approach definition type Trigger struct { //The name of the trigger - Name string + Kind string `json:"kind"` //The parameters with json text format required by the trigger - Param string + Param string `json:"param"` +} + +// Valid ... +func (t *Trigger) Valid(v *validation.Validation) { + if !(t.Kind == replication.TriggerKindImmediately || + t.Kind == replication.TriggerKindManually || + t.Kind == replication.TriggerKindSchedule) { + v.SetError("kind", fmt.Sprintf("invalid trigger kind: %s", t.Kind)) + } } diff --git a/src/replication/policy/manager.go b/src/replication/policy/manager.go index 22890ec73..dcd8a13df 100644 --- a/src/replication/policy/manager.go +++ b/src/replication/policy/manager.go @@ -1,6 +1,11 @@ package policy import ( + "encoding/json" + "time" + + "github.com/vmware/harbor/src/common/dao" + persist_models "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/replication/models" ) @@ -13,30 +18,136 @@ func NewManager() *Manager { } //GetPolicies returns all the policies -func (m *Manager) GetPolicies(query models.QueryParameter) []models.ReplicationPolicy { - return []models.ReplicationPolicy{} +func (m *Manager) GetPolicies(query models.QueryParameter) ([]models.ReplicationPolicy, error) { + result := []models.ReplicationPolicy{} + //TODO support more query conditions other than name and project ID + policies, err := dao.FilterRepPolicies(query.Name, query.ProjectID) + if err != nil { + return result, err + } + + for _, policy := range policies { + ply, err := convertFromPersistModel(policy) + if err != nil { + return []models.ReplicationPolicy{}, err + } + result = append(result, ply) + } + + return result, nil } //GetPolicy returns the policy with the specified ID -func (m *Manager) GetPolicy(policyID int) models.ReplicationPolicy { - return models.ReplicationPolicy{} +func (m *Manager) GetPolicy(policyID int64) (models.ReplicationPolicy, error) { + policy, err := dao.GetRepPolicy(policyID) + if err != nil { + return models.ReplicationPolicy{}, err + } + + return convertFromPersistModel(policy) +} + +// TODO add UT +func convertFromPersistModel(policy *persist_models.RepPolicy) (models.ReplicationPolicy, error) { + if policy == nil { + return models.ReplicationPolicy{}, nil + } + + ply := models.ReplicationPolicy{ + ID: policy.ID, + Name: policy.Name, + Description: policy.Description, + ReplicateDeletion: policy.ReplicateDeletion, + ProjectIDs: []int64{policy.ProjectID}, + TargetIDs: []int64{policy.TargetID}, + CreationTime: policy.CreationTime, + UpdateTime: policy.UpdateTime, + } + + if len(policy.Filters) > 0 { + filters := []models.FilterItem{} + if err := json.Unmarshal([]byte(policy.Filters), &filters); err != nil { + return models.ReplicationPolicy{}, err + } + ply.Filters = filters + } + + if len(policy.Trigger) > 0 { + trigger := &models.Trigger{} + if err := json.Unmarshal([]byte(policy.Trigger), trigger); err != nil { + return models.ReplicationPolicy{}, err + } + ply.Trigger = trigger + } + + return ply, nil +} + +// TODO add ut +func convertToPersistModel(policy models.ReplicationPolicy) (*persist_models.RepPolicy, error) { + ply := &persist_models.RepPolicy{ + ID: policy.ID, + Name: policy.Name, + Description: policy.Description, + ReplicateDeletion: policy.ReplicateDeletion, + CreationTime: policy.CreationTime, + UpdateTime: policy.UpdateTime, + } + + if len(policy.ProjectIDs) > 0 { + ply.ProjectID = policy.ProjectIDs[0] + } + + if len(policy.TargetIDs) > 0 { + ply.TargetID = policy.TargetIDs[0] + } + + if policy.Trigger != nil { + trigger, err := json.Marshal(policy.Trigger) + if err != nil { + return nil, err + } + ply.Trigger = string(trigger) + } + + if len(policy.Filters) > 0 { + filters, err := json.Marshal(policy.Filters) + if err != nil { + return nil, err + } + ply.Filters = string(filters) + } + + return ply, nil } //CreatePolicy creates a new policy with the provided data; //If creating failed, error will be returned; //If creating succeed, ID of the new created policy will be returned. -func (m *Manager) CreatePolicy(policy models.ReplicationPolicy) (int, error) { - return 0, nil +func (m *Manager) CreatePolicy(policy models.ReplicationPolicy) (int64, error) { + now := time.Now() + policy.CreationTime = now + policy.UpdateTime = now + ply, err := convertToPersistModel(policy) + if err != nil { + return 0, err + } + return dao.AddRepPolicy(*ply) } //UpdatePolicy updates the policy; //If updating failed, error will be returned. func (m *Manager) UpdatePolicy(policy models.ReplicationPolicy) error { - return nil + policy.UpdateTime = time.Now() + ply, err := convertToPersistModel(policy) + if err != nil { + return err + } + return dao.UpdateRepPolicy(ply) } //RemovePolicy removes the specified policy; //If removing failed, error will be returned. -func (m *Manager) RemovePolicy(policyID int) error { - return nil +func (m *Manager) RemovePolicy(policyID int64) error { + return dao.DeleteRepPolicy(policyID) } diff --git a/src/replication/trigger/cache.go b/src/replication/trigger/cache.go index 0f74bc79d..ea694f3eb 100644 --- a/src/replication/trigger/cache.go +++ b/src/replication/trigger/cache.go @@ -15,7 +15,7 @@ const ( //Item keeps more metadata of the triggers which are stored in the heap. type Item struct { //Which policy the trigger belong to - policyID int + policyID int64 //Frequency of cache querying //First compration factor @@ -124,7 +124,7 @@ func NewCache(capacity int) *Cache { } //Get the trigger interface with the specified policy ID -func (c *Cache) Get(policyID int) Interface { +func (c *Cache) Get(policyID int64) Interface { if policyID <= 0 { return nil } @@ -144,7 +144,7 @@ func (c *Cache) Get(policyID int) Interface { } //Put the item into cache with ID of ploicy as key -func (c *Cache) Put(policyID int, trigger Interface) { +func (c *Cache) Put(policyID int64, trigger Interface) { if policyID <= 0 || trigger == nil { return } @@ -179,7 +179,7 @@ func (c *Cache) Put(policyID int, trigger Interface) { } //Remove the trigger attached to the specified policy -func (c *Cache) Remove(policyID int) Interface { +func (c *Cache) Remove(policyID int64) Interface { if policyID > 0 { c.lock.Lock() defer c.lock.Unlock() @@ -207,6 +207,6 @@ func (c *Cache) Size() int { } //Generate a hash key with the policy ID -func (c *Cache) key(policyID int) string { +func (c *Cache) key(policyID int64) string { return fmt.Sprintf("trigger-%d", policyID) } diff --git a/src/replication/trigger/manager.go b/src/replication/trigger/manager.go index daed480e0..636fd8632 100644 --- a/src/replication/trigger/manager.go +++ b/src/replication/trigger/manager.go @@ -24,12 +24,12 @@ func NewManager(capacity int) *Manager { } //GetTrigger returns the enabled trigger reference if existing in the cache. -func (m *Manager) GetTrigger(policyID int) Interface { +func (m *Manager) GetTrigger(policyID int64) Interface { return m.cache.Get(policyID) } //RemoveTrigger will disable the trigger and remove it from the cache if existing. -func (m *Manager) RemoveTrigger(policyID int) error { +func (m *Manager) RemoveTrigger(policyID int64) error { trigger := m.cache.Get(policyID) if trigger == nil { return errors.New("Trigger is not cached, please use UnsetTrigger to disable the trigger") @@ -50,16 +50,16 @@ func (m *Manager) RemoveTrigger(policyID int) error { //SetupTrigger will create the new trigger based on the provided json parameters. //If failed, an error will be returned. -func (m *Manager) SetupTrigger(policyID int, trigger models.Trigger) error { +func (m *Manager) SetupTrigger(policyID int64, trigger models.Trigger) error { if policyID <= 0 { return errors.New("Invalid policy ID") } - if len(trigger.Name) == 0 { + if len(trigger.Kind) == 0 { return errors.New("Invalid replication trigger definition") } - switch trigger.Name { + switch trigger.Kind { case replication.TriggerKindSchedule: param := ScheduleParam{} if err := param.Parse(trigger.Param); err != nil { @@ -93,16 +93,16 @@ func (m *Manager) SetupTrigger(policyID int, trigger models.Trigger) error { } //UnsetTrigger will disable the trigger which is not cached in the trigger cache. -func (m *Manager) UnsetTrigger(policyID int, trigger models.Trigger) error { +func (m *Manager) UnsetTrigger(policyID int64, trigger models.Trigger) error { if policyID <= 0 { return errors.New("Invalid policy ID") } - if len(trigger.Name) == 0 { + if len(trigger.Kind) == 0 { return errors.New("Invalid replication trigger definition") } - switch trigger.Name { + switch trigger.Kind { case replication.TriggerKindSchedule: param := ScheduleParam{} if err := param.Parse(trigger.Param); err != nil { diff --git a/src/replication/trigger/trigger_param.go b/src/replication/trigger/trigger_param.go index 10420ef5b..cccf3fca3 100644 --- a/src/replication/trigger/trigger_param.go +++ b/src/replication/trigger/trigger_param.go @@ -3,7 +3,7 @@ package trigger //BasicParam contains the general parameters for all triggers type BasicParam struct { //ID of the related policy - PolicyID int + PolicyID int64 //Whether delete remote replicated images if local ones are deleted OnDeletion bool diff --git a/src/replication/trigger/watch_list.go b/src/replication/trigger/watch_list.go index 6a2009f29..ca6e44cee 100644 --- a/src/replication/trigger/watch_list.go +++ b/src/replication/trigger/watch_list.go @@ -10,7 +10,7 @@ type WatchList struct{} //WatchItem keeps the related data for evaluation in WatchList. type WatchItem struct { //ID of policy - PolicyID int + PolicyID int64 //Corresponding namespace Namespace string diff --git a/src/ui/api/api_test.go b/src/ui/api/api_test.go new file mode 100644 index 000000000..de9956954 --- /dev/null +++ b/src/ui/api/api_test.go @@ -0,0 +1,195 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package api + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "strconv" + "strings" + "testing" + + "github.com/astaxie/beego" + "github.com/dghubble/sling" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" +) + +var ( + nonSysAdminID int64 + sysAdmin = &usrInfo{ + Name: "admin", + Passwd: "Harbor12345", + } + nonSysAdmin = &usrInfo{ + Name: "non_admin", + Passwd: "Harbor12345", + } +) + +type testingRequest struct { + method string + url string + header http.Header + queryStruct interface{} + bodyJSON interface{} + credential *usrInfo +} + +type codeCheckingCase struct { + request *testingRequest + code int + postFunc func(*httptest.ResponseRecorder) error +} + +func newRequest(r *testingRequest) (*http.Request, error) { + if r == nil { + return nil, nil + } + + reqBuilder := sling.New() + switch strings.ToUpper(r.method) { + case "", http.MethodGet: + reqBuilder = reqBuilder.Get(r.url) + case http.MethodPost: + reqBuilder = reqBuilder.Post(r.url) + case http.MethodPut: + reqBuilder = reqBuilder.Put(r.url) + case http.MethodDelete: + reqBuilder = reqBuilder.Delete(r.url) + case http.MethodHead: + reqBuilder = reqBuilder.Head(r.url) + case http.MethodPatch: + reqBuilder = reqBuilder.Patch(r.url) + default: + return nil, fmt.Errorf("unsupported method %s", r.method) + } + + for key, values := range r.header { + for _, value := range values { + reqBuilder = reqBuilder.Add(key, value) + } + } + + if r.queryStruct != nil { + reqBuilder = reqBuilder.QueryStruct(r.queryStruct) + } + + if r.bodyJSON != nil { + reqBuilder = reqBuilder.BodyJSON(r.bodyJSON) + } + + if r.credential != nil { + reqBuilder = reqBuilder.SetBasicAuth(r.credential.Name, r.credential.Passwd) + } + + return reqBuilder.Request() +} + +func handle(r *testingRequest) (*httptest.ResponseRecorder, error) { + req, err := newRequest(r) + if err != nil { + return nil, err + } + + resp := httptest.NewRecorder() + beego.BeeApp.Handlers.ServeHTTP(resp, req) + return resp, nil +} + +func handleAndParse(r *testingRequest, v interface{}) (*httptest.ResponseRecorder, error) { + req, err := newRequest(r) + if err != nil { + return nil, err + } + + resp := httptest.NewRecorder() + beego.BeeApp.Handlers.ServeHTTP(resp, req) + + if resp.Code >= 200 && resp.Code <= 299 { + if err := json.NewDecoder(resp.Body).Decode(v); err != nil { + return nil, err + } + } + + return resp, nil +} + +func runCodeCheckingCases(t *testing.T, cases ...*codeCheckingCase) { + for _, c := range cases { + resp, err := handle(c.request) + require.Nil(t, err) + equal := assert.Equal(t, c.code, resp.Code) + if !equal { + if resp.Body.Len() > 0 { + t.Log(resp.Body.String()) + } + continue + } + + if c.postFunc != nil { + if err := c.postFunc(resp); err != nil { + t.Logf("error in running post function: %v", err) + } + } + } +} + +func parseResourceID(resp *httptest.ResponseRecorder) (int64, error) { + location := resp.Header().Get(http.CanonicalHeaderKey("location")) + if len(location) == 0 { + return 0, fmt.Errorf("empty location header") + } + index := strings.LastIndex(location, "/") + if index == -1 { + return 0, fmt.Errorf("location header %s contains no /", location) + } + + id := strings.TrimPrefix(location, location[:index+1]) + if len(id) == 0 { + return 0, fmt.Errorf("location header %s contains no resource ID", location) + } + + return strconv.ParseInt(id, 10, 64) +} + +func TestMain(m *testing.M) { + if err := prepare(); err != nil { + panic(err) + } + defer clean() + + os.Exit(m.Run()) +} + +func prepare() error { + id, err := dao.Register(models.User{ + Username: nonSysAdmin.Name, + Password: nonSysAdmin.Passwd, + }) + if err != nil { + return err + } + nonSysAdminID = id + return nil +} + +func clean() error { + return dao.DeleteUser(int(nonSysAdminID)) +} diff --git a/src/ui/api/harborapi_test.go b/src/ui/api/harborapi_test.go index fa60ec3c9..10a54b096 100644 --- a/src/ui/api/harborapi_test.go +++ b/src/ui/api/harborapi_test.go @@ -122,7 +122,6 @@ func init() { beego.Router("/api/policies/replication/:id([0-9]+)", &RepPolicyAPI{}) beego.Router("/api/policies/replication", &RepPolicyAPI{}, "get:List") beego.Router("/api/policies/replication", &RepPolicyAPI{}, "post:Post;delete:Delete") - beego.Router("/api/policies/replication/:id([0-9]+)/enablement", &RepPolicyAPI{}, "put:UpdateEnablement") beego.Router("/api/systeminfo", &SystemInfoAPI{}, "get:GetGeneralInfo") beego.Router("/api/systeminfo/volumes", &SystemInfoAPI{}, "get:GetVolumeInfo") beego.Router("/api/systeminfo/getcert", &SystemInfoAPI{}, "get:GetCert") diff --git a/src/ui/api/models/replication_policy.go b/src/ui/api/models/replication_policy.go new file mode 100644 index 000000000..34fa38a7b --- /dev/null +++ b/src/ui/api/models/replication_policy.go @@ -0,0 +1,66 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "time" + + "github.com/astaxie/beego/validation" + common_models "github.com/vmware/harbor/src/common/models" + rep_models "github.com/vmware/harbor/src/replication/models" +) + +// ReplicationPolicy defines the data model used in API level +type ReplicationPolicy struct { + ID int64 `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Filters []rep_models.FilterItem `json:"filters"` + ReplicateDeletion bool `json:"replicate_deletion"` + Trigger *rep_models.Trigger `json:"trigger"` + Projects []*common_models.Project `json:"projects"` + Targets []*common_models.RepTarget `json:"targets"` + CreationTime time.Time `json:"creation_time"` + UpdateTime time.Time `json:"update_time"` + ReplicateExistingImageNow bool `json:"replicate_existing_image_now"` + ErrorJobCount int64 `json:"error_job_count"` +} + +// Valid ... +func (r *ReplicationPolicy) Valid(v *validation.Validation) { + if len(r.Name) == 0 { + v.SetError("name", "can not be empty") + } + + if len(r.Name) > 256 { + v.SetError("name", "max length is 256") + } + + if len(r.Projects) == 0 { + v.SetError("projects", "can not be empty") + } + + if len(r.Targets) == 0 { + v.SetError("targets", "can not be empty") + } + + for _, filter := range r.Filters { + filter.Valid(v) + } + + if r.Trigger != nil { + r.Trigger.Valid(v) + } +} diff --git a/src/ui/api/replication_policy.go b/src/ui/api/replication_policy.go index ebde92340..79b889b8b 100644 --- a/src/ui/api/replication_policy.go +++ b/src/ui/api/replication_policy.go @@ -23,6 +23,10 @@ import ( "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/replication/core" + rep_models "github.com/vmware/harbor/src/replication/models" + api_models "github.com/vmware/harbor/src/ui/api/models" + "github.com/vmware/harbor/src/ui/promgr" ) // RepPolicyAPI handles /api/replicationPolicies /api/replicationPolicies/:id/enablement @@ -47,344 +51,144 @@ func (pa *RepPolicyAPI) Prepare() { // Get ... func (pa *RepPolicyAPI) Get() { id := pa.GetIDFromURL() - policy, err := dao.GetRepPolicy(id) + policy, err := core.DefaultController.GetPolicy(id) if err != nil { log.Errorf("failed to get policy %d: %v", id, err) pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } - if policy == nil { + if policy.ID == 0 { pa.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound)) } - pa.Data["json"] = policy + ply, err := convertFromRepPolicy(pa.ProjectMgr, policy) + if err != nil { + pa.ParseAndHandleError(fmt.Sprintf("failed to convert from replication policy"), err) + return + } + + pa.Data["json"] = ply pa.ServeJSON() } -// List filters policies by name and project_id, if name and project_id -// are nil, List returns all policies +// List ... func (pa *RepPolicyAPI) List() { - name := pa.GetString("name") + queryParam := rep_models.QueryParameter{ + Name: pa.GetString("name"), + } projectIDStr := pa.GetString("project_id") - - var projectID int64 - var err error - - if len(projectIDStr) != 0 { - projectID, err = strconv.ParseInt(projectIDStr, 10, 64) + if len(projectIDStr) > 0 { + projectID, err := strconv.ParseInt(projectIDStr, 10, 64) if err != nil || projectID <= 0 { pa.CustomAbort(http.StatusBadRequest, "invalid project ID") } + queryParam.ProjectID = projectID } - policies, err := dao.FilterRepPolicies(name, projectID) + result := []*api_models.ReplicationPolicy{} + + policies, err := core.DefaultController.GetPolicies(queryParam) if err != nil { - log.Errorf("failed to filter policies %s project ID %d: %v", name, projectID, err) + log.Errorf("failed to get policies: %v, query parameters: %v", err, queryParam) pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } for _, policy := range policies { - project, err := pa.ProjectMgr.Get(policy.ProjectID) + ply, err := convertFromRepPolicy(pa.ProjectMgr, policy) if err != nil { - pa.ParseAndHandleError(fmt.Sprintf( - "failed to get project %d", policy.ProjectID), err) + pa.ParseAndHandleError(fmt.Sprintf("failed to convert from replication policy"), err) return } - if project != nil { - policy.ProjectName = project.Name - } + result = append(result, ply) } - pa.Data["json"] = policies + pa.Data["json"] = result pa.ServeJSON() } -// Post creates a policy, and if it is enbled, the replication will be triggered right now. +// Post creates a replicartion policy func (pa *RepPolicyAPI) Post() { - policy := &models.RepPolicy{} + policy := &api_models.ReplicationPolicy{} pa.DecodeJSONReqAndValidate(policy) - /* - po, err := dao.GetRepPolicyByName(policy.Name) + // check the existence of projects + for _, project := range policy.Projects { + exist, err := pa.ProjectMgr.Exists(project.ProjectID) if err != nil { - log.Errorf("failed to get policy %s: %v", policy.Name, err) - pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + pa.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %d", project.ProjectID), err) + return + } + if !exist { + pa.HandleNotFound(fmt.Sprintf("project %d not found", project.ProjectID)) + return + } + } + + // check the existence of targets + for _, target := range policy.Targets { + t, err := dao.GetRepTarget(target.ID) + if err != nil { + pa.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", target.ID, err)) + return } - if po != nil { - pa.CustomAbort(http.StatusConflict, "name is already used") + if t == nil { + pa.HandleNotFound(fmt.Sprintf("target %d not found", target.ID)) + return } - */ + } - project, err := pa.ProjectMgr.Get(policy.ProjectID) + id, err := core.DefaultController.CreatePolicy(convertToRepPolicy(policy)) if err != nil { - pa.ParseAndHandleError(fmt.Sprintf("failed to get project %d", policy.ProjectID), err) + pa.HandleInternalServerError(fmt.Sprintf("failed to create policy: %v", err)) return } - if project == nil { - pa.CustomAbort(http.StatusBadRequest, fmt.Sprintf("project %d does not exist", policy.ProjectID)) - } + // TODO trigger a replication if ReplicateExistingImageNow is true - target, err := dao.GetRepTarget(policy.TargetID) - if err != nil { - log.Errorf("failed to get target %d: %v", policy.TargetID, err) - pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - if target == nil { - pa.CustomAbort(http.StatusBadRequest, fmt.Sprintf("target %d does not exist", policy.TargetID)) - } - - policies, err := dao.GetRepPolicyByProjectAndTarget(policy.ProjectID, policy.TargetID) - if err != nil { - log.Errorf("failed to get policy [project ID: %d,targetID: %d]: %v", policy.ProjectID, policy.TargetID, err) - pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - if len(policies) > 0 { - pa.CustomAbort(http.StatusConflict, "policy already exists with the same project and target") - } - - pid, err := dao.AddRepPolicy(*policy) - if err != nil { - log.Errorf("Failed to add policy to DB, error: %v", err) - pa.RenderError(http.StatusInternalServerError, "Internal Error") - return - } - - if policy.Enabled == 1 { - go func() { - if err := TriggerReplication(pid, "", nil, models.RepOpTransfer); err != nil { - log.Errorf("failed to trigger replication of %d: %v", pid, err) - } else { - log.Infof("replication of %d triggered", pid) - } - }() - } - - pa.Redirect(http.StatusCreated, strconv.FormatInt(pid, 10)) + pa.Redirect(http.StatusCreated, strconv.FormatInt(id, 10)) } -// Put modifies name, description, target and enablement of policy +// Put updates the replication policy func (pa *RepPolicyAPI) Put() { id := pa.GetIDFromURL() - originalPolicy, err := dao.GetRepPolicy(id) + + originalPolicy, err := core.DefaultController.GetPolicy(id) if err != nil { log.Errorf("failed to get policy %d: %v", id, err) pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } - if originalPolicy == nil { + if originalPolicy.ID == 0 { pa.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound)) } - policy := &models.RepPolicy{} - pa.DecodeJSONReq(policy) - policy.ProjectID = originalPolicy.ProjectID - pa.Validate(policy) - - /* - // check duplicate name - if policy.Name != originalPolicy.Name { - po, err := dao.GetRepPolicyByName(policy.Name) - if err != nil { - log.Errorf("failed to get policy %s: %v", policy.Name, err) - pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - if po != nil { - pa.CustomAbort(http.StatusConflict, "name is already used") - } - } - */ - - if policy.TargetID != originalPolicy.TargetID { - //target of policy can not be modified when the policy is enabled - if originalPolicy.Enabled == 1 { - pa.CustomAbort(http.StatusBadRequest, "target of policy can not be modified when the policy is enabled") - } - - // check the existance of target - target, err := dao.GetRepTarget(policy.TargetID) - if err != nil { - log.Errorf("failed to get target %d: %v", policy.TargetID, err) - pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - if target == nil { - pa.CustomAbort(http.StatusBadRequest, fmt.Sprintf("target %d does not exist", policy.TargetID)) - } - - // check duplicate policy with the same project and target - policies, err := dao.GetRepPolicyByProjectAndTarget(policy.ProjectID, policy.TargetID) - if err != nil { - log.Errorf("failed to get policy [project ID: %d,targetID: %d]: %v", policy.ProjectID, policy.TargetID, err) - pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - if len(policies) > 0 { - pa.CustomAbort(http.StatusConflict, "policy already exists with the same project and target") - } - } + policy := &api_models.ReplicationPolicy{} + pa.DecodeJSONReqAndValidate(policy) policy.ID = id - /* - isTargetChanged := !(policy.TargetID == originalPolicy.TargetID) - isEnablementChanged := !(policy.Enabled == policy.Enabled) - - var shouldStop, shouldTrigger bool - - // if target and enablement are not changed, do nothing - if !isTargetChanged && !isEnablementChanged { - shouldStop = false - shouldTrigger = false - } else if !isTargetChanged && isEnablementChanged { - // target is not changed, but enablement is changed - if policy.Enabled == 0 { - shouldStop = true - shouldTrigger = false - } else { - shouldStop = false - shouldTrigger = true - } - } else if isTargetChanged && !isEnablementChanged { - // target is changed, but enablement is not changed - if policy.Enabled == 0 { - // enablement is 0, do nothing - shouldStop = false - shouldTrigger = false - } else { - // enablement is 1, so stop original target's jobs - // and trigger new target's jobs - shouldStop = true - shouldTrigger = true - } - } else { - // both target and enablement are changed - - // enablement: 1 -> 0 - if policy.Enabled == 0 { - shouldStop = true - shouldTrigger = false - } else { - shouldStop = false - shouldTrigger = true - } - } - - if shouldStop { - if err := postReplicationAction(id, "stop"); err != nil { - log.Errorf("failed to stop replication of %d: %v", id, err) - pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - log.Infof("replication of %d has been stopped", id) - } - - if err = dao.UpdateRepPolicy(policy); err != nil { - log.Errorf("failed to update policy %d: %v", id, err) - pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - if shouldTrigger { - go func() { - if err := TriggerReplication(id, "", nil, models.RepOpTransfer); err != nil { - log.Errorf("failed to trigger replication of %d: %v", id, err) - } else { - log.Infof("replication of %d triggered", id) - } - }() - } - */ - - if err = dao.UpdateRepPolicy(policy); err != nil { - log.Errorf("failed to update policy %d: %v", id, err) - pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - if policy.Enabled != originalPolicy.Enabled && policy.Enabled == 1 { - go func() { - if err := TriggerReplication(id, "", nil, models.RepOpTransfer); err != nil { - log.Errorf("failed to trigger replication of %d: %v", id, err) - } else { - log.Infof("replication of %d triggered", id) - } - }() + if err = core.DefaultController.UpdatePolicy(convertToRepPolicy(policy)); err != nil { + pa.HandleInternalServerError(fmt.Sprintf("failed to update policy %d: %v", id, err)) + return } } -type enablementReq struct { - Enabled int `json:"enabled"` -} - -// UpdateEnablement changes the enablement of the policy -func (pa *RepPolicyAPI) UpdateEnablement() { +// Delete the replication policy +func (pa *RepPolicyAPI) Delete() { id := pa.GetIDFromURL() - policy, err := dao.GetRepPolicy(id) + + policy, err := core.DefaultController.GetPolicy(id) if err != nil { log.Errorf("failed to get policy %d: %v", id, err) pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } - if policy == nil { + if policy.ID == 0 { pa.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound)) } - e := enablementReq{} - pa.DecodeJSONReq(&e) - if e.Enabled != 0 && e.Enabled != 1 { - pa.RenderError(http.StatusBadRequest, "invalid enabled value") - return - } - - if policy.Enabled == e.Enabled { - return - } - - if err := dao.UpdateRepPolicyEnablement(id, e.Enabled); err != nil { - log.Errorf("Failed to update policy enablement in DB, error: %v", err) - pa.RenderError(http.StatusInternalServerError, "Internal Error") - return - } - - if e.Enabled == 1 { - go func() { - if err := TriggerReplication(id, "", nil, models.RepOpTransfer); err != nil { - log.Errorf("failed to trigger replication of %d: %v", id, err) - } else { - log.Infof("replication of %d triggered", id) - } - }() - } else { - go func() { - if err := postReplicationAction(id, "stop"); err != nil { - log.Errorf("failed to stop replication of %d: %v", id, err) - } else { - log.Infof("try to stop replication of %d", id) - } - }() - } -} - -// Delete : policies which are disabled and have no running jobs -// can be deleted -func (pa *RepPolicyAPI) Delete() { - id := pa.GetIDFromURL() - policy, err := dao.GetRepPolicy(id) - if err != nil { - log.Errorf("failed to get policy %d: %v", id, err) - pa.CustomAbort(http.StatusInternalServerError, "") - } - - if policy == nil || policy.Deleted == 1 { - pa.CustomAbort(http.StatusNotFound, "") - } - - if policy.Enabled == 1 { - pa.CustomAbort(http.StatusPreconditionFailed, "plicy is enabled, can not be deleted") - } - + // TODO jobs, err := dao.GetRepJobByPolicy(id) if err != nil { log.Errorf("failed to get jobs of policy %d: %v", id, err) @@ -399,8 +203,82 @@ func (pa *RepPolicyAPI) Delete() { } } - if err = dao.DeleteRepPolicy(id); err != nil { + if err = core.DefaultController.RemovePolicy(id); err != nil { log.Errorf("failed to delete policy %d: %v", id, err) pa.CustomAbort(http.StatusInternalServerError, "") } } + +func convertFromRepPolicy(projectMgr promgr.ProjectManager, policy rep_models.ReplicationPolicy) (*api_models.ReplicationPolicy, error) { + if policy.ID == 0 { + return nil, nil + } + + // populate simple properties + ply := &api_models.ReplicationPolicy{ + ID: policy.ID, + Name: policy.Name, + Description: policy.Description, + Filters: policy.Filters, + ReplicateDeletion: policy.ReplicateDeletion, + Trigger: policy.Trigger, + CreationTime: policy.CreationTime, + UpdateTime: policy.UpdateTime, + } + + // populate projects + for _, projectID := range policy.ProjectIDs { + project, err := projectMgr.Get(projectID) + if err != nil { + return nil, err + } + + ply.Projects = append(ply.Projects, project) + } + + // populate targets + for _, targetID := range policy.TargetIDs { + target, err := dao.GetRepTarget(targetID) + if err != nil { + return nil, err + } + target.Password = "" + ply.Targets = append(ply.Targets, target) + } + + // TODO call the method from replication controller + _, errJobCount, err := dao.FilterRepJobs(policy.ID, "", "error", nil, nil, 0, 0) + if err != nil { + return nil, err + } + ply.ErrorJobCount = errJobCount + + return ply, nil +} + +func convertToRepPolicy(policy *api_models.ReplicationPolicy) rep_models.ReplicationPolicy { + if policy == nil { + return rep_models.ReplicationPolicy{} + } + + ply := rep_models.ReplicationPolicy{ + ID: policy.ID, + Name: policy.Name, + Description: policy.Description, + Filters: policy.Filters, + ReplicateDeletion: policy.ReplicateDeletion, + Trigger: policy.Trigger, + CreationTime: policy.CreationTime, + UpdateTime: policy.UpdateTime, + } + + for _, project := range policy.Projects { + ply.ProjectIDs = append(ply.ProjectIDs, project.ProjectID) + } + + for _, target := range policy.Targets { + ply.TargetIDs = append(ply.TargetIDs, target.ID) + } + + return ply +} diff --git a/src/ui/api/replication_policy_test.go b/src/ui/api/replication_policy_test.go index f3db26711..aca8b1bc2 100644 --- a/src/ui/api/replication_policy_test.go +++ b/src/ui/api/replication_policy_test.go @@ -15,296 +15,498 @@ package api import ( "fmt" - "strconv" + "net/http" + "net/http/httptest" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/replication" - "github.com/vmware/harbor/tests/apitests/apilib" + rep_models "github.com/vmware/harbor/src/replication/models" + api_models "github.com/vmware/harbor/src/ui/api/models" ) -const ( - addPolicyName = "testPolicy" +var ( + repPolicyAPIBasePath = "/api/policies/replication" + policyName = "testPolicy" + projectID int64 = 1 + targetID int64 + policyID int64 ) -var addPolicyID int +func TestRepPolicyAPIPost(t *testing.T) { + postFunc := func(resp *httptest.ResponseRecorder) error { + id, err := parseResourceID(resp) + if err != nil { + return err + } + policyID = id + return nil + } -func TestPoliciesPost(t *testing.T) { - var httpStatusCode int - var err error - - assert := assert.New(t) - apiTest := newHarborAPI() - - //add target CommonAddTarget() - targetID := int64(CommonGetTarget()) - repPolicy := &apilib.RepPolicyPost{int64(1), targetID, addPolicyName, - &models.RepTrigger{ - Type: replication.TriggerKindSchedule, - Params: map[string]interface{}{ - "date": "2:00", + targetID = int64(CommonGetTarget()) + + cases := []*codeCheckingCase{ + // 401 + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPost, + url: repPolicyAPIBasePath, }, + code: http.StatusUnauthorized, }, - []*models.RepFilter{ - &models.RepFilter{ - Type: replication.FilterItemKindRepository, - Value: "library/ubuntu*", + // 403 + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPost, + url: repPolicyAPIBasePath, + credential: nonSysAdmin, }, - }} - - fmt.Println("Testing Policies Post API") - - //-------------------case 1 : response code = 201------------------------// - fmt.Println("case 1 : response code = 201") - httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy) - if err != nil { - t.Error("Error while add policy", err.Error()) - t.Log(err) - } else { - assert.Equal(int(201), httpStatusCode, "httpStatusCode should be 201") - } - - //-------------------case 2 : response code = 409------------------------// - fmt.Println("case 2 : response code = 409:policy already exists") - httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy) - if err != nil { - t.Error("Error while add policy", err.Error()) - t.Log(err) - } else { - assert.Equal(int(409), httpStatusCode, "httpStatusCode should be 409") - } - - //-------------------case 3 : response code = 401------------------------// - fmt.Println("case 3 : response code = 401: User need to log in first.") - httpStatusCode, err = apiTest.AddPolicy(*unknownUsr, *repPolicy) - if err != nil { - t.Error("Error while add policy", err.Error()) - t.Log(err) - } else { - assert.Equal(int(401), httpStatusCode, "httpStatusCode should be 401") - } - - //-------------------case 4 : response code = 400------------------------// - fmt.Println("case 4 : response code = 400:project_id invalid.") - - repPolicy = &apilib.RepPolicyPost{TargetId: targetID, Name: addPolicyName} - httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy) - if err != nil { - t.Error("Error while add policy", err.Error()) - t.Log(err) - } else { - assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400") - } - - //-------------------case 5 : response code = 400------------------------// - fmt.Println("case 5 : response code = 400:project_id does not exist.") - - repPolicy.ProjectId = int64(1111) - httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy) - if err != nil { - t.Error("Error while add policy", err.Error()) - t.Log(err) - } else { - assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400") - } - - //-------------------case 6 : response code = 400------------------------// - fmt.Println("case 6 : response code = 400:target_id invalid.") - - repPolicy = &apilib.RepPolicyPost{ProjectId: int64(1), Name: addPolicyName} - httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy) - if err != nil { - t.Error("Error while add policy", err.Error()) - t.Log(err) - } else { - assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400") - } - - //-------------------case 7 : response code = 400------------------------// - fmt.Println("case 7 : response code = 400:target_id does not exist.") - - repPolicy.TargetId = int64(1111) - httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy) - if err != nil { - t.Error("Error while add policy", err.Error()) - t.Log(err) - } else { - assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400") - } - - fmt.Println("case 8 : response code = 400: invalid filter") - repPolicy = &apilib.RepPolicyPost{int64(1), targetID, addPolicyName, - &models.RepTrigger{ - Type: replication.TriggerKindManually, + code: http.StatusForbidden, }, - []*models.RepFilter{ - &models.RepFilter{ - Type: "replication", - Value: "", + + // 400, invalid name + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPost, + url: repPolicyAPIBasePath, + bodyJSON: &api_models.ReplicationPolicy{}, + credential: sysAdmin, }, - }} - httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy) + code: http.StatusBadRequest, + }, + // 400, invalid projects + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPost, + url: repPolicyAPIBasePath, + bodyJSON: &api_models.ReplicationPolicy{ + Name: policyName, + }, + credential: sysAdmin, + }, + code: http.StatusBadRequest, + }, + // 400, invalid targets + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPost, + url: repPolicyAPIBasePath, + bodyJSON: &api_models.ReplicationPolicy{ + Name: policyName, + Projects: []*models.Project{ + &models.Project{ + ProjectID: projectID, + }, + }, + }, + credential: sysAdmin, + }, + code: http.StatusBadRequest, + }, + // 400, invalid filters + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPost, + url: repPolicyAPIBasePath, + bodyJSON: &api_models.ReplicationPolicy{ + Name: policyName, + Projects: []*models.Project{ + &models.Project{ + ProjectID: projectID, + }, + }, + Targets: []*models.RepTarget{ + &models.RepTarget{ + ID: targetID, + }, + }, + Filters: []rep_models.FilterItem{ + rep_models.FilterItem{ + Kind: "invalid_filter_kind", + Value: "", + }, + }, + }, + credential: sysAdmin, + }, + code: http.StatusBadRequest, + }, + // 400, invalid trigger + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPost, + url: repPolicyAPIBasePath, + bodyJSON: &api_models.ReplicationPolicy{ + Name: policyName, + Projects: []*models.Project{ + &models.Project{ + ProjectID: projectID, + }, + }, + Targets: []*models.RepTarget{ + &models.RepTarget{ + ID: targetID, + }, + }, + Filters: []rep_models.FilterItem{ + rep_models.FilterItem{ + Kind: replication.FilterItemKindRepository, + Value: "*", + }, + }, + Trigger: &rep_models.Trigger{ + Kind: "invalid_trigger_kind", + }, + }, + credential: sysAdmin, + }, + code: http.StatusBadRequest, + }, + // 404, project not found + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPost, + url: repPolicyAPIBasePath, + bodyJSON: &api_models.ReplicationPolicy{ + Name: policyName, + Projects: []*models.Project{ + &models.Project{ + ProjectID: 10000, + }, + }, + Targets: []*models.RepTarget{ + &models.RepTarget{ + ID: targetID, + }, + }, + Filters: []rep_models.FilterItem{ + rep_models.FilterItem{ + Kind: replication.FilterItemKindRepository, + Value: "*", + }, + }, + Trigger: &rep_models.Trigger{ + Kind: replication.TriggerKindManually, + }, + }, + credential: sysAdmin, + }, + code: http.StatusNotFound, + }, + // 404, target not found + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPost, + url: repPolicyAPIBasePath, + bodyJSON: &api_models.ReplicationPolicy{ + Name: policyName, + Projects: []*models.Project{ + &models.Project{ + ProjectID: projectID, + }, + }, + Targets: []*models.RepTarget{ + &models.RepTarget{ + ID: 10000, + }, + }, + Filters: []rep_models.FilterItem{ + rep_models.FilterItem{ + Kind: replication.FilterItemKindRepository, + Value: "*", + }, + }, + Trigger: &rep_models.Trigger{ + Kind: replication.TriggerKindManually, + }, + }, + credential: sysAdmin, + }, + code: http.StatusNotFound, + }, + // 201 + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPost, + url: repPolicyAPIBasePath, + bodyJSON: &api_models.ReplicationPolicy{ + Name: policyName, + Projects: []*models.Project{ + &models.Project{ + ProjectID: projectID, + }, + }, + Targets: []*models.RepTarget{ + &models.RepTarget{ + ID: targetID, + }, + }, + Filters: []rep_models.FilterItem{ + rep_models.FilterItem{ + Kind: replication.FilterItemKindRepository, + Value: "*", + }, + }, + Trigger: &rep_models.Trigger{ + Kind: replication.TriggerKindManually, + }, + }, + credential: sysAdmin, + }, + code: http.StatusCreated, + postFunc: postFunc, + }, + } + + runCodeCheckingCases(t, cases...) +} + +func TestRepPolicyAPIGet(t *testing.T) { + // 404 + runCodeCheckingCases(t, &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodGet, + url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, 10000), + credential: sysAdmin, + }, + code: http.StatusNotFound, + }) + + // 200 + policy := &api_models.ReplicationPolicy{} + resp, err := handleAndParse( + &testingRequest{ + method: http.MethodGet, + url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID), + credential: sysAdmin, + }, policy) require.Nil(t, err) - assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400") + assert.Equal(t, http.StatusOK, resp.Code) + assert.Equal(t, policyID, policy.ID) + assert.Equal(t, policyName, policy.Name) } -func TestPoliciesList(t *testing.T) { - var httpStatusCode int - var err error - var reslut []apilib.RepPolicy +func TestRepPolicyAPIList(t *testing.T) { + // 400: invalid project ID + runCodeCheckingCases(t, &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodGet, + url: repPolicyAPIBasePath, + queryStruct: struct { + ProjectID int64 `url:"project_id"` + }{ + ProjectID: -1, + }, + credential: sysAdmin, + }, + code: http.StatusBadRequest, + }) - assert := assert.New(t) - apiTest := newHarborAPI() - - fmt.Println("Testing Policies Get/List API") - - //-------------------case 1 : response code = 200------------------------// - fmt.Println("case 1 : response code = 200") - projectID := "1" - httpStatusCode, reslut, err = apiTest.ListPolicies(*admin, addPolicyName, projectID) - if err != nil { - t.Error("Error while get policies", err.Error()) - t.Log(err) - } else { - assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") - addPolicyID = int(reslut[0].Id) - } - - //-------------------case 2 : response code = 400------------------------// - fmt.Println("case 2 : response code = 400:invalid projectID") - projectID = "cc" - httpStatusCode, reslut, err = apiTest.ListPolicies(*admin, addPolicyName, projectID) - if err != nil { - t.Error("Error while get policies", err.Error()) - t.Log(err) - } else { - assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400") - } + // 200 + policies := []*api_models.ReplicationPolicy{} + resp, err := handleAndParse( + &testingRequest{ + method: http.MethodGet, + url: repPolicyAPIBasePath, + queryStruct: struct { + ProjectID int64 `url:"project_id"` + Name string `url:"name"` + }{ + ProjectID: projectID, + Name: policyName, + }, + credential: sysAdmin, + }, &policies) + require.Nil(t, err) + assert.Equal(t, http.StatusOK, resp.Code) + require.Equal(t, 1, len(policies)) + assert.Equal(t, policyID, policies[0].ID) + assert.Equal(t, policyName, policies[0].Name) + // 200 + policies = []*api_models.ReplicationPolicy{} + resp, err = handleAndParse( + &testingRequest{ + method: http.MethodGet, + url: repPolicyAPIBasePath, + queryStruct: struct { + ProjectID int64 `url:"project_id"` + Name string `url:"name"` + }{ + ProjectID: projectID, + Name: "non_exist_policy", + }, + credential: sysAdmin, + }, &policies) + require.Nil(t, err) + assert.Equal(t, http.StatusOK, resp.Code) + require.Equal(t, 0, len(policies)) } -func TestPolicyGet(t *testing.T) { - var httpStatusCode int - var err error - - assert := assert.New(t) - apiTest := newHarborAPI() - - fmt.Println("Testing Policy Get API by PolicyID") - - //-------------------case 1 : response code = 200------------------------// - fmt.Println("case 1 : response code = 200") - - policyID := strconv.Itoa(addPolicyID) - httpStatusCode, err = apiTest.GetPolicyByID(*admin, policyID) - if err != nil { - t.Error("Error while get policy", err.Error()) - t.Log(err) - } else { - assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") +func TestRepPolicyAPIPut(t *testing.T) { + cases := []*codeCheckingCase{ + // 404 + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPut, + url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, 10000), + credential: sysAdmin, + }, + code: http.StatusNotFound, + }, + // 400, invalid trigger + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPut, + url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID), + bodyJSON: &api_models.ReplicationPolicy{ + Name: policyName, + Projects: []*models.Project{ + &models.Project{ + ProjectID: projectID, + }, + }, + Targets: []*models.RepTarget{ + &models.RepTarget{ + ID: targetID, + }, + }, + Filters: []rep_models.FilterItem{ + rep_models.FilterItem{ + Kind: replication.FilterItemKindRepository, + Value: "*", + }, + }, + Trigger: &rep_models.Trigger{ + Kind: "invalid_trigger_kind", + }, + }, + credential: sysAdmin, + }, + code: http.StatusBadRequest, + }, + // 200 + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPut, + url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID), + bodyJSON: &api_models.ReplicationPolicy{ + Name: policyName, + Projects: []*models.Project{ + &models.Project{ + ProjectID: projectID, + }, + }, + Targets: []*models.RepTarget{ + &models.RepTarget{ + ID: targetID, + }, + }, + Filters: []rep_models.FilterItem{ + rep_models.FilterItem{ + Kind: replication.FilterItemKindRepository, + Value: "*", + }, + }, + Trigger: &rep_models.Trigger{ + Kind: replication.TriggerKindImmediately, + }, + }, + credential: sysAdmin, + }, + code: http.StatusOK, + }, } + + runCodeCheckingCases(t, cases...) } -func TestPolicyUpdateInfo(t *testing.T) { - var httpStatusCode int - var err error - - targetID := int64(CommonGetTarget()) - policyInfo := &apilib.RepPolicyUpdate{TargetId: targetID, Name: "testNewName"} - - assert := assert.New(t) - apiTest := newHarborAPI() - - fmt.Println("Testing Policy PUT API to update policyInfo") - - //-------------------case 1 : response code = 200------------------------// - fmt.Println("case 1 : response code = 200") - - policyID := strconv.Itoa(addPolicyID) - httpStatusCode, err = apiTest.PutPolicyInfoByID(*admin, policyID, *policyInfo) - if err != nil { - t.Error("Error while update policyInfo", err.Error()) - t.Log(err) - } else { - assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") +func TestRepPolicyAPIDelete(t *testing.T) { + cases := []*codeCheckingCase{ + // 404 + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodDelete, + url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, 10000), + credential: sysAdmin, + }, + code: http.StatusNotFound, + }, + // 200 + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodDelete, + url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID), + credential: sysAdmin, + }, + code: http.StatusOK, + }, } + + runCodeCheckingCases(t, cases...) } -func TestPolicyUpdateEnablement(t *testing.T) { - var httpStatusCode int - var err error - - enablement := &apilib.RepPolicyEnablementReq{int32(0)} - - assert := assert.New(t) - apiTest := newHarborAPI() - - fmt.Println("Testing Policy PUT API to update policy enablement") - - //-------------------case 1 : response code = 200------------------------// - fmt.Println("case 1 : response code = 200") - - policyID := strconv.Itoa(addPolicyID) - httpStatusCode, err = apiTest.PutPolicyEnableByID(*admin, policyID, *enablement) - if err != nil { - t.Error("Error while put policy enablement", err.Error()) - t.Log(err) - } else { - assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") - } - //-------------------case 2 : response code = 404------------------------// - fmt.Println("case 2 : response code = 404,Not Found") - - policyID = "111" - httpStatusCode, err = apiTest.PutPolicyEnableByID(*admin, policyID, *enablement) - if err != nil { - t.Error("Error while put policy enablement", err.Error()) - t.Log(err) - } else { - assert.Equal(int(404), httpStatusCode, "httpStatusCode should be 404") +func TestConvertToRepPolicy(t *testing.T) { + cases := []struct { + input *api_models.ReplicationPolicy + expected rep_models.ReplicationPolicy + }{ + { + input: nil, + expected: rep_models.ReplicationPolicy{}, + }, + { + input: &api_models.ReplicationPolicy{ + ID: 1, + Name: "policy", + Description: "description", + Filters: []rep_models.FilterItem{ + rep_models.FilterItem{ + Kind: "filter_kind_01", + Value: "*", + }, + }, + ReplicateDeletion: true, + Trigger: &rep_models.Trigger{ + Kind: "trigger_kind_01", + Param: "{param}", + }, + Projects: []*models.Project{ + &models.Project{ + ProjectID: 1, + }, + }, + Targets: []*models.RepTarget{ + &models.RepTarget{ + ID: 1, + }, + }, + }, + expected: rep_models.ReplicationPolicy{ + ID: 1, + Name: "policy", + Description: "description", + Filters: []rep_models.FilterItem{ + rep_models.FilterItem{ + Kind: "filter_kind_01", + Value: "*", + }, + }, + ReplicateDeletion: true, + Trigger: &rep_models.Trigger{ + Kind: "trigger_kind_01", + Param: "{param}", + }, + ProjectIDs: []int64{1}, + TargetIDs: []int64{1}, + }, + }, } -} - -func TestPolicyDelete(t *testing.T) { - var httpStatusCode int - var err error - - assert := assert.New(t) - apiTest := newHarborAPI() - - fmt.Println("Testing Policy Delete API") - - //-------------------case 1 : response code = 412------------------------// - fmt.Println("case 1 : response code = 412:policy is enabled, can not be deleted") - - CommonPolicyEabled(addPolicyID, 1) - policyID := strconv.Itoa(addPolicyID) - - httpStatusCode, err = apiTest.DeletePolicyByID(*admin, policyID) - if err != nil { - t.Error("Error while delete policy", err.Error()) - t.Log(err) - } else { - assert.Equal(int(412), httpStatusCode, "httpStatusCode should be 412") - } - - //-------------------case 2 : response code = 200------------------------// - fmt.Println("case 2 : response code = 200") - - CommonPolicyEabled(addPolicyID, 0) - policyID = strconv.Itoa(addPolicyID) - - httpStatusCode, err = apiTest.DeletePolicyByID(*admin, policyID) - if err != nil { - t.Error("Error while delete policy", err.Error()) - t.Log(err) - } else { - assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") - } - - CommonDelTarget() + for _, c := range cases { + assert.EqualValues(t, c.expected, convertToRepPolicy(c.input)) + } } diff --git a/src/ui/router.go b/src/ui/router.go index 0e0298fa6..aaa46f5ae 100644 --- a/src/ui/router.go +++ b/src/ui/router.go @@ -108,7 +108,6 @@ func initRouters() { beego.Router("/api/policies/replication/:id([0-9]+)", &api.RepPolicyAPI{}) beego.Router("/api/policies/replication", &api.RepPolicyAPI{}, "get:List") beego.Router("/api/policies/replication", &api.RepPolicyAPI{}, "post:Post") - beego.Router("/api/policies/replication/:id([0-9]+)/enablement", &api.RepPolicyAPI{}, "put:UpdateEnablement") beego.Router("/api/targets/", &api.TargetAPI{}, "get:List") beego.Router("/api/targets/", &api.TargetAPI{}, "post:Post") beego.Router("/api/targets/:id([0-9]+)", &api.TargetAPI{}) diff --git a/tests/apitests/apilib/rep_policy_post.go b/tests/apitests/apilib/rep_policy_post.go index 365070539..567bab7a4 100644 --- a/tests/apitests/apilib/rep_policy_post.go +++ b/tests/apitests/apilib/rep_policy_post.go @@ -22,10 +22,6 @@ package apilib -import ( - "github.com/vmware/harbor/src/common/models" -) - type RepPolicyPost struct { // The project ID. @@ -36,10 +32,4 @@ type RepPolicyPost struct { // The policy name. Name string `json:"name,omitempty"` - - // Trigger - Trigger *models.RepTrigger `json:"trigger"` - - // Filters - Filters []*models.RepFilter `json:"filters"` } From 59c1160edd90dabf51b71def50c16aff4fd1f5fd Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Mon, 20 Nov 2017 15:47:44 +0800 Subject: [PATCH 12/37] Setup/Unset trigger when CURD policies --- src/replication/core/controller.go | 52 ++++++++++++++-- src/replication/trigger/immediate.go | 2 +- src/replication/trigger/manager.go | 6 +- src/replication/trigger/watch_list.go | 13 ++-- src/ui/api/repository.go | 2 +- src/ui/api/utils.go | 59 ++++++++++++------- .../service/notifications/registry/handler.go | 2 +- 7 files changed, 97 insertions(+), 39 deletions(-) diff --git a/src/replication/core/controller.go b/src/replication/core/controller.go index eb18f6ada..5135510e7 100644 --- a/src/replication/core/controller.go +++ b/src/replication/core/controller.go @@ -91,22 +91,61 @@ func (ctl *Controller) Init() error { //CreatePolicy is used to create a new policy and enable it if necessary func (ctl *Controller) CreatePolicy(newPolicy models.ReplicationPolicy) (int64, error) { - //Validate policy - // TODO + id, err := ctl.policyManager.CreatePolicy(newPolicy) + if err != nil { + return 0, err + } - return ctl.policyManager.CreatePolicy(newPolicy) + if err = ctl.triggerManager.SetupTrigger(id, *newPolicy.Trigger); err != nil { + return 0, err + } + + return id, nil } //UpdatePolicy will update the policy with new content. //Parameter updatedPolicy must have the ID of the updated policy. func (ctl *Controller) UpdatePolicy(updatedPolicy models.ReplicationPolicy) error { // TODO check pre-conditions - return ctl.policyManager.UpdatePolicy(updatedPolicy) + + id := updatedPolicy.ID + originPolicy, err := ctl.policyManager.GetPolicy(id) + if err != nil { + return err + } + + if originPolicy.ID == 0 { + return fmt.Errorf("policy %d not found", id) + } + + if err = ctl.triggerManager.UnsetTrigger(id, *originPolicy.Trigger); err != nil { + return err + } + + if err = ctl.policyManager.UpdatePolicy(updatedPolicy); err != nil { + return err + } + + return ctl.triggerManager.SetupTrigger(id, *updatedPolicy.Trigger) } //RemovePolicy will remove the specified policy and clean the related settings func (ctl *Controller) RemovePolicy(policyID int64) error { // TODO check pre-conditions + + policy, err := ctl.policyManager.GetPolicy(policyID) + if err != nil { + return err + } + + if policy.ID == 0 { + return fmt.Errorf("policy %d not found", policyID) + } + + if err = ctl.triggerManager.UnsetTrigger(policyID, *policy.Trigger); err != nil { + return err + } + return ctl.policyManager.RemovePolicy(policyID) } @@ -122,6 +161,9 @@ func (ctl *Controller) GetPolicies(query models.QueryParameter) ([]models.Replic //Replicate starts one replication defined in the specified policy; //Can be launched by the API layer and related triggers. -func (ctl *Controller) Replicate(policyID int64) error { +func (ctl *Controller) Replicate(policyID int64, item ...*models.FilterItem) error { + + fmt.Printf("replicating %d ...\n", policyID) + return nil } diff --git a/src/replication/trigger/immediate.go b/src/replication/trigger/immediate.go index af4ba3b2b..d73c989b3 100644 --- a/src/replication/trigger/immediate.go +++ b/src/replication/trigger/immediate.go @@ -43,5 +43,5 @@ func (st *ImmediateTrigger) Setup() error { //Unset is the implementation of same method defined in Trigger interface func (st *ImmediateTrigger) Unset() error { - return errors.New("Not implemented") + return DefaultWatchList.Remove(st.params.PolicyID) } diff --git a/src/replication/trigger/manager.go b/src/replication/trigger/manager.go index 636fd8632..ac1e08628 100644 --- a/src/replication/trigger/manager.go +++ b/src/replication/trigger/manager.go @@ -12,17 +12,18 @@ import ( //with json format. type Manager struct { //Cache for triggers - cache *Cache + //cache *Cache } //NewManager is the constructor of trigger manager. //capacity is the max number of trigger references manager can keep in memory func NewManager(capacity int) *Manager { return &Manager{ - cache: NewCache(capacity), + //cache: NewCache(capacity), } } +/* //GetTrigger returns the enabled trigger reference if existing in the cache. func (m *Manager) GetTrigger(policyID int64) Interface { return m.cache.Get(policyID) @@ -47,6 +48,7 @@ func (m *Manager) RemoveTrigger(policyID int64) error { return nil } +*/ //SetupTrigger will create the new trigger based on the provided json parameters. //If failed, an error will be returned. diff --git a/src/replication/trigger/watch_list.go b/src/replication/trigger/watch_list.go index ca6e44cee..76b3eb8bd 100644 --- a/src/replication/trigger/watch_list.go +++ b/src/replication/trigger/watch_list.go @@ -28,16 +28,11 @@ func (wl *WatchList) Add(item WatchItem) error { } //Remove the specified watch item from list -func (wl *WatchList) Remove() WatchItem { - return WatchItem{} -} - -//Update the watch item in the list -func (wl *WatchList) Update(updatedItem WatchItem) error { +func (wl *WatchList) Remove(policyID int64) error { return nil } -//Get the specified watch item -func (wl *WatchList) Get(namespace string) WatchItem { - return WatchItem{} +//Get the watch items according to the namespace and operation +func (wl *WatchList) Get(namespace, operation string) ([]WatchItem, error) { + return []WatchItem{}, nil } diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index 7d01cd45e..958a6f5ef 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -255,7 +255,7 @@ func (ra *RepositoryAPI) Delete() { } log.Infof("delete tag: %s:%s", repoName, t) - go TriggerReplicationByRepository(project.ProjectID, repoName, []string{t}, models.RepOpDelete) + go CheckAndTriggerReplication(repoName+":"+t, "delete") go func(tag string) { if err := dao.AddAccessLog(models.AccessLog{ diff --git a/src/ui/api/utils.go b/src/ui/api/utils.go index a0bb4fb39..b66990155 100644 --- a/src/ui/api/utils.go +++ b/src/ui/api/utils.go @@ -32,6 +32,10 @@ import ( "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/common/utils/registry" "github.com/vmware/harbor/src/common/utils/registry/auth" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/core" + rep_models "github.com/vmware/harbor/src/replication/models" + "github.com/vmware/harbor/src/replication/trigger" "github.com/vmware/harbor/src/ui/config" "github.com/vmware/harbor/src/ui/promgr" "github.com/vmware/harbor/src/ui/service/token" @@ -77,7 +81,41 @@ func checkUserExists(name string) int { return 0 } +// CheckAndTriggerReplication checks whether replication policy is set +// on the resource, if is, trigger the replication +func CheckAndTriggerReplication(image, operation string) { + project, _ := utils.ParseRepository(image) + watchItems, err := trigger.DefaultWatchList.Get(project, operation) + if err != nil { + log.Errorf("failed to get watch list for resource %s, operation %s: %v", image, operation, err) + return + } + if len(watchItems) == 0 { + log.Debugf("no replication should be triggered for resource %s, operation %s, skip", image, operation) + return + } + + for _, watchItem := range watchItems { + // TODO define a new type ReplicationItem to wrap FilterItem and operation. + // Maybe change the FilterItem to interface and define a type Resource to + // implement FilterItem is better? + item := &rep_models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: image, + Metadata: map[string]interface{}{ + "operation": operation, + }, + } + if err := core.DefaultController.Replicate(watchItem.PolicyID, item); err != nil { + log.Errorf("failed to trigger replication for resource: %s, operation: %s: %v", image, operation, err) + return + } + log.Infof("replication for resource: %s, operation: %s triggered", image, operation) + } +} + // TriggerReplication triggers the replication according to the policy +// TODO remove func TriggerReplication(policyID int64, repository string, tags []string, operation string) error { data := struct { @@ -101,26 +139,7 @@ func TriggerReplication(policyID int64, repository string, return uiutils.RequestAsUI("POST", url, bytes.NewBuffer(b), uiutils.NewStatusRespHandler(http.StatusOK)) } -// TriggerReplicationByRepository triggers the replication according to the repository -func TriggerReplicationByRepository(projectID int64, repository string, tags []string, operation string) { - policies, err := dao.GetRepPolicyByProject(projectID) - if err != nil { - log.Errorf("failed to get policies for repository %s: %v", repository, err) - return - } - - for _, policy := range policies { - if policy.Enabled == 0 { - continue - } - if err := TriggerReplication(policy.ID, repository, tags, operation); err != nil { - log.Errorf("failed to trigger replication of policy %d for %s: %v", policy.ID, repository, err) - } else { - log.Infof("replication of policy %d for %s triggered", policy.ID, repository) - } - } -} - +// TODO remove func postReplicationAction(policyID int64, acton string) error { data := struct { PolicyID int64 `json:"policy_id"` diff --git a/src/ui/service/notifications/registry/handler.go b/src/ui/service/notifications/registry/handler.go index 4e532242c..6cfc2433d 100644 --- a/src/ui/service/notifications/registry/handler.go +++ b/src/ui/service/notifications/registry/handler.go @@ -104,7 +104,7 @@ func (n *NotificationHandler) Post() { } }() - go api.TriggerReplicationByRepository(pro.ProjectID, repository, []string{tag}, models.RepOpTransfer) + go api.CheckAndTriggerReplication(repository+":"+tag, "push") if autoScanEnabled(pro) { last, err := clairdao.GetLastUpdate() From 1c338ed30becdb3b4b427d8fc28baf39d5a0b58e Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 23 Nov 2017 13:47:56 +0800 Subject: [PATCH 13/37] Add replication manual trigger API & update replication/policy API docs --- docs/swagger.yaml | 165 ++++++++----------------------- src/ui/api/harborapi_test.go | 1 + src/ui/api/models/replication.go | 31 ++++++ src/ui/api/replication.go | 63 ++++++++++++ src/ui/api/replication_test.go | 92 +++++++++++++++++ src/ui/router.go | 1 + 6 files changed, 229 insertions(+), 124 deletions(-) create mode 100644 src/ui/api/models/replication.go create mode 100644 src/ui/api/replication.go create mode 100644 src/ui/api/replication_test.go diff --git a/docs/swagger.yaml b/docs/swagger.yaml index c6de6a1c1..9120fc964 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1484,7 +1484,7 @@ paths: description: Create new policy. required: true schema: - $ref: '#/definitions/RepPolicyPost' + $ref: '#/definitions/RepPolicy' tags: - Products responses: @@ -1539,10 +1539,10 @@ paths: description: policy ID - name: policyupdate in: body - description: 'Update policy name, description, target and enablement.' + description: 'Updated properties of the replication policy.' required: true schema: - $ref: '#/definitions/RepPolicyUpdate' + $ref: '#/definitions/RepPolicy' tags: - Products responses: @@ -1560,35 +1560,27 @@ paths: project and target. '500': description: Unexpected internal errors. - /policies/replication/{id}/enablement: - put: - summary: Put modifies enablement of the policy. + /replications: + post: + summary: Trigger the replication according to the specified policy. description: | - This endpoint let user update policy enablement flag. + This endpoint is used to trigger a replication. parameters: - - name: id - in: path - type: integer - format: int64 - required: true - description: policy ID - - name: enabledflag + - name: policy ID in: body - description: The policy enablement flag. + description: The ID of replication policy. required: true schema: - $ref: '#/definitions/RepPolicyEnablementReq' + $ref: '#/definitions/Replication' tags: - Products responses: '200': - description: Update job policy enablement successfully. - '400': - description: Invalid enabled value. + description: Trigger the replication successfully. '401': description: User need to log in first. '404': - description: The specific repository ID's policy does not exist. + description: The policy does not exist. '500': description: Unexpected internal errors. /targets: @@ -2376,27 +2368,22 @@ definitions: type: integer format: int64 description: The policy ID. - project_id: - type: integer - format: int64 - description: The project ID. - project_name: - type: string - description: The project name. - target_id: - type: integer - format: int64 - description: The target ID. name: type: string description: The policy name. - enabled: - type: integer - format: int - description: The policy's enabled status. description: type: string description: The description of the policy. + projects: + type: object + description: The project list that the policy applys to. + items: + $ref: '#/definitions/Project' + targets: + type: object + description: The target list. + items: + $ref: '#/definitions/RepTarget' trigger: type: object description: The trigger for schedule job. @@ -2408,14 +2395,11 @@ definitions: items: $ref: '#/definitions/RepFilter' replicate_existing_image_now: - type: string + type: boolean description: Whether to replicate the existing images now. replicate_deletion: - type: string + type: boolean description: Whether to replicate the deletion operation. - start_time: - type: string - description: The start time of the policy. creation_time: type: string description: The create time of the policy. @@ -2425,100 +2409,27 @@ definitions: error_job_count: format: int description: The error job count number for the policy. - deleted: - type: integer - RepPolicyPost: - type: object - properties: - project_id: - type: integer - format: int64 - description: The project ID. - target_id: - type: integer - format: int64 - description: The target ID. - name: - type: string - description: The policy name. - trigger: - type: object - description: The trigger for schedule job. - items: - $ref: '#/definitions/RepTrigger' - filters: - type: array - description: The replication policy filter array. - items: - $ref: '#/definitions/RepFilter' - replicate_existing_image_now: - type: string - description: Whether to replicate the existing images now. - replicate_deletion: - type: string - description: Whether replication deletion operation. - enabled: - type: integer - format: int - description: '1-enable, 0-disable' - RepPolicyUpdate: - type: object - properties: - target_id: - type: integer - format: int64 - description: The target ID. - name: - type: string - description: The policy name. - enabled: - type: integer - format: int - description: The policy's enabled status. - description: - type: string - description: The description of the policy. - trigger: - type: object - description: The trigger for schedule job. - items: - $ref: '#/definitions/RepTrigger' - filters: - type: array - description: The replication policy filter array. - items: - $ref: '#/definitions/RepFilter' - replicate_existing_image_now: - type: string - description: Whether to replicate the existing images now. - replicate_deletion: - type: string - description: Whether replication deletion operation. RepTrigger: type: object properties: - type: + kind: type: string - description: The replication policy trigger type. - params: - type: object - description: The map is the replication policy trigger parameters. + description: The replication policy trigger kind. + param: + type: string + description: The replication policy trigger parameters. RepFilter: type: object properties: - type: + kind: type: string - description: The replication policy filter type. + description: The replication policy filter kind. value: type: string description: The replication policy filter value. - RepPolicyEnablementReq: - type: object - properties: - enabled: - type: integer - format: int - description: The policy enablement flag. + metadata: + type: object + description: This map object is the replication policy filter metadata. RepTarget: type: object properties: @@ -3001,6 +2912,12 @@ definitions: type: integer description: The offest in seconds of UTC 0 o'clock, only valid when the policy type is "daily" description: The parameters of the policy, the values are dependant on the type of the policy. - + Replication: + type: object + properties: + policy_id: + type: integer + description: The ID of replication policy + diff --git a/src/ui/api/harborapi_test.go b/src/ui/api/harborapi_test.go index 10a54b096..a62e11816 100644 --- a/src/ui/api/harborapi_test.go +++ b/src/ui/api/harborapi_test.go @@ -129,6 +129,7 @@ func init() { beego.Router("/api/configurations", &ConfigAPI{}) beego.Router("/api/configurations/reset", &ConfigAPI{}, "post:Reset") beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping") + beego.Router("/api/replications", &ReplicationAPI{}) _ = updateInitPassword(1, "Harbor12345") diff --git a/src/ui/api/models/replication.go b/src/ui/api/models/replication.go new file mode 100644 index 000000000..c19584ea2 --- /dev/null +++ b/src/ui/api/models/replication.go @@ -0,0 +1,31 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "github.com/astaxie/beego/validation" +) + +// Replication defines the properties of model used in replication API +type Replication struct { + PolicyID int64 `json:"policy_id"` +} + +// Valid ... +func (r *Replication) Valid(v *validation.Validation) { + if r.PolicyID <= 0 { + v.SetError("policy_id", "invalid value") + } +} diff --git a/src/ui/api/replication.go b/src/ui/api/replication.go new file mode 100644 index 000000000..49e1b80dc --- /dev/null +++ b/src/ui/api/replication.go @@ -0,0 +1,63 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "fmt" + + "github.com/vmware/harbor/src/replication/core" + "github.com/vmware/harbor/src/ui/api/models" +) + +// ReplicationAPI handles API calls for replication +type ReplicationAPI struct { + BaseController +} + +// Prepare does authentication and authorization works +func (r *ReplicationAPI) Prepare() { + r.BaseController.Prepare() + if !r.SecurityCtx.IsAuthenticated() { + r.HandleUnauthorized() + return + } + + if !r.SecurityCtx.IsSysAdmin() { + r.HandleForbidden(r.SecurityCtx.GetUsername()) + return + } +} + +// Post trigger a replication according to the specified policy +func (r *ReplicationAPI) Post() { + replication := &models.Replication{} + r.DecodeJSONReqAndValidate(replication) + + policy, err := core.DefaultController.GetPolicy(replication.PolicyID) + if err != nil { + r.HandleInternalServerError(fmt.Sprintf("failed to get replication policy %d: %v", replication.PolicyID, err)) + return + } + + if policy.ID == 0 { + r.HandleNotFound(fmt.Sprintf("replication policy %d not found", replication.PolicyID)) + return + } + + if err = core.DefaultController.Replicate(replication.PolicyID); err != nil { + r.HandleInternalServerError(fmt.Sprintf("failed to trigger the replication policy %d: %v", replication.PolicyID, err)) + return + } +} diff --git a/src/ui/api/replication_test.go b/src/ui/api/replication_test.go new file mode 100644 index 000000000..ce9330f44 --- /dev/null +++ b/src/ui/api/replication_test.go @@ -0,0 +1,92 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package api + +import ( + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/replication" + api_models "github.com/vmware/harbor/src/ui/api/models" +) + +const ( + replicationAPIBaseURL = "/api/replications" +) + +func TestReplicationAPIPost(t *testing.T) { + targetID, err := dao.AddRepTarget( + models.RepTarget{ + Name: "test_replication_target", + URL: "127.0.0.1", + Username: "username", + Password: "password", + }) + require.Nil(t, err) + defer dao.DeleteRepTarget(targetID) + + policyID, err := dao.AddRepPolicy( + models.RepPolicy{ + Name: "test_replication_policy", + ProjectID: 1, + TargetID: targetID, + Trigger: fmt.Sprintf("{\"kind\":\"%s\"}", replication.TriggerKindManual), + }) + require.Nil(t, err) + defer dao.DeleteRepPolicy(policyID) + + cases := []*codeCheckingCase{ + // 401 + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPost, + url: replicationAPIBaseURL, + bodyJSON: &api_models.Replication{ + PolicyID: policyID, + }, + }, + code: http.StatusUnauthorized, + }, + // 404 + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPost, + url: replicationAPIBaseURL, + bodyJSON: &api_models.Replication{ + PolicyID: 10000, + }, + credential: admin, + }, + code: http.StatusNotFound, + }, + // 200 + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodPost, + url: replicationAPIBaseURL, + bodyJSON: &api_models.Replication{ + PolicyID: policyID, + }, + credential: admin, + }, + code: http.StatusOK, + }, + } + + runCodeCheckingCases(t, cases...) +} diff --git a/src/ui/router.go b/src/ui/router.go index aaa46f5ae..fc243afcf 100644 --- a/src/ui/router.go +++ b/src/ui/router.go @@ -118,6 +118,7 @@ func initRouters() { beego.Router("/api/configurations", &api.ConfigAPI{}) beego.Router("/api/configurations/reset", &api.ConfigAPI{}, "post:Reset") beego.Router("/api/statistics", &api.StatisticAPI{}) + beego.Router("/api/replications", &api.ReplicationAPI{}) beego.Router("/api/systeminfo", &api.SystemInfoAPI{}, "get:GetGeneralInfo") beego.Router("/api/systeminfo/volumes", &api.SystemInfoAPI{}, "get:GetVolumeInfo") From 6b0ee138e50852aa86dee919eb8031fde0794dda Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Tue, 21 Nov 2017 13:08:09 +0800 Subject: [PATCH 14/37] Implement immediate trigger and the methods of WatchList --- make/common/db/registry.sql | 11 +++ make/common/db/registry_sqlite.sql | 10 ++ src/common/dao/watch_item.go | 62 ++++++++++++ src/common/dao/watch_item_test.go | 71 ++++++++++++++ src/common/models/base.go | 3 +- src/common/models/watch_item.go | 35 +++++++ src/replication/consts.go | 14 +-- src/replication/core/controller.go | 35 +++++-- src/replication/models/policy.go | 1 + src/replication/models/trigger.go | 4 +- src/replication/trigger/immediate.go | 25 +++-- src/replication/trigger/manager.go | 32 +++--- src/replication/trigger/param_immediate.go | 10 +- src/replication/trigger/watch_list.go | 33 ++++++- src/replication/trigger/watch_list_test.go | 108 +++++++++++++++++++++ src/ui/api/replication_policy.go | 34 ++++++- src/ui/api/replication_policy_test.go | 10 +- tools/migration/changelog.md | 1 + 18 files changed, 435 insertions(+), 64 deletions(-) create mode 100644 src/common/dao/watch_item.go create mode 100644 src/common/dao/watch_item_test.go create mode 100644 src/common/models/watch_item.go create mode 100644 src/replication/trigger/watch_list_test.go diff --git a/make/common/db/registry.sql b/make/common/db/registry.sql index 994b108d7..37d3e1072 100644 --- a/make/common/db/registry.sql +++ b/make/common/db/registry.sql @@ -184,6 +184,17 @@ create table replication_job ( INDEX policy (policy_id), INDEX poid_uptime (policy_id, update_time) ); + +create table replication_immediate_trigger ( + id int NOT NULL AUTO_INCREMENT, + policy_id int NOT NULL, + namespace varchar(256) NOT NULL, + on_push tinyint(1) NOT NULL DEFAULT 0, + on_deletion tinyint(1) NOT NULL DEFAULT 0, + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, + PRIMARY KEY (id) + ); create table img_scan_job ( id int NOT NULL AUTO_INCREMENT, diff --git a/make/common/db/registry_sqlite.sql b/make/common/db/registry_sqlite.sql index 2528a845d..efe0f09fd 100644 --- a/make/common/db/registry_sqlite.sql +++ b/make/common/db/registry_sqlite.sql @@ -176,6 +176,16 @@ create table replication_job ( update_time timestamp default CURRENT_TIMESTAMP ); +create table replication_immediate_trigger ( + id INTEGER PRIMARY KEY, + policy_id int NOT NULL, + namespace varchar(256) NOT NULL, + on_push tinyint(1) NOT NULL DEFAULT 0, + on_deletion tinyint(1) NOT NULL DEFAULT 0, + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP + ); + create table img_scan_job ( id INTEGER PRIMARY KEY, diff --git a/src/common/dao/watch_item.go b/src/common/dao/watch_item.go new file mode 100644 index 000000000..dd16ae66b --- /dev/null +++ b/src/common/dao/watch_item.go @@ -0,0 +1,62 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "time" + + "github.com/vmware/harbor/src/common/models" +) + +// DefaultDatabaseWatchItemDAO is an instance of DatabaseWatchItemDAO +var DefaultDatabaseWatchItemDAO WatchItemDAO = &DatabaseWatchItemDAO{} + +// WatchItemDAO defines operations about WatchItem +type WatchItemDAO interface { + Add(*models.WatchItem) (int64, error) + DeleteByPolicyID(int64) error + Get(namespace, operation string) ([]models.WatchItem, error) +} + +// DatabaseWatchItemDAO implements interface WatchItemDAO for database +type DatabaseWatchItemDAO struct{} + +// Add a WatchItem +func (d *DatabaseWatchItemDAO) Add(item *models.WatchItem) (int64, error) { + now := time.Now() + item.CreationTime = now + item.UpdateTime = now + return GetOrmer().Insert(item) +} + +// DeleteByPolicyID deletes the WatchItem specified by policy ID +func (d *DatabaseWatchItemDAO) DeleteByPolicyID(policyID int64) error { + _, err := GetOrmer().QueryTable(&models.WatchItem{}).Filter("PolicyID", policyID).Delete() + return err +} + +// Get returns WatchItem list according to the namespace and operation +func (d *DatabaseWatchItemDAO) Get(namespace, operation string) ([]models.WatchItem, error) { + qs := GetOrmer().QueryTable(&models.WatchItem{}).Filter("Namespace", namespace) + if operation == "push" { + qs = qs.Filter("OnPush", 1) + } else if operation == "delete" { + qs = qs.Filter("OnDeletion", 1) + } + + items := []models.WatchItem{} + _, err := qs.All(&items) + return items, err +} diff --git a/src/common/dao/watch_item_test.go b/src/common/dao/watch_item_test.go new file mode 100644 index 000000000..b5b9b1b84 --- /dev/null +++ b/src/common/dao/watch_item_test.go @@ -0,0 +1,71 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmware/harbor/src/common/models" +) + +func TestMethodsOfWatchItem(t *testing.T) { + targetID, err := AddRepTarget(models.RepTarget{ + Name: "test_target_for_watch_item", + URL: "http://127.0.0.1", + }) + require.Nil(t, err) + defer DeleteRepTarget(targetID) + + policyID, err := AddRepPolicy(models.RepPolicy{ + Name: "test_policy_for_watch_item", + ProjectID: 1, + TargetID: targetID, + }) + require.Nil(t, err) + defer DeleteRepPolicy(policyID) + + item := &models.WatchItem{ + PolicyID: policyID, + Namespace: "library", + OnPush: false, + OnDeletion: true, + } + + // test Add + id, err := DefaultDatabaseWatchItemDAO.Add(item) + require.Nil(t, err) + + // test Get: operation-push + items, err := DefaultDatabaseWatchItemDAO.Get("library", "push") + require.Nil(t, err) + assert.Equal(t, 0, len(items)) + + // test Get: operation-delete + items, err = DefaultDatabaseWatchItemDAO.Get("library", "delete") + require.Nil(t, err) + assert.Equal(t, 1, len(items)) + assert.Equal(t, id, items[0].ID) + assert.Equal(t, "library", items[0].Namespace) + assert.True(t, items[0].OnDeletion) + + // test DeleteByPolicyID + err = DefaultDatabaseWatchItemDAO.DeleteByPolicyID(policyID) + require.Nil(t, err) + items, err = DefaultDatabaseWatchItemDAO.Get("library", "delete") + require.Nil(t, err) + assert.Equal(t, 0, len(items)) +} diff --git a/src/common/models/base.go b/src/common/models/base.go index f3cbfb935..202e24961 100644 --- a/src/common/models/base.go +++ b/src/common/models/base.go @@ -29,5 +29,6 @@ func init() { new(ScanJob), new(RepoRecord), new(ImgScanOverview), - new(ClairVulnTimestamp)) + new(ClairVulnTimestamp), + new(WatchItem)) } diff --git a/src/common/models/watch_item.go b/src/common/models/watch_item.go new file mode 100644 index 000000000..75f22dcbe --- /dev/null +++ b/src/common/models/watch_item.go @@ -0,0 +1,35 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "time" +) + +// WatchItem ... +type WatchItem struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + PolicyID int64 `orm:"column(policy_id)" json:"policy_id"` + Namespace string `orm:"column(namespace)" json:"namespace"` + OnDeletion bool `orm:"column(on_deletion)" json:"on_deletion"` + OnPush bool `orm:"column(on_push)" json:"on_push"` + CreationTime time.Time `orm:"column(creation_time)" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time)" json:"update_time"` +} + +//TableName ... +func (w *WatchItem) TableName() string { + return "replication_immediate_trigger" +} diff --git a/src/replication/consts.go b/src/replication/consts.go index 64800b994..b9a8a36a7 100644 --- a/src/replication/consts.go +++ b/src/replication/consts.go @@ -8,22 +8,16 @@ const ( //FilterItemKindTag : Kind of filter item is 'tag' FilterItemKindTag = "tag" - //TODO: Refactor constants - - //TriggerKindManually : kind of trigger is 'manully' - TriggerKindManually = "manually" - //TriggerKindImmediately : kind of trigger is 'immediately' - TriggerKindImmediately = "immediately" - //AdaptorKindHarbor : Kind of adaptor of Harbor AdaptorKindHarbor = "Harbor" //TriggerKindImmediate : Kind of trigger is 'Immediate' - TriggerKindImmediate = "Immediate" + TriggerKindImmediate = "immediate" //TriggerKindSchedule : Kind of trigger is 'Schedule' - TriggerKindSchedule = "Schedule" + TriggerKindSchedule = "schedule" //TriggerKindManual : Kind of trigger is 'Manual' - TriggerKindManual = "Manual" + TriggerKindManual = "manual" + //TriggerScheduleDaily : type of scheduling is 'daily' TriggerScheduleDaily = "daily" //TriggerScheduleWeekly : type of scheduling is 'weekly' diff --git a/src/replication/core/controller.go b/src/replication/core/controller.go index 5135510e7..ad2cf67ee 100644 --- a/src/replication/core/controller.go +++ b/src/replication/core/controller.go @@ -55,7 +55,6 @@ func (ctl *Controller) Init() error { //Build query parameters triggerNames := []string{ - replication.TriggerKindImmediate, replication.TriggerKindSchedule, } queryName := "" @@ -73,7 +72,7 @@ func (ctl *Controller) Init() error { } if policies != nil && len(policies) > 0 { for _, policy := range policies { - if err := ctl.triggerManager.SetupTrigger(policy.ID, *policy.Trigger); err != nil { + if err := ctl.triggerManager.SetupTrigger(&policy); err != nil { //TODO: Log error fmt.Printf("Error: %s", err) //TODO:Update the status of policy @@ -96,7 +95,8 @@ func (ctl *Controller) CreatePolicy(newPolicy models.ReplicationPolicy) (int64, return 0, err } - if err = ctl.triggerManager.SetupTrigger(id, *newPolicy.Trigger); err != nil { + newPolicy.ID = id + if err = ctl.triggerManager.SetupTrigger(&newPolicy); err != nil { return 0, err } @@ -118,15 +118,34 @@ func (ctl *Controller) UpdatePolicy(updatedPolicy models.ReplicationPolicy) erro return fmt.Errorf("policy %d not found", id) } - if err = ctl.triggerManager.UnsetTrigger(id, *originPolicy.Trigger); err != nil { - return err + reset := false + if updatedPolicy.Trigger.Kind != originPolicy.Trigger.Kind { + reset = true + } else { + switch updatedPolicy.Trigger.Kind { + case replication.TriggerKindSchedule: + if updatedPolicy.Trigger.Param != originPolicy.Trigger.Param { + reset = true + } + case replication.TriggerKindImmediate: + // Always reset immediate trigger as it is relevent with namespaces + reset = true + default: + // manual trigger, no need to reset + } } - if err = ctl.policyManager.UpdatePolicy(updatedPolicy); err != nil { - return err + if reset { + if err = ctl.triggerManager.UnsetTrigger(id, *originPolicy.Trigger); err != nil { + return err + } + if err = ctl.policyManager.UpdatePolicy(updatedPolicy); err != nil { + return err + } + return ctl.triggerManager.SetupTrigger(&updatedPolicy) } - return ctl.triggerManager.SetupTrigger(id, *updatedPolicy.Trigger) + return ctl.policyManager.UpdatePolicy(updatedPolicy) } //RemovePolicy will remove the specified policy and clean the related settings diff --git a/src/replication/models/policy.go b/src/replication/models/policy.go index faa0295b1..27c157a64 100644 --- a/src/replication/models/policy.go +++ b/src/replication/models/policy.go @@ -14,6 +14,7 @@ type ReplicationPolicy struct { Trigger *Trigger //The trigger of the replication ProjectIDs []int64 //Projects attached to this policy TargetIDs []int64 + Namespaces []string // The namespaces are used to set immediate trigger CreationTime time.Time UpdateTime time.Time } diff --git a/src/replication/models/trigger.go b/src/replication/models/trigger.go index 4477f6363..fa62166cb 100644 --- a/src/replication/models/trigger.go +++ b/src/replication/models/trigger.go @@ -18,8 +18,8 @@ type Trigger struct { // Valid ... func (t *Trigger) Valid(v *validation.Validation) { - if !(t.Kind == replication.TriggerKindImmediately || - t.Kind == replication.TriggerKindManually || + if !(t.Kind == replication.TriggerKindImmediate || + t.Kind == replication.TriggerKindManual || t.Kind == replication.TriggerKindSchedule) { v.SetError("kind", fmt.Sprintf("invalid trigger kind: %s", t.Kind)) } diff --git a/src/replication/trigger/immediate.go b/src/replication/trigger/immediate.go index d73c989b3..f10753a95 100644 --- a/src/replication/trigger/immediate.go +++ b/src/replication/trigger/immediate.go @@ -1,8 +1,6 @@ package trigger import ( - "errors" - "github.com/vmware/harbor/src/replication" ) @@ -26,19 +24,20 @@ func (st *ImmediateTrigger) Kind() string { //Setup is the implementation of same method defined in Trigger interface func (st *ImmediateTrigger) Setup() error { - if st.params.PolicyID <= 0 || len(st.params.Namespace) == 0 { - return errors.New("Invalid parameters for Immediate trigger") - } - //TODO: Need more complicated logic here to handle partial updates - wt := WatchItem{ - PolicyID: st.params.PolicyID, - Namespace: st.params.Namespace, - OnDeletion: st.params.OnDeletion, - OnPush: true, - } + for _, namespace := range st.params.Namespaces { + wt := WatchItem{ + PolicyID: st.params.PolicyID, + Namespace: namespace, + OnDeletion: st.params.OnDeletion, + OnPush: true, + } - return DefaultWatchList.Add(wt) + if err := DefaultWatchList.Add(wt); err != nil { + return err + } + } + return nil } //Unset is the implementation of same method defined in Trigger interface diff --git a/src/replication/trigger/manager.go b/src/replication/trigger/manager.go index ac1e08628..83969855c 100644 --- a/src/replication/trigger/manager.go +++ b/src/replication/trigger/manager.go @@ -2,7 +2,9 @@ package trigger import ( "errors" + "fmt" + "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/replication" "github.com/vmware/harbor/src/replication/models" ) @@ -50,25 +52,24 @@ func (m *Manager) RemoveTrigger(policyID int64) error { } */ -//SetupTrigger will create the new trigger based on the provided json parameters. +//SetupTrigger will create the new trigger based on the provided policy. //If failed, an error will be returned. -func (m *Manager) SetupTrigger(policyID int64, trigger models.Trigger) error { - if policyID <= 0 { - return errors.New("Invalid policy ID") - } - - if len(trigger.Kind) == 0 { - return errors.New("Invalid replication trigger definition") +func (m *Manager) SetupTrigger(policy *models.ReplicationPolicy) error { + if policy == nil || policy.Trigger == nil { + log.Debug("empty policy or trigger, skip trigger setup") + return nil } + trigger := policy.Trigger switch trigger.Kind { case replication.TriggerKindSchedule: param := ScheduleParam{} if err := param.Parse(trigger.Param); err != nil { return err } - //Append policy ID info - param.PolicyID = policyID + //Append policy ID and whether replicate deletion + param.PolicyID = policy.ID + param.OnDeletion = policy.ReplicateDeletion newTrigger := NewScheduleTrigger(param) if err := newTrigger.Setup(); err != nil { @@ -79,16 +80,19 @@ func (m *Manager) SetupTrigger(policyID int64, trigger models.Trigger) error { if err := param.Parse(trigger.Param); err != nil { return err } - //Append policy ID info - param.PolicyID = policyID + //Append policy ID and whether replicate deletion + param.PolicyID = policy.ID + param.OnDeletion = policy.ReplicateDeletion + param.Namespaces = policy.Namespaces newTrigger := NewImmediateTrigger(param) if err := newTrigger.Setup(); err != nil { return err } + case replication.TriggerKindManual: + // do nothing default: - //Treat as manual trigger - break + return fmt.Errorf("invalid trigger type: %s", policy.Trigger.Kind) } return nil diff --git a/src/replication/trigger/param_immediate.go b/src/replication/trigger/param_immediate.go index 492c65872..bb9b248a5 100644 --- a/src/replication/trigger/param_immediate.go +++ b/src/replication/trigger/param_immediate.go @@ -1,9 +1,5 @@ package trigger -import ( - "errors" -) - //NOTES: Whether replicate the existing images when the type of trigger is //'Immediate' is a once-effective setting which will not be persisted // and kept as one parameter of 'Immediate' trigger. It will only be @@ -14,13 +10,13 @@ type ImmediateParam struct { //Basic parameters BasicParam - //Namepace - Namespace string + //Namepaces + Namespaces []string } //Parse is the implementation of same method in TriggerParam interface //NOTES: No need to implement this method for 'Immediate' trigger as //it does not have any parameters with json format. func (ip ImmediateParam) Parse(param string) error { - return errors.New("Should NOT be called as it's not implemented") + return nil } diff --git a/src/replication/trigger/watch_list.go b/src/replication/trigger/watch_list.go index 76b3eb8bd..95f902179 100644 --- a/src/replication/trigger/watch_list.go +++ b/src/replication/trigger/watch_list.go @@ -1,5 +1,10 @@ package trigger +import ( + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" +) + //DefaultWatchList is the default instance of WatchList var DefaultWatchList = &WatchList{} @@ -24,15 +29,37 @@ type WatchItem struct { //Add item to the list and persist into DB func (wl *WatchList) Add(item WatchItem) error { - return nil + _, err := dao.DefaultDatabaseWatchItemDAO.Add( + &models.WatchItem{ + PolicyID: item.PolicyID, + Namespace: item.Namespace, + OnPush: item.OnPush, + OnDeletion: item.OnDeletion, + }) + return err } //Remove the specified watch item from list func (wl *WatchList) Remove(policyID int64) error { - return nil + return dao.DefaultDatabaseWatchItemDAO.DeleteByPolicyID(policyID) } //Get the watch items according to the namespace and operation func (wl *WatchList) Get(namespace, operation string) ([]WatchItem, error) { - return []WatchItem{}, nil + items, err := dao.DefaultDatabaseWatchItemDAO.Get(namespace, operation) + if err != nil { + return nil, err + } + + watchItems := []WatchItem{} + for _, item := range items { + watchItems = append(watchItems, WatchItem{ + PolicyID: item.PolicyID, + Namespace: item.Namespace, + OnPush: item.OnPush, + OnDeletion: item.OnDeletion, + }) + } + + return watchItems, nil } diff --git a/src/replication/trigger/watch_list_test.go b/src/replication/trigger/watch_list_test.go new file mode 100644 index 000000000..2e5c4e3a5 --- /dev/null +++ b/src/replication/trigger/watch_list_test.go @@ -0,0 +1,108 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package trigger + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" +) + +type fakeWatchItemDAO struct { + items []models.WatchItem +} + +func (f *fakeWatchItemDAO) Add(item *models.WatchItem) (int64, error) { + f.items = append(f.items, *item) + return int64(len(f.items) + 1), nil +} + +// Delete the WatchItem specified by policy ID +func (f *fakeWatchItemDAO) DeleteByPolicyID(policyID int64) error { + for i, item := range f.items { + if item.PolicyID == policyID { + f.items = append(f.items[:i], f.items[i+1:]...) + break + } + } + return nil +} + +// Get returns WatchItem list according to the namespace and operation +func (f *fakeWatchItemDAO) Get(namespace, operation string) ([]models.WatchItem, error) { + items := []models.WatchItem{} + for _, item := range f.items { + if item.Namespace != namespace { + continue + } + + if operation == "push" { + if item.OnPush { + items = append(items, item) + } + } + + if operation == "delete" { + if item.OnDeletion { + items = append(items, item) + } + } + } + + return items, nil +} + +func TestMethodsOfWatchList(t *testing.T) { + dao.DefaultDatabaseWatchItemDAO = &fakeWatchItemDAO{} + + var policyID int64 = 1 + + // test Add + item := WatchItem{ + PolicyID: policyID, + Namespace: "library", + OnDeletion: true, + OnPush: false, + } + + err := DefaultWatchList.Add(item) + require.Nil(t, err) + + // test Get: non-exist namespace + items, err := DefaultWatchList.Get("non-exist-namespace", "delete") + require.Nil(t, err) + assert.Equal(t, 0, len(items)) + + // test Get: non-exist operation + items, err = DefaultWatchList.Get("library", "non-exist-operation") + require.Nil(t, err) + assert.Equal(t, 0, len(items)) + + // test Get: valid params + items, err = DefaultWatchList.Get("library", "delete") + require.Nil(t, err) + assert.Equal(t, 1, len(items)) + assert.Equal(t, policyID, items[0].PolicyID) + + // test Remove + err = DefaultWatchList.Remove(policyID) + require.Nil(t, err) + items, err = DefaultWatchList.Get("library", "delete") + require.Nil(t, err) + assert.Equal(t, 0, len(items)) +} diff --git a/src/ui/api/replication_policy.go b/src/ui/api/replication_policy.go index 79b889b8b..955ed2fec 100644 --- a/src/ui/api/replication_policy.go +++ b/src/ui/api/replication_policy.go @@ -113,15 +113,16 @@ func (pa *RepPolicyAPI) Post() { // check the existence of projects for _, project := range policy.Projects { - exist, err := pa.ProjectMgr.Exists(project.ProjectID) + pro, err := pa.ProjectMgr.Get(project.ProjectID) if err != nil { pa.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %d", project.ProjectID), err) return } - if !exist { + if pro == nil { pa.HandleNotFound(fmt.Sprintf("project %d not found", project.ProjectID)) return } + project.Name = pro.Name } // check the existence of targets @@ -168,6 +169,34 @@ func (pa *RepPolicyAPI) Put() { policy.ID = id + // check the existence of projects + for _, project := range policy.Projects { + pro, err := pa.ProjectMgr.Get(project.ProjectID) + if err != nil { + pa.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %d", project.ProjectID), err) + return + } + if pro == nil { + pa.HandleNotFound(fmt.Sprintf("project %d not found", project.ProjectID)) + return + } + project.Name = pro.Name + } + + // check the existence of targets + for _, target := range policy.Targets { + t, err := dao.GetRepTarget(target.ID) + if err != nil { + pa.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", target.ID, err)) + return + } + + if t == nil { + pa.HandleNotFound(fmt.Sprintf("target %d not found", target.ID)) + return + } + } + if err = core.DefaultController.UpdatePolicy(convertToRepPolicy(policy)); err != nil { pa.HandleInternalServerError(fmt.Sprintf("failed to update policy %d: %v", id, err)) return @@ -274,6 +303,7 @@ func convertToRepPolicy(policy *api_models.ReplicationPolicy) rep_models.Replica for _, project := range policy.Projects { ply.ProjectIDs = append(ply.ProjectIDs, project.ProjectID) + ply.Namespaces = append(ply.Namespaces, project.Name) } for _, target := range policy.Targets { diff --git a/src/ui/api/replication_policy_test.go b/src/ui/api/replication_policy_test.go index aca8b1bc2..5e2b2f2f5 100644 --- a/src/ui/api/replication_policy_test.go +++ b/src/ui/api/replication_policy_test.go @@ -189,7 +189,7 @@ func TestRepPolicyAPIPost(t *testing.T) { }, }, Trigger: &rep_models.Trigger{ - Kind: replication.TriggerKindManually, + Kind: replication.TriggerKindManual, }, }, credential: sysAdmin, @@ -220,7 +220,7 @@ func TestRepPolicyAPIPost(t *testing.T) { }, }, Trigger: &rep_models.Trigger{ - Kind: replication.TriggerKindManually, + Kind: replication.TriggerKindManual, }, }, credential: sysAdmin, @@ -251,7 +251,7 @@ func TestRepPolicyAPIPost(t *testing.T) { }, }, Trigger: &rep_models.Trigger{ - Kind: replication.TriggerKindManually, + Kind: replication.TriggerKindManual, }, }, credential: sysAdmin, @@ -412,7 +412,7 @@ func TestRepPolicyAPIPut(t *testing.T) { }, }, Trigger: &rep_models.Trigger{ - Kind: replication.TriggerKindImmediately, + Kind: replication.TriggerKindImmediate, }, }, credential: sysAdmin, @@ -477,6 +477,7 @@ func TestConvertToRepPolicy(t *testing.T) { Projects: []*models.Project{ &models.Project{ ProjectID: 1, + Name: "library", }, }, Targets: []*models.RepTarget{ @@ -501,6 +502,7 @@ func TestConvertToRepPolicy(t *testing.T) { Param: "{param}", }, ProjectIDs: []int64{1}, + Namespaces: []string{"library"}, TargetIDs: []int64{1}, }, }, diff --git a/tools/migration/changelog.md b/tools/migration/changelog.md index d2e9a0b70..8cebdb593 100644 --- a/tools/migration/changelog.md +++ b/tools/migration/changelog.md @@ -61,3 +61,4 @@ Changelog for harbor database schema - add column `filters` to table `replication_policy` - add column `replicate_deletion` to table `replication_policy` + - create table `replication_immediate_trigger` \ No newline at end of file From 4902fdf2ef26431ca1cbe6e955cc4ab1a463e0d9 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Mon, 27 Nov 2017 16:12:10 +0800 Subject: [PATCH 15/37] Implement schedual trigger for replication --- src/replication/trigger/schedule.go | 29 +++++++++-- src/replication/trigger/schedule_test.go | 63 ++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 src/replication/trigger/schedule_test.go diff --git a/src/replication/trigger/schedule.go b/src/replication/trigger/schedule.go index d7c07cadd..3178b4d57 100644 --- a/src/replication/trigger/schedule.go +++ b/src/replication/trigger/schedule.go @@ -1,8 +1,11 @@ package trigger import ( - "errors" + "fmt" + "github.com/vmware/harbor/src/common/scheduler" + "github.com/vmware/harbor/src/common/scheduler/policy" + "github.com/vmware/harbor/src/common/scheduler/task" "github.com/vmware/harbor/src/replication" ) @@ -25,10 +28,30 @@ func (st *ScheduleTrigger) Kind() string { //Setup is the implementation of same method defined in Trigger interface func (st *ScheduleTrigger) Setup() error { - return errors.New("Not implemented") + config := &policy.AlternatePolicyConfiguration{} + switch st.params.Type { + case replication.TriggerScheduleDaily: + config.Duration = 24 * 3600 + config.OffsetTime = st.params.Offtime + case replication.TriggerScheduleWeekly: + config.Duration = 7 * 24 * 3600 + config.OffsetTime = st.params.Offtime + config.Weekday = st.params.Weekday + default: + return fmt.Errorf("unsupported schedual trigger type: %s", st.params.Type) + } + + schedulePolicy := policy.NewAlternatePolicy(assembleName(st.params.PolicyID), config) + attachTask := task.NewReplicationTask() + schedulePolicy.AttachTasks(attachTask) + return scheduler.DefaultScheduler.Schedule(schedulePolicy) } //Unset is the implementation of same method defined in Trigger interface func (st *ScheduleTrigger) Unset() error { - return errors.New("Not implemented") + return scheduler.DefaultScheduler.UnSchedule(assembleName(st.params.PolicyID)) +} + +func assembleName(policyID int64) string { + return fmt.Sprintf("replication_policy_%d", policyID) } diff --git a/src/replication/trigger/schedule_test.go b/src/replication/trigger/schedule_test.go new file mode 100644 index 000000000..5fecd934b --- /dev/null +++ b/src/replication/trigger/schedule_test.go @@ -0,0 +1,63 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package trigger + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmware/harbor/src/common/scheduler" + "github.com/vmware/harbor/src/replication" +) + +func TestAssembleName(t *testing.T) { + assert.Equal(t, "replication_policy_1", assembleName(1)) +} + +func TestKindOfScheduleTrigger(t *testing.T) { + trigger := NewScheduleTrigger(ScheduleParam{}) + assert.Equal(t, replication.TriggerKindSchedule, trigger.Kind()) +} + +func TestSetupAndUnSetOfScheduleTrigger(t *testing.T) { + // invalid schedule param + trigger := NewScheduleTrigger(ScheduleParam{}) + assert.NotNil(t, trigger.Setup()) + + // valid schedule param + var policyID int64 = 1 + trigger = NewScheduleTrigger(ScheduleParam{ + BasicParam: BasicParam{ + PolicyID: policyID, + }, + Type: replication.TriggerScheduleWeekly, + Weekday: (int8(time.Now().Weekday()) + 1) % 7, + Offtime: 0, + }) + + count := scheduler.DefaultScheduler.PolicyCount() + require.Nil(t, scheduler.DefaultScheduler.GetPolicy(assembleName(policyID))) + + require.Nil(t, trigger.Setup()) + + assert.Equal(t, count+1, scheduler.DefaultScheduler.PolicyCount()) + assert.NotNil(t, scheduler.DefaultScheduler.GetPolicy(assembleName(policyID))) + + require.Nil(t, trigger.Unset()) + assert.Equal(t, count, scheduler.DefaultScheduler.PolicyCount()) + assert.Nil(t, scheduler.DefaultScheduler.GetPolicy(assembleName(policyID))) +} From 594d2136307f01ebdfa3c2ed9ae27eaa670d965f Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Tue, 28 Nov 2017 14:38:24 +0800 Subject: [PATCH 16/37] Publish replication notification for manual, scheduel and immediate trigger --- src/common/notifier/init.go | 13 --- .../task/replication/replication_task.go | 31 +++++++ .../replication_task_test.go | 6 +- src/common/scheduler/task/replication_task.go | 24 ----- src/common/utils/test/watch_item.go | 65 +++++++++++++ src/replication/consts.go | 5 + src/replication/core/controller.go | 14 +-- src/replication/event/init.go | 39 ++++++++ .../event/notification/notification.go | 34 +++++++ src/replication/event/on_deletion_handler.go | 52 ++++------- .../event/on_deletion_handler_test.go | 43 +++++++++ src/replication/event/on_push_handler.go | 91 +++++++++++++------ src/replication/event/on_push_handler_test.go | 43 +++++++++ .../event/start_replication_handler.go | 31 ++++--- .../event/start_replication_handler_test.go | 41 +++++++++ src/replication/event/{ => topic}/topics.go | 2 +- src/replication/trigger/schedule.go | 4 +- src/replication/trigger/watch_list_test.go | 48 +--------- src/ui/api/harborapi_test.go | 1 + src/ui/api/replication.go | 11 ++- src/ui/api/repository.go | 15 ++- src/ui/api/utils.go | 37 -------- src/ui/main.go | 1 + .../service/notifications/registry/handler.go | 15 ++- 24 files changed, 458 insertions(+), 208 deletions(-) delete mode 100644 src/common/notifier/init.go create mode 100644 src/common/scheduler/task/replication/replication_task.go rename src/common/scheduler/task/{ => replication}/replication_task_test.go (54%) delete mode 100644 src/common/scheduler/task/replication_task.go create mode 100644 src/common/utils/test/watch_item.go create mode 100644 src/replication/event/init.go create mode 100644 src/replication/event/notification/notification.go create mode 100644 src/replication/event/on_deletion_handler_test.go create mode 100644 src/replication/event/on_push_handler_test.go create mode 100644 src/replication/event/start_replication_handler_test.go rename src/replication/event/{ => topic}/topics.go (95%) diff --git a/src/common/notifier/init.go b/src/common/notifier/init.go deleted file mode 100644 index b36d1f8de..000000000 --- a/src/common/notifier/init.go +++ /dev/null @@ -1,13 +0,0 @@ -package notifier - -import ( - "github.com/vmware/harbor/src/replication/event" -) - -//Subscribe related topics -func init() { - //Listen the related event topics - Subscribe(event.StartReplicationTopic, &event.StartReplicationHandler{}) - Subscribe(event.ReplicationEventTopicOnPush, &event.OnPushHandler{}) - Subscribe(event.ReplicationEventTopicOnDeletion, &event.OnDeletionHandler{}) -} diff --git a/src/common/scheduler/task/replication/replication_task.go b/src/common/scheduler/task/replication/replication_task.go new file mode 100644 index 000000000..1d4917420 --- /dev/null +++ b/src/common/scheduler/task/replication/replication_task.go @@ -0,0 +1,31 @@ +package replication + +import ( + "github.com/vmware/harbor/src/common/notifier" + "github.com/vmware/harbor/src/replication/event/notification" + "github.com/vmware/harbor/src/replication/event/topic" +) + +//Task is the task for triggering one replication +type Task struct { + PolicyID int64 +} + +//NewTask is constructor of creating ReplicationTask +func NewTask(policyID int64) *Task { + return &Task{ + PolicyID: policyID, + } +} + +//Name returns the name of this task +func (t *Task) Name() string { + return "replication" +} + +//Run the actions here +func (t *Task) Run() error { + return notifier.Publish(topic.StartReplicationTopic, notification.StartReplicationNotification{ + PolicyID: t.PolicyID, + }) +} diff --git a/src/common/scheduler/task/replication_task_test.go b/src/common/scheduler/task/replication/replication_task_test.go similarity index 54% rename from src/common/scheduler/task/replication_task_test.go rename to src/common/scheduler/task/replication/replication_task_test.go index cd0acd2c0..a914b46f9 100644 --- a/src/common/scheduler/task/replication_task_test.go +++ b/src/common/scheduler/task/replication/replication_task_test.go @@ -1,9 +1,9 @@ -package task +package replication import "testing" -func TestReplicationTask(t *testing.T) { - tk := NewReplicationTask() +func TestTask(t *testing.T) { + tk := NewTask(1) if tk == nil { t.Fail() } diff --git a/src/common/scheduler/task/replication_task.go b/src/common/scheduler/task/replication_task.go deleted file mode 100644 index aaefcc571..000000000 --- a/src/common/scheduler/task/replication_task.go +++ /dev/null @@ -1,24 +0,0 @@ -package task - -import ( - "errors" -) - -//ReplicationTask is the task for triggering one replication -type ReplicationTask struct{} - -//NewReplicationTask is constructor of creating ReplicationTask -func NewReplicationTask() *ReplicationTask { - return &ReplicationTask{} -} - -//Name returns the name of this task -func (rt *ReplicationTask) Name() string { - return "replication" -} - -//Run the actions here -func (rt *ReplicationTask) Run() error { - //Trigger the replication here - return errors.New("Not implemented") -} diff --git a/src/common/utils/test/watch_item.go b/src/common/utils/test/watch_item.go new file mode 100644 index 000000000..c801a9761 --- /dev/null +++ b/src/common/utils/test/watch_item.go @@ -0,0 +1,65 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "github.com/vmware/harbor/src/common/models" +) + +// FakeWatchItemDAO is the fake implement for the dao.WatchItemDAO +type FakeWatchItemDAO struct { + items []models.WatchItem +} + +// Add ... +func (f *FakeWatchItemDAO) Add(item *models.WatchItem) (int64, error) { + f.items = append(f.items, *item) + return int64(len(f.items) + 1), nil +} + +// DeleteByPolicyID : delete the WatchItem specified by policy ID +func (f *FakeWatchItemDAO) DeleteByPolicyID(policyID int64) error { + for i, item := range f.items { + if item.PolicyID == policyID { + f.items = append(f.items[:i], f.items[i+1:]...) + break + } + } + return nil +} + +// Get returns WatchItem list according to the namespace and operation +func (f *FakeWatchItemDAO) Get(namespace, operation string) ([]models.WatchItem, error) { + items := []models.WatchItem{} + for _, item := range f.items { + if item.Namespace != namespace { + continue + } + + if operation == "push" { + if item.OnPush { + items = append(items, item) + } + } + + if operation == "delete" { + if item.OnDeletion { + items = append(items, item) + } + } + } + + return items, nil +} diff --git a/src/replication/consts.go b/src/replication/consts.go index b9a8a36a7..813988d54 100644 --- a/src/replication/consts.go +++ b/src/replication/consts.go @@ -22,4 +22,9 @@ const ( TriggerScheduleDaily = "daily" //TriggerScheduleWeekly : type of scheduling is 'weekly' TriggerScheduleWeekly = "weekly" + + //OperationPush : operation for pushing images + OperationPush = "push" + //OperationDelete : operation for deleting images + OperationDelete = "delete" ) diff --git a/src/replication/core/controller.go b/src/replication/core/controller.go index ad2cf67ee..d06727380 100644 --- a/src/replication/core/controller.go +++ b/src/replication/core/controller.go @@ -135,17 +135,19 @@ func (ctl *Controller) UpdatePolicy(updatedPolicy models.ReplicationPolicy) erro } } + if err = ctl.policyManager.UpdatePolicy(updatedPolicy); err != nil { + return err + } + if reset { if err = ctl.triggerManager.UnsetTrigger(id, *originPolicy.Trigger); err != nil { return err } - if err = ctl.policyManager.UpdatePolicy(updatedPolicy); err != nil { - return err - } + return ctl.triggerManager.SetupTrigger(&updatedPolicy) } - return ctl.policyManager.UpdatePolicy(updatedPolicy) + return nil } //RemovePolicy will remove the specified policy and clean the related settings @@ -180,9 +182,9 @@ func (ctl *Controller) GetPolicies(query models.QueryParameter) ([]models.Replic //Replicate starts one replication defined in the specified policy; //Can be launched by the API layer and related triggers. -func (ctl *Controller) Replicate(policyID int64, item ...*models.FilterItem) error { +func (ctl *Controller) Replicate(policyID int64, metadate ...map[string]interface{}) error { - fmt.Printf("replicating %d ...\n", policyID) + fmt.Printf("replicating %d, metadata: %v ...\n", policyID, metadate) return nil } diff --git a/src/replication/event/init.go b/src/replication/event/init.go new file mode 100644 index 000000000..0f79739e3 --- /dev/null +++ b/src/replication/event/init.go @@ -0,0 +1,39 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package event + +import ( + "github.com/vmware/harbor/src/common/notifier" + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/replication/event/topic" +) + +//Subscribe related topics +func init() { + //Listen the related event topics + handlers := map[string]notifier.NotificationHandler{ + topic.StartReplicationTopic: &StartReplicationHandler{}, + topic.ReplicationEventTopicOnPush: &OnPushHandler{}, + topic.ReplicationEventTopicOnDeletion: &OnDeletionHandler{}, + } + + for topic, handler := range handlers { + if err := notifier.Subscribe(topic, handler); err != nil { + log.Errorf("failed to subscribe topic %s: %v", topic, err) + continue + } + log.Debugf("topic %s is subscribed", topic) + } +} diff --git a/src/replication/event/notification/notification.go b/src/replication/event/notification/notification.go new file mode 100644 index 000000000..fa2309b78 --- /dev/null +++ b/src/replication/event/notification/notification.go @@ -0,0 +1,34 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package notification + +//OnPushNotification contains the data required by this handler +type OnPushNotification struct { + //The name of the image that is being pushed + Image string +} + +//OnDeletionNotification contains the data required by this handler +type OnDeletionNotification struct { + //The name of the image that is being deleted + Image string +} + +//StartReplicationNotification contains data required by this handler +type StartReplicationNotification struct { + //ID of the policy + PolicyID int64 + Metadata map[string]interface{} +} diff --git a/src/replication/event/on_deletion_handler.go b/src/replication/event/on_deletion_handler.go index e16af893c..356b34d25 100644 --- a/src/replication/event/on_deletion_handler.go +++ b/src/replication/event/on_deletion_handler.go @@ -1,3 +1,17 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package event import ( @@ -5,19 +19,13 @@ import ( "fmt" "reflect" - "github.com/vmware/harbor/src/replication/core" - "github.com/vmware/harbor/src/replication/models" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/event/notification" ) //OnDeletionHandler implements the notification handler interface to handle image on push event. type OnDeletionHandler struct{} -//OnDeletionNotification contains the data required by this handler -type OnDeletionNotification struct { - //The name of the project where the being pushed images are located - ProjectName string -} - //Handle implements the same method of notification handler interface func (oph *OnDeletionHandler) Handle(value interface{}) error { if value == nil { @@ -25,32 +33,12 @@ func (oph *OnDeletionHandler) Handle(value interface{}) error { } vType := reflect.TypeOf(value) - if vType.Kind() != reflect.Struct || vType.String() != "event.OnDeletionNotification" { - return fmt.Errorf("Mismatch value type of OnDeletionHandler, expect %s but got %s", "event.OnDeletionNotification", vType.String()) + if vType.Kind() != reflect.Struct || vType.String() != "notification.OnDeletionNotification" { + return fmt.Errorf("Mismatch value type of OnDeletionHandler, expect %s but got %s", "notification.OnDeletionNotification", vType.String()) } - notification := value.(OnDeletionNotification) - //TODO:Call projectManager to get the projectID - fmt.Println(notification.ProjectName) - query := models.QueryParameter{ - ProjectID: 0, - } - - policies, err := core.DefaultController.GetPolicies(query) - if err != nil { - return err - } - if policies != nil && len(policies) > 0 { - for _, p := range policies { - //Error accumulated and then return? - if err := core.DefaultController.Replicate(p.ID); err != nil { - //TODO:Log error - fmt.Println(err.Error()) - } - } - } - - return nil + notification := value.(notification.OnDeletionNotification) + return checkAndTriggerReplication(notification.Image, replication.OperationDelete) } //IsStateful implements the same method of notification handler interface diff --git a/src/replication/event/on_deletion_handler_test.go b/src/replication/event/on_deletion_handler_test.go new file mode 100644 index 000000000..e34792f07 --- /dev/null +++ b/src/replication/event/on_deletion_handler_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package event + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/utils/test" + "github.com/vmware/harbor/src/replication/event/notification" +) + +func TestHandleOfOnDeletionHandler(t *testing.T) { + dao.DefaultDatabaseWatchItemDAO = &test.FakeWatchItemDAO{} + + handler := &OnDeletionHandler{} + + assert.NotNil(t, handler.Handle(nil)) + assert.NotNil(t, handler.Handle(map[string]string{})) + assert.NotNil(t, handler.Handle(struct{}{})) + + assert.Nil(t, handler.Handle(notification.OnDeletionNotification{ + Image: "library/hello-world:latest", + })) +} + +func TestIsStatefulOfOnDeletionHandler(t *testing.T) { + handler := &OnDeletionHandler{} + assert.False(t, handler.IsStateful()) +} diff --git a/src/replication/event/on_push_handler.go b/src/replication/event/on_push_handler.go index d8c4d3aae..306a52341 100644 --- a/src/replication/event/on_push_handler.go +++ b/src/replication/event/on_push_handler.go @@ -1,3 +1,17 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package event import ( @@ -5,19 +19,19 @@ import ( "fmt" "reflect" - "github.com/vmware/harbor/src/replication/core" + "github.com/vmware/harbor/src/common/notifier" + "github.com/vmware/harbor/src/common/utils" + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/event/notification" + "github.com/vmware/harbor/src/replication/event/topic" "github.com/vmware/harbor/src/replication/models" + "github.com/vmware/harbor/src/replication/trigger" ) //OnPushHandler implements the notification handler interface to handle image on push event. type OnPushHandler struct{} -//OnPushNotification contains the data required by this handler -type OnPushNotification struct { - //The ID of the project where the being pushed images are located - ProjectID int -} - //Handle implements the same method of notification handler interface func (oph *OnPushHandler) Handle(value interface{}) error { if value == nil { @@ -25,31 +39,13 @@ func (oph *OnPushHandler) Handle(value interface{}) error { } vType := reflect.TypeOf(value) - if vType.Kind() != reflect.Struct || vType.String() != "event.OnPushNotification" { - return fmt.Errorf("Mismatch value type of OnPushHandler, expect %s but got %s", "event.OnPushNotification", vType.String()) + if vType.Kind() != reflect.Struct || vType.String() != "notification.OnPushNotification" { + return fmt.Errorf("Mismatch value type of OnPushHandler, expect %s but got %s", "notification.OnPushNotification", vType.String()) } - notification := value.(OnDeletionNotification) - //TODO:Call projectManager to get the projectID - fmt.Println(notification.ProjectName) - query := models.QueryParameter{ - ProjectID: 0, - } + notification := value.(notification.OnPushNotification) - policies, err := core.DefaultController.GetPolicies(query) - if err != nil { - return err - } - if policies != nil && len(policies) > 0 { - for _, p := range policies { - if err := core.DefaultController.Replicate(p.ID); err != nil { - //TODO:Log error - fmt.Println(err.Error()) - } - } - } - - return nil + return checkAndTriggerReplication(notification.Image, replication.OperationPush) } //IsStateful implements the same method of notification handler interface @@ -57,3 +53,40 @@ func (oph *OnPushHandler) IsStateful() bool { //Statless return false } + +// checks whether replication policy is set on the resource, if is, trigger the replication +func checkAndTriggerReplication(image, operation string) error { + project, _ := utils.ParseRepository(image) + watchItems, err := trigger.DefaultWatchList.Get(project, operation) + if err != nil { + return fmt.Errorf("failed to get watch list for resource %s, operation %s: %v", + image, operation, err) + } + if len(watchItems) == 0 { + log.Debugf("no replication should be triggered for resource %s, operation %s, skip", image, operation) + return nil + } + + for _, watchItem := range watchItems { + item := &models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: image, + Metadata: map[string]interface{}{ + "operation": operation, + }, + } + + if err := notifier.Publish(topic.StartReplicationTopic, notification.StartReplicationNotification{ + PolicyID: watchItem.PolicyID, + Metadata: map[string]interface{}{ + "": []*models.FilterItem{item}, + }, + }); err != nil { + return fmt.Errorf("failed to publish replication topic for resource %s, operation %s, policy %d: %v", + image, operation, watchItem.PolicyID, err) + } + log.Infof("replication topic for resource %s, operation %s, policy %d triggered", + image, operation, watchItem.PolicyID) + } + return nil +} diff --git a/src/replication/event/on_push_handler_test.go b/src/replication/event/on_push_handler_test.go new file mode 100644 index 000000000..bcd57605c --- /dev/null +++ b/src/replication/event/on_push_handler_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package event + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/utils/test" + "github.com/vmware/harbor/src/replication/event/notification" +) + +func TestHandleOfOnPushHandler(t *testing.T) { + dao.DefaultDatabaseWatchItemDAO = &test.FakeWatchItemDAO{} + + handler := &OnPushHandler{} + + assert.NotNil(t, handler.Handle(nil)) + assert.NotNil(t, handler.Handle(map[string]string{})) + assert.NotNil(t, handler.Handle(struct{}{})) + + assert.Nil(t, handler.Handle(notification.OnPushNotification{ + Image: "library/hello-world:latest", + })) +} + +func TestIsStatefulOfOnPushHandler(t *testing.T) { + handler := &OnPushHandler{} + assert.False(t, handler.IsStateful()) +} diff --git a/src/replication/event/start_replication_handler.go b/src/replication/event/start_replication_handler.go index 93658a593..82b8bce40 100644 --- a/src/replication/event/start_replication_handler.go +++ b/src/replication/event/start_replication_handler.go @@ -1,3 +1,17 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package event import ( @@ -6,17 +20,12 @@ import ( "reflect" "github.com/vmware/harbor/src/replication/core" + "github.com/vmware/harbor/src/replication/event/notification" ) //StartReplicationHandler implements the notification handler interface to handle start replication requests. type StartReplicationHandler struct{} -//StartReplicationNotification contains data required by this handler -type StartReplicationNotification struct { - //ID of the policy - PolicyID int64 -} - //Handle implements the same method of notification handler interface func (srh *StartReplicationHandler) Handle(value interface{}) error { if value == nil { @@ -24,18 +33,18 @@ func (srh *StartReplicationHandler) Handle(value interface{}) error { } vType := reflect.TypeOf(value) - if vType.Kind() != reflect.Struct || vType.String() != "core.StartReplicationNotification" { - return fmt.Errorf("Mismatch value type of StartReplicationHandler, expect %s but got %s", "core.StartReplicationNotification", vType.String()) + if vType.Kind() != reflect.Struct || vType.String() != "notification.StartReplicationNotification" { + return fmt.Errorf("Mismatch value type of StartReplicationHandler, expect %s but got %s", "notification.StartReplicationNotification", vType.String()) } - notification := value.(StartReplicationNotification) + notification := value.(notification.StartReplicationNotification) if notification.PolicyID <= 0 { return errors.New("Invalid policy") } //Start replication - //TODO: - return core.DefaultController.Replicate(notification.PolicyID) + + return core.DefaultController.Replicate(notification.PolicyID, notification.Metadata) } //IsStateful implements the same method of notification handler interface diff --git a/src/replication/event/start_replication_handler_test.go b/src/replication/event/start_replication_handler_test.go new file mode 100644 index 000000000..30404b270 --- /dev/null +++ b/src/replication/event/start_replication_handler_test.go @@ -0,0 +1,41 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package event + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/replication/event/notification" +) + +func TestHandle(t *testing.T) { + handler := &StartReplicationHandler{} + + assert.NotNil(t, handler.Handle(nil)) + assert.NotNil(t, handler.Handle(map[string]string{})) + assert.NotNil(t, handler.Handle(struct{}{})) + assert.NotNil(t, handler.Handle(notification.StartReplicationNotification{ + PolicyID: -1, + })) + assert.Nil(t, handler.Handle(notification.StartReplicationNotification{ + PolicyID: 1, + })) +} + +func TestIsStateful(t *testing.T) { + handler := &StartReplicationHandler{} + assert.False(t, handler.IsStateful()) +} diff --git a/src/replication/event/topics.go b/src/replication/event/topic/topics.go similarity index 95% rename from src/replication/event/topics.go rename to src/replication/event/topic/topics.go index 5a6aa954e..fce3b9c81 100644 --- a/src/replication/event/topics.go +++ b/src/replication/event/topic/topics.go @@ -1,4 +1,4 @@ -package event +package topic const ( //ReplicationEventTopicOnPush : OnPush event diff --git a/src/replication/trigger/schedule.go b/src/replication/trigger/schedule.go index 3178b4d57..3a8480919 100644 --- a/src/replication/trigger/schedule.go +++ b/src/replication/trigger/schedule.go @@ -5,7 +5,7 @@ import ( "github.com/vmware/harbor/src/common/scheduler" "github.com/vmware/harbor/src/common/scheduler/policy" - "github.com/vmware/harbor/src/common/scheduler/task" + replication_task "github.com/vmware/harbor/src/common/scheduler/task/replication" "github.com/vmware/harbor/src/replication" ) @@ -42,7 +42,7 @@ func (st *ScheduleTrigger) Setup() error { } schedulePolicy := policy.NewAlternatePolicy(assembleName(st.params.PolicyID), config) - attachTask := task.NewReplicationTask() + attachTask := replication_task.NewTask(st.params.PolicyID) schedulePolicy.AttachTasks(attachTask) return scheduler.DefaultScheduler.Schedule(schedulePolicy) } diff --git a/src/replication/trigger/watch_list_test.go b/src/replication/trigger/watch_list_test.go index 2e5c4e3a5..e8f1b9aed 100644 --- a/src/replication/trigger/watch_list_test.go +++ b/src/replication/trigger/watch_list_test.go @@ -20,55 +20,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware/harbor/src/common/dao" - "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/test" ) -type fakeWatchItemDAO struct { - items []models.WatchItem -} - -func (f *fakeWatchItemDAO) Add(item *models.WatchItem) (int64, error) { - f.items = append(f.items, *item) - return int64(len(f.items) + 1), nil -} - -// Delete the WatchItem specified by policy ID -func (f *fakeWatchItemDAO) DeleteByPolicyID(policyID int64) error { - for i, item := range f.items { - if item.PolicyID == policyID { - f.items = append(f.items[:i], f.items[i+1:]...) - break - } - } - return nil -} - -// Get returns WatchItem list according to the namespace and operation -func (f *fakeWatchItemDAO) Get(namespace, operation string) ([]models.WatchItem, error) { - items := []models.WatchItem{} - for _, item := range f.items { - if item.Namespace != namespace { - continue - } - - if operation == "push" { - if item.OnPush { - items = append(items, item) - } - } - - if operation == "delete" { - if item.OnDeletion { - items = append(items, item) - } - } - } - - return items, nil -} - func TestMethodsOfWatchList(t *testing.T) { - dao.DefaultDatabaseWatchItemDAO = &fakeWatchItemDAO{} + dao.DefaultDatabaseWatchItemDAO = &test.FakeWatchItemDAO{} var policyID int64 = 1 diff --git a/src/ui/api/harborapi_test.go b/src/ui/api/harborapi_test.go index a62e11816..d521f6883 100644 --- a/src/ui/api/harborapi_test.go +++ b/src/ui/api/harborapi_test.go @@ -40,6 +40,7 @@ import ( "github.com/dghubble/sling" //for test env prepare + _ "github.com/vmware/harbor/src/replication/event" _ "github.com/vmware/harbor/src/ui/auth/db" _ "github.com/vmware/harbor/src/ui/auth/ldap" ) diff --git a/src/ui/api/replication.go b/src/ui/api/replication.go index 49e1b80dc..43bd2289c 100644 --- a/src/ui/api/replication.go +++ b/src/ui/api/replication.go @@ -17,7 +17,11 @@ package api import ( "fmt" + "github.com/vmware/harbor/src/common/notifier" + "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/replication/core" + "github.com/vmware/harbor/src/replication/event/notification" + "github.com/vmware/harbor/src/replication/event/topic" "github.com/vmware/harbor/src/ui/api/models" ) @@ -56,8 +60,11 @@ func (r *ReplicationAPI) Post() { return } - if err = core.DefaultController.Replicate(replication.PolicyID); err != nil { - r.HandleInternalServerError(fmt.Sprintf("failed to trigger the replication policy %d: %v", replication.PolicyID, err)) + if err = notifier.Publish(topic.StartReplicationTopic, notification.StartReplicationNotification{ + PolicyID: replication.PolicyID, + }); err != nil { + r.HandleInternalServerError(fmt.Sprintf("failed to publish replication topic for policy %d: %v", replication.PolicyID, err)) return } + log.Infof("replication topic for policy %d triggered", replication.PolicyID) } diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index 958a6f5ef..4a56e379d 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -26,12 +26,15 @@ import ( "github.com/docker/distribution/manifest/schema2" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/notifier" "github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/clair" registry_error "github.com/vmware/harbor/src/common/utils/error" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/common/utils/notary" "github.com/vmware/harbor/src/common/utils/registry" + "github.com/vmware/harbor/src/replication/event/notification" + "github.com/vmware/harbor/src/replication/event/topic" "github.com/vmware/harbor/src/ui/config" uiutils "github.com/vmware/harbor/src/ui/utils" ) @@ -255,7 +258,17 @@ func (ra *RepositoryAPI) Delete() { } log.Infof("delete tag: %s:%s", repoName, t) - go CheckAndTriggerReplication(repoName+":"+t, "delete") + go func() { + image := repoName + ":" + t + err := notifier.Publish(topic.ReplicationEventTopicOnDeletion, notification.OnDeletionNotification{ + Image: image, + }) + if err != nil { + log.Errorf("failed to publish on deletion topic for resource %s: %v", image, err) + return + } + log.Debugf("the on deletion topic for resource %s published", image) + }() go func(tag string) { if err := dao.AddAccessLog(models.AccessLog{ diff --git a/src/ui/api/utils.go b/src/ui/api/utils.go index b66990155..5ef259aeb 100644 --- a/src/ui/api/utils.go +++ b/src/ui/api/utils.go @@ -32,10 +32,6 @@ import ( "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/common/utils/registry" "github.com/vmware/harbor/src/common/utils/registry/auth" - "github.com/vmware/harbor/src/replication" - "github.com/vmware/harbor/src/replication/core" - rep_models "github.com/vmware/harbor/src/replication/models" - "github.com/vmware/harbor/src/replication/trigger" "github.com/vmware/harbor/src/ui/config" "github.com/vmware/harbor/src/ui/promgr" "github.com/vmware/harbor/src/ui/service/token" @@ -81,39 +77,6 @@ func checkUserExists(name string) int { return 0 } -// CheckAndTriggerReplication checks whether replication policy is set -// on the resource, if is, trigger the replication -func CheckAndTriggerReplication(image, operation string) { - project, _ := utils.ParseRepository(image) - watchItems, err := trigger.DefaultWatchList.Get(project, operation) - if err != nil { - log.Errorf("failed to get watch list for resource %s, operation %s: %v", image, operation, err) - return - } - if len(watchItems) == 0 { - log.Debugf("no replication should be triggered for resource %s, operation %s, skip", image, operation) - return - } - - for _, watchItem := range watchItems { - // TODO define a new type ReplicationItem to wrap FilterItem and operation. - // Maybe change the FilterItem to interface and define a type Resource to - // implement FilterItem is better? - item := &rep_models.FilterItem{ - Kind: replication.FilterItemKindTag, - Value: image, - Metadata: map[string]interface{}{ - "operation": operation, - }, - } - if err := core.DefaultController.Replicate(watchItem.PolicyID, item); err != nil { - log.Errorf("failed to trigger replication for resource: %s, operation: %s: %v", image, operation, err) - return - } - log.Infof("replication for resource: %s, operation: %s triggered", image, operation) - } -} - // TriggerReplication triggers the replication according to the policy // TODO remove func TriggerReplication(policyID int64, repository string, diff --git a/src/ui/main.go b/src/ui/main.go index a9f7fb859..26f7ff5a4 100644 --- a/src/ui/main.go +++ b/src/ui/main.go @@ -29,6 +29,7 @@ import ( "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/notifier" "github.com/vmware/harbor/src/common/scheduler" + _ "github.com/vmware/harbor/src/replication/event" "github.com/vmware/harbor/src/ui/api" _ "github.com/vmware/harbor/src/ui/auth/db" _ "github.com/vmware/harbor/src/ui/auth/ldap" diff --git a/src/ui/service/notifications/registry/handler.go b/src/ui/service/notifications/registry/handler.go index 6cfc2433d..750bbf47b 100644 --- a/src/ui/service/notifications/registry/handler.go +++ b/src/ui/service/notifications/registry/handler.go @@ -23,8 +23,11 @@ import ( "github.com/vmware/harbor/src/common/dao" clairdao "github.com/vmware/harbor/src/common/dao/clair" "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/notifier" "github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" + rep_notification "github.com/vmware/harbor/src/replication/event/notification" + "github.com/vmware/harbor/src/replication/event/topic" "github.com/vmware/harbor/src/ui/api" "github.com/vmware/harbor/src/ui/config" uiutils "github.com/vmware/harbor/src/ui/utils" @@ -104,7 +107,17 @@ func (n *NotificationHandler) Post() { } }() - go api.CheckAndTriggerReplication(repository+":"+tag, "push") + go func() { + image := repository + ":" + tag + err := notifier.Publish(topic.ReplicationEventTopicOnPush, rep_notification.OnPushNotification{ + Image: image, + }) + if err != nil { + log.Errorf("failed to publish on push topic for resource %s: %v", image, err) + return + } + log.Debugf("the on push topic for resource %s published", image) + }() if autoScanEnabled(pro) { last, err := clairdao.GetLastUpdate() From 2ba3758472e0cf88e3367446ba92d0cbcad396f7 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Mon, 4 Dec 2017 15:39:40 +0800 Subject: [PATCH 17/37] Update the replicatoin APIs in swagger.yaml --- docs/swagger.yaml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 9120fc964..af2356bcf 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2375,20 +2375,17 @@ definitions: type: string description: The description of the policy. projects: - type: object + type: array description: The project list that the policy applys to. items: $ref: '#/definitions/Project' targets: - type: object + type: array description: The target list. items: $ref: '#/definitions/RepTarget' trigger: - type: object - description: The trigger for schedule job. - items: - $ref: '#/definitions/RepTrigger' + $ref: '#/definitions/RepTrigger' filters: type: array description: The replication policy filter array. @@ -2407,14 +2404,14 @@ definitions: type: string description: The update time of the policy. error_job_count: - format: int + type: integer description: The error job count number for the policy. RepTrigger: type: object properties: kind: type: string - description: The replication policy trigger kind. + description: The replication policy trigger kind. The valid values are manual, immediate and schedule. param: type: string description: The replication policy trigger parameters. @@ -2423,10 +2420,10 @@ definitions: properties: kind: type: string - description: The replication policy filter kind. - value: + description: The replication policy filter kind. The valid values are project, repository and tag. + pattern: type: string - description: The replication policy filter value. + description: The replication policy filter pattern. metadata: type: object description: This map object is the replication policy filter metadata. From a384325a1ea19b77fb2bb4ee31e7c513de4aca2b Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Tue, 28 Nov 2017 14:38:24 +0800 Subject: [PATCH 18/37] Publish replication notification for manual, scheduel and immediate trigger --- src/ui/api/utils.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/ui/api/utils.go b/src/ui/api/utils.go index 5ef259aeb..76a0cabf5 100644 --- a/src/ui/api/utils.go +++ b/src/ui/api/utils.go @@ -77,6 +77,50 @@ func checkUserExists(name string) int { return 0 } +<<<<<<< HEAD +======= +// CheckAndTriggerReplication checks whether replication policy is set +// on the resource, if is, trigger the replication +func CheckAndTriggerReplication(image, operation string) { + project, _ := utils.ParseRepository(image) + watchItems, err := trigger.DefaultWatchList.Get(project, operation) + if err != nil { + log.Errorf("failed to get watch list for resource %s, operation %s: %v", image, operation, err) + return + } + if len(watchItems) == 0 { + log.Debugf("no replication should be triggered for resource %s, operation %s, skip", image, operation) + return + } + + for _, watchItem := range watchItems { + // TODO define a new type ReplicationItem to wrap FilterItem and operation. + // Maybe change the FilterItem to interface and define a type Resource to + // implement FilterItem is better? + item := &rep_models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: image, + Metadata: map[string]interface{}{ + "operation": operation, + }, + } + + if err := notifier.Publish(topic.StartReplicationTopic, notification.StartReplicationNotification{ + PolicyID: watchItem.PolicyID, + Metadata: map[string]interface{}{ + "": []*rep_models.FilterItem{item}, + }, + }); err != nil { + log.Errorf("failed to publish replication topic for resource %s, operation %s, policy %d: %v", + image, operation, watchItem.PolicyID, err) + return + } + log.Infof("replication topic for resource %s, operation %s, policy %d triggered", + image, operation, watchItem.PolicyID) + } +} + +>>>>>>> 3409fa1... Publish replication notification for manual, scheduel and immediate trigger // TriggerReplication triggers the replication according to the policy // TODO remove func TriggerReplication(policyID int64, repository string, From c5ccb7e53c286b7c21f32f2c8e5201c86385aa80 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 29 Nov 2017 15:14:13 +0800 Subject: [PATCH 19/37] Enable filter chain in replication --- src/replication/consts.go | 7 +- src/replication/core/controller.go | 111 ++++++++++++++++- src/replication/models/filter.go | 41 ++++++ src/replication/models/filter_item.go | 2 + src/replication/models/policy.go | 2 +- src/replication/policy/manager.go | 9 +- src/replication/registry/harbor_adaptor.go | 40 +++++- .../source/repository_convertor.go | 11 +- .../source/tag_combination_filter.go | 1 + src/replication/source/tag_convertor.go | 9 +- src/ui/api/models/replication_policy.go | 2 +- src/ui/api/utils.go | 117 ------------------ src/ui/main.go | 5 + 13 files changed, 219 insertions(+), 138 deletions(-) create mode 100644 src/replication/models/filter.go diff --git a/src/replication/consts.go b/src/replication/consts.go index 813988d54..a2148823b 100644 --- a/src/replication/consts.go +++ b/src/replication/consts.go @@ -23,8 +23,11 @@ const ( //TriggerScheduleWeekly : type of scheduling is 'weekly' TriggerScheduleWeekly = "weekly" - //OperationPush : operation for pushing images + //OperationPush : push operation OperationPush = "push" - //OperationDelete : operation for deleting images + //OperationDelete : delete operation OperationDelete = "delete" + + // PatternMatchAll : the pattern that match all + PatternMatchAll = ".*" ) diff --git a/src/replication/core/controller.go b/src/replication/core/controller.go index d06727380..1bcc094a8 100644 --- a/src/replication/core/controller.go +++ b/src/replication/core/controller.go @@ -3,6 +3,9 @@ package core import ( "fmt" + "github.com/vmware/harbor/src/common/dao" + common_models "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/replication" "github.com/vmware/harbor/src/replication/models" "github.com/vmware/harbor/src/replication/policy" @@ -182,9 +185,113 @@ func (ctl *Controller) GetPolicies(query models.QueryParameter) ([]models.Replic //Replicate starts one replication defined in the specified policy; //Can be launched by the API layer and related triggers. -func (ctl *Controller) Replicate(policyID int64, metadate ...map[string]interface{}) error { +func (ctl *Controller) Replicate(policyID int64, metadata ...map[string]interface{}) error { + policy, err := ctl.GetPolicy(policyID) + if err != nil { + return err + } + if policy.ID == 0 { + return fmt.Errorf("policy %d not found", policyID) + } - fmt.Printf("replicating %d, metadata: %v ...\n", policyID, metadate) + candidates := []models.FilterItem{} + if len(metadata) > 0 { + meta := metadata[0]["candidates"] + if meta != nil { + cands, ok := meta.([]models.FilterItem) + if ok { + candidates = append(candidates, cands...) + } + } + } + + // prepare candidates for replication + candidates = getCandidates(&policy, ctl.sourcer, candidates...) + + targets := []*common_models.RepTarget{} + for _, targetID := range policy.TargetIDs { + target, err := dao.GetRepTarget(targetID) + if err != nil { + return err + } + targets = append(targets, target) + } + + // TODO merge tags whose repository is same into one struct + + // call job service to do the replication + return replicate(candidates, targets) +} + +func getCandidates(policy *models.ReplicationPolicy, sourcer *source.Sourcer, candidates ...models.FilterItem) []models.FilterItem { + if len(candidates) == 0 { + for _, namespace := range policy.Namespaces { + candidates = append(candidates, models.FilterItem{ + Kind: replication.FilterItemKindProject, + Value: namespace, + Operation: replication.OperationPush, + }) + } + } + + filterChain := buildFilterChain(policy, sourcer) + + return filterChain.DoFilter(candidates) +} + +func buildFilterChain(policy *models.ReplicationPolicy, sourcer *source.Sourcer) source.FilterChain { + filters := []source.Filter{} + + patternMap := map[string]string{} + for _, f := range policy.Filters { + patternMap[f.Kind] = f.Pattern + } + + // TODO convert wildcard to regex expression + projectPattern, exist := patternMap[replication.FilterItemKindProject] + if !exist { + projectPattern = replication.PatternMatchAll + } + + repositoryPattern, exist := patternMap[replication.FilterItemKindRepository] + if !exist { + repositoryPattern = replication.PatternMatchAll + } + repositoryPattern = fmt.Sprintf("%s/%s", projectPattern, repositoryPattern) + + tagPattern, exist := patternMap[replication.FilterItemKindProject] + if !exist { + tagPattern = replication.PatternMatchAll + } + tagPattern = fmt.Sprintf("%s:%s", repositoryPattern, tagPattern) + + if policy.Trigger.Kind == replication.TriggerKindImmediate { + // build filter chain for immediate trigger policy + filters = append(filters, + source.NewPatternFilter(replication.FilterItemKindTag, tagPattern)) + } else { + // build filter chain for manual and schedule trigger policy + + // append project filter + filters = append(filters, + source.NewPatternFilter(replication.FilterItemKindProject, projectPattern)) + // append repository filter + filters = append(filters, + source.NewPatternFilter(replication.FilterItemKindRepository, + repositoryPattern, source.NewRepositoryConvertor(sourcer.GetAdaptor(replication.AdaptorKindHarbor)))) + // append tag filter + filters = append(filters, + source.NewPatternFilter(replication.FilterItemKindTag, + tagPattern, source.NewTagConvertor(sourcer.GetAdaptor(replication.AdaptorKindHarbor)))) + } + + return source.NewDefaultFilterChain(filters) +} + +func replicate(candidates []models.FilterItem, targets []*common_models.RepTarget) error { + // TODO + + log.Infof("replicate candidates %v to targets %v", candidates, targets) return nil } diff --git a/src/replication/models/filter.go b/src/replication/models/filter.go new file mode 100644 index 000000000..648d6247c --- /dev/null +++ b/src/replication/models/filter.go @@ -0,0 +1,41 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "fmt" + + "github.com/astaxie/beego/validation" + "github.com/vmware/harbor/src/replication" +) + +// Filter is the data model represents the filter defined by user. +type Filter struct { + Kind string `json:"kind"` + Pattern string `json:"pattern"` +} + +// Valid ... +func (f *Filter) Valid(v *validation.Validation) { + if !(f.Kind == replication.FilterItemKindProject || + f.Kind == replication.FilterItemKindRepository || + f.Kind == replication.FilterItemKindTag) { + v.SetError("kind", fmt.Sprintf("invalid filter kind: %s", f.Kind)) + } + + if len(f.Pattern) == 0 { + v.SetError("pattern", "filter pattern can not be empty") + } +} diff --git a/src/replication/models/filter_item.go b/src/replication/models/filter_item.go index d142fb70f..101fd6438 100644 --- a/src/replication/models/filter_item.go +++ b/src/replication/models/filter_item.go @@ -20,6 +20,8 @@ type FilterItem struct { //kind == 'tag', value will be tag name. Value string `json:"value"` + Operation string `json:"operation"` + //Extension placeholder. //To append more additional information if required by the filter. Metadata map[string]interface{} `json:"metadata"` diff --git a/src/replication/models/policy.go b/src/replication/models/policy.go index 27c157a64..5e6777703 100644 --- a/src/replication/models/policy.go +++ b/src/replication/models/policy.go @@ -9,7 +9,7 @@ type ReplicationPolicy struct { ID int64 //UUID of the policy Name string Description string - Filters []FilterItem + Filters []Filter ReplicateDeletion bool Trigger *Trigger //The trigger of the replication ProjectIDs []int64 //Projects attached to this policy diff --git a/src/replication/policy/manager.go b/src/replication/policy/manager.go index dcd8a13df..3f8a35f17 100644 --- a/src/replication/policy/manager.go +++ b/src/replication/policy/manager.go @@ -7,6 +7,7 @@ import ( "github.com/vmware/harbor/src/common/dao" persist_models "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/replication/models" + "github.com/vmware/harbor/src/ui/config" ) //Manager provides replication policy CURD capabilities. @@ -64,8 +65,14 @@ func convertFromPersistModel(policy *persist_models.RepPolicy) (models.Replicati UpdateTime: policy.UpdateTime, } + project, err := config.GlobalProjectMgr.Get(policy.ProjectID) + if err != nil { + return models.ReplicationPolicy{}, err + } + ply.Namespaces = []string{project.Name} + if len(policy.Filters) > 0 { - filters := []models.FilterItem{} + filters := []models.Filter{} if err := json.Unmarshal([]byte(policy.Filters), &filters); err != nil { return models.ReplicationPolicy{}, err } diff --git a/src/replication/registry/harbor_adaptor.go b/src/replication/registry/harbor_adaptor.go index 6c7ffe09d..e28622f93 100644 --- a/src/replication/registry/harbor_adaptor.go +++ b/src/replication/registry/harbor_adaptor.go @@ -1,10 +1,15 @@ package registry import ( + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/replication" "github.com/vmware/harbor/src/replication/models" + "github.com/vmware/harbor/src/ui/utils" ) +// TODO refacotor the methods of HarborAdaptor by caling Harbor's API + //HarborAdaptor is defined to adapt the Harbor registry type HarborAdaptor struct{} @@ -25,7 +30,19 @@ func (ha *HarborAdaptor) GetNamespace(name string) models.Namespace { //GetRepositories is used to get all the repositories under the specified namespace func (ha *HarborAdaptor) GetRepositories(namespace string) []models.Repository { - return nil + repos, err := dao.GetRepositoryByProjectName(namespace) + if err != nil { + log.Errorf("failed to get repositories under namespace %s: %v", namespace, err) + return nil + } + + repositories := []models.Repository{} + for _, repo := range repos { + repositories = append(repositories, models.Repository{ + Name: repo.Name, + }) + } + return repositories } //GetRepository is used to get the repository with the specified name under the specified namespace @@ -35,7 +52,26 @@ func (ha *HarborAdaptor) GetRepository(name string, namespace string) models.Rep //GetTags is used to get all the tags of the specified repository under the namespace func (ha *HarborAdaptor) GetTags(repositoryName string, namespace string) []models.Tag { - return nil + client, err := utils.NewRepositoryClientForUI("harbor-ui", repositoryName) + if err != nil { + log.Errorf("failed to create registry client: %v", err) + return nil + } + + ts, err := client.ListTag() + if err != nil { + log.Errorf("failed to get tags of repository %s: %v", repositoryName, err) + return nil + } + + tags := []models.Tag{} + for _, t := range ts { + tags = append(tags, models.Tag{ + Name: t, + }) + } + + return tags } //GetTag is used to get the tag with the specified name of the repository under the namespace diff --git a/src/replication/source/repository_convertor.go b/src/replication/source/repository_convertor.go index 12e5bda28..b2ff6cd58 100644 --- a/src/replication/source/repository_convertor.go +++ b/src/replication/source/repository_convertor.go @@ -35,6 +35,8 @@ func NewRepositoryConvertor(registry registry.Adaptor) *RepositoryConvertor { // Convert projects to repositories func (r *RepositoryConvertor) Convert(items []models.FilterItem) []models.FilterItem { + // TODO get repositories from database where the push/deletion operations are recorded + // if support replicate deletion result := []models.FilterItem{} for _, item := range items { if item.Kind != replication.FilterItemKindProject { @@ -46,12 +48,9 @@ func (r *RepositoryConvertor) Convert(items []models.FilterItem) []models.Filter repositories := r.registry.GetRepositories(item.Value) for _, repository := range repositories { result = append(result, models.FilterItem{ - Kind: replication.FilterItemKindRepository, - Value: repository.Name, - // public is used to create project if it does not exist when replicating - Metadata: map[string]interface{}{ - "public": item.Metadata["public"], - }, + Kind: replication.FilterItemKindRepository, + Value: repository.Name, + Operation: item.Operation, }) } } diff --git a/src/replication/source/tag_combination_filter.go b/src/replication/source/tag_combination_filter.go index ced506e44..8bdddadd6 100644 --- a/src/replication/source/tag_combination_filter.go +++ b/src/replication/source/tag_combination_filter.go @@ -60,6 +60,7 @@ func (t *TagCombinationFilter) DoFilter(filterItems []models.FilterItem) []model repos[strs[0]] = append(repos[strs[0]], strs[1]) } + // TODO append operation items := []models.FilterItem{} for repo, tags := range repos { items = append(items, models.FilterItem{ diff --git a/src/replication/source/tag_convertor.go b/src/replication/source/tag_convertor.go index 236e04e3c..09adcecf9 100644 --- a/src/replication/source/tag_convertor.go +++ b/src/replication/source/tag_convertor.go @@ -46,12 +46,9 @@ func (t *TagConvertor) Convert(items []models.FilterItem) []models.FilterItem { tags := t.registry.GetTags(item.Value, "") for _, tag := range tags { result = append(result, models.FilterItem{ - Kind: replication.FilterItemKindTag, - Value: tag.Name, - // public is used to create project if it does not exist when replicating - Metadata: map[string]interface{}{ - "public": item.Metadata["public"], - }, + Kind: replication.FilterItemKindTag, + Value: item.Value + ":" + tag.Name, + Operation: item.Operation, }) } } diff --git a/src/ui/api/models/replication_policy.go b/src/ui/api/models/replication_policy.go index 34fa38a7b..c8da8a03a 100644 --- a/src/ui/api/models/replication_policy.go +++ b/src/ui/api/models/replication_policy.go @@ -27,7 +27,7 @@ type ReplicationPolicy struct { ID int64 `json:"id"` Name string `json:"name"` Description string `json:"description"` - Filters []rep_models.FilterItem `json:"filters"` + Filters []rep_models.Filter `json:"filters"` ReplicateDeletion bool `json:"replicate_deletion"` Trigger *rep_models.Trigger `json:"trigger"` Projects []*common_models.Project `json:"projects"` diff --git a/src/ui/api/utils.go b/src/ui/api/utils.go index 76a0cabf5..7f883109b 100644 --- a/src/ui/api/utils.go +++ b/src/ui/api/utils.go @@ -15,10 +15,7 @@ package api import ( - "bytes" - "encoding/json" "fmt" - "io/ioutil" "net/http" "sort" "strings" @@ -77,120 +74,6 @@ func checkUserExists(name string) int { return 0 } -<<<<<<< HEAD -======= -// CheckAndTriggerReplication checks whether replication policy is set -// on the resource, if is, trigger the replication -func CheckAndTriggerReplication(image, operation string) { - project, _ := utils.ParseRepository(image) - watchItems, err := trigger.DefaultWatchList.Get(project, operation) - if err != nil { - log.Errorf("failed to get watch list for resource %s, operation %s: %v", image, operation, err) - return - } - if len(watchItems) == 0 { - log.Debugf("no replication should be triggered for resource %s, operation %s, skip", image, operation) - return - } - - for _, watchItem := range watchItems { - // TODO define a new type ReplicationItem to wrap FilterItem and operation. - // Maybe change the FilterItem to interface and define a type Resource to - // implement FilterItem is better? - item := &rep_models.FilterItem{ - Kind: replication.FilterItemKindTag, - Value: image, - Metadata: map[string]interface{}{ - "operation": operation, - }, - } - - if err := notifier.Publish(topic.StartReplicationTopic, notification.StartReplicationNotification{ - PolicyID: watchItem.PolicyID, - Metadata: map[string]interface{}{ - "": []*rep_models.FilterItem{item}, - }, - }); err != nil { - log.Errorf("failed to publish replication topic for resource %s, operation %s, policy %d: %v", - image, operation, watchItem.PolicyID, err) - return - } - log.Infof("replication topic for resource %s, operation %s, policy %d triggered", - image, operation, watchItem.PolicyID) - } -} - ->>>>>>> 3409fa1... Publish replication notification for manual, scheduel and immediate trigger -// TriggerReplication triggers the replication according to the policy -// TODO remove -func TriggerReplication(policyID int64, repository string, - tags []string, operation string) error { - data := struct { - PolicyID int64 `json:"policy_id"` - Repo string `json:"repository"` - Operation string `json:"operation"` - TagList []string `json:"tags"` - }{ - PolicyID: policyID, - Repo: repository, - TagList: tags, - Operation: operation, - } - - b, err := json.Marshal(&data) - if err != nil { - return err - } - url := buildReplicationURL() - - return uiutils.RequestAsUI("POST", url, bytes.NewBuffer(b), uiutils.NewStatusRespHandler(http.StatusOK)) -} - -// TODO remove -func postReplicationAction(policyID int64, acton string) error { - data := struct { - PolicyID int64 `json:"policy_id"` - Action string `json:"action"` - }{ - PolicyID: policyID, - Action: acton, - } - - b, err := json.Marshal(&data) - if err != nil { - return err - } - - url := buildReplicationActionURL() - - req, err := http.NewRequest("POST", url, bytes.NewBuffer(b)) - if err != nil { - return err - } - - uiutils.AddUISecret(req) - - client := &http.Client{} - - resp, err := client.Do(req) - if err != nil { - return err - } - - defer resp.Body.Close() - - if resp.StatusCode == http.StatusOK { - return nil - } - - b, err = ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - - return fmt.Errorf("%d %s", resp.StatusCode, string(b)) -} - // SyncRegistry syncs the repositories of registry with database. func SyncRegistry(pm promgr.ProjectManager) error { diff --git a/src/ui/main.go b/src/ui/main.go index 8a47ecca5..e25a10e29 100644 --- a/src/ui/main.go +++ b/src/ui/main.go @@ -29,6 +29,7 @@ import ( "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/notifier" "github.com/vmware/harbor/src/common/scheduler" + "github.com/vmware/harbor/src/replication/core" _ "github.com/vmware/harbor/src/replication/event" "github.com/vmware/harbor/src/ui/api" _ "github.com/vmware/harbor/src/ui/auth/db" @@ -131,6 +132,10 @@ func main() { notifier.Publish(notifier.ScanAllPolicyTopic, notifier.ScanPolicyNotification{Type: scanAllPolicy.Type, DailyTime: (int64)(dailyTime)}) } + if err := core.DefaultController.Init(); err != nil { + log.Errorf("failed to initialize DefaultContorllter: %v", err) + } + filter.Init() beego.InsertFilter("/*", beego.BeforeRouter, filter.SecurityFilter) beego.InsertFilter("/api/*", beego.BeforeRouter, filter.MediaTypeFilter("application/json")) From 8b4fdfc2cc0a05e7953b6a9c3ede73a0d7042b34 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Tue, 5 Dec 2017 14:13:01 +0800 Subject: [PATCH 20/37] Add unit tests for replication related methods --- .travis.yml | 4 +- src/common/utils/test/policy_manager.go | 45 ++++++ .../utils/test/replication_controllter.go | 26 ++++ src/replication/core/controller.go | 63 +++++--- src/replication/core/controller_test.go | 144 ++++++++++++++++++ .../event/start_replication_handler.go | 3 +- .../event/start_replication_handler_test.go | 4 + src/replication/models/filter_item.go | 34 ++--- src/replication/models/filter_test.go | 45 ++++++ src/replication/models/trigger.go | 14 ++ src/replication/models/trigger_test.go | 41 +++++ src/replication/policy/manager.go | 45 ++++-- src/replication/policy/manager_test.go | 60 ++++++++ .../source/repository_convertor_test.go | 13 +- src/replication/source/tag_convertor_test.go | 9 -- src/replication/target/target.go | 38 +++++ src/ui/api/harborapi_test.go | 5 + src/ui/api/replication.go | 2 +- src/ui/api/replication_policy.go | 14 +- src/ui/api/replication_policy_test.go | 72 ++++----- src/ui/main.go | 4 +- 21 files changed, 564 insertions(+), 121 deletions(-) create mode 100644 src/common/utils/test/policy_manager.go create mode 100644 src/common/utils/test/replication_controllter.go create mode 100644 src/replication/core/controller_test.go create mode 100644 src/replication/models/filter_test.go create mode 100644 src/replication/models/trigger_test.go create mode 100644 src/replication/policy/manager_test.go create mode 100644 src/replication/target/target.go diff --git a/.travis.yml b/.travis.yml index 849deb7f3..294124bba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,8 +89,8 @@ script: - sudo mkdir -p ./make/common/config/registry/ - sudo mv ./tests/reg_config.yml ./make/common/config/registry/config.yml - sudo docker-compose -f ./make/docker-compose.test.yml up -d - - go list ./... | grep -v -E 'vendor|tests' | xargs -L1 fgt golint - - go list ./... | grep -v -E 'vendor|tests' | xargs -L1 go vet + - go list ./... | grep -v -E 'vendor|tests|test' | xargs -L1 fgt golint + - go list ./... | grep -v -E 'vendor|tests|test' | xargs -L1 go vet - export MYSQL_HOST=$IP - export REGISTRY_URL=$IP:5000 - echo $REGISTRY_URL diff --git a/src/common/utils/test/policy_manager.go b/src/common/utils/test/policy_manager.go new file mode 100644 index 000000000..492c88e00 --- /dev/null +++ b/src/common/utils/test/policy_manager.go @@ -0,0 +1,45 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" +) + +type FakePolicyManager struct { +} + +func (f *FakePolicyManager) GetPolicies(query models.QueryParameter) ([]models.ReplicationPolicy, error) { + return []models.ReplicationPolicy{}, nil +} + +func (f *FakePolicyManager) GetPolicy(id int64) (models.ReplicationPolicy, error) { + return models.ReplicationPolicy{ + ID: 1, + Trigger: &models.Trigger{ + Kind: replication.TriggerKindManual, + }, + }, nil +} +func (f *FakePolicyManager) CreatePolicy(policy models.ReplicationPolicy) (int64, error) { + return 1, nil +} +func (f *FakePolicyManager) UpdatePolicy(models.ReplicationPolicy) error { + return nil +} +func (f *FakePolicyManager) RemovePolicy(int64) error { + return nil +} diff --git a/src/common/utils/test/replication_controllter.go b/src/common/utils/test/replication_controllter.go new file mode 100644 index 000000000..adff7ae7a --- /dev/null +++ b/src/common/utils/test/replication_controllter.go @@ -0,0 +1,26 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +type FakeReplicatoinController struct { + FakePolicyManager +} + +func (f *FakeReplicatoinController) Init() error { + return nil +} +func (f *FakeReplicatoinController) Replicate(policyID int64, metadata ...map[string]interface{}) error { + return nil +} diff --git a/src/replication/core/controller.go b/src/replication/core/controller.go index 1bcc094a8..a3db273c9 100644 --- a/src/replication/core/controller.go +++ b/src/replication/core/controller.go @@ -1,26 +1,50 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package core import ( "fmt" - "github.com/vmware/harbor/src/common/dao" common_models "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/replication" "github.com/vmware/harbor/src/replication/models" "github.com/vmware/harbor/src/replication/policy" "github.com/vmware/harbor/src/replication/source" + "github.com/vmware/harbor/src/replication/target" "github.com/vmware/harbor/src/replication/trigger" ) -//Controller is core module to cordinate and control the overall workflow of the +// Controller defines the methods that a replicatoin controllter should implement +type Controller interface { + policy.Manager + Init() error + Replicate(policyID int64, metadata ...map[string]interface{}) error +} + +//DefaultController is core module to cordinate and control the overall workflow of the //replication modules. -type Controller struct { +type DefaultController struct { //Indicate whether the controller has been initialized or not initialized bool //Manage the policies - policyManager *policy.Manager + policyManager policy.Manager + + //Manage the targets + targetManager target.Manager //Handle the things related with source sourcer *source.Sourcer @@ -31,7 +55,7 @@ type Controller struct { //Keep controller as singleton instance var ( - DefaultController = NewController(ControllerConfig{}) //Use default data + GlobalController Controller = NewDefaultController(ControllerConfig{}) //Use default data ) //ControllerConfig includes related configurations required by the controller @@ -40,18 +64,19 @@ type ControllerConfig struct { CacheCapacity int } -//NewController is the constructor of Controller. -func NewController(config ControllerConfig) *Controller { +//NewDefaultController is the constructor of DefaultController. +func NewDefaultController(config ControllerConfig) *DefaultController { //Controller refer the default instances - return &Controller{ - policyManager: policy.NewManager(), + return &DefaultController{ + policyManager: policy.NewDefaultManager(), + targetManager: target.NewDefaultManager(), sourcer: source.NewSourcer(), triggerManager: trigger.NewManager(config.CacheCapacity), } } //Init will initialize the controller and the sub components -func (ctl *Controller) Init() error { +func (ctl *DefaultController) Init() error { if ctl.initialized { return nil } @@ -92,7 +117,7 @@ func (ctl *Controller) Init() error { } //CreatePolicy is used to create a new policy and enable it if necessary -func (ctl *Controller) CreatePolicy(newPolicy models.ReplicationPolicy) (int64, error) { +func (ctl *DefaultController) CreatePolicy(newPolicy models.ReplicationPolicy) (int64, error) { id, err := ctl.policyManager.CreatePolicy(newPolicy) if err != nil { return 0, err @@ -108,7 +133,7 @@ func (ctl *Controller) CreatePolicy(newPolicy models.ReplicationPolicy) (int64, //UpdatePolicy will update the policy with new content. //Parameter updatedPolicy must have the ID of the updated policy. -func (ctl *Controller) UpdatePolicy(updatedPolicy models.ReplicationPolicy) error { +func (ctl *DefaultController) UpdatePolicy(updatedPolicy models.ReplicationPolicy) error { // TODO check pre-conditions id := updatedPolicy.ID @@ -154,7 +179,7 @@ func (ctl *Controller) UpdatePolicy(updatedPolicy models.ReplicationPolicy) erro } //RemovePolicy will remove the specified policy and clean the related settings -func (ctl *Controller) RemovePolicy(policyID int64) error { +func (ctl *DefaultController) RemovePolicy(policyID int64) error { // TODO check pre-conditions policy, err := ctl.policyManager.GetPolicy(policyID) @@ -174,18 +199,18 @@ func (ctl *Controller) RemovePolicy(policyID int64) error { } //GetPolicy is delegation of GetPolicy of Policy.Manager -func (ctl *Controller) GetPolicy(policyID int64) (models.ReplicationPolicy, error) { +func (ctl *DefaultController) GetPolicy(policyID int64) (models.ReplicationPolicy, error) { return ctl.policyManager.GetPolicy(policyID) } //GetPolicies is delegation of GetPoliciemodels.ReplicationPolicy{}s of Policy.Manager -func (ctl *Controller) GetPolicies(query models.QueryParameter) ([]models.ReplicationPolicy, error) { +func (ctl *DefaultController) GetPolicies(query models.QueryParameter) ([]models.ReplicationPolicy, error) { return ctl.policyManager.GetPolicies(query) } //Replicate starts one replication defined in the specified policy; //Can be launched by the API layer and related triggers. -func (ctl *Controller) Replicate(policyID int64, metadata ...map[string]interface{}) error { +func (ctl *DefaultController) Replicate(policyID int64, metadata ...map[string]interface{}) error { policy, err := ctl.GetPolicy(policyID) if err != nil { return err @@ -210,7 +235,7 @@ func (ctl *Controller) Replicate(policyID int64, metadata ...map[string]interfac targets := []*common_models.RepTarget{} for _, targetID := range policy.TargetIDs { - target, err := dao.GetRepTarget(targetID) + target, err := ctl.targetManager.GetTarget(targetID) if err != nil { return err } @@ -259,13 +284,13 @@ func buildFilterChain(policy *models.ReplicationPolicy, sourcer *source.Sourcer) } repositoryPattern = fmt.Sprintf("%s/%s", projectPattern, repositoryPattern) - tagPattern, exist := patternMap[replication.FilterItemKindProject] + tagPattern, exist := patternMap[replication.FilterItemKindTag] if !exist { tagPattern = replication.PatternMatchAll } tagPattern = fmt.Sprintf("%s:%s", repositoryPattern, tagPattern) - if policy.Trigger.Kind == replication.TriggerKindImmediate { + if policy.Trigger != nil && policy.Trigger.Kind == replication.TriggerKindImmediate { // build filter chain for immediate trigger policy filters = append(filters, source.NewPatternFilter(replication.FilterItemKindTag, tagPattern)) diff --git a/src/replication/core/controller_test.go b/src/replication/core/controller_test.go new file mode 100644 index 000000000..bbfa671f8 --- /dev/null +++ b/src/replication/core/controller_test.go @@ -0,0 +1,144 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/common/utils/test" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" + "github.com/vmware/harbor/src/replication/source" +) + +func TestMain(m *testing.M) { + // set the policy manager used by GlobalController with a fake policy manager + controller := GlobalController.(*DefaultController) + controller.policyManager = &test.FakePolicyManager{} + os.Exit(m.Run()) +} + +func TestNewDefaultController(t *testing.T) { + controller := NewDefaultController(ControllerConfig{}) + assert.NotNil(t, controller) +} + +func TestInit(t *testing.T) { + assert.Nil(t, GlobalController.Init()) +} + +func TestCreatePolicy(t *testing.T) { + _, err := GlobalController.CreatePolicy(models.ReplicationPolicy{}) + assert.Nil(t, err) +} + +func TestUpdatePolicy(t *testing.T) { + assert.Nil(t, GlobalController.UpdatePolicy(models.ReplicationPolicy{ + ID: 2, + Trigger: &models.Trigger{ + Kind: replication.TriggerKindManual, + }, + })) +} + +func TestRemovePolicy(t *testing.T) { + assert.Nil(t, GlobalController.RemovePolicy(1)) +} + +func TestGetPolicy(t *testing.T) { + _, err := GlobalController.GetPolicy(1) + assert.Nil(t, err) +} + +func TestGetPolicies(t *testing.T) { + _, err := GlobalController.GetPolicies(models.QueryParameter{}) + assert.Nil(t, err) +} + +func TestReplicate(t *testing.T) { + // TODO +} + +func TestGetCandidates(t *testing.T) { + policy := &models.ReplicationPolicy{ + ID: 1, + Filters: []models.Filter{ + models.Filter{ + Kind: replication.FilterItemKindTag, + Pattern: ".*", + }, + }, + Trigger: &models.Trigger{ + Kind: replication.TriggerKindImmediate, + }, + } + + sourcer := source.NewSourcer() + + candidates := []models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/hello-world:release-1.0", + }, + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/hello-world:latest", + }, + } + result := getCandidates(policy, sourcer, candidates...) + assert.Equal(t, 2, len(result)) + + policy.Filters = []models.Filter{ + models.Filter{ + Kind: replication.FilterItemKindTag, + Pattern: "release-.*", + }, + } + result = getCandidates(policy, sourcer, candidates...) + assert.Equal(t, 1, len(result)) +} + +func TestBuildFilterChain(t *testing.T) { + policy := &models.ReplicationPolicy{ + ID: 1, + Filters: []models.Filter{ + models.Filter{ + Kind: replication.FilterItemKindProject, + Pattern: "*", + }, + models.Filter{ + Kind: replication.FilterItemKindRepository, + Pattern: "*", + }, + models.Filter{ + Kind: replication.FilterItemKindTag, + Pattern: "*", + }, + }, + } + + sourcer := source.NewSourcer() + + chain := buildFilterChain(policy, sourcer) + assert.Equal(t, 3, len(chain.Filters())) + + policy.Trigger = &models.Trigger{ + Kind: replication.TriggerKindImmediate, + } + chain = buildFilterChain(policy, sourcer) + assert.Equal(t, 1, len(chain.Filters())) +} diff --git a/src/replication/event/start_replication_handler.go b/src/replication/event/start_replication_handler.go index 82b8bce40..7e29ef542 100644 --- a/src/replication/event/start_replication_handler.go +++ b/src/replication/event/start_replication_handler.go @@ -43,8 +43,7 @@ func (srh *StartReplicationHandler) Handle(value interface{}) error { } //Start replication - - return core.DefaultController.Replicate(notification.PolicyID, notification.Metadata) + return core.GlobalController.Replicate(notification.PolicyID, notification.Metadata) } //IsStateful implements the same method of notification handler interface diff --git a/src/replication/event/start_replication_handler_test.go b/src/replication/event/start_replication_handler_test.go index 30404b270..88ba61fbc 100644 --- a/src/replication/event/start_replication_handler_test.go +++ b/src/replication/event/start_replication_handler_test.go @@ -18,10 +18,14 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/common/utils/test" + "github.com/vmware/harbor/src/replication/core" "github.com/vmware/harbor/src/replication/event/notification" ) func TestHandle(t *testing.T) { + core.GlobalController = &test.FakeReplicatoinController{} + handler := &StartReplicationHandler{} assert.NotNil(t, handler.Handle(nil)) diff --git a/src/replication/models/filter_item.go b/src/replication/models/filter_item.go index 101fd6438..82497dd90 100644 --- a/src/replication/models/filter_item.go +++ b/src/replication/models/filter_item.go @@ -1,12 +1,19 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package models -import ( - "fmt" - - "github.com/astaxie/beego/validation" - "github.com/vmware/harbor/src/replication" -) - //FilterItem is the general data model represents the filtering resources which are used as input and output for the filters. type FilterItem struct { @@ -26,16 +33,3 @@ type FilterItem struct { //To append more additional information if required by the filter. Metadata map[string]interface{} `json:"metadata"` } - -// Valid ... -func (f *FilterItem) Valid(v *validation.Validation) { - if !(f.Kind == replication.FilterItemKindProject || - f.Kind == replication.FilterItemKindRepository || - f.Kind == replication.FilterItemKindTag) { - v.SetError("kind", fmt.Sprintf("invalid filter kind: %s", f.Kind)) - } - - if len(f.Value) == 0 { - v.SetError("value", "filter value can not be empty") - } -} diff --git a/src/replication/models/filter_test.go b/src/replication/models/filter_test.go new file mode 100644 index 000000000..4026f0e20 --- /dev/null +++ b/src/replication/models/filter_test.go @@ -0,0 +1,45 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "testing" + + "github.com/astaxie/beego/validation" + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/replication" +) + +func TestValid(t *testing.T) { + cases := map[*Filter]bool{ + &Filter{}: true, + &Filter{ + Kind: "invalid_kind", + }: true, + &Filter{ + Kind: replication.FilterItemKindRepository, + }: true, + &Filter{ + Kind: replication.FilterItemKindRepository, + Pattern: "*", + }: false, + } + + for filter, hasError := range cases { + v := &validation.Validation{} + filter.Valid(v) + assert.Equal(t, hasError, v.HasErrors()) + } +} diff --git a/src/replication/models/trigger.go b/src/replication/models/trigger.go index fa62166cb..27c2afa69 100644 --- a/src/replication/models/trigger.go +++ b/src/replication/models/trigger.go @@ -1,3 +1,17 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package models import ( diff --git a/src/replication/models/trigger_test.go b/src/replication/models/trigger_test.go new file mode 100644 index 000000000..b591995e9 --- /dev/null +++ b/src/replication/models/trigger_test.go @@ -0,0 +1,41 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "testing" + + "github.com/astaxie/beego/validation" + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/replication" +) + +func TestValidOfTrigger(t *testing.T) { + cases := map[*Trigger]bool{ + &Trigger{}: true, + &Trigger{ + Kind: "invalid_kind", + }: true, + &Trigger{ + Kind: replication.TriggerKindImmediate, + }: false, + } + + for filter, hasError := range cases { + v := &validation.Validation{} + filter.Valid(v) + assert.Equal(t, hasError, v.HasErrors()) + } +} diff --git a/src/replication/policy/manager.go b/src/replication/policy/manager.go index 3f8a35f17..bdce2740a 100644 --- a/src/replication/policy/manager.go +++ b/src/replication/policy/manager.go @@ -1,3 +1,17 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package policy import ( @@ -10,16 +24,25 @@ import ( "github.com/vmware/harbor/src/ui/config" ) -//Manager provides replication policy CURD capabilities. -type Manager struct{} +// Manager defines the method a policy manger should implement +type Manager interface { + GetPolicies(models.QueryParameter) ([]models.ReplicationPolicy, error) + GetPolicy(int64) (models.ReplicationPolicy, error) + CreatePolicy(models.ReplicationPolicy) (int64, error) + UpdatePolicy(models.ReplicationPolicy) error + RemovePolicy(int64) error +} -//NewManager is the constructor of Manager. -func NewManager() *Manager { - return &Manager{} +//DefaultManager provides replication policy CURD capabilities. +type DefaultManager struct{} + +//NewDefaultManager is the constructor of DefaultManager. +func NewDefaultManager() *DefaultManager { + return &DefaultManager{} } //GetPolicies returns all the policies -func (m *Manager) GetPolicies(query models.QueryParameter) ([]models.ReplicationPolicy, error) { +func (m *DefaultManager) GetPolicies(query models.QueryParameter) ([]models.ReplicationPolicy, error) { result := []models.ReplicationPolicy{} //TODO support more query conditions other than name and project ID policies, err := dao.FilterRepPolicies(query.Name, query.ProjectID) @@ -39,7 +62,7 @@ func (m *Manager) GetPolicies(query models.QueryParameter) ([]models.Replication } //GetPolicy returns the policy with the specified ID -func (m *Manager) GetPolicy(policyID int64) (models.ReplicationPolicy, error) { +func (m *DefaultManager) GetPolicy(policyID int64) (models.ReplicationPolicy, error) { policy, err := dao.GetRepPolicy(policyID) if err != nil { return models.ReplicationPolicy{}, err @@ -48,7 +71,6 @@ func (m *Manager) GetPolicy(policyID int64) (models.ReplicationPolicy, error) { return convertFromPersistModel(policy) } -// TODO add UT func convertFromPersistModel(policy *persist_models.RepPolicy) (models.ReplicationPolicy, error) { if policy == nil { return models.ReplicationPolicy{}, nil @@ -90,7 +112,6 @@ func convertFromPersistModel(policy *persist_models.RepPolicy) (models.Replicati return ply, nil } -// TODO add ut func convertToPersistModel(policy models.ReplicationPolicy) (*persist_models.RepPolicy, error) { ply := &persist_models.RepPolicy{ ID: policy.ID, @@ -131,7 +152,7 @@ func convertToPersistModel(policy models.ReplicationPolicy) (*persist_models.Rep //CreatePolicy creates a new policy with the provided data; //If creating failed, error will be returned; //If creating succeed, ID of the new created policy will be returned. -func (m *Manager) CreatePolicy(policy models.ReplicationPolicy) (int64, error) { +func (m *DefaultManager) CreatePolicy(policy models.ReplicationPolicy) (int64, error) { now := time.Now() policy.CreationTime = now policy.UpdateTime = now @@ -144,7 +165,7 @@ func (m *Manager) CreatePolicy(policy models.ReplicationPolicy) (int64, error) { //UpdatePolicy updates the policy; //If updating failed, error will be returned. -func (m *Manager) UpdatePolicy(policy models.ReplicationPolicy) error { +func (m *DefaultManager) UpdatePolicy(policy models.ReplicationPolicy) error { policy.UpdateTime = time.Now() ply, err := convertToPersistModel(policy) if err != nil { @@ -155,6 +176,6 @@ func (m *Manager) UpdatePolicy(policy models.ReplicationPolicy) error { //RemovePolicy removes the specified policy; //If removing failed, error will be returned. -func (m *Manager) RemovePolicy(policyID int64) error { +func (m *DefaultManager) RemovePolicy(policyID int64) error { return dao.DeleteRepPolicy(policyID) } diff --git a/src/replication/policy/manager_test.go b/src/replication/policy/manager_test.go new file mode 100644 index 000000000..62337aff7 --- /dev/null +++ b/src/replication/policy/manager_test.go @@ -0,0 +1,60 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policy + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmware/harbor/src/replication/models" +) + +func TestConvertToPersistModel(t *testing.T) { + var id, projectID, targetID int64 = 1, 1, 1 + name := "policy01" + replicateDeletion := true + trigger := &models.Trigger{ + Kind: "trigger_kind", + } + filters := []models.Filter{ + models.Filter{ + Kind: "filter_kind", + Pattern: "filter_pattern", + }, + } + policy := models.ReplicationPolicy{ + ID: id, + Name: name, + ReplicateDeletion: replicateDeletion, + ProjectIDs: []int64{projectID}, + TargetIDs: []int64{targetID}, + Trigger: trigger, + Filters: filters, + } + + ply, err := convertToPersistModel(policy) + require.Nil(t, err) + assert.Equal(t, id, ply.ID) + assert.Equal(t, name, ply.Name) + assert.Equal(t, replicateDeletion, ply.ReplicateDeletion) + assert.Equal(t, projectID, ply.ProjectID) + assert.Equal(t, targetID, ply.TargetID) + tg, _ := json.Marshal(trigger) + assert.Equal(t, string(tg), ply.Trigger) + ft, _ := json.Marshal(filters) + assert.Equal(t, string(ft), ply.Filters) +} diff --git a/src/replication/source/repository_convertor_test.go b/src/replication/source/repository_convertor_test.go index 48e223232..f260f113d 100644 --- a/src/replication/source/repository_convertor_test.go +++ b/src/replication/source/repository_convertor_test.go @@ -27,9 +27,6 @@ func TestRepositoryConvert(t *testing.T) { models.FilterItem{ Kind: replication.FilterItemKindProject, Value: "library", - Metadata: map[string]interface{}{ - "public": true, - }, }, models.FilterItem{ Kind: replication.FilterItemKindRepository, @@ -39,16 +36,10 @@ func TestRepositoryConvert(t *testing.T) { models.FilterItem{ Kind: replication.FilterItemKindRepository, Value: "library/ubuntu", - Metadata: map[string]interface{}{ - "public": true, - }, }, models.FilterItem{ Kind: replication.FilterItemKindRepository, Value: "library/centos", - Metadata: map[string]interface{}{ - "public": true, - }, }, } @@ -88,10 +79,10 @@ func (f *fakeRegistryAdaptor) GetRepository(name string, namespace string) model func (f *fakeRegistryAdaptor) GetTags(repositoryName string, namespace string) []models.Tag { return []models.Tag{ models.Tag{ - Name: "library/ubuntu:14.04", + Name: "14.04", }, models.Tag{ - Name: "library/ubuntu:16.04", + Name: "16.04", }, } } diff --git a/src/replication/source/tag_convertor_test.go b/src/replication/source/tag_convertor_test.go index 151192bbb..6e64f86d2 100644 --- a/src/replication/source/tag_convertor_test.go +++ b/src/replication/source/tag_convertor_test.go @@ -27,9 +27,6 @@ func TestTagConvert(t *testing.T) { models.FilterItem{ Kind: replication.FilterItemKindRepository, Value: "library/ubuntu", - Metadata: map[string]interface{}{ - "public": true, - }, }, models.FilterItem{ Kind: replication.FilterItemKindProject, @@ -39,16 +36,10 @@ func TestTagConvert(t *testing.T) { models.FilterItem{ Kind: replication.FilterItemKindTag, Value: "library/ubuntu:14.04", - Metadata: map[string]interface{}{ - "public": true, - }, }, models.FilterItem{ Kind: replication.FilterItemKindTag, Value: "library/ubuntu:16.04", - Metadata: map[string]interface{}{ - "public": true, - }, }, } diff --git a/src/replication/target/target.go b/src/replication/target/target.go new file mode 100644 index 000000000..ab8e815e5 --- /dev/null +++ b/src/replication/target/target.go @@ -0,0 +1,38 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package target + +import ( + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" +) + +// Manager defines the methods that a target manager should implement +type Manager interface { + GetTarget(int64) (*models.RepTarget, error) +} + +// DefaultManager implement the Manager interface +type DefaultManager struct{} + +// NewDefaultManager returns an instance of DefaultManger +func NewDefaultManager() *DefaultManager { + return &DefaultManager{} +} + +// GetTarget ... +func (d *DefaultManager) GetTarget(id int64) (*models.RepTarget, error) { + return dao.GetRepTarget(id) +} diff --git a/src/ui/api/harborapi_test.go b/src/ui/api/harborapi_test.go index 987b69ca2..7e62499c5 100644 --- a/src/ui/api/harborapi_test.go +++ b/src/ui/api/harborapi_test.go @@ -40,6 +40,7 @@ import ( "github.com/dghubble/sling" //for test env prepare + "github.com/vmware/harbor/src/replication/core" _ "github.com/vmware/harbor/src/replication/event" _ "github.com/vmware/harbor/src/ui/auth/db" _ "github.com/vmware/harbor/src/ui/auth/ldap" @@ -133,6 +134,10 @@ func init() { _ = updateInitPassword(1, "Harbor12345") + if err := core.GlobalController.Init(); err != nil { + log.Fatalf("failed to initialize GlobalController: %v", err) + } + //syncRegistry if err := SyncRegistry(config.GlobalProjectMgr); err != nil { log.Fatalf("failed to sync repositories from registry: %v", err) diff --git a/src/ui/api/replication.go b/src/ui/api/replication.go index 43bd2289c..37a8ec6ce 100644 --- a/src/ui/api/replication.go +++ b/src/ui/api/replication.go @@ -49,7 +49,7 @@ func (r *ReplicationAPI) Post() { replication := &models.Replication{} r.DecodeJSONReqAndValidate(replication) - policy, err := core.DefaultController.GetPolicy(replication.PolicyID) + policy, err := core.GlobalController.GetPolicy(replication.PolicyID) if err != nil { r.HandleInternalServerError(fmt.Sprintf("failed to get replication policy %d: %v", replication.PolicyID, err)) return diff --git a/src/ui/api/replication_policy.go b/src/ui/api/replication_policy.go index 955ed2fec..da452331b 100644 --- a/src/ui/api/replication_policy.go +++ b/src/ui/api/replication_policy.go @@ -51,7 +51,7 @@ func (pa *RepPolicyAPI) Prepare() { // Get ... func (pa *RepPolicyAPI) Get() { id := pa.GetIDFromURL() - policy, err := core.DefaultController.GetPolicy(id) + policy, err := core.GlobalController.GetPolicy(id) if err != nil { log.Errorf("failed to get policy %d: %v", id, err) pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) @@ -87,7 +87,7 @@ func (pa *RepPolicyAPI) List() { result := []*api_models.ReplicationPolicy{} - policies, err := core.DefaultController.GetPolicies(queryParam) + policies, err := core.GlobalController.GetPolicies(queryParam) if err != nil { log.Errorf("failed to get policies: %v, query parameters: %v", err, queryParam) pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) @@ -139,7 +139,7 @@ func (pa *RepPolicyAPI) Post() { } } - id, err := core.DefaultController.CreatePolicy(convertToRepPolicy(policy)) + id, err := core.GlobalController.CreatePolicy(convertToRepPolicy(policy)) if err != nil { pa.HandleInternalServerError(fmt.Sprintf("failed to create policy: %v", err)) return @@ -154,7 +154,7 @@ func (pa *RepPolicyAPI) Post() { func (pa *RepPolicyAPI) Put() { id := pa.GetIDFromURL() - originalPolicy, err := core.DefaultController.GetPolicy(id) + originalPolicy, err := core.GlobalController.GetPolicy(id) if err != nil { log.Errorf("failed to get policy %d: %v", id, err) pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) @@ -197,7 +197,7 @@ func (pa *RepPolicyAPI) Put() { } } - if err = core.DefaultController.UpdatePolicy(convertToRepPolicy(policy)); err != nil { + if err = core.GlobalController.UpdatePolicy(convertToRepPolicy(policy)); err != nil { pa.HandleInternalServerError(fmt.Sprintf("failed to update policy %d: %v", id, err)) return } @@ -207,7 +207,7 @@ func (pa *RepPolicyAPI) Put() { func (pa *RepPolicyAPI) Delete() { id := pa.GetIDFromURL() - policy, err := core.DefaultController.GetPolicy(id) + policy, err := core.GlobalController.GetPolicy(id) if err != nil { log.Errorf("failed to get policy %d: %v", id, err) pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) @@ -232,7 +232,7 @@ func (pa *RepPolicyAPI) Delete() { } } - if err = core.DefaultController.RemovePolicy(id); err != nil { + if err = core.GlobalController.RemovePolicy(id); err != nil { log.Errorf("failed to delete policy %d: %v", id, err) pa.CustomAbort(http.StatusInternalServerError, "") } diff --git a/src/ui/api/replication_policy_test.go b/src/ui/api/replication_policy_test.go index 5e2b2f2f5..0bf688105 100644 --- a/src/ui/api/replication_policy_test.go +++ b/src/ui/api/replication_policy_test.go @@ -123,10 +123,10 @@ func TestRepPolicyAPIPost(t *testing.T) { ID: targetID, }, }, - Filters: []rep_models.FilterItem{ - rep_models.FilterItem{ - Kind: "invalid_filter_kind", - Value: "", + Filters: []rep_models.Filter{ + rep_models.Filter{ + Kind: "invalid_filter_kind", + Pattern: "", }, }, }, @@ -151,10 +151,10 @@ func TestRepPolicyAPIPost(t *testing.T) { ID: targetID, }, }, - Filters: []rep_models.FilterItem{ - rep_models.FilterItem{ - Kind: replication.FilterItemKindRepository, - Value: "*", + Filters: []rep_models.Filter{ + rep_models.Filter{ + Kind: replication.FilterItemKindRepository, + Pattern: "*", }, }, Trigger: &rep_models.Trigger{ @@ -182,10 +182,10 @@ func TestRepPolicyAPIPost(t *testing.T) { ID: targetID, }, }, - Filters: []rep_models.FilterItem{ - rep_models.FilterItem{ - Kind: replication.FilterItemKindRepository, - Value: "*", + Filters: []rep_models.Filter{ + rep_models.Filter{ + Kind: replication.FilterItemKindRepository, + Pattern: "*", }, }, Trigger: &rep_models.Trigger{ @@ -213,10 +213,10 @@ func TestRepPolicyAPIPost(t *testing.T) { ID: 10000, }, }, - Filters: []rep_models.FilterItem{ - rep_models.FilterItem{ - Kind: replication.FilterItemKindRepository, - Value: "*", + Filters: []rep_models.Filter{ + rep_models.Filter{ + Kind: replication.FilterItemKindRepository, + Pattern: "*", }, }, Trigger: &rep_models.Trigger{ @@ -244,10 +244,10 @@ func TestRepPolicyAPIPost(t *testing.T) { ID: targetID, }, }, - Filters: []rep_models.FilterItem{ - rep_models.FilterItem{ - Kind: replication.FilterItemKindRepository, - Value: "*", + Filters: []rep_models.Filter{ + rep_models.Filter{ + Kind: replication.FilterItemKindRepository, + Pattern: "*", }, }, Trigger: &rep_models.Trigger{ @@ -374,10 +374,10 @@ func TestRepPolicyAPIPut(t *testing.T) { ID: targetID, }, }, - Filters: []rep_models.FilterItem{ - rep_models.FilterItem{ - Kind: replication.FilterItemKindRepository, - Value: "*", + Filters: []rep_models.Filter{ + rep_models.Filter{ + Kind: replication.FilterItemKindRepository, + Pattern: "*", }, }, Trigger: &rep_models.Trigger{ @@ -405,10 +405,10 @@ func TestRepPolicyAPIPut(t *testing.T) { ID: targetID, }, }, - Filters: []rep_models.FilterItem{ - rep_models.FilterItem{ - Kind: replication.FilterItemKindRepository, - Value: "*", + Filters: []rep_models.Filter{ + rep_models.Filter{ + Kind: replication.FilterItemKindRepository, + Pattern: "*", }, }, Trigger: &rep_models.Trigger{ @@ -463,10 +463,10 @@ func TestConvertToRepPolicy(t *testing.T) { ID: 1, Name: "policy", Description: "description", - Filters: []rep_models.FilterItem{ - rep_models.FilterItem{ - Kind: "filter_kind_01", - Value: "*", + Filters: []rep_models.Filter{ + rep_models.Filter{ + Kind: "filter_kind_01", + Pattern: "*", }, }, ReplicateDeletion: true, @@ -490,10 +490,10 @@ func TestConvertToRepPolicy(t *testing.T) { ID: 1, Name: "policy", Description: "description", - Filters: []rep_models.FilterItem{ - rep_models.FilterItem{ - Kind: "filter_kind_01", - Value: "*", + Filters: []rep_models.Filter{ + rep_models.Filter{ + Kind: "filter_kind_01", + Pattern: "*", }, }, ReplicateDeletion: true, diff --git a/src/ui/main.go b/src/ui/main.go index e25a10e29..12f500de3 100644 --- a/src/ui/main.go +++ b/src/ui/main.go @@ -132,8 +132,8 @@ func main() { notifier.Publish(notifier.ScanAllPolicyTopic, notifier.ScanPolicyNotification{Type: scanAllPolicy.Type, DailyTime: (int64)(dailyTime)}) } - if err := core.DefaultController.Init(); err != nil { - log.Errorf("failed to initialize DefaultContorllter: %v", err) + if err := core.GlobalController.Init(); err != nil { + log.Errorf("failed to initialize the replication controller: %v", err) } filter.Init() From fe10c2e7f5e187ab5584f8526309359fed2feece Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 7 Dec 2017 19:07:51 +0800 Subject: [PATCH 21/37] Create replicator to submit replication job to jobservice --- src/adminserver/client/client.go | 19 ++-- src/adminserver/client/client_test.go | 3 +- src/common/http/client/auth/auth.go | 73 +++++++++++++++ src/common/http/client/auth/auth_test.go | 56 ++++++++++++ src/common/http/client/client.go | 56 ++++++++++++ .../auth/auth.go => common/http/error.go} | 38 ++------ src/jobservice/client/client.go | 88 +++++++++++++++++++ src/jobservice/client/client_test.go | 55 ++++++++++++ src/jobservice/config/config.go | 7 +- src/replication/consts.go | 5 -- src/replication/core/controller.go | 80 +++++++++++------ src/replication/event/on_deletion_handler.go | 4 +- src/replication/event/on_push_handler.go | 7 +- src/replication/models/policy.go | 4 +- src/replication/policy/manager.go | 7 ++ src/replication/replicator/replicator.go | 43 +++++++++ .../replicator/replicator_test.go} | 31 +++---- src/ui/config/config.go | 7 +- 18 files changed, 476 insertions(+), 107 deletions(-) create mode 100644 src/common/http/client/auth/auth.go create mode 100644 src/common/http/client/auth/auth_test.go create mode 100644 src/common/http/client/client.go rename src/{adminserver/client/auth/auth.go => common/http/error.go} (51%) create mode 100644 src/jobservice/client/client.go create mode 100644 src/jobservice/client/client_test.go create mode 100644 src/replication/replicator/replicator.go rename src/{adminserver/client/auth/auth_test.go => replication/replicator/replicator_test.go} (57%) diff --git a/src/adminserver/client/client.go b/src/adminserver/client/client.go index 95fbdf0f8..ba82d283c 100644 --- a/src/adminserver/client/client.go +++ b/src/adminserver/client/client.go @@ -23,8 +23,8 @@ import ( "net/http" "strings" - "github.com/vmware/harbor/src/adminserver/client/auth" "github.com/vmware/harbor/src/adminserver/systeminfo/imagestorage" + httpclient "github.com/vmware/harbor/src/common/http/client" "github.com/vmware/harbor/src/common/utils" ) @@ -43,22 +43,20 @@ type Client interface { } // NewClient return an instance of Adminserver client -func NewClient(baseURL string, authorizer auth.Authorizer) Client { +func NewClient(baseURL string, c httpclient.Client) Client { baseURL = strings.TrimRight(baseURL, "/") if !strings.Contains(baseURL, "://") { baseURL = "http://" + baseURL } return &client{ - baseURL: baseURL, - client: &http.Client{}, - authorizer: authorizer, + baseURL: baseURL, + client: c, } } type client struct { - baseURL string - client *http.Client - authorizer auth.Authorizer + baseURL string + client httpclient.Client } // do creates request and authorizes it if authorizer is not nil @@ -69,11 +67,6 @@ func (c *client) do(method, relativePath string, body io.Reader) (*http.Response return nil, err } - if c.authorizer != nil { - if err := c.authorizer.Authorize(req); err != nil { - return nil, err - } - } return c.client.Do(req) } diff --git a/src/adminserver/client/client_test.go b/src/adminserver/client/client_test.go index c232288c1..b74c187b4 100644 --- a/src/adminserver/client/client_test.go +++ b/src/adminserver/client/client_test.go @@ -16,6 +16,7 @@ package client import ( "fmt" + "net/http" "os" "testing" @@ -34,7 +35,7 @@ func TestMain(m *testing.M) { os.Exit(1) } - c = NewClient(server.URL, nil) + c = NewClient(server.URL, &http.Client{}) os.Exit(m.Run()) } diff --git a/src/common/http/client/auth/auth.go b/src/common/http/client/auth/auth.go new file mode 100644 index 000000000..7106a7df6 --- /dev/null +++ b/src/common/http/client/auth/auth.go @@ -0,0 +1,73 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "errors" + "net/http" +) + +const ( + secretCookieName = "secret" +) + +// Authorizer authorizes the requests +type Authorizer interface { + Authorize(*http.Request) error +} + +// CookieAuthorizer authorizes the requests by adding cookie specified by name and value +type CookieAuthorizer struct { + name string + value string +} + +// NewCookieAuthorizer returns an instance of CookieAuthorizer +func NewCookieAuthorizer(name, value string) *CookieAuthorizer { + return &CookieAuthorizer{ + name: name, + value: value, + } +} + +// Authorize the request with the cookie +func (c *CookieAuthorizer) Authorize(req *http.Request) error { + if req == nil { + return errors.New("the request is null") + } + + req.AddCookie(&http.Cookie{ + Name: c.name, + Value: c.value, + }) + return nil +} + +// SecretAuthorizer authorizes the requests with the specified secret +type SecretAuthorizer struct { + authorizer *CookieAuthorizer +} + +// NewSecretAuthorizer returns an instance of SecretAuthorizer +func NewSecretAuthorizer(secret string) *SecretAuthorizer { + return &SecretAuthorizer{ + authorizer: NewCookieAuthorizer(secretCookieName, secret), + } +} + +// Authorize the request with the secret +func (s *SecretAuthorizer) Authorize(req *http.Request) error { + return s.authorizer.Authorize(req) +} diff --git a/src/common/http/client/auth/auth_test.go b/src/common/http/client/auth/auth_test.go new file mode 100644 index 000000000..7754a663b --- /dev/null +++ b/src/common/http/client/auth/auth_test.go @@ -0,0 +1,56 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAuthorizeOfCookieAuthorizer(t *testing.T) { + name, value := "name", "value" + authorizer := NewCookieAuthorizer(name, value) + + // nil request + require.NotNil(t, authorizer.Authorize(nil)) + + // valid request + req, err := http.NewRequest("", "", nil) + require.Nil(t, err) + require.Nil(t, authorizer.Authorize(req)) + require.Equal(t, 1, len(req.Cookies())) + v, err := req.Cookie(name) + require.Nil(t, err) + assert.Equal(t, value, v.Value) +} + +func TestAuthorizeOfSecretAuthorizer(t *testing.T) { + secret := "secret" + authorizer := NewSecretAuthorizer(secret) + + // nil request + require.NotNil(t, authorizer.Authorize(nil)) + + // valid request + req, err := http.NewRequest("", "", nil) + require.Nil(t, err) + require.Nil(t, authorizer.Authorize(req)) + require.Equal(t, 1, len(req.Cookies())) + v, err := req.Cookie(secretCookieName) + require.Nil(t, err) + assert.Equal(t, secret, v.Value) +} diff --git a/src/common/http/client/client.go b/src/common/http/client/client.go new file mode 100644 index 000000000..b8e86a596 --- /dev/null +++ b/src/common/http/client/client.go @@ -0,0 +1,56 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "net/http" + + "github.com/vmware/harbor/src/common/http/client/auth" +) + +// Client defines the method that a HTTP client should implement +type Client interface { + Do(*http.Request) (*http.Response, error) +} + +// AuthorizedClient authorizes the requests before sending them +type AuthorizedClient struct { + client *http.Client + authorizer auth.Authorizer +} + +// NewAuthorizedClient returns an instance of the AuthorizedClient +func NewAuthorizedClient(authorizer auth.Authorizer, client ...*http.Client) *AuthorizedClient { + c := &AuthorizedClient{ + authorizer: authorizer, + } + if len(client) > 0 { + c.client = client[0] + } + if c.client == nil { + c.client = &http.Client{} + } + return c +} + +// Do authorizes the request before sending it +func (a *AuthorizedClient) Do(req *http.Request) (*http.Response, error) { + if a.authorizer != nil { + if err := a.authorizer.Authorize(req); err != nil { + return nil, err + } + } + return a.client.Do(req) +} diff --git a/src/adminserver/client/auth/auth.go b/src/common/http/error.go similarity index 51% rename from src/adminserver/client/auth/auth.go rename to src/common/http/error.go index 038a8c122..67e5da8e0 100644 --- a/src/adminserver/client/auth/auth.go +++ b/src/common/http/error.go @@ -12,39 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package auth +package http import ( - "net/http" + "fmt" ) -// Authorizer authorizes request -type Authorizer interface { - Authorize(*http.Request) error +// Error wrap HTTP status code and message as an error +type Error struct { + Code int + Message string } -// NewSecretAuthorizer returns an instance of secretAuthorizer -func NewSecretAuthorizer(cookieName, secret string) Authorizer { - return &secretAuthorizer{ - cookieName: cookieName, - secret: secret, - } -} - -type secretAuthorizer struct { - cookieName string - secret string -} - -func (s *secretAuthorizer) Authorize(req *http.Request) error { - if req == nil { - return nil - } - - req.AddCookie(&http.Cookie{ - Name: s.cookieName, - Value: s.secret, - }) - - return nil +// Error ... +func (e *Error) Error() string { + return fmt.Sprintf("http error: code %d, message %s", e.Code, e.Message) } diff --git a/src/jobservice/client/client.go b/src/jobservice/client/client.go new file mode 100644 index 000000000..5288a2f92 --- /dev/null +++ b/src/jobservice/client/client.go @@ -0,0 +1,88 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + + commonhttp "github.com/vmware/harbor/src/common/http" + "github.com/vmware/harbor/src/common/http/client" + "github.com/vmware/harbor/src/jobservice/api" +) + +// Client defines the methods that a jobservice client should implement +type Client interface { + SubmitReplicationJob(*api.ReplicationReq) error +} + +// DefaultClient provides a default implement for the interface Client +type DefaultClient struct { + endpoint string + client client.Client +} + +// NewDefaultClient returns an instance of DefaultClient +func NewDefaultClient(endpoint string, client ...client.Client) *DefaultClient { + c := &DefaultClient{ + endpoint: endpoint, + } + + if len(client) > 0 { + c.client = client[0] + } + + if c.client == nil { + c.client = &http.Client{} + } + + return c +} + +// SubmitReplicationJob submits a replication job to the jobservice +func (d *DefaultClient) SubmitReplicationJob(replication *api.ReplicationReq) error { + url := d.endpoint + "/api/jobs/replication" + + buffer := &bytes.Buffer{} + if err := json.NewEncoder(buffer).Encode(replication); err != nil { + return err + } + + req, err := http.NewRequest(http.MethodPost, url, buffer) + if err != nil { + return err + } + + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + message, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + return &commonhttp.Error{ + Code: resp.StatusCode, + Message: string(message), + } + } + + return nil +} diff --git a/src/jobservice/client/client_test.go b/src/jobservice/client/client_test.go new file mode 100644 index 000000000..0cde379cf --- /dev/null +++ b/src/jobservice/client/client_test.go @@ -0,0 +1,55 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "encoding/json" + "net/http" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/common/utils/test" + "github.com/vmware/harbor/src/jobservice/api" +) + +var url string + +func TestMain(m *testing.M) { + requestMapping := []*test.RequestHandlerMapping{ + &test.RequestHandlerMapping{ + Method: http.MethodPost, + Pattern: "/api/jobs/replication", + Handler: func(w http.ResponseWriter, r *http.Request) { + replication := &api.ReplicationReq{} + if err := json.NewDecoder(r.Body).Decode(replication); err != nil { + w.WriteHeader(http.StatusInternalServerError) + } + }, + }, + } + server := test.NewServer(requestMapping...) + defer server.Close() + + url = server.URL + + os.Exit(m.Run()) +} + +func TestSubmitReplicationJob(t *testing.T) { + client := NewDefaultClient(url) + err := client.SubmitReplicationJob(&api.ReplicationReq{}) + assert.Nil(t, err) +} diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index 674cd058d..4ece1a814 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -20,9 +20,10 @@ import ( "strings" "github.com/vmware/harbor/src/adminserver/client" - "github.com/vmware/harbor/src/adminserver/client/auth" "github.com/vmware/harbor/src/common" comcfg "github.com/vmware/harbor/src/common/config" + httpclient "github.com/vmware/harbor/src/common/http/client" + "github.com/vmware/harbor/src/common/http/client/auth" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" ) @@ -50,8 +51,8 @@ func Init() error { adminServerURL = "http://adminserver" } log.Infof("initializing client for adminserver %s ...", adminServerURL) - authorizer := auth.NewSecretAuthorizer(secretCookieName, UISecret()) - AdminserverClient = client.NewClient(adminServerURL, authorizer) + authorizer := auth.NewSecretAuthorizer(UISecret()) + AdminserverClient = client.NewClient(adminServerURL, httpclient.NewAuthorizedClient(authorizer)) if err := AdminserverClient.Ping(); err != nil { return fmt.Errorf("failed to ping adminserver: %v", err) } diff --git a/src/replication/consts.go b/src/replication/consts.go index a2148823b..e7c142110 100644 --- a/src/replication/consts.go +++ b/src/replication/consts.go @@ -23,11 +23,6 @@ const ( //TriggerScheduleWeekly : type of scheduling is 'weekly' TriggerScheduleWeekly = "weekly" - //OperationPush : push operation - OperationPush = "push" - //OperationDelete : delete operation - OperationDelete = "delete" - // PatternMatchAll : the pattern that match all PatternMatchAll = ".*" ) diff --git a/src/replication/core/controller.go b/src/replication/core/controller.go index a3db273c9..17e40d8fd 100644 --- a/src/replication/core/controller.go +++ b/src/replication/core/controller.go @@ -16,15 +16,21 @@ package core import ( "fmt" + "strings" + "github.com/vmware/harbor/src/common/http/client" + "github.com/vmware/harbor/src/common/http/client/auth" common_models "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/api" "github.com/vmware/harbor/src/replication" "github.com/vmware/harbor/src/replication/models" "github.com/vmware/harbor/src/replication/policy" + "github.com/vmware/harbor/src/replication/replicator" "github.com/vmware/harbor/src/replication/source" "github.com/vmware/harbor/src/replication/target" "github.com/vmware/harbor/src/replication/trigger" + "github.com/vmware/harbor/src/ui/config" ) // Controller defines the methods that a replicatoin controllter should implement @@ -51,6 +57,9 @@ type DefaultController struct { //Manage the triggers of policies triggerManager *trigger.Manager + + //Handle the replication work + replicator replicator.Replicator } //Keep controller as singleton instance @@ -65,14 +74,20 @@ type ControllerConfig struct { } //NewDefaultController is the constructor of DefaultController. -func NewDefaultController(config ControllerConfig) *DefaultController { +func NewDefaultController(cfg ControllerConfig) *DefaultController { //Controller refer the default instances - return &DefaultController{ + ctl := &DefaultController{ policyManager: policy.NewDefaultManager(), targetManager: target.NewDefaultManager(), sourcer: source.NewSourcer(), - triggerManager: trigger.NewManager(config.CacheCapacity), + triggerManager: trigger.NewManager(cfg.CacheCapacity), } + + endpoint := config.InternalJobServiceURL() + client := client.NewAuthorizedClient(auth.NewSecretAuthorizer(config.UISecret())) + ctl.replicator = replicator.NewDefaultReplicator(endpoint, client) + + return ctl } //Init will initialize the controller and the sub components @@ -82,16 +97,8 @@ func (ctl *DefaultController) Init() error { } //Build query parameters - triggerNames := []string{ - replication.TriggerKindSchedule, - } - queryName := "" - for _, name := range triggerNames { - queryName = fmt.Sprintf("%s,%s", queryName, name) - } - //Enable the triggers query := models.QueryParameter{ - TriggerName: queryName, + TriggerType: replication.TriggerKindSchedule, } policies, err := ctl.policyManager.GetPolicies(query) @@ -233,19 +240,22 @@ func (ctl *DefaultController) Replicate(policyID int64, metadata ...map[string]i // prepare candidates for replication candidates = getCandidates(&policy, ctl.sourcer, candidates...) - targets := []*common_models.RepTarget{} - for _, targetID := range policy.TargetIDs { - target, err := ctl.targetManager.GetTarget(targetID) - if err != nil { - return err + // TODO + /* + targets := []*common_models.RepTarget{} + for _, targetID := range policy.TargetIDs { + target, err := ctl.targetManager.GetTarget(targetID) + if err != nil { + return err + } + targets = append(targets, target) } - targets = append(targets, target) - } + */ // TODO merge tags whose repository is same into one struct - // call job service to do the replication - return replicate(candidates, targets) + // submit the replication + return replicate(ctl.replicator, policyID, candidates) } func getCandidates(policy *models.ReplicationPolicy, sourcer *source.Sourcer, candidates ...models.FilterItem) []models.FilterItem { @@ -254,7 +264,7 @@ func getCandidates(policy *models.ReplicationPolicy, sourcer *source.Sourcer, ca candidates = append(candidates, models.FilterItem{ Kind: replication.FilterItemKindProject, Value: namespace, - Operation: replication.OperationPush, + Operation: common_models.RepOpTransfer, }) } } @@ -313,10 +323,28 @@ func buildFilterChain(policy *models.ReplicationPolicy, sourcer *source.Sourcer) return source.NewDefaultFilterChain(filters) } -func replicate(candidates []models.FilterItem, targets []*common_models.RepTarget) error { - // TODO - - log.Infof("replicate candidates %v to targets %v", candidates, targets) +func replicate(replicator replicator.Replicator, policyID int64, candidates []models.FilterItem) error { + repositories := map[string][]string{} + // TODO the operation of all candidates are same for now. Update it after supporting + // replicate deletion + operation := "" + for _, candidate := range candidates { + strs := strings.SplitN(candidate.Value, ":", 2) + repositories[strs[0]] = append(repositories[strs[0]], strs[1]) + operation = candidate.Operation + } + for repository, tags := range repositories { + replication := &api.ReplicationReq{ + PolicyID: policyID, + Repo: repository, + Operation: operation, + TagList: tags, + } + log.Debugf("submiting replication job to jobservice: %v", replication) + if err := replicator.Replicate(replication); err != nil { + return err + } + } return nil } diff --git a/src/replication/event/on_deletion_handler.go b/src/replication/event/on_deletion_handler.go index 356b34d25..ac7b9697b 100644 --- a/src/replication/event/on_deletion_handler.go +++ b/src/replication/event/on_deletion_handler.go @@ -19,7 +19,7 @@ import ( "fmt" "reflect" - "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/replication/event/notification" ) @@ -38,7 +38,7 @@ func (oph *OnDeletionHandler) Handle(value interface{}) error { } notification := value.(notification.OnDeletionNotification) - return checkAndTriggerReplication(notification.Image, replication.OperationDelete) + return checkAndTriggerReplication(notification.Image, models.RepOpDelete) } //IsStateful implements the same method of notification handler interface diff --git a/src/replication/event/on_push_handler.go b/src/replication/event/on_push_handler.go index 306a52341..30ae3447a 100644 --- a/src/replication/event/on_push_handler.go +++ b/src/replication/event/on_push_handler.go @@ -19,6 +19,7 @@ import ( "fmt" "reflect" + common_models "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/notifier" "github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" @@ -45,7 +46,7 @@ func (oph *OnPushHandler) Handle(value interface{}) error { notification := value.(notification.OnPushNotification) - return checkAndTriggerReplication(notification.Image, replication.OperationPush) + return checkAndTriggerReplication(notification.Image, common_models.RepOpTransfer) } //IsStateful implements the same method of notification handler interface @@ -68,7 +69,7 @@ func checkAndTriggerReplication(image, operation string) error { } for _, watchItem := range watchItems { - item := &models.FilterItem{ + item := models.FilterItem{ Kind: replication.FilterItemKindTag, Value: image, Metadata: map[string]interface{}{ @@ -79,7 +80,7 @@ func checkAndTriggerReplication(image, operation string) error { if err := notifier.Publish(topic.StartReplicationTopic, notification.StartReplicationNotification{ PolicyID: watchItem.PolicyID, Metadata: map[string]interface{}{ - "": []*models.FilterItem{item}, + "candidates": []models.FilterItem{item}, }, }); err != nil { return fmt.Errorf("failed to publish replication topic for resource %s, operation %s, policy %d: %v", diff --git a/src/replication/models/policy.go b/src/replication/models/policy.go index 5e6777703..f6ab0217d 100644 --- a/src/replication/models/policy.go +++ b/src/replication/models/policy.go @@ -27,8 +27,8 @@ type QueryParameter struct { //Size of each page, couple with page PageSize int64 - //Query by the name of trigger - TriggerName string + //Query by the type of trigger + TriggerType string //Query by project ID ProjectID int64 diff --git a/src/replication/policy/manager.go b/src/replication/policy/manager.go index bdce2740a..a6049dbfe 100644 --- a/src/replication/policy/manager.go +++ b/src/replication/policy/manager.go @@ -55,6 +55,13 @@ func (m *DefaultManager) GetPolicies(query models.QueryParameter) ([]models.Repl if err != nil { return []models.ReplicationPolicy{}, err } + + if len(query.TriggerType) > 0 { + if ply.Trigger.Kind != query.TriggerType { + continue + } + } + result = append(result, ply) } diff --git a/src/replication/replicator/replicator.go b/src/replication/replicator/replicator.go new file mode 100644 index 000000000..e27ba4da2 --- /dev/null +++ b/src/replication/replicator/replicator.go @@ -0,0 +1,43 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package replicator + +import ( + "github.com/vmware/harbor/src/common/http/client" + "github.com/vmware/harbor/src/jobservice/api" + jobserviceclient "github.com/vmware/harbor/src/jobservice/client" +) + +// Replicator submits the replication work to the jobservice +type Replicator interface { + Replicate(*api.ReplicationReq) error +} + +// DefaultReplicator provides a default implement for Replicator +type DefaultReplicator struct { + client jobserviceclient.Client +} + +// NewDefaultReplicator returns an instance of DefaultReplicator +func NewDefaultReplicator(endpoint string, client ...client.Client) *DefaultReplicator { + return &DefaultReplicator{ + client: jobserviceclient.NewDefaultClient(endpoint, client...), + } +} + +// Replicate ... +func (d *DefaultReplicator) Replicate(replication *api.ReplicationReq) error { + return d.client.SubmitReplicationJob(replication) +} diff --git a/src/adminserver/client/auth/auth_test.go b/src/replication/replicator/replicator_test.go similarity index 57% rename from src/adminserver/client/auth/auth_test.go rename to src/replication/replicator/replicator_test.go index f5743cccc..b2a229cce 100644 --- a/src/adminserver/client/auth/auth_test.go +++ b/src/replication/replicator/replicator_test.go @@ -12,32 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -package auth +package replicator import ( - "net/http" "testing" "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/jobservice/api" ) -func TestAuthorize(t *testing.T) { - cookieName := "secret" - secret := "secret" - authorizer := NewSecretAuthorizer(cookieName, secret) - req, err := http.NewRequest("", "", nil) - if !assert.Nil(t, err, "unexpected error") { - return - } +type fakeJobserviceClient struct{} - err = authorizer.Authorize(req) - if !assert.Nil(t, err, "unexpected error") { - return - } - - cookie, err := req.Cookie(cookieName) - if !assert.Nil(t, err, "unexpected error") { - return - } - assert.Equal(t, secret, cookie.Value, "unexpected cookie") +func (f *fakeJobserviceClient) SubmitReplicationJob(replication *api.ReplicationReq) error { + return nil +} + +func TestReplicate(t *testing.T) { + replicator := NewDefaultReplicator("http://jobservice") + replicator.client = &fakeJobserviceClient{} + assert.Nil(t, replicator.Replicate(&api.ReplicationReq{})) } diff --git a/src/ui/config/config.go b/src/ui/config/config.go index 1fdb34c13..1b798115a 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -23,9 +23,10 @@ import ( "strings" "github.com/vmware/harbor/src/adminserver/client" - "github.com/vmware/harbor/src/adminserver/client/auth" "github.com/vmware/harbor/src/common" comcfg "github.com/vmware/harbor/src/common/config" + httpclient "github.com/vmware/harbor/src/common/http/client" + "github.com/vmware/harbor/src/common/http/client/auth" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/secret" "github.com/vmware/harbor/src/common/utils/log" @@ -73,8 +74,8 @@ func Init() error { // InitByURL Init configurations with given url func InitByURL(adminServerURL string) error { log.Infof("initializing client for adminserver %s ...", adminServerURL) - authorizer := auth.NewSecretAuthorizer(secretCookieName, UISecret()) - AdminserverClient = client.NewClient(adminServerURL, authorizer) + authorizer := auth.NewSecretAuthorizer(UISecret()) + AdminserverClient = client.NewClient(adminServerURL, httpclient.NewAuthorizedClient(authorizer)) if err := AdminserverClient.Ping(); err != nil { return fmt.Errorf("failed to ping adminserver: %v", err) } From 76c88d8d49edf2d288a064912a1cd6e00f32a1de Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 8 Dec 2017 15:30:41 +0800 Subject: [PATCH 22/37] Fix bugs of replicaton --- src/replication/core/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/replication/core/controller.go b/src/replication/core/controller.go index 17e40d8fd..638c41553 100644 --- a/src/replication/core/controller.go +++ b/src/replication/core/controller.go @@ -83,7 +83,7 @@ func NewDefaultController(cfg ControllerConfig) *DefaultController { triggerManager: trigger.NewManager(cfg.CacheCapacity), } - endpoint := config.InternalJobServiceURL() + endpoint := "http://jobservice" client := client.NewAuthorizedClient(auth.NewSecretAuthorizer(config.UISecret())) ctl.replicator = replicator.NewDefaultReplicator(endpoint, client) From 1d2e206ce43c64922dcb21d3ac810e4a1a2c5773 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Mon, 11 Dec 2017 17:11:50 +0800 Subject: [PATCH 23/37] Refactor replication filter chain --- src/replication/consts.go | 3 - src/replication/core/controller.go | 83 ++++++------------ src/replication/core/controller_test.go | 23 ++--- src/replication/source/match.go | 23 +++++ .../source/repository_convertor.go | 5 +- .../source/repository_convertor_test.go | 3 + src/replication/source/repository_filter.go | 80 +++++++++++++++++ .../source/repository_filter_test.go | 75 ++++++++++++++++ src/replication/source/tag_convertor.go | 5 +- src/replication/source/tag_convertor_test.go | 3 + src/replication/source/tag_filter.go | 78 +++++++++++++++++ src/replication/source/tag_filter_test.go | 85 +++++++++++++++++++ 12 files changed, 387 insertions(+), 79 deletions(-) create mode 100644 src/replication/source/match.go create mode 100644 src/replication/source/repository_filter.go create mode 100644 src/replication/source/repository_filter_test.go create mode 100644 src/replication/source/tag_filter.go create mode 100644 src/replication/source/tag_filter_test.go diff --git a/src/replication/consts.go b/src/replication/consts.go index e7c142110..b9a8a36a7 100644 --- a/src/replication/consts.go +++ b/src/replication/consts.go @@ -22,7 +22,4 @@ const ( TriggerScheduleDaily = "daily" //TriggerScheduleWeekly : type of scheduling is 'weekly' TriggerScheduleWeekly = "weekly" - - // PatternMatchAll : the pattern that match all - PatternMatchAll = ".*" ) diff --git a/src/replication/core/controller.go b/src/replication/core/controller.go index 638c41553..33b486003 100644 --- a/src/replication/core/controller.go +++ b/src/replication/core/controller.go @@ -83,7 +83,8 @@ func NewDefaultController(cfg ControllerConfig) *DefaultController { triggerManager: trigger.NewManager(cfg.CacheCapacity), } - endpoint := "http://jobservice" + // TODO read from configuration + endpoint := "http://jobservice:8080" client := client.NewAuthorizedClient(auth.NewSecretAuthorizer(config.UISecret())) ctl.replicator = replicator.NewDefaultReplicator(endpoint, client) @@ -226,19 +227,8 @@ func (ctl *DefaultController) Replicate(policyID int64, metadata ...map[string]i return fmt.Errorf("policy %d not found", policyID) } - candidates := []models.FilterItem{} - if len(metadata) > 0 { - meta := metadata[0]["candidates"] - if meta != nil { - cands, ok := meta.([]models.FilterItem) - if ok { - candidates = append(candidates, cands...) - } - } - } - // prepare candidates for replication - candidates = getCandidates(&policy, ctl.sourcer, candidates...) + candidates := getCandidates(&policy, ctl.sourcer, metadata...) // TODO /* @@ -252,13 +242,23 @@ func (ctl *DefaultController) Replicate(policyID int64, metadata ...map[string]i } */ - // TODO merge tags whose repository is same into one struct - // submit the replication return replicate(ctl.replicator, policyID, candidates) } -func getCandidates(policy *models.ReplicationPolicy, sourcer *source.Sourcer, candidates ...models.FilterItem) []models.FilterItem { +func getCandidates(policy *models.ReplicationPolicy, sourcer *source.Sourcer, + metadata ...map[string]interface{}) []models.FilterItem { + candidates := []models.FilterItem{} + if len(metadata) > 0 { + meta := metadata[0]["candidates"] + if meta != nil { + cands, ok := meta.([]models.FilterItem) + if ok { + candidates = append(candidates, cands...) + } + } + } + if len(candidates) == 0 { for _, namespace := range policy.Namespaces { candidates = append(candidates, models.FilterItem{ @@ -277,53 +277,26 @@ func getCandidates(policy *models.ReplicationPolicy, sourcer *source.Sourcer, ca func buildFilterChain(policy *models.ReplicationPolicy, sourcer *source.Sourcer) source.FilterChain { filters := []source.Filter{} - patternMap := map[string]string{} + patterns := map[string]string{} for _, f := range policy.Filters { - patternMap[f.Kind] = f.Pattern + patterns[f.Kind] = f.Pattern } - // TODO convert wildcard to regex expression - projectPattern, exist := patternMap[replication.FilterItemKindProject] - if !exist { - projectPattern = replication.PatternMatchAll - } - - repositoryPattern, exist := patternMap[replication.FilterItemKindRepository] - if !exist { - repositoryPattern = replication.PatternMatchAll - } - repositoryPattern = fmt.Sprintf("%s/%s", projectPattern, repositoryPattern) - - tagPattern, exist := patternMap[replication.FilterItemKindTag] - if !exist { - tagPattern = replication.PatternMatchAll - } - tagPattern = fmt.Sprintf("%s:%s", repositoryPattern, tagPattern) - - if policy.Trigger != nil && policy.Trigger.Kind == replication.TriggerKindImmediate { - // build filter chain for immediate trigger policy - filters = append(filters, - source.NewPatternFilter(replication.FilterItemKindTag, tagPattern)) - } else { - // build filter chain for manual and schedule trigger policy - - // append project filter - filters = append(filters, - source.NewPatternFilter(replication.FilterItemKindProject, projectPattern)) - // append repository filter - filters = append(filters, - source.NewPatternFilter(replication.FilterItemKindRepository, - repositoryPattern, source.NewRepositoryConvertor(sourcer.GetAdaptor(replication.AdaptorKindHarbor)))) - // append tag filter - filters = append(filters, - source.NewPatternFilter(replication.FilterItemKindTag, - tagPattern, source.NewTagConvertor(sourcer.GetAdaptor(replication.AdaptorKindHarbor)))) - } + registry := sourcer.GetAdaptor(replication.AdaptorKindHarbor) + // only support repository and tag filter for now + filters = append(filters, + source.NewRepositoryFilter(patterns[replication.FilterItemKindRepository], registry)) + filters = append(filters, + source.NewTagFilter(patterns[replication.FilterItemKindTag], registry)) return source.NewDefaultFilterChain(filters) } func replicate(replicator replicator.Replicator, policyID int64, candidates []models.FilterItem) error { + if len(candidates) == 0 { + log.Debugf("replicaton candidates are null, no further action needed") + } + repositories := map[string][]string{} // TODO the operation of all candidates are same for now. Update it after supporting // replicate deletion diff --git a/src/replication/core/controller_test.go b/src/replication/core/controller_test.go index bbfa671f8..2404ebed7 100644 --- a/src/replication/core/controller_test.go +++ b/src/replication/core/controller_test.go @@ -79,7 +79,7 @@ func TestGetCandidates(t *testing.T) { Filters: []models.Filter{ models.Filter{ Kind: replication.FilterItemKindTag, - Pattern: ".*", + Pattern: "*", }, }, Trigger: &models.Trigger{ @@ -99,16 +99,19 @@ func TestGetCandidates(t *testing.T) { Value: "library/hello-world:latest", }, } - result := getCandidates(policy, sourcer, candidates...) + metadata := map[string]interface{}{ + "candidates": candidates, + } + result := getCandidates(policy, sourcer, metadata) assert.Equal(t, 2, len(result)) policy.Filters = []models.Filter{ models.Filter{ Kind: replication.FilterItemKindTag, - Pattern: "release-.*", + Pattern: "release-*", }, } - result = getCandidates(policy, sourcer, candidates...) + result = getCandidates(policy, sourcer, metadata) assert.Equal(t, 1, len(result)) } @@ -116,10 +119,6 @@ func TestBuildFilterChain(t *testing.T) { policy := &models.ReplicationPolicy{ ID: 1, Filters: []models.Filter{ - models.Filter{ - Kind: replication.FilterItemKindProject, - Pattern: "*", - }, models.Filter{ Kind: replication.FilterItemKindRepository, Pattern: "*", @@ -134,11 +133,5 @@ func TestBuildFilterChain(t *testing.T) { sourcer := source.NewSourcer() chain := buildFilterChain(policy, sourcer) - assert.Equal(t, 3, len(chain.Filters())) - - policy.Trigger = &models.Trigger{ - Kind: replication.TriggerKindImmediate, - } - chain = buildFilterChain(policy, sourcer) - assert.Equal(t, 1, len(chain.Filters())) + assert.Equal(t, 2, len(chain.Filters())) } diff --git a/src/replication/source/match.go b/src/replication/source/match.go new file mode 100644 index 000000000..9e09dcbd2 --- /dev/null +++ b/src/replication/source/match.go @@ -0,0 +1,23 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "path/filepath" +) + +func match(pattern, str string) (bool, error) { + return filepath.Match(pattern, str) +} diff --git a/src/replication/source/repository_convertor.go b/src/replication/source/repository_convertor.go index b2ff6cd58..db39d775d 100644 --- a/src/replication/source/repository_convertor.go +++ b/src/replication/source/repository_convertor.go @@ -15,7 +15,6 @@ package source import ( - "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/replication" "github.com/vmware/harbor/src/replication/models" "github.com/vmware/harbor/src/replication/registry" @@ -39,9 +38,9 @@ func (r *RepositoryConvertor) Convert(items []models.FilterItem) []models.Filter // if support replicate deletion result := []models.FilterItem{} for _, item := range items { + // just put it to the result list if the item is not a project if item.Kind != replication.FilterItemKindProject { - log.Warningf("unexpected filter item kind for repository convertor, expected %s got %s, skip", - replication.FilterItemKindProject, item.Kind) + result = append(result, item) continue } diff --git a/src/replication/source/repository_convertor_test.go b/src/replication/source/repository_convertor_test.go index f260f113d..2ee1f5183 100644 --- a/src/replication/source/repository_convertor_test.go +++ b/src/replication/source/repository_convertor_test.go @@ -41,6 +41,9 @@ func TestRepositoryConvert(t *testing.T) { Kind: replication.FilterItemKindRepository, Value: "library/centos", }, + models.FilterItem{ + Kind: replication.FilterItemKindRepository, + }, } convertor := NewRepositoryConvertor(&fakeRegistryAdaptor{}) diff --git a/src/replication/source/repository_filter.go b/src/replication/source/repository_filter.go new file mode 100644 index 000000000..48af422cc --- /dev/null +++ b/src/replication/source/repository_filter.go @@ -0,0 +1,80 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "strings" + + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" + "github.com/vmware/harbor/src/replication/registry" +) + +// RepositoryFilter implement Filter interface to filter repository +type RepositoryFilter struct { + pattern string + convertor Convertor +} + +// NewRepositoryFilter returns an instance of RepositoryFilter +func NewRepositoryFilter(pattern string, registry registry.Adaptor) *RepositoryFilter { + return &RepositoryFilter{ + pattern: pattern, + convertor: NewRepositoryConvertor(registry), + } +} + +// Init ... +func (r *RepositoryFilter) Init() error { + return nil +} + +// GetConvertor ... +func (r *RepositoryFilter) GetConvertor() Convertor { + return r.convertor +} + +// DoFilter filters repository and image(according to the repository part) and drops any other resource types +func (r *RepositoryFilter) DoFilter(items []models.FilterItem) []models.FilterItem { + result := []models.FilterItem{} + for _, item := range items { + if item.Kind != replication.FilterItemKindRepository && item.Kind != replication.FilterItemKindTag { + log.Warningf("unsupported type %s for repository filter, drop", item.Kind) + continue + } + + repository := item.Value + if item.Kind == replication.FilterItemKindTag { + repository = strings.SplitN(repository, ":", 2)[0] + } + + if len(r.pattern) == 0 { + log.Debugf("pattern is null, add %s to the repository filter result list", item.Value) + result = append(result, item) + } else { + matched, err := match(r.pattern, repository) + if err != nil { + log.Errorf("failed to match pattern %s to value %s: %v", r.pattern, repository, err) + break + } + if matched { + log.Debugf("pattern %s matched, add %s to the repository filter result list", r.pattern, item.Value) + result = append(result, item) + } + } + } + return result +} diff --git a/src/replication/source/repository_filter_test.go b/src/replication/source/repository_filter_test.go new file mode 100644 index 000000000..6ab515323 --- /dev/null +++ b/src/replication/source/repository_filter_test.go @@ -0,0 +1,75 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" + "github.com/vmware/harbor/src/replication/registry" +) + +func TestInitOfRepositoryFilter(t *testing.T) { + filter := NewRepositoryFilter("", ®istry.HarborAdaptor{}) + assert.Nil(t, filter.Init()) +} + +func TestGetConvertorOfRepositoryFilter(t *testing.T) { + filter := NewRepositoryFilter("", ®istry.HarborAdaptor{}) + assert.NotNil(t, filter.GetConvertor()) +} + +func TestDoFilterOfRepositoryFilter(t *testing.T) { + // invalid filter item type + filter := NewRepositoryFilter("", ®istry.HarborAdaptor{}) + items := filter.DoFilter([]models.FilterItem{ + models.FilterItem{ + Kind: "invalid_type", + }, + }) + assert.Equal(t, 0, len(items)) + + // empty pattern + filter = NewRepositoryFilter("", ®istry.HarborAdaptor{}) + items = filter.DoFilter([]models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindRepository, + Value: "library/hello-world", + }, + }) + assert.Equal(t, 1, len(items)) + + // non-empty pattern + filter = NewRepositoryFilter("library/*", ®istry.HarborAdaptor{}) + items = filter.DoFilter([]models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/hello-world", + }, + }) + assert.Equal(t, 1, len(items)) + + // non-empty pattern + filter = NewRepositoryFilter("library/*", ®istry.HarborAdaptor{}) + items = filter.DoFilter([]models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/hello-world:latest", + }, + }) + assert.Equal(t, 1, len(items)) +} diff --git a/src/replication/source/tag_convertor.go b/src/replication/source/tag_convertor.go index 09adcecf9..80e7f29f7 100644 --- a/src/replication/source/tag_convertor.go +++ b/src/replication/source/tag_convertor.go @@ -15,7 +15,6 @@ package source import ( - "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/replication" "github.com/vmware/harbor/src/replication/models" "github.com/vmware/harbor/src/replication/registry" @@ -38,8 +37,8 @@ func (t *TagConvertor) Convert(items []models.FilterItem) []models.FilterItem { result := []models.FilterItem{} for _, item := range items { if item.Kind != replication.FilterItemKindRepository { - log.Warningf("unexpected filter item kind for tag convertor, expected %s got %s, skip", - replication.FilterItemKindRepository, item.Kind) + // just put it to the result list if the item is not a repository + result = append(result, item) continue } diff --git a/src/replication/source/tag_convertor_test.go b/src/replication/source/tag_convertor_test.go index 6e64f86d2..17c244c1d 100644 --- a/src/replication/source/tag_convertor_test.go +++ b/src/replication/source/tag_convertor_test.go @@ -41,6 +41,9 @@ func TestTagConvert(t *testing.T) { Kind: replication.FilterItemKindTag, Value: "library/ubuntu:16.04", }, + models.FilterItem{ + Kind: replication.FilterItemKindProject, + }, } convertor := NewTagConvertor(&fakeRegistryAdaptor{}) diff --git a/src/replication/source/tag_filter.go b/src/replication/source/tag_filter.go new file mode 100644 index 000000000..bdf9158e0 --- /dev/null +++ b/src/replication/source/tag_filter.go @@ -0,0 +1,78 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "strings" + + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" + "github.com/vmware/harbor/src/replication/registry" +) + +// TagFilter implements Filter interface to filter tag +type TagFilter struct { + pattern string + convertor Convertor +} + +// NewTagFilter returns an instance of TagFilter +func NewTagFilter(pattern string, registry registry.Adaptor) *TagFilter { + return &TagFilter{ + pattern: pattern, + convertor: NewTagConvertor(registry), + } +} + +// Init ... +func (t *TagFilter) Init() error { + return nil +} + +// GetConvertor ... +func (t *TagFilter) GetConvertor() Convertor { + return t.convertor +} + +// DoFilter filters tag of the image +func (t *TagFilter) DoFilter(items []models.FilterItem) []models.FilterItem { + result := []models.FilterItem{} + for _, item := range items { + if item.Kind != replication.FilterItemKindTag { + log.Warningf("unsupported type %s for tag filter, dropped", item.Kind) + continue + } + + if len(t.pattern) == 0 { + log.Debugf("pattern is null, add %s to the tag filter result list", item.Value) + result = append(result, item) + continue + } + + tag := strings.SplitN(item.Value, ":", 2)[1] + matched, err := match(t.pattern, tag) + if err != nil { + log.Errorf("failed to match pattern %s to value %s: %v", t.pattern, tag, err) + continue + } + + if matched { + log.Debugf("pattern %s matched, add %s to the tag filter result list", t.pattern, item.Value) + result = append(result, item) + } + } + return result +} diff --git a/src/replication/source/tag_filter_test.go b/src/replication/source/tag_filter_test.go new file mode 100644 index 000000000..8114852ce --- /dev/null +++ b/src/replication/source/tag_filter_test.go @@ -0,0 +1,85 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/src/replication/models" + "github.com/vmware/harbor/src/replication/registry" +) + +func TestInitOfTagFilter(t *testing.T) { + filter := NewTagFilter("", ®istry.HarborAdaptor{}) + assert.Nil(t, filter.Init()) +} + +func TestGetConvertorOfTagFilter(t *testing.T) { + filter := NewTagFilter("", ®istry.HarborAdaptor{}) + assert.NotNil(t, filter.GetConvertor()) +} + +func TestDoFilterOfTagFilter(t *testing.T) { + // invalid filter item type + filter := NewTagFilter("", ®istry.HarborAdaptor{}) + items := filter.DoFilter([]models.FilterItem{ + models.FilterItem{ + Kind: "invalid_type", + }, + }) + assert.Equal(t, 0, len(items)) + + // empty pattern + filter = NewTagFilter("", ®istry.HarborAdaptor{}) + items = filter.DoFilter([]models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/hello-world:latest", + }, + }) + assert.Equal(t, 1, len(items)) + + // non-empty pattern + filter = NewTagFilter("l*t", ®istry.HarborAdaptor{}) + items = filter.DoFilter([]models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/hello-world:latest", + }, + }) + assert.Equal(t, 1, len(items)) + + // non-empty pattern + filter = NewTagFilter("lates?", ®istry.HarborAdaptor{}) + items = filter.DoFilter([]models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/hello-world:latest", + }, + }) + assert.Equal(t, 1, len(items)) + + // non-empty pattern + filter = NewTagFilter("latest?", ®istry.HarborAdaptor{}) + items = filter.DoFilter([]models.FilterItem{ + models.FilterItem{ + Kind: replication.FilterItemKindTag, + Value: "library/hello-world:latest", + }, + }) + assert.Equal(t, 0, len(items)) +} From 055ab0ba15d49b0fecae3afe9109bcf728fb9d78 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Tue, 12 Dec 2017 15:56:46 +0800 Subject: [PATCH 24/37] Refine replication schedule trigger API --- src/replication/core/controller.go | 7 +- src/replication/core/controller_test.go | 6 +- src/replication/models/trigger.go | 49 ++++++++- src/replication/models/trigger_test.go | 36 ++++++ src/replication/source/repository_filter.go | 6 + src/replication/source/tag_filter.go | 6 + src/replication/trigger/manager.go | 116 +++++++++----------- src/replication/trigger/schedule.go | 5 +- src/ui/api/replication_policy_test.go | 6 +- 9 files changed, 154 insertions(+), 83 deletions(-) diff --git a/src/replication/core/controller.go b/src/replication/core/controller.go index 33b486003..38a0a93f9 100644 --- a/src/replication/core/controller.go +++ b/src/replication/core/controller.go @@ -160,7 +160,7 @@ func (ctl *DefaultController) UpdatePolicy(updatedPolicy models.ReplicationPolic } else { switch updatedPolicy.Trigger.Kind { case replication.TriggerKindSchedule: - if updatedPolicy.Trigger.Param != originPolicy.Trigger.Param { + if !originPolicy.Trigger.ScheduleParam.Equal(updatedPolicy.Trigger.ScheduleParam) { reset = true } case replication.TriggerKindImmediate: @@ -176,7 +176,7 @@ func (ctl *DefaultController) UpdatePolicy(updatedPolicy models.ReplicationPolic } if reset { - if err = ctl.triggerManager.UnsetTrigger(id, *originPolicy.Trigger); err != nil { + if err = ctl.triggerManager.UnsetTrigger(&originPolicy); err != nil { return err } @@ -199,7 +199,7 @@ func (ctl *DefaultController) RemovePolicy(policyID int64) error { return fmt.Errorf("policy %d not found", policyID) } - if err = ctl.triggerManager.UnsetTrigger(policyID, *policy.Trigger); err != nil { + if err = ctl.triggerManager.UnsetTrigger(&policy); err != nil { return err } @@ -230,7 +230,6 @@ func (ctl *DefaultController) Replicate(policyID int64, metadata ...map[string]i // prepare candidates for replication candidates := getCandidates(&policy, ctl.sourcer, metadata...) - // TODO /* targets := []*common_models.RepTarget{} for _, targetID := range policy.TargetIDs { diff --git a/src/replication/core/controller_test.go b/src/replication/core/controller_test.go index 2404ebed7..58b1935a5 100644 --- a/src/replication/core/controller_test.go +++ b/src/replication/core/controller_test.go @@ -42,7 +42,11 @@ func TestInit(t *testing.T) { } func TestCreatePolicy(t *testing.T) { - _, err := GlobalController.CreatePolicy(models.ReplicationPolicy{}) + _, err := GlobalController.CreatePolicy(models.ReplicationPolicy{ + Trigger: &models.Trigger{ + Kind: replication.TriggerKindManual, + }, + }) assert.Nil(t, err) } diff --git a/src/replication/models/trigger.go b/src/replication/models/trigger.go index 27c2afa69..4af3e5329 100644 --- a/src/replication/models/trigger.go +++ b/src/replication/models/trigger.go @@ -23,11 +23,8 @@ import ( //Trigger is replication launching approach definition type Trigger struct { - //The name of the trigger - Kind string `json:"kind"` - - //The parameters with json text format required by the trigger - Param string `json:"param"` + Kind string `json:"kind"` // the type of the trigger + ScheduleParam *ScheduleParam `json:"schedule_param"` // optional, only used when kind is 'schedule' } // Valid ... @@ -37,4 +34,46 @@ func (t *Trigger) Valid(v *validation.Validation) { t.Kind == replication.TriggerKindSchedule) { v.SetError("kind", fmt.Sprintf("invalid trigger kind: %s", t.Kind)) } + + if t.Kind == replication.TriggerKindSchedule { + if t.ScheduleParam == nil { + v.SetError("schedule_param", "empty schedule_param") + } else { + t.ScheduleParam.Valid(v) + } + } +} + +// ScheduleParam defines the parameters used by schedule trigger +type ScheduleParam struct { + Type string `json:"type"` //daily or weekly + Weekday int8 `json:"weekday"` //Optional, only used when type is 'weekly' + Offtime int64 `json:"offtime"` //The time offset with the UTC 00:00 in seconds +} + +// Valid ... +func (s *ScheduleParam) Valid(v *validation.Validation) { + if !(s.Type == replication.TriggerScheduleDaily || + s.Type == replication.TriggerScheduleWeekly) { + v.SetError("type", fmt.Sprintf("invalid schedule trigger parameter type: %s", s.Type)) + } + + if s.Type == replication.TriggerScheduleWeekly { + if s.Weekday < 1 || s.Weekday > 7 { + v.SetError("weekday", fmt.Sprintf("invalid schedule trigger parameter weekday: %d", s.Weekday)) + } + } + + if s.Offtime < 0 || s.Offtime > 3600*24 { + v.SetError("offtime", fmt.Sprintf("invalid schedule trigger parameter offtime: %d", s.Offtime)) + } +} + +// Equal ... +func (s *ScheduleParam) Equal(param *ScheduleParam) bool { + if param == nil { + return false + } + + return s.Type == param.Type && s.Weekday == param.Weekday && s.Offtime == param.Offtime } diff --git a/src/replication/models/trigger_test.go b/src/replication/models/trigger_test.go index b591995e9..2aba67a44 100644 --- a/src/replication/models/trigger_test.go +++ b/src/replication/models/trigger_test.go @@ -31,6 +31,9 @@ func TestValidOfTrigger(t *testing.T) { &Trigger{ Kind: replication.TriggerKindImmediate, }: false, + &Trigger{ + Kind: replication.TriggerKindSchedule, + }: true, } for filter, hasError := range cases { @@ -39,3 +42,36 @@ func TestValidOfTrigger(t *testing.T) { assert.Equal(t, hasError, v.HasErrors()) } } + +func TestValidOfScheduleParam(t *testing.T) { + cases := map[*ScheduleParam]bool{ + &ScheduleParam{}: true, + &ScheduleParam{ + Type: "invalid_type", + }: true, + &ScheduleParam{ + Type: replication.TriggerScheduleDaily, + Offtime: 3600*24 + 1, + }: true, + &ScheduleParam{ + Type: replication.TriggerScheduleDaily, + Offtime: 3600 * 2, + }: false, + &ScheduleParam{ + Type: replication.TriggerScheduleWeekly, + Weekday: 0, + Offtime: 3600 * 2, + }: true, + &ScheduleParam{ + Type: replication.TriggerScheduleWeekly, + Weekday: 7, + Offtime: 3600 * 2, + }: false, + } + + for param, hasError := range cases { + v := &validation.Validation{} + param.Valid(v) + assert.Equal(t, hasError, v.HasErrors()) + } +} diff --git a/src/replication/source/repository_filter.go b/src/replication/source/repository_filter.go index 48af422cc..78258dfe2 100644 --- a/src/replication/source/repository_filter.go +++ b/src/replication/source/repository_filter.go @@ -49,6 +49,12 @@ func (r *RepositoryFilter) GetConvertor() Convertor { // DoFilter filters repository and image(according to the repository part) and drops any other resource types func (r *RepositoryFilter) DoFilter(items []models.FilterItem) []models.FilterItem { + candidates := []string{} + for _, item := range items { + candidates = append(candidates, item.Value) + } + log.Debugf("repository filter candidates: %v", candidates) + result := []models.FilterItem{} for _, item := range items { if item.Kind != replication.FilterItemKindRepository && item.Kind != replication.FilterItemKindTag { diff --git a/src/replication/source/tag_filter.go b/src/replication/source/tag_filter.go index bdf9158e0..f6fc5db92 100644 --- a/src/replication/source/tag_filter.go +++ b/src/replication/source/tag_filter.go @@ -49,6 +49,12 @@ func (t *TagFilter) GetConvertor() Convertor { // DoFilter filters tag of the image func (t *TagFilter) DoFilter(items []models.FilterItem) []models.FilterItem { + candidates := []string{} + for _, item := range items { + candidates = append(candidates, item.Value) + } + log.Debugf("tag filter candidates: %v", candidates) + result := []models.FilterItem{} for _, item := range items { if item.Kind != replication.FilterItemKindTag { diff --git a/src/replication/trigger/manager.go b/src/replication/trigger/manager.go index 83969855c..10f6eaf83 100644 --- a/src/replication/trigger/manager.go +++ b/src/replication/trigger/manager.go @@ -1,7 +1,6 @@ package trigger import ( - "errors" "fmt" "github.com/vmware/harbor/src/common/utils/log" @@ -55,88 +54,71 @@ func (m *Manager) RemoveTrigger(policyID int64) error { //SetupTrigger will create the new trigger based on the provided policy. //If failed, an error will be returned. func (m *Manager) SetupTrigger(policy *models.ReplicationPolicy) error { - if policy == nil || policy.Trigger == nil { - log.Debug("empty policy or trigger, skip trigger setup") + trigger, err := createTrigger(policy) + if err != nil { + return err + } + + // manual trigger, do nothing + if trigger == nil { return nil } + tg := trigger.(Interface) + if err = tg.Setup(); err != nil { + return err + } + + log.Debugf("%s trigger for policy %d is set", tg.Kind(), policy.ID) + return nil +} + +//UnsetTrigger will disable the trigger which is not cached in the trigger cache. +func (m *Manager) UnsetTrigger(policy *models.ReplicationPolicy) error { + trigger, err := createTrigger(policy) + if err != nil { + return err + } + + // manual trigger, do nothing + if trigger == nil { + return nil + } + + tg := trigger.(Interface) + if err = tg.Unset(); err != nil { + return err + } + + log.Debugf("%s trigger for policy %d is unset", tg.Kind(), policy.ID) + return nil +} + +func createTrigger(policy *models.ReplicationPolicy) (interface{}, error) { + if policy == nil || policy.Trigger == nil { + return nil, fmt.Errorf("empty policy or trigger") + } + trigger := policy.Trigger switch trigger.Kind { case replication.TriggerKindSchedule: param := ScheduleParam{} - if err := param.Parse(trigger.Param); err != nil { - return err - } - //Append policy ID and whether replicate deletion param.PolicyID = policy.ID - param.OnDeletion = policy.ReplicateDeletion + param.Type = trigger.ScheduleParam.Type + param.Weekday = trigger.ScheduleParam.Weekday + param.Offtime = trigger.ScheduleParam.Offtime - newTrigger := NewScheduleTrigger(param) - if err := newTrigger.Setup(); err != nil { - return err - } + return NewScheduleTrigger(param), nil case replication.TriggerKindImmediate: param := ImmediateParam{} - if err := param.Parse(trigger.Param); err != nil { - return err - } - //Append policy ID and whether replicate deletion param.PolicyID = policy.ID param.OnDeletion = policy.ReplicateDeletion param.Namespaces = policy.Namespaces - newTrigger := NewImmediateTrigger(param) - if err := newTrigger.Setup(); err != nil { - return err - } + return NewImmediateTrigger(param), nil case replication.TriggerKindManual: - // do nothing + return nil, nil default: - return fmt.Errorf("invalid trigger type: %s", policy.Trigger.Kind) + return nil, fmt.Errorf("invalid trigger type: %s", trigger.Kind) } - - return nil -} - -//UnsetTrigger will disable the trigger which is not cached in the trigger cache. -func (m *Manager) UnsetTrigger(policyID int64, trigger models.Trigger) error { - if policyID <= 0 { - return errors.New("Invalid policy ID") - } - - if len(trigger.Kind) == 0 { - return errors.New("Invalid replication trigger definition") - } - - switch trigger.Kind { - case replication.TriggerKindSchedule: - param := ScheduleParam{} - if err := param.Parse(trigger.Param); err != nil { - return err - } - //Append policy ID info - param.PolicyID = policyID - - newTrigger := NewScheduleTrigger(param) - if err := newTrigger.Unset(); err != nil { - return err - } - case replication.TriggerKindImmediate: - param := ImmediateParam{} - if err := param.Parse(trigger.Param); err != nil { - return err - } - //Append policy ID info - param.PolicyID = policyID - - newTrigger := NewImmediateTrigger(param) - if err := newTrigger.Unset(); err != nil { - return err - } - default: - //Treat as manual trigger - break - } - - return nil } diff --git a/src/replication/trigger/schedule.go b/src/replication/trigger/schedule.go index 3a8480919..5d5b24aad 100644 --- a/src/replication/trigger/schedule.go +++ b/src/replication/trigger/schedule.go @@ -2,6 +2,7 @@ package trigger import ( "fmt" + "time" "github.com/vmware/harbor/src/common/scheduler" "github.com/vmware/harbor/src/common/scheduler/policy" @@ -31,10 +32,10 @@ func (st *ScheduleTrigger) Setup() error { config := &policy.AlternatePolicyConfiguration{} switch st.params.Type { case replication.TriggerScheduleDaily: - config.Duration = 24 * 3600 + config.Duration = 24 * 3600 * time.Second config.OffsetTime = st.params.Offtime case replication.TriggerScheduleWeekly: - config.Duration = 7 * 24 * 3600 + config.Duration = 7 * 24 * 3600 * time.Second config.OffsetTime = st.params.Offtime config.Weekday = st.params.Weekday default: diff --git a/src/ui/api/replication_policy_test.go b/src/ui/api/replication_policy_test.go index 0bf688105..9698d5df1 100644 --- a/src/ui/api/replication_policy_test.go +++ b/src/ui/api/replication_policy_test.go @@ -471,8 +471,7 @@ func TestConvertToRepPolicy(t *testing.T) { }, ReplicateDeletion: true, Trigger: &rep_models.Trigger{ - Kind: "trigger_kind_01", - Param: "{param}", + Kind: "trigger_kind_01", }, Projects: []*models.Project{ &models.Project{ @@ -498,8 +497,7 @@ func TestConvertToRepPolicy(t *testing.T) { }, ReplicateDeletion: true, Trigger: &rep_models.Trigger{ - Kind: "trigger_kind_01", - Param: "{param}", + Kind: "trigger_kind_01", }, ProjectIDs: []int64{1}, Namespaces: []string{"library"}, From b5e7de331e263a9c9484517ef96478168d3eec65 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Tue, 12 Dec 2017 20:44:41 +0800 Subject: [PATCH 25/37] Delete enabled and start_time properties of replication rule --- src/common/dao/dao_test.go | 51 ++---------------------- src/common/dao/replication_job.go | 50 ++++------------------- src/common/models/replication_job.go | 2 - src/jobservice/job/job_test.go | 2 - src/jobservice/job/jobs.go | 6 --- src/jobservice/job/statemachine.go | 10 ----- src/replication/event/on_push_handler.go | 8 ++-- src/ui/api/dataprepare_test.go | 4 -- src/ui/api/target.go | 17 -------- 9 files changed, 14 insertions(+), 136 deletions(-) diff --git a/src/common/dao/dao_test.go b/src/common/dao/dao_test.go index 03252787e..c169f11ba 100644 --- a/src/common/dao/dao_test.go +++ b/src/common/dao/dao_test.go @@ -941,7 +941,6 @@ func TestFilterRepTargets(t *testing.T) { func TestAddRepPolicy(t *testing.T) { policy := models.RepPolicy{ ProjectID: 1, - Enabled: 1, TargetID: targetID, Description: "whatever", Name: "mypolicy", @@ -961,15 +960,10 @@ func TestAddRepPolicy(t *testing.T) { t.Errorf("Unable to find a policy with id: %d", id) } - if p.Name != "mypolicy" || p.TargetID != targetID || p.Enabled != 1 || p.Description != "whatever" { - t.Errorf("The data does not match, expected: Name: mypolicy, TargetID: %d, Enabled: 1, Description: whatever;\n result: Name: %s, TargetID: %d, Enabled: %d, Description: %s", - targetID, p.Name, p.TargetID, p.Enabled, p.Description) + if p.Name != "mypolicy" || p.TargetID != targetID || p.Description != "whatever" { + t.Errorf("The data does not match, expected: Name: mypolicy, TargetID: %d, Description: whatever;\n result: Name: %s, TargetID: %d, Description: %s", + targetID, p.Name, p.TargetID, p.Description) } - var tm = time.Now().AddDate(0, 0, -1) - if !p.StartTime.After(tm) { - t.Errorf("Unexpected start_time: %v", p.StartTime) - } - } func TestGetRepPolicyByTarget(t *testing.T) { @@ -1019,44 +1013,9 @@ func TestGetRepPolicyByName(t *testing.T) { } -func TestDisableRepPolicy(t *testing.T) { - err := DisableRepPolicy(policyID) - if err != nil { - t.Errorf("Failed to disable policy, id: %d", policyID) - } - p, err := GetRepPolicy(policyID) - if err != nil { - t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, policyID) - } - if p == nil { - t.Errorf("Unable to find a policy with id: %d", policyID) - } - if p.Enabled == 1 { - t.Errorf("The Enabled value of replication policy is still 1 after disabled, id: %d", policyID) - } -} - -func TestEnableRepPolicy(t *testing.T) { - err := EnableRepPolicy(policyID) - if err != nil { - t.Errorf("Failed to disable policy, id: %d", policyID) - } - p, err := GetRepPolicy(policyID) - if err != nil { - t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, policyID) - } - if p == nil { - t.Errorf("Unable to find a policy with id: %d", policyID) - } - if p.Enabled == 0 { - t.Errorf("The Enabled value of replication policy is still 0 after disabled, id: %d", policyID) - } -} - func TestAddRepPolicy2(t *testing.T) { policy2 := models.RepPolicy{ ProjectID: 3, - Enabled: 0, TargetID: 3, Description: "whatever", Name: "mypolicy", @@ -1073,10 +1032,6 @@ func TestAddRepPolicy2(t *testing.T) { if p == nil { t.Errorf("Unable to find a policy with id: %d", policyID2) } - var tm time.Time - if p.StartTime.After(tm) { - t.Errorf("Unexpected start_time: %v", p.StartTime) - } } func TestAddRepJob(t *testing.T) { diff --git a/src/common/dao/replication_job.go b/src/common/dao/replication_job.go index 4087e3d13..cfdadc3eb 100644 --- a/src/common/dao/replication_job.go +++ b/src/common/dao/replication_job.go @@ -106,17 +106,13 @@ func FilterRepTargets(name string) ([]*models.RepTarget, error) { // AddRepPolicy ... func AddRepPolicy(policy models.RepPolicy) (int64, error) { o := GetOrmer() - sql := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, start_time, creation_time, update_time, filters, replicate_deletion) - values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + sql := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, creation_time, update_time, filters, replicate_deletion) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` params := []interface{}{} - params = append(params, policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.Trigger) now := time.Now() - if policy.Enabled == 1 { - params = append(params, now) - } else { - params = append(params, nil) - } - params = append(params, now, now, policy.Filters, policy.ReplicateDeletion) + params = append(params, policy.Name, policy.ProjectID, policy.TargetID, 1, + policy.Description, policy.Trigger, now, now, policy.Filters, + policy.ReplicateDeletion) result, err := o.Raw(sql, params...).Exec() if err != nil { @@ -150,8 +146,8 @@ func FilterRepPolicies(name string, projectID int64) ([]*models.RepPolicy, error var args []interface{} sql := `select rp.id, rp.project_id, rp.target_id, - rt.name as target_name, rp.name, rp.enabled, rp.description, - rp.cron_str, rp.filters, rp.replicate_deletion,rp.start_time, + rt.name as target_name, rp.name, rp.description, + rp.cron_str, rp.filters, rp.replicate_deletion, rp.creation_time, rp.update_time, count(rj.status) as error_job_count from replication_policy rp @@ -245,7 +241,7 @@ func GetRepPolicyByProjectAndTarget(projectID, targetID int64) ([]*models.RepPol func UpdateRepPolicy(policy *models.RepPolicy) error { o := GetOrmer() policy.UpdateTime = time.Now() - _, err := o.Update(policy, "TargetID", "Name", "Enabled", "Description", + _, err := o.Update(policy, "TargetID", "Name", "Description", "Trigger", "Filters", "ReplicateDeletion", "UpdateTime") return err } @@ -262,36 +258,6 @@ func DeleteRepPolicy(id int64) error { return err } -// UpdateRepPolicyEnablement ... -func UpdateRepPolicyEnablement(id int64, enabled int) error { - o := GetOrmer() - p := models.RepPolicy{ - ID: id, - Enabled: enabled, - UpdateTime: time.Now(), - } - - var err error - if enabled == 1 { - p.StartTime = time.Now() - _, err = o.Update(&p, "Enabled", "StartTime") - } else { - _, err = o.Update(&p, "Enabled") - } - - return err -} - -// EnableRepPolicy ... -func EnableRepPolicy(id int64) error { - return UpdateRepPolicyEnablement(id, 1) -} - -// DisableRepPolicy ... -func DisableRepPolicy(id int64) error { - return UpdateRepPolicyEnablement(id, 0) -} - // AddRepJob ... func AddRepJob(job models.RepJob) (int64, error) { o := GetOrmer() diff --git a/src/common/models/replication_job.go b/src/common/models/replication_job.go index abf011a34..8c5744800 100644 --- a/src/common/models/replication_job.go +++ b/src/common/models/replication_job.go @@ -42,12 +42,10 @@ type RepPolicy struct { ProjectID int64 `orm:"column(project_id)" ` TargetID int64 `orm:"column(target_id)"` Name string `orm:"column(name)"` - Enabled int `orm:"column(enabled)"` Description string `orm:"column(description)"` Trigger string `orm:"column(cron_str)"` Filters string `orm:"column(filters)"` ReplicateDeletion bool `orm:"column(replicate_deletion)"` - StartTime time.Time `orm:"column(start_time)"` CreationTime time.Time `orm:"column(creation_time);auto_now_add"` UpdateTime time.Time `orm:"column(update_time);auto_now"` Deleted int `orm:"column(deleted)"` diff --git a/src/jobservice/job/job_test.go b/src/jobservice/job/job_test.go index ab52c9487..eb29c93cc 100644 --- a/src/jobservice/job/job_test.go +++ b/src/jobservice/job/job_test.go @@ -105,7 +105,6 @@ func TestRepJob(t *testing.T) { assert.Nil(err) j, err := dao.GetRepJob(repJobID) assert.Equal(models.JobRetrying, j.Status) - assert.Equal(1, rj.parm.Enabled) assert.False(rj.parm.Insecure) rj2 := NewRepJob(99999) err = rj2.Init() @@ -163,7 +162,6 @@ func prepareRepJobData() error { } policy := models.RepPolicy{ ProjectID: 1, - Enabled: 1, TargetID: targetID, Description: "whatever", Name: "mypolicy", diff --git a/src/jobservice/job/jobs.go b/src/jobservice/job/jobs.go index 96a3bc821..1752f9164 100644 --- a/src/jobservice/job/jobs.go +++ b/src/jobservice/job/jobs.go @@ -62,7 +62,6 @@ type RepJobParm struct { TargetPassword string Repository string Tags []string - Enabled int Operation string Insecure bool } @@ -124,13 +123,8 @@ func (rj *RepJob) Init() error { LocalRegURL: regURL, Repository: job.Repository, Tags: job.TagList, - Enabled: policy.Enabled, Operation: job.Operation, } - if policy.Enabled == 0 { - //worker will cancel this job - return nil - } target, err := dao.GetRepTarget(policy.TargetID) if err != nil { return fmt.Errorf("Failed to get target, error: %v", err) diff --git a/src/jobservice/job/statemachine.go b/src/jobservice/job/statemachine.go index 3db88a78f..8b41ebdf7 100644 --- a/src/jobservice/job/statemachine.go +++ b/src/jobservice/job/statemachine.go @@ -208,16 +208,6 @@ func (sm *SM) Reset(j Job) error { } func (sm *SM) kickOff() error { - if repJob, ok := sm.CurrentJob.(*RepJob); ok { - if repJob.parm.Enabled == 0 { - log.Debugf("The policy of job:%v is disabled, will cancel the job", repJob) - _, err := sm.EnterState(models.JobCanceled) - if err != nil { - log.Warningf("For job: %v, failed to update state to 'canceled', error: %v", repJob, err) - } - return err - } - } log.Debugf("In kickOff: will start job: %v", sm.CurrentJob) sm.Start(models.JobRunning) return nil diff --git a/src/replication/event/on_push_handler.go b/src/replication/event/on_push_handler.go index 30ae3447a..f00468ca2 100644 --- a/src/replication/event/on_push_handler.go +++ b/src/replication/event/on_push_handler.go @@ -70,11 +70,9 @@ func checkAndTriggerReplication(image, operation string) error { for _, watchItem := range watchItems { item := models.FilterItem{ - Kind: replication.FilterItemKindTag, - Value: image, - Metadata: map[string]interface{}{ - "operation": operation, - }, + Kind: replication.FilterItemKindTag, + Value: image, + Operation: operation, } if err := notifier.Publish(topic.StartReplicationTopic, notification.StartReplicationNotification{ diff --git a/src/ui/api/dataprepare_test.go b/src/ui/api/dataprepare_test.go index 67e41c9c9..711a74bed 100644 --- a/src/ui/api/dataprepare_test.go +++ b/src/ui/api/dataprepare_test.go @@ -118,10 +118,6 @@ func CommonDelTarget() { _ = dao.DeleteRepTarget(target.ID) } -func CommonPolicyEabled(policyID int, enabled int) { - _ = dao.UpdateRepPolicyEnablement(int64(policyID), enabled) -} - func CommonAddRepository() { commonRepository := &models.RepoRecord{ RepositoryID: 1, diff --git a/src/ui/api/target.go b/src/ui/api/target.go index 68c3c2689..61f8939eb 100644 --- a/src/ui/api/target.go +++ b/src/ui/api/target.go @@ -231,23 +231,6 @@ func (t *TargetAPI) Put() { t.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound)) } - policies, err := dao.GetRepPolicyByTarget(id) - if err != nil { - log.Errorf("failed to get policies according target %d: %v", id, err) - t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - hasEnabledPolicy := false - for _, policy := range policies { - if policy.Enabled == 1 { - hasEnabledPolicy = true - break - } - } - - if hasEnabledPolicy { - t.CustomAbort(http.StatusBadRequest, "the target is associated with policy which is enabled") - } if len(target.Password) != 0 { target.Password, err = utils.ReversibleDecrypt(target.Password, t.secretKey) if err != nil { From c4dc95f4f9e1687e3037f87c7512688b51c85ac0 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 13 Dec 2017 14:56:45 +0800 Subject: [PATCH 26/37] Add implement for supporting replicatie the existing images now --- src/ui/api/replication.go | 13 +++++++++---- src/ui/api/replication_policy.go | 20 +++++++++++++++++++- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/ui/api/replication.go b/src/ui/api/replication.go index 37a8ec6ce..ffe9165d7 100644 --- a/src/ui/api/replication.go +++ b/src/ui/api/replication.go @@ -60,11 +60,16 @@ func (r *ReplicationAPI) Post() { return } - if err = notifier.Publish(topic.StartReplicationTopic, notification.StartReplicationNotification{ - PolicyID: replication.PolicyID, - }); err != nil { + if err = startReplication(replication.PolicyID); err != nil { r.HandleInternalServerError(fmt.Sprintf("failed to publish replication topic for policy %d: %v", replication.PolicyID, err)) return } - log.Infof("replication topic for policy %d triggered", replication.PolicyID) + log.Infof("replication signal for policy %d sent", replication.PolicyID) +} + +func startReplication(policyID int64) error { + return notifier.Publish(topic.StartReplicationTopic, + notification.StartReplicationNotification{ + PolicyID: policyID, + }) } diff --git a/src/ui/api/replication_policy.go b/src/ui/api/replication_policy.go index da452331b..b349f45c5 100644 --- a/src/ui/api/replication_policy.go +++ b/src/ui/api/replication_policy.go @@ -145,7 +145,15 @@ func (pa *RepPolicyAPI) Post() { return } - // TODO trigger a replication if ReplicateExistingImageNow is true + if policy.ReplicateExistingImageNow { + go func() { + if err = startReplication(id); err != nil { + log.Errorf("failed to send replication signal for policy %d: %v", id, err) + return + } + log.Infof("replication signal for policy %d sent", id) + }() + } pa.Redirect(http.StatusCreated, strconv.FormatInt(id, 10)) } @@ -201,6 +209,16 @@ func (pa *RepPolicyAPI) Put() { pa.HandleInternalServerError(fmt.Sprintf("failed to update policy %d: %v", id, err)) return } + + if policy.ReplicateExistingImageNow { + go func() { + if err = startReplication(id); err != nil { + log.Errorf("failed to send replication signal for policy %d: %v", id, err) + return + } + log.Infof("replication signal for policy %d sent", id) + }() + } } // Delete the replication policy From a736cb7b09d26cbe0e3b5e989dde788f033debd1 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 15 Dec 2017 06:22:37 +0800 Subject: [PATCH 27/37] Update the HTTP client according to the comments --- src/adminserver/client/client.go | 22 +++- src/adminserver/client/client_test.go | 3 +- src/common/http/client.go | 101 ++++++++++++++++++ src/common/http/client/client.go | 56 ---------- .../http/{client => modifier}/auth/auth.go | 53 +++------ .../{client => modifier}/auth/auth_test.go | 21 +--- .../registry => http/modifier}/modifier.go | 2 +- .../utils/registry/auth/tokenauthorizer.go | 6 +- src/common/utils/registry/transport.go | 5 +- src/jobservice/client/client.go | 26 ++--- src/jobservice/client/client_test.go | 2 +- src/jobservice/config/config.go | 8 +- src/replication/core/controller.go | 9 +- src/replication/replicator/replicator.go | 9 +- src/replication/replicator/replicator_test.go | 3 +- src/ui/config/config.go | 8 +- 16 files changed, 176 insertions(+), 158 deletions(-) create mode 100644 src/common/http/client.go delete mode 100644 src/common/http/client/client.go rename src/common/http/{client => modifier}/auth/auth.go (56%) rename src/common/http/{client => modifier}/auth/auth_test.go (65%) rename src/common/{utils/registry => http/modifier}/modifier.go (97%) diff --git a/src/adminserver/client/client.go b/src/adminserver/client/client.go index ba82d283c..d2a448eb9 100644 --- a/src/adminserver/client/client.go +++ b/src/adminserver/client/client.go @@ -24,7 +24,8 @@ import ( "strings" "github.com/vmware/harbor/src/adminserver/systeminfo/imagestorage" - httpclient "github.com/vmware/harbor/src/common/http/client" + common_http "github.com/vmware/harbor/src/common/http" + "github.com/vmware/harbor/src/common/http/modifier/auth" "github.com/vmware/harbor/src/common/utils" ) @@ -43,22 +44,33 @@ type Client interface { } // NewClient return an instance of Adminserver client -func NewClient(baseURL string, c httpclient.Client) Client { +func NewClient(baseURL string, cfg *Config) Client { baseURL = strings.TrimRight(baseURL, "/") if !strings.Contains(baseURL, "://") { baseURL = "http://" + baseURL } - return &client{ + client := &client{ baseURL: baseURL, - client: c, } + if cfg != nil { + authorizer := auth.NewSecretAuthorizer(cfg.Secret) + client.client = common_http.NewClient(nil, authorizer) + } + return client } type client struct { baseURL string - client httpclient.Client + client *common_http.Client } +// Config contains configurations needed for client +type Config struct { + Secret string +} + +// TODO refactor the codes with methods of common_http.Client + // do creates request and authorizes it if authorizer is not nil func (c *client) do(method, relativePath string, body io.Reader) (*http.Response, error) { url := c.baseURL + relativePath diff --git a/src/adminserver/client/client_test.go b/src/adminserver/client/client_test.go index b74c187b4..59d407a34 100644 --- a/src/adminserver/client/client_test.go +++ b/src/adminserver/client/client_test.go @@ -16,7 +16,6 @@ package client import ( "fmt" - "net/http" "os" "testing" @@ -35,7 +34,7 @@ func TestMain(m *testing.M) { os.Exit(1) } - c = NewClient(server.URL, &http.Client{}) + c = NewClient(server.URL, &Config{}) os.Exit(m.Run()) } diff --git a/src/common/http/client.go b/src/common/http/client.go new file mode 100644 index 000000000..70ec6e528 --- /dev/null +++ b/src/common/http/client.go @@ -0,0 +1,101 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "io" + "net/http" + + "github.com/vmware/harbor/src/common/http/modifier" +) + +// Client wraps net/http.Client with modifiers, modifiers the request before sending it +type Client struct { + modifiers []modifier.Modifier + client *http.Client +} + +// NewClient creates an instance of Client. Use net/http.Client as the default value +// if c is nil. +func NewClient(c *http.Client, modifiers ...modifier.Modifier) *Client { + client := &Client{ + client: c, + } + if client.client == nil { + client.client = &http.Client{} + } + if len(modifiers) > 0 { + client.modifiers = modifiers + } + return client +} + +// Do ... +func (c *Client) Do(req *http.Request) (*http.Response, error) { + for _, modifier := range c.modifiers { + if err := modifier.Modify(req); err != nil { + return nil, err + } + } + + return c.client.Do(req) +} + +// Get ... +func (c *Client) Get(url string) (*http.Response, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + return c.Do(req) +} + +// Head ... +func (c *Client) Head(url string) (*http.Response, error) { + req, err := http.NewRequest(http.MethodHead, url, nil) + if err != nil { + return nil, err + } + return c.Do(req) +} + +// Post ... +func (c *Client) Post(url, bodyType string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPost, url, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", bodyType) + return c.Do(req) +} + +// Put ... +func (c *Client) Put(url, bodyType string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPut, url, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", bodyType) + return c.Do(req) +} + +// Delete ... +func (c *Client) Delete(url string) (*http.Response, error) { + req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return nil, err + } + return c.Do(req) +} diff --git a/src/common/http/client/client.go b/src/common/http/client/client.go deleted file mode 100644 index b8e86a596..000000000 --- a/src/common/http/client/client.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) 2017 VMware, Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package client - -import ( - "net/http" - - "github.com/vmware/harbor/src/common/http/client/auth" -) - -// Client defines the method that a HTTP client should implement -type Client interface { - Do(*http.Request) (*http.Response, error) -} - -// AuthorizedClient authorizes the requests before sending them -type AuthorizedClient struct { - client *http.Client - authorizer auth.Authorizer -} - -// NewAuthorizedClient returns an instance of the AuthorizedClient -func NewAuthorizedClient(authorizer auth.Authorizer, client ...*http.Client) *AuthorizedClient { - c := &AuthorizedClient{ - authorizer: authorizer, - } - if len(client) > 0 { - c.client = client[0] - } - if c.client == nil { - c.client = &http.Client{} - } - return c -} - -// Do authorizes the request before sending it -func (a *AuthorizedClient) Do(req *http.Request) (*http.Response, error) { - if a.authorizer != nil { - if err := a.authorizer.Authorize(req); err != nil { - return nil, err - } - } - return a.client.Do(req) -} diff --git a/src/common/http/client/auth/auth.go b/src/common/http/modifier/auth/auth.go similarity index 56% rename from src/common/http/client/auth/auth.go rename to src/common/http/modifier/auth/auth.go index 7106a7df6..b28253fdf 100644 --- a/src/common/http/client/auth/auth.go +++ b/src/common/http/modifier/auth/auth.go @@ -17,57 +17,38 @@ package auth import ( "errors" "net/http" + + "github.com/vmware/harbor/src/common/http/modifier" ) const ( secretCookieName = "secret" ) -// Authorizer authorizes the requests -type Authorizer interface { - Authorize(*http.Request) error -} - -// CookieAuthorizer authorizes the requests by adding cookie specified by name and value -type CookieAuthorizer struct { - name string - value string -} - -// NewCookieAuthorizer returns an instance of CookieAuthorizer -func NewCookieAuthorizer(name, value string) *CookieAuthorizer { - return &CookieAuthorizer{ - name: name, - value: value, - } -} - -// Authorize the request with the cookie -func (c *CookieAuthorizer) Authorize(req *http.Request) error { - if req == nil { - return errors.New("the request is null") - } - - req.AddCookie(&http.Cookie{ - Name: c.name, - Value: c.value, - }) - return nil -} +// Authorizer is a kind of Modifier used to authorize the requests +type Authorizer modifier.Modifier // SecretAuthorizer authorizes the requests with the specified secret type SecretAuthorizer struct { - authorizer *CookieAuthorizer + secret string } // NewSecretAuthorizer returns an instance of SecretAuthorizer func NewSecretAuthorizer(secret string) *SecretAuthorizer { return &SecretAuthorizer{ - authorizer: NewCookieAuthorizer(secretCookieName, secret), + secret: secret, } } -// Authorize the request with the secret -func (s *SecretAuthorizer) Authorize(req *http.Request) error { - return s.authorizer.Authorize(req) +// Modify the request by adding secret authentication information +func (s *SecretAuthorizer) Modify(req *http.Request) error { + if req == nil { + return errors.New("the request is null") + } + + req.AddCookie(&http.Cookie{ + Name: secretCookieName, + Value: s.secret, + }) + return nil } diff --git a/src/common/http/client/auth/auth_test.go b/src/common/http/modifier/auth/auth_test.go similarity index 65% rename from src/common/http/client/auth/auth_test.go rename to src/common/http/modifier/auth/auth_test.go index 7754a663b..c0a68bc65 100644 --- a/src/common/http/client/auth/auth_test.go +++ b/src/common/http/modifier/auth/auth_test.go @@ -21,34 +21,17 @@ import ( "github.com/stretchr/testify/require" ) -func TestAuthorizeOfCookieAuthorizer(t *testing.T) { - name, value := "name", "value" - authorizer := NewCookieAuthorizer(name, value) - - // nil request - require.NotNil(t, authorizer.Authorize(nil)) - - // valid request - req, err := http.NewRequest("", "", nil) - require.Nil(t, err) - require.Nil(t, authorizer.Authorize(req)) - require.Equal(t, 1, len(req.Cookies())) - v, err := req.Cookie(name) - require.Nil(t, err) - assert.Equal(t, value, v.Value) -} - func TestAuthorizeOfSecretAuthorizer(t *testing.T) { secret := "secret" authorizer := NewSecretAuthorizer(secret) // nil request - require.NotNil(t, authorizer.Authorize(nil)) + require.NotNil(t, authorizer.Modify(nil)) // valid request req, err := http.NewRequest("", "", nil) require.Nil(t, err) - require.Nil(t, authorizer.Authorize(req)) + require.Nil(t, authorizer.Modify(req)) require.Equal(t, 1, len(req.Cookies())) v, err := req.Cookie(secretCookieName) require.Nil(t, err) diff --git a/src/common/utils/registry/modifier.go b/src/common/http/modifier/modifier.go similarity index 97% rename from src/common/utils/registry/modifier.go rename to src/common/http/modifier/modifier.go index a15a9a0fb..ccd34a87c 100644 --- a/src/common/utils/registry/modifier.go +++ b/src/common/http/modifier/modifier.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package registry +package modifier import ( "net/http" diff --git a/src/common/utils/registry/auth/tokenauthorizer.go b/src/common/utils/registry/auth/tokenauthorizer.go index bbdb69d9f..4e34a9f4d 100644 --- a/src/common/utils/registry/auth/tokenauthorizer.go +++ b/src/common/utils/registry/auth/tokenauthorizer.go @@ -23,9 +23,9 @@ import ( "time" "github.com/docker/distribution/registry/auth/token" + "github.com/vmware/harbor/src/common/http/modifier" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" - "github.com/vmware/harbor/src/common/utils/registry" token_util "github.com/vmware/harbor/src/ui/service/token" ) @@ -254,7 +254,7 @@ func ping(client *http.Client, endpoint string) (string, string, error) { // from token server and add it to the origin request // If customizedTokenService is set, the token request will be sent to it instead of the server get from authorizer func NewStandardTokenAuthorizer(client *http.Client, credential Credential, - customizedTokenService ...string) registry.Modifier { + customizedTokenService ...string) modifier.Modifier { generator := &standardTokenGenerator{ credential: credential, client: client, @@ -309,7 +309,7 @@ func (s *standardTokenGenerator) generate(scopes []*token.ResourceActions, endpo // NewRawTokenAuthorizer returns a token authorizer which calls method to create // token directly -func NewRawTokenAuthorizer(username, service string) registry.Modifier { +func NewRawTokenAuthorizer(username, service string) modifier.Modifier { generator := &rawTokenGenerator{ service: service, username: username, diff --git a/src/common/utils/registry/transport.go b/src/common/utils/registry/transport.go index 3ae8a8d50..d844f9308 100644 --- a/src/common/utils/registry/transport.go +++ b/src/common/utils/registry/transport.go @@ -17,17 +17,18 @@ package registry import ( "net/http" + "github.com/vmware/harbor/src/common/http/modifier" "github.com/vmware/harbor/src/common/utils/log" ) // Transport holds information about base transport and modifiers type Transport struct { transport http.RoundTripper - modifiers []Modifier + modifiers []modifier.Modifier } // NewTransport ... -func NewTransport(transport http.RoundTripper, modifiers ...Modifier) *Transport { +func NewTransport(transport http.RoundTripper, modifiers ...modifier.Modifier) *Transport { return &Transport{ transport: transport, modifiers: modifiers, diff --git a/src/jobservice/client/client.go b/src/jobservice/client/client.go index 5288a2f92..83dec41cd 100644 --- a/src/jobservice/client/client.go +++ b/src/jobservice/client/client.go @@ -21,7 +21,7 @@ import ( "net/http" commonhttp "github.com/vmware/harbor/src/common/http" - "github.com/vmware/harbor/src/common/http/client" + "github.com/vmware/harbor/src/common/http/modifier/auth" "github.com/vmware/harbor/src/jobservice/api" ) @@ -33,21 +33,22 @@ type Client interface { // DefaultClient provides a default implement for the interface Client type DefaultClient struct { endpoint string - client client.Client + client *commonhttp.Client +} + +// Config contains configuration items needed for DefaultClient +type Config struct { + Secret string } // NewDefaultClient returns an instance of DefaultClient -func NewDefaultClient(endpoint string, client ...client.Client) *DefaultClient { +func NewDefaultClient(endpoint string, cfg *Config) *DefaultClient { c := &DefaultClient{ endpoint: endpoint, } - if len(client) > 0 { - c.client = client[0] - } - - if c.client == nil { - c.client = &http.Client{} + if cfg != nil { + c.client = commonhttp.NewClient(nil, auth.NewSecretAuthorizer(cfg.Secret)) } return c @@ -62,12 +63,7 @@ func (d *DefaultClient) SubmitReplicationJob(replication *api.ReplicationReq) er return err } - req, err := http.NewRequest(http.MethodPost, url, buffer) - if err != nil { - return err - } - - resp, err := d.client.Do(req) + resp, err := d.client.Post(url, "application/json", buffer) if err != nil { return err } diff --git a/src/jobservice/client/client_test.go b/src/jobservice/client/client_test.go index 0cde379cf..4d8ab5fb0 100644 --- a/src/jobservice/client/client_test.go +++ b/src/jobservice/client/client_test.go @@ -49,7 +49,7 @@ func TestMain(m *testing.M) { } func TestSubmitReplicationJob(t *testing.T) { - client := NewDefaultClient(url) + client := NewDefaultClient(url, &Config{}) err := client.SubmitReplicationJob(&api.ReplicationReq{}) assert.Nil(t, err) } diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index 4ece1a814..4992c2363 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -22,8 +22,6 @@ import ( "github.com/vmware/harbor/src/adminserver/client" "github.com/vmware/harbor/src/common" comcfg "github.com/vmware/harbor/src/common/config" - httpclient "github.com/vmware/harbor/src/common/http/client" - "github.com/vmware/harbor/src/common/http/client/auth" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" ) @@ -51,8 +49,10 @@ func Init() error { adminServerURL = "http://adminserver" } log.Infof("initializing client for adminserver %s ...", adminServerURL) - authorizer := auth.NewSecretAuthorizer(UISecret()) - AdminserverClient = client.NewClient(adminServerURL, httpclient.NewAuthorizedClient(authorizer)) + cfg := &client.Config{ + Secret: UISecret(), + } + AdminserverClient = client.NewClient(adminServerURL, cfg) if err := AdminserverClient.Ping(); err != nil { return fmt.Errorf("failed to ping adminserver: %v", err) } diff --git a/src/replication/core/controller.go b/src/replication/core/controller.go index 38a0a93f9..6945ae75b 100644 --- a/src/replication/core/controller.go +++ b/src/replication/core/controller.go @@ -18,11 +18,10 @@ import ( "fmt" "strings" - "github.com/vmware/harbor/src/common/http/client" - "github.com/vmware/harbor/src/common/http/client/auth" common_models "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/jobservice/api" + "github.com/vmware/harbor/src/jobservice/client" "github.com/vmware/harbor/src/replication" "github.com/vmware/harbor/src/replication/models" "github.com/vmware/harbor/src/replication/policy" @@ -85,8 +84,10 @@ func NewDefaultController(cfg ControllerConfig) *DefaultController { // TODO read from configuration endpoint := "http://jobservice:8080" - client := client.NewAuthorizedClient(auth.NewSecretAuthorizer(config.UISecret())) - ctl.replicator = replicator.NewDefaultReplicator(endpoint, client) + ctl.replicator = replicator.NewDefaultReplicator(endpoint, + &client.Config{ + Secret: config.UISecret(), + }) return ctl } diff --git a/src/replication/replicator/replicator.go b/src/replication/replicator/replicator.go index e27ba4da2..5edef542f 100644 --- a/src/replication/replicator/replicator.go +++ b/src/replication/replicator/replicator.go @@ -15,9 +15,8 @@ package replicator import ( - "github.com/vmware/harbor/src/common/http/client" "github.com/vmware/harbor/src/jobservice/api" - jobserviceclient "github.com/vmware/harbor/src/jobservice/client" + "github.com/vmware/harbor/src/jobservice/client" ) // Replicator submits the replication work to the jobservice @@ -27,13 +26,13 @@ type Replicator interface { // DefaultReplicator provides a default implement for Replicator type DefaultReplicator struct { - client jobserviceclient.Client + client client.Client } // NewDefaultReplicator returns an instance of DefaultReplicator -func NewDefaultReplicator(endpoint string, client ...client.Client) *DefaultReplicator { +func NewDefaultReplicator(endpoint string, cfg *client.Config) *DefaultReplicator { return &DefaultReplicator{ - client: jobserviceclient.NewDefaultClient(endpoint, client...), + client: client.NewDefaultClient(endpoint, cfg), } } diff --git a/src/replication/replicator/replicator_test.go b/src/replication/replicator/replicator_test.go index b2a229cce..726f12b7a 100644 --- a/src/replication/replicator/replicator_test.go +++ b/src/replication/replicator/replicator_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/vmware/harbor/src/jobservice/api" + "github.com/vmware/harbor/src/jobservice/client" ) type fakeJobserviceClient struct{} @@ -28,7 +29,7 @@ func (f *fakeJobserviceClient) SubmitReplicationJob(replication *api.Replication } func TestReplicate(t *testing.T) { - replicator := NewDefaultReplicator("http://jobservice") + replicator := NewDefaultReplicator("http://jobservice", &client.Config{}) replicator.client = &fakeJobserviceClient{} assert.Nil(t, replicator.Replicate(&api.ReplicationReq{})) } diff --git a/src/ui/config/config.go b/src/ui/config/config.go index 1b798115a..207e8067d 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -25,8 +25,6 @@ import ( "github.com/vmware/harbor/src/adminserver/client" "github.com/vmware/harbor/src/common" comcfg "github.com/vmware/harbor/src/common/config" - httpclient "github.com/vmware/harbor/src/common/http/client" - "github.com/vmware/harbor/src/common/http/client/auth" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/secret" "github.com/vmware/harbor/src/common/utils/log" @@ -74,8 +72,10 @@ func Init() error { // InitByURL Init configurations with given url func InitByURL(adminServerURL string) error { log.Infof("initializing client for adminserver %s ...", adminServerURL) - authorizer := auth.NewSecretAuthorizer(UISecret()) - AdminserverClient = client.NewClient(adminServerURL, httpclient.NewAuthorizedClient(authorizer)) + cfg := &client.Config{ + Secret: UISecret(), + } + AdminserverClient = client.NewClient(adminServerURL, cfg) if err := AdminserverClient.Ping(); err != nil { return fmt.Errorf("failed to ping adminserver: %v", err) } From 260ef561c4298304434366980cc816467d949447 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Sat, 16 Dec 2017 06:45:59 +0800 Subject: [PATCH 28/37] Update the HTTP client for easy use by add more util functions --- src/adminserver/client/client.go | 105 ++++-------------------------- src/common/http/client.go | 107 ++++++++++++++++++++++++------- src/jobservice/client/client.go | 36 ++--------- 3 files changed, 99 insertions(+), 149 deletions(-) diff --git a/src/adminserver/client/client.go b/src/adminserver/client/client.go index d2a448eb9..115e903b6 100644 --- a/src/adminserver/client/client.go +++ b/src/adminserver/client/client.go @@ -15,16 +15,10 @@ package client import ( - "bytes" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" "strings" "github.com/vmware/harbor/src/adminserver/systeminfo/imagestorage" - common_http "github.com/vmware/harbor/src/common/http" + "github.com/vmware/harbor/src/common/http" "github.com/vmware/harbor/src/common/http/modifier/auth" "github.com/vmware/harbor/src/common/utils" ) @@ -54,14 +48,14 @@ func NewClient(baseURL string, cfg *Config) Client { } if cfg != nil { authorizer := auth.NewSecretAuthorizer(cfg.Secret) - client.client = common_http.NewClient(nil, authorizer) + client.client = http.NewClient(nil, authorizer) } return client } type client struct { baseURL string - client *common_http.Client + client *http.Client } // Config contains configurations needed for client @@ -69,19 +63,6 @@ type Config struct { Secret string } -// TODO refactor the codes with methods of common_http.Client - -// do creates request and authorizes it if authorizer is not nil -func (c *client) do(method, relativePath string, body io.Reader) (*http.Response, error) { - url := c.baseURL + relativePath - req, err := http.NewRequest(method, url, body) - if err != nil { - return nil, err - } - - return c.client.Do(req) -} - func (c *client) Ping() error { addr := strings.Split(c.baseURL, "://")[1] if !strings.Contains(addr, ":") { @@ -93,96 +74,32 @@ func (c *client) Ping() error { // GetCfgs ... func (c *client) GetCfgs() (map[string]interface{}, error) { - resp, err := c.do(http.MethodGet, "/api/configurations", nil) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to get configurations: %d %s", - resp.StatusCode, b) - } - + url := c.baseURL + "/api/configurations" cfgs := map[string]interface{}{} - if err = json.Unmarshal(b, &cfgs); err != nil { + if err := c.client.Get(url, &cfgs); err != nil { return nil, err } - return cfgs, nil } // UpdateCfgs ... func (c *client) UpdateCfgs(cfgs map[string]interface{}) error { - data, err := json.Marshal(cfgs) - if err != nil { - return err - } - - resp, err := c.do(http.MethodPut, "/api/configurations", bytes.NewReader(data)) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - return fmt.Errorf("failed to update configurations: %d %s", - resp.StatusCode, b) - } - - return nil + url := c.baseURL + "/api/configurations" + return c.client.Put(url, cfgs) } // ResetCfgs ... func (c *client) ResetCfgs() error { - resp, err := c.do(http.MethodPost, "/api/configurations/reset", nil) - if err != nil { - return err - } - - if resp.StatusCode != http.StatusOK { - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - return fmt.Errorf("failed to reset configurations: %d %s", - resp.StatusCode, b) - } - - return nil + url := c.baseURL + "/api/configurations/reset" + return c.client.Post(url) } // Capacity ... func (c *client) Capacity() (*imagestorage.Capacity, error) { - resp, err := c.do(http.MethodGet, "/api/systeminfo/capacity", nil) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to get capacity: %d %s", - resp.StatusCode, b) - } - + url := c.baseURL + "/api/systeminfo/capacity" capacity := &imagestorage.Capacity{} - if err = json.Unmarshal(b, capacity); err != nil { + if err := c.client.Get(url, capacity); err != nil { return nil, err } - return capacity, nil } diff --git a/src/common/http/client.go b/src/common/http/client.go index 70ec6e528..a5971cbaf 100644 --- a/src/common/http/client.go +++ b/src/common/http/client.go @@ -15,20 +15,25 @@ package http import ( + "bytes" + "encoding/json" "io" + "io/ioutil" "net/http" "github.com/vmware/harbor/src/common/http/modifier" ) -// Client wraps net/http.Client with modifiers, modifiers the request before sending it +// Client is a util for common HTTP operations, such Get, Head, Post, Put and Delete. +// Use Do instead if those methods can not meet your requirement type Client struct { modifiers []modifier.Modifier client *http.Client } -// NewClient creates an instance of Client. Use net/http.Client as the default value -// if c is nil. +// NewClient creates an instance of Client. +// Use net/http.Client as the default value if c is nil. +// Modifiers modify the request before sending it. func NewClient(c *http.Client, modifiers ...modifier.Modifier) *Client { client := &Client{ client: c, @@ -54,48 +59,104 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) { } // Get ... -func (c *Client) Get(url string) (*http.Response, error) { +func (c *Client) Get(url string, v ...interface{}) error { req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { - return nil, err + return err } - return c.Do(req) + + data, err := c.do(req) + if err != nil { + return err + } + + if len(v) == 0 { + return nil + } + + return json.Unmarshal(data, v[0]) } // Head ... -func (c *Client) Head(url string) (*http.Response, error) { +func (c *Client) Head(url string) error { req, err := http.NewRequest(http.MethodHead, url, nil) if err != nil { - return nil, err + return err } - return c.Do(req) + _, err = c.do(req) + return err } // Post ... -func (c *Client) Post(url, bodyType string, body io.Reader) (*http.Response, error) { - req, err := http.NewRequest(http.MethodPost, url, body) - if err != nil { - return nil, err +func (c *Client) Post(url string, v ...interface{}) error { + var reader io.Reader + if len(v) > 0 { + data, err := json.Marshal(v[0]) + if err != nil { + return err + } + + reader = bytes.NewReader(data) } - req.Header.Set("Content-Type", bodyType) - return c.Do(req) + + req, err := http.NewRequest(http.MethodPost, url, reader) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + _, err = c.do(req) + return err } // Put ... -func (c *Client) Put(url, bodyType string, body io.Reader) (*http.Response, error) { - req, err := http.NewRequest(http.MethodPut, url, body) - if err != nil { - return nil, err +func (c *Client) Put(url string, v ...interface{}) error { + var reader io.Reader + if len(v) > 0 { + data := []byte{} + data, err := json.Marshal(v[0]) + if err != nil { + return err + } + reader = bytes.NewReader(data) } - req.Header.Set("Content-Type", bodyType) - return c.Do(req) + + req, err := http.NewRequest(http.MethodPut, url, reader) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + _, err = c.do(req) + return err } // Delete ... -func (c *Client) Delete(url string) (*http.Response, error) { +func (c *Client) Delete(url string) error { req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return err + } + _, err = c.do(req) + return err +} + +func (c *Client) do(req *http.Request) ([]byte, error) { + resp, err := c.Do(req) if err != nil { return nil, err } - return c.Do(req) + defer resp.Body.Close() + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return nil, &Error{ + Code: resp.StatusCode, + Message: string(data), + } + } + + return data, nil } diff --git a/src/jobservice/client/client.go b/src/jobservice/client/client.go index 83dec41cd..7ace3d3a2 100644 --- a/src/jobservice/client/client.go +++ b/src/jobservice/client/client.go @@ -15,12 +15,7 @@ package client import ( - "bytes" - "encoding/json" - "io/ioutil" - "net/http" - - commonhttp "github.com/vmware/harbor/src/common/http" + "github.com/vmware/harbor/src/common/http" "github.com/vmware/harbor/src/common/http/modifier/auth" "github.com/vmware/harbor/src/jobservice/api" ) @@ -33,7 +28,7 @@ type Client interface { // DefaultClient provides a default implement for the interface Client type DefaultClient struct { endpoint string - client *commonhttp.Client + client *http.Client } // Config contains configuration items needed for DefaultClient @@ -48,7 +43,7 @@ func NewDefaultClient(endpoint string, cfg *Config) *DefaultClient { } if cfg != nil { - c.client = commonhttp.NewClient(nil, auth.NewSecretAuthorizer(cfg.Secret)) + c.client = http.NewClient(nil, auth.NewSecretAuthorizer(cfg.Secret)) } return c @@ -57,28 +52,5 @@ func NewDefaultClient(endpoint string, cfg *Config) *DefaultClient { // SubmitReplicationJob submits a replication job to the jobservice func (d *DefaultClient) SubmitReplicationJob(replication *api.ReplicationReq) error { url := d.endpoint + "/api/jobs/replication" - - buffer := &bytes.Buffer{} - if err := json.NewEncoder(buffer).Encode(replication); err != nil { - return err - } - - resp, err := d.client.Post(url, "application/json", buffer) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - message, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - return &commonhttp.Error{ - Code: resp.StatusCode, - Message: string(message), - } - } - - return nil + return d.client.Post(url, replication) } From 831f69595aa0d23a3af59259cda447b3b794fdb9 Mon Sep 17 00:00:00 2001 From: "Fuhui Peng (c)" Date: Thu, 30 Nov 2017 11:05:44 +0800 Subject: [PATCH 29/37] Add ui code about replication enhancement --- .../list-replication-rule.component.html.ts | 17 +- .../replication/replication.component.html.ts | 3 +- .../src/replication/replication.component.ts | 17 +- src/ui_ng/package.json | 2 +- .../harbor-shell/harbor-shell.component.html | 1 + src/ui_ng/src/app/harbor-routing.module.ts | 37 +- .../project-detail.component.html | 2 +- .../project-detail.component.ts | 7 +- .../destination-page.component.html | 1 + .../replication-page.component.html | 2 +- .../replication-rule.component.ts | 528 ++++++++++++++++++ .../replication-rule/replication-rule.css | 30 + .../replication-rule/replication-rule.html | 122 ++++ .../replication-rule.service.ts | 75 +++ .../replication-rule/replication-rule.ts | 48 ++ .../src/app/replication/replication.module.ts | 15 +- .../total-replication-page.component.html | 3 +- .../total-replication-page.component.ts | 14 +- .../total-replication.component.css | 5 +- .../route/system-admin-activate.service.ts | 5 +- src/ui_ng/src/app/shared/shared.utils.ts | 22 + src/ui_ng/src/i18n/lang/en-us-lang.json | 4 +- src/ui_ng/src/i18n/lang/es-es-lang.json | 4 +- src/ui_ng/src/i18n/lang/zh-cn-lang.json | 4 +- .../resources/Harbor-Pages/Replication.robot | 26 +- .../Harbor-Pages/Replication_Elements.robot | 8 +- 26 files changed, 937 insertions(+), 65 deletions(-) create mode 100644 src/ui_ng/src/app/replication/replication-rule/replication-rule.component.ts create mode 100644 src/ui_ng/src/app/replication/replication-rule/replication-rule.css create mode 100644 src/ui_ng/src/app/replication/replication-rule/replication-rule.html create mode 100644 src/ui_ng/src/app/replication/replication-rule/replication-rule.service.ts create mode 100644 src/ui_ng/src/app/replication/replication-rule/replication-rule.ts diff --git a/src/ui_ng/lib/src/list-replication-rule/list-replication-rule.component.html.ts b/src/ui_ng/lib/src/list-replication-rule/list-replication-rule.component.html.ts index 410e2aeeb..8687a9a86 100644 --- a/src/ui_ng/lib/src/list-replication-rule/list-replication-rule.component.html.ts +++ b/src/ui_ng/lib/src/list-replication-rule/list-replication-rule.component.html.ts @@ -2,9 +2,9 @@ export const LIST_REPLICATION_RULE_TEMPLATE: string = `
{{'REPLICATION.NAME' | translate}} - {{'REPLICATION.PROJECT' | translate}} + {{'REPLICATION.PROJECT' | translate}} {{'REPLICATION.DESCRIPTION' | translate}} - {{'REPLICATION.DESTINATION_NAME' | translate}} + {{'REPLICATION.DESTINATION_NAME' | translate}} {{'REPLICATION.LAST_START_TIME' | translate}} {{'REPLICATION.ACTIVATION' | translate}} {{'REPLICATION.PLACEHOLDER' | translate }} @@ -14,17 +14,12 @@ export const LIST_REPLICATION_RULE_TEMPLATE: string = ` - - - {{p.name}} - - - {{p.name}} - + {{p.name}} + + {{p.projects[0].name}} - {{p.project_name}} {{p.description ? p.description : '-'}} - {{p.target_name}} + {{p.targets[0].name}} - {{p.start_time | date: 'short'}} diff --git a/src/ui_ng/lib/src/replication/replication.component.html.ts b/src/ui_ng/lib/src/replication/replication.component.html.ts index cb374aa27..1d4877c13 100644 --- a/src/ui_ng/lib/src/replication/replication.component.html.ts +++ b/src/ui_ng/lib/src/replication/replication.component.html.ts @@ -3,8 +3,7 @@ export const REPLICATION_TEMPLATE: string = `
- - +
diff --git a/src/ui_ng/lib/src/replication/replication.component.ts b/src/ui_ng/lib/src/replication/replication.component.ts index 9e6ce673f..8427c5ecf 100644 --- a/src/ui_ng/lib/src/replication/replication.component.ts +++ b/src/ui_ng/lib/src/replication/replication.component.ts @@ -88,6 +88,8 @@ export class ReplicationComponent implements OnInit, OnDestroy { @Input() readonly: boolean; @Output() redirect = new EventEmitter(); + @Output() openCreateRule = new EventEmitter(); + @Output() openEdit = new EventEmitter(); search: SearchOption = new SearchOption(); @@ -111,8 +113,8 @@ export class ReplicationComponent implements OnInit, OnDestroy { @ViewChild(ListReplicationRuleComponent) listReplicationRule: ListReplicationRuleComponent; - @ViewChild(CreateEditRuleComponent) - createEditPolicyComponent: CreateEditRuleComponent; +/* @ViewChild(CreateEditRuleComponent) + createEditPolicyComponent: CreateEditRuleComponent;*/ @ViewChild("replicationLogViewer") replicationLogViewer: JobLogViewerComponent; @@ -134,9 +136,9 @@ export class ReplicationComponent implements OnInit, OnDestroy { private translateService: TranslateService) { } - public get creationAvailable(): boolean { + /*public get creationAvailable(): boolean { return !this.readonly && this.projectId ? true : false; - } + }*/ public get showPaginationIndex(): boolean { return this.totalCount > 0; @@ -146,6 +148,7 @@ export class ReplicationComponent implements OnInit, OnDestroy { this.currentRuleStatus = this.ruleStatus[0]; this.currentJobStatus = this.jobStatus[0]; this.currentJobSearchOption = 0; + console.log('readonly', this.readonly); } ngOnDestroy() { @@ -155,7 +158,8 @@ export class ReplicationComponent implements OnInit, OnDestroy { } openModal(): void { - this.createEditPolicyComponent.openCreateEditRule(true); + this.openCreateRule.emit(); + // this.createEditPolicyComponent.openCreateEditRule(true); } openEditRule(rule: ReplicationRule) { @@ -164,7 +168,8 @@ export class ReplicationComponent implements OnInit, OnDestroy { if (rule.enabled === 1) { editable = false; } - this.createEditPolicyComponent.openCreateEditRule(editable, rule.id); + this.openEdit.emit(rule.id); + // this.createEditPolicyComponent.openCreateEditRule(editable, rule.id); } } diff --git a/src/ui_ng/package.json b/src/ui_ng/package.json index f8f2b596a..b91e5fd80 100644 --- a/src/ui_ng/package.json +++ b/src/ui_ng/package.json @@ -31,7 +31,7 @@ "clarity-icons": "^0.9.8", "clarity-ui": "^0.9.8", "core-js": "^2.4.1", - "harbor-ui": "0.4.91", + "harbor-ui": "^0.5.9-test-31", "intl": "^1.2.5", "mutationobserver-shim": "^0.3.2", "ngx-cookie": "^1.0.0", diff --git a/src/ui_ng/src/app/base/harbor-shell/harbor-shell.component.html b/src/ui_ng/src/app/base/harbor-shell/harbor-shell.component.html index efe767bf4..7ce4a0cc0 100644 --- a/src/ui_ng/src/app/base/harbor-shell/harbor-shell.component.html +++ b/src/ui_ng/src/app/base/harbor-shell/harbor-shell.component.html @@ -18,6 +18,7 @@ diff --git a/src/ui_ng/src/app/harbor-routing.module.ts b/src/ui_ng/src/app/harbor-routing.module.ts index ed9f876ba..57711c35b 100644 --- a/src/ui_ng/src/app/harbor-routing.module.ts +++ b/src/ui_ng/src/app/harbor-routing.module.ts @@ -50,6 +50,7 @@ import { LeavingConfigRouteDeactivate } from './shared/route/leaving-config-deac import { MemberGuard } from './shared/route/member-guard-activate.service'; import { TagDetailPageComponent } from './repository/tag-detail/tag-detail-page.component'; +import { ReplicationRuleComponent} from "./replication/replication-rule/replication-rule.component"; const harborRoutes: Routes = [ { path: '', redirectTo: 'harbor', pathMatch: 'full' }, @@ -80,23 +81,23 @@ const harborRoutes: Routes = [ }, { path: 'replications', - component: ReplicationManagementComponent, + component: TotalReplicationPageComponent, canActivate: [SystemAdminGuard], canActivateChild: [SystemAdminGuard], - children: [ - { - path: 'rules', - component: TotalReplicationPageComponent - }, - { - path: 'endpoints', - component: DestinationPageComponent - }, - { - path: '**', - redirectTo: 'endpoints' - } - ] + }, + { + path: 'replications/:id/rule', + component: ReplicationRuleComponent, + canActivate: [SystemAdminGuard], + canActivateChild: [SystemAdminGuard], + + }, + { + path: 'replications/new-rule', + component: ReplicationRuleComponent, + canActivate: [SystemAdminGuard], + canActivateChild: [SystemAdminGuard], + }, { path: 'tags/:id/:repo', @@ -146,6 +147,12 @@ const harborRoutes: Routes = [ component: ConfigurationComponent, canActivate: [SystemAdminGuard], canDeactivate: [LeavingConfigRouteDeactivate] + }, + { + path: 'registry', + component: DestinationPageComponent, + canActivate: [SystemAdminGuard], + canActivateChild: [SystemAdminGuard], } ] }, diff --git a/src/ui_ng/src/app/project/project-detail/project-detail.component.html b/src/ui_ng/src/app/project/project-detail/project-detail.component.html index 317f03b8a..e4d291458 100644 --- a/src/ui_ng/src/app/project/project-detail/project-detail.component.html +++ b/src/ui_ng/src/app/project/project-detail/project-detail.component.html @@ -13,7 +13,7 @@ -