upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor synced 2024-09-21 05:52:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refine scheduler

Browse Source
This commit is contained in:
Steven Zou 2017-07-19 16:02:19 +08:00
parent 5c876621ec
commit 8f921db588
11 changed files with 135 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/common/notifier/config_watcher.go
View File

@ -16,31 +16,23 @@ func WatchConfigChanges(cfg map[string]interface{}) error {
//Currently only watch the scan all policy change.
if v, ok := cfg[ScanAllPolicyTopic]; ok {
if reflect.TypeOf(v).Kind() == reflect.Map {
policyCfg := &models.ScanAllPolicy{}
vMap := v.(map[string]interface{})
//Reset filed name.
if pv, yes := vMap["parameter"]; yes {
vMap["Parm"] = pv
delete(vMap, "parameter")
}
if err := utils.ConvertMapToStruct(policyCfg, vMap); err != nil {
return err
}
policyNotification := ScanPolicyNotification{
Type: policyCfg.Type,
DailyTime: 0,
}
if t, yes := policyCfg.Parm["daily_time"]; yes {
if reflect.TypeOf(t).Kind() == reflect.Int {
policyNotification.DailyTime = (int64)(t.(int))
}
}
return Publish(ScanAllPolicyTopic, policyNotification)
policyCfg := &models.ScanAllPolicy{}
if err := utils.ConvertMapToStruct(policyCfg, v); err != nil {
return err
}
policyNotification := ScanPolicyNotification{
Type: policyCfg.Type,
DailyTime: 0,
}
if t, yes := policyCfg.Parm["daily_time"]; yes {
if reflect.TypeOf(t).Kind() == reflect.Int {
policyNotification.DailyTime = (int64)(t.(int))
}
}
return Publish(ScanAllPolicyTopic, policyNotification)
}
return nil

3
src/common/notifier/notifier_test.go
View File

@ -192,8 +192,6 @@ func TestConcurrentPublishWithScanPolicyHandler(t *testing.T) {
if len(notificationWatcher.handlers) != 1 {
t.Fatal("Handler is not registered")
}
//Wating for everything is ready.
<-time.After(1 * time.Second)
utcTime := time.Now().UTC().Unix()
notification := ScanPolicyNotification{"daily", utcTime + 3600}
@ -206,6 +204,7 @@ func TestConcurrentPublishWithScanPolicyHandler(t *testing.T) {
//Wating for everything is ready.
<-time.After(2 * time.Second)
if err := UnSubscribe("testing_topic", ""); err != nil {
t.Fatal(err.Error())
}

37
src/common/notifier/scan_policy_notitification_handler.go
View File

@ -63,33 +63,22 @@ func (s *ScanPolicyNotificationHandler) Handle(value interface{}) error {
//To check and compare if the related parameter is changed.
if pl := scheduler.DefaultScheduler.GetPolicy(alternatePolicy); pl != nil {
if reflect.TypeOf(pl).Kind() == reflect.Ptr &&
reflect.TypeOf(pl).String() == "*policy.AlternatePolicy" {
spl := pl.(*policy.AlternatePolicy)
plConfig := spl.GetConfig()
if plConfig != nil {
if plConfig.OffsetTime != notification.DailyTime {
//Parameter changed.
//Unschedule policy.
if err := scheduler.DefaultScheduler.UnSchedule(alternatePolicy); err != nil {
return err
}
//Waiting for async action is done!
<-time.After(500 * time.Millisecond)
//Reschedule policy.
return schedulePolicy(notification)
}
//No change, do nothing.
return nil
policyCandidate := policy.NewAlternatePolicy(&policy.AlternatePolicyConfiguration{
Duration: 24 * time.Hour,
OffsetTime: notification.DailyTime,
})
if !pl.Equal(policyCandidate) {
//Parameter changed.
//Unschedule policy.
if err := scheduler.DefaultScheduler.UnSchedule(alternatePolicy); err != nil {
return err
}
return errors.New("*policy.AlternatePolicy should not have nil configuration")
//Schedule a new policy.
return schedulePolicy(notification)
}
return fmt.Errorf("Invalid policy type: %s", reflect.TypeOf(pl).String())
//Same policy configuration, do nothing
return nil
}
return errors.New("Inconsistent policy scheduling status")

6
src/common/notifier/scan_policy_notitification_handler_test.go
View File

@ -28,8 +28,6 @@ func TestScanPolicyNotificationHandler(t *testing.T) {
t.Fatal(err)
}
//Waiting for everything is ready.
<-time.After(1 * time.Second)
if !testingScheduler.HasScheduled("Alternate Policy") {
t.Fatal("Handler does not work")
}
@ -40,8 +38,6 @@ func TestScanPolicyNotificationHandler(t *testing.T) {
t.Fatal(err)
}
//Waiting for everything is ready.
<-time.After(1 * time.Second)
if !testingScheduler.HasScheduled("Alternate Policy") {
t.Fatal("Handler does not work [2]")
}
@ -63,8 +59,6 @@ func TestScanPolicyNotificationHandler(t *testing.T) {
t.Fatal(err)
}
//Waiting for everything is ready.
<-time.After(1 * time.Second)
if testingScheduler.HasScheduled("Alternate Policy") {
t.Fail()
}

20
src/common/scheduler/policy/alternate_policy.go
View File

@ -152,3 +152,23 @@ func (alp *AlternatePolicy) Evaluate() (<-chan bool, error) {
return alp.evaluation, nil
}
//Equal is an implementation of same method in policy interface.
func (alp *AlternatePolicy) Equal(p Policy) bool {
if p == nil {
return false
}
pl, ok := p.(*AlternatePolicy)
if !ok {
return false
}
cfg := pl.GetConfig()
cfg2 := alp.GetConfig()
if (cfg == nil && cfg2 != nil) || (cfg != nil && cfg2 == nil) {
return false
}
return cfg == nil || (cfg.Duration == cfg2.Duration && cfg.OffsetTime == cfg2.OffsetTime)
}

7
src/common/scheduler/policy/policy.go
View File

@ -34,6 +34,11 @@ type Policy interface {
Evaluate() (<-chan bool, error)
//Disable the enabled policy and release all the allocated resources.
//Disable should also send signal to the terminated channel which returned by Done.
Disable() error
//Equal will compare the two policies based on related factors if existing such as confgiuration etc.
//to determine whether the two policies are same ones or not. Please pay attention that, not every policy
//needs to support this method. If no need, please directly return false to indicate each policies are
//different.
Equal(p Policy) bool
}

79
src/common/scheduler/scheduler.go
View File

@ -61,9 +61,6 @@ type Scheduler struct {
//Store to keep the references of scheduled policies.
policies Store
//Queue for accepting the scheduling polices.
scheduleQueue chan policy.Policy
//Queue for receiving policy unschedule request or complete signal.
unscheduleQueue chan string
@ -90,7 +87,6 @@ func NewScheduler(config *Configuration) *Scheduler {
qSize = config.QueueSize
}
sq := make(chan policy.Policy, qSize)
usq := make(chan string, qSize)
stChan := make(chan *StatItem, 4)
tc := make(chan bool, 1)
@ -99,7 +95,6 @@ func NewScheduler(config *Configuration) *Scheduler {
return &Scheduler{
config: config,
policies: store,
scheduleQueue: sq,
unscheduleQueue: usq,
statChan: stChan,
terminateChan: tc,
@ -125,37 +120,25 @@ func (sch *Scheduler) Start() {
}
}()
defer func() {
//Exit and clear.
sch.isRunning = false
//Stop all watchers.
for _, wt := range sch.policies.GetAll() {
wt.Stop()
}
//Clear resources
sch.policies.Clear()
log.Infof("Policy scheduler stop at %s\n", time.Now().UTC().Format(time.RFC3339))
}()
for {
select {
case <-sch.terminateChan:
//Exit
return
case p := <-sch.scheduleQueue:
if !sch.policies.Exists(p.Name()) {
//Schedule the policy.
watcher := NewWatcher(p, sch.statChan, sch.unscheduleQueue)
//Keep the policy for future use after it's successfully scheduled.
sch.policies.Put(p.Name(), watcher)
//Enable it.
watcher.Start()
//Update stats and log info.
log.Infof("Policy %s is scheduled", p.Name())
sch.statChan <- &StatItem{statSchedulePolicy, 1, nil}
}
case name := <-sch.unscheduleQueue:
//Find the watcher.
watcher := sch.policies.Remove(name)
if watcher != nil && watcher.IsRunning() {
watcher.Stop()
//Update stats and log info.
log.Infof("Policy %s is unscheduled", name)
sch.statChan <- &StatItem{statUnSchedulePolicy, 1, nil}
//Unscheduled when policy is completed.
if err := sch.UnSchedule(name); err != nil {
log.Error(err.Error())
}
case stat := <-sch.statChan:
{
@ -204,18 +187,8 @@ func (sch *Scheduler) Stop() {
return
}
//Terminate damon firstly to stop receiving signals.
//Terminate damon to stop receiving signals.
sch.terminateChan <- true
//Stop all watchers.
for _, wt := range sch.policies.GetAll() {
wt.Stop()
}
//Clear resources
sch.policies.Clear()
log.Infof("Policy scheduler stop at %s\n", time.Now().UTC().Format(time.RFC3339))
}
//Schedule and enable the policy.
@ -238,7 +211,16 @@ func (sch *Scheduler) Schedule(scheduledPolicy policy.Policy) error {
}
//Schedule the policy.
sch.scheduleQueue <- scheduledPolicy
watcher := NewWatcher(scheduledPolicy, sch.statChan, sch.unscheduleQueue)
//Enable it.
watcher.Start()
//Keep the policy for future use after it's successfully scheduled.
sch.policies.Put(scheduledPolicy.Name(), watcher)
//Update stats and log info.
log.Infof("Policy %s is scheduled", scheduledPolicy.Name())
sch.statChan <- &StatItem{statSchedulePolicy, 1, nil}
return nil
}
@ -254,7 +236,17 @@ func (sch *Scheduler) UnSchedule(policyName string) error {
}
//Unschedule the policy.
sch.unscheduleQueue <- policyName
//Find the watcher.
watcher := sch.policies.Remove(policyName)
if watcher != nil && watcher.IsRunning() {
watcher.Stop()
//Update stats and log info.
log.Infof("Policy %s is unscheduled", policyName)
sch.statChan <- &StatItem{statUnSchedulePolicy, 1, nil}
} else {
log.Warningf("Inconsistent worker status for policy '%s'.\n", policyName)
}
return nil
}
@ -271,8 +263,9 @@ func (sch *Scheduler) HasScheduled(policyName string) bool {
//GetPolicy is used to get related policy reference by its name.
func (sch *Scheduler) GetPolicy(policyName string) policy.Policy {
if sch.policies.Exists(policyName) {
return sch.policies.Get(policyName).p
wk := sch.policies.Get(policyName)
if wk != nil {
return wk.p
}
return nil

15
src/common/scheduler/scheduler_test.go
View File

@ -4,6 +4,7 @@ import (
"testing"
"time"
"github.com/vmware/harbor/src/common/scheduler/policy"
"github.com/vmware/harbor/src/common/scheduler/task"
)
@ -61,6 +62,10 @@ func (fp *fakePolicy) Disable() error {
return nil
}
func (fp *fakePolicy) Equal(policy.Policy) bool {
return false
}
type fakeTask struct {
number int
}
@ -98,14 +103,9 @@ func TestScheduler(t *testing.T) {
if DefaultScheduler.Schedule(fp) != nil {
t.Fatal("Schedule policy failed")
}
//Waiting for everything is stable
time.Sleep(1 * time.Second)
if DefaultScheduler.policies.Size() == 0 {
t.Fatal("No policy in the store after calling Schedule()")
}
if DefaultScheduler.stats.PolicyCount != 1 {
t.Fatal("Policy stats do not match")
}
if DefaultScheduler.GetPolicy(fp.Name()) == nil {
t.Fatal("Failed to get poicy by name")
}
@ -124,12 +124,11 @@ func TestScheduler(t *testing.T) {
if DefaultScheduler.UnSchedule(fp.Name()) != nil {
t.Fatal("Unschedule policy failed")
}
//Waiting for everything is stable
time.Sleep(1 * time.Second)
if DefaultScheduler.stats.PolicyCount != 0 {
if DefaultScheduler.policies.Size() != 0 {
t.Fatal("Policy count does not match after calling UnSchedule()")
}
<-time.After(1 * time.Second)
copiedValue := DefaultScheduler.stats.CompletedTasks
<-time.After(2 * time.Second)

1
src/common/scheduler/watcher.go
View File

@ -56,6 +56,7 @@ func (wc *Watcher) Start() {
defer func() {
wc.isRunning = false
log.Infof("Work for policy %s is stopped.\n", wc.p.Name())
}()
evalChan, err := pl.Evaluate()

40
src/common/utils/utils.go
View File

@ -16,6 +16,7 @@ package utils
import (
"crypto/rand"
"encoding/json"
"errors"
"fmt"
"net"
@ -129,46 +130,21 @@ func ParseTimeStamp(timestamp string) (*time.Time, error) {
}
//ConvertMapToStruct is used to fill the specified struct with map.
//Only support the exported fields.
func ConvertMapToStruct(object interface{}, valuesInMap map[string]interface{}) error {
func ConvertMapToStruct(object interface{}, values interface{}) error {
if object == nil {
return fmt.Errorf("nil struct is not supported")
return errors.New("nil struct is not supported")
}
if reflect.TypeOf(object).Kind() != reflect.Ptr {
return fmt.Errorf("object should be referred by pointer")
return errors.New("object should be referred by pointer")
}
for k, v := range valuesInMap {
if err := setField(object, strings.Title(strings.ToLower(k)), v); err != nil {
return err
}
bytes, err := json.Marshal(values)
if err != nil {
return err
}
return nil
}
func setField(object interface{}, field string, value interface{}) error {
structValue := reflect.ValueOf(object).Elem()
structFieldValue := structValue.FieldByName(field)
if !structFieldValue.IsValid() {
return fmt.Errorf("No such field: %s in obj", field)
}
if !structFieldValue.CanSet() {
return fmt.Errorf("Cannot set value for field %s", field)
}
structFieldType := structFieldValue.Type()
val := reflect.ValueOf(value)
if structFieldType != val.Type() {
return errors.New("Provided value type didn't match object field type")
}
structFieldValue.Set(val)
return nil
return json.Unmarshal(bytes, object)
}
// ParseProjectIDOrName parses value to ID(int64) or name(string)

53
src/ui/router.go
View File

@ -30,34 +30,35 @@ func initRouters() {
beego.SetStaticPath("/static", "./static")
beego.SetStaticPath("/i18n", "./static/i18n")
//Page Controllers:
beego.Router("/", &controllers.IndexController{})
beego.Router("/sign-in", &controllers.IndexController{})
beego.Router("/sign-up", &controllers.IndexController{})
beego.Router("/reset_password", &controllers.IndexController{})
beego.Router("/harbor", &controllers.IndexController{})
beego.Router("/harbor/sign-in", &controllers.IndexController{})
beego.Router("/harbor/sign-up", &controllers.IndexController{})
beego.Router("/harbor/dashboard", &controllers.IndexController{})
beego.Router("/harbor/projects", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/repositories", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/replications", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/members", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/logs", &controllers.IndexController{})
beego.Router("/harbor/tags/:id/*", &controllers.IndexController{})
beego.Router("/harbor/users", &controllers.IndexController{})
beego.Router("/harbor/logs", &controllers.IndexController{})
beego.Router("/harbor/replications", &controllers.IndexController{})
beego.Router("/harbor/replications/endpoints", &controllers.IndexController{})
beego.Router("/harbor/replications/rules", &controllers.IndexController{})
beego.Router("/harbor/tags", &controllers.IndexController{})
beego.Router("/harbor/configs", &controllers.IndexController{})
// standalone
if !config.WithAdmiral() {
//Disable page access in integration mode.
//Page Controllers:
beego.Router("/", &controllers.IndexController{})
beego.Router("/sign-in", &controllers.IndexController{})
beego.Router("/sign-up", &controllers.IndexController{})
beego.Router("/reset_password", &controllers.IndexController{})
beego.Router("/harbor", &controllers.IndexController{})
beego.Router("/harbor/sign-in", &controllers.IndexController{})
beego.Router("/harbor/sign-up", &controllers.IndexController{})
beego.Router("/harbor/dashboard", &controllers.IndexController{})
beego.Router("/harbor/projects", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/repositories", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/replications", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/members", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/logs", &controllers.IndexController{})
beego.Router("/harbor/tags/:id/*", &controllers.IndexController{})
beego.Router("/harbor/users", &controllers.IndexController{})
beego.Router("/harbor/logs", &controllers.IndexController{})
beego.Router("/harbor/replications", &controllers.IndexController{})
beego.Router("/harbor/replications/endpoints", &controllers.IndexController{})
beego.Router("/harbor/replications/rules", &controllers.IndexController{})
beego.Router("/harbor/tags", &controllers.IndexController{})
beego.Router("/harbor/configs", &controllers.IndexController{})
beego.Router("/login", &controllers.CommonController{}, "post:Login")
beego.Router("/log_out", &controllers.CommonController{}, "get:LogOut")
beego.Router("/reset", &controllers.CommonController{}, "post:ResetPassword")