Merge pull request #4910 from vmware/fix_none_zero_issue

Improve the error handling capabilities of job service

src/jobservice/job/impl/context.go

@@ -6,7 +6,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
 	"reflect"
+	"time"
 
 	"github.com/vmware/harbor/src/adminserver/client"
 	"github.com/vmware/harbor/src/common"
@@ -19,6 +21,10 @@ import (
 	"github.com/vmware/harbor/src/jobservice/logger"
 )
 
+const (
+	maxRetryTimes = 5
+)
+
 //Context ...
 type Context struct {
 	//System context
@@ -51,9 +57,25 @@ func NewContext(sysCtx context.Context, adminClient client.Client) *Context {
 
 //Init ...
 func (c *Context) Init() error {
-	configs, err := c.adminClient.GetCfgs()
-	if err != nil {
-		return err
+	var (
+		counter = 0
+		err     error
+		configs map[string]interface{}
+	)
+
+	for counter == 0 || err != nil {
+		counter++
+		configs, err = c.adminClient.GetCfgs()
+		if err != nil {
+			logger.Errorf("Job context initialization error: %s\n", err.Error())
+			if counter < maxRetryTimes {
+				backoff := (int)(math.Pow(2, (float64)(counter))) + 2*counter + 5
+				logger.Infof("Retry in %d seconds", backoff)
+				time.Sleep(time.Duration(backoff) * time.Second)
+			} else {
+				return fmt.Errorf("job context initialization error: %s (%d times tried)", err.Error(), counter)
+			}
+		}
 	}
 
 	db := getDBFromConfig(configs)

src/jobservice/main.go

@@ -3,7 +3,6 @@ package main
 import (
 	"errors"
 	"flag"
-	"fmt"
 
 	"github.com/vmware/harbor/src/adminserver/client"
 	"github.com/vmware/harbor/src/jobservice/config"
@@ -22,15 +21,13 @@ func main() {
 
 	//Missing config file
 	if configPath == nil || utils.IsEmptyStr(*configPath) {
-		fmt.Println("Config file should be specified")
 		flag.Usage()
-		return
+		logger.Fatal("Config file should be specified")
 	}
 
 	//Load configurations
 	if err := config.DefaultConfig.Load(*configPath, true); err != nil {
-		fmt.Printf("Failed to load configurations with error: %s\n", err)
-		return
+		logger.Fatalf("Failed to load configurations with error: %s\n", err)
 	}
 
 	//Set job context initializer

src/jobservice/runtime/bootstrap.go

@@ -91,13 +91,14 @@ func (bs *Bootstrap) LoadAndRun() {
 	logSweeper := logger.NewSweeper(ctx, config.GetLogBasePath(), config.GetLogArchivePeriod())
 	logSweeper.Start()
 
+	//To indicate if any errors occurred
+	var err error
 	//Block here
 	sig := make(chan os.Signal, 1)
 	signal.Notify(sig, os.Interrupt, syscall.SIGTERM, os.Kill)
 	select {
 	case <-sig:
-	case err := <-rootContext.ErrorChan:
-		logger.Errorf("Server error:%s\n", err)
+	case err = <-rootContext.ErrorChan:
 	}
 
 	//Call cancel to send termination signal to other interested parts.
@@ -125,6 +126,10 @@ func (bs *Bootstrap) LoadAndRun() {
 	rootContext.WG.Wait()
 	close <- true
 
+	if err != nil {
+		logger.Fatalf("Server exit with error: %s\n", err)
+	}
+
 	logger.Infof("Server gracefully exit")
 }