add ping for adapter to wait for service ready

Signed-off-by: wang yan <wangyan@vmware.com>

src/core/api/health.go

@@ -34,8 +34,9 @@ import (
 )
 
 var (
-	timeout               = 60 * time.Second
+	timeout = 60 * time.Second
-	healthCheckerRegistry = map[string]health.Checker{}
+	// HealthCheckerRegistry ...
+	HealthCheckerRegistry = map[string]health.Checker{}
 )
 
 type overallHealthStatus struct {
@@ -67,11 +68,11 @@ type HealthAPI struct {
 func (h *HealthAPI) CheckHealth() {
 	var isHealthy healthy = true
 	components := []*componentHealthStatus{}
-	c := make(chan *componentHealthStatus, len(healthCheckerRegistry))
+	c := make(chan *componentHealthStatus, len(HealthCheckerRegistry))
-	for name, checker := range healthCheckerRegistry {
+	for name, checker := range HealthCheckerRegistry {
 		go check(name, checker, timeout, c)
 	}
-	for i := 0; i < len(healthCheckerRegistry); i++ {
+	for i := 0; i < len(HealthCheckerRegistry); i++ {
 		componentStatus := <-c
 		if len(componentStatus.Error) != 0 {
 			isHealthy = false
@@ -290,21 +291,21 @@ func redisHealthChecker() health.Checker {
 }
 
 func registerHealthCheckers() {
-	healthCheckerRegistry["core"] = coreHealthChecker()
+	HealthCheckerRegistry["core"] = coreHealthChecker()
-	healthCheckerRegistry["portal"] = portalHealthChecker()
+	HealthCheckerRegistry["portal"] = portalHealthChecker()
-	healthCheckerRegistry["jobservice"] = jobserviceHealthChecker()
+	HealthCheckerRegistry["jobservice"] = jobserviceHealthChecker()
-	healthCheckerRegistry["registry"] = registryHealthChecker()
+	HealthCheckerRegistry["registry"] = registryHealthChecker()
-	healthCheckerRegistry["registryctl"] = registryCtlHealthChecker()
+	HealthCheckerRegistry["registryctl"] = registryCtlHealthChecker()
-	healthCheckerRegistry["database"] = databaseHealthChecker()
+	HealthCheckerRegistry["database"] = databaseHealthChecker()
-	healthCheckerRegistry["redis"] = redisHealthChecker()
+	HealthCheckerRegistry["redis"] = redisHealthChecker()
 	if config.WithChartMuseum() {
-		healthCheckerRegistry["chartmuseum"] = chartmuseumHealthChecker()
+		HealthCheckerRegistry["chartmuseum"] = chartmuseumHealthChecker()
 	}
 	if config.WithClair() {
-		healthCheckerRegistry["clair"] = clairHealthChecker()
+		HealthCheckerRegistry["clair"] = clairHealthChecker()
 	}
 	if config.WithNotary() {
-		healthCheckerRegistry["notary"] = notaryHealthChecker()
+		HealthCheckerRegistry["notary"] = notaryHealthChecker()
 	}
 }
 

src/core/api/health_test.go

@@ -92,9 +92,9 @@ func fakeHealthChecker(healthy bool) health.Checker {
 }
 func TestCheckHealth(t *testing.T) {
 	// component01: healthy, component02: healthy => status: healthy
-	healthCheckerRegistry = map[string]health.Checker{}
+	HealthCheckerRegistry = map[string]health.Checker{}
-	healthCheckerRegistry["component01"] = fakeHealthChecker(true)
+	HealthCheckerRegistry["component01"] = fakeHealthChecker(true)
-	healthCheckerRegistry["component02"] = fakeHealthChecker(true)
+	HealthCheckerRegistry["component02"] = fakeHealthChecker(true)
 	status := map[string]interface{}{}
 	err := handleAndParse(&testingRequest{
 		method: http.MethodGet,
@@ -104,9 +104,9 @@ func TestCheckHealth(t *testing.T) {
 	assert.Equal(t, "healthy", status["status"].(string))
 
 	// component01: healthy, component02: unhealthy => status: unhealthy
-	healthCheckerRegistry = map[string]health.Checker{}
+	HealthCheckerRegistry = map[string]health.Checker{}
-	healthCheckerRegistry["component01"] = fakeHealthChecker(true)
+	HealthCheckerRegistry["component01"] = fakeHealthChecker(true)
-	healthCheckerRegistry["component02"] = fakeHealthChecker(false)
+	HealthCheckerRegistry["component02"] = fakeHealthChecker(false)
 	status = map[string]interface{}{}
 	err = handleAndParse(&testingRequest{
 		method: http.MethodGet,
@@ -128,7 +128,7 @@ func TestDatabaseHealthChecker(t *testing.T) {
 }
 
 func TestRegisterHealthCheckers(t *testing.T) {
-	healthCheckerRegistry = map[string]health.Checker{}
+	HealthCheckerRegistry = map[string]health.Checker{}
 	registerHealthCheckers()
-	assert.NotNil(t, healthCheckerRegistry["core"])
+	assert.NotNil(t, HealthCheckerRegistry["core"])
 }

src/core/api/quota/chart/chart.go

@@ -21,6 +21,7 @@ import (
 	"github.com/goharbor/harbor/src/common/models"
 	common_quota "github.com/goharbor/harbor/src/common/quota"
 	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/core/api"
 	quota "github.com/goharbor/harbor/src/core/api/quota"
 	"github.com/goharbor/harbor/src/core/config"
 	"github.com/goharbor/harbor/src/core/promgr"
@@ -49,6 +50,11 @@ var (
 	controllerOnce sync.Once
 )
 
+// Ping ...
+func (rm *Migrator) Ping() error {
+	return api.HealthCheckerRegistry["chartmuseum"].Check()
+}
+
 // Dump ...
 // Depends on DB to dump chart data, as chart cannot get all of namespaces.
 func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
@@ -99,8 +105,6 @@ func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
 			defer wg.Done()
 
 			var repos []quota.RepoData
-			var afs []*models.Artifact
-
 			ctr, err := chartController()
 			if err != nil {
 				errChan <- err
@@ -115,6 +119,7 @@ func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
 
 			// repo
 			for _, chart := range chartInfo {
+				var afs []*models.Artifact
 				chartVersions, err := ctr.GetChart(project.Name, chart.Name)
 				if err != nil {
 					errChan <- err
@@ -171,7 +176,8 @@ func (rm *Migrator) Usage(projects []quota.ProjectInfo) ([]quota.ProjectUsage, e
 		proUsage := quota.ProjectUsage{
 			Project: project.Name,
 			Used: common_quota.ResourceList{
-				common_quota.ResourceCount: count,
+				common_quota.ResourceCount:   count,
+				common_quota.ResourceStorage: 0,
 			},
 		}
 		pros = append(pros, proUsage)

src/core/api/quota/migrator.go

@@ -21,11 +21,15 @@ import (
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/core/config"
 	"github.com/goharbor/harbor/src/core/promgr"
+	"github.com/goharbor/harbor/src/pkg/types"
 	"strconv"
 )
 
 // QuotaMigrator ...
 type QuotaMigrator interface {
+	// Ping validates and wait for backend service ready.
+	Ping() error
+
 	// Dump exports all data from backend service, registry, chartmuseum
 	Dump() ([]ProjectInfo, error)
 
@@ -74,6 +78,7 @@ func Register(name string, adapter Instance) {
 
 // Sync ...
 func Sync(pm promgr.ProjectManager, populate bool) error {
+	totalUsage := make(map[string][]ProjectUsage)
 	for name, instanceFunc := range adapters {
 		if !config.WithChartMuseum() {
 			if name == "chart" {
@@ -81,6 +86,9 @@ func Sync(pm promgr.ProjectManager, populate bool) error {
 			}
 		}
 		adapter := instanceFunc(pm)
+		if err := adapter.Ping(); err != nil {
+			return err
+		}
 		data, err := adapter.Dump()
 		if err != nil {
 			return err
@@ -89,18 +97,58 @@ func Sync(pm promgr.ProjectManager, populate bool) error {
 		if err != nil {
 			return err
 		}
-		if err := ensureQuota(usage); err != nil {
+		totalUsage[name] = usage
-			return err
-		}
 		if populate {
 			if err := adapter.Persist(data); err != nil {
 				return err
 			}
 		}
 	}
+	merged := mergeUsage(totalUsage)
+	if err := ensureQuota(merged); err != nil {
+		return err
+	}
 	return nil
 }
 
+// mergeUsage merges the usage of adapters
+func mergeUsage(total map[string][]ProjectUsage) []ProjectUsage {
+	if !config.WithChartMuseum() {
+		return total["registry"]
+	}
+	regUsgs := total["registry"]
+	chartUsgs := total["chart"]
+
+	var mergedUsage []ProjectUsage
+	temp := make(map[string]quota.ResourceList)
+
+	for _, regUsg := range regUsgs {
+		_, exist := temp[regUsg.Project]
+		if !exist {
+			temp[regUsg.Project] = regUsg.Used
+			mergedUsage = append(mergedUsage, ProjectUsage{
+				Project: regUsg.Project,
+				Used:    regUsg.Used,
+			})
+		}
+	}
+	for _, chartUsg := range chartUsgs {
+		var usedTemp quota.ResourceList
+		_, exist := temp[chartUsg.Project]
+		if !exist {
+			usedTemp = chartUsg.Used
+		} else {
+			usedTemp = types.Add(temp[chartUsg.Project], chartUsg.Used)
+		}
+		temp[chartUsg.Project] = usedTemp
+		mergedUsage = append(mergedUsage, ProjectUsage{
+			Project: chartUsg.Project,
+			Used:    usedTemp,
+		})
+	}
+	return mergedUsage
+}
+
 // ensureQuota updates the quota and quota usage in the data base.
 func ensureQuota(usages []ProjectUsage) error {
 	var pid int64

src/core/api/quota/registry/registry.go

@@ -46,6 +46,11 @@ func NewRegistryMigrator(pm promgr.ProjectManager) quota.QuotaMigrator {
 	return &migrator
 }
 
+// Ping ...
+func (rm *Migrator) Ping() error {
+	return api.HealthCheckerRegistry["registry"].Check()
+}
+
 // Dump ...
 func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
 	var (