fix(replication): list projects before replicate to reduce create duplicate project and requests to target registry (#15934)

Signed-off-by: chlins <chenyuzh@vmware.com>

src/common/http/client.go

@@ -226,6 +226,14 @@ func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error {
 		for _, link := range links {
 			if link.Rel == "next" {
 				endpoint = url.Scheme + "://" + url.Host + link.URL
+				url, err = url.Parse(endpoint)
+				if err != nil {
+					return err
+				}
+				// encode the query parameters to avoid bad request
+				// e.g. ?q=name={p1 p2 p3} need to be encoded to ?q=name%3D%7Bp1+p2+p3%7D
+				url.RawQuery = url.Query().Encode()
+				endpoint = url.String()
 				break
 			}
 		}

src/pkg/reg/adapter/harbor/base/adapter.go

@@ -15,7 +15,7 @@
 package base
 
 import (
-	"errors"
+	"fmt"
 	"net/http"
 	"os"
 	"strconv"
@@ -24,6 +24,7 @@ import (
 	common_http "github.com/goharbor/harbor/src/common/http"
 	"github.com/goharbor/harbor/src/common/http/modifier"
 	common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth"
+	"github.com/goharbor/harbor/src/lib/errors"
 	"github.com/goharbor/harbor/src/lib/log"
 	"github.com/goharbor/harbor/src/pkg/reg/adapter/native"
 	"github.com/goharbor/harbor/src/pkg/reg/model"
@@ -169,7 +170,32 @@ func (a *Adapter) PrepareForPush(resources []*model.Resource) error {
 			Metadata: metadata,
 		}
 	}
-	for _, project := range projects {
+
+	var ps []string
+	for p := range projects {
+		ps = append(ps, p)
+	}
+	q := fmt.Sprintf("name={%s}", strings.Join(ps, " "))
+	// get exist projects
+	queryProjects, err := a.Client.ListProjectsWithQuery(q, false)
+	if err != nil {
+		return errors.Wrapf(err, "list projects with query %s", q)
+	}
+
+	existProjects := make(map[string]*Project)
+	for _, p := range queryProjects {
+		existProjects[p.Name] = p
+	}
+
+	var notExistProjects []*Project
+	for _, p := range projects {
+		_, exist := existProjects[p.Name]
+		if !exist {
+			notExistProjects = append(notExistProjects, p)
+		}
+	}
+
+	for _, project := range notExistProjects {
 		if err := a.Client.CreateProject(project.Name, project.Metadata); err != nil {
 			if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict {
 				log.Debugf("got 409 when trying to create project %s", project.Name)

src/pkg/reg/adapter/harbor/base/adapter_test.go

@@ -89,7 +89,16 @@ func TestPrepareForPush(t *testing.T) {
 		Handler: func(w http.ResponseWriter, r *http.Request) {
 			w.WriteHeader(http.StatusCreated)
 		},
-	})
+	},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/api/projects",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusOK)
+				w.Write([]byte(`[]`))
+			},
+		},
+	)
 	registry := &model.Registry{
 		URL: server.URL,
 	}
@@ -138,10 +147,11 @@ func TestPrepareForPush(t *testing.T) {
 
 	// project already exists
 	server = test.NewServer(&test.RequestHandlerMapping{
-		Method:  http.MethodPost,
+		Method:  http.MethodGet,
 		Pattern: "/api/projects",
 		Handler: func(w http.ResponseWriter, r *http.Request) {
-			w.WriteHeader(http.StatusConflict)
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(`[{"name": "library"}]`))
 		},
 	})
 	registry = &model.Registry{

src/pkg/reg/adapter/harbor/base/client.go

@@ -17,6 +17,7 @@ package base
 import (
 	"fmt"
 	"net/http"
+	"net/url"
 	"strings"
 
 	common_http "github.com/goharbor/harbor/src/common/http"
@@ -112,6 +113,19 @@ func (c *Client) ListProjects(name string) ([]*Project, error) {
 	return projects, nil
 }
 
+// ListProjectsWithQuery lists projects with query
+func (c *Client) ListProjectsWithQuery(q string, with_detail bool) ([]*Project, error) {
+	projects := []*Project{}
+	// if old version does not support query, it will fallback to normal
+	// list(list all).
+	url := fmt.Sprintf("%s/projects?q=%s&with_detail=%t", c.BasePath(), url.QueryEscape(q), with_detail)
+	if err := c.C.GetAndIteratePagination(url, &projects); err != nil {
+		return nil, err
+	}
+
+	return projects, nil
+}
+
 // GetProject gets the specific project
 func (c *Client) GetProject(name string) (*Project, error) {
 	projects, err := c.ListProjects(name)