mirror of
https://github.com/goharbor/harbor
synced 2024-09-21 00:39:58 +00:00
fix(replication): list projects before replicate to reduce create duplicate project and requests to target registry (#15934)
Signed-off-by: chlins <chenyuzh@vmware.com>
This commit is contained in:
parent
b2268dbf8e
commit
3aa698c7c9
|
@ -226,6 +226,14 @@ func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error {
|
||||||
for _, link := range links {
|
for _, link := range links {
|
||||||
if link.Rel == "next" {
|
if link.Rel == "next" {
|
||||||
endpoint = url.Scheme + "://" + url.Host + link.URL
|
endpoint = url.Scheme + "://" + url.Host + link.URL
|
||||||
|
url, err = url.Parse(endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// encode the query parameters to avoid bad request
|
||||||
|
// e.g. ?q=name={p1 p2 p3} need to be encoded to ?q=name%3D%7Bp1+p2+p3%7D
|
||||||
|
url.RawQuery = url.Query().Encode()
|
||||||
|
endpoint = url.String()
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,7 +15,7 @@
|
||||||
package base
|
package base
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
@ -24,6 +24,7 @@ import (
|
||||||
common_http "github.com/goharbor/harbor/src/common/http"
|
common_http "github.com/goharbor/harbor/src/common/http"
|
||||||
"github.com/goharbor/harbor/src/common/http/modifier"
|
"github.com/goharbor/harbor/src/common/http/modifier"
|
||||||
common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth"
|
common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth"
|
||||||
|
"github.com/goharbor/harbor/src/lib/errors"
|
||||||
"github.com/goharbor/harbor/src/lib/log"
|
"github.com/goharbor/harbor/src/lib/log"
|
||||||
"github.com/goharbor/harbor/src/pkg/reg/adapter/native"
|
"github.com/goharbor/harbor/src/pkg/reg/adapter/native"
|
||||||
"github.com/goharbor/harbor/src/pkg/reg/model"
|
"github.com/goharbor/harbor/src/pkg/reg/model"
|
||||||
|
@ -169,7 +170,32 @@ func (a *Adapter) PrepareForPush(resources []*model.Resource) error {
|
||||||
Metadata: metadata,
|
Metadata: metadata,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, project := range projects {
|
|
||||||
|
var ps []string
|
||||||
|
for p := range projects {
|
||||||
|
ps = append(ps, p)
|
||||||
|
}
|
||||||
|
q := fmt.Sprintf("name={%s}", strings.Join(ps, " "))
|
||||||
|
// get exist projects
|
||||||
|
queryProjects, err := a.Client.ListProjectsWithQuery(q, false)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "list projects with query %s", q)
|
||||||
|
}
|
||||||
|
|
||||||
|
existProjects := make(map[string]*Project)
|
||||||
|
for _, p := range queryProjects {
|
||||||
|
existProjects[p.Name] = p
|
||||||
|
}
|
||||||
|
|
||||||
|
var notExistProjects []*Project
|
||||||
|
for _, p := range projects {
|
||||||
|
_, exist := existProjects[p.Name]
|
||||||
|
if !exist {
|
||||||
|
notExistProjects = append(notExistProjects, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, project := range notExistProjects {
|
||||||
if err := a.Client.CreateProject(project.Name, project.Metadata); err != nil {
|
if err := a.Client.CreateProject(project.Name, project.Metadata); err != nil {
|
||||||
if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict {
|
if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict {
|
||||||
log.Debugf("got 409 when trying to create project %s", project.Name)
|
log.Debugf("got 409 when trying to create project %s", project.Name)
|
||||||
|
|
|
@ -89,7 +89,16 @@ func TestPrepareForPush(t *testing.T) {
|
||||||
Handler: func(w http.ResponseWriter, r *http.Request) {
|
Handler: func(w http.ResponseWriter, r *http.Request) {
|
||||||
w.WriteHeader(http.StatusCreated)
|
w.WriteHeader(http.StatusCreated)
|
||||||
},
|
},
|
||||||
})
|
},
|
||||||
|
&test.RequestHandlerMapping{
|
||||||
|
Method: http.MethodGet,
|
||||||
|
Pattern: "/api/projects",
|
||||||
|
Handler: func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.Write([]byte(`[]`))
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
registry := &model.Registry{
|
registry := &model.Registry{
|
||||||
URL: server.URL,
|
URL: server.URL,
|
||||||
}
|
}
|
||||||
|
@ -138,10 +147,11 @@ func TestPrepareForPush(t *testing.T) {
|
||||||
|
|
||||||
// project already exists
|
// project already exists
|
||||||
server = test.NewServer(&test.RequestHandlerMapping{
|
server = test.NewServer(&test.RequestHandlerMapping{
|
||||||
Method: http.MethodPost,
|
Method: http.MethodGet,
|
||||||
Pattern: "/api/projects",
|
Pattern: "/api/projects",
|
||||||
Handler: func(w http.ResponseWriter, r *http.Request) {
|
Handler: func(w http.ResponseWriter, r *http.Request) {
|
||||||
w.WriteHeader(http.StatusConflict)
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.Write([]byte(`[{"name": "library"}]`))
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
registry = &model.Registry{
|
registry = &model.Registry{
|
||||||
|
|
|
@ -17,6 +17,7 @@ package base
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
common_http "github.com/goharbor/harbor/src/common/http"
|
common_http "github.com/goharbor/harbor/src/common/http"
|
||||||
|
@ -112,6 +113,19 @@ func (c *Client) ListProjects(name string) ([]*Project, error) {
|
||||||
return projects, nil
|
return projects, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListProjectsWithQuery lists projects with query
|
||||||
|
func (c *Client) ListProjectsWithQuery(q string, with_detail bool) ([]*Project, error) {
|
||||||
|
projects := []*Project{}
|
||||||
|
// if old version does not support query, it will fallback to normal
|
||||||
|
// list(list all).
|
||||||
|
url := fmt.Sprintf("%s/projects?q=%s&with_detail=%t", c.BasePath(), url.QueryEscape(q), with_detail)
|
||||||
|
if err := c.C.GetAndIteratePagination(url, &projects); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return projects, nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetProject gets the specific project
|
// GetProject gets the specific project
|
||||||
func (c *Client) GetProject(name string) (*Project, error) {
|
func (c *Client) GetProject(name string) (*Project, error) {
|
||||||
projects, err := c.ListProjects(name)
|
projects, err := c.ListProjects(name)
|
||||||
|
|
Loading…
Reference in New Issue
Block a user