From 769c9c63f12fadcd0bd622ae5cd50a333cb8b44d Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Mon, 10 Oct 2016 15:40:18 +0800 Subject: [PATCH] fix #856 --- api/base.go | 4 +- api/jobs/replication.go | 79 +++++++++++++++++----------------- dao/dao_test.go | 2 +- utils/link.go | 95 +++++++++++++++++++++++++++++++++++++++++ utils/utils_test.go | 38 +++++++++++++++++ 5 files changed, 176 insertions(+), 42 deletions(-) create mode 100644 utils/link.go diff --git a/api/base.go b/api/base.go index 9e8b96123..52e1711d0 100644 --- a/api/base.go +++ b/api/base.go @@ -32,8 +32,8 @@ import ( ) const ( - defaultPageSize int64 = 10 - maxPageSize int64 = 100 + defaultPageSize int64 = 500 + maxPageSize int64 = 500 ) // BaseAPI wraps common methods for controllers to host API diff --git a/api/jobs/replication.go b/api/jobs/replication.go index 98b8d6320..09d2469f6 100644 --- a/api/jobs/replication.go +++ b/api/jobs/replication.go @@ -29,6 +29,7 @@ import ( "github.com/vmware/harbor/job/config" "github.com/vmware/harbor/job/utils" "github.com/vmware/harbor/models" + u "github.com/vmware/harbor/utils" "github.com/vmware/harbor/utils/log" ) @@ -176,46 +177,46 @@ func (rj *ReplicationJob) GetLog() { // calls the api from UI to get repo list func getRepoList(projectID int64) ([]string, error) { - /* - uiUser := os.Getenv("UI_USR") - if len(uiUser) == 0 { - uiUser = "admin" - } - uiPwd := os.Getenv("UI_PWD") - if len(uiPwd) == 0 { - uiPwd = "Harbor12345" - } - */ - uiURL := config.LocalUIURL() + repositories := []string{} + client := &http.Client{} - req, err := http.NewRequest("GET", uiURL+"/api/repositories?project_id="+strconv.Itoa(int(projectID)), nil) - if err != nil { - log.Errorf("Error when creating request: %v", err) - return nil, err - } - //req.SetBasicAuth(uiUser, uiPwd) - req.AddCookie(&http.Cookie{Name: models.UISecretCookie, Value: config.UISecret()}) - //dump, err := httputil.DumpRequest(req, true) - //log.Debugf("req: %q", dump) - resp, err := client.Do(req) - if err != nil { - log.Errorf("Error when calling UI api to get repositories, error: %v", err) - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - log.Errorf("Unexpected status code: %d", resp.StatusCode) - dump, _ := httputil.DumpResponse(resp, true) - log.Debugf("response: %q", dump) - return nil, fmt.Errorf("Unexpected status code when getting repository list: %d", resp.StatusCode) + uiURL := config.LocalUIURL() + next := "/api/repositories?project_id=" + strconv.Itoa(int(projectID)) + for len(next) != 0 { + req, err := http.NewRequest("GET", uiURL+next, nil) + if err != nil { + return repositories, err + } + + req.AddCookie(&http.Cookie{Name: models.UISecretCookie, Value: config.UISecret()}) + + resp, err := client.Do(req) + if err != nil { + return repositories, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + dump, _ := httputil.DumpResponse(resp, true) + log.Debugf("response: %q", dump) + return repositories, fmt.Errorf("Unexpected status code when getting repository list: %d", resp.StatusCode) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return repositories, err + } + + var list []string + if err = json.Unmarshal(body, &list); err != nil { + return repositories, err + } + + repositories = append(repositories, list...) + + links := u.ParseLink(resp.Header.Get(http.CanonicalHeaderKey("link"))) + next = links.Next() } - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Errorf("Failed to read the response body, error: %v", err) - return nil, err - } - var repoList []string - err = json.Unmarshal(body, &repoList) - return repoList, err + return repositories, nil } diff --git a/dao/dao_test.go b/dao/dao_test.go index edae0f858..42aa0b238 100644 --- a/dao/dao_test.go +++ b/dao/dao_test.go @@ -1586,7 +1586,7 @@ func TestRepositoryExists(t *testing.T) { var exists bool exists = RepositoryExists(currentRepository.Name) if !exists { - t.Errorf("The repository with name: %d, does not exist", currentRepository.Name) + t.Errorf("The repository with name: %s, does not exist", currentRepository.Name) } } diff --git a/utils/link.go b/utils/link.go new file mode 100644 index 000000000..fad043915 --- /dev/null +++ b/utils/link.go @@ -0,0 +1,95 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package utils + +import ( + "strings" +) + +// Link : HTTP link header +type Link struct { + // URL : url part of header + URL string + // Rel : prev or next + Rel string +} + +// Links : multiple link +type Links []*Link + +// Prev returns the URL indicated by "prev" rel. +func (l Links) Prev() string { + prev := "" + for _, link := range l { + if strings.ToLower(link.Rel) == "prev" { + prev = link.URL + break + } + } + return prev +} + +// Next returns the URL indicated by "next" rel. +func (l Links) Next() string { + next := "" + for _, link := range l { + if link.Rel == "next" { + next = link.URL + break + } + } + return next +} + +// ParseLink parses the raw link header to Links +func ParseLink(raw string) Links { + links := Links{} + + for _, l := range strings.Split(raw, ",") { + link := parseSingleLink(l) + if link != nil { + links = append(links, link) + } + } + + return links +} + +func parseSingleLink(raw string) *Link { + link := &Link{} + + for _, str := range strings.Split(raw, ";") { + str = strings.TrimSpace(str) + if strings.HasPrefix(str, "<") && strings.HasSuffix(str, ">") { + str = strings.Trim(str, "<>") + link.URL = str + continue + } + + parts := strings.SplitN(str, "=", 2) + if len(parts) != 2 || strings.ToLower(parts[0]) != "rel" { + continue + } + + link.Rel = strings.ToLower(strings.Trim(parts[1], "\"")) + } + + if len(link.URL) == 0 || len(link.Rel) == 0 { + link = nil + } + + return link +} diff --git a/utils/utils_test.go b/utils/utils_test.go index d151b8452..d5de02319 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -134,3 +134,41 @@ func TestReversibleEncrypt(t *testing.T) { t.Errorf("decrypted password: %s, is not identical to original", decrypted) } } + +func TestParseLink(t *testing.T) { + raw := "" + links := ParseLink(raw) + if len(links) != 0 { + t.Errorf("unexpected length: %d != %d", len(links), 0) + } + + raw = "a;b,c" + links = ParseLink(raw) + if len(links) != 0 { + t.Errorf("unexpected length: %d != %d", len(links), 0) + } + + raw = `; rel="prev"` + links = ParseLink(raw) + if len(links) != 1 { + t.Errorf("unexpected length: %d != %d", len(links), 1) + } + prev := `/api/users?page=1&page_size=100` + if links.Prev() != prev { + t.Errorf("unexpected prev: %s != %s", links.Prev(), prev) + } + + raw = `; rel="prev", ; rel="next"` + links = ParseLink(raw) + if len(links) != 2 { + t.Errorf("unexpected length: %d != %d", len(links), 2) + } + prev = `/api/users?page=1&page_size=100` + if links.Prev() != prev { + t.Errorf("unexpected prev: %s != %s", links.Prev(), prev) + } + next := `/api/users?page=3&page_size=100` + if links.Next() != next { + t.Errorf("unexpected prev: %s != %s", links.Next(), next) + } +}