fix #856

api/base.go

@@ -32,8 +32,8 @@ import (
 )
 
 const (
-	defaultPageSize int64 = 10
-	maxPageSize     int64 = 100
+	defaultPageSize int64 = 500
+	maxPageSize     int64 = 500
 )
 
 // BaseAPI wraps common methods for controllers to host API

api/jobs/replication.go

@@ -29,6 +29,7 @@ import (
 	"github.com/vmware/harbor/job/config"
 	"github.com/vmware/harbor/job/utils"
 	"github.com/vmware/harbor/models"
+	u "github.com/vmware/harbor/utils"
 	"github.com/vmware/harbor/utils/log"
 )
 
@@ -176,46 +177,46 @@ func (rj *ReplicationJob) GetLog() {
 
 // calls the api from UI to get repo list
 func getRepoList(projectID int64) ([]string, error) {
-	/*
-		uiUser := os.Getenv("UI_USR")
-		if len(uiUser) == 0 {
-			uiUser = "admin"
-		}
-		uiPwd := os.Getenv("UI_PWD")
-		if len(uiPwd) == 0 {
-			uiPwd = "Harbor12345"
-		}
-	*/
-	uiURL := config.LocalUIURL()
+	repositories := []string{}
+
 	client := &http.Client{}
-	req, err := http.NewRequest("GET", uiURL+"/api/repositories?project_id="+strconv.Itoa(int(projectID)), nil)
-	if err != nil {
-		log.Errorf("Error when creating request: %v", err)
-		return nil, err
-	}
-	//req.SetBasicAuth(uiUser, uiPwd)
-	req.AddCookie(&http.Cookie{Name: models.UISecretCookie, Value: config.UISecret()})
-	//dump, err := httputil.DumpRequest(req, true)
-	//log.Debugf("req: %q", dump)
-	resp, err := client.Do(req)
-	if err != nil {
-		log.Errorf("Error when calling UI api to get repositories, error: %v", err)
-		return nil, err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		log.Errorf("Unexpected status code: %d", resp.StatusCode)
-		dump, _ := httputil.DumpResponse(resp, true)
-		log.Debugf("response: %q", dump)
-		return nil, fmt.Errorf("Unexpected status code when getting repository list: %d", resp.StatusCode)
+	uiURL := config.LocalUIURL()
+	next := "/api/repositories?project_id=" + strconv.Itoa(int(projectID))
+	for len(next) != 0 {
+		req, err := http.NewRequest("GET", uiURL+next, nil)
+		if err != nil {
+			return repositories, err
+		}
+
+		req.AddCookie(&http.Cookie{Name: models.UISecretCookie, Value: config.UISecret()})
+
+		resp, err := client.Do(req)
+		if err != nil {
+			return repositories, err
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != http.StatusOK {
+			dump, _ := httputil.DumpResponse(resp, true)
+			log.Debugf("response: %q", dump)
+			return repositories, fmt.Errorf("Unexpected status code when getting repository list: %d", resp.StatusCode)
+		}
+
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return repositories, err
+		}
+
+		var list []string
+		if err = json.Unmarshal(body, &list); err != nil {
+			return repositories, err
+		}
+
+		repositories = append(repositories, list...)
+
+		links := u.ParseLink(resp.Header.Get(http.CanonicalHeaderKey("link")))
+		next = links.Next()
 	}
 
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		log.Errorf("Failed to read the response body, error: %v", err)
-		return nil, err
-	}
-	var repoList []string
-	err = json.Unmarshal(body, &repoList)
-	return repoList, err
+	return repositories, nil
 }

dao/dao_test.go

@@ -1586,7 +1586,7 @@ func TestRepositoryExists(t *testing.T) {
 	var exists bool
 	exists = RepositoryExists(currentRepository.Name)
 	if !exists {
-		t.Errorf("The repository with name: %d, does not exist", currentRepository.Name)
+		t.Errorf("The repository with name: %s, does not exist", currentRepository.Name)
 	}
 }
 

utils/link.go

@@ -0,0 +1,95 @@
+/*
+   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package utils
+
+import (
+	"strings"
+)
+
+// Link : HTTP link header
+type Link struct {
+	// URL : url part of header
+	URL string
+	// Rel : prev or next
+	Rel string
+}
+
+// Links : multiple link
+type Links []*Link
+
+// Prev returns the URL indicated by "prev" rel.
+func (l Links) Prev() string {
+	prev := ""
+	for _, link := range l {
+		if strings.ToLower(link.Rel) == "prev" {
+			prev = link.URL
+			break
+		}
+	}
+	return prev
+}
+
+// Next returns the URL indicated by "next" rel.
+func (l Links) Next() string {
+	next := ""
+	for _, link := range l {
+		if link.Rel == "next" {
+			next = link.URL
+			break
+		}
+	}
+	return next
+}
+
+// ParseLink parses the raw link header to Links
+func ParseLink(raw string) Links {
+	links := Links{}
+
+	for _, l := range strings.Split(raw, ",") {
+		link := parseSingleLink(l)
+		if link != nil {
+			links = append(links, link)
+		}
+	}
+
+	return links
+}
+
+func parseSingleLink(raw string) *Link {
+	link := &Link{}
+
+	for _, str := range strings.Split(raw, ";") {
+		str = strings.TrimSpace(str)
+		if strings.HasPrefix(str, "<") && strings.HasSuffix(str, ">") {
+			str = strings.Trim(str, "<>")
+			link.URL = str
+			continue
+		}
+
+		parts := strings.SplitN(str, "=", 2)
+		if len(parts) != 2 || strings.ToLower(parts[0]) != "rel" {
+			continue
+		}
+
+		link.Rel = strings.ToLower(strings.Trim(parts[1], "\""))
+	}
+
+	if len(link.URL) == 0 || len(link.Rel) == 0 {
+		link = nil
+	}
+
+	return link
+}

utils/utils_test.go

@@ -134,3 +134,41 @@ func TestReversibleEncrypt(t *testing.T) {
 		t.Errorf("decrypted password: %s, is not identical to original", decrypted)
 	}
 }
+
+func TestParseLink(t *testing.T) {
+	raw := ""
+	links := ParseLink(raw)
+	if len(links) != 0 {
+		t.Errorf("unexpected length: %d != %d", len(links), 0)
+	}
+
+	raw = "a;b,c"
+	links = ParseLink(raw)
+	if len(links) != 0 {
+		t.Errorf("unexpected length: %d != %d", len(links), 0)
+	}
+
+	raw = `</api/users?page=1&page_size=100>; rel="prev"`
+	links = ParseLink(raw)
+	if len(links) != 1 {
+		t.Errorf("unexpected length: %d != %d", len(links), 1)
+	}
+	prev := `/api/users?page=1&page_size=100`
+	if links.Prev() != prev {
+		t.Errorf("unexpected prev: %s != %s", links.Prev(), prev)
+	}
+
+	raw = `</api/users?page=1&page_size=100>; rel="prev", </api/users?page=3&page_size=100>; rel="next"`
+	links = ParseLink(raw)
+	if len(links) != 2 {
+		t.Errorf("unexpected length: %d != %d", len(links), 2)
+	}
+	prev = `/api/users?page=1&page_size=100`
+	if links.Prev() != prev {
+		t.Errorf("unexpected prev: %s != %s", links.Prev(), prev)
+	}
+	next := `/api/users?page=3&page_size=100`
+	if links.Next() != next {
+		t.Errorf("unexpected prev: %s != %s", links.Next(), next)
+	}
+}