upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor synced 2025-04-20 06:30:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Changed logic search projects in gitlab adapter (#18529)

Browse Source 
* fix(gitlab): change logic search projects usage search_namespaces

Signed-off-by: lxShaDoWxl <lxshadowxkingxl@gmail.com>

* tests(gitlab): remove old data and actualization test

Signed-off-by: lxShaDoWxl <lxshadowxkingxl@gmail.com>

* refactor(gitlab): added debug log

Signed-off-by: lxShaDoWxl <lxshadowxkingxl@gmail.com>
(cherry picked from commit 732806221b1c1633682fe1adad4cbcd3a4687d0b)

* lint(gitlab): fix import order

Signed-off-by: lxShaDoWxl <lxshadowxkingxl@gmail.com>

---------

Signed-off-by: lxShaDoWxl <lxshadowxkingxl@gmail.com>
This commit is contained in:
Viktor 2023-05-15 11:59:13 +06:00 committed by GitHub
parent 7b0692d698
commit f8e28dbede
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 24 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/pkg/reg/adapter/gitlab/adapter.go
View File

@ -15,6 +15,7 @@
package gitlab package gitlab
import ( import (
"net/url"
"strings" "strings"
"github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/log"
@ -127,6 +128,7 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
} else { } else {
pathPatterns = append(pathPatterns, nameFilter) pathPatterns = append(pathPatterns, nameFilter)
} }
log.Debugf("Patterns: %v", pathPatterns)
for _, project := range projects { for _, project := range projects {
if !project.RegistryEnabled { if !project.RegistryEnabled {
@ -143,8 +145,10 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
} }
for _, repository := range repositories { for _, repository := range repositories {
if !existPatterns(repository.Path, pathPatterns) { if !existPatterns(repository.Path, pathPatterns) {
log.Debugf("Skipping repository path=%s and id=%d", repository.Path, repository.ID)
continue continue
} }
log.Debugf("Search tags repository path=%s and id=%d", repository.Path, repository.ID)
vTags, err := a.clientGitlabAPI.getTags(project.ID, repository.ID) vTags, err := a.clientGitlabAPI.getTags(project.ID, repository.ID)
if err != nil { if err != nil {
return nil, err return nil, err
@ -183,21 +187,12 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
func (a *adapter) getProjectsByPattern(pattern string) ([]*Project, error) { func (a *adapter) getProjectsByPattern(pattern string) ([]*Project, error) {
var projects []*Project var projects []*Project
projectset := make(map[string]bool)
var err error var err error
if len(pattern) > 0 { if len(pattern) > 0 {
names, ok := util.IsSpecificPath(pattern) names, ok := util.IsSpecificPath(pattern)
if ok { if ok {
for _, name := range names { for _, name := range names {
substrings := strings.Split(name, "/") var projectsByName, err = a.clientGitlabAPI.getProjectsByName(url.QueryEscape(name))
if len(substrings) < 2 {
continue
}
if _, ok := projectset[substrings[1]]; ok {
continue
}
projectset[substrings[1]] = true
var projectsByName, err = a.clientGitlabAPI.getProjectsByName(substrings[1])
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -207,20 +202,20 @@ func (a *adapter) getProjectsByPattern(pattern string) ([]*Project, error) {
projects = append(projects, projectsByName...) projects = append(projects, projectsByName...)
} }
} else { } else {
substrings := strings.Split(pattern, "/") projectName := ""
if len(substrings) < 2 { for i, substring := range strings.Split(pattern, "/") {
if strings.Contains(substring, "*") {
if i != 0 {
break
}
} else {
projectName += substring + "/"
}
}
if projectName == "" {
return projects, nil return projects, nil
} }
projectName := substrings[1] projects, err = a.clientGitlabAPI.getProjectsByName(url.QueryEscape(projectName))
if projectName == "*" {
return projects, nil
}
projectName = strings.Trim(projectName, "*")
if strings.Contains(projectName, "*") {
return projects, nil
}
projects, err = a.clientGitlabAPI.getProjectsByName(projectName)
if err != nil { if err != nil {
return nil, err return nil, err
} }

10
src/pkg/reg/adapter/gitlab/adapter_test.go
View File

@ -38,16 +38,10 @@ func getServer(t *testing.T) *httptest.Server {
w.Header().Set("X-Next-Page", "") w.Header().Set("X-Next-Page", "")
switch search { switch search {
case "dev-docker": case "library/dev-docker", "library", "library/", "dev-docker/", "dev-docker":
mustWriteHTTPResponse(t, w, "testdata/projects/dev-docker.json") mustWriteHTTPResponse(t, w, "testdata/projects/dev-docker.json")
break break
case "dev-": case "", "library/dockers":
mustWriteHTTPResponse(t, w, "testdata/projects/dev-docker.json")
break
case "-docker":
mustWriteHTTPResponse(t, w, "testdata/projects/-docker.json")
break
case "":
mustWriteHTTPResponse(t, w, "testdata/projects/all.json") mustWriteHTTPResponse(t, w, "testdata/projects/all.json")
break break
default: default:

8
src/pkg/reg/adapter/gitlab/client.go
View File

@ -25,6 +25,7 @@ import (
common_http "github.com/goharbor/harbor/src/common/http" common_http "github.com/goharbor/harbor/src/common/http"
liberrors "github.com/goharbor/harbor/src/lib/errors" liberrors "github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/reg/model" "github.com/goharbor/harbor/src/pkg/reg/model"
"github.com/goharbor/harbor/src/pkg/reg/util" "github.com/goharbor/harbor/src/pkg/reg/util"
) )
@ -82,7 +83,7 @@ func (c *Client) getProjects() ([]*Project, error) {
func (c *Client) getProjectsByName(name string) ([]*Project, error) { func (c *Client) getProjectsByName(name string) ([]*Project, error) {
var projects []*Project var projects []*Project
urlAPI := fmt.Sprintf("%s/api/v4/projects?search=%s&membership=1&per_page=50", c.url, name) urlAPI := fmt.Sprintf("%s/api/v4/projects?search=%s&membership=true&search_namespaces=true&per_page=50", c.url, name)
if err := c.GetAndIteratePagination(urlAPI, &projects); err != nil { if err := c.GetAndIteratePagination(urlAPI, &projects); err != nil {
return nil, err return nil, err
} }
@ -94,6 +95,7 @@ func (c *Client) getRepositories(projectID int64) ([]*Repository, error) {
if err := c.GetAndIteratePagination(urlAPI, &repositories); err != nil { if err := c.GetAndIteratePagination(urlAPI, &repositories); err != nil {
return nil, err return nil, err
} }
log.Debugf("Count repositories %d in project %d", len(repositories), projectID)
return repositories, nil return repositories, nil
} }
@ -103,6 +105,7 @@ func (c *Client) getTags(projectID int64, repositoryID int64) ([]*Tag, error) {
if err := c.GetAndIteratePagination(urlAPI, &tags); err != nil { if err := c.GetAndIteratePagination(urlAPI, &tags); err != nil {
return nil, err return nil, err
} }
log.Debugf("Count tags %d in repository %d, and project %d", len(tags), repositoryID, projectID)
return tags, nil return tags, nil
} }
@ -113,7 +116,6 @@ func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error {
if err != nil { if err != nil {
return err return err
} }
rv := reflect.ValueOf(v) rv := reflect.ValueOf(v)
if rv.Kind() != reflect.Ptr { if rv.Kind() != reflect.Ptr {
return errors.New("v should be a pointer to a slice") return errors.New("v should be a pointer to a slice")
@ -122,7 +124,7 @@ func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error {
if elemType.Kind() != reflect.Slice { if elemType.Kind() != reflect.Slice {
return errors.New("v should be a pointer to a slice") return errors.New("v should be a pointer to a slice")
} }
log.Debugf("Gitlab request %s", urlAPI)
resources := reflect.Indirect(reflect.New(elemType)) resources := reflect.Indirect(reflect.New(elemType))
for len(endpoint) > 0 { for len(endpoint) > 0 {
req, err := c.newRequest(http.MethodGet, endpoint, nil) req, err := c.newRequest(http.MethodGet, endpoint, nil)

97
src/pkg/reg/adapter/gitlab/testdata/projects/-docker.json vendored
View File

@ -1,97 +0,0 @@
[
{
"id": 3,
"description": "",
"name": "dev-docker",
"name_with_namespace": "Library / dev-docker",
"path": "dev-docker",
"path_with_namespace": "library/dev-docker",
"created_at": "2019-01-17T09:47:07.504Z",
"default_branch": "master",
"tag_list": [],
"avatar_url": null,
"star_count": 0,
"forks_count": 0,
"last_activity_at": "2019-06-09T15:18:10.045Z",
"empty_repo": false,
"archived": false,
"visibility": "private",
"resolve_outdated_diff_discussions": false,
"container_registry_enabled": true,
"issues_enabled": true,
"merge_requests_enabled": true,
"wiki_enabled": true,
"jobs_enabled": true,
"snippets_enabled": true,
"shared_runners_enabled": true,
"lfs_enabled": true,
"creator_id": 1,
"forked_from_project": {},
"import_status": "finished",
"open_issues_count": 0,
"ci_default_git_depth": null,
"public_jobs": true,
"ci_config_path": null,
"shared_with_groups": [],
"only_allow_merge_if_pipeline_succeeds": false,
"request_access_enabled": false,
"only_allow_merge_if_all_discussions_are_resolved": false,
"printing_merge_request_link_enabled": true,
"merge_method": "merge",
"external_authorization_classification_label": "",
"permissions": {
"project_access": null,
"group_access": null
},
"mirror": false
},
{
"id": 4,
"description": "",
"name": "dev-docker-test",
"name_with_namespace": "Library / dev-docker-test",
"path": "dev-docker-test",
"path_with_namespace": "library/dev-docker-test",
"created_at": "2019-01-17T09:47:07.504Z",
"default_branch": "master",
"tag_list": [],
"avatar_url": null,
"star_count": 0,
"forks_count": 0,
"last_activity_at": "2019-06-09T15:18:10.045Z",
"empty_repo": false,
"archived": false,
"visibility": "private",
"resolve_outdated_diff_discussions": false,
"container_registry_enabled": true,
"issues_enabled": true,
"merge_requests_enabled": true,
"wiki_enabled": true,
"jobs_enabled": true,
"snippets_enabled": true,
"shared_runners_enabled": true,
"lfs_enabled": true,
"creator_id": 1,
"forked_from_project": {},
"import_status": "finished",
"open_issues_count": 0,
"ci_default_git_depth": null,
"public_jobs": true,
"ci_config_path": null,
"shared_with_groups": [],
"only_allow_merge_if_pipeline_succeeds": false,
"request_access_enabled": false,
"only_allow_merge_if_all_discussions_are_resolved": false,
"printing_merge_request_link_enabled": true,
"merge_method": "merge",
"external_authorization_classification_label": "",
"permissions": {
"project_access": null,
"group_access": null
},
"mirror": false
}
]

96
src/pkg/reg/adapter/gitlab/testdata/projects/dev-.json vendored
View File

@ -1,96 +0,0 @@
[
{
"id": 3,
"description": "",
"name": "dev-docker",
"name_with_namespace": "Library / dev-docker",
"path": "dev-docker",
"path_with_namespace": "library/dev-docker",
"created_at": "2019-01-17T09:47:07.504Z",
"default_branch": "master",
"tag_list": [],
"avatar_url": null,
"star_count": 0,
"forks_count": 0,
"last_activity_at": "2019-06-09T15:18:10.045Z",
"empty_repo": false,
"archived": false,
"visibility": "private",
"resolve_outdated_diff_discussions": false,
"container_registry_enabled": true,
"issues_enabled": true,
"merge_requests_enabled": true,
"wiki_enabled": true,
"jobs_enabled": true,
"snippets_enabled": true,
"shared_runners_enabled": true,
"lfs_enabled": true,
"creator_id": 1,
"forked_from_project": {},
"import_status": "finished",
"open_issues_count": 0,
"ci_default_git_depth": null,
"public_jobs": true,
"ci_config_path": null,
"shared_with_groups": [],
"only_allow_merge_if_pipeline_succeeds": false,
"request_access_enabled": false,
"only_allow_merge_if_all_discussions_are_resolved": false,
"printing_merge_request_link_enabled": true,
"merge_method": "merge",
"external_authorization_classification_label": "",
"permissions": {
"project_access": null,
"group_access": null
},
"mirror": false
},
{
"id": 4,
"description": "",
"name": "dev-docker-test",
"name_with_namespace": "Library / dev-docker-test",
"path": "dev-docker-test",
"path_with_namespace": "library/dev-docker-test",
"created_at": "2019-01-17T09:47:07.504Z",
"default_branch": "master",
"tag_list": [],
"avatar_url": null,
"star_count": 0,
"forks_count": 0,
"last_activity_at": "2019-06-09T15:18:10.045Z",
"empty_repo": false,
"archived": false,
"visibility": "private",
"resolve_outdated_diff_discussions": false,
"container_registry_enabled": true,
"issues_enabled": true,
"merge_requests_enabled": true,
"wiki_enabled": true,
"jobs_enabled": true,
"snippets_enabled": true,
"shared_runners_enabled": true,
"lfs_enabled": true,
"creator_id": 1,
"forked_from_project": {},
"import_status": "finished",
"open_issues_count": 0,
"ci_default_git_depth": null,
"public_jobs": true,
"ci_config_path": null,
"shared_with_groups": [],
"only_allow_merge_if_pipeline_succeeds": false,
"request_access_enabled": false,
"only_allow_merge_if_all_discussions_are_resolved": false,
"printing_merge_request_link_enabled": true,
"merge_method": "merge",
"external_authorization_classification_label": "",
"permissions": {
"project_access": null,
"group_access": null
},
"mirror": false
}
]