upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor synced 2024-09-21 21:19:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/goharbor/harbor into project-quota-dev

Browse Source
This commit is contained in:
wang yan 2019-07-10 10:57:10 +08:00
commit 6d0271ee5c
33 changed files with 922 additions and 1373 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/common/dao/group/usergroup.go
View File

@ -15,10 +15,8 @@
package group
import (
"strings"
"time"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/dao"
@ -139,20 +137,3 @@ func OnBoardUserGroup(g *models.UserGroup, keyAttribute string, combinedKeyAttri
return nil
}
// GetGroupDNQueryCondition get the part of IN ('XXX', 'XXX') condition
func GetGroupDNQueryCondition(userGroupList []*models.UserGroup) string {
result := make([]string, 0)
count := 0
for _, userGroup := range userGroupList {
if userGroup.GroupType == common.LdapGroupType {
result = append(result, "'"+userGroup.LdapGroupDN+"'")
count++
}
}
// No LDAP Group found
if count == 0 {
return ""
}
return strings.Join(result, ",")
}

111
src/common/dao/group/usergroup_test.go
View File

@ -47,6 +47,8 @@ func TestMain(m *testing.M) {
initSqls := []string{
"insert into harbor_user (username, email, password, realname) values ('member_test_01', 'member_test_01@example.com', '123456', 'member_test_01')",
"insert into project (name, owner_id) values ('member_test_01', 1)",
`insert into project (name, owner_id) values ('group_project2', 1)`,
`insert into project (name, owner_id) values ('group_project_private', 1)`,
"insert into user_group (group_name, group_type, ldap_group_dn) values ('test_group_01', 1, 'cn=harbor_users,ou=sample,ou=vmware,dc=harbor,dc=com')",
"update project set owner_id = (select user_id from harbor_user where username = 'member_test_01') where name = 'member_test_01'",
"insert into project_member (project_id, entity_id, entity_type, role) values ( (select project_id from project where name = 'member_test_01') , (select user_id from harbor_user where username = 'member_test_01'), 'u', 1)",
@ -55,6 +57,8 @@ func TestMain(m *testing.M) {
clearSqls := []string{
"delete from project where name='member_test_01'",
"delete from project where name='group_project2'",
"delete from project where name='group_project_private'",
"delete from harbor_user where username='member_test_01' or username='pm_sample'",
"delete from user_group",
"delete from project_member",
@ -175,7 +179,7 @@ func TestUpdateUserGroup(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
fmt.Printf("id=%v", createdUserGroupID)
fmt.Printf("id=%v\n", createdUserGroupID)
if err := UpdateUserGroupName(tt.args.id, tt.args.groupName); (err != nil) != tt.wantErr {
t.Errorf("UpdateUserGroup() error = %v, wantErr %v", err, tt.wantErr)
userGroup, err := GetUserGroup(tt.args.id)
@ -249,40 +253,6 @@ func TestOnBoardUserGroup(t *testing.T) {
}
}
func TestGetGroupDNQueryCondition(t *testing.T) {
userGroupList := []*models.UserGroup{
{
GroupName: "sample1",
GroupType: 1,
LdapGroupDN: "cn=sample1_users,ou=groups,dc=example,dc=com",
},
{
GroupName: "sample2",
GroupType: 1,
LdapGroupDN: "cn=sample2_users,ou=groups,dc=example,dc=com",
},
{
GroupName: "sample3",
GroupType: 0,
LdapGroupDN: "cn=sample3_users,ou=groups,dc=example,dc=com",
},
}
groupQueryConditions := GetGroupDNQueryCondition(userGroupList)
expectedConditions := `'cn=sample1_users,ou=groups,dc=example,dc=com','cn=sample2_users,ou=groups,dc=example,dc=com'`
if groupQueryConditions != expectedConditions {
t.Errorf("Failed to GetGroupDNQueryCondition, expected %v, actual %v", expectedConditions, groupQueryConditions)
}
var userGroupList2 []*models.UserGroup
groupQueryCondition2 := GetGroupDNQueryCondition(userGroupList2)
if len(groupQueryCondition2) > 0 {
t.Errorf("Failed to GetGroupDNQueryCondition, expected %v, actual %v", "", groupQueryCondition2)
}
groupQueryCondition3 := GetGroupDNQueryCondition(nil)
if len(groupQueryCondition3) > 0 {
t.Errorf("Failed to GetGroupDNQueryCondition, expected %v, actual %v", "", groupQueryCondition3)
}
}
func TestGetGroupProjects(t *testing.T) {
userID, err := dao.Register(models.User{
Username: "grouptestu09",
@ -322,8 +292,7 @@ func TestGetGroupProjects(t *testing.T) {
})
defer project.DeleteProjectMemberByID(pmid)
type args struct {
groupDNCondition string
query *models.ProjectQueryParam
query *models.ProjectQueryParam
}
member := &models.MemberQuery{
Name: "grouptestu09",
@ -335,19 +304,17 @@ func TestGetGroupProjects(t *testing.T) {
wantErr bool
}{
{"Query with group DN",
args{"'cn=harbor_users,ou=groups,dc=example,dc=com'",
&models.ProjectQueryParam{
Member: member,
}},
args{&models.ProjectQueryParam{
Member: member,
}},
1, false},
{"Query without group DN",
args{"",
&models.ProjectQueryParam{}},
args{&models.ProjectQueryParam{}},
1, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := dao.GetGroupProjects(tt.args.groupDNCondition, tt.args.query)
got, err := dao.GetGroupProjects([]int{groupID}, tt.args.query)
if (err != nil) != tt.wantErr {
t.Errorf("GetGroupProjects() error = %v, wantErr %v", err, tt.wantErr)
return
@ -392,8 +359,7 @@ func TestGetTotalGroupProjects(t *testing.T) {
})
defer project.DeleteProjectMemberByID(pmid)
type args struct {
groupDNCondition string
query *models.ProjectQueryParam
query *models.ProjectQueryParam
}
tests := []struct {
name string
@ -401,18 +367,16 @@ func TestGetTotalGroupProjects(t *testing.T) {
wantSize int
wantErr bool
}{
{"Query with group DN",
args{"'cn=harbor_users,ou=groups,dc=example,dc=com'",
&models.ProjectQueryParam{}},
{"Query with group ID",
args{&models.ProjectQueryParam{}},
1, false},
{"Query without group DN",
args{"",
&models.ProjectQueryParam{}},
{"Query without group ID",
args{&models.ProjectQueryParam{}},
1, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := dao.GetTotalGroupProjects(tt.args.groupDNCondition, tt.args.query)
got, err := dao.GetTotalGroupProjects([]int{groupID}, tt.args.query)
if (err != nil) != tt.wantErr {
t.Errorf("GetGroupProjects() error = %v, wantErr %v", err, tt.wantErr)
return
@ -423,3 +387,44 @@ func TestGetTotalGroupProjects(t *testing.T) {
})
}
}
func TestGetRolesByLDAPGroup(t *testing.T) {
userGroupList, err := QueryUserGroup(models.UserGroup{LdapGroupDN: "cn=harbor_users,ou=sample,ou=vmware,dc=harbor,dc=com", GroupType: 1})
if err != nil || len(userGroupList) < 1 {
t.Errorf("failed to query user group, err %v", err)
}
project, err := dao.GetProjectByName("member_test_01")
if err != nil {
t.Errorf("Error occurred when Get project by name: %v", err)
}
privateProject, err := dao.GetProjectByName("group_project_private")
if err != nil {
t.Errorf("Error occurred when Get project by name: %v", err)
}
type args struct {
projectID int64
groupIDs []int
}
tests := []struct {
name string
args args
wantSize int
wantErr bool
}{
{"Check normal", args{projectID: project.ProjectID, groupIDs: []int{userGroupList[0].ID}}, 1, false},
{"Check non exist", args{projectID: privateProject.ProjectID, groupIDs: []int{9999}}, 0, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := dao.GetRolesByGroupID(tt.args.projectID, tt.args.groupIDs)
if (err != nil) != tt.wantErr {
t.Errorf("TestGetRolesByLDAPGroup() error = %v, wantErr %v", err, tt.wantErr)
return
}
if len(got) != tt.wantSize {
t.Errorf("TestGetRolesByLDAPGroup() = %v, want %v", len(got), tt.wantSize)
}
})
}
}

38
src/common/dao/project.go
View File

@ -156,18 +156,19 @@ func GetProjects(query *models.ProjectQueryParam) ([]*models.Project, error) {
// GetGroupProjects - Get user's all projects, including user is the user member of this project
// and the user is in the group which is a group member of this project.
func GetGroupProjects(groupDNCondition string, query *models.ProjectQueryParam) ([]*models.Project, error) {
func GetGroupProjects(groupIDs []int, query *models.ProjectQueryParam) ([]*models.Project, error) {
sql, params := projectQueryConditions(query)
sql = `select distinct p.project_id, p.name, p.owner_id,
p.creation_time, p.update_time ` + sql
if len(groupDNCondition) > 0 {
groupIDCondition := JoinNumberConditions(groupIDs)
if len(groupIDs) > 0 {
sql = fmt.Sprintf(
`%s union select distinct p.project_id, p.name, p.owner_id, p.creation_time, p.update_time
from project p
left join project_member pm on p.project_id = pm.project_id
left join user_group ug on ug.id = pm.entity_id and pm.entity_type = 'g' and ug.group_type = 1
where ug.ldap_group_dn in ( %s ) order by name`,
sql, groupDNCondition)
left join user_group ug on ug.id = pm.entity_id and pm.entity_type = 'g'
where ug.id in ( %s ) order by name`,
sql, groupIDCondition)
}
sqlStr, queryParams := CreatePagination(query, sql, params)
log.Debugf("query sql:%v", sql)
@ -178,10 +179,11 @@ func GetGroupProjects(groupDNCondition string, query *models.ProjectQueryParam)
// GetTotalGroupProjects - Get the total count of projects, including user is the member of this project and the
// user is in the group, which is the group member of this project.
func GetTotalGroupProjects(groupDNCondition string, query *models.ProjectQueryParam) (int, error) {
func GetTotalGroupProjects(groupIDs []int, query *models.ProjectQueryParam) (int, error) {
var sql string
sqlCondition, params := projectQueryConditions(query)
if len(groupDNCondition) == 0 {
groupIDCondition := JoinNumberConditions(groupIDs)
if len(groupIDs) == 0 {
sql = `select count(1) ` + sqlCondition
} else {
sql = fmt.Sprintf(
@ -189,9 +191,9 @@ func GetTotalGroupProjects(groupDNCondition string, query *models.ProjectQueryPa
from ( select p.project_id %s union select p.project_id
from project p
left join project_member pm on p.project_id = pm.project_id
left join user_group ug on ug.id = pm.entity_id and pm.entity_type = 'g' and ug.group_type = 1
where ug.ldap_group_dn in ( %s )) t`,
sqlCondition, groupDNCondition)
left join user_group ug on ug.id = pm.entity_id and pm.entity_type = 'g'
where ug.id in ( %s )) t`,
sqlCondition, groupIDCondition)
}
log.Debugf("query sql:%v", sql)
var count int
@ -291,24 +293,24 @@ func DeleteProject(id int64) error {
return err
}
// GetRolesByLDAPGroup - Get Project roles of the
// specified group DN is a member of current project
func GetRolesByLDAPGroup(projectID int64, groupDNCondition string) ([]int, error) {
// GetRolesByGroupID - Get Project roles of the
// specified group is a member of current project
func GetRolesByGroupID(projectID int64, groupIDs []int) ([]int, error) {
var roles []int
if len(groupDNCondition) == 0 {
if len(groupIDs) == 0 {
return roles, nil
}
groupIDCondition := JoinNumberConditions(groupIDs)
o := GetOrmer()
// Because an LDAP user can be memberof multiple groups,
// the role is in descent order (1-admin, 2-developer, 3-guest, 4-master), use min to select the max privilege role.
sql := fmt.Sprintf(
`select min(pm.role) from project_member pm
left join user_group ug on pm.entity_type = 'g' and pm.entity_id = ug.id
where ug.ldap_group_dn in ( %s ) and pm.project_id = ? `,
groupDNCondition)
where ug.id in ( %s ) and pm.project_id = ?`,
groupIDCondition)
log.Debugf("sql:%v", sql)
if _, err := o.Raw(sql, projectID).QueryRows(&roles); err != nil {
log.Warningf("Error in GetRolesByLDAPGroup, error: %v", err)
log.Warningf("Error in GetRolesByGroupID, error: %v", err)
return nil, err
}
// If there is no row selected, the min returns an empty row, to avoid return 0 as role

13
src/common/dao/project/projectmember.go
View File

@ -148,16 +148,3 @@ func SearchMemberByName(projectID int64, entityName string) ([]*models.Member, e
_, err := o.Raw(sql, queryParam).QueryRows(&members)
return members, err
}
// GetRolesByGroup -- Query group roles
func GetRolesByGroup(projectID int64, groupDNCondition string) []int {
var roles []int
o := dao.GetOrmer()
sql := `select role from project_member pm
left join user_group ug on pm.project_id = ?
where ug.group_type = 1 and ug.ldap_group_dn in (` + groupDNCondition + `)`
if _, err := o.Raw(sql, projectID).QueryRows(&roles); err != nil {
return roles
}
return roles
}

27
src/common/dao/project/projectmember_test.go
View File

@ -305,30 +305,3 @@ func PrepareGroupTest() {
}
dao.PrepareTestData(clearSqls, initSqls)
}
func TestGetRolesByGroup(t *testing.T) {
PrepareGroupTest()
project, err := dao.GetProjectByName("group_project")
if err != nil {
t.Errorf("Error occurred when GetProjectByName : %v", err)
}
type args struct {
projectID int64
groupDNCondition string
}
tests := []struct {
name string
args args
want []int
}{
{"Query group with role", args{project.ProjectID, "'cn=harbor_user,dc=example,dc=com'"}, []int{2}},
{"Query group no role", args{project.ProjectID, "'cn=another_user,dc=example,dc=com'"}, []int{}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := GetRolesByGroup(tt.args.projectID, tt.args.groupDNCondition); !dao.ArrayEqual(got, tt.want) {
t.Errorf("GetRolesByGroup() = %v, want %v", got, tt.want)
}
})
}
}

97
src/common/dao/project_test.go
View File

@ -118,36 +118,6 @@ func Test_projectQueryConditions(t *testing.T) {
}
}
func TestGetGroupProjects(t *testing.T) {
prepareGroupTest()
query := &models.ProjectQueryParam{Member: &models.MemberQuery{Name: "sample_group"}}
type args struct {
groupDNCondition string
query *models.ProjectQueryParam
}
tests := []struct {
name string
args args
wantSize int
wantErr bool
}{
{"Verify correct sql", args{groupDNCondition: "'cn=harbor_user,dc=example,dc=com'", query: query}, 1, false},
{"Verify missed sql", args{groupDNCondition: "'cn=another_user,dc=example,dc=com'", query: query}, 0, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := GetGroupProjects(tt.args.groupDNCondition, tt.args.query)
if (err != nil) != tt.wantErr {
t.Errorf("GetGroupProjects() error = %v, wantErr %v", err, tt.wantErr)
return
}
if len(got) != tt.wantSize {
t.Errorf("GetGroupProjects() = %v, want %v", got, tt.wantSize)
}
})
}
}
func prepareGroupTest() {
initSqls := []string{
`insert into user_group (group_name, group_type, ldap_group_dn) values ('harbor_group_01', 1, 'cn=harbor_user,dc=example,dc=com')`,
@ -169,73 +139,6 @@ func prepareGroupTest() {
PrepareTestData(clearSqls, initSqls)
}
func TestGetTotalGroupProjects(t *testing.T) {
prepareGroupTest()
query := &models.ProjectQueryParam{Member: &models.MemberQuery{Name: "sample_group"}}
type args struct {
groupDNCondition string
query *models.ProjectQueryParam
}
tests := []struct {
name string
args args
want int
wantErr bool
}{
{"Verify correct sql", args{groupDNCondition: "'cn=harbor_user,dc=example,dc=com'", query: query}, 1, false},
{"Verify missed sql", args{groupDNCondition: "'cn=another_user,dc=example,dc=com'", query: query}, 0, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := GetTotalGroupProjects(tt.args.groupDNCondition, tt.args.query)
if (err != nil) != tt.wantErr {
t.Errorf("GetTotalGroupProjects() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("GetTotalGroupProjects() = %v, want %v", got, tt.want)
}
})
}
}
func TestGetRolesByLDAPGroup(t *testing.T) {
prepareGroupTest()
project, err := GetProjectByName("group_project")
if err != nil {
t.Errorf("Error occurred when Get project by name: %v", err)
}
privateProject, err := GetProjectByName("group_project_private")
if err != nil {
t.Errorf("Error occurred when Get project by name: %v", err)
}
type args struct {
projectID int64
groupDNCondition string
}
tests := []struct {
name string
args args
wantSize int
wantErr bool
}{
{"Check normal", args{project.ProjectID, "'cn=harbor_user,dc=example,dc=com'"}, 1, false},
{"Check non exist", args{privateProject.ProjectID, "'cn=not_harbor_user,dc=example,dc=com'"}, 0, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := GetRolesByLDAPGroup(tt.args.projectID, tt.args.groupDNCondition)
if (err != nil) != tt.wantErr {
t.Errorf("TestGetRolesByLDAPGroup() error = %v, wantErr %v", err, tt.wantErr)
return
}
if len(got) != tt.wantSize {
t.Errorf("TestGetRolesByLDAPGroup() = %v, want %v", len(got), tt.wantSize)
}
})
}
}
func TestProjetExistsByName(t *testing.T) {
name := "project_exist_by_name_test"
exist := ProjectExistsByName(name)

11
src/common/dao/utils.go Normal file
View File

@ -0,0 +1,11 @@
package dao
import (
"fmt"
"strings"
)
// JoinNumberConditions - To join number condition into string,used in sql query
func JoinNumberConditions(ids []int) string {
return strings.Trim(strings.Replace(fmt.Sprint(ids), " ", ",", -1), "[]")
}

24
src/common/dao/utils_test.go Normal file
View File

@ -0,0 +1,24 @@
package dao
import "testing"
func TestJoinNumberConditions(t *testing.T) {
type args struct {
ids []int
}
tests := []struct {
name string
args args
want string
}{
{name: "normal test", args: args{[]int{1, 2, 3}}, want: "1,2,3"},
{name: "dummy test", args: args{[]int{}}, want: ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := JoinNumberConditions(tt.args.ids); got != tt.want {
t.Errorf("JoinNumberConditions() = %v, want %v", got, tt.want)
}
})
}
}

6
src/common/models/project.go
View File

@ -128,9 +128,9 @@ type ProjectQueryParam struct {
// MemberQuery filter by member's username and role
type MemberQuery struct {
Name string // the username of member
Role int // the role of the member has to the project
GroupList []*UserGroup // the group list of current user
Name string // the username of member
Role int // the role of the member has to the project
GroupIDs []int // the group ID of current user belongs to
}
// Pagination ...

14
src/common/models/user.go
View File

@ -35,13 +35,13 @@ type User struct {
// to it.
Role int `orm:"-" json:"role_id"`
// RoleList []Role `json:"role_list"`
HasAdminRole bool `orm:"column(sysadmin_flag)" json:"has_admin_role"`
ResetUUID string `orm:"column(reset_uuid)" json:"reset_uuid"`
Salt string `orm:"column(salt)" json:"-"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
GroupList []*UserGroup `orm:"-" json:"-"`
OIDCUserMeta *OIDCUser `orm:"-" json:"oidc_user_meta,omitempty"`
HasAdminRole bool `orm:"column(sysadmin_flag)" json:"has_admin_role"`
ResetUUID string `orm:"column(reset_uuid)" json:"reset_uuid"`
Salt string `orm:"column(salt)" json:"-"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
GroupIDs []int `orm:"-" json:"-"`
OIDCUserMeta *OIDCUser `orm:"-" json:"oidc_user_meta,omitempty"`
}
// UserQuery ...

12
src/common/security/local/context.go
View File

@ -17,7 +17,6 @@ package local
import (
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/dao/group"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/rbac/project"
@ -140,12 +139,11 @@ func (s *SecurityContext) GetRolesByGroup(projectIDOrName interface{}) []int {
user := s.user
project, err := s.pm.Get(projectIDOrName)
// No user, group or project info
if err != nil || project == nil || user == nil || len(user.GroupList) == 0 {
if err != nil || project == nil || user == nil || len(user.GroupIDs) == 0 {
return roles
}
// Get role by LDAP group
groupDNConditions := group.GetGroupDNQueryCondition(user.GroupList)
roles, err = dao.GetRolesByLDAPGroup(project.ProjectID, groupDNConditions)
// Get role by Group ID
roles, err = dao.GetRolesByGroupID(project.ProjectID, user.GroupIDs)
if err != nil {
return nil
}
@ -157,8 +155,8 @@ func (s *SecurityContext) GetMyProjects() ([]*models.Project, error) {
result, err := s.pm.List(
&models.ProjectQueryParam{
Member: &models.MemberQuery{
Name: s.GetUsername(),
GroupList: s.user.GroupList,
Name: s.GetUsername(),
GroupIDs: s.user.GroupIDs,
},
})
if err != nil {

22
src/common/security/local/context_test.go
View File

@ -20,6 +20,7 @@ import (
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/dao/group"
"github.com/goharbor/harbor/src/common/dao/project"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/rbac"
@ -253,9 +254,16 @@ func TestHasPushPullPermWithGroup(t *testing.T) {
if err != nil {
t.Errorf("Error occurred when GetUser: %v", err)
}
developer.GroupList = []*models.UserGroup{
{GroupName: "test_group", GroupType: 1, LdapGroupDN: "cn=harbor_user,dc=example,dc=com"},
userGroups, err := group.QueryUserGroup(models.UserGroup{GroupType: common.LdapGroupType, LdapGroupDN: "cn=harbor_user,dc=example,dc=com"})
if err != nil {
t.Errorf("Failed to query user group %v", err)
}
if len(userGroups) < 1 {
t.Errorf("Failed to retrieve user group")
}
developer.GroupIDs = []int{userGroups[0].ID}
resource := rbac.NewProjectNamespace(project.Name).Resource(rbac.ResourceRepository)
@ -332,9 +340,15 @@ func TestSecurityContext_GetRolesByGroup(t *testing.T) {
if err != nil {
t.Errorf("Error occurred when GetUser: %v", err)
}
developer.GroupList = []*models.UserGroup{
{GroupName: "test_group", GroupType: 1, LdapGroupDN: "cn=harbor_user,dc=example,dc=com"},
userGroups, err := group.QueryUserGroup(models.UserGroup{GroupType: common.LdapGroupType, LdapGroupDN: "cn=harbor_user,dc=example,dc=com"})
if err != nil {
t.Errorf("Failed to query user group %v", err)
}
if len(userGroups) < 1 {
t.Errorf("Failed to retrieve user group")
}
developer.GroupIDs = []int{userGroups[0].ID}
type fields struct {
user *models.User
pm promgr.ProjectManager

15
src/core/auth/ldap/ldap.go
View File

@ -20,11 +20,11 @@ import (
"strings"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/dao/group"
"github.com/goharbor/harbor/src/common/utils"
goldap "gopkg.in/ldap.v2"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/dao/group"
"github.com/goharbor/harbor/src/common/models"
ldapUtils "github.com/goharbor/harbor/src/common/utils/ldap"
"github.com/goharbor/harbor/src/common/utils/log"
@ -79,7 +79,7 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) {
u.Username = ldapUsers[0].Username
u.Email = strings.TrimSpace(ldapUsers[0].Email)
u.Realname = ldapUsers[0].Realname
userGroups := make([]*models.UserGroup, 0)
ugIDs := []int{}
dn := ldapUsers[0].DN
if err = ldapSession.Bind(dn, m.Password); err != nil {
@ -95,6 +95,7 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) {
for _, groupDN := range ldapUsers[0].GroupDNList {
groupDN = utils.TrimLower(groupDN)
// Attach LDAP group admin
if len(groupAdminDN) > 0 && groupAdminDN == groupDN {
u.HasAdminRole = true
}
@ -103,16 +104,16 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) {
GroupType: 1,
LdapGroupDN: groupDN,
}
userGroupList, err := group.QueryUserGroup(userGroupQuery)
userGroups, err := group.QueryUserGroup(userGroupQuery)
if err != nil {
continue
}
if len(userGroupList) == 0 {
if len(userGroups) == 0 {
continue
}
userGroups = append(userGroups, userGroupList[0])
ugIDs = append(ugIDs, userGroups[0].ID)
}
u.GroupList = userGroups
u.GroupIDs = ugIDs
return &u, nil
}

12
src/core/promgr/pmsdriver/local/local.go
View File

@ -20,7 +20,6 @@ import (
"time"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/dao/group"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils"
errutil "github.com/goharbor/harbor/src/common/utils/error"
@ -132,19 +131,16 @@ func (d *driver) Update(projectIDOrName interface{},
func (d *driver) List(query *models.ProjectQueryParam) (*models.ProjectQueryResult, error) {
var total int64
var projects []*models.Project
var groupDNCondition string
// List with LDAP group projects
var groupIDs []int
if query != nil && query.Member != nil {
groupDNCondition = group.GetGroupDNQueryCondition(query.Member.GroupList)
groupIDs = query.Member.GroupIDs
}
count, err := dao.GetTotalGroupProjects(groupDNCondition, query)
count, err := dao.GetTotalGroupProjects(groupIDs, query)
if err != nil {
return nil, err
}
total = int64(count)
projects, err = dao.GetGroupProjects(groupDNCondition, query)
projects, err = dao.GetGroupProjects(groupIDs, query)
if err != nil {
return nil, err
}

83
src/replication/adapter/adapter.go
View File

@ -17,10 +17,18 @@ package adapter
import (
"errors"
"fmt"
"io"
"github.com/docker/distribution"
"github.com/goharbor/harbor/src/replication/filter"
"github.com/goharbor/harbor/src/replication/model"
)
// const definition
const (
UserAgentReplication = "harbor-replication-service"
)
var registry = map[model.RegistryType]Factory{}
// Factory creates a specific Adapter according to the params
@ -37,6 +45,81 @@ type Adapter interface {
HealthCheck() (model.HealthStatus, error)
}
// ImageRegistry defines the capabilities that an image registry should have
type ImageRegistry interface {
FetchImages(filters []*model.Filter) ([]*model.Resource, error)
ManifestExist(repository, reference string) (exist bool, digest string, err error)
PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error)
PushManifest(repository, reference, mediaType string, payload []byte) error
// the "reference" can be "tag" or "digest", the function needs to handle both
DeleteManifest(repository, reference string) error
BlobExist(repository, digest string) (exist bool, err error)
PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error)
PushBlob(repository, digest string, size int64, blob io.Reader) error
}
// ChartRegistry defines the capabilities that a chart registry should have
type ChartRegistry interface {
FetchCharts(filters []*model.Filter) ([]*model.Resource, error)
ChartExist(name, version string) (bool, error)
DownloadChart(name, version string) (io.ReadCloser, error)
UploadChart(name, version string, chart io.Reader) error
DeleteChart(name, version string) error
}
// Repository defines an repository object, it can be image repository, chart repository and etc.
type Repository struct {
ResourceType string `json:"resource_type"`
Name string `json:"name"`
}
// GetName returns the name
func (r *Repository) GetName() string {
return r.Name
}
// GetFilterableType returns the filterable type
func (r *Repository) GetFilterableType() filter.FilterableType {
return filter.FilterableTypeRepository
}
// GetResourceType returns the resource type
func (r *Repository) GetResourceType() string {
return r.ResourceType
}
// GetLabels returns the labels
func (r *Repository) GetLabels() []string {
return nil
}
// VTag defines an vTag object, it can be image tag, chart version and etc.
type VTag struct {
ResourceType string `json:"resource_type"`
Name string `json:"name"`
Labels []string `json:"labels"`
}
// GetFilterableType returns the filterable type
func (v *VTag) GetFilterableType() filter.FilterableType {
return filter.FilterableTypeVTag
}
// GetResourceType returns the resource type
func (v *VTag) GetResourceType() string {
return v.ResourceType
}
// GetName returns the name
func (v *VTag) GetName() string {
return v.Name
}
// GetLabels returns the labels
func (v *VTag) GetLabels() []string {
return v.Labels
}
// RegisterFactory registers one adapter factory to the registry
func RegisterFactory(t model.RegistryType, factory Factory) error {
if len(t) == 0 {

16
src/replication/adapter/awsecr/adapter.go
View File

@ -16,6 +16,9 @@ package awsecr
import (
"errors"
"net/http"
"regexp"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
@ -24,9 +27,8 @@ import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
"net/http"
"regexp"
)
func init() {
@ -45,14 +47,14 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
return nil, err
}
authorizer := NewAuth(region, registry.Credential.AccessKey, registry.Credential.AccessSecret, registry.Insecure)
reg, err := adp.NewDefaultImageRegistryWithCustomizedAuthorizer(registry, authorizer)
dockerRegistry, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer)
if err != nil {
return nil, err
}
return &adapter{
registry: registry,
DefaultImageRegistry: reg,
region: region,
registry: registry,
Adapter: dockerRegistry,
region: region,
}, nil
}
@ -66,7 +68,7 @@ func parseRegion(url string) (string, error) {
}
type adapter struct {
*adp.DefaultImageRegistry
*native.Adapter
registry *model.Registry
region string
forceEndpoint *string

15
src/replication/adapter/awsecr/adapter_test.go
View File

@ -2,8 +2,6 @@ package awsecr
import (
"fmt"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/stretchr/testify/assert"
"io"
"io/ioutil"
"net/http"
@ -11,8 +9,11 @@ import (
"testing"
"time"
"github.com/goharbor/harbor/src/common/utils/test"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
"github.com/stretchr/testify/assert"
)
func TestAdapter_NewAdapter(t *testing.T) {
@ -130,15 +131,15 @@ func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Ser
AccessSecret: "ppp",
}
}
reg, err := adp.NewDefaultImageRegistry(registry)
dockerRegistryAdapter, err := native.NewAdapter(registry)
if err != nil {
panic(err)
}
return &adapter{
registry: registry,
DefaultImageRegistry: reg,
region: "test-region",
forceEndpoint: &server.URL,
registry: registry,
Adapter: dockerRegistryAdapter,
region: "test-region",
forceEndpoint: &server.URL,
}, server
}

113
src/replication/adapter/awsecr/image_registry.go
View File

@ -1,113 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package awsecr
import (
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
)
var _ adp.ImageRegistry = adapter{}
func (a adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
nameFilterPattern := ""
tagFilterPattern := ""
for _, filter := range filters {
switch filter.Type {
case model.FilterTypeName:
nameFilterPattern = filter.Value.(string)
case model.FilterTypeTag:
tagFilterPattern = filter.Value.(string)
}
}
repositories, err := a.filterRepositories(nameFilterPattern)
if err != nil {
return nil, err
}
var resources []*model.Resource
for _, repository := range repositories {
tags, err := a.filterTags(repository, tagFilterPattern)
if err != nil {
return nil, err
}
if len(tags) == 0 {
continue
}
resources = append(resources, &model.Resource{
Type: model.ResourceTypeImage,
Registry: a.registry,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: repository,
},
Vtags: tags,
},
})
}
return resources, nil
}
func (a adapter) filterRepositories(pattern string) ([]string, error) {
// if the pattern is a specific repository name, just returns the parsed repositories
// and will check the existence later when filtering the tags
if repositories, ok := util.IsSpecificPath(pattern); ok {
return repositories, nil
}
// search repositories from catalog api
repositories, err := a.Catalog()
if err != nil {
return nil, err
}
// if the pattern is null, just return the result of catalog API
if len(pattern) == 0 {
return repositories, nil
}
result := []string{}
for _, repository := range repositories {
match, err := util.Match(pattern, repository)
if err != nil {
return nil, err
}
if match {
result = append(result, repository)
}
}
return result, nil
}
func (a adapter) filterTags(repository, pattern string) ([]string, error) {
tags, err := a.ListTag(repository)
if err != nil {
return nil, err
}
if len(pattern) == 0 {
return tags, nil
}
var result []string
for _, tag := range tags {
match, err := util.Match(pattern, tag)
if err != nil {
return nil, err
}
if match {
result = append(result, tag)
}
}
return result, nil
}

19
src/replication/adapter/azurecr/adapter.go
View File

@ -24,24 +24,26 @@ func init() {
}
func factory(registry *model.Registry) (adp.Adapter, error) {
client, err := getClient(registry)
if err != nil {
return nil, err
if registry.Credential == nil || len(registry.Credential.AccessKey) == 0 ||
len(registry.Credential.AccessSecret) == 0 {
return nil, fmt.Errorf("credential is necessary for registry %s", registry.URL)
}
reg, err := native.NewWithClient(registry, client)
authorizer := auth.NewBasicAuthCredential(registry.Credential.AccessKey,
registry.Credential.AccessSecret)
dockerRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer)
if err != nil {
return nil, err
}
return &adapter{
registry: registry,
Native: reg,
Adapter: dockerRegistryAdapter,
}, nil
}
type adapter struct {
*native.Native
*native.Adapter
registry *model.Registry
}
@ -72,11 +74,6 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
}, nil
}
// PrepareForPush no preparation needed for Azure container registry
func (a *adapter) PrepareForPush(resources []*model.Resource) error {
return nil
}
// HealthCheck checks health status of a registry
func (a *adapter) HealthCheck() (model.HealthStatus, error) {
err := a.PingGet()

30
src/replication/adapter/chart_registry.go
View File

@ -1,30 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package adapter
import (
"io"
"github.com/goharbor/harbor/src/replication/model"
)
// ChartRegistry defines the capabilities that a chart registry should have
type ChartRegistry interface {
FetchCharts(filters []*model.Filter) ([]*model.Resource, error)
ChartExist(name, version string) (bool, error)
DownloadChart(name, version string) (io.ReadCloser, error)
UploadChart(name, version string, chart io.Reader) error
DeleteChart(name, version string) error
}

11
src/replication/adapter/dockerhub/adapter.go
View File

@ -12,6 +12,7 @@ import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
)
@ -47,7 +48,7 @@ func factory(registry *model.Registry) (adp.Adapter, error) {
Transport: util.GetHTTPTransport(registry.Insecure),
}, credential)
reg, err := adp.NewDefaultImageRegistryWithCustomizedAuthorizer(&model.Registry{
dockerRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(&model.Registry{
Name: registry.Name,
URL: registryURL, // specify the URL of Docker Hub registry service
Credential: registry.Credential,
@ -58,14 +59,14 @@ func factory(registry *model.Registry) (adp.Adapter, error) {
}
return &adapter{
client: client,
registry: registry,
DefaultImageRegistry: reg,
client: client,
registry: registry,
Adapter: dockerRegistryAdapter,
}, nil
}
type adapter struct {
*adp.DefaultImageRegistry
*native.Adapter
registry *model.Registry
client *Client
}

12
src/replication/adapter/googlegcr/adapter.go
View File

@ -15,12 +15,14 @@
package googlegcr
import (
"net/http"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
"net/http"
)
func init() {
@ -44,19 +46,19 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
Transport: util.GetHTTPTransport(registry.Insecure),
}, credential)
reg, err := adp.NewDefaultImageRegistryWithCustomizedAuthorizer(registry, authorizer)
dockerRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer)
if err != nil {
return nil, err
}
return &adapter{
registry: registry,
DefaultImageRegistry: reg,
registry: registry,
Adapter: dockerRegistryAdapter,
}, nil
}
type adapter struct {
*adp.DefaultImageRegistry
*native.Adapter
registry *model.Registry
}

113
src/replication/adapter/googlegcr/image_registry.go
View File

@ -1,113 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package googlegcr
import (
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
)
var _ adp.ImageRegistry = adapter{}
func (a adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
nameFilterPattern := ""
tagFilterPattern := ""
for _, filter := range filters {
switch filter.Type {
case model.FilterTypeName:
nameFilterPattern = filter.Value.(string)
case model.FilterTypeTag:
tagFilterPattern = filter.Value.(string)
}
}
repositories, err := a.filterRepositories(nameFilterPattern)
if err != nil {
return nil, err
}
var resources []*model.Resource
for _, repository := range repositories {
tags, err := a.filterTags(repository, tagFilterPattern)
if err != nil {
return nil, err
}
if len(tags) == 0 {
continue
}
resources = append(resources, &model.Resource{
Type: model.ResourceTypeImage,
Registry: a.registry,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: repository,
},
Vtags: tags,
},
})
}
return resources, nil
}
func (a adapter) filterRepositories(pattern string) ([]string, error) {
// if the pattern is a specific repository name, just returns the parsed repositories
// and will check the existence later when filtering the tags
if repositories, ok := util.IsSpecificPath(pattern); ok {
return repositories, nil
}
// search repositories from catalog api
repositories, err := a.Catalog()
if err != nil {
return nil, err
}
// if the pattern is null, just return the result of catalog API
if len(pattern) == 0 {
return repositories, nil
}
result := []string{}
for _, repository := range repositories {
match, err := util.Match(pattern, repository)
if err != nil {
return nil, err
}
if match {
result = append(result, repository)
}
}
return result, nil
}
func (a adapter) filterTags(repository, pattern string) ([]string, error) {
tags, err := a.ListTag(repository)
if err != nil {
return nil, err
}
if len(pattern) == 0 {
return tags, nil
}
var result []string
for _, tag := range tags {
match, err := util.Match(pattern, tag)
if err != nil {
return nil, err
}
if match {
result = append(result, tag)
}
}
return result, nil
}

7
src/replication/adapter/harbor/adapter.go
View File

@ -27,6 +27,7 @@ import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
)
@ -42,7 +43,7 @@ func init() {
}
type adapter struct {
*adp.DefaultImageRegistry
*native.Adapter
registry *model.Registry
url string
client *common_http.Client
@ -67,7 +68,7 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
modifiers = append(modifiers, authorizer)
}
reg, err := adp.NewDefaultImageRegistry(registry)
dockerRegistryAdapter, err := native.NewAdapter(registry)
if err != nil {
return nil, err
}
@ -78,7 +79,7 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
&http.Client{
Transport: transport,
}, modifiers...),
DefaultImageRegistry: reg,
Adapter: dockerRegistryAdapter,
}, nil
}

9
src/replication/adapter/huawei/huawei_adapter.go
View File

@ -12,6 +12,7 @@ import (
"github.com/goharbor/harbor/src/common/utils/log"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
)
@ -27,7 +28,7 @@ func init() {
// Adapter is for images replications between harbor and Huawei image repository(SWR)
type adapter struct {
*adp.DefaultImageRegistry
*native.Adapter
registry *model.Registry
}
@ -232,13 +233,13 @@ func (a *adapter) HealthCheck() (model.HealthStatus, error) {
// AdapterFactory is the factory for huawei adapter
func AdapterFactory(registry *model.Registry) (adp.Adapter, error) {
reg, err := adp.NewDefaultImageRegistry(registry)
dockerRegistryAdapter, err := native.NewAdapter(registry)
if err != nil {
return nil, err
}
return &adapter{
registry: registry,
DefaultImageRegistry: reg,
registry: registry,
Adapter: dockerRegistryAdapter,
}, nil
}

325
src/replication/adapter/image_registry.go
View File

@ -1,325 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package adapter
import (
"errors"
"io"
"net/http"
"strings"
"sync"
"github.com/goharbor/harbor/src/replication/filter"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/goharbor/harbor/src/common/http/modifier"
common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth"
"github.com/goharbor/harbor/src/common/utils/log"
registry_pkg "github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
)
// const definition
const (
UserAgentReplication = "harbor-replication-service"
)
// ImageRegistry defines the capabilities that an image registry should have
type ImageRegistry interface {
FetchImages(filters []*model.Filter) ([]*model.Resource, error)
ManifestExist(repository, reference string) (exist bool, digest string, err error)
PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error)
PushManifest(repository, reference, mediaType string, payload []byte) error
// the "reference" can be "tag" or "digest", the function needs to handle both
DeleteManifest(repository, reference string) error
BlobExist(repository, digest string) (exist bool, err error)
PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error)
PushBlob(repository, digest string, size int64, blob io.Reader) error
}
// Repository defines an repository object, it can be image repository, chart repository and etc.
type Repository struct {
ResourceType string `json:"resource_type"`
Name string `json:"name"`
}
// GetName returns the name
func (r *Repository) GetName() string {
return r.Name
}
// GetFilterableType returns the filterable type
func (r *Repository) GetFilterableType() filter.FilterableType {
return filter.FilterableTypeRepository
}
// GetResourceType returns the resource type
func (r *Repository) GetResourceType() string {
return r.ResourceType
}
// GetLabels returns the labels
func (r *Repository) GetLabels() []string {
return nil
}
// VTag defines an vTag object, it can be image tag, chart version and etc.
type VTag struct {
ResourceType string `json:"resource_type"`
Name string `json:"name"`
Labels []string `json:"labels"`
}
// GetFilterableType returns the filterable type
func (v *VTag) GetFilterableType() filter.FilterableType {
return filter.FilterableTypeVTag
}
// GetResourceType returns the resource type
func (v *VTag) GetResourceType() string {
return v.ResourceType
}
// GetName returns the name
func (v *VTag) GetName() string {
return v.Name
}
// GetLabels returns the labels
func (v *VTag) GetLabels() []string {
return v.Labels
}
// DefaultImageRegistry provides a default implementation for interface ImageRegistry
type DefaultImageRegistry struct {
sync.RWMutex
*registry_pkg.Registry
registry *model.Registry
client *http.Client
clients map[string]*registry_pkg.Repository
}
// NewDefaultRegistryWithClient returns an instance of DefaultImageRegistry
func NewDefaultRegistryWithClient(registry *model.Registry, client *http.Client) (*DefaultImageRegistry, error) {
reg, err := registry_pkg.NewRegistry(registry.URL, client)
if err != nil {
return nil, err
}
return &DefaultImageRegistry{
Registry: reg,
client: client,
registry: registry,
clients: map[string]*registry_pkg.Repository{},
}, nil
}
// NewDefaultImageRegistry returns an instance of DefaultImageRegistry
func NewDefaultImageRegistry(registry *model.Registry) (*DefaultImageRegistry, error) {
var authorizer modifier.Modifier
if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 {
var cred modifier.Modifier
if registry.Credential.Type == model.CredentialTypeSecret {
cred = common_http_auth.NewSecretAuthorizer(registry.Credential.AccessSecret)
} else {
cred = auth.NewBasicAuthCredential(
registry.Credential.AccessKey,
registry.Credential.AccessSecret)
}
authorizer = auth.NewStandardTokenAuthorizer(&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
}, cred, registry.TokenServiceURL)
}
return NewDefaultImageRegistryWithCustomizedAuthorizer(registry, authorizer)
}
// NewDefaultImageRegistryWithCustomizedAuthorizer returns an instance of DefaultImageRegistry with the customized authorizer
func NewDefaultImageRegistryWithCustomizedAuthorizer(registry *model.Registry, authorizer modifier.Modifier) (*DefaultImageRegistry, error) {
transport := util.GetHTTPTransport(registry.Insecure)
modifiers := []modifier.Modifier{
&auth.UserAgentModifier{
UserAgent: UserAgentReplication,
},
}
if authorizer != nil {
modifiers = append(modifiers, authorizer)
}
client := &http.Client{
Transport: registry_pkg.NewTransport(transport, modifiers...),
}
reg, err := registry_pkg.NewRegistry(registry.URL, client)
if err != nil {
return nil, err
}
return &DefaultImageRegistry{
Registry: reg,
client: client,
registry: registry,
clients: map[string]*registry_pkg.Repository{},
}, nil
}
func (d *DefaultImageRegistry) getClient(repository string) (*registry_pkg.Repository, error) {
d.RLock()
client, exist := d.clients[repository]
d.RUnlock()
if exist {
return client, nil
}
return d.create(repository)
}
func (d *DefaultImageRegistry) create(repository string) (*registry_pkg.Repository, error) {
d.Lock()
defer d.Unlock()
// double check
client, exist := d.clients[repository]
if exist {
return client, nil
}
client, err := registry_pkg.NewRepository(repository, d.registry.URL, d.client)
if err != nil {
return nil, err
}
d.clients[repository] = client
return client, nil
}
// HealthCheck checks health status of a registry
func (d *DefaultImageRegistry) HealthCheck() (model.HealthStatus, error) {
var err error
if d.registry.Credential == nil ||
(len(d.registry.Credential.AccessKey) == 0 && len(d.registry.Credential.AccessSecret) == 0) {
err = d.PingSimple()
} else {
err = d.Ping()
}
if err != nil {
log.Errorf("failed to ping registry %s: %v", d.registry.URL, err)
return model.Unhealthy, nil
}
return model.Healthy, nil
}
// FetchImages ...
func (d *DefaultImageRegistry) FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) {
return nil, errors.New("not implemented")
}
// ManifestExist ...
func (d *DefaultImageRegistry) ManifestExist(repository, reference string) (bool, string, error) {
client, err := d.getClient(repository)
if err != nil {
return false, "", err
}
digest, exist, err := client.ManifestExist(reference)
return exist, digest, err
}
// PullManifest ...
func (d *DefaultImageRegistry) PullManifest(repository, reference string, accepttedMediaTypes []string) (distribution.Manifest, string, error) {
client, err := d.getClient(repository)
if err != nil {
return nil, "", err
}
digest, mediaType, payload, err := client.PullManifest(reference, accepttedMediaTypes)
if err != nil {
return nil, "", err
}
if strings.Contains(mediaType, "application/json") {
mediaType = schema1.MediaTypeManifest
}
manifest, _, err := registry_pkg.UnMarshal(mediaType, payload)
if err != nil {
return nil, "", err
}
return manifest, digest, nil
}
// PushManifest ...
func (d *DefaultImageRegistry) PushManifest(repository, reference, mediaType string, payload []byte) error {
client, err := d.getClient(repository)
if err != nil {
return err
}
_, err = client.PushManifest(reference, mediaType, payload)
return err
}
// DeleteManifest ...
func (d *DefaultImageRegistry) DeleteManifest(repository, reference string) error {
client, err := d.getClient(repository)
if err != nil {
return err
}
digest := reference
if !isDigest(digest) {
dgt, exist, err := client.ManifestExist(reference)
if err != nil {
return err
}
if !exist {
log.Debugf("the manifest of %s:%s doesn't exist", repository, reference)
return nil
}
digest = dgt
}
return client.DeleteManifest(digest)
}
// BlobExist ...
func (d *DefaultImageRegistry) BlobExist(repository, digest string) (bool, error) {
client, err := d.getClient(repository)
if err != nil {
return false, err
}
return client.BlobExist(digest)
}
// PullBlob ...
func (d *DefaultImageRegistry) PullBlob(repository, digest string) (int64, io.ReadCloser, error) {
client, err := d.getClient(repository)
if err != nil {
return 0, nil, err
}
return client.PullBlob(digest)
}
// PushBlob ...
func (d *DefaultImageRegistry) PushBlob(repository, digest string, size int64, blob io.Reader) error {
client, err := d.getClient(repository)
if err != nil {
return err
}
return client.PushBlob(digest, size, blob)
}
func isDigest(str string) bool {
return strings.Contains(str, ":")
}
// ListTag ...
func (d *DefaultImageRegistry) ListTag(repository string) ([]string, error) {
client, err := d.getClient(repository)
if err != nil {
return []string{}, err
}
return client.ListTag()
}

46
src/replication/adapter/image_registry_test.go
View File

@ -1,46 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package adapter
import (
"testing"
"github.com/stretchr/testify/assert"
)
// TODO add UT
func TestIsDigest(t *testing.T) {
cases := []struct {
str string
isDigest bool
}{
{
str: "",
isDigest: false,
},
{
str: "latest",
isDigest: false,
},
{
str: "sha256:fea8895f450959fa676bcc1df0611ea93823a735a01205fd8622846041d0c7cf",
isDigest: true,
},
}
for _, c := range cases {
assert.Equal(t, c.isDigest, isDigest(c.str))
}
}

343
src/replication/adapter/native/adapter.go
View File

@ -15,16 +15,26 @@
package native
import (
"io"
"net/http"
"strings"
"sync"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/goharbor/harbor/src/common/http/modifier"
common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth"
"github.com/goharbor/harbor/src/common/utils/log"
registry_pkg "github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
)
func init() {
if err := adp.RegisterFactory(model.RegistryTypeDockerRegistry, func(registry *model.Registry) (adp.Adapter, error) {
return newAdapter(registry)
return NewAdapter(registry)
}); err != nil {
log.Errorf("failed to register factory for %s: %v", model.RegistryTypeDockerRegistry, err)
return
@ -32,39 +42,65 @@ func init() {
log.Infof("the factory for adapter %s registered", model.RegistryTypeDockerRegistry)
}
func newAdapter(registry *model.Registry) (*Native, error) {
reg, err := adp.NewDefaultImageRegistry(registry)
if err != nil {
return nil, err
}
return &Native{
registry: registry,
DefaultImageRegistry: reg,
}, nil
}
var _ adp.Adapter = &Adapter{}
// NewWithClient ...
func NewWithClient(registry *model.Registry, client *http.Client) (*Native, error) {
reg, err := adp.NewDefaultRegistryWithClient(registry, client)
if err != nil {
return nil, err
}
return &Native{
registry: registry,
DefaultImageRegistry: reg,
}, nil
}
// Native is adapter to native docker registry
type Native struct {
*adp.DefaultImageRegistry
// Adapter implements an adapter for Docker registry. It can be used to all registries
// that implement the registry V2 API
type Adapter struct {
sync.RWMutex
*registry_pkg.Registry
registry *model.Registry
client *http.Client
clients map[string]*registry_pkg.Repository // client for repositories
}
var _ adp.Adapter = Native{}
// NewAdapter returns an instance of the Adapter
func NewAdapter(registry *model.Registry) (*Adapter, error) {
var authorizer modifier.Modifier
if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 {
var cred modifier.Modifier
if registry.Credential.Type == model.CredentialTypeSecret {
cred = common_http_auth.NewSecretAuthorizer(registry.Credential.AccessSecret)
} else {
cred = auth.NewBasicAuthCredential(
registry.Credential.AccessKey,
registry.Credential.AccessSecret)
}
authorizer = auth.NewStandardTokenAuthorizer(&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
}, cred, registry.TokenServiceURL)
}
return NewAdapterWithCustomizedAuthorizer(registry, authorizer)
}
// Info ...
func (Native) Info() (info *model.RegistryInfo, err error) {
// NewAdapterWithCustomizedAuthorizer returns an instance of the Adapter with the customized authorizer
func NewAdapterWithCustomizedAuthorizer(registry *model.Registry, authorizer modifier.Modifier) (*Adapter, error) {
transport := util.GetHTTPTransport(registry.Insecure)
modifiers := []modifier.Modifier{
&auth.UserAgentModifier{
UserAgent: adp.UserAgentReplication,
},
}
if authorizer != nil {
modifiers = append(modifiers, authorizer)
}
client := &http.Client{
Transport: registry_pkg.NewTransport(transport, modifiers...),
}
reg, err := registry_pkg.NewRegistry(registry.URL, client)
if err != nil {
return nil, err
}
return &Adapter{
Registry: reg,
registry: registry,
client: client,
clients: map[string]*registry_pkg.Repository{},
}, nil
}
// Info returns the basic information about the adapter
func (a *Adapter) Info() (info *model.RegistryInfo, err error) {
return &model.RegistryInfo{
Type: model.RegistryTypeDockerRegistry,
SupportedResourceTypes: []model.ResourceType{
@ -87,5 +123,250 @@ func (Native) Info() (info *model.RegistryInfo, err error) {
}, nil
}
// PrepareForPush nothing need to do.
func (Native) PrepareForPush([]*model.Resource) error { return nil }
// PrepareForPush does nothing
func (a *Adapter) PrepareForPush([]*model.Resource) error {
return nil
}
// HealthCheck checks health status of a registry
func (a *Adapter) HealthCheck() (model.HealthStatus, error) {
var err error
if a.registry.Credential == nil ||
(len(a.registry.Credential.AccessKey) == 0 && len(a.registry.Credential.AccessSecret) == 0) {
err = a.PingSimple()
} else {
err = a.Ping()
}
if err != nil {
log.Errorf("failed to ping registry %s: %v", a.registry.URL, err)
return model.Unhealthy, nil
}
return model.Healthy, nil
}
// FetchImages ...
func (a *Adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
repositories, err := a.getRepositories(filters)
if err != nil {
return nil, err
}
if len(repositories) == 0 {
return nil, nil
}
for _, filter := range filters {
if err = filter.DoFilter(&repositories); err != nil {
return nil, err
}
}
var resources []*model.Resource
for _, repository := range repositories {
vTags, err := a.getVTags(repository.Name)
if err != nil {
return nil, err
}
if len(vTags) == 0 {
continue
}
for _, filter := range filters {
if err = filter.DoFilter(&vTags); err != nil {
return nil, err
}
}
if len(vTags) == 0 {
continue
}
tags := []string{}
for _, vTag := range vTags {
tags = append(tags, vTag.Name)
}
resources = append(resources, &model.Resource{
Type: model.ResourceTypeImage,
Registry: a.registry,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: repository.Name,
},
Vtags: tags,
},
})
}
return resources, nil
}
func (a *Adapter) getRepositories(filters []*model.Filter) ([]*adp.Repository, error) {
pattern := ""
for _, filter := range filters {
if filter.Type == model.FilterTypeName {
pattern = filter.Value.(string)
break
}
}
var repositories []string
var err error
// if the pattern of repository name filter is a specific repository name, just returns
// the parsed repositories and will check the existence later when filtering the tags
if paths, ok := util.IsSpecificPath(pattern); ok {
repositories = paths
} else {
// search repositories from catalog API
repositories, err = a.Catalog()
if err != nil {
return nil, err
}
}
result := []*adp.Repository{}
for _, repository := range repositories {
result = append(result, &adp.Repository{
ResourceType: string(model.ResourceTypeImage),
Name: repository,
})
}
return result, nil
}
func (a *Adapter) getVTags(repository string) ([]*adp.VTag, error) {
tags, err := a.ListTag(repository)
if err != nil {
return nil, err
}
var result []*adp.VTag
for _, tag := range tags {
result = append(result, &adp.VTag{
ResourceType: string(model.ResourceTypeImage),
Name: tag,
})
}
return result, nil
}
// ManifestExist ...
func (a *Adapter) ManifestExist(repository, reference string) (bool, string, error) {
client, err := a.getClient(repository)
if err != nil {
return false, "", err
}
digest, exist, err := client.ManifestExist(reference)
return exist, digest, err
}
// PullManifest ...
func (a *Adapter) PullManifest(repository, reference string, accepttedMediaTypes []string) (distribution.Manifest, string, error) {
client, err := a.getClient(repository)
if err != nil {
return nil, "", err
}
digest, mediaType, payload, err := client.PullManifest(reference, accepttedMediaTypes)
if err != nil {
return nil, "", err
}
if strings.Contains(mediaType, "application/json") {
mediaType = schema1.MediaTypeManifest
}
manifest, _, err := registry_pkg.UnMarshal(mediaType, payload)
if err != nil {
return nil, "", err
}
return manifest, digest, nil
}
// PushManifest ...
func (a *Adapter) PushManifest(repository, reference, mediaType string, payload []byte) error {
client, err := a.getClient(repository)
if err != nil {
return err
}
_, err = client.PushManifest(reference, mediaType, payload)
return err
}
// DeleteManifest ...
func (a *Adapter) DeleteManifest(repository, reference string) error {
client, err := a.getClient(repository)
if err != nil {
return err
}
digest := reference
if !isDigest(digest) {
dgt, exist, err := client.ManifestExist(reference)
if err != nil {
return err
}
if !exist {
log.Debugf("the manifest of %s:%s doesn't exist", repository, reference)
return nil
}
digest = dgt
}
return client.DeleteManifest(digest)
}
// BlobExist ...
func (a *Adapter) BlobExist(repository, digest string) (bool, error) {
client, err := a.getClient(repository)
if err != nil {
return false, err
}
return client.BlobExist(digest)
}
// PullBlob ...
func (a *Adapter) PullBlob(repository, digest string) (int64, io.ReadCloser, error) {
client, err := a.getClient(repository)
if err != nil {
return 0, nil, err
}
return client.PullBlob(digest)
}
// PushBlob ...
func (a *Adapter) PushBlob(repository, digest string, size int64, blob io.Reader) error {
client, err := a.getClient(repository)
if err != nil {
return err
}
return client.PushBlob(digest, size, blob)
}
func isDigest(str string) bool {
return strings.Contains(str, ":")
}
// ListTag ...
func (a *Adapter) ListTag(repository string) ([]string, error) {
client, err := a.getClient(repository)
if err != nil {
return []string{}, err
}
return client.ListTag()
}
func (a *Adapter) getClient(repository string) (*registry_pkg.Repository, error) {
a.RLock()
client, exist := a.clients[repository]
a.RUnlock()
if exist {
return client, nil
}
return a.create(repository)
}
func (a *Adapter) create(repository string) (*registry_pkg.Repository, error) {
a.Lock()
defer a.Unlock()
// double check
client, exist := a.clients[repository]
if exist {
return client, nil
}
client, err := registry_pkg.NewRepository(repository, a.registry.URL, a.client)
if err != nil {
return nil, err
}
a.clients[repository] = client
return client, nil
}

295
src/replication/adapter/native/adapter_test.go
View File

@ -15,11 +15,15 @@
package native
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/replication/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_newAdapter(t *testing.T) {
@ -33,7 +37,7 @@ func Test_newAdapter(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := newAdapter(tt.registry)
got, err := NewAdapter(tt.registry)
if tt.wantErr {
assert.NotNil(t, err)
assert.Nil(t, got)
@ -47,14 +51,11 @@ func Test_newAdapter(t *testing.T) {
func Test_native_Info(t *testing.T) {
var registry = &model.Registry{URL: "abc"}
var reg, _ = adp.NewDefaultImageRegistry(registry)
var adapter = Native{
DefaultImageRegistry: reg,
registry: registry,
}
adapter, err := NewAdapter(registry)
require.Nil(t, err)
assert.NotNil(t, adapter)
var info, err = adapter.Info()
info, err := adapter.Info()
assert.Nil(t, err)
assert.NotNil(t, info)
assert.Equal(t, model.RegistryTypeDockerRegistry, info.Type)
@ -66,13 +67,279 @@ func Test_native_Info(t *testing.T) {
func Test_native_PrepareForPush(t *testing.T) {
var registry = &model.Registry{URL: "abc"}
var reg, _ = adp.NewDefaultImageRegistry(registry)
var adapter = Native{
DefaultImageRegistry: reg,
registry: registry,
}
adapter, err := NewAdapter(registry)
require.Nil(t, err)
assert.NotNil(t, adapter)
var err = adapter.PrepareForPush(nil)
err = adapter.PrepareForPush(nil)
assert.Nil(t, err)
}
func mockNativeRegistry() (mock *httptest.Server) {
return test.NewServer(
&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/v2/_catalog",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`{"repositories":["test/a1","test/b2","test/c3/3level"]}`))
},
},
&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/v2/test/a1/tags/list",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`{"name":"test/a1","tags":["tag11"]}`))
},
},
&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/v2/test/b2/tags/list",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`{"name":"test/b2","tags":["tag11","tag2","tag13"]}`))
},
},
&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/v2/test/c3/3level/tags/list",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`{"name":"test/c3/3level","tags":["tag4"]}`))
},
},
)
}
func Test_native_FetchImages(t *testing.T) {
var mock = mockNativeRegistry()
defer mock.Close()
fmt.Println("mockNativeRegistry URL: ", mock.URL)
var registry = &model.Registry{
Type: model.RegistryTypeDockerRegistry,
URL: mock.URL,
Insecure: true,
}
adapter, err := NewAdapter(registry)
assert.Nil(t, err)
assert.NotNil(t, adapter)
tests := []struct {
name string
filters []*model.Filter
want []*model.Resource
wantErr bool
}{
{
name: "repository not exist",
filters: []*model.Filter{
{
Type: model.FilterTypeName,
Value: "b1",
},
},
wantErr: false,
},
{
name: "tag not exist",
filters: []*model.Filter{
{
Type: model.FilterTypeTag,
Value: "this_tag_not_exist_in_the_mock_server",
},
},
wantErr: false,
},
{
name: "no filters",
filters: []*model.Filter{},
want: []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/a1"},
Vtags: []string{"tag11"},
},
},
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag11", "tag2", "tag13"},
},
},
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/c3/3level"},
Vtags: []string{"tag4"},
},
},
},
wantErr: false,
},
{
name: "only special repository",
filters: []*model.Filter{
{
Type: model.FilterTypeName,
Value: "test/a1",
},
},
want: []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/a1"},
Vtags: []string{"tag11"},
},
},
},
wantErr: false,
},
{
name: "only special tag",
filters: []*model.Filter{
{
Type: model.FilterTypeTag,
Value: "tag11",
},
},
want: []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/a1"},
Vtags: []string{"tag11"},
},
},
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag11"},
},
},
},
wantErr: false,
},
{
name: "special repository and special tag",
filters: []*model.Filter{
{
Type: model.FilterTypeName,
Value: "test/b2",
},
{
Type: model.FilterTypeTag,
Value: "tag2",
},
},
want: []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag2"},
},
},
},
wantErr: false,
},
{
name: "only wildcard repository",
filters: []*model.Filter{
{
Type: model.FilterTypeName,
Value: "test/b*",
},
},
want: []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag11", "tag2", "tag13"},
},
},
},
wantErr: false,
},
{
name: "only wildcard tag",
filters: []*model.Filter{
{
Type: model.FilterTypeTag,
Value: "tag1*",
},
},
want: []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/a1"},
Vtags: []string{"tag11"},
},
},
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag11", "tag13"},
},
},
},
wantErr: false,
},
{
name: "wildcard repository and wildcard tag",
filters: []*model.Filter{
{
Type: model.FilterTypeName,
Value: "test/b*",
},
{
Type: model.FilterTypeTag,
Value: "tag1*",
},
},
want: []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag11", "tag13"},
},
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var resources, err = adapter.FetchImages(tt.filters)
if tt.wantErr {
require.Len(t, resources, 0)
require.NotNil(t, err)
} else {
require.Equal(t, len(tt.want), len(resources))
for i, resource := range resources {
require.NotNil(t, resource.Metadata)
assert.Equal(t, tt.want[i].Metadata.Repository, resource.Metadata.Repository)
assert.Equal(t, tt.want[i].Metadata.Vtags, resource.Metadata.Vtags)
}
}
})
}
}
func TestIsDigest(t *testing.T) {
cases := []struct {
str string
isDigest bool
}{
{
str: "",
isDigest: false,
},
{
str: "latest",
isDigest: false,
},
{
str: "sha256:fea8895f450959fa676bcc1df0611ea93823a735a01205fd8622846041d0c7cf",
isDigest: true,
},
}
for _, c := range cases {
assert.Equal(t, c.isDigest, isDigest(c.str))
}
}

114
src/replication/adapter/native/image_registry.go
View File

@ -1,114 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package native
import (
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
)
var _ adp.ImageRegistry = Native{}
// FetchImages ...
func (n Native) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
nameFilterPattern := ""
tagFilterPattern := ""
for _, filter := range filters {
switch filter.Type {
case model.FilterTypeName:
nameFilterPattern = filter.Value.(string)
case model.FilterTypeTag:
tagFilterPattern = filter.Value.(string)
}
}
repositories, err := n.filterRepositories(nameFilterPattern)
if err != nil {
return nil, err
}
var resources []*model.Resource
for _, repository := range repositories {
tags, err := n.filterTags(repository, tagFilterPattern)
if err != nil {
return nil, err
}
if len(tags) == 0 {
continue
}
resources = append(resources, &model.Resource{
Type: model.ResourceTypeImage,
Registry: n.registry,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: repository,
},
Vtags: tags,
},
})
}
return resources, nil
}
func (n Native) filterRepositories(pattern string) ([]string, error) {
// if the pattern is a specific repository name, just returns the parsed repositories
// and will check the existence later when filtering the tags
if repositories, ok := util.IsSpecificPath(pattern); ok {
return repositories, nil
}
// search repositories from catalog api
repositories, err := n.Catalog()
if err != nil {
return nil, err
}
// if the pattern is null, just return the result of catalog API
if len(pattern) == 0 {
return repositories, nil
}
result := []string{}
for _, repository := range repositories {
match, err := util.Match(pattern, repository)
if err != nil {
return nil, err
}
if match {
result = append(result, repository)
}
}
return result, nil
}
func (n Native) filterTags(repository, pattern string) ([]string, error) {
tags, err := n.ListTag(repository)
if err != nil {
return nil, err
}
if len(pattern) == 0 {
return tags, nil
}
var result []string
for _, tag := range tags {
match, err := util.Match(pattern, tag)
if err != nil {
return nil, err
}
if match {
result = append(result, tag)
}
}
return result, nil
}

279
src/replication/adapter/native/image_registry_test.go
View File

@ -1,279 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package native
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/goharbor/harbor/src/common/utils/test"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func mockNativeRegistry() (mock *httptest.Server) {
return test.NewServer(
&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/v2/_catalog",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`{"repositories":["test/a1","test/b2","test/c3/3level"]}`))
},
},
&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/v2/test/a1/tags/list",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`{"name":"test/a1","tags":["tag11"]}`))
},
},
&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/v2/test/b2/tags/list",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`{"name":"test/b2","tags":["tag11","tag2","tag13"]}`))
},
},
&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/v2/test/c3/3level/tags/list",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`{"name":"test/c3/3level","tags":["tag4"]}`))
},
},
)
}
func Test_native_FetchImages(t *testing.T) {
var mock = mockNativeRegistry()
defer mock.Close()
fmt.Println("mockNativeRegistry URL: ", mock.URL)
var registry = &model.Registry{
Type: model.RegistryTypeDockerRegistry,
URL: mock.URL,
Insecure: true,
}
var reg, err = adp.NewDefaultImageRegistry(registry)
assert.NotNil(t, reg)
assert.Nil(t, err)
var adapter = Native{
DefaultImageRegistry: reg,
registry: registry,
}
assert.NotNil(t, adapter)
tests := []struct {
name string
filters []*model.Filter
want []*model.Resource
wantErr bool
}{
{
name: "repository not exist",
filters: []*model.Filter{
{
Type: model.FilterTypeName,
Value: "b1",
},
},
wantErr: false,
},
{
name: "tag not exist",
filters: []*model.Filter{
{
Type: model.FilterTypeTag,
Value: "this_tag_not_exist_in_the_mock_server",
},
},
wantErr: false,
},
{
name: "no filters",
filters: []*model.Filter{},
want: []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/a1"},
Vtags: []string{"tag11"},
},
},
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag11", "tag2", "tag13"},
},
},
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/c3/3level"},
Vtags: []string{"tag4"},
},
},
},
wantErr: false,
},
{
name: "only special repository",
filters: []*model.Filter{
{
Type: model.FilterTypeName,
Value: "test/a1",
},
},
want: []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/a1"},
Vtags: []string{"tag11"},
},
},
},
wantErr: false,
},
{
name: "only special tag",
filters: []*model.Filter{
{
Type: model.FilterTypeTag,
Value: "tag11",
},
},
want: []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/a1"},
Vtags: []string{"tag11"},
},
},
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag11"},
},
},
},
wantErr: false,
},
{
name: "special repository and special tag",
filters: []*model.Filter{
{
Type: model.FilterTypeName,
Value: "test/b2",
},
{
Type: model.FilterTypeTag,
Value: "tag2",
},
},
want: []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag2"},
},
},
},
wantErr: false,
},
{
name: "only wildcard repository",
filters: []*model.Filter{
{
Type: model.FilterTypeName,
Value: "test/b*",
},
},
want: []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag11", "tag2", "tag13"},
},
},
},
wantErr: false,
},
{
name: "only wildcard tag",
filters: []*model.Filter{
{
Type: model.FilterTypeTag,
Value: "tag1*",
},
},
want: []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/a1"},
Vtags: []string{"tag11"},
},
},
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag11", "tag13"},
},
},
},
wantErr: false,
},
{
name: "wildcard repository and wildcard tag",
filters: []*model.Filter{
{
Type: model.FilterTypeName,
Value: "test/b*",
},
{
Type: model.FilterTypeTag,
Value: "tag1*",
},
},
want: []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag11", "tag13"},
},
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var resources, err = adapter.FetchImages(tt.filters)
if tt.wantErr {
require.Len(t, resources, 0)
require.NotNil(t, err)
} else {
require.Equal(t, len(tt.want), len(resources))
for i, resource := range resources {
require.NotNil(t, resource.Metadata)
assert.Equal(t, tt.want[i].Metadata.Repository, resource.Metadata.Repository)
assert.Equal(t, tt.want[i].Metadata.Vtags, resource.Metadata.Vtags)
}
}
})
}
}

40
tests/ldap_test.ldif
View File

@ -19,7 +19,7 @@ dn: ou=people,dc=example,dc=com
objectClass: organizationalUnit
ou: People
# OU for Groups
# OU for Groups
dn: ou=groups,dc=example,dc=com
objectClass: organizationalUnit
ou: Groups
@ -84,6 +84,8 @@ member: cn=user023,ou=people,dc=example,dc=com
member: cn=user024,ou=people,dc=example,dc=com
member: cn=user025,ou=people,dc=example,dc=com
member: cn=user026,ou=people,dc=example,dc=com
member: cn=user027,ou=people,dc=example,dc=com
member: cn=user028,ou=people,dc=example,dc=com
objectclass: groupOfNames
objectclass: top
@ -678,6 +680,42 @@ uid: user026
uidnumber: 5030
userpassword: {MD5}jMx5MaPXabPyX7F0SoFxYQ==
dn: cn=user027,ou=people,dc=example,dc=com
cn: user027
gidnumber: 10000
givenname: user027
homedirectory: /home/user027
loginshell: /bin/bash
mail: user027@example.com
objectclass: top
objectclass: posixAccount
objectclass: shadowAccount
objectclass: inetOrgPerson
objectclass: organizationalPerson
objectclass: person
sn: user027
uid: user027
uidnumber: 5031
userpassword: {MD5}jMx5MaPXabPyX7F0SoFxYQ==
dn: cn=user028,ou=people,dc=example,dc=com
cn: user028
gidnumber: 10000
givenname: user028
homedirectory: /home/user028
loginshell: /bin/bash
mail: user028@example.com
objectclass: top
objectclass: posixAccount
objectclass: shadowAccount
objectclass: inetOrgPerson
objectclass: organizationalPerson
objectclass: person
sn: user028
uid: user028
uidnumber: 5032
userpassword: {MD5}jMx5MaPXabPyX7F0SoFxYQ==
dn: cn=guest_user,ou=people,dc=example,dc=com
cn: guest_user
gidnumber: 10000

4
tests/resources/Harbor-Pages/ToolKit.robot
View File

@ -77,8 +77,8 @@ Multi-delete Member
[Arguments] @{obj}
:For ${obj} in @{obj}
\ Retry Element Click //clr-dg-row[contains(.,'${obj}')]//label
Retry Element Click ${member_action_xpath}
Retry Element Click ${delete_action_xpath}
Retry Double Keywords When Error Retry Element Click ${member_action_xpath} Retry Wait Until Page Contains Element ${delete_action_xpath}
Retry Double Keywords When Error Retry Element Click ${delete_action_xpath} Retry Wait Until Page Contains Element ${delete_btn}
Retry Double Keywords When Error Retry Element Click ${delete_btn} Retry Wait Until Page Not Contains Element ${delete_btn}