upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor synced 2025-04-16 11:09:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/goharbor/harbor into remove-tri-sec

Browse Source
This commit is contained in:
wang yan 2020-09-07 11:57:22 +08:00
commit 827158badc
16 changed files with 154 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
VERSION
View File

@ -1 +1 @@
v2.1.0
v2.2.0

5
src/common/models/project.go
View File

@ -80,6 +80,11 @@ func (p *Project) IsPublic() bool {
return isTrue(public)
}
// IsProxy returns true when the project type is proxy cache
func (p *Project) IsProxy() bool {
return p.RegistryID > 0
}
// ContentTrustEnabled ...
func (p *Project) ContentTrustEnabled() bool {
enabled, exist := p.GetMetadata(ProMetaEnableContentTrust)

17
src/controller/p2p/preheat/controller.go
View File

@ -117,6 +117,8 @@ type Controller interface {
ListPoliciesByProject(ctx context.Context, project int64, query *q.Query) ([]*policyModels.Schema, error)
// CheckHealth checks the instance health, for test connection
CheckHealth(ctx context.Context, instance *providerModels.Instance) error
// DeletePoliciesOfProject delete all policies under one project
DeletePoliciesOfProject(ctx context.Context, project int64) error
}
var _ Controller = (*controller)(nil)
@ -419,6 +421,21 @@ func (c *controller) DeletePolicy(ctx context.Context, id int64) error {
return c.pManager.Delete(ctx, id)
}
// DeletePoliciesOfProject deletes all the policy under project.
func (c *controller) DeletePoliciesOfProject(ctx context.Context, project int64) error {
policies, err := c.ListPoliciesByProject(ctx, project, nil)
if err != nil {
return err
}
for _, p := range policies {
if err = c.DeletePolicy(ctx, p.ID); err != nil {
return err
}
}
return nil
}
// deleteExecs delete executions
func (c *controller) deleteExecs(ctx context.Context, vendorID int64) error {
executions, err := c.executionMgr.List(ctx, &q.Query{

16
src/controller/p2p/preheat/controllor_test.go
View File

@ -335,6 +335,22 @@ func (s *preheatSuite) TestListPoliciesByProject() {
s.NotNil(p)
}
func (s *preheatSuite) TestDeletePoliciesOfProject() {
fakePolicies := []*policy.Schema{
{ID: 1000, Name: "1-should-delete", ProjectID: 10},
{ID: 1001, Name: "2-should-delete", ProjectID: 10},
}
s.fakePolicyMgr.On("ListPoliciesByProject", s.ctx, int64(10), mock.Anything).Return(fakePolicies, nil)
for _, p := range fakePolicies {
s.fakePolicyMgr.On("Get", s.ctx, p.ID).Return(p, nil)
s.fakePolicyMgr.On("Delete", s.ctx, p.ID).Return(nil)
s.fakeExecutionMgr.On("List", s.ctx, &q.Query{Keywords: map[string]interface{}{"VendorID": p.ID, "VendorType": "P2P_PREHEAT"}}).Return([]*taskModel.Execution{}, nil)
}
err := s.controller.DeletePoliciesOfProject(s.ctx, 10)
s.NoError(err)
}
func (s *preheatSuite) TestCheckHealth() {
// if instance is nil
var instance *providerModel.Instance

4
src/core/api/base.go
View File

@ -35,11 +35,11 @@ import (
"github.com/goharbor/harbor/src/pkg/repository"
"github.com/goharbor/harbor/src/pkg/retention"
"github.com/goharbor/harbor/src/pkg/scheduler"
sec "github.com/goharbor/harbor/src/server/middleware/security"
)
const (
yamlFileContentType = "application/x-yaml"
userSessionKey = "user"
)
// the managers/controllers used globally
@ -176,7 +176,7 @@ func (b *BaseController) WriteYamlData(object interface{}) {
// PopulateUserSession generates a new session ID and fill the user model in parm to the session
func (b *BaseController) PopulateUserSession(u models.User) {
b.SessionRegenerateID()
b.SetSession(userSessionKey, u)
b.SetSession(sec.UserIDSessionKey, u.UserID)
}
// Init related objects/configurations for the API controllers

3
src/server/middleware/contenttrust/contenttrust.go
View File

@ -21,6 +21,9 @@ var (
if err != nil {
return false, err
}
if len(art.Tag) > 0 {
return checker.IsTagSigned(art.Tag, art.Digest), nil
}
return checker.IsArtifactSigned(art.Digest), nil
}
)

22
src/server/middleware/repoproxy/proxy.go
View File

@ -17,20 +17,20 @@ package repoproxy
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/common/security"
"github.com/goharbor/harbor/src/common/security/proxycachesecret"
"github.com/goharbor/harbor/src/lib/errors"
httpLib "github.com/goharbor/harbor/src/lib/http"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/registry"
"io"
"net/http"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/security"
"github.com/goharbor/harbor/src/common/security/proxycachesecret"
"github.com/goharbor/harbor/src/controller/project"
"github.com/goharbor/harbor/src/controller/proxy"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/lib/errors"
httpLib "github.com/goharbor/harbor/src/lib/http"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/registry"
"github.com/goharbor/harbor/src/server/middleware"
)
@ -163,14 +163,6 @@ func setHeaders(w http.ResponseWriter, size int64, mediaType string, dig string)
h.Set("Etag", dig)
}
// isProxyProject check the project is a proxy project
func isProxyProject(p *models.Project) bool {
if p == nil {
return false
}
return p.RegistryID > 0
}
// isProxySession check if current security context is proxy session
func isProxySession(ctx context.Context) bool {
sc, ok := security.FromContext(ctx)
@ -194,7 +186,7 @@ func DisableBlobAndManifestUploadMiddleware() func(http.Handler) http.Handler {
httpLib.SendError(w, err)
return
}
if isProxyProject(p) && !isProxySession(ctx) {
if p.IsProxy() && !isProxySession(ctx) {
httpLib.SendError(w,
errors.DeniedError(
errors.Errorf("can not push artifact to a proxy project: %v", p.Name)))

32
src/server/middleware/repoproxy/proxy_test.go
View File

@ -18,44 +18,12 @@ import (
"context"
"testing"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/security"
"github.com/goharbor/harbor/src/common/security/proxycachesecret"
securitySecret "github.com/goharbor/harbor/src/common/security/secret"
"github.com/goharbor/harbor/src/core/config"
)
func TestIsProxyProject(t *testing.T) {
cases := []struct {
name string
in *models.Project
want bool
}{
{
name: `no proxy`,
in: &models.Project{RegistryID: 0},
want: false,
},
{
name: `normal proxy`,
in: &models.Project{RegistryID: 1},
want: true,
},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
got := isProxyProject(tt.in)
if got != tt.want {
t.Errorf(`(%v) = %v; want "%v"`, tt.in, got, tt.want)
}
})
}
}
func TestIsProxySession(t *testing.T) {
config.Init()
sc1 := securitySecret.NewSecurityContext("123456789", config.SecretStore)

16
src/server/middleware/security/auth_proxy_test.go
View File

@ -17,25 +17,23 @@ package security
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/test"
_ "github.com/goharbor/harbor/src/core/auth/authproxy"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/lib"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"io/ioutil"
"k8s.io/api/authentication/v1beta1"
"net/http"
"net/http/httptest"
"net/url"
"testing"
)
func TestAuthProxy(t *testing.T) {
config.Init()
test.InitDatabaseFromEnv()
authProxy := &authProxy{}
server, err := newAuthProxyTestServer()
@ -49,7 +47,7 @@ func TestAuthProxy(t *testing.T) {
common.HTTPAuthProxyTokenReviewEndpoint: server.URL,
common.AUTHMode: common.HTTPAuth,
}
config.Upload(c)
config.InitWithSettings(c)
v, e := config.HTTPAuthProxySetting()
require.Nil(t, e)
assert.Equal(t, *v, models.HTTPAuthProxy{

23
src/server/middleware/security/basic_auth_test.go
View File

@ -15,18 +15,39 @@
package security
import (
"fmt"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
_ "github.com/goharbor/harbor/src/core/auth/db"
"github.com/goharbor/harbor/src/core/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"net/http"
"testing"
)
func TestBasicAuth(t *testing.T) {
c := map[string]interface{}{
common.AUTHMode: common.DBAuth,
}
config.InitWithSettings(c)
user := models.User{
Username: "tester",
Password: "Harbor12345",
}
uid, err := dao.Register(user)
defer func(id int64) {
sql := fmt.Sprintf("DELETE FROM harbor_user WHERE user_id=%d", id)
dao.ExecuteBatchSQL([]string{sql})
}(uid)
basicAuth := &basicAuth{}
req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1/api/projects/", nil)
require.Nil(t, err)
req.SetBasicAuth("admin", "Harbor12345")
req.SetBasicAuth("tester", "Harbor12345")
ctx := basicAuth.Generate(req)
assert.NotNil(t, ctx)
}

8
src/server/middleware/security/security_test.go
View File

@ -15,13 +15,21 @@
package security
import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/security"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"net/http"
"os"
"testing"
)
func TestMain(m *testing.M) {
dao.PrepareTestForPostgresSQL()
os.Exit(m.Run())
}
func TestSecurity(t *testing.T) {
var ctx security.Context
var exist bool

21
src/server/middleware/security/session.go
View File

@ -19,6 +19,7 @@ import (
"net/http/httptest"
"github.com/astaxie/beego"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/security"
"github.com/goharbor/harbor/src/common/security/local"
@ -26,6 +27,9 @@ import (
"github.com/goharbor/harbor/src/lib/log"
)
// UserIDSessionKey is the key for storing user id in session
const UserIDSessionKey = "user_id"
type session struct{}
func (s *session) Generate(req *http.Request) security.Context {
@ -35,15 +39,24 @@ func (s *session) Generate(req *http.Request) security.Context {
log.Errorf("failed to get the session store for request: %v", err)
return nil
}
userInterface := store.Get("user")
userInterface := store.Get(UserIDSessionKey)
if userInterface == nil {
return nil
}
user, ok := userInterface.(models.User)
uid, ok := userInterface.(int)
if !ok {
log.Warning("can not convert the user in session to user model")
log.Warning("can not convert the data in session to user id")
return nil
}
user, err := dao.GetUser(models.User{UserID: uid})
if err != nil {
log.Errorf("failed to get user info, error: %v, skip generating security context via session", err)
return nil
}
if user == nil {
log.Errorf("the user id from session: %d, not exist in DB", uid)
return nil
}
log.Debugf("a session security context generated for request %s %s", req.Method, req.URL.Path)
return local.NewSecurityContext(&user, config.GlobalProjectMgr)
return local.NewSecurityContext(user, config.GlobalProjectMgr)
}

25
src/server/middleware/security/session_test.go
View File

@ -15,11 +15,15 @@
package security
import (
"fmt"
"github.com/astaxie/beego"
beegosession "github.com/astaxie/beego/session"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"net/http"
"net/http/httptest"
"path/filepath"
@ -41,20 +45,31 @@ func TestSession(t *testing.T) {
beego.GlobalSessions, err = beegosession.NewManager("memory", conf)
require.Nil(t, err)
user := models.User{
Username: "admin",
UserID: 1,
Email: "admin@example.com",
user := &models.User{
Username: "session_test",
Email: "session_test@example.com",
SysAdminFlag: true,
}
err = dao.OnBoardUser(user)
require.Nil(t, err)
defer func(id int) {
sql := fmt.Sprintf("DELETE FROM harbor_user WHERE user_id=%d", id)
dao.ExecuteBatchSQL([]string{sql})
}(user.UserID)
req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1/api/projects/", nil)
require.Nil(t, err)
store, err := beego.GlobalSessions.SessionStart(httptest.NewRecorder(), req)
require.Nil(t, err)
err = store.Set("user", user)
err = store.Set(UserIDSessionKey, 999)
require.Nil(t, err)
session := &session{}
ctx := session.Generate(req)
assert.Nil(t, ctx)
err = store.Set(UserIDSessionKey, user.UserID)
require.Nil(t, err)
ctx = session.Generate(req)
assert.NotNil(t, ctx)
}

8
src/server/v2.0/handler/artifact.go
View File

@ -17,9 +17,6 @@ package handler
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/controller/event/metadata"
"github.com/goharbor/harbor/src/controller/project"
"github.com/goharbor/harbor/src/pkg/notification"
"net/http"
"strings"
"time"
@ -31,10 +28,13 @@ import (
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/controller/artifact"
"github.com/goharbor/harbor/src/controller/artifact/processor"
"github.com/goharbor/harbor/src/controller/event/metadata"
"github.com/goharbor/harbor/src/controller/project"
"github.com/goharbor/harbor/src/controller/repository"
"github.com/goharbor/harbor/src/controller/scan"
"github.com/goharbor/harbor/src/controller/tag"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/server/v2.0/handler/assembler"
"github.com/goharbor/harbor/src/server/v2.0/handler/model"
"github.com/goharbor/harbor/src/server/v2.0/models"
@ -244,7 +244,7 @@ func (a *artifactAPI) requireNonProxyCacheProject(ctx context.Context, name stri
if err != nil {
return err
}
if pro.RegistryID > 0 {
if pro.IsProxy() {
return errors.New(nil).WithCode(errors.MethodNotAllowedCode).
WithMessage("the operation isn't supported for a proxy cache project")
}

23
src/server/v2.0/handler/project.go
View File

@ -3,7 +3,6 @@ package handler
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/pkg/robot"
"strconv"
"strings"
"sync"
@ -15,6 +14,7 @@ import (
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/security"
"github.com/goharbor/harbor/src/common/security/local"
"github.com/goharbor/harbor/src/controller/p2p/preheat"
"github.com/goharbor/harbor/src/controller/project"
"github.com/goharbor/harbor/src/controller/quota"
"github.com/goharbor/harbor/src/controller/repository"
@ -29,6 +29,7 @@ import (
"github.com/goharbor/harbor/src/pkg/project/metadata"
"github.com/goharbor/harbor/src/pkg/quota/types"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/robot"
"github.com/goharbor/harbor/src/pkg/user"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/server/v2.0/handler/model"
@ -48,6 +49,7 @@ func newProjectAPI() *projectAPI {
projectCtl: project.Ctl,
quotaCtl: quota.Ctl,
robotMgr: robot.Mgr,
preheatCtl: preheat.Ctl,
}
}
@ -60,6 +62,7 @@ type projectAPI struct {
projectCtl project.Controller
quotaCtl quota.Controller
robotMgr robot.Manager
preheatCtl preheat.Controller
}
func (a *projectAPI) CreateProject(ctx context.Context, params operation.CreateProjectParams) middleware.Responder {
@ -116,6 +119,12 @@ func (a *projectAPI) CreateProject(ctx context.Context, params operation.CreateP
req.Metadata.Public = strconv.FormatBool(false)
}
// ignore enable_content_trust metadata for proxy cache project
// see https://github.com/goharbor/harbor/issues/12940 to get more info
if req.RegistryID != nil {
req.Metadata.EnableContentTrust = nil
}
// validate the RegistryID and StorageLimit in the body of the request
if err := a.validateProjectReq(ctx, req); err != nil {
return a.SendError(ctx, err)
@ -212,6 +221,11 @@ func (a *projectAPI) DeleteProject(ctx context.Context, params operation.DeleteP
}
}
// preheat policies under the project should be deleted after deleting the project
if err = a.preheatCtl.DeletePoliciesOfProject(ctx, params.ProjectID); err != nil {
return a.SendError(ctx, err)
}
return operation.NewDeleteProjectOK()
}
@ -306,7 +320,7 @@ func (a *projectAPI) GetProjectSummary(ctx context.Context, params operation.Get
fetchSummaries = append(fetchSummaries, getProjectMemberSummary)
}
if p.RegistryID > 0 {
if p.IsProxy() {
fetchSummaries = append(fetchSummaries, getProjectRegistrySummary)
}
@ -450,6 +464,11 @@ func (a *projectAPI) UpdateProject(ctx context.Context, params operation.UpdateP
}
}
// ignore enable_content_trust metadata for proxy cache project
// see https://github.com/goharbor/harbor/issues/12940 to get more info
if params.Project.Metadata != nil && p.IsProxy() {
params.Project.Metadata.EnableContentTrust = nil
}
lib.JSONCopy(&p.Metadata, params.Project.Metadata)
if err := a.projectCtl.Update(ctx, p); err != nil {

5
tests/ci/api_common_install.sh
View File

@ -25,7 +25,10 @@ sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && su
sudo make swagger_client
#TODO: Swagger python package used to installed into dist-packages, but it's changed into site-packages all in a sudden, we havn't found the root cause.
# so current workround is to copy swagger packages from site-packages to dist-packages.
sudo cp -r /usr/lib/python3.7/site-packages/* /usr/local/lib/python3.7/dist-packages
package_dir=/usr/lib/python3.7/site-packages
if [ -d $package_dir ] && [ $(find $package_dir -type f -name "*client*.egg" | wc -l) -gt 0 ];then
sudo cp -rf ${package_dir}/* /usr/local/lib/python3.7/dist-packages
fi
if [ $GITHUB_TOKEN ];
then