upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor synced 2024-09-20 20:15:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/job-service' into sync_image

Browse Source
This commit is contained in:
Wenkai Yin 2016-05-25 15:41:11 +08:00
commit 84cfafaeb0
15 changed files with 127 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
api/jobs/replication.go
View File

@ -6,15 +6,14 @@ import (
"github.com/vmware/harbor/api"
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/job"
"github.com/vmware/harbor/job/config"
"github.com/vmware/harbor/job/utils"
"github.com/vmware/harbor/models"
"github.com/vmware/harbor/utils/log"
"io/ioutil"
"net/http"
"net/http/httputil"
"os"
"strconv"
"strings"
)
type ReplicationJob struct {
@ -131,29 +130,27 @@ func (rj *ReplicationJob) GetLog() {
// calls the api from UI to get repo list
func getRepoList(projectID int64) ([]string, error) {
uiURL := os.Getenv("UI_URL")
if len(uiURL) == 0 {
uiURL = "ui"
}
if !strings.HasSuffix(uiURL, "/") {
uiURL += "/"
}
//TODO:Use secret key instead
uiUser := os.Getenv("UI_USR")
if len(uiUser) == 0 {
uiUser = "admin"
}
uiPwd := os.Getenv("UI_PWD")
if len(uiPwd) == 0 {
uiPwd = "Harbor12345"
}
/*
uiUser := os.Getenv("UI_USR")
if len(uiUser) == 0 {
uiUser = "admin"
}
uiPwd := os.Getenv("UI_PWD")
if len(uiPwd) == 0 {
uiPwd = "Harbor12345"
}
*/
uiURL := config.LocalHarborURL()
client := &http.Client{}
req, err := http.NewRequest("GET", uiURL+"api/repositories?project_id="+strconv.Itoa(int(projectID)), nil)
if err != nil {
log.Errorf("Error when creating request: %v")
return nil, err
}
req.SetBasicAuth(uiUser, uiPwd)
//req.SetBasicAuth(uiUser, uiPwd)
req.AddCookie(&http.Cookie{Name: models.UISecretCookie, Value: config.UISecret()})
//dump, err := httputil.DumpRequest(req, true)
//log.Debugf("req: %q", dump)
resp, err := client.Do(req)
if err != nil {
log.Errorf("Error when calling UI api to get repositories, error: %v", err)

11
api/repository.go
View File

@ -26,6 +26,7 @@ import (
"github.com/docker/distribution/manifest/schema1"
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/models"
"github.com/vmware/harbor/service/cache"
svc_utils "github.com/vmware/harbor/service/utils"
"github.com/vmware/harbor/utils/log"
"github.com/vmware/harbor/utils/registry"
@ -43,7 +44,11 @@ type RepositoryAPI struct {
// Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission.
func (ra *RepositoryAPI) Prepare() {
ra.userID = ra.ValidateUser()
if svc_utils.VerifySecret(ra.Ctx.Request) {
ra.userID = 1
} else {
ra.userID = ra.ValidateUser()
}
}
// Get ...
@ -69,7 +74,7 @@ func (ra *RepositoryAPI) Get() {
return
}
repoList, err := svc_utils.GetRepoFromCache()
repoList, err := cache.GetRepoFromCache()
if err != nil {
log.Errorf("Failed to get repo from cache, error: %v", err)
ra.RenderError(http.StatusInternalServerError, "internal sever error")
@ -142,7 +147,7 @@ func (ra *RepositoryAPI) Delete() {
go func() {
log.Debug("refreshing catalog cache")
if err := svc_utils.RefreshCatalogCache(); err != nil {
if err := cache.RefreshCatalogCache(); err != nil {
log.Errorf("error occurred while refresh catalog cache: %v", err)
}
}()

4
api/search.go
View File

@ -22,7 +22,7 @@ import (
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/models"
svc_utils "github.com/vmware/harbor/service/utils"
"github.com/vmware/harbor/service/cache"
"github.com/vmware/harbor/utils"
"github.com/vmware/harbor/utils/log"
)
@ -85,7 +85,7 @@ func (s *SearchAPI) Get() {
}
}
repositories, err2 := svc_utils.GetRepoFromCache()
repositories, err2 := cache.GetRepoFromCache()
if err2 != nil {
log.Errorf("Failed to get repos from cache, error: %v", err2)
s.CustomAbort(http.StatusInternalServerError, "Failed to get repositories search result")

6
api/statistic.go
View File

@ -21,7 +21,7 @@ import (
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/models"
svc_utils "github.com/vmware/harbor/service/utils"
"github.com/vmware/harbor/service/cache"
"github.com/vmware/harbor/utils/log"
)
@ -88,7 +88,7 @@ func (s *StatisticAPI) Get() {
//getReposByProject returns repo numbers of specified project
func getRepoCountByProject(projectName string) int {
repoList, err := svc_utils.GetRepoFromCache()
repoList, err := cache.GetRepoFromCache()
if err != nil {
log.Errorf("Failed to get repo from cache, error: %v", err)
return 0
@ -107,7 +107,7 @@ func getRepoCountByProject(projectName string) int {
//getTotalRepoCount returns total repo count
func getTotalRepoCount() int {
repoList, err := svc_utils.GetRepoFromCache()
repoList, err := cache.GetRepoFromCache()
if err != nil {
log.Errorf("Failed to get repo from cache, error: %v", err)
return 0

25
job/config/config.go
View File

@ -11,8 +11,9 @@ import (
const defaultMaxWorkers int = 10
var maxJobWorkers int
var localRegURL string
var localURL string
var logDir string
var uiSecret string
func init() {
maxWorkersEnv := os.Getenv("MAX_JOB_WORKERS")
@ -23,9 +24,9 @@ func init() {
maxJobWorkers = defaultMaxWorkers
}
localRegURL = os.Getenv("LOCAL_REGISTRY_URL")
if len(localRegURL) == 0 {
localRegURL = "http://registry:5000/"
localURL = os.Getenv("LOCAL_HARBOR_URL")
if len(localURL) == 0 {
localURL = "http://ui/"
}
logDir = os.Getenv("LOG_DIR")
@ -46,19 +47,29 @@ func init() {
panic(fmt.Sprintf("%s is not a direcotry", logDir))
}
uiSecret = os.Getenv("UI_SECRET")
if len(uiSecret) == 0 {
panic("UI Secret is not set")
}
log.Debugf("config: maxJobWorkers: %d", maxJobWorkers)
log.Debugf("config: localRegURL: %s", localRegURL)
log.Debugf("config: localHarborURL: %s", localURL)
log.Debugf("config: logDir: %s", logDir)
log.Debugf("config: uiSecret: ******")
}
func MaxJobWorkers() int {
return maxJobWorkers
}
func LocalRegURL() string {
return localRegURL
func LocalHarborURL() string {
return localURL
}
func LogDir() string {
return logDir
}
func UISecret() string {
return uiSecret
}

13
job/replication/statehandlers.go
View File

@ -49,12 +49,11 @@ type BaseHandler struct {
repository string // prject_name/repo_name
tags []string
srcURL string // url of source registry
srcSecretKey string // secretKey ...
srcURL string // url of source registry
dstURL string // url of target registry
dstUsr string // username ...
dstPwd string // password ...
dstPwd string // username ...
srcClient *registry.Repository
dstClient *registry.Repository
@ -69,7 +68,7 @@ type BaseHandler struct {
// InitBaseHandler initializes a BaseHandler: creating clients for source and destination registry,
// listing tags of the repository if parameter tags is nil.
func InitBaseHandler(repository, srcURL, srcSecretKey,
func InitBaseHandler(repository, srcURL, srcSecret,
dstURL, dstUsr, dstPwd string, tags []string, logger *log.Logger) (*BaseHandler, error) {
logger.Infof("initializing: repository: %s, tags: %v, source URL: %s, destination URL: %s, destination user: %s",
@ -79,7 +78,6 @@ func InitBaseHandler(repository, srcURL, srcSecretKey,
repository: repository,
tags: tags,
srcURL: srcURL,
srcSecretKey: srcSecretKey,
dstURL: dstURL,
dstUsr: dstUsr,
dstPwd: dstPwd,
@ -89,8 +87,9 @@ func InitBaseHandler(repository, srcURL, srcSecretKey,
base.project = getProjectName(base.repository)
//TODO using secret key
srcCred := auth.NewBasicAuthCredential("admin", "Harbor12345")
c := &http.Cookie{Name: models.UISecretCookie, Value: srcSecret}
srcCred := auth.NewCookieCredential(c)
// srcCred := auth.NewBasicAuthCredential("admin", "Harbor12345")
srcClient, err := registry.NewRepositoryWithCredential(base.repository, base.srcURL, srcCred)
if err != nil {
return nil, err

4
job/statemachine.go
View File

@ -180,7 +180,7 @@ func (sm *JobSM) Reset(jid int64) error {
return fmt.Errorf("The policy doesn't exist in DB, policy id:%d", job.PolicyID)
}
sm.Parms = &RepJobParm{
LocalRegURL: config.LocalRegURL(),
LocalRegURL: config.LocalHarborURL(),
Repository: job.Repository,
Enabled: policy.Enabled,
Operation: job.Operation,
@ -224,7 +224,7 @@ func (sm *JobSM) Reset(jid int64) error {
}
func addImgOutTransition(sm *JobSM) error {
base, err := replication.InitBaseHandler(sm.Parms.Repository, sm.Parms.LocalRegURL, "",
base, err := replication.InitBaseHandler(sm.Parms.Repository, sm.Parms.LocalRegURL, config.UISecret(),
sm.Parms.TargetURL, sm.Parms.TargetUsername, sm.Parms.TargetPassword,
nil, sm.Logger)
if err != nil {

7
jobservice/my_start.sh
View File

@ -3,9 +3,10 @@ export MYSQL_PORT=3306
export MYSQL_USR=root
export MYSQL_PWD=root123
export LOG_LEVEL=debug
export UI_URL=http://127.0.0.1/
export UI_USR=admin
export UI_PWD=Harbor12345
export LOCAL_HARBOR_URL=http://127.0.0.1/
export UI_SECRET=abcdef
#export UI_USR=admin
#export UI_PWD=Harbor12345
export MAX_JOB_WORKERS=1
./jobservice

2
jobservice/populate.sql
View File

@ -1,3 +1,3 @@
use registry;
insert into replication_target (name, url, username, password) values ('test', '192.168.0.2:5000', 'testuser', 'passw0rd');
insert into replication_target (name, url, username, password) values ('test', 'http://10.117.171.31', 'admin', 'Harbor12345');
insert into replication_policy (name, project_id, target_id, enabled, start_time) value ('test_policy', 1, 1, 1, NOW());

2
models/replication_job.go
View File

@ -15,6 +15,8 @@ const (
JobContinue string = "_continue"
RepOpTransfer string = "transfer"
RepOpDelete string = "delete"
//cookie name to contain the UI secret
UISecretCookie string = "uisecret"
)
type RepPolicy struct {

2
service/utils/cache.go → service/cache/cache.go vendored
View File

@ -13,7 +13,7 @@
limitations under the License.
*/
package utils
package cache
import (
"os"

4
service/notification.go
View File

@ -23,7 +23,7 @@ import (
"github.com/vmware/harbor/api"
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/models"
svc_utils "github.com/vmware/harbor/service/utils"
"github.com/vmware/harbor/service/cache"
"github.com/vmware/harbor/utils/log"
"github.com/vmware/harbor/utils/registry"
@ -73,7 +73,7 @@ func (n *NotificationHandler) Post() {
go dao.AccessLog(username, project, repo, repoTag, action)
if action == "push" {
go func() {
err2 := svc_utils.RefreshCatalogCache()
err2 := cache.RefreshCatalogCache()
if err2 != nil {
log.Errorf("Error happens when refreshing cache: %v", err2)
}

29
service/token/token.go
View File

@ -21,7 +21,7 @@ import (
"github.com/vmware/harbor/auth"
"github.com/vmware/harbor/models"
//svc_utils "github.com/vmware/harbor/service/utils"
svc_utils "github.com/vmware/harbor/service/utils"
"github.com/vmware/harbor/utils/log"
"github.com/astaxie/beego"
@ -38,20 +38,27 @@ type Handler struct {
// checkes the permission agains local DB and generates jwt token.
func (h *Handler) Get() {
var username string
request := h.Ctx.Request
log.Infof("request url: %v", request.URL.String())
username, password, _ := request.BasicAuth()
authenticated := authenticate(username, password)
service := h.GetString("service")
scopes := h.GetStrings("scope")
if len(scopes) == 0 && !authenticated {
log.Info("login request with invalid credentials")
h.CustomAbort(http.StatusUnauthorized, "")
}
access := GetResourceActions(scopes)
for _, a := range access {
FilterAccess(username, authenticated, a)
log.Infof("request url: %v", request.URL.String())
if svc_utils.VerifySecret(request) {
log.Debugf("Will grant all access as this request is from job service with legal secret.")
username = "job-service-user"
} else {
username, password, _ := request.BasicAuth()
authenticated := authenticate(username, password)
if len(scopes) == 0 && !authenticated {
log.Info("login request with invalid credentials")
h.CustomAbort(http.StatusUnauthorized, "")
}
for _, a := range access {
FilterAccess(username, authenticated, a)
}
}
h.serveToken(username, service, access)
}

30
service/utils/utils.go Normal file
View File

@ -0,0 +1,30 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package utils
import (
"github.com/vmware/harbor/utils/log"
"net/http"
"os"
)
func VerifySecret(r *http.Request) bool {
secret := os.Getenv("UI_SECRET")
c, err := r.Cookie("uisecret")
if err != nil {
log.Errorf("Failed to get secret cookie, error: %v", err)
}
return c != nil && c.Value == secret
}

14
utils/registry/auth/credential.go
View File

@ -42,3 +42,17 @@ func NewBasicAuthCredential(username, password string) Credential {
func (b *basicAuthCredential) AddAuthorization(req *http.Request) {
req.SetBasicAuth(b.username, b.password)
}
type cookieCredential struct {
cookie *http.Cookie
}
func NewCookieCredential(c *http.Cookie) Credential {
return &cookieCredential{
cookie: c,
}
}
func (c *cookieCredential) AddAuthorization(req *http.Request) {
req.AddCookie(c.cookie)
}