From fbe341bf5d8c449807a81106d9ab912f341da7b4 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 28 Apr 2016 18:49:59 +0800 Subject: [PATCH 01/15] update --- utils/registry/auth/authorizer.go | 6 +- utils/registry/auth/tokenhandler.go | 106 +++++++++------------------- utils/registry/registry.go | 43 +++++++---- utils/registry/repository.go | 36 ++-------- 4 files changed, 71 insertions(+), 120 deletions(-) diff --git a/utils/registry/auth/authorizer.go b/utils/registry/auth/authorizer.go index e560355ae..af3a604c3 100644 --- a/utils/registry/auth/authorizer.go +++ b/utils/registry/auth/authorizer.go @@ -23,8 +23,8 @@ import ( // Handler authorizes requests according to the schema type Handler interface { - // Schema : basic, bearer - Schema() string + // Scheme : basic, bearer + Scheme() string //AuthorizeRequest adds basic auth or token auth to the header of request AuthorizeRequest(req *http.Request, params map[string]string) error } @@ -47,7 +47,7 @@ func NewRequestAuthorizer(handlers []Handler, challenges []au.Challenge) *Reques func (r *RequestAuthorizer) ModifyRequest(req *http.Request) error { for _, handler := range r.handlers { for _, challenge := range r.challenges { - if handler.Schema() == challenge.Scheme { + if handler.Scheme() == challenge.Scheme { if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil { return err } diff --git a/utils/registry/auth/tokenhandler.go b/utils/registry/auth/tokenhandler.go index 3e0b6c30f..8cdd808a8 100644 --- a/utils/registry/auth/tokenhandler.go +++ b/utils/registry/auth/tokenhandler.go @@ -39,80 +39,59 @@ func (s *scope) string() string { return fmt.Sprintf("%s:%s:%s", s.Type, s.Name, strings.Join(s.Actions, ",")) } -type token struct { - token string - expiresIn time.Time -} - -type tokenGenerator func(realm, service string, scopes []string) (*token, error) +type tokenGenerator func(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) type tokenHandler struct { - scope *scope - cache map[string]*token - tg tokenGenerator + scope *scope + tg tokenGenerator + cache string // cached token + expiresIn int // The duration in seconds since the token was issued that it will remain valid + issuedAt *time.Time // The RFC3339-serialized UTC standard time at which a given token was issued } -// Schema returns the schema that the handler can handle -func (t *tokenHandler) Schema() string { +// Scheme returns the scheme that the handler can handle +func (t *tokenHandler) Scheme() string { return "bearer" } // AuthorizeRequest will add authorization header which contains a token before the request is sent func (t *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { - var token string var scopes []*scope // TODO handle additional scope: xxx.xxx.xxx?from=repo scopes = append(scopes, t.scope) - key := cacheKey(scopes) - value, ok := t.cache[key] - var expired bool + expired := true - if ok { - expired = value.expiresIn.Before(time.Now()) + if t.expiresIn != 0 && t.issuedAt != nil { + expired = t.issuedAt.Add(time.Duration(t.expiresIn) * time.Second).Before(time.Now().UTC()) } - if ok && !expired { - token = value.token - log.Debugf("get token from cache: %s", key) - } else { - if ok && expired { - delete(t.cache, key) - log.Debugf("token is expired, remove from cache: %s", key) - } - + if expired { scopeStrs := []string{} for _, scope := range scopes { scopeStrs = append(scopeStrs, scope.string()) } - tk, err := t.tg(params["realm"], params["service"], scopeStrs) + token, expiresIn, issuedAt, err := t.tg(params["realm"], params["service"], scopeStrs) if err != nil { return err } - token = tk.token - t.cache[key] = tk - log.Debugf("add token to cache: %s", key) + t.cache = token + t.expiresIn = expiresIn + t.issuedAt = issuedAt } - req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token)) + if !expired { + log.Debug("get token from cache") + } + + req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", t.cache)) log.Debugf("add token to request: %s %s", req.Method, req.URL.String()) return nil } -// cacheKey returns a string which can identify the scope array and can be used as the key in cache map -func cacheKey(scopes []*scope) string { - key := "" - for _, scope := range scopes { - key = key + scope.string() + "|" - } - key = strings.TrimRight(key, "|") - - return key -} - type standardTokenHandler struct { tokenHandler client *http.Client @@ -135,16 +114,15 @@ func NewStandardTokenHandler(credential Credential, scopeType, scopeName string, Name: scopeName, Actions: scopeActions, } - handler.cache = make(map[string]*token, 1) handler.tg = handler.generateToken return handler } -func (s *standardTokenHandler) generateToken(realm, service string, scopes []string) (*token, error) { +func (s *standardTokenHandler) generateToken(realm, service string, scopes []string) (string, int, *time.Time, error) { u, err := url.Parse(realm) if err != nil { - return nil, err + return "", 0, nil, err } q := u.Query() q.Add("service", service) @@ -154,44 +132,39 @@ func (s *standardTokenHandler) generateToken(realm, service string, scopes []str u.RawQuery = q.Encode() r, err := http.NewRequest("GET", u.String(), nil) if err != nil { - return nil, err + return "", 0, nil, err } s.credential.AddAuthorization(r) resp, err := s.client.Do(r) if err != nil { - return nil, err + return "", 0, nil, err } defer resp.Body.Close() b, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, err + return "", 0, nil, err } if resp.StatusCode != http.StatusOK { - return nil, registry_errors.Error{ + return "", 0, nil, registry_errors.Error{ StatusCode: resp.StatusCode, Message: string(b), } } + // TODO tk := struct { Token string `json:"token"` }{} if err = json.Unmarshal(b, &tk); err != nil { - return nil, err - } - - t := &token{ - token: tk.Token, - // TODO handle the expires time - expiresIn: time.Now().Add(5 * time.Minute), + return "", 0, nil, err } log.Debug("get token from token server") - return t, nil + return tk.Token, 0, nil, nil } type usernameTokenHandler struct { @@ -211,26 +184,15 @@ func NewUsernameTokenHandler(username string, scopeType, scopeName string, scope Name: scopeName, Actions: scopeActions, } - handler.cache = make(map[string]*token, 1) handler.tg = handler.generateToken return handler } -func (u *usernameTokenHandler) generateToken(realm, service string, scopes []string) (*token, error) { - tk, err := token_util.GenTokenForUI(u.username, service, scopes) - if err != nil { - return nil, err - } - - t := &token{ - token: tk, - // TODO handle the expires time - expiresIn: time.Now().Add(5 * time.Minute), - } - +func (u *usernameTokenHandler) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) { + // TODO + token, err = token_util.GenTokenForUI(u.username, service, scopes) log.Debug("get token by calling GenTokenForUI directly") - - return t, nil + return } diff --git a/utils/registry/registry.go b/utils/registry/registry.go index 201e29f20..62118ecbc 100644 --- a/utils/registry/registry.go +++ b/utils/registry/registry.go @@ -63,25 +63,14 @@ func NewRegistryWithUsername(endpoint, username string) (*Registry, error) { return nil, err } - resp, err := http.Get(buildPingURL(endpoint)) + client, err := newClient(endpoint, username, nil, "registry", "catalog", "*") if err != nil { return nil, err } - var handlers []auth.Handler - handler := auth.NewUsernameTokenHandler(username, "registry", "catalog", "*") - handlers = append(handlers, handler) - - challenges := auth.ParseChallengeFromResponse(resp) - authorizer := auth.NewRequestAuthorizer(handlers, challenges) - - transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer}) - registry := &Registry{ Endpoint: u, - client: &http.Client{ - Transport: transport, - }, + client: client, } return registry, nil @@ -131,3 +120,31 @@ func (r *Registry) Catalog() ([]string, error) { func buildCatalogURL(endpoint string) string { return fmt.Sprintf("%s/v2/_catalog", endpoint) } + +func newClient(endpoint, username string, credential auth.Credential, + scopeType, scopeName string, scopeActions ...string) (*http.Client, error) { + + endpoint = strings.TrimRight(endpoint, "/") + resp, err := http.Get(buildPingURL(endpoint)) + if err != nil { + return nil, err + } + + var handlers []auth.Handler + var handler auth.Handler + if credential != nil { + handler = auth.NewStandardTokenHandler(credential, scopeType, scopeName, scopeActions...) + } else { + handler = auth.NewUsernameTokenHandler(username, scopeType, scopeName, scopeActions...) + } + + handlers = append(handlers, handler) + + challenges := auth.ParseChallengeFromResponse(resp) + authorizer := auth.NewRequestAuthorizer(handlers, challenges) + + transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer}) + return &http.Client{ + Transport: transport, + }, nil +} diff --git a/utils/registry/repository.go b/utils/registry/repository.go index 121b26741..04d8f9fdd 100644 --- a/utils/registry/repository.go +++ b/utils/registry/repository.go @@ -71,26 +71,12 @@ func NewRepositoryWithCredential(name, endpoint string, credential auth.Credenti return nil, err } - resp, err := http.Get(buildPingURL(endpoint)) - if err != nil { - return nil, err - } - - var handlers []auth.Handler - handler := auth.NewStandardTokenHandler(credential, "repository", name, "pull", "push") - handlers = append(handlers, handler) - - challenges := auth.ParseChallengeFromResponse(resp) - authorizer := auth.NewRequestAuthorizer(handlers, challenges) - - transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer}) + client, err := newClient(endpoint, "", credential, "repository", name, "pull", "push") repository := &Repository{ Name: name, Endpoint: u, - client: &http.Client{ - Transport: transport, - }, + client: client, } log.Debugf("initialized a repository client with credential: %s %s", endpoint, name) @@ -109,26 +95,12 @@ func NewRepositoryWithUsername(name, endpoint, username string) (*Repository, er return nil, err } - resp, err := http.Get(buildPingURL(endpoint)) - if err != nil { - return nil, err - } - - var handlers []auth.Handler - handler := auth.NewUsernameTokenHandler(username, "repository", name, "pull", "push") - handlers = append(handlers, handler) - - challenges := auth.ParseChallengeFromResponse(resp) - authorizer := auth.NewRequestAuthorizer(handlers, challenges) - - transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer}) + client, err := newClient(endpoint, username, nil, "repository", name, "pull", "push") repository := &Repository{ Name: name, Endpoint: u, - client: &http.Client{ - Transport: transport, - }, + client: client, } log.Debugf("initialized a repository client with username: %s %s", endpoint, name) From 545ca4135cbb6da4490bbd61e5afecbd3b10473e Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 29 Apr 2016 14:59:00 +0800 Subject: [PATCH 02/15] token expires --- service/token/authutils.go | 29 ++++++++++--------- service/token/token.go | 7 +++-- utils/registry/auth/authorizer.go | 3 +- utils/registry/auth/credential.go | 1 + utils/registry/auth/tokenhandler.go | 45 ++++++++++++++++++++++------- 5 files changed, 58 insertions(+), 27 deletions(-) diff --git a/service/token/authutils.go b/service/token/authutils.go index bb9d8ad92..bfad2ea90 100644 --- a/service/token/authutils.go +++ b/service/token/authutils.go @@ -108,7 +108,7 @@ func FilterAccess(username string, authenticated bool, a *token.ResourceActions) } // GenTokenForUI is for the UI process to call, so it won't establish a https connection from UI to proxy. -func GenTokenForUI(username string, service string, scopes []string) (string, error) { +func GenTokenForUI(username string, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) { access := GetResourceActions(scopes) for _, a := range access { FilterAccess(username, true, a) @@ -117,22 +117,22 @@ func GenTokenForUI(username string, service string, scopes []string) (string, er } // MakeToken makes a valid jwt token based on parms. -func MakeToken(username, service string, access []*token.ResourceActions) (string, error) { +func MakeToken(username, service string, access []*token.ResourceActions) (token string, expiresIn int, issuedAt *time.Time, err error) { pk, err := libtrust.LoadKeyFile(privateKey) if err != nil { - return "", err + return "", 0, nil, err } - tk, err := makeTokenCore(issuer, username, service, expiration, access, pk) + tk, expiresIn, issuedAt, err := makeTokenCore(issuer, username, service, expiration, access, pk) if err != nil { - return "", err + return "", 0, nil, err } rs := fmt.Sprintf("%s.%s", tk.Raw, base64UrlEncode(tk.Signature)) - return rs, nil + return rs, expiresIn, issuedAt, nil } //make token core func makeTokenCore(issuer, subject, audience string, expiration int, - access []*token.ResourceActions, signingKey libtrust.PrivateKey) (*token.Token, error) { + access []*token.ResourceActions, signingKey libtrust.PrivateKey) (t *token.Token, expiresIn int, issuedAt *time.Time, err error) { joseHeader := &token.Header{ Type: "JWT", @@ -142,10 +142,12 @@ func makeTokenCore(issuer, subject, audience string, expiration int, jwtID, err := randString(16) if err != nil { - return nil, fmt.Errorf("Error to generate jwt id: %s", err) + return nil, 0, nil, fmt.Errorf("Error to generate jwt id: %s", err) } - now := time.Now() + now := time.Now().UTC() + issuedAt = &now + expiresIn = expiration * 60 claimSet := &token.ClaimSet{ Issuer: issuer, @@ -161,10 +163,10 @@ func makeTokenCore(issuer, subject, audience string, expiration int, var joseHeaderBytes, claimSetBytes []byte if joseHeaderBytes, err = json.Marshal(joseHeader); err != nil { - return nil, fmt.Errorf("unable to marshal jose header: %s", err) + return nil, 0, nil, fmt.Errorf("unable to marshal jose header: %s", err) } if claimSetBytes, err = json.Marshal(claimSet); err != nil { - return nil, fmt.Errorf("unable to marshal claim set: %s", err) + return nil, 0, nil, fmt.Errorf("unable to marshal claim set: %s", err) } encodedJoseHeader := base64UrlEncode(joseHeaderBytes) @@ -173,12 +175,13 @@ func makeTokenCore(issuer, subject, audience string, expiration int, var signatureBytes []byte if signatureBytes, _, err = signingKey.Sign(strings.NewReader(payload), crypto.SHA256); err != nil { - return nil, fmt.Errorf("unable to sign jwt payload: %s", err) + return nil, 0, nil, fmt.Errorf("unable to sign jwt payload: %s", err) } signature := base64UrlEncode(signatureBytes) tokenString := fmt.Sprintf("%s.%s", payload, signature) - return token.NewToken(tokenString) + t, err = token.NewToken(tokenString) + return } func randString(length int) (string, error) { diff --git a/service/token/token.go b/service/token/token.go index c12e4d7ad..1b4dbf927 100644 --- a/service/token/token.go +++ b/service/token/token.go @@ -17,6 +17,7 @@ package token import ( "net/http" + "time" "github.com/vmware/harbor/auth" "github.com/vmware/harbor/models" @@ -59,14 +60,16 @@ func (h *Handler) Get() { func (h *Handler) serveToken(username, service string, access []*token.ResourceActions) { writer := h.Ctx.ResponseWriter //create token - rawToken, err := MakeToken(username, service, access) + rawToken, expiresIn, issuedAt, err := MakeToken(username, service, access) if err != nil { log.Errorf("Failed to make token, error: %v", err) writer.WriteHeader(http.StatusInternalServerError) return } - tk := make(map[string]string) + tk := make(map[string]interface{}) tk["token"] = rawToken + tk["expires_in"] = expiresIn + tk["issued_at"] = issuedAt.Format(time.RFC3339) h.Data["json"] = tk h.ServeJSON() } diff --git a/utils/registry/auth/authorizer.go b/utils/registry/auth/authorizer.go index af3a604c3..cea731246 100644 --- a/utils/registry/auth/authorizer.go +++ b/utils/registry/auth/authorizer.go @@ -29,7 +29,8 @@ type Handler interface { AuthorizeRequest(req *http.Request, params map[string]string) error } -// RequestAuthorizer holds a handler list, which will authorize request +// RequestAuthorizer holds a handler list, which will authorize request. +// Implements interface RequestModifier type RequestAuthorizer struct { handlers []Handler challenges []au.Challenge diff --git a/utils/registry/auth/credential.go b/utils/registry/auth/credential.go index 68143e2f9..6dd867136 100644 --- a/utils/registry/auth/credential.go +++ b/utils/registry/auth/credential.go @@ -25,6 +25,7 @@ type Credential interface { AddAuthorization(req *http.Request) } +// Implements interface Credential type basicAuthCredential struct { username string password string diff --git a/utils/registry/auth/tokenhandler.go b/utils/registry/auth/tokenhandler.go index 8cdd808a8..98954fae6 100644 --- a/utils/registry/auth/tokenhandler.go +++ b/utils/registry/auth/tokenhandler.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "net/http" "net/url" + "strconv" "strings" "time" @@ -41,6 +42,7 @@ func (s *scope) string() string { type tokenGenerator func(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) +// Implements interface Handler type tokenHandler struct { scope *scope tg tokenGenerator @@ -92,6 +94,7 @@ func (t *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]str return nil } +// Implements interface Handler type standardTokenHandler struct { tokenHandler client *http.Client @@ -119,10 +122,10 @@ func NewStandardTokenHandler(credential Credential, scopeType, scopeName string, return handler } -func (s *standardTokenHandler) generateToken(realm, service string, scopes []string) (string, int, *time.Time, error) { +func (s *standardTokenHandler) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) { u, err := url.Parse(realm) if err != nil { - return "", 0, nil, err + return } q := u.Query() q.Add("service", service) @@ -132,41 +135,61 @@ func (s *standardTokenHandler) generateToken(realm, service string, scopes []str u.RawQuery = q.Encode() r, err := http.NewRequest("GET", u.String(), nil) if err != nil { - return "", 0, nil, err + return } s.credential.AddAuthorization(r) resp, err := s.client.Do(r) if err != nil { - return "", 0, nil, err + return } defer resp.Body.Close() b, err := ioutil.ReadAll(resp.Body) if err != nil { - return "", 0, nil, err + return } if resp.StatusCode != http.StatusOK { - return "", 0, nil, registry_errors.Error{ + err = registry_errors.Error{ StatusCode: resp.StatusCode, Message: string(b), } + return } - // TODO tk := struct { - Token string `json:"token"` + Token string `json:"token"` + ExpiresIn string `json:"expires_in"` + IssuedAt string `json:"issued_at"` }{} if err = json.Unmarshal(b, &tk); err != nil { - return "", 0, nil, err + return + } + + token = tk.Token + + expiresIn, err = strconv.Atoi(tk.ExpiresIn) + if err != nil { + expiresIn = 0 + log.Errorf("error occurred while converting expires_in: %v", err) + err = nil + } else { + t, err := time.Parse(time.RFC3339, tk.IssuedAt) + if err != nil { + log.Errorf("error occurred while parsing issued_at: %v", err) + err = nil + } else { + issuedAt = &t + } } log.Debug("get token from token server") - return tk.Token, 0, nil, nil + return } +// Implements interface Handler type usernameTokenHandler struct { tokenHandler username string @@ -192,7 +215,7 @@ func NewUsernameTokenHandler(username string, scopeType, scopeName string, scope func (u *usernameTokenHandler) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) { // TODO - token, err = token_util.GenTokenForUI(u.username, service, scopes) + token, expiresIn, issuedAt, err = token_util.GenTokenForUI(u.username, service, scopes) log.Debug("get token by calling GenTokenForUI directly") return } From bf53ca9a47dba8c9917423bbc5f655765846ecd5 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 29 Apr 2016 16:59:54 +0800 Subject: [PATCH 03/15] handler from=repo --- service/utils/cache.go | 4 ++-- utils/registry/auth/tokenhandler.go | 32 +++++++++++++++++++++-------- utils/registry/registry.go | 2 ++ utils/registry/repository.go | 2 +- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/service/utils/cache.go b/service/utils/cache.go index bbb9bd0b8..a19753f59 100644 --- a/service/utils/cache.go +++ b/service/utils/cache.go @@ -75,14 +75,14 @@ func RefreshCatalogCache() error { rc, err = registry.NewRepositoryWithUsername(repo, endpoint, username) if err != nil { log.Errorf("error occurred while initializing repository client used by cache: %s %v", repo, err) - return err + continue } repositoryClients[repo] = rc } tags, err := rc.ListTag() if err != nil { log.Errorf("error occurred while list tag for %s: %v", repo, err) - return err + continue } if len(tags) != 0 { diff --git a/utils/registry/auth/tokenhandler.go b/utils/registry/auth/tokenhandler.go index 98954fae6..f0ee50b33 100644 --- a/utils/registry/auth/tokenhandler.go +++ b/utils/registry/auth/tokenhandler.go @@ -59,8 +59,20 @@ func (t *tokenHandler) Scheme() string { // AuthorizeRequest will add authorization header which contains a token before the request is sent func (t *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { var scopes []*scope + var token string - // TODO handle additional scope: xxx.xxx.xxx?from=repo + hasFrom := false + from := req.URL.Query().Get("from") + if len(from) != 0 { + s := &scope{ + Type: "repository", + Name: from, + Actions: []string{"pull"}, + } + scopes = append(scopes, s) + // do not cache the token if "from" appears + hasFrom = true + } scopes = append(scopes, t.scope) @@ -70,7 +82,7 @@ func (t *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]str expired = t.issuedAt.Add(time.Duration(t.expiresIn) * time.Second).Before(time.Now().UTC()) } - if expired { + if expired || hasFrom { scopeStrs := []string{} for _, scope := range scopes { scopeStrs = append(scopeStrs, scope.string()) @@ -79,16 +91,19 @@ func (t *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]str if err != nil { return err } - t.cache = token - t.expiresIn = expiresIn - t.issuedAt = issuedAt - } - if !expired { + if !hasFrom { + t.cache = token + t.expiresIn = expiresIn + t.issuedAt = issuedAt + log.Debug("add token to cache") + } + } else { + token = t.cache log.Debug("get token from cache") } - req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", t.cache)) + req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token)) log.Debugf("add token to request: %s %s", req.Method, req.URL.String()) return nil @@ -214,7 +229,6 @@ func NewUsernameTokenHandler(username string, scopeType, scopeName string, scope } func (u *usernameTokenHandler) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) { - // TODO token, expiresIn, issuedAt, err = token_util.GenTokenForUI(u.username, service, scopes) log.Debug("get token by calling GenTokenForUI directly") return diff --git a/utils/registry/registry.go b/utils/registry/registry.go index 62118ecbc..1ee01892e 100644 --- a/utils/registry/registry.go +++ b/utils/registry/registry.go @@ -73,6 +73,8 @@ func NewRegistryWithUsername(endpoint, username string) (*Registry, error) { client: client, } + log.Debugf("initialized a registry client with username: %s %s", endpoint, username) + return registry, nil } diff --git a/utils/registry/repository.go b/utils/registry/repository.go index 04d8f9fdd..0a1b9f096 100644 --- a/utils/registry/repository.go +++ b/utils/registry/repository.go @@ -103,7 +103,7 @@ func NewRepositoryWithUsername(name, endpoint, username string) (*Repository, er client: client, } - log.Debugf("initialized a repository client with username: %s %s", endpoint, name) + log.Debugf("initialized a repository client with username: %s %s", endpoint, name, username) return repository, nil } From 9fbebad626005e3674aa871eb44675d5b43767be Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 29 Apr 2016 17:03:11 +0800 Subject: [PATCH 04/15] set cgo as the default DNS resolver --- Deploy/docker-compose.yml | 2 ++ Deploy/templates/ui/env | 1 + 2 files changed, 3 insertions(+) diff --git a/Deploy/docker-compose.yml b/Deploy/docker-compose.yml index 2e9147ee1..2dc09b620 100644 --- a/Deploy/docker-compose.yml +++ b/Deploy/docker-compose.yml @@ -11,6 +11,8 @@ services: volumes: - /data/registry:/storage - ./config/registry/:/etc/registry/ + environment: + - GODEBUG=netdns=cgo ports: - 5001:5001 command: diff --git a/Deploy/templates/ui/env b/Deploy/templates/ui/env index 383e5f15a..fe91f5965 100644 --- a/Deploy/templates/ui/env +++ b/Deploy/templates/ui/env @@ -12,3 +12,4 @@ LDAP_URL=$ldap_url LDAP_BASE_DN=$ldap_basedn SELF_REGISTRATION=$self_registration LOG_LEVEL=debug +GODEBUG=netdns=cgo \ No newline at end of file From 0f0bc86aded2924bb5dcd1c21406cc4c8c003ff0 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 29 Apr 2016 17:22:17 +0800 Subject: [PATCH 05/15] add logs --- service/token/authutils.go | 2 ++ service/token/token.go | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/service/token/authutils.go b/service/token/authutils.go index bfad2ea90..3ea15c294 100644 --- a/service/token/authutils.go +++ b/service/token/authutils.go @@ -39,6 +39,7 @@ const ( // GetResourceActions ... func GetResourceActions(scopes []string) []*token.ResourceActions { + log.Debugf("scopes: %+v", scopes) var res []*token.ResourceActions for _, s := range scopes { if s == "" { @@ -59,6 +60,7 @@ func GetResourceActions(scopes []string) []*token.ResourceActions { func FilterAccess(username string, authenticated bool, a *token.ResourceActions) { if a.Type == "registry" && a.Name == "catalog" { + log.Infof("current access, type: %s, name:%s, actions:%v \n", a.Type, a.Name, a.Actions) return } diff --git a/service/token/token.go b/service/token/token.go index 1b4dbf927..2d4c77673 100644 --- a/service/token/token.go +++ b/service/token/token.go @@ -44,7 +44,6 @@ func (h *Handler) Get() { authenticated := authenticate(username, password) service := h.GetString("service") scopes := h.GetStrings("scope") - log.Debugf("scopes: %+v", scopes) if len(scopes) == 0 && !authenticated { log.Info("login request with invalid credentials") From f2a180e6173db889a6031f4a49e5447a5c19d563 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 29 Apr 2016 17:46:00 +0800 Subject: [PATCH 06/15] update --- utils/registry/auth/tokenhandler.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/registry/auth/tokenhandler.go b/utils/registry/auth/tokenhandler.go index f0ee50b33..c33627491 100644 --- a/utils/registry/auth/tokenhandler.go +++ b/utils/registry/auth/tokenhandler.go @@ -87,10 +87,11 @@ func (t *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]str for _, scope := range scopes { scopeStrs = append(scopeStrs, scope.string()) } - token, expiresIn, issuedAt, err := t.tg(params["realm"], params["service"], scopeStrs) + to, expiresIn, issuedAt, err := t.tg(params["realm"], params["service"], scopeStrs) if err != nil { return err } + token = to if !hasFrom { t.cache = token From 2a461ee53e9b8ea790008374ba98b3e21e1d6bea Mon Sep 17 00:00:00 2001 From: wy65701436 Date: Mon, 2 May 2016 20:58:50 -0700 Subject: [PATCH 07/15] update API for creating user, updating password and comment post return. --- docs/swagger.yaml | 94 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 404a0b431..588565b55 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -109,7 +109,7 @@ paths: tags: - Products responses: - 200: + 201: description: Project created successfully. 400: description: Unsatisfied with constraints of the project creation. @@ -383,6 +383,56 @@ paths: description: User does not have permission of admin role. 500: description: Unexpected internal errors. + post: + summary: Creates a new user account. + description: | + This endpoint is to create a user if the user does not already exist. + parameters: + - name: username + in: body + type: string + format: string + required: true + description: The username of the new created user. Maximum of 20 + characters. + - name: password + in: body + type: string + format: string + required: true + description: The password of the new created user. Maximum of 20 + characters. + - name: email + in: body + type: string + format: string + required: true + description: Publicly visible email address of the new created user. + - name: realname + in: body + type: string + format: string + required: true + description: Full name associated with the new created user. + Maximum of 20 characters. + - name: comment + in: body + type: string + format: string + required: false + description: Comments associated with the new created user. + Maximum of 30 characters. + tags: + - Products + responses: + 201: + description: User created successfully. + 400: + description: Unsatisfied with constraints of the user creation. + 403: + description: User registration can only be used by admin role user when self-registration is off. + 500: + description: Unexpected internal errors. /users/{user_id}: put: summary: Update a registered user to change to be an administrator of Harbor. @@ -438,6 +488,48 @@ paths: description: User ID does not exist. 500: description: Unexpected internal errors. + /users/{user_id}/password: + put: + summary: Change the password on a user that already exists. + description: | + This endpoint is for user to update password. Users with the admin + role can change any user's password. Guest users can change only + their own password. + parameters: + - name: user_id + in: path + type: integer + format: int32 + required: true + description: Registered user ID. + parameters: + - name: old_password + in: body + type: string + format: string + required: true + description: The user's existing password. + parameters: + - name: new_password + in: body + type: string + format: string + required: true + description: New password for marking as to be updated. + tags: + - Products + responses: + 200: + description: Updated password successfully. + 400: + description: Invalid user ID; Old password is blank; New password + is blank. + 401: + description: Old password is not correct. + 403: + description: Guests can only change their own account. + 500: + description: Unexpected internal errors. /repositories: get: summary: Get repositories accompany with relevant project and repo name. From ff429234015fc440d98939f3f18ae5e6dab5a80c Mon Sep 17 00:00:00 2001 From: wy65701436 Date: Mon, 2 May 2016 21:05:06 -0700 Subject: [PATCH 08/15] update API for creating user, updating password and comment post return. --- docs/swagger.yaml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 588565b55..aeb95ea92 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -393,15 +393,13 @@ paths: type: string format: string required: true - description: The username of the new created user. Maximum of 20 - characters. + description: The username of the new created user. Maximum of 20 characters. - name: password in: body type: string format: string required: true - description: The password of the new created user. Maximum of 20 - characters. + description: The password of the new created user. Maximum of 20 characters. - name: email in: body type: string @@ -413,15 +411,13 @@ paths: type: string format: string required: true - description: Full name associated with the new created user. - Maximum of 20 characters. + description: Full name associated with the new created user. Maximum of 20 characters. - name: comment in: body type: string format: string required: false - description: Comments associated with the new created user. - Maximum of 30 characters. + description: Comments associated with the new created user. Maximum of 30 characters. tags: - Products responses: @@ -522,8 +518,7 @@ paths: 200: description: Updated password successfully. 400: - description: Invalid user ID; Old password is blank; New password - is blank. + description: Invalid user ID; Old password is blank; New password is blank. 401: description: Old password is not correct. 403: From 73fe2a30d6b50e87ec3792cbb9da9543c588e728 Mon Sep 17 00:00:00 2001 From: wy65701436 Date: Mon, 2 May 2016 22:43:50 -0700 Subject: [PATCH 09/15] remove duplicate parameters lines. --- docs/swagger.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index aeb95ea92..8c648819f 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -498,14 +498,12 @@ paths: format: int32 required: true description: Registered user ID. - parameters: - name: old_password in: body type: string format: string required: true description: The user's existing password. - parameters: - name: new_password in: body type: string From 83ba19516647e7be55a560ed0034f3a587f04aed Mon Sep 17 00:00:00 2001 From: hmwenchen Date: Tue, 3 May 2016 16:54:01 +0800 Subject: [PATCH 10/15] Add unit test for AccessLog --- dao/accesslog.go | 8 +++++++ dao/dao_test.go | 62 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/dao/accesslog.go b/dao/accesslog.go index dbf447353..908fb9280 100644 --- a/dao/accesslog.go +++ b/dao/accesslog.go @@ -62,6 +62,14 @@ func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) { sql += ` and u.username like ? ` queryParam = append(queryParam, accessLog.Username) } + if accessLog.RepoName != "" { + sql += ` and a.repo_name = ? ` + queryParam = append(queryParam, accessLog.RepoName) + } + if accessLog.RepoTag != "" { + sql += ` and a.repo_tag = ? ` + queryParam = append(queryParam, accessLog.RepoTag) + } if accessLog.Keywords != "" { sql += ` and a.operation in ( ` keywordList := strings.Split(accessLog.Keywords, "/") diff --git a/dao/dao_test.go b/dao/dao_test.go index da0ad6993..99ece3348 100644 --- a/dao/dao_test.go +++ b/dao/dao_test.go @@ -102,6 +102,8 @@ func clearUp(username string) { const username string = "Tester01" const projectName string = "test_project" +const repoTag string = "test1.1" +const repoTag2 string = "test1.2" const SysAdmin int = 1 const projectAdmin int = 2 const developer int = 3 @@ -419,6 +421,66 @@ func TestGetAccessLog(t *testing.T) { } } +func TestAddAccessLog(t *testing.T) { + var err error + var accessLogList []models.AccessLog + accessLog := models.AccessLog{ + UserID: currentUser.UserID, + ProjectID: currentProject.ProjectID, + RepoName: currentProject.Name + "/", + RepoTag: repoTag, + GUID: "N/A", + Operation: "create", + OpTime: time.Now(), + } + err = AddAccessLog(accessLog) + if err != nil { + t.Errorf("Error occurred in AddAccessLog: %v", err) + } + accessLogList, err = GetAccessLogs(accessLog) + if err != nil { + t.Errorf("Error occurred in GetAccessLog: %v", err) + } + if len(accessLogList) != 1 { + t.Errorf("The length of accesslog list should be 1, actual: %d", len(accessLogList)) + } + if accessLogList[0].RepoName != projectName+"/" { + t.Errorf("The project name does not match, expected: %s, actual: %s", projectName+"/", accessLogList[0].RepoName) + } + if accessLogList[0].RepoTag != repoTag { + t.Errorf("The repo tag does not match, expected: %s, actual: %s", repoTag, accessLogList[0].RepoTag) + } +} + +func TestAccessLog(t *testing.T) { + var err error + var accessLogList []models.AccessLog + accessLog := models.AccessLog{ + UserID: currentUser.UserID, + ProjectID: currentProject.ProjectID, + RepoName: currentProject.Name + "/", + RepoTag: repoTag2, + Operation: "create", + } + err = AccessLog(currentUser.Username, currentProject.Name, currentProject.Name+"/", repoTag2, "create") + if err != nil { + t.Errorf("Error occurred in AccessLog: %v", err) + } + accessLogList, err = GetAccessLogs(accessLog) + if err != nil { + t.Errorf("Error occurred in GetAccessLog: %v", err) + } + if len(accessLogList) != 1 { + t.Errorf("The length of accesslog list should be 1, actual: %d", len(accessLogList)) + } + if accessLogList[0].RepoName != projectName+"/" { + t.Errorf("The project name does not match, expected: %s, actual: %s", projectName+"/", accessLogList[0].RepoName) + } + if accessLogList[0].RepoTag != repoTag2 { + t.Errorf("The repo tag does not match, expected: %s, actual: %s", repoTag2, accessLogList[0].RepoTag) + } +} + func TestProjectExists(t *testing.T) { var exists bool var err error From c7da848dd592d421022ef58e1ff3eb59474d110c Mon Sep 17 00:00:00 2001 From: wy65701436 Date: Tue, 3 May 2016 18:27:07 -0700 Subject: [PATCH 11/15] pass on the swagger editor check. --- docs/swagger.yaml | 61 +++++++++++++++-------------------------------- 1 file changed, 19 insertions(+), 42 deletions(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 8c648819f..88bbb78ec 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -388,36 +388,12 @@ paths: description: | This endpoint is to create a user if the user does not already exist. parameters: - - name: username + - name: user in: body - type: string - format: string + description: New created user. required: true - description: The username of the new created user. Maximum of 20 characters. - - name: password - in: body - type: string - format: string - required: true - description: The password of the new created user. Maximum of 20 characters. - - name: email - in: body - type: string - format: string - required: true - description: Publicly visible email address of the new created user. - - name: realname - in: body - type: string - format: string - required: true - description: Full name associated with the new created user. Maximum of 20 characters. - - name: comment - in: body - type: string - format: string - required: false - description: Comments associated with the new created user. Maximum of 30 characters. + schema: + $ref: '#/definitions/User' tags: - Products responses: @@ -488,9 +464,7 @@ paths: put: summary: Change the password on a user that already exists. description: | - This endpoint is for user to update password. Users with the admin - role can change any user's password. Guest users can change only - their own password. + This endpoint is for user to update password. Users with the admin role can change any user's password. Guest users can change only their own password. parameters: - name: user_id in: path @@ -498,18 +472,12 @@ paths: format: int32 required: true description: Registered user ID. - - name: old_password + - name: password in: body - type: string - format: string + description: New updated password. required: true - description: The user's existing password. - - name: new_password - in: body - type: string - format: string - required: true - description: New password for marking as to be updated. + schema: + $ref: '#/definitions/Password' tags: - Products responses: @@ -522,7 +490,7 @@ paths: 403: description: Guests can only change their own account. 500: - description: Unexpected internal errors. + description: Unexpected internal errors. /repositories: get: summary: Get repositories accompany with relevant project and repo name. @@ -725,6 +693,15 @@ definitions: deleted: type: integer format: int32 + Password: + type: object + properties: + old_password: + type: string + description: The user's existing password. + new_password: + type: string + description: New password for marking as to be updated. AccessLog: type: object properties: From f4f716ad31c3918a1f54593404612d56798951bd Mon Sep 17 00:00:00 2001 From: wy65701436 Date: Tue, 3 May 2016 18:39:37 -0700 Subject: [PATCH 12/15] pass on the swagger editor check. --- docs/swagger.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 88bbb78ec..98bc392b1 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -474,7 +474,7 @@ paths: description: Registered user ID. - name: password in: body - description: New updated password. + description: Password for updating. required: true schema: $ref: '#/definitions/Password' From 36160ea64c441cd7ae7e7963275c242f1e7df4a0 Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Wed, 4 May 2016 14:18:52 +0800 Subject: [PATCH 13/15] bump to golang 1.6.2 image to fix compilation error --- Dockerfile.ui | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile.ui b/Dockerfile.ui index 459158279..c5b087bd3 100644 --- a/Dockerfile.ui +++ b/Dockerfile.ui @@ -1,4 +1,4 @@ -FROM golang:1.5.1 +FROM golang:1.6.2 MAINTAINER jiangd@vmware.com @@ -11,7 +11,6 @@ COPY . /go/src/github.com/vmware/harbor COPY ./vendor/golang.org /go/src/golang.org WORKDIR /go/src/github.com/vmware/harbor/ui -ENV GO15VENDOREXPERIMENT 1 RUN go get -d github.com/docker/distribution \ && go get -d github.com/docker/libtrust \ && go get -d github.com/go-sql-driver/mysql \ From a5cb8e16eb277af3d6f4431a3a1a9f161ddf4e1e Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Wed, 4 May 2016 14:25:54 +0800 Subject: [PATCH 14/15] bump up go version in travis.yml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6e536c507..aff9e1f81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,14 @@ language: go go: - - 1.5.3 + - 1.6.2 go_import_path: github.com/vmware/harbor service: - mysql -env: GO15VENDOREXPERIMENT=1 DB_HOST=127.0.0.1 DB_PORT=3306 DB_USR=root DB_PWD= +env: DB_HOST=127.0.0.1 DB_PORT=3306 DB_USR=root DB_PWD= install: - sudo apt-get update && sudo apt-get install -y libldap2-dev From b68f2e1d224102df041110d1db7eed03c8c31acb Mon Sep 17 00:00:00 2001 From: wy65701436 Date: Tue, 3 May 2016 23:43:37 -0700 Subject: [PATCH 15/15] pass on the swagger parser check. --- docs/swagger.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 98bc392b1..20307cc49 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -474,7 +474,7 @@ paths: description: Registered user ID. - name: password in: body - description: Password for updating. + description: Password to be updated. required: true schema: $ref: '#/definitions/Password'