From ab6bb58c3636875d6aba25aa6260687783d79164 Mon Sep 17 00:00:00 2001 From: hmwenchen Date: Fri, 22 Apr 2016 00:28:59 +0800 Subject: [PATCH 1/6] Add tag to accesslog --- Deploy/db/registry.sql | 1 + api/project.go | 3 + api/repository.go | 143 +++-------------------------- dao/accesslog.go | 22 +++-- dao/project.go | 2 +- models/accesslog.go | 1 + models/notification.go | 1 + models/repo.go | 30 ++++++ service/notification.go | 82 ++++++++++++++++- static/i18n/locale_en-US.ini | 1 + static/i18n/locale_zh-CN.ini | 3 +- static/resources/js/item-detail.js | 1 + views/item-detail.tpl | 9 +- 13 files changed, 153 insertions(+), 146 deletions(-) diff --git a/Deploy/db/registry.sql b/Deploy/db/registry.sql index 9e4a342b3..bd0644b33 100644 --- a/Deploy/db/registry.sql +++ b/Deploy/db/registry.sql @@ -94,6 +94,7 @@ create table access_log ( user_id int NOT NULL, project_id int NOT NULL, repo_name varchar (40), + repo_tag varchar (20), GUID varchar(64), operation varchar(20) NOT NULL, op_time timestamp, diff --git a/api/project.go b/api/project.go index dbc0a4ecb..2d2aa1e47 100644 --- a/api/project.go +++ b/api/project.go @@ -31,6 +31,7 @@ import ( type ProjectAPI struct { BaseAPI userID int + username string projectID int64 } @@ -183,6 +184,8 @@ func (p *ProjectAPI) FilterAccessLog() { p.CustomAbort(http.StatusInternalServerError, "Internal error.") } p.Data["json"] = accessLogList + + log.Errorf("--- accessLog first record: %v ---", accessLogList[0]) p.ServeJSON() } diff --git a/api/repository.go b/api/repository.go index 6058b3608..252cb2fbc 100644 --- a/api/repository.go +++ b/api/repository.go @@ -18,7 +18,6 @@ package api import ( "encoding/json" "net/http" - "os" "strconv" "strings" "time" @@ -27,9 +26,6 @@ import ( "github.com/vmware/harbor/models" svc_utils "github.com/vmware/harbor/service/utils" "github.com/vmware/harbor/utils/log" - "github.com/vmware/harbor/utils/registry" - "github.com/vmware/harbor/utils/registry/auth" - "github.com/vmware/harbor/utils/registry/errors" ) // RepositoryAPI handles request to /api/repositories /api/repositories/tags /api/repositories/manifests, the parm has to be put @@ -40,7 +36,6 @@ type RepositoryAPI struct { BaseAPI userID int username string - registry *registry.Registry } // Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission. @@ -58,43 +53,6 @@ func (ra *RepositoryAPI) Prepare() { } else { ra.username = username } - - var client *http.Client - - //no session, initialize a standard auth handler - if ra.userID == dao.NonExistUserID && len(ra.username) == 0 { - username, password, _ := ra.Ctx.Request.BasicAuth() - - credential := auth.NewBasicAuthCredential(username, password) - client = registry.NewClientStandardAuthHandlerEmbeded(credential) - log.Debug("initializing standard auth handler") - - } else { - // session works, initialize a username auth handler - username := ra.username - if len(username) == 0 { - user, err := dao.GetUser(models.User{ - UserID: ra.userID, - }) - if err != nil { - log.Errorf("error occurred whiling geting user for initializing a username auth handler: %v", err) - return - } - - username = user.Username - } - - client = registry.NewClientUsernameAuthHandlerEmbeded(username) - log.Debug("initializing username auth handler: %s", username) - } - - endpoint := os.Getenv("REGISTRY_URL") - r, err := registry.New(endpoint, client) - if err != nil { - log.Fatalf("error occurred while initializing auth handler for repository API: %v", err) - } - - ra.registry = r } // Get ... @@ -119,13 +77,11 @@ func (ra *RepositoryAPI) Get() { ra.RenderError(http.StatusForbidden, "") return } - repoList, err := svc_utils.GetRepoFromCache() if err != nil { log.Errorf("Failed to get repo from cache, error: %v", err) ra.RenderError(http.StatusInternalServerError, "internal sever error") } - projectName := p.Name q := ra.GetString("q") var resp []string @@ -149,92 +105,26 @@ func (ra *RepositoryAPI) Get() { ra.ServeJSON() } -// Delete ... -func (ra *RepositoryAPI) Delete() { - repoName := ra.GetString("repo_name") - if len(repoName) == 0 { - ra.CustomAbort(http.StatusBadRequest, "repo_name is nil") - } - - tags := []string{} - tag := ra.GetString("tag") - if len(tag) == 0 { - tagList, err := ra.registry.ListTag(repoName) - if err != nil { - e, ok := errors.ParseError(err) - if ok { - log.Info(e) - ra.CustomAbort(e.StatusCode, e.Message) - } else { - log.Error(err) - ra.CustomAbort(http.StatusInternalServerError, "internal error") - } - - } - tags = append(tags, tagList...) - - } else { - tags = append(tags, tag) - } - - for _, t := range tags { - if err := ra.registry.DeleteTag(repoName, t); err != nil { - e, ok := errors.ParseError(err) - if ok { - ra.CustomAbort(e.StatusCode, e.Message) - } else { - log.Error(err) - ra.CustomAbort(http.StatusInternalServerError, "internal error") - } - } - log.Infof("delete tag: %s %s", repoName, t) - } - - go func() { - log.Debug("refreshing catalog cache") - if err := svc_utils.RefreshCatalogCache(); err != nil { - log.Errorf("error occurred while refresh catalog cache: %v", err) - } - }() - -} - type tag struct { Name string `json:"name"` Tags []string `json:"tags"` } -type histroyItem struct { - V1Compatibility string `json:"v1Compatibility"` -} - -type manifest struct { - Name string `json:"name"` - Tag string `json:"tag"` - Architecture string `json:"architecture"` - SchemaVersion int `json:"schemaVersion"` - History []histroyItem `json:"history"` -} - // GetTags handles GET /api/repositories/tags func (ra *RepositoryAPI) GetTags() { var tags []string repoName := ra.GetString("repo_name") - - tags, err := ra.registry.ListTag(repoName) + result, err := svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repoName, "tags", "list"), ra.username) if err != nil { - e, ok := errors.ParseError(err) - if ok { - log.Info(e) - ra.CustomAbort(e.StatusCode, e.Message) - } else { - log.Error(err) - ra.CustomAbort(http.StatusInternalServerError, "internal error") - } + log.Errorf("Failed to get repo tags, repo name: %s, error: %v", repoName, err) + ra.RenderError(http.StatusInternalServerError, "Failed to get repo tags") + } else { + t := tag{} + json.Unmarshal(result, &t) + tags = t.Tags } - ra.Data["json"] = tags ra.ServeJSON() } @@ -246,20 +136,14 @@ func (ra *RepositoryAPI) GetManifests() { item := models.RepoItem{} - _, _, payload, err := ra.registry.PullManifest(repoName, tag, registry.ManifestVersion1) + result, err := svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repoName, "manifests", tag), ra.username) if err != nil { - e, ok := errors.ParseError(err) - if ok { - log.Info(e) - ra.CustomAbort(e.StatusCode, e.Message) - } else { - log.Error(err) - ra.CustomAbort(http.StatusInternalServerError, "internal error") - } + log.Errorf("Failed to get manifests for repo, repo name: %s, tag: %s, error: %v", repoName, tag, err) + ra.RenderError(http.StatusInternalServerError, "Internal Server Error") + return } - - mani := manifest{} - err = json.Unmarshal(payload, &mani) + mani := models.Manifest{} + err = json.Unmarshal(result, &mani) if err != nil { log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repoName, tag, err) ra.RenderError(http.StatusInternalServerError, "Internal Server Error") @@ -273,6 +157,7 @@ func (ra *RepositoryAPI) GetManifests() { ra.RenderError(http.StatusInternalServerError, "Internal Server Error") return } + item.CreatedStr = item.Created.Format("2006-01-02 15:04:05") item.DurationDays = strconv.Itoa(int(time.Since(item.Created).Hours()/24)) + " days" ra.Data["json"] = item diff --git a/dao/accesslog.go b/dao/accesslog.go index 721ac92cd..20fb20655 100644 --- a/dao/accesslog.go +++ b/dao/accesslog.go @@ -19,6 +19,7 @@ import ( "strings" "github.com/vmware/harbor/models" + "github.com/vmware/harbor/utils/log" "github.com/astaxie/beego/orm" ) @@ -27,14 +28,14 @@ import ( func AddAccessLog(accessLog models.AccessLog) error { o := orm.NewOrm() p, err := o.Raw(`insert into access_log - (user_id, project_id, repo_name, guid, operation, op_time) - values (?, ?, ?, ?, ?, now())`).Prepare() + (user_id, project_id, repo_name, repo_tag, guid, operation, op_time) + values (?, ?, ?, ?, ?, ?, now())`).Prepare() if err != nil { return err } defer p.Close() - _, err = p.Exec(accessLog.UserID, accessLog.ProjectID, accessLog.RepoName, accessLog.GUID, accessLog.Operation) + _, err = p.Exec(accessLog.UserID, accessLog.ProjectID, accessLog.RepoName, accessLog.RepoTag, accessLog.GUID, accessLog.Operation) return err } @@ -43,7 +44,7 @@ func AddAccessLog(accessLog models.AccessLog) error { func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) { o := orm.NewOrm() - sql := `select a.log_id, u.username, a.repo_name, a.operation, a.op_time + sql := `select a.log_id, u.username, a.repo_name, a.repo_tag, a.operation, a.op_time from access_log a left join user u on a.user_id = u.user_id where a.project_id = ? ` queryParam := make([]interface{}, 1) @@ -95,13 +96,16 @@ func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) { return accessLogList, nil } -// AccessLog ... -func AccessLog(username, projectName, repoName, action string) error { +// AccessLog, invoked in service/notification.go +func AccessLog(username, projectName, repoName, repoTag, action string) error { o := orm.NewOrm() - sql := "insert into access_log (user_id, project_id, repo_name, operation, op_time) " + + sql := "insert into access_log (user_id, project_id, repo_name, repo_tag, operation, op_time) " + "select (select user_id as user_id from user where username=?), " + - "(select project_id as project_id from project where name=?), ?, ?, now() " - _, err := o.Raw(sql, username, projectName, repoName, action).Exec() + "(select project_id as project_id from project where name=?), ?, ?, ?, now() " + _, err := o.Raw(sql, username, projectName, repoName, repoTag, action).Exec() + if err != nil { + log.Errorf("error in AccessLog: %v ", err) + } return err } diff --git a/dao/project.go b/dao/project.go index d2a611e9f..a0e6e6b25 100644 --- a/dao/project.go +++ b/dao/project.go @@ -60,7 +60,7 @@ func AddProject(project models.Project) error { return err } - accessLog := models.AccessLog{UserID: project.OwnerID, ProjectID: projectID, RepoName: project.Name + "/", GUID: "N/A", Operation: "create", OpTime: time.Now()} + accessLog := models.AccessLog{UserID: project.OwnerID, ProjectID: projectID, RepoName: project.Name + "/", RepoTag: "N/A", GUID: "N/A", Operation: "create", OpTime: time.Now()} err = AddAccessLog(accessLog) return err diff --git a/models/accesslog.go b/models/accesslog.go index 150f3b588..3111e3a44 100644 --- a/models/accesslog.go +++ b/models/accesslog.go @@ -25,6 +25,7 @@ type AccessLog struct { UserID int `orm:"column(user_id)" json:"UserId"` ProjectID int64 `orm:"column(project_id)" json:"ProjectId"` RepoName string `orm:"column(repo_name)"` + RepoTag string `orm:"column(repo_tag)"` GUID string `orm:"column(GUID)" json:"Guid"` Operation string `orm:"column(operation)"` OpTime time.Time `orm:"column(op_time)"` diff --git a/models/notification.go b/models/notification.go index 0f608e2c6..3f30ae7e4 100644 --- a/models/notification.go +++ b/models/notification.go @@ -40,6 +40,7 @@ type Target struct { Digest string Repository string URL string `json:"Url"` + Tag string } // Actor holds information about actor. diff --git a/models/repo.go b/models/repo.go index 6b8de742b..44abd9d9b 100644 --- a/models/repo.go +++ b/models/repo.go @@ -29,6 +29,7 @@ type RepoItem struct { ID string `json:"Id"` Parent string `json:"Parent"` Created time.Time `json:"Created"` + CreatedStr string `json:"CreatedStr"` DurationDays string `json:"Duration Days"` Author string `json:"Author"` Architecture string `json:"Architecture"` @@ -42,3 +43,32 @@ type Tag struct { Version string `json:"version"` ImageID string `json:"image_id"` } + +type Manifest struct { + SchemaVersion int `json:"schemaVersion"` + Name string `json:"name"` + Tag string `json:"tag"` + Architecture string `json:"architecture"` + FsLayers []blobSumItem `json:"fsLayers"` + History []histroyItem `json:"history"` +} + +type histroyItem struct { + V1Compatibility string `json:"v1Compatibility"` +} + +type blobSumItem struct { + BlobSum string `json:"blobSum"` +} + +type ManifestDigest struct { + MediaType string `json:"mediaType"` + SchemaVersion int `json:"schemaVersion"` + Layers []layerItem `json:"layers"` +} + +type layerItem struct { + MediaType string `json:"mediaType"` + Size int `json:"size"` + Digest string `json:"digest"` +} diff --git a/service/notification.go b/service/notification.go index beab28778..04611ec21 100644 --- a/service/notification.go +++ b/service/notification.go @@ -18,6 +18,7 @@ package service import ( "encoding/json" "regexp" + "sort" "strings" "github.com/vmware/harbor/dao" @@ -33,6 +34,11 @@ type NotificationHandler struct { beego.Controller } +type taglist struct { + Name string `json:"name"` + Tags []string `json:"tags"` +} + const manifestPattern = `^application/vnd.docker.distribution.manifest.v\d\+json` // Post handles POST request, and records audit log or refreshes cache based on event. @@ -46,7 +52,7 @@ func (n *NotificationHandler) Post() { log.Errorf("error while decoding json: %v", err) return } - var username, action, repo, project string + var username, action, repo, project, repo_tag, tag_url string var matched bool for _, e := range notification.Events { matched, err = regexp.MatchString(manifestPattern, e.Target.MediaType) @@ -58,13 +64,67 @@ func (n *NotificationHandler) Post() { username = e.Actor.Name action = e.Action repo = e.Target.Repository + tag_url = e.Target.URL + result, err1 := svc_utils.RegistryAPIGet(tag_url, username) + + if err1 != nil { + log.Errorf("Failed to get manifests for repo, repo name: %s, tag: %s, error: %v", repo, tag_url, err1) + return + } + + maniDig := models.ManifestDigest{} + err = json.Unmarshal(result, &maniDig) + if err != nil { + log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repo, tag_url, err) + return + } + + var digestLayers []string + var tagLayers []string + for _, diglayer := range maniDig.Layers { + digestLayers = append(digestLayers, diglayer.Digest) + } + + result, err = svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repo, "tags", "list"), username) + if err != nil { + log.Errorf("Failed to get repo tags, repo name: %s, error: %v", repo, err) + } else { + t := taglist{} + json.Unmarshal(result, &t) + for _, tag := range t.Tags { + result, err = svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repo, "manifests", tag), username) + if err != nil { + log.Errorf("Failed to get repo tags, repo name: %s, error: %v", repo, err) + continue + } + taginfo := models.Manifest{} + err = json.Unmarshal(result, &taginfo) + if err != nil { + log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repo, tag, err) + continue + } + for _, fslayer := range taginfo.FsLayers { + tagLayers = append(tagLayers, fslayer.BlobSum) + } + + sort.Strings(digestLayers) + sort.Strings(tagLayers) + eq := compStringArray(digestLayers, tagLayers) + if eq { + repo_tag = tag + break + } + } + } + if strings.Contains(repo, "/") { project = repo[0:strings.LastIndex(repo, "/")] } if username == "" { username = "anonymous" } - go dao.AccessLog(username, project, repo, action) + log.Debugf("repo tag is : %v ", repo_tag) + go dao.AccessLog(username, project, repo, repo_tag, action) if action == "push" { go func() { err2 := svc_utils.RefreshCatalogCache() @@ -82,3 +142,21 @@ func (n *NotificationHandler) Post() { func (n *NotificationHandler) Render() error { return nil } + +func compStringArray(a, b []string) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true +} diff --git a/static/i18n/locale_en-US.ini b/static/i18n/locale_en-US.ini index 981213cf0..e7999607d 100644 --- a/static/i18n/locale_en-US.ini +++ b/static/i18n/locale_en-US.ini @@ -59,6 +59,7 @@ repo = Repositories user = Users logs = Logs repo_name = Repository Name +repo_tag = Tag add_members = Add Members operation = Operation advance = Advanced Search diff --git a/static/i18n/locale_zh-CN.ini b/static/i18n/locale_zh-CN.ini index 90ea4fc04..3f3ef904b 100644 --- a/static/i18n/locale_zh-CN.ini +++ b/static/i18n/locale_zh-CN.ini @@ -59,6 +59,7 @@ repo = 镜像仓库 user = 用户 logs = 日志 repo_name = 镜像名称 +repo_tag = 镜像标签 add_members = 添加成员 operation = 操作 advance = 高级检索 @@ -82,4 +83,4 @@ index_desc_2 = 2. 效率: 搭建组织内部的私有容器Registry服务，可 index_desc_3 = 3. 访问控制: 提供基于角色的访问控制，可集成企业目前拥有的用户管理系统（如:AD/LDAP）。 index_desc_4 = 4. 审计: 所有访问Registry服务的操作均被记录，便于日后审计。 index_desc_5 = 5. 管理界面: 具有友好易用图形管理界面。 -index_title = 企业级 Registry 服务 \ No newline at end of file +index_title = 企业级 Registry 服务 diff --git a/static/resources/js/item-detail.js b/static/resources/js/item-detail.js index 1b6ffa771..6f445f717 100644 --- a/static/resources/js/item-detail.js +++ b/static/resources/js/item-detail.js @@ -286,6 +286,7 @@ jQuery(function(){ '' + '' + e.Username + '' + '' + e.RepoName + '' + + '' + e.RepoTag + '' + '' + e.Operation + '' + '' + moment(new Date(e.OpTime)).format("YYYY-MM-DD HH:mm:ss") + '' + ''); diff --git a/views/item-detail.tpl b/views/item-detail.tpl index a1ce83e96..e9cdb2753 100644 --- a/views/item-detail.tpl +++ b/views/item-detail.tpl @@ -159,10 +159,11 @@ - - - - + + + + + From dc96ded36a4bcc111ed58d5dc60c728a0070c868 Mon Sep 17 00:00:00 2001 From: hmwenchen Date: Fri, 22 Apr 2016 09:18:51 +0800 Subject: [PATCH 2/6] Merge with origin master --- api/project.go | 2 - api/repository.go | 127 ++++++++++++++++++++++++++++++++++++---- models/notification.go | 1 - models/repo.go | 1 - service/notification.go | 99 +++++++++++++++++++------------ 5 files changed, 176 insertions(+), 54 deletions(-) diff --git a/api/project.go b/api/project.go index 2d2aa1e47..9c1a4f678 100644 --- a/api/project.go +++ b/api/project.go @@ -31,7 +31,6 @@ import ( type ProjectAPI struct { BaseAPI userID int - username string projectID int64 } @@ -185,7 +184,6 @@ func (p *ProjectAPI) FilterAccessLog() { } p.Data["json"] = accessLogList - log.Errorf("--- accessLog first record: %v ---", accessLogList[0]) p.ServeJSON() } diff --git a/api/repository.go b/api/repository.go index 252cb2fbc..e259067a4 100644 --- a/api/repository.go +++ b/api/repository.go @@ -18,6 +18,7 @@ package api import ( "encoding/json" "net/http" + "os" "strconv" "strings" "time" @@ -26,6 +27,9 @@ import ( "github.com/vmware/harbor/models" svc_utils "github.com/vmware/harbor/service/utils" "github.com/vmware/harbor/utils/log" + "github.com/vmware/harbor/utils/registry" + "github.com/vmware/harbor/utils/registry/auth" + "github.com/vmware/harbor/utils/registry/errors" ) // RepositoryAPI handles request to /api/repositories /api/repositories/tags /api/repositories/manifests, the parm has to be put @@ -36,6 +40,7 @@ type RepositoryAPI struct { BaseAPI userID int username string + registry *registry.Registry } // Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission. @@ -53,6 +58,43 @@ func (ra *RepositoryAPI) Prepare() { } else { ra.username = username } + + var client *http.Client + + //no session, initialize a standard auth handler + if ra.userID == dao.NonExistUserID && len(ra.username) == 0 { + username, password, _ := ra.Ctx.Request.BasicAuth() + + credential := auth.NewBasicAuthCredential(username, password) + client = registry.NewClientStandardAuthHandlerEmbeded(credential) + log.Debug("initializing standard auth handler") + + } else { + // session works, initialize a username auth handler + username := ra.username + if len(username) == 0 { + user, err := dao.GetUser(models.User{ + UserID: ra.userID, + }) + if err != nil { + log.Errorf("error occurred whiling geting user for initializing a username auth handler: %v", err) + return + } + + username = user.Username + } + + client = registry.NewClientUsernameAuthHandlerEmbeded(username) + log.Debug("initializing username auth handler: %s", username) + } + + endpoint := os.Getenv("REGISTRY_URL") + r, err := registry.New(endpoint, client) + if err != nil { + log.Fatalf("error occurred while initializing auth handler for repository API: %v", err) + } + + ra.registry = r } // Get ... @@ -77,11 +119,13 @@ func (ra *RepositoryAPI) Get() { ra.RenderError(http.StatusForbidden, "") return } + repoList, err := svc_utils.GetRepoFromCache() if err != nil { log.Errorf("Failed to get repo from cache, error: %v", err) ra.RenderError(http.StatusInternalServerError, "internal sever error") } + projectName := p.Name q := ra.GetString("q") var resp []string @@ -105,6 +149,56 @@ func (ra *RepositoryAPI) Get() { ra.ServeJSON() } +// Delete ... +func (ra *RepositoryAPI) Delete() { + repoName := ra.GetString("repo_name") + if len(repoName) == 0 { + ra.CustomAbort(http.StatusBadRequest, "repo_name is nil") + } + + tags := []string{} + tag := ra.GetString("tag") + if len(tag) == 0 { + tagList, err := ra.registry.ListTag(repoName) + if err != nil { + e, ok := errors.ParseError(err) + if ok { + log.Info(e) + ra.CustomAbort(e.StatusCode, e.Message) + } else { + log.Error(err) + ra.CustomAbort(http.StatusInternalServerError, "internal error") + } + + } + tags = append(tags, tagList...) + + } else { + tags = append(tags, tag) + } + + for _, t := range tags { + if err := ra.registry.DeleteTag(repoName, t); err != nil { + e, ok := errors.ParseError(err) + if ok { + ra.CustomAbort(e.StatusCode, e.Message) + } else { + log.Error(err) + ra.CustomAbort(http.StatusInternalServerError, "internal error") + } + } + log.Infof("delete tag: %s %s", repoName, t) + } + + go func() { + log.Debug("refreshing catalog cache") + if err := svc_utils.RefreshCatalogCache(); err != nil { + log.Errorf("error occurred while refresh catalog cache: %v", err) + } + }() + +} + type tag struct { Name string `json:"name"` Tags []string `json:"tags"` @@ -116,15 +210,18 @@ func (ra *RepositoryAPI) GetTags() { var tags []string repoName := ra.GetString("repo_name") - result, err := svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repoName, "tags", "list"), ra.username) + tags, err := ra.registry.ListTag(repoName) if err != nil { - log.Errorf("Failed to get repo tags, repo name: %s, error: %v", repoName, err) - ra.RenderError(http.StatusInternalServerError, "Failed to get repo tags") - } else { - t := tag{} - json.Unmarshal(result, &t) - tags = t.Tags + e, ok := errors.ParseError(err) + if ok { + log.Info(e) + ra.CustomAbort(e.StatusCode, e.Message) + } else { + log.Error(err) + ra.CustomAbort(http.StatusInternalServerError, "internal error") + } } + ra.Data["json"] = tags ra.ServeJSON() } @@ -136,14 +233,19 @@ func (ra *RepositoryAPI) GetManifests() { item := models.RepoItem{} - result, err := svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repoName, "manifests", tag), ra.username) + _, _, payload, err := ra.registry.PullManifest(repoName, tag, registry.ManifestVersion1) if err != nil { - log.Errorf("Failed to get manifests for repo, repo name: %s, tag: %s, error: %v", repoName, tag, err) - ra.RenderError(http.StatusInternalServerError, "Internal Server Error") - return + e, ok := errors.ParseError(err) + if ok { + log.Info(e) + ra.CustomAbort(e.StatusCode, e.Message) + } else { + log.Error(err) + ra.CustomAbort(http.StatusInternalServerError, "internal error") + } } mani := models.Manifest{} - err = json.Unmarshal(result, &mani) + err = json.Unmarshal(payload, &mani) if err != nil { log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repoName, tag, err) ra.RenderError(http.StatusInternalServerError, "Internal Server Error") @@ -157,7 +259,6 @@ func (ra *RepositoryAPI) GetManifests() { ra.RenderError(http.StatusInternalServerError, "Internal Server Error") return } - item.CreatedStr = item.Created.Format("2006-01-02 15:04:05") item.DurationDays = strconv.Itoa(int(time.Since(item.Created).Hours()/24)) + " days" ra.Data["json"] = item diff --git a/models/notification.go b/models/notification.go index 3f30ae7e4..0f608e2c6 100644 --- a/models/notification.go +++ b/models/notification.go @@ -40,7 +40,6 @@ type Target struct { Digest string Repository string URL string `json:"Url"` - Tag string } // Actor holds information about actor. diff --git a/models/repo.go b/models/repo.go index 44abd9d9b..c60910b82 100644 --- a/models/repo.go +++ b/models/repo.go @@ -29,7 +29,6 @@ type RepoItem struct { ID string `json:"Id"` Parent string `json:"Parent"` Created time.Time `json:"Created"` - CreatedStr string `json:"CreatedStr"` DurationDays string `json:"Duration Days"` Author string `json:"Author"` Architecture string `json:"Architecture"` diff --git a/service/notification.go b/service/notification.go index 04611ec21..ecd19d9b8 100644 --- a/service/notification.go +++ b/service/notification.go @@ -17,6 +17,8 @@ package service import ( "encoding/json" + "net/http" + "os" "regexp" "sort" "strings" @@ -25,6 +27,8 @@ import ( "github.com/vmware/harbor/models" svc_utils "github.com/vmware/harbor/service/utils" "github.com/vmware/harbor/utils/log" + "github.com/vmware/harbor/utils/registry" + "github.com/vmware/harbor/utils/registry/errors" "github.com/astaxie/beego" ) @@ -32,11 +36,7 @@ import ( // NotificationHandler handles request on /service/notifications/, which listens to registry's events. type NotificationHandler struct { beego.Controller -} - -type taglist struct { - Name string `json:"name"` - Tags []string `json:"tags"` + registry *registry.Registry } const manifestPattern = `^application/vnd.docker.distribution.manifest.v\d\+json` @@ -52,8 +52,9 @@ func (n *NotificationHandler) Post() { log.Errorf("error while decoding json: %v", err) return } - var username, action, repo, project, repo_tag, tag_url string + var username, action, repo, project, repo_tag, tag_url, digest string var matched bool + var client *http.Client for _, e := range notification.Events { matched, err = regexp.MatchString(manifestPattern, e.Target.MediaType) if err != nil { @@ -65,15 +66,27 @@ func (n *NotificationHandler) Post() { action = e.Action repo = e.Target.Repository tag_url = e.Target.URL - result, err1 := svc_utils.RegistryAPIGet(tag_url, username) + digest = e.Target.Digest + client = registry.NewClientUsernameAuthHandlerEmbeded(username) + log.Debug("initializing username auth handler: %s", username) + endpoint := os.Getenv("REGISTRY_URL") + r, err1 := registry.New(endpoint, client) if err1 != nil { - log.Errorf("Failed to get manifests for repo, repo name: %s, tag: %s, error: %v", repo, tag_url, err1) + log.Fatalf("error occurred while initializing auth handler for repository API: %v", err1) + + } + n.registry = r + + _, _, payload, err2 := n.registry.PullManifest(repo, digest, registry.ManifestVersion1) + + if err2 != nil { + log.Errorf("Failed to get manifests for repo, repo name: %s, tag: %s, error: %v", repo, tag_url, err2) return } maniDig := models.ManifestDigest{} - err = json.Unmarshal(result, &maniDig) + err = json.Unmarshal(payload, &maniDig) if err != nil { log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repo, tag_url, err) return @@ -85,36 +98,48 @@ func (n *NotificationHandler) Post() { digestLayers = append(digestLayers, diglayer.Digest) } - result, err = svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repo, "tags", "list"), username) + tags, err := n.registry.ListTag(repo) if err != nil { - log.Errorf("Failed to get repo tags, repo name: %s, error: %v", repo, err) - } else { - t := taglist{} - json.Unmarshal(result, &t) - for _, tag := range t.Tags { - result, err = svc_utils.RegistryAPIGet(svc_utils.BuildRegistryURL(repo, "manifests", tag), username) - if err != nil { - log.Errorf("Failed to get repo tags, repo name: %s, error: %v", repo, err) - continue - } - taginfo := models.Manifest{} - err = json.Unmarshal(result, &taginfo) - if err != nil { - log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repo, tag, err) - continue - } - for _, fslayer := range taginfo.FsLayers { - tagLayers = append(tagLayers, fslayer.BlobSum) - } - - sort.Strings(digestLayers) - sort.Strings(tagLayers) - eq := compStringArray(digestLayers, tagLayers) - if eq { - repo_tag = tag - break - } + e, ok := errors.ParseError(err) + if ok { + log.Info(e) + } else { + log.Error(err) } + return + } + + log.Infof("tags : %v ", tags) + + for _, tag := range tags { + _, _, payload, err := n.registry.PullManifest(repo, tag, registry.ManifestVersion1) + if err != nil { + e, ok := errors.ParseError(err) + if ok { + log.Info(e) + } else { + log.Error(err) + } + continue + } + taginfo := models.Manifest{} + err = json.Unmarshal(payload, &taginfo) + if err != nil { + log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repo, tag, err) + continue + } + for _, fslayer := range taginfo.FsLayers { + tagLayers = append(tagLayers, fslayer.BlobSum) + } + + sort.Strings(digestLayers) + sort.Strings(tagLayers) + eq := compStringArray(digestLayers, tagLayers) + if eq { + repo_tag = tag + break + } + } if strings.Contains(repo, "/") { From 957df50958ce113919cc8318a61bfaba95425314 Mon Sep 17 00:00:00 2001 From: hmwenchen Date: Fri, 22 Apr 2016 20:26:28 +0800 Subject: [PATCH 3/6] Minor change to comply with CI requirements --- dao/accesslog.go | 2 +- models/repo.go | 2 ++ service/notification.go | 14 +++++++------- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/dao/accesslog.go b/dao/accesslog.go index 20fb20655..dbf447353 100644 --- a/dao/accesslog.go +++ b/dao/accesslog.go @@ -96,7 +96,7 @@ func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) { return accessLogList, nil } -// AccessLog, invoked in service/notification.go +// AccessLog ... func AccessLog(username, projectName, repoName, repoTag, action string) error { o := orm.NewOrm() sql := "insert into access_log (user_id, project_id, repo_name, repo_tag, operation, op_time) " + diff --git a/models/repo.go b/models/repo.go index c60910b82..4ace8d1b6 100644 --- a/models/repo.go +++ b/models/repo.go @@ -43,6 +43,7 @@ type Tag struct { ImageID string `json:"image_id"` } +// Manifest ... type Manifest struct { SchemaVersion int `json:"schemaVersion"` Name string `json:"name"` @@ -60,6 +61,7 @@ type blobSumItem struct { BlobSum string `json:"blobSum"` } +// ManifestDigest ... type ManifestDigest struct { MediaType string `json:"mediaType"` SchemaVersion int `json:"schemaVersion"` diff --git a/service/notification.go b/service/notification.go index ecd19d9b8..f47ddd627 100644 --- a/service/notification.go +++ b/service/notification.go @@ -52,7 +52,7 @@ func (n *NotificationHandler) Post() { log.Errorf("error while decoding json: %v", err) return } - var username, action, repo, project, repo_tag, tag_url, digest string + var username, action, repo, project, repoTag, tagURL, digest string var matched bool var client *http.Client for _, e := range notification.Events { @@ -65,7 +65,7 @@ func (n *NotificationHandler) Post() { username = e.Actor.Name action = e.Action repo = e.Target.Repository - tag_url = e.Target.URL + tagURL = e.Target.URL digest = e.Target.Digest client = registry.NewClientUsernameAuthHandlerEmbeded(username) @@ -81,14 +81,14 @@ func (n *NotificationHandler) Post() { _, _, payload, err2 := n.registry.PullManifest(repo, digest, registry.ManifestVersion1) if err2 != nil { - log.Errorf("Failed to get manifests for repo, repo name: %s, tag: %s, error: %v", repo, tag_url, err2) + log.Errorf("Failed to get manifests for repo, repo name: %s, tag: %s, error: %v", repo, tagURL, err2) return } maniDig := models.ManifestDigest{} err = json.Unmarshal(payload, &maniDig) if err != nil { - log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repo, tag_url, err) + log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repo, tagURL, err) return } @@ -136,7 +136,7 @@ func (n *NotificationHandler) Post() { sort.Strings(tagLayers) eq := compStringArray(digestLayers, tagLayers) if eq { - repo_tag = tag + repoTag = tag break } @@ -148,8 +148,8 @@ func (n *NotificationHandler) Post() { if username == "" { username = "anonymous" } - log.Debugf("repo tag is : %v ", repo_tag) - go dao.AccessLog(username, project, repo, repo_tag, action) + log.Debugf("repo tag is : %v ", repoTag) + go dao.AccessLog(username, project, repo, repoTag, action) if action == "push" { go func() { err2 := svc_utils.RefreshCatalogCache() From 87b213c3ae85fbf63f61a7fba25f47edc4b471d4 Mon Sep 17 00:00:00 2001 From: hmwenchen Date: Tue, 26 Apr 2016 20:06:21 +0800 Subject: [PATCH 4/6] Use registry 2.4 toget tag info from events --- models/notification.go | 1 + models/repo.go | 13 ------ service/notification.go | 81 ++------------------------------------ utils/registry/registry.go | 3 ++ 4 files changed, 7 insertions(+), 91 deletions(-) diff --git a/models/notification.go b/models/notification.go index 0f608e2c6..3f30ae7e4 100644 --- a/models/notification.go +++ b/models/notification.go @@ -40,6 +40,7 @@ type Target struct { Digest string Repository string URL string `json:"Url"` + Tag string } // Actor holds information about actor. diff --git a/models/repo.go b/models/repo.go index 4ace8d1b6..f224c902e 100644 --- a/models/repo.go +++ b/models/repo.go @@ -60,16 +60,3 @@ type histroyItem struct { type blobSumItem struct { BlobSum string `json:"blobSum"` } - -// ManifestDigest ... -type ManifestDigest struct { - MediaType string `json:"mediaType"` - SchemaVersion int `json:"schemaVersion"` - Layers []layerItem `json:"layers"` -} - -type layerItem struct { - MediaType string `json:"mediaType"` - Size int `json:"size"` - Digest string `json:"digest"` -} diff --git a/service/notification.go b/service/notification.go index f47ddd627..d307cb487 100644 --- a/service/notification.go +++ b/service/notification.go @@ -52,7 +52,7 @@ func (n *NotificationHandler) Post() { log.Errorf("error while decoding json: %v", err) return } - var username, action, repo, project, repoTag, tagURL, digest string + var username, action, repo, project, repoTag string var matched bool var client *http.Client for _, e := range notification.Events { @@ -65,82 +65,8 @@ func (n *NotificationHandler) Post() { username = e.Actor.Name action = e.Action repo = e.Target.Repository - tagURL = e.Target.URL - digest = e.Target.Digest - - client = registry.NewClientUsernameAuthHandlerEmbeded(username) - log.Debug("initializing username auth handler: %s", username) - endpoint := os.Getenv("REGISTRY_URL") - r, err1 := registry.New(endpoint, client) - if err1 != nil { - log.Fatalf("error occurred while initializing auth handler for repository API: %v", err1) - - } - n.registry = r - - _, _, payload, err2 := n.registry.PullManifest(repo, digest, registry.ManifestVersion1) - - if err2 != nil { - log.Errorf("Failed to get manifests for repo, repo name: %s, tag: %s, error: %v", repo, tagURL, err2) - return - } - - maniDig := models.ManifestDigest{} - err = json.Unmarshal(payload, &maniDig) - if err != nil { - log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repo, tagURL, err) - return - } - - var digestLayers []string - var tagLayers []string - for _, diglayer := range maniDig.Layers { - digestLayers = append(digestLayers, diglayer.Digest) - } - - tags, err := n.registry.ListTag(repo) - if err != nil { - e, ok := errors.ParseError(err) - if ok { - log.Info(e) - } else { - log.Error(err) - } - return - } - - log.Infof("tags : %v ", tags) - - for _, tag := range tags { - _, _, payload, err := n.registry.PullManifest(repo, tag, registry.ManifestVersion1) - if err != nil { - e, ok := errors.ParseError(err) - if ok { - log.Info(e) - } else { - log.Error(err) - } - continue - } - taginfo := models.Manifest{} - err = json.Unmarshal(payload, &taginfo) - if err != nil { - log.Errorf("Failed to decode json from response for manifests, repo name: %s, tag: %s, error: %v", repo, tag, err) - continue - } - for _, fslayer := range taginfo.FsLayers { - tagLayers = append(tagLayers, fslayer.BlobSum) - } - - sort.Strings(digestLayers) - sort.Strings(tagLayers) - eq := compStringArray(digestLayers, tagLayers) - if eq { - repoTag = tag - break - } - - } + repoTag = e.Target.Tag + log.Debugf("repo tag is : %v ", repoTag) if strings.Contains(repo, "/") { project = repo[0:strings.LastIndex(repo, "/")] @@ -148,7 +74,6 @@ func (n *NotificationHandler) Post() { if username == "" { username = "anonymous" } - log.Debugf("repo tag is : %v ", repoTag) go dao.AccessLog(username, project, repo, repoTag, action) if action == "push" { go func() { diff --git a/utils/registry/registry.go b/utils/registry/registry.go index e390c4c0d..b4f41131b 100644 --- a/utils/registry/registry.go +++ b/utils/registry/registry.go @@ -25,6 +25,7 @@ import ( "github.com/docker/distribution/manifest" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" + "github.com/vmware/harbor/utils/log" "github.com/vmware/harbor/utils/registry/errors" ) @@ -197,6 +198,8 @@ func (r *Registry) PullManifest(name, reference string, version manifest.Version } // if the registry does not support schema 2, schema 1 manifest will be returned + log.Debugf("--- Pull request headers: %v ---", req.Header) + log.Debugf("--- Pull request Accept header: %v ---", http.CanonicalHeaderKey("Accept")) req.Header.Set(http.CanonicalHeaderKey("Accept"), version.MediaType) resp, err := r.client.Do(req) From 53bb0c1e6d7c24fddc1cf775a544dbb92eab171c Mon Sep 17 00:00:00 2001 From: hmwenchen Date: Tue, 26 Apr 2016 20:43:03 +0800 Subject: [PATCH 5/6] Remove unused packages --- service/notification.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/service/notification.go b/service/notification.go index d307cb487..ceda490d0 100644 --- a/service/notification.go +++ b/service/notification.go @@ -17,10 +17,7 @@ package service import ( "encoding/json" - "net/http" - "os" "regexp" - "sort" "strings" "github.com/vmware/harbor/dao" @@ -28,7 +25,6 @@ import ( svc_utils "github.com/vmware/harbor/service/utils" "github.com/vmware/harbor/utils/log" "github.com/vmware/harbor/utils/registry" - "github.com/vmware/harbor/utils/registry/errors" "github.com/astaxie/beego" ) @@ -54,7 +50,6 @@ func (n *NotificationHandler) Post() { } var username, action, repo, project, repoTag string var matched bool - var client *http.Client for _, e := range notification.Events { matched, err = regexp.MatchString(manifestPattern, e.Target.MediaType) if err != nil { From b4cc422a976bff0fc029517d332e41cda2709160 Mon Sep 17 00:00:00 2001 From: hmwenchen Date: Tue, 26 Apr 2016 21:00:29 +0800 Subject: [PATCH 6/6] Remove debug info --- service/notification.go | 20 -------------------- utils/registry/registry.go | 3 --- 2 files changed, 23 deletions(-) diff --git a/service/notification.go b/service/notification.go index ceda490d0..913d45b04 100644 --- a/service/notification.go +++ b/service/notification.go @@ -24,7 +24,6 @@ import ( "github.com/vmware/harbor/models" svc_utils "github.com/vmware/harbor/service/utils" "github.com/vmware/harbor/utils/log" - "github.com/vmware/harbor/utils/registry" "github.com/astaxie/beego" ) @@ -32,7 +31,6 @@ import ( // NotificationHandler handles request on /service/notifications/, which listens to registry's events. type NotificationHandler struct { beego.Controller - registry *registry.Registry } const manifestPattern = `^application/vnd.docker.distribution.manifest.v\d\+json` @@ -87,21 +85,3 @@ func (n *NotificationHandler) Post() { func (n *NotificationHandler) Render() error { return nil } - -func compStringArray(a, b []string) bool { - if a == nil && b == nil { - return true - } - if a == nil || b == nil { - return false - } - if len(a) != len(b) { - return false - } - for i, v := range a { - if v != b[i] { - return false - } - } - return true -} diff --git a/utils/registry/registry.go b/utils/registry/registry.go index b4f41131b..e390c4c0d 100644 --- a/utils/registry/registry.go +++ b/utils/registry/registry.go @@ -25,7 +25,6 @@ import ( "github.com/docker/distribution/manifest" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" - "github.com/vmware/harbor/utils/log" "github.com/vmware/harbor/utils/registry/errors" ) @@ -198,8 +197,6 @@ func (r *Registry) PullManifest(name, reference string, version manifest.Version } // if the registry does not support schema 2, schema 1 manifest will be returned - log.Debugf("--- Pull request headers: %v ---", req.Header) - log.Debugf("--- Pull request Accept header: %v ---", http.CanonicalHeaderKey("Accept")) req.Header.Set(http.CanonicalHeaderKey("Accept"), version.MediaType) resp, err := r.client.Do(req)
{{i18n .Lang "username"}}{{i18n .Lang "repo_name"}}{{i18n .Lang "operation"}}{{i18n .Lang "timestamp"}}{{i18n .Lang "username"}}{{i18n .Lang "repo_name"}}{{i18n .Lang "repo_tag"}}{{i18n .Lang "operation"}}{{i18n .Lang "timestamp"}}