From 4f9d9ed5d8267eb5b7f65187c6fc4f09e7f17f52 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 27 Apr 2017 18:19:28 +0800 Subject: [PATCH] delete owner_id column from table repository --- make/common/db/registry.sql | 1 - make/common/db/registry_sqlite.sql | 2 - src/common/dao/accesslog.go | 18 -- src/common/dao/dao_test.go | 25 +-- src/common/dao/repository.go | 16 +- src/common/dao/repository_test.go | 35 ++-- src/common/models/accesslog.go | 2 +- src/common/models/project.go | 2 +- src/common/models/replication_job.go | 6 +- src/common/models/repo.go | 5 +- src/common/models/role.go | 2 +- src/common/models/user.go | 2 +- src/ui/api/dataprepare_test.go | 251 +++++++++++++-------------- src/ui/api/repository.go | 4 +- src/ui/api/utils.go | 19 +- src/ui/service/notification.go | 10 +- tools/migration/changelog.md | 4 + 17 files changed, 182 insertions(+), 222 deletions(-) diff --git a/make/common/db/registry.sql b/make/common/db/registry.sql index ca30ce855..cb4e91cc8 100644 --- a/make/common/db/registry.sql +++ b/make/common/db/registry.sql @@ -116,7 +116,6 @@ create table repository ( repository_id int NOT NULL AUTO_INCREMENT, name varchar(255) NOT NULL, project_id int NOT NULL, - owner_id int NOT NULL, description text, pull_count int DEFAULT 0 NOT NULL, star_count int DEFAULT 0 NOT NULL, diff --git a/make/common/db/registry_sqlite.sql b/make/common/db/registry_sqlite.sql index de14e28ee..497565ee6 100644 --- a/make/common/db/registry_sqlite.sql +++ b/make/common/db/registry_sqlite.sql @@ -113,13 +113,11 @@ create table repository ( repository_id INTEGER PRIMARY KEY, name varchar(255) NOT NULL, project_id int NOT NULL, - owner_id int NOT NULL, description text, pull_count int DEFAULT 0 NOT NULL, star_count int DEFAULT 0 NOT NULL, creation_time timestamp default CURRENT_TIMESTAMP, update_time timestamp default CURRENT_TIMESTAMP, - FOREIGN KEY (owner_id) REFERENCES user(user_id), FOREIGN KEY (project_id) REFERENCES project(project_id), UNIQUE (name) ); diff --git a/src/common/dao/accesslog.go b/src/common/dao/accesslog.go index 15921294b..f0e7b1a3c 100644 --- a/src/common/dao/accesslog.go +++ b/src/common/dao/accesslog.go @@ -215,24 +215,6 @@ func GetRecentLogs(userID, linesNum int, startTime, endTime string) ([]models.Ac return logs, nil } -// GetAccessLogCreator ... -func GetAccessLogCreator(repoName string) (string, error) { - o := GetOrmer() - sql := "select * from user where user_id = (select user_id from access_log where operation = 'push' and repo_name = ? order by op_time desc limit 1)" - - var u []models.User - n, err := o.Raw(sql, repoName).QueryRows(&u) - - if err != nil { - return "", err - } - if n == 0 { - return "", nil - } - - return u[0].Username, nil -} - // CountPull ... func CountPull(repoName string) (int64, error) { o := GetOrmer() diff --git a/src/common/dao/dao_test.go b/src/common/dao/dao_test.go index 8f40406f2..a8cd269d2 100644 --- a/src/common/dao/dao_test.go +++ b/src/common/dao/dao_test.go @@ -192,7 +192,7 @@ func testForMySQL(m *testing.M) int { }, } - log.Infof("MYSQL_HOST: %s, MYSQL_USR: %s, MYSQL_PORT: %s, MYSQL_PWD: %s\n", dbHost, dbUser, dbPort, dbPassword) + log.Infof("MYSQL_HOST: %s, MYSQL_USR: %s, MYSQL_PORT: %d, MYSQL_PWD: %s\n", dbHost, dbUser, dbPort, dbPassword) return testForAll(m, database) } @@ -650,26 +650,6 @@ func TestAccessLog(t *testing.T) { } } -func TestGetAccessLogCreator(t *testing.T) { - var err error - err = AccessLog(currentUser.Username, currentProject.Name, currentProject.Name+"/tomcat", repoTag2, "push") - if err != nil { - t.Errorf("Error occurred in AccessLog: %v", err) - } - err = AccessLog(currentUser.Username, currentProject.Name, currentProject.Name+"/tomcat", repoTag2, "push") - if err != nil { - t.Errorf("Error occurred in AccessLog: %v", err) - } - - user, err := GetAccessLogCreator(currentProject.Name + "/tomcat") - if err != nil { - t.Errorf("Error occurred in GetAccessLogCreator: %v", err) - } - if user != currentUser.Username { - t.Errorf("The access log creator does not match, expected: %s, actual: %s", currentUser.Username, user) - } -} - func TestCountPull(t *testing.T) { var err error err = AccessLog(currentUser.Username, currentProject.Name, currentProject.Name+"/tomcat", repoTag2, "pull") @@ -1595,8 +1575,7 @@ func TestGetOrmer(t *testing.T) { func TestAddRepository(t *testing.T) { repoRecord := models.RepoRecord{ Name: currentProject.Name + "/" + repositoryName, - OwnerName: currentUser.Username, - ProjectName: currentProject.Name, + ProjectID: currentProject.ProjectID, Description: "testing repo", PullCount: 0, StarCount: 0, diff --git a/src/common/dao/repository.go b/src/common/dao/repository.go index cb993d393..b0d5c1f24 100644 --- a/src/common/dao/repository.go +++ b/src/common/dao/repository.go @@ -24,13 +24,15 @@ import ( // AddRepository adds a repo to the database. func AddRepository(repo models.RepoRecord) error { - o := GetOrmer() - sql := "insert into repository (owner_id, project_id, name, description, pull_count, star_count, creation_time, update_time) " + - "select (select user_id as owner_id from user where username=?), " + - "(select project_id as project_id from project where name=?), ?, ?, ?, ?, ?, NULL " + if repo.ProjectID == 0 { + return fmt.Errorf("invalid project ID: %d", repo.ProjectID) + } - _, err := o.Raw(sql, repo.OwnerName, repo.ProjectName, repo.Name, repo.Description, - repo.PullCount, repo.StarCount, time.Now()).Exec() + o := GetOrmer() + now := time.Now() + repo.CreationTime = now + repo.UpdateTime = now + _, err := o.Insert(&repo) return err } @@ -104,7 +106,7 @@ func GetRepositoryByProjectName(name string) ([]*models.RepoRecord, error) { //GetTopRepos returns the most popular repositories func GetTopRepos(userID int, count int) ([]*models.RepoRecord, error) { sql := - `select r.repository_id, r.name, r.owner_id, + `select r.repository_id, r.name, r.project_id, r.description, r.pull_count, r.star_count, r.creation_time, r.update_time from repository r diff --git a/src/common/dao/repository_test.go b/src/common/dao/repository_test.go index 9d8db1864..e13cee3ef 100644 --- a/src/common/dao/repository_test.go +++ b/src/common/dao/repository_test.go @@ -27,9 +27,8 @@ var ( project = "library" name = "library/repository-test" repository = &models.RepoRecord{ - Name: name, - OwnerName: "admin", - ProjectName: project, + Name: name, + ProjectID: 1, } ) @@ -214,9 +213,8 @@ func TestGetTopRepos(t *testing.T) { require.NoError(err) repository1 := &models.RepoRecord{ - Name: fmt.Sprintf("%v/repository1", project1.Name), - OwnerName: admin.Username, - ProjectName: project1.Name, + Name: fmt.Sprintf("%v/repository1", project1.Name), + ProjectID: project1.ProjectID, } err = AddRepository(*repository1) require.NoError(err) @@ -225,9 +223,8 @@ func TestGetTopRepos(t *testing.T) { require.NoError(err) repository2 := &models.RepoRecord{ - Name: fmt.Sprintf("%v/repository2", project1.Name), - OwnerName: admin.Username, - ProjectName: project1.Name, + Name: fmt.Sprintf("%v/repository2", project1.Name), + ProjectID: project1.ProjectID, } err = AddRepository(*repository2) require.NoError(err) @@ -237,9 +234,8 @@ func TestGetTopRepos(t *testing.T) { require.NoError(err) repository3 := &models.RepoRecord{ - Name: fmt.Sprintf("%v/repository3", project2.Name), - OwnerName: admin.Username, - ProjectName: project2.Name, + Name: fmt.Sprintf("%v/repository3", project2.Name), + ProjectID: project2.ProjectID, } err = AddRepository(*repository3) require.NoError(err) @@ -259,9 +255,8 @@ func TestGetTopRepos(t *testing.T) { deletedPublicProject.ProjectID, err = AddProject(deletedPublicProject) require.NoError(err) deletedPublicRepository1 := &models.RepoRecord{ - Name: fmt.Sprintf("%v/repository1", deletedPublicProject.Name), - OwnerName: admin.Username, - ProjectName: deletedPublicProject.Name, + Name: fmt.Sprintf("%v/repository1", deletedPublicProject.Name), + ProjectID: deletedPublicProject.ProjectID, } err = AddRepository(*deletedPublicRepository1) require.NoError(err) @@ -303,9 +298,8 @@ func TestGetTotalOfRepositoriesByProject(t *testing.T) { } if err := addRepository(&models.RepoRecord{ - Name: repoName, - OwnerName: "admin", - ProjectName: "library", + Name: repoName, + ProjectID: projectID, }); err != nil { t.Errorf("failed to add repository %s: %v", repoName, err) return @@ -333,9 +327,8 @@ func TestGetRepositoriesByProject(t *testing.T) { repoName := "library/repository" if err := addRepository(&models.RepoRecord{ - Name: repoName, - OwnerName: "admin", - ProjectName: "library", + Name: repoName, + ProjectID: projectID, }); err != nil { t.Errorf("failed to add repository %s: %v", repoName, err) return diff --git a/src/common/models/accesslog.go b/src/common/models/accesslog.go index d6a649ff3..54d1f4ec6 100644 --- a/src/common/models/accesslog.go +++ b/src/common/models/accesslog.go @@ -20,7 +20,7 @@ import ( // AccessLog holds information about logs which are used to record the actions that user take to the resourses. type AccessLog struct { - LogID int `orm:"pk;column(log_id)" json:"log_id"` + LogID int `orm:"pk;auto;column(log_id)" json:"log_id"` UserID int `orm:"column(user_id)" json:"user_id"` ProjectID int64 `orm:"column(project_id)" json:"project_id"` RepoName string `orm:"column(repo_name)" json:"repo_name"` diff --git a/src/common/models/project.go b/src/common/models/project.go index 6aeb26881..33caea050 100644 --- a/src/common/models/project.go +++ b/src/common/models/project.go @@ -20,7 +20,7 @@ import ( // Project holds the details of a project. type Project struct { - ProjectID int64 `orm:"pk;column(project_id)" json:"project_id"` + ProjectID int64 `orm:"pk;auto;column(project_id)" json:"project_id"` OwnerID int `orm:"column(owner_id)" json:"owner_id"` Name string `orm:"column(name)" json:"name"` CreationTime time.Time `orm:"column(creation_time)" json:"creation_time"` diff --git a/src/common/models/replication_job.go b/src/common/models/replication_job.go index 639f8eb87..bd391e9d1 100644 --- a/src/common/models/replication_job.go +++ b/src/common/models/replication_job.go @@ -48,7 +48,7 @@ const ( // RepPolicy is the model for a replication policy, which associate to a project and a target (destination) type RepPolicy struct { - ID int64 `orm:"column(id)" json:"id"` + ID int64 `orm:"pk;auto;column(id)" json:"id"` ProjectID int64 `orm:"column(project_id)" json:"project_id"` ProjectName string `json:"project_name,omitempty"` TargetID int64 `orm:"column(target_id)" json:"target_id"` @@ -95,7 +95,7 @@ func (r *RepPolicy) Valid(v *validation.Validation) { // RepJob is the model for a replication job, which is the execution unit on job service, currently it is used to transfer/remove // a repository to/from a remote registry instance. type RepJob struct { - ID int64 `orm:"column(id)" json:"id"` + ID int64 `orm:"pk;auto;column(id)" json:"id"` Status string `orm:"column(status)" json:"status"` Repository string `orm:"column(repository)" json:"repository"` PolicyID int64 `orm:"column(policy_id)" json:"policy_id"` @@ -109,7 +109,7 @@ type RepJob struct { // RepTarget is the model for a replication targe, i.e. destination, which wraps the endpoint URL and username/password of a remote registry. type RepTarget struct { - ID int64 `orm:"column(id)" json:"id"` + ID int64 `orm:"pk;auto;column(id)" json:"id"` URL string `orm:"column(url)" json:"endpoint"` Name string `orm:"column(name)" json:"name"` Username string `orm:"column(username)" json:"username"` diff --git a/src/common/models/repo.go b/src/common/models/repo.go index 00c01ae13..11aaffacc 100644 --- a/src/common/models/repo.go +++ b/src/common/models/repo.go @@ -20,11 +20,8 @@ import ( // RepoRecord holds the record of an repository in DB, all the infors are from the registry notification event. type RepoRecord struct { - RepositoryID string `orm:"column(repository_id);pk" json:"repository_id"` + RepositoryID int64 `orm:"pk;auto;column(repository_id)" json:"repository_id"` Name string `orm:"column(name)" json:"name"` - OwnerName string `orm:"-"` - OwnerID int64 `orm:"column(owner_id)" json:"owner_id"` - ProjectName string `orm:"-"` ProjectID int64 `orm:"column(project_id)" json:"project_id"` Description string `orm:"column(description)" json:"description"` PullCount int64 `orm:"column(pull_count)" json:"pull_count"` diff --git a/src/common/models/role.go b/src/common/models/role.go index 90b5438d1..27dc7a43b 100644 --- a/src/common/models/role.go +++ b/src/common/models/role.go @@ -25,7 +25,7 @@ const ( // Role holds the details of a role. type Role struct { - RoleID int `orm:"pk;column(role_id)" json:"role_id"` + RoleID int `orm:"pk;auto;column(role_id)" json:"role_id"` RoleCode string `orm:"column(role_code)" json:"role_code"` Name string `orm:"column(name)" json:"role_name"` diff --git a/src/common/models/user.go b/src/common/models/user.go index 67fdfb761..bdfcf5917 100644 --- a/src/common/models/user.go +++ b/src/common/models/user.go @@ -20,7 +20,7 @@ import ( // User holds the details of a user. type User struct { - UserID int `orm:"pk;column(user_id)" json:"user_id"` + UserID int `orm:"pk;auto;column(user_id)" json:"user_id"` Username string `orm:"column(username)" json:"username"` Email string `orm:"column(email)" json:"email"` Password string `orm:"column(password)" json:"password"` diff --git a/src/ui/api/dataprepare_test.go b/src/ui/api/dataprepare_test.go index 8d566288a..67e41c9c9 100644 --- a/src/ui/api/dataprepare_test.go +++ b/src/ui/api/dataprepare_test.go @@ -11,130 +11,127 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -/* - Copyright (c) 2016 VMware, Inc. All Rights Reserved. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package api - -import ( - "os" - - "github.com/vmware/harbor/src/common/dao" - "github.com/vmware/harbor/src/common/models" -) - -const ( - //Prepare Test info - TestUserName = "testUser0001" - TestUserPwd = "testUser0001" - TestUserEmail = "testUser0001@mydomain.com" - TestProName = "testProject0001" - TestTargetName = "testTarget0001" - TestRepoName = "testRepo0001" - AdminName = "admin" - DefaultProjectName = "library" -) - -func CommonAddUser() { - - commonUser := models.User{ - Username: TestUserName, - Password: TestUserPwd, - Email: TestUserEmail, - } - - _, _ = dao.Register(commonUser) - -} - -func CommonGetUserID() int { - queryUser := &models.User{ - Username: TestUserName, - } - commonUser, _ := dao.GetUser(*queryUser) - return commonUser.UserID -} - -func CommonDelUser() { - queryUser := &models.User{ - Username: TestUserName, - } - commonUser, _ := dao.GetUser(*queryUser) - _ = dao.DeleteUser(commonUser.UserID) - -} - -func CommonAddProject() { - - queryUser := &models.User{ - Username: "admin", - } - adminUser, _ := dao.GetUser(*queryUser) - commonProject := &models.Project{ - Name: TestProName, - OwnerID: adminUser.UserID, - } - - _, _ = dao.AddProject(*commonProject) - -} - -func CommonDelProject() { - commonProject, _ := dao.GetProjectByName(TestProName) - - _ = dao.DeleteProject(commonProject.ProjectID) -} - -func CommonAddTarget() { - endPoint := os.Getenv("REGISTRY_URL") - commonTarget := &models.RepTarget{ - URL: endPoint, - Name: TestTargetName, - Username: adminName, - Password: adminPwd, - } - _, _ = dao.AddRepTarget(*commonTarget) -} - -func CommonGetTarget() int { - target, _ := dao.GetRepTargetByName(TestTargetName) - return int(target.ID) -} - -func CommonDelTarget() { - target, _ := dao.GetRepTargetByName(TestTargetName) - _ = dao.DeleteRepTarget(target.ID) -} - -func CommonPolicyEabled(policyID int, enabled int) { - _ = dao.UpdateRepPolicyEnablement(int64(policyID), enabled) -} - -func CommonAddRepository() { - commonRepository := &models.RepoRecord{ - RepositoryID: "1", - Name: TestRepoName, - OwnerName: AdminName, - OwnerID: 1, - ProjectName: DefaultProjectName, - ProjectID: 1, - PullCount: 1, - } - _ = dao.AddRepository(*commonRepository) -} - -func CommonDelRepository() { - _ = dao.DeleteRepository(TestRepoName) -} +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "os" + + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" +) + +const ( + //Prepare Test info + TestUserName = "testUser0001" + TestUserPwd = "testUser0001" + TestUserEmail = "testUser0001@mydomain.com" + TestProName = "testProject0001" + TestTargetName = "testTarget0001" + TestRepoName = "testRepo0001" + AdminName = "admin" + DefaultProjectName = "library" +) + +func CommonAddUser() { + + commonUser := models.User{ + Username: TestUserName, + Password: TestUserPwd, + Email: TestUserEmail, + } + + _, _ = dao.Register(commonUser) + +} + +func CommonGetUserID() int { + queryUser := &models.User{ + Username: TestUserName, + } + commonUser, _ := dao.GetUser(*queryUser) + return commonUser.UserID +} + +func CommonDelUser() { + queryUser := &models.User{ + Username: TestUserName, + } + commonUser, _ := dao.GetUser(*queryUser) + _ = dao.DeleteUser(commonUser.UserID) + +} + +func CommonAddProject() { + + queryUser := &models.User{ + Username: "admin", + } + adminUser, _ := dao.GetUser(*queryUser) + commonProject := &models.Project{ + Name: TestProName, + OwnerID: adminUser.UserID, + } + + _, _ = dao.AddProject(*commonProject) + +} + +func CommonDelProject() { + commonProject, _ := dao.GetProjectByName(TestProName) + + _ = dao.DeleteProject(commonProject.ProjectID) +} + +func CommonAddTarget() { + endPoint := os.Getenv("REGISTRY_URL") + commonTarget := &models.RepTarget{ + URL: endPoint, + Name: TestTargetName, + Username: adminName, + Password: adminPwd, + } + _, _ = dao.AddRepTarget(*commonTarget) +} + +func CommonGetTarget() int { + target, _ := dao.GetRepTargetByName(TestTargetName) + return int(target.ID) +} + +func CommonDelTarget() { + target, _ := dao.GetRepTargetByName(TestTargetName) + _ = dao.DeleteRepTarget(target.ID) +} + +func CommonPolicyEabled(policyID int, enabled int) { + _ = dao.UpdateRepPolicyEnablement(int64(policyID), enabled) +} + +func CommonAddRepository() { + commonRepository := &models.RepoRecord{ + RepositoryID: 1, + Name: TestRepoName, + ProjectID: 1, + PullCount: 1, + } + _ = dao.AddRepository(*commonRepository) +} + +func CommonDelRepository() { + _ = dao.DeleteRepository(TestRepoName) +} diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index 1d64eff41..2b6300dad 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -45,9 +45,8 @@ type RepositoryAPI struct { } type repoResp struct { - ID string `json:"id"` + ID int64 `json:"id"` Name string `json:"name"` - OwnerID int64 `json:"owner_id"` ProjectID int64 `json:"project_id"` Description string `json:"description"` PullCount int64 `json:"pull_count"` @@ -147,7 +146,6 @@ func populateTagsCount(repositories []*models.RepoRecord) ([]*repoResp, error) { repo := &repoResp{ ID: repository.RepositoryID, Name: repository.Name, - OwnerID: repository.OwnerID, ProjectID: repository.ProjectID, Description: repository.Description, PullCount: repository.PullCount, diff --git a/src/ui/api/utils.go b/src/ui/api/utils.go index bc9b2218b..62d7cbb2c 100644 --- a/src/ui/api/utils.go +++ b/src/ui/api/utils.go @@ -270,18 +270,21 @@ func SyncRegistry() error { log.Debugf("Start adding repositories into DB... ") for _, repoToAdd := range reposToAdd { project, _ := utils.ParseRepository(repoToAdd) - user, err := dao.GetAccessLogCreator(repoToAdd) - if err != nil { - log.Errorf("Error happens when getting the repository owner from access log: %v", err) - } - if len(user) == 0 { - user = "anonymous" - } pullCount, err := dao.CountPull(repoToAdd) if err != nil { log.Errorf("Error happens when counting pull count from access log: %v", err) } - repoRecord := models.RepoRecord{Name: repoToAdd, OwnerName: user, ProjectName: project, PullCount: pullCount} + pro, err := dao.GetProjectByName(project) + if err != nil { + log.Errorf("failed to get project %s: %v", project, err) + continue + } + repoRecord := models.RepoRecord{ + Name: repoToAdd, + ProjectID: pro.ProjectID, + PullCount: pullCount, + } + if err := dao.AddRepository(repoRecord); err != nil { log.Errorf("Error happens when adding the missing repository: %v", err) } else { diff --git a/src/ui/service/notification.go b/src/ui/service/notification.go index 58e5ba72b..285c52cc4 100644 --- a/src/ui/service/notification.go +++ b/src/ui/service/notification.go @@ -76,7 +76,15 @@ func (n *NotificationHandler) Post() { return } log.Debugf("Add repository %s into DB.", repository) - repoRecord := models.RepoRecord{Name: repository, OwnerName: user, ProjectName: project} + pro, err := dao.GetProjectByName(project) + if err != nil { + log.Errorf("failed to get project %s: %v", project, err) + return + } + repoRecord := models.RepoRecord{ + Name: repository, + ProjectID: pro.ProjectID, + } if err := dao.AddRepository(repoRecord); err != nil { log.Errorf("Error happens when adding repository: %v", err) } diff --git a/tools/migration/changelog.md b/tools/migration/changelog.md index 696faf6b9..1ced7351a 100644 --- a/tools/migration/changelog.md +++ b/tools/migration/changelog.md @@ -37,3 +37,7 @@ Changelog for harbor database schema - alter column `name` on table `project`: varchar(30)->varchar(41) - create table `repository` - alter column `password` on table `replication_target`: varchar(40)->varchar(128) + +## 1.2.0 + + - delete column `owner_id` from table `repository`