upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor synced 2024-09-21 05:39:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #752 from ywk253100/dev_b

Browse Source 
Recover lost commit: improve the performance of database when paginating large records for access logs, jobs and getting the top repositories
This commit is contained in:
Wenkai Yin 2016-09-02 18:48:14 +08:00 committed by GitHub
commit 8dd75c9b85
4 changed files with 102 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Deploy/db/registry.sql
View File

@ -107,6 +107,7 @@ create table access_log (
operation varchar(20) NOT NULL,
op_time timestamp,
primary key (log_id),
INDEX pid_optime (project_id, op_time),
FOREIGN KEY (user_id) REFERENCES user(user_id),
FOREIGN KEY (project_id) REFERENCES project (project_id)
);
@ -169,7 +170,8 @@ create table replication_job (
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
PRIMARY KEY (id),
INDEX policy (policy_id)
INDEX policy (policy_id),
INDEX poid_uptime (policy_id, update_time)
);
create table properties (

9
api/project.go
View File

@ -339,13 +339,18 @@ func (p *ProjectAPI) FilterAccessLog() {
p.DecodeJSONReq(&query)
query.ProjectID = p.projectID
query.Username = "%" + query.Username + "%"
query.BeginTime = time.Unix(query.BeginTimestamp, 0)
query.EndTime = time.Unix(query.EndTimestamp, 0)
page, pageSize := p.getPaginationParams()
logs, total, err := dao.GetAccessLogs(query, pageSize, pageSize*(page-1))
total, err := dao.GetTotalOfAccessLogs(query)
if err != nil {
log.Errorf("failed to get total of access log: %v", err)
p.CustomAbort(http.StatusInternalServerError, "")
}
logs, err := dao.GetAccessLogs(query, pageSize, pageSize*(page-1))
if err != nil {
log.Errorf("failed to get access log: %v", err)
p.CustomAbort(http.StatusInternalServerError, "")

121
dao/accesslog.go
View File

@ -38,79 +38,106 @@ func AddAccessLog(accessLog models.AccessLog) error {
return err
}
//GetAccessLogs gets access logs according to different conditions
func GetAccessLogs(query models.AccessLog, limit, offset int64) ([]models.AccessLog, int64, error) {
// GetTotalOfAccessLogs ...
func GetTotalOfAccessLogs(query models.AccessLog) (int64, error) {
o := GetOrmer()
condition := ` from access_log a left join user u on a.user_id = u.user_id
where a.project_id = ? `
queryParam := make([]interface{}, 1)
queryParam := []interface{}{}
sql := `select count(*) from access_log al
where al.project_id = ?`
queryParam = append(queryParam, query.ProjectID)
if query.UserID != 0 {
condition += ` and a.user_id = ? `
queryParam = append(queryParam, query.UserID)
}
if query.Operation != "" {
condition += ` and a.operation = ? `
queryParam = append(queryParam, query.Operation)
}
if query.Username != "" {
condition += ` and u.username like ? `
queryParam = append(queryParam, query.Username)
sql = `select count(*) from access_log al
left join user u
on al.user_id = u.user_id
where al.project_id = ? and u.username like ? `
queryParam = append(queryParam, "%"+query.Username+"%")
}
sql += genFilterClauses(query, &queryParam)
var total int64
if err := o.Raw(sql, queryParam).QueryRow(&total); err != nil {
return 0, err
}
return total, nil
}
//GetAccessLogs gets access logs according to different conditions
func GetAccessLogs(query models.AccessLog, limit, offset int64) ([]models.AccessLog, error) {
o := GetOrmer()
queryParam := []interface{}{}
sql := `select al.log_id, u.username, al.repo_name,
al.repo_tag, al.operation, al.op_time
from access_log al
left join user u
on al.user_id = u.user_id
where al.project_id = ? `
queryParam = append(queryParam, query.ProjectID)
if query.Username != "" {
sql += ` and u.username like ? `
queryParam = append(queryParam, "%"+query.Username+"%")
}
sql += genFilterClauses(query, &queryParam)
sql += ` order by al.op_time desc `
sql = paginateForRawSQL(sql, limit, offset)
logs := []models.AccessLog{}
_, err := o.Raw(sql, queryParam).QueryRows(&logs)
if err != nil {
return logs, err
}
return logs, nil
}
func genFilterClauses(query models.AccessLog, queryParam *[]interface{}) string {
sql := ""
if query.Operation != "" {
sql += ` and al.operation = ? `
*queryParam = append(*queryParam, query.Operation)
}
if query.RepoName != "" {
condition += ` and a.repo_name = ? `
queryParam = append(queryParam, query.RepoName)
sql += ` and al.repo_name = ? `
*queryParam = append(*queryParam, query.RepoName)
}
if query.RepoTag != "" {
condition += ` and a.repo_tag = ? `
queryParam = append(queryParam, query.RepoTag)
sql += ` and al.repo_tag = ? `
*queryParam = append(*queryParam, query.RepoTag)
}
if query.Keywords != "" {
condition += ` and a.operation in ( `
sql += ` and al.operation in ( `
keywordList := strings.Split(query.Keywords, "/")
num := len(keywordList)
for i := 0; i < num; i++ {
if keywordList[i] != "" {
if i == num-1 {
condition += `?)`
sql += `?)`
} else {
condition += `?,`
sql += `?,`
}
queryParam = append(queryParam, keywordList[i])
*queryParam = append(*queryParam, keywordList[i])
}
}
}
if query.BeginTimestamp > 0 {
condition += ` and a.op_time >= ? `
queryParam = append(queryParam, query.BeginTime)
sql += ` and al.op_time >= ? `
*queryParam = append(*queryParam, query.BeginTime)
}
if query.EndTimestamp > 0 {
condition += ` and a.op_time <= ? `
queryParam = append(queryParam, query.EndTime)
sql += ` and al.op_time <= ? `
*queryParam = append(*queryParam, query.EndTime)
}
condition += ` order by a.op_time desc `
totalSQL := `select count(*) ` + condition
logs := []models.AccessLog{}
var total int64
if err := o.Raw(totalSQL, queryParam).QueryRow(&total); err != nil {
return logs, 0, err
}
condition = paginateForRawSQL(condition, limit, offset)
recordsSQL := `select a.log_id, u.username, a.repo_name, a.repo_tag, a.operation, a.op_time ` + condition
_, err := o.Raw(recordsSQL, queryParam).QueryRows(&logs)
if err != nil {
return logs, 0, err
}
return logs, total, nil
return sql
}
// AccessLog ...

21
dao/dao_test.go
View File

@ -458,7 +458,7 @@ func TestGetAccessLog(t *testing.T) {
UserID: currentUser.UserID,
ProjectID: currentProject.ProjectID,
}
accessLogs, _, err := GetAccessLogs(queryAccessLog, 1000, 0)
accessLogs, err := GetAccessLogs(queryAccessLog, 1000, 0)
if err != nil {
t.Errorf("Error occurred in GetAccessLog: %v", err)
}
@ -470,6 +470,21 @@ func TestGetAccessLog(t *testing.T) {
}
}
func TestGetTotalOfAccessLogs(t *testing.T) {
queryAccessLog := models.AccessLog{
UserID: currentUser.UserID,
ProjectID: currentProject.ProjectID,
}
total, err := GetTotalOfAccessLogs(queryAccessLog)
if err != nil {
t.Fatalf("failed to get total of access log: %v", err)
}
if total != 1 {
t.Errorf("unexpected total %d != %d", total, 1)
}
}
func TestAddAccessLog(t *testing.T) {
var err error
var accessLogList []models.AccessLog
@ -486,7 +501,7 @@ func TestAddAccessLog(t *testing.T) {
if err != nil {
t.Errorf("Error occurred in AddAccessLog: %v", err)
}
accessLogList, _, err = GetAccessLogs(accessLog, 1000, 0)
accessLogList, err = GetAccessLogs(accessLog, 1000, 0)
if err != nil {
t.Errorf("Error occurred in GetAccessLog: %v", err)
}
@ -515,7 +530,7 @@ func TestAccessLog(t *testing.T) {
if err != nil {
t.Errorf("Error occurred in AccessLog: %v", err)
}
accessLogList, _, err = GetAccessLogs(accessLog, 1000, 0)
accessLogList, err = GetAccessLogs(accessLog, 1000, 0)
if err != nil {
t.Errorf("Error occurred in GetAccessLog: %v", err)
}