Merge pull request #2461 from ywk253100/170607_log_api

Support query logs according to different conditions

docs/swagger.yaml

@@ -217,8 +217,8 @@ paths:
           description: Project ID does not exist.
         500:
           description: Unexpected internal errors.
-  /projects/{project_id}/logs/filter:
+  /projects/{project_id}/logs:
-    post:
+    get:
       summary: Get access logs accompany with a relevant project.
       description: |
         This endpoint let user search access logs filtered by operations and date time ranges.
@@ -229,11 +229,36 @@ paths:
           format: int64
           required: true
           description: Relevant project ID
-        - name: access_log
+        - name: username
-          in: body
+          in: query
-          schema:
+          type: string
-            $ref: '#/definitions/AccessLogFilter'
+          required: false
-          description: Search results of access logs.
+          description: Username of the operator.
+        - name: repository
+          in: query
+          type: string
+          required: false
+          description: The name of repository
+        - name: tag
+          in: query
+          type: string
+          required: false
+          description: The name of tag
+        - name: operation
+          in: query
+          type: string
+          required: false
+          description: The operation
+        - name: begin_timestamp
+          in: query
+          type: string
+          required: false
+          description: The begin timestamp
+        - name: end_timestamp
+          in: query
+          type: string
+          required: false
+          description: The end timestamp
         - name: page
           in: query
           type: integer
@@ -861,6 +886,36 @@ paths:
       description: |
         This endpoint let user see the recent operation logs of the projects which he is member of 
       parameters:
+        - name: username
+          in: query
+          type: string
+          required: false
+          description: Username of the operator.
+        - name: repository
+          in: query
+          type: string
+          required: false
+          description: The name of repository
+        - name: tag
+          in: query
+          type: string
+          required: false
+          description: The name of tag
+        - name: operation
+          in: query
+          type: string
+          required: false
+          description: The operation
+        - name: begin_timestamp
+          in: query
+          type: string
+          required: false
+          description: The begin timestamp
+        - name: end_timestamp
+          in: query
+          type: string
+          required: false
+          description: The end timestamp
         - name: page
           in: query
           type: integer

src/common/dao/accesslog.go

@@ -67,13 +67,19 @@ func logQueryConditions(query *models.LogQueryParam) orm.QuerySeter {
 		qs = qs.Filter("username__contains", query.Username)
 	}
 	if len(query.Repository) != 0 {
-		qs = qs.Filter("repo_name", query.Repository)
+		qs = qs.Filter("repo_name__contains", query.Repository)
 	}
 	if len(query.Tag) != 0 {
-		qs = qs.Filter("repo_tag", query.Tag)
+		qs = qs.Filter("repo_tag__contains", query.Tag)
 	}
-	if len(query.Operations) > 0 {
+	operations := []string{}
-		qs = qs.Filter("operation__in", query.Operations)
+	for _, operation := range query.Operations {
+		if len(operation) > 0 {
+			operations = append(operations, operation)
+		}
+	}
+	if len(operations) > 0 {
+		qs = qs.Filter("operation__in", operations)
 	}
 	if query.BeginTime != nil {
 		qs = qs.Filter("op_time__gte", query.BeginTime)

src/common/models/accesslog.go

@@ -19,19 +19,15 @@ import (
 )
 
 // AccessLog holds information about logs which are used to record the actions that user take to the resourses.
-// TODO remove useless attrs
 type AccessLog struct {
-	LogID          int       `orm:"pk;auto;column(log_id)" json:"log_id"`
+	LogID     int       `orm:"pk;auto;column(log_id)" json:"log_id"`
-	Username       string    `orm:"column(username)"  json:"username"`
+	Username  string    `orm:"column(username)"  json:"username"`
-	ProjectID      int64     `orm:"column(project_id)"  json:"project_id"`
+	ProjectID int64     `orm:"column(project_id)"  json:"project_id"`
-	RepoName       string    `orm:"column(repo_name)" json:"repo_name"`
+	RepoName  string    `orm:"column(repo_name)" json:"repo_name"`
-	RepoTag        string    `orm:"column(repo_tag)" json:"repo_tag"`
+	RepoTag   string    `orm:"column(repo_tag)" json:"repo_tag"`
-	GUID           string    `orm:"column(GUID)"  json:"guid"`
+	GUID      string    `orm:"column(GUID)"  json:"guid"`
-	Operation      string    `orm:"column(operation)" json:"operation"`
+	Operation string    `orm:"column(operation)" json:"operation"`
-	OpTime         time.Time `orm:"column(op_time)" json:"op_time"`
+	OpTime    time.Time `orm:"column(op_time)" json:"op_time"`
-	Keywords       string    `orm:"-" json:"keywords"`
-	BeginTimestamp int64     `orm:"-" json:"begin_timestamp"`
-	EndTimestamp   int64     `orm:"-" json:"end_timestamp"`
 }
 
 // LogQueryParam is used to set query conditions when listing

src/common/utils/utils.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
@@ -114,3 +115,13 @@ func TestTCPConn(addr string, timeout, interval int) error {
 		return fmt.Errorf("failed to connect to tcp:%s after %d seconds", addr, timeout)
 	}
 }
+
+// ParseTimeStamp parse timestamp to time
+func ParseTimeStamp(timestamp string) (*time.Time, error) {
+	i, err := strconv.ParseInt(timestamp, 10, 64)
+	if err != nil {
+		return nil, err
+	}
+	t := time.Unix(i, 0)
+	return &t, nil
+}

src/common/utils/utils_test.go

@@ -17,8 +17,12 @@ package utils
 import (
 	"encoding/base64"
 	"net/http/httptest"
+	"strconv"
 	"strings"
 	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestParseEndpoint(t *testing.T) {
@@ -191,3 +195,19 @@ func TestTestTCPConn(t *testing.T) {
 		t.Fatalf("failed to test tcp connection of %s: %v", addr, err)
 	}
 }
+
+func TestParseTimeStamp(t *testing.T) {
+	// invalid input
+	_, err := ParseTimeStamp("")
+	assert.NotNil(t, err)
+
+	// invalid input
+	_, err = ParseTimeStamp("invalid")
+	assert.NotNil(t, err)
+
+	// valid
+	now := time.Now().Unix()
+	result, err := ParseTimeStamp(strconv.FormatInt(now, 10))
+	assert.Nil(t, err)
+	assert.Equal(t, now, result.Unix())
+}

src/ui/api/harborapi_test.go

@@ -96,7 +96,7 @@ func init() {
 	beego.Router("/api/users/:id([0-9]+)/password", &UserAPI{}, "put:ChangePassword")
 	beego.Router("/api/users/:id/sysadmin", &UserAPI{}, "put:ToggleUserAdminRole")
 	beego.Router("/api/projects/:id/publicity", &ProjectAPI{}, "put:ToggleProjectPublic")
-	beego.Router("/api/projects/:id([0-9]+)/logs/filter", &ProjectAPI{}, "post:FilterAccessLog")
+	beego.Router("/api/projects/:id([0-9]+)/logs", &ProjectAPI{}, "get:Logs")
 	beego.Router("/api/projects/:pid([0-9]+)/members/?:mid", &ProjectMemberAPI{}, "get:Get;post:Post;delete:Delete;put:Put")
 	beego.Router("/api/repositories", &RepositoryAPI{})
 	beego.Router("/api/statistics", &StatisticAPI{})
@@ -379,27 +379,12 @@ func (a testapi) ToggleProjectPublicity(prjUsr usrInfo, projectID string, ispubl
 }
 
 //Get access logs accompany with a relevant project.
-func (a testapi) ProjectLogsFilter(prjUsr usrInfo, projectID string, accessLog apilib.AccessLogFilter) (int, []byte, error) {
+func (a testapi) ProjectLogs(prjUsr usrInfo, projectID string, query *apilib.LogQuery) (int, []byte, error) {
-	//func (a testapi) ProjectLogsFilter(prjUsr usrInfo, projectID string, accessLog apilib.AccessLog) (int, apilib.AccessLog, error) {
+	_sling := sling.New().Get(a.basePath).
-	_sling := sling.New().Post(a.basePath)
+		Path("/api/projects/" + projectID + "/logs").
+		QueryStruct(query)
 
-	path := "/api/projects/" + projectID + "/logs/filter"
+	return request(_sling, jsonAcceptHeader, prjUsr)
-
-	_sling = _sling.Path(path)
-
-	// body params
-	_sling = _sling.BodyJSON(accessLog)
-
-	//var successPayload []apilib.AccessLog
-
-	httpStatusCode, body, err := request(_sling, jsonAcceptHeader, prjUsr)
-	/*
-		if err == nil && httpStatusCode == 200 {
-			err = json.Unmarshal(body, &successPayload)
-		}
-	*/
-	return httpStatusCode, body, err
-	//	return httpStatusCode, successPayload, err
 }
 
 //-------------------------Member Test---------------------------------------//

src/ui/api/log.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils"
 )
 
 //LogAPI handles request api/logs
@@ -43,12 +44,36 @@ func (l *LogAPI) Prepare() {
 func (l *LogAPI) Get() {
 	page, size := l.GetPaginationParams()
 	query := &models.LogQueryParam{
+		Username:   l.GetString("username"),
+		Repository: l.GetString("repository"),
+		Tag:        l.GetString("tag"),
+		Operations: l.GetStrings("operation"),
 		Pagination: &models.Pagination{
 			Page: page,
 			Size: size,
 		},
 	}
 
+	timestamp := l.GetString("begin_timestamp")
+	if len(timestamp) > 0 {
+		t, err := utils.ParseTimeStamp(timestamp)
+		if err != nil {
+			l.HandleBadRequest(fmt.Sprintf("invalid begin_timestamp: %s", timestamp))
+			return
+		}
+		query.BeginTime = t
+	}
+
+	timestamp = l.GetString("end_timestamp")
+	if len(timestamp) > 0 {
+		t, err := utils.ParseTimeStamp(timestamp)
+		if err != nil {
+			l.HandleBadRequest(fmt.Sprintf("invalid end_timestamp: %s", timestamp))
+			return
+		}
+		query.EndTime = t
+	}
+
 	if !l.isSysAdmin {
 		projects, err := l.ProjectMgr.GetByMember(l.username)
 		if err != nil {

src/ui/api/project.go

@@ -18,11 +18,11 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 
 	"github.com/vmware/harbor/src/common"
 	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/ui/config"
 
@@ -368,8 +368,8 @@ func (p *ProjectAPI) ToggleProjectPublic() {
 	}
 }
 
-// FilterAccessLog handles GET to /api/projects/{}/logs
+// Logs ...
-func (p *ProjectAPI) FilterAccessLog() {
+func (p *ProjectAPI) Logs() {
 	if !p.SecurityCtx.IsAuthenticated() {
 		p.HandleUnauthorized()
 		return
@@ -380,51 +380,54 @@ func (p *ProjectAPI) FilterAccessLog() {
 		return
 	}
 
-	var query models.AccessLog
+	page, size := p.GetPaginationParams()
-	p.DecodeJSONReq(&query)
+	query := &models.LogQueryParam{
-
-	queryParm := &models.LogQueryParam{
 		ProjectIDs: []int64{p.project.ProjectID},
-		Username:   query.Username,
+		Username:   p.GetString("username"),
-		Repository: query.RepoName,
+		Repository: p.GetString("repository"),
-		Tag:        query.RepoTag,
+		Tag:        p.GetString("tag"),
+		Operations: p.GetStrings("operation"),
+		Pagination: &models.Pagination{
+			Page: page,
+			Size: size,
+		},
 	}
 
-	if len(query.Keywords) > 0 {
+	timestamp := p.GetString("begin_timestamp")
-		queryParm.Operations = strings.Split(query.Keywords, "/")
+	if len(timestamp) > 0 {
+		t, err := utils.ParseTimeStamp(timestamp)
+		if err != nil {
+			p.HandleBadRequest(fmt.Sprintf("invalid begin_timestamp: %s", timestamp))
+			return
+		}
+		query.BeginTime = t
 	}
 
-	if query.BeginTimestamp > 0 {
+	timestamp = p.GetString("end_timestamp")
-		beginTime := time.Unix(query.BeginTimestamp, 0)
+	if len(timestamp) > 0 {
-		queryParm.BeginTime = &beginTime
+		t, err := utils.ParseTimeStamp(timestamp)
+		if err != nil {
+			p.HandleBadRequest(fmt.Sprintf("invalid end_timestamp: %s", timestamp))
+			return
+		}
+		query.EndTime = t
 	}
 
-	if query.EndTimestamp > 0 {
+	total, err := dao.GetTotalOfAccessLogs(query)
-		endTime := time.Unix(query.EndTimestamp, 0)
-		queryParm.EndTime = &endTime
-	}
-
-	page, pageSize := p.GetPaginationParams()
-	queryParm.Pagination = &models.Pagination{
-		Page: page,
-		Size: pageSize,
-	}
-
-	total, err := dao.GetTotalOfAccessLogs(queryParm)
 	if err != nil {
 		p.HandleInternalServerError(fmt.Sprintf(
 			"failed to get total of access log: %v", err))
 		return
 	}
 
-	logs, err := dao.GetAccessLogs(queryParm)
+	logs, err := dao.GetAccessLogs(query)
 	if err != nil {
 		p.HandleInternalServerError(fmt.Sprintf(
 			"failed to get access log: %v", err))
 		return
 	}
 
-	p.SetPaginationHeader(total, page, pageSize)
+	p.SetPaginationHeader(total, page, size)
 	p.Data["json"] = logs
 	p.ServeJSON()
 }

src/ui/api/project_test.go

@@ -320,19 +320,19 @@ func TestProjectLogsFilter(t *testing.T) {
 
 	apiTest := newHarborAPI()
 
-	endTimestamp := time.Now().Unix()
+	query := &apilib.LogQuery{
-	startTimestamp := endTimestamp - 3600
-	accessLog := &apilib.AccessLogFilter{
 		Username:       "admin",
-		Keywords:       "",
+		Repository:     "",
-		BeginTimestamp: startTimestamp,
+		Tag:            "",
-		EndTimestamp:   endTimestamp,
+		Operation:      []string{""},
+		BeginTimestamp: 0,
+		EndTimestamp:   time.Now().Unix(),
 	}
 
 	//-------------------case1: Response Code=200------------------------------//
 	fmt.Println("case 1: respose code:200")
 	projectID := "1"
-	httpStatusCode, _, err := apiTest.ProjectLogsFilter(*admin, projectID, *accessLog)
+	httpStatusCode, _, err := apiTest.ProjectLogs(*admin, projectID, query)
 	if err != nil {
 		t.Error("Error while search access logs")
 		t.Log(err)
@@ -342,7 +342,7 @@ func TestProjectLogsFilter(t *testing.T) {
 	//-------------------case2: Response Code=401:User need to log in first.------------------------------//
 	fmt.Println("case 2: respose code:401:User need to log in first.")
 	projectID = "1"
-	httpStatusCode, _, err = apiTest.ProjectLogsFilter(*unknownUsr, projectID, *accessLog)
+	httpStatusCode, _, err = apiTest.ProjectLogs(*unknownUsr, projectID, query)
 	if err != nil {
 		t.Error("Error while search access logs")
 		t.Log(err)
@@ -352,7 +352,7 @@ func TestProjectLogsFilter(t *testing.T) {
 	//-------------------case3: Response Code=404:Project does not exist.-------------------------//
 	fmt.Println("case 3: respose code:404:Illegal format of provided ID value.")
 	projectID = "11111"
-	httpStatusCode, _, err = apiTest.ProjectLogsFilter(*admin, projectID, *accessLog)
+	httpStatusCode, _, err = apiTest.ProjectLogs(*admin, projectID, query)
 	if err != nil {
 		t.Error("Error while search access logs")
 		t.Log(err)

src/ui/router.go

@@ -66,7 +66,7 @@ func initRouters() {
 	beego.Router("/api/projects/", &api.ProjectAPI{}, "get:List;post:Post;head:Head")
 	beego.Router("/api/projects/:id([0-9]+)", &api.ProjectAPI{})
 	beego.Router("/api/projects/:id([0-9]+)/publicity", &api.ProjectAPI{}, "put:ToggleProjectPublic")
-	beego.Router("/api/projects/:id([0-9]+)/logs/filter", &api.ProjectAPI{}, "post:FilterAccessLog")
+	beego.Router("/api/projects/:id([0-9]+)/logs", &api.ProjectAPI{}, "get:Logs")
 	beego.Router("/api/statistics", &api.StatisticAPI{})
 	beego.Router("/api/users/?:id", &api.UserAPI{})
 	beego.Router("/api/users/:id([0-9]+)/password", &api.UserAPI{}, "put:ChangePassword")

tests/apitests/apilib/access_log_filter.go

@@ -1,10 +1,10 @@
-/* 
+/*
  * Harbor API
  *
  * These APIs provide services for manipulating Harbor project.
  *
  * OpenAPI spec version: 0.3.0
- * 
+ *
  * Generated by: https://github.com/swagger-api/swagger-codegen.git
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,17 +22,13 @@
 
 package apilib
 
-type AccessLogFilter struct {
+type LogQuery struct {
-
+	Username       string   `json:"username"`
-	// Relevant user's name that accessed this project.
+	Repository     string   `json:"repository"`
-	Username string `json:"username,omitempty"`
+	Tag            string   `json:"tag"`
-
+	Operation      []string `json:"operation"`
-	// Operation name specified when project created.
+	BeginTimestamp int64    `json:"begin_timestamp"`
-	Keywords string `json:"keywords,omitempty"`
+	EndTimestamp   int64    `json:"end_timestamp"`
-
+	Page           int64    `json:"page"`
-	// Begin timestamp for querying access logs.
+	PageSize       int64    `json:"page_size"`
-	BeginTimestamp int64 `json:"begin_timestamp,omitempty"`
-
-	// End timestamp for querying accessl logs.
-	EndTimestamp int64 `json:"end_timestamp,omitempty"`
 }