upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor synced 2024-09-20 18:39:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #8536 from bitsf/tag_retention_task_num

Browse Source 
add task retain num
This commit is contained in:
Steven Zou 2019-08-07 17:39:39 +08:00 committed by GitHub
commit f3ba25f656
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 197 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

207
make/migrations/postgresql/0010_1.9.0_schema.up.sql
View File

@ -1,138 +1,141 @@
/* add table for CVE whitelist */
CREATE TABLE cve_whitelist (
id SERIAL PRIMARY KEY NOT NULL,
project_id int,
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP,
expires_at bigint,
items text NOT NULL,
UNIQUE (project_id)
CREATE TABLE cve_whitelist
(
id SERIAL PRIMARY KEY NOT NULL,
project_id int,
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP,
expires_at bigint,
items text NOT NULL,
UNIQUE (project_id)
);
CREATE TABLE blob (
id SERIAL PRIMARY KEY NOT NULL,
/*
digest of config, layer, manifest
*/
digest varchar(255) NOT NULL,
content_type varchar(255) NOT NULL,
size int NOT NULL,
creation_time timestamp default CURRENT_TIMESTAMP,
UNIQUE (digest)
CREATE TABLE blob
(
id SERIAL PRIMARY KEY NOT NULL,
/*
digest of config, layer, manifest
*/
digest varchar(255) NOT NULL,
content_type varchar(255) NOT NULL,
size int NOT NULL,
creation_time timestamp default CURRENT_TIMESTAMP,
UNIQUE (digest)
);
CREATE TABLE artifact (
id SERIAL PRIMARY KEY NOT NULL,
project_id int NOT NULL,
repo varchar(255) NOT NULL,
tag varchar(255) NOT NULL,
/*
digest of manifest
*/
digest varchar(255) NOT NULL,
/*
kind of artifact, image, chart, etc..
*/
kind varchar(255) NOT NULL,
creation_time timestamp default CURRENT_TIMESTAMP,
pull_time timestamp,
push_time timestamp,
CONSTRAINT unique_artifact UNIQUE (project_id, repo, tag)
CREATE TABLE artifact
(
id SERIAL PRIMARY KEY NOT NULL,
project_id int NOT NULL,
repo varchar(255) NOT NULL,
tag varchar(255) NOT NULL,
/*
digest of manifest
*/
digest varchar(255) NOT NULL,
/*
kind of artifact, image, chart, etc..
*/
kind varchar(255) NOT NULL,
creation_time timestamp default CURRENT_TIMESTAMP,
pull_time timestamp,
push_time timestamp,
CONSTRAINT unique_artifact UNIQUE (project_id, repo, tag)
);
/* add the table for relation of artifact and blob */
CREATE TABLE artifact_blob (
id SERIAL PRIMARY KEY NOT NULL,
digest_af varchar(255) NOT NULL,
digest_blob varchar(255) NOT NULL,
creation_time timestamp default CURRENT_TIMESTAMP,
CONSTRAINT unique_artifact_blob UNIQUE (digest_af, digest_blob)
CREATE TABLE artifact_blob
(
id SERIAL PRIMARY KEY NOT NULL,
digest_af varchar(255) NOT NULL,
digest_blob varchar(255) NOT NULL,
creation_time timestamp default CURRENT_TIMESTAMP,
CONSTRAINT unique_artifact_blob UNIQUE (digest_af, digest_blob)
);
/* add quota table */
CREATE TABLE quota (
id SERIAL PRIMARY KEY NOT NULL,
reference VARCHAR(255) NOT NULL,
reference_id VARCHAR(255) NOT NULL,
hard JSONB NOT NULL,
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP,
UNIQUE(reference, reference_id)
CREATE TABLE quota
(
id SERIAL PRIMARY KEY NOT NULL,
reference VARCHAR(255) NOT NULL,
reference_id VARCHAR(255) NOT NULL,
hard JSONB NOT NULL,
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP,
UNIQUE (reference, reference_id)
);
/* add quota usage table */
CREATE TABLE quota_usage (
id SERIAL PRIMARY KEY NOT NULL,
reference VARCHAR(255) NOT NULL,
reference_id VARCHAR(255) NOT NULL,
used JSONB NOT NULL,
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP,
UNIQUE(reference, reference_id)
CREATE TABLE quota_usage
(
id SERIAL PRIMARY KEY NOT NULL,
reference VARCHAR(255) NOT NULL,
reference_id VARCHAR(255) NOT NULL,
used JSONB NOT NULL,
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP,
UNIQUE (reference, reference_id)
);
INSERT INTO quota (reference, reference_id, hard, creation_time, update_time)
SELECT
'project',
CAST(project_id AS VARCHAR),
'{"count": -1, "storage": -1}',
NOW(),
NOW()
FROM
project
WHERE
deleted = 'f';
SELECT 'project',
CAST(project_id AS VARCHAR),
'{"count": -1, "storage": -1}',
NOW(),
NOW()
FROM project
WHERE deleted = 'f';
INSERT INTO quota_usage (id, reference, reference_id, used, creation_time, update_time)
SELECT
id,
reference,
reference_id,
'{"count": 0, "storage": 0}',
creation_time,
update_time
FROM
quota;
SELECT id,
reference,
reference_id,
'{"count": 0, "storage": 0}',
creation_time,
update_time
FROM quota;
create table retention_policy
(
id serial PRIMARY KEY NOT NULL,
scope_level varchar(20),
scope_reference integer,
trigger_kind varchar(20),
data text,
create_time time,
update_time time
id serial PRIMARY KEY NOT NULL,
scope_level varchar(20),
scope_reference integer,
trigger_kind varchar(20),
data text,
create_time time,
update_time time
);
create table retention_execution
(
id serial PRIMARY KEY NOT NULL,
policy_id integer,
dry_run boolean,
trigger varchar(20),
start_time timestamp
id serial PRIMARY KEY NOT NULL,
policy_id integer,
dry_run boolean,
trigger varchar(20),
start_time timestamp
);
create table retention_task
(
id SERIAL NOT NULL,
execution_id integer,
repository varchar(255),
job_id varchar(64),
status varchar(32),
start_time timestamp default CURRENT_TIMESTAMP,
end_time timestamp default CURRENT_TIMESTAMP,
PRIMARY KEY (id)
id SERIAL NOT NULL,
execution_id integer,
repository varchar(255),
job_id varchar(64),
status varchar(32),
start_time timestamp default CURRENT_TIMESTAMP,
end_time timestamp default CURRENT_TIMESTAMP,
total integer,
retained integer,
PRIMARY KEY (id)
);
create table schedule
(
id SERIAL NOT NULL,
job_id varchar(64),
status varchar(64),
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP,
PRIMARY KEY (id)
id SERIAL NOT NULL,
job_id varchar(64),
status varchar(64),
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP,
PRIMARY KEY (id)
);

23
src/core/service/notifications/jobs/handler.go
View File

@ -46,6 +46,7 @@ type Handler struct {
id int64
status string
rawStatus string
checkIn string
}
// Prepare ...
@ -73,6 +74,7 @@ func (h *Handler) Prepare() {
return
}
h.status = status
h.checkIn = data.CheckIn
}
// HandleScan handles the webhook of scan job
@ -117,8 +119,27 @@ func (h *Handler) HandleRetentionTask() {
if h.status == models.JobFinished || h.status == models.JobError ||
h.status == models.JobStopped {
task.EndTime = time.Now()
props = append(props, "EndTime")
} else if h.status == models.JobRunning {
if h.checkIn != "" {
var retainObj struct {
Total int `json:"total"`
Retained int `json:"retained"`
}
if err := json.Unmarshal([]byte(h.checkIn), &retainObj); err != nil {
log.Errorf("failed to resolve checkin of retention task %d: %v", h.id, err)
} else {
if retainObj.Total > 0 {
task.Total = retainObj.Total
props = append(props, "Total")
}
if retainObj.Retained > 0 {
task.Retained = retainObj.Retained
props = append(props, "Retained")
}
}
}
}
props = append(props, "EndTime")
if err := mgr.UpdateTask(task, props...); err != nil {
log.Errorf("failed to update the status of retention task %d: %v", h.id, err)
h.SendInternalServerError(err)

2
src/pkg/retention/dao/models/retention.go
View File

@ -56,4 +56,6 @@ type RetentionTask struct {
Status string `orm:"column(status)"`
StartTime time.Time `orm:"column(start_time)"`
EndTime time.Time `orm:"column(end_time)"`
Total int `orm:"column(total)"`
Retained int `orm:"column(retained)"`
}

24
src/pkg/retention/job.go
View File

@ -16,6 +16,7 @@ package retention
import (
"bytes"
"encoding/json"
"fmt"
"strings"
"time"
@ -111,6 +112,29 @@ func (pj *Job) Run(ctx job.Context, params job.Parameters) error {
// Log stage: results with table view
logResults(myLogger, allCandidates, results)
// Save retain and total num in DB
return saveRetainNum(ctx, results, allCandidates)
}
func saveRetainNum(ctx job.Context, retained []*res.Result, allCandidates []*res.Candidate) error {
var delNum int
for _, r := range retained {
if r.Error == nil {
delNum++
}
}
retainObj := struct {
Total int `json:"total"`
Retained int `json:"retained"`
}{
Total: len(allCandidates),
Retained: len(allCandidates) - delNum,
}
c, err := json.Marshal(retainObj)
if err != nil {
return err
}
_ = ctx.Checkin(string(c))
return nil
}

8
src/pkg/retention/manager.go
View File

@ -194,6 +194,8 @@ func (d *DefaultManager) CreateTask(task *Task) (int64, error) {
Status: task.Status,
StartTime: task.StartTime,
EndTime: task.EndTime,
Total: task.Total,
Retained: task.Retained,
}
return dao.CreateTask(t)
}
@ -217,6 +219,8 @@ func (d *DefaultManager) ListTasks(query ...*q.TaskQuery) ([]*Task, error) {
Status: t.Status,
StartTime: t.StartTime,
EndTime: t.EndTime,
Total: t.Total,
Retained: t.Retained,
})
}
return tasks, nil
@ -243,6 +247,8 @@ func (d *DefaultManager) UpdateTask(task *Task, cols ...string) error {
Status: task.Status,
StartTime: task.StartTime,
EndTime: task.EndTime,
Total: task.Total,
Retained: task.Retained,
}, cols...)
}
@ -263,6 +269,8 @@ func (d *DefaultManager) GetTask(taskID int64) (*Task, error) {
Status: task.Status,
StartTime: task.StartTime,
EndTime: task.EndTime,
Total: task.Total,
Retained: task.Retained,
}, nil
}

2
src/pkg/retention/models.go
View File

@ -56,6 +56,8 @@ type Task struct {
Status string `json:"status"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
Total int `json:"total"`
Retained int `json:"retained"`
}
// History of retention

93
src/portal/package-lock.json generated
View File

@ -176,15 +176,13 @@
"version": "1.40.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz",
"integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==",
"dev": true,
"optional": true
"dev": true
},
"mime-types": {
"version": "2.1.24",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz",
"integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==",
"dev": true,
"optional": true,
"requires": {
"mime-db": "1.40.0"
}
@ -9254,10 +9252,13 @@
}
},
"karma-jasmine": {
"version": "1.1.2",
"resolved": "http://registry.npm.taobao.org/karma-jasmine/download/karma-jasmine-1.1.2.tgz",
"integrity": "sha1-OU8rJf+0pkS5rabyLUQ+L9CIhsM=",
"dev": true
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/karma-jasmine/-/karma-jasmine-2.0.1.tgz",
"integrity": "sha512-iuC0hmr9b+SNn1DaUD2QEYtUxkS1J+bSJSn7ejdEexs7P8EYvA1CWkEdrDQ+8jVH3AgWlCNwjYsT1chjcNW9lA==",
"dev": true,
"requires": {
"jasmine-core": "^3.3"
}
},
"karma-jasmine-html-reporter": {
"version": "0.2.2",
@ -9266,6 +9267,14 @@
"dev": true,
"requires": {
"karma-jasmine": "^1.0.2"
},
"dependencies": {
"karma-jasmine": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/karma-jasmine/-/karma-jasmine-1.1.2.tgz",
"integrity": "sha1-OU8rJf+0pkS5rabyLUQ+L9CIhsM=",
"dev": true
}
}
},
"karma-mocha-reporter": {
@ -10654,8 +10663,7 @@
},
"ansi-regex": {
"version": "2.1.1",
"bundled": true,
"optional": true
"bundled": true
},
"aproba": {
"version": "1.2.0",
@ -10679,7 +10687,6 @@
"brace-expansion": {
"version": "1.1.11",
"bundled": true,
"optional": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@ -10692,8 +10699,7 @@
},
"code-point-at": {
"version": "1.1.0",
"bundled": true,
"optional": true
"bundled": true
},
"concat-map": {
"version": "0.0.1",
@ -10702,8 +10708,7 @@
},
"console-control-strings": {
"version": "1.1.0",
"bundled": true,
"optional": true
"bundled": true
},
"core-util-is": {
"version": "1.0.2",
@ -10806,8 +10811,7 @@
},
"inherits": {
"version": "2.0.3",
"bundled": true,
"optional": true
"bundled": true
},
"ini": {
"version": "1.3.5",
@ -10817,7 +10821,6 @@
"is-fullwidth-code-point": {
"version": "1.0.0",
"bundled": true,
"optional": true,
"requires": {
"number-is-nan": "^1.0.0"
}
@ -10830,20 +10833,17 @@
"minimatch": {
"version": "3.0.4",
"bundled": true,
"optional": true,
"requires": {
"brace-expansion": "^1.1.7"
}
},
"minimist": {
"version": "0.0.8",
"bundled": true,
"optional": true
"bundled": true
},
"minipass": {
"version": "2.3.5",
"bundled": true,
"optional": true,
"requires": {
"safe-buffer": "^5.1.2",
"yallist": "^3.0.0"
@ -10860,7 +10860,6 @@
"mkdirp": {
"version": "0.5.1",
"bundled": true,
"optional": true,
"requires": {
"minimist": "0.0.8"
}
@ -10933,8 +10932,7 @@
},
"number-is-nan": {
"version": "1.0.1",
"bundled": true,
"optional": true
"bundled": true
},
"object-assign": {
"version": "4.1.1",
@ -10944,7 +10942,6 @@
"once": {
"version": "1.4.0",
"bundled": true,
"optional": true,
"requires": {
"wrappy": "1"
}
@ -11020,8 +11017,7 @@
},
"safe-buffer": {
"version": "5.1.2",
"bundled": true,
"optional": true
"bundled": true
},
"safer-buffer": {
"version": "2.1.2",
@ -11051,7 +11047,6 @@
"string-width": {
"version": "1.0.2",
"bundled": true,
"optional": true,
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
@ -11069,7 +11064,6 @@
"strip-ansi": {
"version": "3.0.1",
"bundled": true,
"optional": true,
"requires": {
"ansi-regex": "^2.0.0"
}
@ -11108,13 +11102,11 @@
},
"wrappy": {
"version": "1.0.2",
"bundled": true,
"optional": true
"bundled": true
},
"yallist": {
"version": "3.0.3",
"bundled": true,
"optional": true
"bundled": true
}
}
},
@ -18778,8 +18770,7 @@
"ansi-regex": {
"version": "2.1.1",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"aproba": {
"version": "1.2.0",
@ -18807,7 +18798,6 @@
"version": "1.1.11",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@ -18822,8 +18812,7 @@
"code-point-at": {
"version": "1.1.0",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"concat-map": {
"version": "0.0.1",
@ -18834,8 +18823,7 @@
"console-control-strings": {
"version": "1.1.0",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"core-util-is": {
"version": "1.0.2",
@ -18952,8 +18940,7 @@
"inherits": {
"version": "2.0.3",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"ini": {
"version": "1.3.5",
@ -18965,7 +18952,6 @@
"version": "1.0.0",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"number-is-nan": "^1.0.0"
}
@ -18980,7 +18966,6 @@
"version": "3.0.4",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"brace-expansion": "^1.1.7"
}
@ -18988,14 +18973,12 @@
"minimist": {
"version": "0.0.8",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"minipass": {
"version": "2.3.5",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"safe-buffer": "^5.1.2",
"yallist": "^3.0.0"
@ -19014,7 +18997,6 @@
"version": "0.5.1",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"minimist": "0.0.8"
}
@ -19095,8 +19077,7 @@
"number-is-nan": {
"version": "1.0.1",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"object-assign": {
"version": "4.1.1",
@ -19108,7 +19089,6 @@
"version": "1.4.0",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"wrappy": "1"
}
@ -19194,8 +19174,7 @@
"safe-buffer": {
"version": "5.1.2",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"safer-buffer": {
"version": "2.1.2",
@ -19231,7 +19210,6 @@
"version": "1.0.2",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
@ -19251,7 +19229,6 @@
"version": "3.0.1",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"ansi-regex": "^2.0.0"
}
@ -19295,14 +19272,12 @@
"wrappy": {
"version": "1.0.2",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"yallist": {
"version": "3.0.3",
"bundled": true,
"dev": true,
"optional": true
"dev": true
}
}
},