diff --git a/make/migrations/postgresql/0010_1.9.0_schema.up.sql b/make/migrations/postgresql/0010_1.9.0_schema.up.sql index 249bf4878..7ec76adeb 100644 --- a/make/migrations/postgresql/0010_1.9.0_schema.up.sql +++ b/make/migrations/postgresql/0010_1.9.0_schema.up.sql @@ -1,138 +1,141 @@ /* add table for CVE whitelist */ -CREATE TABLE cve_whitelist ( - id SERIAL PRIMARY KEY NOT NULL, - project_id int, - creation_time timestamp default CURRENT_TIMESTAMP, - update_time timestamp default CURRENT_TIMESTAMP, - expires_at bigint, - items text NOT NULL, - UNIQUE (project_id) +CREATE TABLE cve_whitelist +( + id SERIAL PRIMARY KEY NOT NULL, + project_id int, + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP, + expires_at bigint, + items text NOT NULL, + UNIQUE (project_id) ); -CREATE TABLE blob ( - id SERIAL PRIMARY KEY NOT NULL, - /* - digest of config, layer, manifest - */ - digest varchar(255) NOT NULL, - content_type varchar(255) NOT NULL, - size int NOT NULL, - creation_time timestamp default CURRENT_TIMESTAMP, - UNIQUE (digest) +CREATE TABLE blob +( + id SERIAL PRIMARY KEY NOT NULL, + /* + digest of config, layer, manifest + */ + digest varchar(255) NOT NULL, + content_type varchar(255) NOT NULL, + size int NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + UNIQUE (digest) ); -CREATE TABLE artifact ( - id SERIAL PRIMARY KEY NOT NULL, - project_id int NOT NULL, - repo varchar(255) NOT NULL, - tag varchar(255) NOT NULL, - /* - digest of manifest - */ - digest varchar(255) NOT NULL, - /* - kind of artifact, image, chart, etc.. - */ - kind varchar(255) NOT NULL, - creation_time timestamp default CURRENT_TIMESTAMP, - pull_time timestamp, - push_time timestamp, - CONSTRAINT unique_artifact UNIQUE (project_id, repo, tag) +CREATE TABLE artifact +( + id SERIAL PRIMARY KEY NOT NULL, + project_id int NOT NULL, + repo varchar(255) NOT NULL, + tag varchar(255) NOT NULL, + /* + digest of manifest + */ + digest varchar(255) NOT NULL, + /* + kind of artifact, image, chart, etc.. + */ + kind varchar(255) NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + pull_time timestamp, + push_time timestamp, + CONSTRAINT unique_artifact UNIQUE (project_id, repo, tag) ); /* add the table for relation of artifact and blob */ -CREATE TABLE artifact_blob ( - id SERIAL PRIMARY KEY NOT NULL, - digest_af varchar(255) NOT NULL, - digest_blob varchar(255) NOT NULL, - creation_time timestamp default CURRENT_TIMESTAMP, - CONSTRAINT unique_artifact_blob UNIQUE (digest_af, digest_blob) +CREATE TABLE artifact_blob +( + id SERIAL PRIMARY KEY NOT NULL, + digest_af varchar(255) NOT NULL, + digest_blob varchar(255) NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + CONSTRAINT unique_artifact_blob UNIQUE (digest_af, digest_blob) ); /* add quota table */ -CREATE TABLE quota ( - id SERIAL PRIMARY KEY NOT NULL, - reference VARCHAR(255) NOT NULL, - reference_id VARCHAR(255) NOT NULL, - hard JSONB NOT NULL, - creation_time timestamp default CURRENT_TIMESTAMP, - update_time timestamp default CURRENT_TIMESTAMP, - UNIQUE(reference, reference_id) +CREATE TABLE quota +( + id SERIAL PRIMARY KEY NOT NULL, + reference VARCHAR(255) NOT NULL, + reference_id VARCHAR(255) NOT NULL, + hard JSONB NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP, + UNIQUE (reference, reference_id) ); /* add quota usage table */ -CREATE TABLE quota_usage ( - id SERIAL PRIMARY KEY NOT NULL, - reference VARCHAR(255) NOT NULL, - reference_id VARCHAR(255) NOT NULL, - used JSONB NOT NULL, - creation_time timestamp default CURRENT_TIMESTAMP, - update_time timestamp default CURRENT_TIMESTAMP, - UNIQUE(reference, reference_id) +CREATE TABLE quota_usage +( + id SERIAL PRIMARY KEY NOT NULL, + reference VARCHAR(255) NOT NULL, + reference_id VARCHAR(255) NOT NULL, + used JSONB NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP, + UNIQUE (reference, reference_id) ); INSERT INTO quota (reference, reference_id, hard, creation_time, update_time) -SELECT - 'project', - CAST(project_id AS VARCHAR), - '{"count": -1, "storage": -1}', - NOW(), - NOW() -FROM - project -WHERE - deleted = 'f'; +SELECT 'project', + CAST(project_id AS VARCHAR), + '{"count": -1, "storage": -1}', + NOW(), + NOW() +FROM project +WHERE deleted = 'f'; INSERT INTO quota_usage (id, reference, reference_id, used, creation_time, update_time) -SELECT - id, - reference, - reference_id, -'{"count": 0, "storage": 0}', - creation_time, - update_time -FROM - quota; +SELECT id, + reference, + reference_id, + '{"count": 0, "storage": 0}', + creation_time, + update_time +FROM quota; create table retention_policy ( - id serial PRIMARY KEY NOT NULL, - scope_level varchar(20), - scope_reference integer, - trigger_kind varchar(20), - data text, - create_time time, - update_time time + id serial PRIMARY KEY NOT NULL, + scope_level varchar(20), + scope_reference integer, + trigger_kind varchar(20), + data text, + create_time time, + update_time time ); create table retention_execution ( - id serial PRIMARY KEY NOT NULL, - policy_id integer, - dry_run boolean, - trigger varchar(20), - start_time timestamp + id serial PRIMARY KEY NOT NULL, + policy_id integer, + dry_run boolean, + trigger varchar(20), + start_time timestamp ); create table retention_task ( - id SERIAL NOT NULL, - execution_id integer, - repository varchar(255), - job_id varchar(64), - status varchar(32), - start_time timestamp default CURRENT_TIMESTAMP, - end_time timestamp default CURRENT_TIMESTAMP, - PRIMARY KEY (id) + id SERIAL NOT NULL, + execution_id integer, + repository varchar(255), + job_id varchar(64), + status varchar(32), + start_time timestamp default CURRENT_TIMESTAMP, + end_time timestamp default CURRENT_TIMESTAMP, + total integer, + retained integer, + PRIMARY KEY (id) ); create table schedule ( - id SERIAL NOT NULL, - job_id varchar(64), - status varchar(64), - creation_time timestamp default CURRENT_TIMESTAMP, - update_time timestamp default CURRENT_TIMESTAMP, - PRIMARY KEY (id) + id SERIAL NOT NULL, + job_id varchar(64), + status varchar(64), + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP, + PRIMARY KEY (id) ); diff --git a/src/core/service/notifications/jobs/handler.go b/src/core/service/notifications/jobs/handler.go index ac153ecb9..d35147f9d 100644 --- a/src/core/service/notifications/jobs/handler.go +++ b/src/core/service/notifications/jobs/handler.go @@ -46,6 +46,7 @@ type Handler struct { id int64 status string rawStatus string + checkIn string } // Prepare ... @@ -73,6 +74,7 @@ func (h *Handler) Prepare() { return } h.status = status + h.checkIn = data.CheckIn } // HandleScan handles the webhook of scan job @@ -117,8 +119,27 @@ func (h *Handler) HandleRetentionTask() { if h.status == models.JobFinished || h.status == models.JobError || h.status == models.JobStopped { task.EndTime = time.Now() + props = append(props, "EndTime") + } else if h.status == models.JobRunning { + if h.checkIn != "" { + var retainObj struct { + Total int `json:"total"` + Retained int `json:"retained"` + } + if err := json.Unmarshal([]byte(h.checkIn), &retainObj); err != nil { + log.Errorf("failed to resolve checkin of retention task %d: %v", h.id, err) + } else { + if retainObj.Total > 0 { + task.Total = retainObj.Total + props = append(props, "Total") + } + if retainObj.Retained > 0 { + task.Retained = retainObj.Retained + props = append(props, "Retained") + } + } + } } - props = append(props, "EndTime") if err := mgr.UpdateTask(task, props...); err != nil { log.Errorf("failed to update the status of retention task %d: %v", h.id, err) h.SendInternalServerError(err) diff --git a/src/pkg/retention/dao/models/retention.go b/src/pkg/retention/dao/models/retention.go index c0eba65a0..f1c5cce7b 100644 --- a/src/pkg/retention/dao/models/retention.go +++ b/src/pkg/retention/dao/models/retention.go @@ -56,4 +56,6 @@ type RetentionTask struct { Status string `orm:"column(status)"` StartTime time.Time `orm:"column(start_time)"` EndTime time.Time `orm:"column(end_time)"` + Total int `orm:"column(total)"` + Retained int `orm:"column(retained)"` } diff --git a/src/pkg/retention/job.go b/src/pkg/retention/job.go index cc56fe14f..533e9dddc 100644 --- a/src/pkg/retention/job.go +++ b/src/pkg/retention/job.go @@ -16,6 +16,7 @@ package retention import ( "bytes" + "encoding/json" "fmt" "strings" "time" @@ -111,6 +112,29 @@ func (pj *Job) Run(ctx job.Context, params job.Parameters) error { // Log stage: results with table view logResults(myLogger, allCandidates, results) + // Save retain and total num in DB + return saveRetainNum(ctx, results, allCandidates) +} + +func saveRetainNum(ctx job.Context, retained []*res.Result, allCandidates []*res.Candidate) error { + var delNum int + for _, r := range retained { + if r.Error == nil { + delNum++ + } + } + retainObj := struct { + Total int `json:"total"` + Retained int `json:"retained"` + }{ + Total: len(allCandidates), + Retained: len(allCandidates) - delNum, + } + c, err := json.Marshal(retainObj) + if err != nil { + return err + } + _ = ctx.Checkin(string(c)) return nil } diff --git a/src/pkg/retention/manager.go b/src/pkg/retention/manager.go index 5ac0ee79a..ccb2f6339 100644 --- a/src/pkg/retention/manager.go +++ b/src/pkg/retention/manager.go @@ -194,6 +194,8 @@ func (d *DefaultManager) CreateTask(task *Task) (int64, error) { Status: task.Status, StartTime: task.StartTime, EndTime: task.EndTime, + Total: task.Total, + Retained: task.Retained, } return dao.CreateTask(t) } @@ -217,6 +219,8 @@ func (d *DefaultManager) ListTasks(query ...*q.TaskQuery) ([]*Task, error) { Status: t.Status, StartTime: t.StartTime, EndTime: t.EndTime, + Total: t.Total, + Retained: t.Retained, }) } return tasks, nil @@ -243,6 +247,8 @@ func (d *DefaultManager) UpdateTask(task *Task, cols ...string) error { Status: task.Status, StartTime: task.StartTime, EndTime: task.EndTime, + Total: task.Total, + Retained: task.Retained, }, cols...) } @@ -263,6 +269,8 @@ func (d *DefaultManager) GetTask(taskID int64) (*Task, error) { Status: task.Status, StartTime: task.StartTime, EndTime: task.EndTime, + Total: task.Total, + Retained: task.Retained, }, nil } diff --git a/src/pkg/retention/models.go b/src/pkg/retention/models.go index 63b544aa4..6b2daab9a 100644 --- a/src/pkg/retention/models.go +++ b/src/pkg/retention/models.go @@ -56,6 +56,8 @@ type Task struct { Status string `json:"status"` StartTime time.Time `json:"start_time"` EndTime time.Time `json:"end_time"` + Total int `json:"total"` + Retained int `json:"retained"` } // History of retention diff --git a/src/portal/package-lock.json b/src/portal/package-lock.json index 31054d2c4..ca4c3d186 100644 --- a/src/portal/package-lock.json +++ b/src/portal/package-lock.json @@ -176,15 +176,13 @@ "version": "1.40.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==", - "dev": true, - "optional": true + "dev": true }, "mime-types": { "version": "2.1.24", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz", "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==", "dev": true, - "optional": true, "requires": { "mime-db": "1.40.0" } @@ -9254,10 +9252,13 @@ } }, "karma-jasmine": { - "version": "1.1.2", - "resolved": "http://registry.npm.taobao.org/karma-jasmine/download/karma-jasmine-1.1.2.tgz", - "integrity": "sha1-OU8rJf+0pkS5rabyLUQ+L9CIhsM=", - "dev": true + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/karma-jasmine/-/karma-jasmine-2.0.1.tgz", + "integrity": "sha512-iuC0hmr9b+SNn1DaUD2QEYtUxkS1J+bSJSn7ejdEexs7P8EYvA1CWkEdrDQ+8jVH3AgWlCNwjYsT1chjcNW9lA==", + "dev": true, + "requires": { + "jasmine-core": "^3.3" + } }, "karma-jasmine-html-reporter": { "version": "0.2.2", @@ -9266,6 +9267,14 @@ "dev": true, "requires": { "karma-jasmine": "^1.0.2" + }, + "dependencies": { + "karma-jasmine": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/karma-jasmine/-/karma-jasmine-1.1.2.tgz", + "integrity": "sha1-OU8rJf+0pkS5rabyLUQ+L9CIhsM=", + "dev": true + } } }, "karma-mocha-reporter": { @@ -10654,8 +10663,7 @@ }, "ansi-regex": { "version": "2.1.1", - "bundled": true, - "optional": true + "bundled": true }, "aproba": { "version": "1.2.0", @@ -10679,7 +10687,6 @@ "brace-expansion": { "version": "1.1.11", "bundled": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -10692,8 +10699,7 @@ }, "code-point-at": { "version": "1.1.0", - "bundled": true, - "optional": true + "bundled": true }, "concat-map": { "version": "0.0.1", @@ -10702,8 +10708,7 @@ }, "console-control-strings": { "version": "1.1.0", - "bundled": true, - "optional": true + "bundled": true }, "core-util-is": { "version": "1.0.2", @@ -10806,8 +10811,7 @@ }, "inherits": { "version": "2.0.3", - "bundled": true, - "optional": true + "bundled": true }, "ini": { "version": "1.3.5", @@ -10817,7 +10821,6 @@ "is-fullwidth-code-point": { "version": "1.0.0", "bundled": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -10830,20 +10833,17 @@ "minimatch": { "version": "3.0.4", "bundled": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } }, "minimist": { "version": "0.0.8", - "bundled": true, - "optional": true + "bundled": true }, "minipass": { "version": "2.3.5", "bundled": true, - "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -10860,7 +10860,6 @@ "mkdirp": { "version": "0.5.1", "bundled": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -10933,8 +10932,7 @@ }, "number-is-nan": { "version": "1.0.1", - "bundled": true, - "optional": true + "bundled": true }, "object-assign": { "version": "4.1.1", @@ -10944,7 +10942,6 @@ "once": { "version": "1.4.0", "bundled": true, - "optional": true, "requires": { "wrappy": "1" } @@ -11020,8 +11017,7 @@ }, "safe-buffer": { "version": "5.1.2", - "bundled": true, - "optional": true + "bundled": true }, "safer-buffer": { "version": "2.1.2", @@ -11051,7 +11047,6 @@ "string-width": { "version": "1.0.2", "bundled": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -11069,7 +11064,6 @@ "strip-ansi": { "version": "3.0.1", "bundled": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -11108,13 +11102,11 @@ }, "wrappy": { "version": "1.0.2", - "bundled": true, - "optional": true + "bundled": true }, "yallist": { "version": "3.0.3", - "bundled": true, - "optional": true + "bundled": true } } }, @@ -18778,8 +18770,7 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "aproba": { "version": "1.2.0", @@ -18807,7 +18798,6 @@ "version": "1.1.11", "bundled": true, "dev": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -18822,8 +18812,7 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "concat-map": { "version": "0.0.1", @@ -18834,8 +18823,7 @@ "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "core-util-is": { "version": "1.0.2", @@ -18952,8 +18940,7 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "ini": { "version": "1.3.5", @@ -18965,7 +18952,6 @@ "version": "1.0.0", "bundled": true, "dev": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -18980,7 +18966,6 @@ "version": "3.0.4", "bundled": true, "dev": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -18988,14 +18973,12 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "minipass": { "version": "2.3.5", "bundled": true, "dev": true, - "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -19014,7 +18997,6 @@ "version": "0.5.1", "bundled": true, "dev": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -19095,8 +19077,7 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "object-assign": { "version": "4.1.1", @@ -19108,7 +19089,6 @@ "version": "1.4.0", "bundled": true, "dev": true, - "optional": true, "requires": { "wrappy": "1" } @@ -19194,8 +19174,7 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "safer-buffer": { "version": "2.1.2", @@ -19231,7 +19210,6 @@ "version": "1.0.2", "bundled": true, "dev": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -19251,7 +19229,6 @@ "version": "3.0.1", "bundled": true, "dev": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -19295,14 +19272,12 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "yallist": { "version": "3.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true } } },