From a3ac9639f17d4b78676dc4ee6af065edc245a7c2 Mon Sep 17 00:00:00 2001 From: yixingj Date: Fri, 14 Jul 2017 16:43:03 +0800 Subject: [PATCH 01/21] Clair offline data update guide --- docs/clair_offlinedata_guide.md | 57 +++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 docs/clair_offlinedata_guide.md diff --git a/docs/clair_offlinedata_guide.md b/docs/clair_offlinedata_guide.md new file mode 100644 index 000000000..db37454a7 --- /dev/null +++ b/docs/clair_offlinedata_guide.md @@ -0,0 +1,57 @@ +## Guild for update clair offline data +### Summary + +In some case when user install harbor in an environment without internet access. Then Clair will not be able to fetch the latest vulnerability database. In this circumstance user need manfully update the Clair database. + +This document is a step by step instruction on update Clair vulnerability database in Harbor v1.2. + +### Preparation + +A. User need to install Clair 2.0.1 ( if you have a harbor1.2 instance with internet access will also works.) + +B. Check the Clair already update the vulnerability to the latest. + a. 'docker ps' to list the Clair container. Get the Clair container id . + b. Check the log of the Clair container. + c. If you are using harbor you can find the latest Clair log under /var/log/harbor/2017--xx-xx/clair.log + d. You will find some logs like follow: + ``` + Jul 3 20:40:45 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:45.890364","updater name":"rhel"} + Jul 3 20:40:46 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:46.768924","updater name":"alpine"} + Jul 3 20:40:47 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:47.190982","updater name":"oracle"} + Jul 3 20:41:07 172.18.0.1 clair[3516]: {"Event":"Debian buster is not mapped to any version number (eg. Jessie-\u003e8). Please update me.","Level":"warning","Location":"debian.go:128","Time":"2017-07-04 03:41:07.833720"} + Jul 3 20:41:07 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:41:07.833975","updater name":"debian"} + Jul 4 00:26:17 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 07:26:17.596986","updater name":"ubuntu"} + Jul 4 00:26:18 172.18.0.1 clair[3516]: {"Event":"adding metadata to vulnerabilities","Level":"info","Location":"updater.go:253","Time":"2017-07-04 07:26:18.060810"} + Jul 4 00:38:05 172.18.0.1 clair[3516]: {"Event":"update finished","Level":"info","Location":"updater.go:198","Time":"2017-07-04 07:38:05.251580"} + ``` + e. The update finished indicate that Clair has finished an vulnerability update round. You need to check that logs above it to make sure all the endpoints are update correctly. + +### Data dump + +A. Login into the PostgreSQL container of Clair. +* $>docker exec -it clair-db bash + +B. Dump the Clair vulnerability database by the follow command +* $> pg_dump -U postgres -a -t feature -t keyvalue -t namespace -t schema_migrations -t vulnerability -t vulnerability_fixedin_feature > vulnerability.sqll +* $> pg_dump -U postgres -c -s > clear.sql + +C. Collect the offline data +* Exit the container and back to the host where the container running. (assume you are using aufs as storage driver) +* cd /var/lib/docker/aufs/mnt +* find . -name clear.sql +* find . -name vulnerability.sql +* Copy the above two file to the host where Harbor is running. + +### Back Up Clair DB +A. Before update the offline data, user are strongly suggested to backup their Clair db. +* pg_dump -U postgres -c > all.sql + +### Update Clair DB +A. Copy the vulnerability.sql and clear.sql to the clair-db container which you want to update. +(if you check the harbor docker compose file you will find clair-db has two volumes: ./common/config/clair/postgresql-init.d/:/docker-entrypoint-initdb.d and /data/clair-db:/var/lib/postgresql/data, you can put the above sql file in ether of this two folder as a transfer) +B. Get the clair db shell by "docker exec –it clair-db bash" +C. $>psql -U postgres < clear.sql +D. $>psql –U postgres < vulnerability.sql + +### Rescan +After update the offline data, user need to trigger the "rescan all" functionality to scan all the images and Harbor reflect the new changes automatically after the scan finished.(Otherwise the vulnerability detail will not show up) From dc8ec8921b11774a5f43185203bb8f92cd0f55d1 Mon Sep 17 00:00:00 2001 From: yixingj Date: Fri, 14 Jul 2017 16:48:00 +0800 Subject: [PATCH 02/21] refactor format --- docs/clair_offlinedata_guide.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/clair_offlinedata_guide.md b/docs/clair_offlinedata_guide.md index db37454a7..46dac02c5 100644 --- a/docs/clair_offlinedata_guide.md +++ b/docs/clair_offlinedata_guide.md @@ -12,8 +12,11 @@ A. User need to install Clair 2.0.1 ( if you have a harbor1.2 instance with inte B. Check the Clair already update the vulnerability to the latest. a. 'docker ps' to list the Clair container. Get the Clair container id . b. Check the log of the Clair container. + c. If you are using harbor you can find the latest Clair log under /var/log/harbor/2017--xx-xx/clair.log + d. You will find some logs like follow: + ``` Jul 3 20:40:45 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:45.890364","updater name":"rhel"} Jul 3 20:40:46 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:46.768924","updater name":"alpine"} @@ -24,7 +27,8 @@ B. Check the Clair already update the vulnerability to the latest. Jul 4 00:26:18 172.18.0.1 clair[3516]: {"Event":"adding metadata to vulnerabilities","Level":"info","Location":"updater.go:253","Time":"2017-07-04 07:26:18.060810"} Jul 4 00:38:05 172.18.0.1 clair[3516]: {"Event":"update finished","Level":"info","Location":"updater.go:198","Time":"2017-07-04 07:38:05.251580"} ``` - e. The update finished indicate that Clair has finished an vulnerability update round. You need to check that logs above it to make sure all the endpoints are update correctly. + +e. The update finished indicate that Clair has finished an vulnerability update round. You need to check that logs above it to make sure all the endpoints are update correctly. ### Data dump @@ -49,8 +53,11 @@ A. Before update the offline data, user are strongly suggested to backup their C ### Update Clair DB A. Copy the vulnerability.sql and clear.sql to the clair-db container which you want to update. (if you check the harbor docker compose file you will find clair-db has two volumes: ./common/config/clair/postgresql-init.d/:/docker-entrypoint-initdb.d and /data/clair-db:/var/lib/postgresql/data, you can put the above sql file in ether of this two folder as a transfer) + B. Get the clair db shell by "docker exec –it clair-db bash" + C. $>psql -U postgres < clear.sql + D. $>psql –U postgres < vulnerability.sql ### Rescan From bfb1b7e75bdc0e014f7a83be59c84e3f081babec Mon Sep 17 00:00:00 2001 From: yixingj Date: Fri, 14 Jul 2017 16:50:04 +0800 Subject: [PATCH 03/21] refactor format --- docs/clair_offlinedata_guide.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/clair_offlinedata_guide.md b/docs/clair_offlinedata_guide.md index 46dac02c5..46525ea62 100644 --- a/docs/clair_offlinedata_guide.md +++ b/docs/clair_offlinedata_guide.md @@ -10,12 +10,14 @@ This document is a step by step instruction on update Clair vulnerability databa A. User need to install Clair 2.0.1 ( if you have a harbor1.2 instance with internet access will also works.) B. Check the Clair already update the vulnerability to the latest. - a. 'docker ps' to list the Clair container. Get the Clair container id . + + a. 'docker ps' to list the Clair container. Get the Clair container id . + b. Check the log of the Clair container. - c. If you are using harbor you can find the latest Clair log under /var/log/harbor/2017--xx-xx/clair.log + c. If you are using harbor you can find the latest Clair log under /var/log/harbor/2017--xx-xx/clair.log - d. You will find some logs like follow: + d. You will find some logs like follow: ``` Jul 3 20:40:45 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:45.890364","updater name":"rhel"} @@ -28,7 +30,7 @@ B. Check the Clair already update the vulnerability to the latest. Jul 4 00:38:05 172.18.0.1 clair[3516]: {"Event":"update finished","Level":"info","Location":"updater.go:198","Time":"2017-07-04 07:38:05.251580"} ``` -e. The update finished indicate that Clair has finished an vulnerability update round. You need to check that logs above it to make sure all the endpoints are update correctly. + e. The update finished indicate that Clair has finished an vulnerability update round. You need to check that logs above it to make sure all the endpoints are update correctly. ### Data dump From 008f386f7e7f387f7e568c475f32c060a35f92c8 Mon Sep 17 00:00:00 2001 From: yixingj Date: Fri, 14 Jul 2017 16:54:13 +0800 Subject: [PATCH 04/21] refactor --- docs/clair_offlinedata_guide.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/clair_offlinedata_guide.md b/docs/clair_offlinedata_guide.md index 46525ea62..68934f445 100644 --- a/docs/clair_offlinedata_guide.md +++ b/docs/clair_offlinedata_guide.md @@ -19,16 +19,16 @@ B. Check the Clair already update the vulnerability to the latest. d. You will find some logs like follow: - ``` - Jul 3 20:40:45 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:45.890364","updater name":"rhel"} - Jul 3 20:40:46 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:46.768924","updater name":"alpine"} - Jul 3 20:40:47 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:47.190982","updater name":"oracle"} - Jul 3 20:41:07 172.18.0.1 clair[3516]: {"Event":"Debian buster is not mapped to any version number (eg. Jessie-\u003e8). Please update me.","Level":"warning","Location":"debian.go:128","Time":"2017-07-04 03:41:07.833720"} - Jul 3 20:41:07 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:41:07.833975","updater name":"debian"} - Jul 4 00:26:17 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 07:26:17.596986","updater name":"ubuntu"} - Jul 4 00:26:18 172.18.0.1 clair[3516]: {"Event":"adding metadata to vulnerabilities","Level":"info","Location":"updater.go:253","Time":"2017-07-04 07:26:18.060810"} - Jul 4 00:38:05 172.18.0.1 clair[3516]: {"Event":"update finished","Level":"info","Location":"updater.go:198","Time":"2017-07-04 07:38:05.251580"} - ``` + ``` + Jul 3 20:40:45 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:45.890364","updater name":"rhel"} + Jul 3 20:40:46 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:46.768924","updater name":"alpine"} + Jul 3 20:40:47 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:47.190982","updater name":"oracle"} + Jul 3 20:41:07 172.18.0.1 clair[3516]: {"Event":"Debian buster is not mapped to any version number (eg. Jessie-\u003e8). Please update me.","Level":"warning","Location":"debian.go:128","Time":"2017-07-04 03:41:07.833720"} + Jul 3 20:41:07 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:41:07.833975","updater name":"debian"} + Jul 4 00:26:17 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 07:26:17.596986","updater name":"ubuntu"} + Jul 4 00:26:18 172.18.0.1 clair[3516]: {"Event":"adding metadata to vulnerabilities","Level":"info","Location":"updater.go:253","Time":"2017-07-04 07:26:18.060810"} + Jul 4 00:38:05 172.18.0.1 clair[3516]: {"Event":"update finished","Level":"info","Location":"updater.go:198","Time":"2017-07-04 07:38:05.251580"} + ``` e. The update finished indicate that Clair has finished an vulnerability update round. You need to check that logs above it to make sure all the endpoints are update correctly. From 19a5591ef504b20c3662a9ee65e1ef34a09032b3 Mon Sep 17 00:00:00 2001 From: yixingj Date: Fri, 14 Jul 2017 16:57:43 +0800 Subject: [PATCH 05/21] Update Clair offline data update doc --- docs/clair_offlinedata_guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/clair_offlinedata_guide.md b/docs/clair_offlinedata_guide.md index 68934f445..bfa69873b 100644 --- a/docs/clair_offlinedata_guide.md +++ b/docs/clair_offlinedata_guide.md @@ -10,7 +10,7 @@ This document is a step by step instruction on update Clair vulnerability databa A. User need to install Clair 2.0.1 ( if you have a harbor1.2 instance with internet access will also works.) B. Check the Clair already update the vulnerability to the latest. - + a. 'docker ps' to list the Clair container. Get the Clair container id . b. Check the log of the Clair container. From bfb0c0a32d12342c166ff16ff20f70ce422d5e32 Mon Sep 17 00:00:00 2001 From: yixingj Date: Fri, 14 Jul 2017 17:04:37 +0800 Subject: [PATCH 06/21] Update Clair offiline date guide --- docs/clair_offlinedata_guide.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/clair_offlinedata_guide.md b/docs/clair_offlinedata_guide.md index bfa69873b..d3274aaa9 100644 --- a/docs/clair_offlinedata_guide.md +++ b/docs/clair_offlinedata_guide.md @@ -1,4 +1,4 @@ -## Guild for update clair offline data +## Guild for update Clair offline data ### Summary In some case when user install harbor in an environment without internet access. Then Clair will not be able to fetch the latest vulnerability database. In this circumstance user need manfully update the Clair database. @@ -18,7 +18,6 @@ B. Check the Clair already update the vulnerability to the latest. c. If you are using harbor you can find the latest Clair log under /var/log/harbor/2017--xx-xx/clair.log d. You will find some logs like follow: - ``` Jul 3 20:40:45 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:45.890364","updater name":"rhel"} Jul 3 20:40:46 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:46.768924","updater name":"alpine"} @@ -29,7 +28,6 @@ B. Check the Clair already update the vulnerability to the latest. Jul 4 00:26:18 172.18.0.1 clair[3516]: {"Event":"adding metadata to vulnerabilities","Level":"info","Location":"updater.go:253","Time":"2017-07-04 07:26:18.060810"} Jul 4 00:38:05 172.18.0.1 clair[3516]: {"Event":"update finished","Level":"info","Location":"updater.go:198","Time":"2017-07-04 07:38:05.251580"} ``` - e. The update finished indicate that Clair has finished an vulnerability update round. You need to check that logs above it to make sure all the endpoints are update correctly. ### Data dump @@ -56,7 +54,7 @@ A. Before update the offline data, user are strongly suggested to backup their C A. Copy the vulnerability.sql and clear.sql to the clair-db container which you want to update. (if you check the harbor docker compose file you will find clair-db has two volumes: ./common/config/clair/postgresql-init.d/:/docker-entrypoint-initdb.d and /data/clair-db:/var/lib/postgresql/data, you can put the above sql file in ether of this two folder as a transfer) -B. Get the clair db shell by "docker exec –it clair-db bash" +B. Get the Clair db shell by "docker exec –it clair-db bash" C. $>psql -U postgres < clear.sql From 9b65a6ae756d92904771fed73614b7a92e8f84b5 Mon Sep 17 00:00:00 2001 From: yixingj Date: Tue, 18 Jul 2017 14:34:24 +0800 Subject: [PATCH 07/21] Update the docs --- docs/clair_offlinedata_guide.md | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/docs/clair_offlinedata_guide.md b/docs/clair_offlinedata_guide.md index d3274aaa9..ce99a4fb5 100644 --- a/docs/clair_offlinedata_guide.md +++ b/docs/clair_offlinedata_guide.md @@ -32,33 +32,24 @@ B. Check the Clair already update the vulnerability to the latest. ### Data dump -A. Login into the PostgreSQL container of Clair. -* $>docker exec -it clair-db bash +A. Login into the host where Clair db is running. B. Dump the Clair vulnerability database by the follow command -* $> pg_dump -U postgres -a -t feature -t keyvalue -t namespace -t schema_migrations -t vulnerability -t vulnerability_fixedin_feature > vulnerability.sqll -* $> pg_dump -U postgres -c -s > clear.sql +* $>docker exec clair-db /bin/bash -c "pg_dump -U postgres -a -t feature -t keyvalue -t namespace -t schema_migrations -t vulnerability -t vulnerability_fixedin_feature" > vulnerability.sql +* $>docker exec clair-db /bin/bash -c "pg_dump -U postgres -c -s" > clear.sql -C. Collect the offline data -* Exit the container and back to the host where the container running. (assume you are using aufs as storage driver) -* cd /var/lib/docker/aufs/mnt -* find . -name clear.sql -* find . -name vulnerability.sql -* Copy the above two file to the host where Harbor is running. ### Back Up Clair DB A. Before update the offline data, user are strongly suggested to backup their Clair db. -* pg_dump -U postgres -c > all.sql +* docker exec clair-db /bin/bash -c "pg_dump -U postgres -c" > all.sql ### Update Clair DB -A. Copy the vulnerability.sql and clear.sql to the clair-db container which you want to update. -(if you check the harbor docker compose file you will find clair-db has two volumes: ./common/config/clair/postgresql-init.d/:/docker-entrypoint-initdb.d and /data/clair-db:/var/lib/postgresql/data, you can put the above sql file in ether of this two folder as a transfer) +A. Copy the vulnerability.sql and clear.sql to the host where clair-db container which you want to update is running on. -B. Get the Clair db shell by "docker exec –it clair-db bash" -C. $>psql -U postgres < clear.sql +C. $>docker exec -i clair-db psql -U postgres < clear.sql -D. $>psql –U postgres < vulnerability.sql +D. $>docker exec -i clair-db psql –U postgres < vulnerability.sql ### Rescan After update the offline data, user need to trigger the "rescan all" functionality to scan all the images and Harbor reflect the new changes automatically after the scan finished.(Otherwise the vulnerability detail will not show up) From 97a9052050f301bf1ec2728b1c26dfb070c1f175 Mon Sep 17 00:00:00 2001 From: Steven Zou Date: Mon, 24 Jul 2017 13:39:49 +0800 Subject: [PATCH 08/21] Enhance scanning status controlling --- .../config/registry-config.component.html.ts | 2 +- .../config/registry-config.component.spec.ts | 5 +- .../src/config/registry-config.component.ts | 26 +++--- ...vulnerability-config.component.template.ts | 7 +- .../vulnerability-config.component.ts | 93 ++++++++++++++----- .../create-edit-rule.component.spec.ts | 10 +- src/ui_ng/lib/src/harbor-library.module.ts | 14 ++- .../job-log-viewer.component.spec.ts | 12 +-- .../job-log-viewer.component.template.ts | 2 +- .../job-log-viewer.component.ts | 30 +++++- .../replication/replication.component.spec.ts | 4 +- .../repository-stackview.component.spec.ts | 4 +- src/ui_ng/lib/src/service.config.ts | 8 ++ src/ui_ng/lib/src/service/index.ts | 3 +- src/ui_ng/lib/src/service/interface.ts | 1 + .../lib/src/service/job-log.service.spec.ts | 39 ++++++++ src/ui_ng/lib/src/service/job-log.service.ts | 84 +++++++++++++++++ src/ui_ng/lib/src/tag/tag.component.css.ts | 1 - src/ui_ng/lib/src/tag/tag.component.spec.ts | 5 +- .../result-bar-chart.component.spec.ts | 11 ++- .../result-bar-chart.component.ts | 14 ++- .../vulnerability-scanning/scanning.css.ts | 1 + .../vulnerability-scanning/scanning.html.ts | 7 +- src/ui_ng/package.json | 2 +- src/ui_ng/src/app/app-config.ts | 2 + .../src/app/config/config.component.html | 2 +- src/ui_ng/src/app/config/config.component.ts | 4 - .../tag-repository.component.html | 2 +- .../tag-repository.component.ts | 16 +++- src/ui_ng/src/i18n/lang/en-us-lang.json | 8 +- src/ui_ng/src/i18n/lang/es-es-lang.json | 8 +- src/ui_ng/src/i18n/lang/zh-cn-lang.json | 8 +- 32 files changed, 345 insertions(+), 90 deletions(-) create mode 100644 src/ui_ng/lib/src/service/job-log.service.spec.ts create mode 100644 src/ui_ng/lib/src/service/job-log.service.ts diff --git a/src/ui_ng/lib/src/config/registry-config.component.html.ts b/src/ui_ng/lib/src/config/registry-config.component.html.ts index b2cdccecc..a647ca0bb 100644 --- a/src/ui_ng/lib/src/config/registry-config.component.html.ts +++ b/src/ui_ng/lib/src/config/registry-config.component.html.ts @@ -2,7 +2,7 @@ export const REGISTRY_CONFIG_HTML: string = `
- +
diff --git a/src/ui_ng/lib/src/config/registry-config.component.spec.ts b/src/ui_ng/lib/src/config/registry-config.component.spec.ts index c29041130..41e9fda50 100644 --- a/src/ui_ng/lib/src/config/registry-config.component.spec.ts +++ b/src/ui_ng/lib/src/config/registry-config.component.spec.ts @@ -52,7 +52,8 @@ describe('RegistryConfigComponent (inline template)', () => { "project_creation_restriction": "everyone", "self_registration": true, "has_ca_root": true, - "harbor_version": "v1.1.1-rc1-160-g565110d" + "harbor_version": "v1.1.1-rc1-160-g565110d", + "next_scan_all": 0 }; beforeEach(async(() => { @@ -85,7 +86,7 @@ describe('RegistryConfigComponent (inline template)', () => { systemInfoService = fixture.debugElement.injector.get(SystemInfoService); spy = spyOn(cfgService, 'getConfigurations').and.returnValue(Promise.resolve(mockConfig)); saveSpy = spyOn(cfgService, 'saveConfigurations').and.returnValue(Promise.resolve(true)); - spySystemInfo = spyOn(systemInfoService, 'getSystemInfo').and.returnValues(Promise.resolve(mockSystemInfo)); + spySystemInfo = spyOn(systemInfoService, 'getSystemInfo').and.returnValue(Promise.resolve(mockSystemInfo)); fixture.detectChanges(); }); diff --git a/src/ui_ng/lib/src/config/registry-config.component.ts b/src/ui_ng/lib/src/config/registry-config.component.ts index dcfa80c03..d963d7ce4 100644 --- a/src/ui_ng/lib/src/config/registry-config.component.ts +++ b/src/ui_ng/lib/src/config/registry-config.component.ts @@ -54,17 +54,8 @@ export class RegistryConfigComponent implements OnInit { return this.systemInfo && this.systemInfo.with_clair; } - get clairDB(): ClairDBStatus { - return this.systemInfo && this.systemInfo.clair_vulnerability_status ? - this.systemInfo.clair_vulnerability_status : null; - } - ngOnInit(): void { - //Get system info - toPromise(this.systemInfoService.getSystemInfo()) - .then((info: SystemInfo) => this.systemInfo = info) - .catch(error => this.errorHandler.error(error)); - + this.loadSystemInfo(); //Initialize this.load(); } @@ -82,20 +73,25 @@ export class RegistryConfigComponent implements OnInit { return !this._isEmptyObject(this.getChanges()); } + //Get system info + loadSystemInfo(): void { + toPromise(this.systemInfoService.getSystemInfo()) + .then((info: SystemInfo) => this.systemInfo = info) + .catch(error => this.errorHandler.error(error)); + } + //Load configurations load(): void { this.onGoing = true; toPromise(this.configService.getConfigurations()) .then((config: Configuration) => { - this.onGoing = false; - this.configCopy = this._clone(config); this.config = config; + this.onGoing = false; }) .catch(error => { - this.onGoing = false; - this.errorHandler.error(error); + this.onGoing = false; }); } @@ -118,6 +114,8 @@ export class RegistryConfigComponent implements OnInit { }); //Reload to fetch all the updates this.load(); + //Reload all system info + //this.loadSystemInfo(); }) .catch(error => { this.onGoing = false; diff --git a/src/ui_ng/lib/src/config/vulnerability/vulnerability-config.component.template.ts b/src/ui_ng/lib/src/config/vulnerability/vulnerability-config.component.template.ts index d6f750647..566a6c581 100644 --- a/src/ui_ng/lib/src/config/vulnerability/vulnerability-config.component.template.ts +++ b/src/ui_ng/lib/src/config/vulnerability/vulnerability-config.component.template.ts @@ -12,13 +12,13 @@ export const VULNERABILITY_CONFIG_HTML: string = `
{{nt.namespace}} - {{convertToLocalTime(nt.last_update*1000)}} + {{ convertToLocalTime(nt.last_update) | date:'MM/dd/y HH:mm:ss'}}
@@ -38,7 +38,8 @@ export const VULNERABILITY_CONFIG_HTML: string = `
- + + {{ 'CONFIG.SCANNING.NEXT_SCAN' | translate }} {{ nextScanTimestamp | date:'HH:mm' }}
diff --git a/src/ui_ng/lib/src/config/vulnerability/vulnerability-config.component.ts b/src/ui_ng/lib/src/config/vulnerability/vulnerability-config.component.ts index 6d30ca18a..625bffff4 100644 --- a/src/ui_ng/lib/src/config/vulnerability/vulnerability-config.component.ts +++ b/src/ui_ng/lib/src/config/vulnerability/vulnerability-config.component.ts @@ -1,9 +1,13 @@ -import { Component, Input, Output, EventEmitter, ViewChild } from '@angular/core'; +import { Component, Input, Output, EventEmitter, ViewChild, OnInit } from '@angular/core'; import { NgForm } from '@angular/forms'; import { Configuration } from '../config'; import { VULNERABILITY_CONFIG_HTML, VULNERABILITY_CONFIG_STYLES } from './vulnerability-config.component.template'; -import { ScanningResultService } from '../../service/scanning.service'; +import { + ScanningResultService, + SystemInfo, + SystemInfoService +} from '../../service/index'; import { ErrorHandler } from '../../error-handler'; import { toPromise } from '../../utils'; import { TranslateService } from '@ngx-translate/core'; @@ -19,9 +23,10 @@ const ONE_DAY_SECONDS: number = 24 * ONE_HOUR_SECONDS; template: VULNERABILITY_CONFIG_HTML, styles: [VULNERABILITY_CONFIG_STYLES, REGISTRY_CONFIG_STYLES] }) -export class VulnerabilityConfigComponent { +export class VulnerabilityConfigComponent implements OnInit { _localTime: Date = new Date(); + onSubmitting: boolean = false; config: Configuration; openState: boolean = false; @Output() configChange: EventEmitter = new EventEmitter(); @@ -46,21 +51,37 @@ export class VulnerabilityConfigComponent { } @Input() showSubTitle: boolean = false; - @Input() clairDBStatus: ClairDBStatus; + systemInfo: SystemInfo; - get updatedTimestamp(): string { - if (this.clairDBStatus && this.clairDBStatus.overall_last_update > 0) { - return this.convertToLocalTime(this.clairDBStatus.overall_last_update*1000); + get scanAvailable(): boolean { + let dt: Date = new Date(); + return !this.onSubmitting && (this.nextScanTime <= 0 || dt.getTime() > ((this.nextScanTime + 30) * 1000)); + } + + get nextScanTimestamp(): Date { + return this.nextScanTime > 0 ? this.convertToLocalTime(this.nextScanTime) : null; + } + + get nextScanTime(): number { + return this.systemInfo && this.systemInfo.next_scan_all ? this.systemInfo.next_scan_all : 0; + } + + get updatedTimestamp(): Date { + if (this.systemInfo && + this.systemInfo.clair_vulnerability_status && + this.systemInfo.clair_vulnerability_status.overall_last_update > 0) { + return this.convertToLocalTime(this.systemInfo.clair_vulnerability_status.overall_last_update); } - return "--"; + return null; } get namespaceTimestamps(): ClairDetail[] { - if (this.clairDBStatus && - this.clairDBStatus.details && - this.clairDBStatus.details.length > 0) { - return this.clairDBStatus.details; + if (this.systemInfo && + this.systemInfo.clair_vulnerability_status && + this.systemInfo.clair_vulnerability_status.details && + this.systemInfo.clair_vulnerability_status.details.length > 0) { + return this.systemInfo.clair_vulnerability_status.details; } return []; @@ -207,31 +228,57 @@ export class VulnerabilityConfigComponent { } get isClairDBFullyReady(): boolean { - return this.clairDBStatus && this.clairDBStatus.overall_last_update > 0; + return this.systemInfo && + this.systemInfo.clair_vulnerability_status && + this.systemInfo.clair_vulnerability_status.overall_last_update > 0; } constructor( private scanningService: ScanningResultService, private errorHandler: ErrorHandler, - private translate: TranslateService) { } + private translate: TranslateService, + private systemInfoService: SystemInfoService + ) { } - convertToLocalTime(utcTime: number): string { - let offset: number = this._localTime.getTimezoneOffset() * 60; - let timeWithLocal: number = utcTime - offset; - let dt = new Date(); - dt.setTime(timeWithLocal); - return dt.toLocaleString(); + ngOnInit(): void { + this.getSystemInfo(); + } + + convertToLocalTime(utcTime: number): Date { + let dt: Date = new Date(); + dt.setTime(utcTime * 1000); + + return dt; } scanNow(): void { + if (this.onSubmitting) { + return;//Aoid duplicated submitting + } + + this.onSubmitting = true; toPromise(this.scanningService.startScanningAll()) .then(() => { this.translate.get("CONFIG.SCANNING.TRIGGER_SCAN_ALL_SUCCESS").subscribe((res: string) => { this.errorHandler.info(res); }); - //TODO: - //Change button disable status. + + //Update system info + this.getSystemInfo().then(() => { + this.onSubmitting = false; + }).catch(() => { + this.onSubmitting = false; + }); }) - .catch(error => this.errorHandler.error(error)) + .catch(error => { + this.errorHandler.error(error); + this.onSubmitting = false; + }); + } + + getSystemInfo(): Promise { + return toPromise(this.systemInfoService.getSystemInfo()) + .then((info: SystemInfo) => this.systemInfo = info) + .catch(error => this.errorHandler.error(error)); } } \ No newline at end of file diff --git a/src/ui_ng/lib/src/create-edit-rule/create-edit-rule.component.spec.ts b/src/ui_ng/lib/src/create-edit-rule/create-edit-rule.component.spec.ts index 0c88b2f17..58d3f66a5 100644 --- a/src/ui_ng/lib/src/create-edit-rule/create-edit-rule.component.spec.ts +++ b/src/ui_ng/lib/src/create-edit-rule/create-edit-rule.component.spec.ts @@ -18,7 +18,12 @@ import { ReplicationRule, ReplicationJob, Endpoint } from '../service/interface' import { ErrorHandler } from '../error-handler/error-handler'; import { SERVICE_CONFIG, IServiceConfig } from '../service.config'; -import { ReplicationService, ReplicationDefaultService } from '../service/replication.service'; +import { + ReplicationService, + ReplicationDefaultService, + JobLogService, + JobLogDefaultService + } from '../service/index'; import { EndpointService, EndpointDefaultService } from '../service/endpoint.service'; import { JobLogViewerComponent } from '../job-log-viewer/job-log-viewer.component'; @@ -183,7 +188,8 @@ describe('CreateEditRuleComponent (inline template)', ()=>{ ErrorHandler, { provide: SERVICE_CONFIG, useValue: config }, { provide: ReplicationService, useClass: ReplicationDefaultService }, - { provide: EndpointService, useClass: EndpointDefaultService } + { provide: EndpointService, useClass: EndpointDefaultService }, + { provide: JobLogService, useClass: JobLogDefaultService } ] }); })); diff --git a/src/ui_ng/lib/src/harbor-library.module.ts b/src/ui_ng/lib/src/harbor-library.module.ts index 1af8b4a9f..4b08acc0f 100644 --- a/src/ui_ng/lib/src/harbor-library.module.ts +++ b/src/ui_ng/lib/src/harbor-library.module.ts @@ -41,7 +41,9 @@ import { ScanningResultService, ScanningResultDefaultService, ConfigurationService, - ConfigurationDefaultService + ConfigurationDefaultService, + JobLogService, + JobLogDefaultService } from './service/index'; import { ErrorHandler, @@ -74,7 +76,8 @@ export const DefaultServiceConfig: IServiceConfig = { langMessagePathForHttpLoader: "i18n/langs/", langMessageFileSuffixForHttpLoader: "-lang.json", localI18nMessageVariableMap: {}, - configurationEndpoint: "/api/configurations" + configurationEndpoint: "/api/configurations", + scanJobEndpoint: "/api/jobs/scan" }; /** @@ -112,7 +115,10 @@ export interface HarborModuleConfig { scanningService?: Provider, //Service implementation for configuration - configService?: Provider + configService?: Provider, + + //Service implementation for job log + jobLogService?: Provider } /** @@ -197,6 +203,7 @@ export class HarborLibraryModule { config.tagService || { provide: TagService, useClass: TagDefaultService }, config.scanningService || { provide: ScanningResultService, useClass: ScanningResultDefaultService }, config.configService || { provide: ConfigurationService, useClass: ConfigurationDefaultService }, + config.jobLogService || { provide: JobLogService, useClass: JobLogDefaultService }, //Do initializing TranslateServiceInitializer, { @@ -224,6 +231,7 @@ export class HarborLibraryModule { config.tagService || { provide: TagService, useClass: TagDefaultService }, config.scanningService || { provide: ScanningResultService, useClass: ScanningResultDefaultService }, config.configService || { provide: ConfigurationService, useClass: ConfigurationDefaultService }, + config.jobLogService || { provide: JobLogService, useClass: JobLogDefaultService }, ChannelService ] }; diff --git a/src/ui_ng/lib/src/job-log-viewer/job-log-viewer.component.spec.ts b/src/ui_ng/lib/src/job-log-viewer/job-log-viewer.component.spec.ts index 02defb2d5..2780bbc9a 100644 --- a/src/ui_ng/lib/src/job-log-viewer/job-log-viewer.component.spec.ts +++ b/src/ui_ng/lib/src/job-log-viewer/job-log-viewer.component.spec.ts @@ -2,7 +2,7 @@ import { async, ComponentFixture, TestBed } from '@angular/core/testing'; import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; import { DebugElement } from '@angular/core'; import { Observable } from 'rxjs/Observable'; -import { ReplicationService, ReplicationDefaultService } from '../service/index'; +import { JobLogService, JobLogDefaultService } from '../service/index'; import { JobLogViewerComponent } from './job-log-viewer.component'; import { SERVICE_CONFIG, IServiceConfig } from '../service.config'; @@ -13,7 +13,7 @@ describe('JobLogViewerComponent (inline template)', () => { let component: JobLogViewerComponent; let fixture: ComponentFixture; let serviceConfig: IServiceConfig; - let replicationService: ReplicationService; + let jobLogService: JobLogDefaultService; let spy: jasmine.Spy; let testConfig: IServiceConfig = { replicationJobEndpoint: "/api/jobs/replication/testing" @@ -29,7 +29,7 @@ describe('JobLogViewerComponent (inline template)', () => { providers: [ ErrorHandler, { provide: SERVICE_CONFIG, useValue: testConfig }, - { provide: ReplicationService, useClass: ReplicationDefaultService } + { provide: JobLogService, useClass: JobLogDefaultService } ] }); })); @@ -39,9 +39,9 @@ describe('JobLogViewerComponent (inline template)', () => { component = fixture.componentInstance; serviceConfig = TestBed.get(SERVICE_CONFIG); - replicationService = fixture.debugElement.injector.get(ReplicationService); - spy = spyOn(replicationService, 'getJobLog') - .and.returnValues(Promise.resolve("job log text")); + jobLogService = fixture.debugElement.injector.get(JobLogService); + spy = spyOn(jobLogService, 'getJobLog') + .and.returnValue(Promise.resolve("job log text")); fixture.detectChanges(); }); diff --git a/src/ui_ng/lib/src/job-log-viewer/job-log-viewer.component.template.ts b/src/ui_ng/lib/src/job-log-viewer/job-log-viewer.component.template.ts index 5b4624edb..567ee1a94 100644 --- a/src/ui_ng/lib/src/job-log-viewer/job-log-viewer.component.template.ts +++ b/src/ui_ng/lib/src/job-log-viewer/job-log-viewer.component.template.ts @@ -1,6 +1,6 @@ export const JOB_LOG_VIEWER_TEMPLATE: string = ` -

{{'REPLICATION.JOB_LOG_VIEWER' | translate }}

+

{{title | translate }}

diff --git a/src/ui_ng/lib/src/job-log-viewer/job-log-viewer.component.ts b/src/ui_ng/lib/src/job-log-viewer/job-log-viewer.component.ts index f17ff3a8d..260c836bf 100644 --- a/src/ui_ng/lib/src/job-log-viewer/job-log-viewer.component.ts +++ b/src/ui_ng/lib/src/job-log-viewer/job-log-viewer.component.ts @@ -11,13 +11,15 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -import { Component } from '@angular/core'; +import { Component, Input } from '@angular/core'; import { JOB_LOG_VIEWER_TEMPLATE, JOB_LOG_VIEWER_STYLES } from './job-log-viewer.component.template'; -import { ReplicationService } from '../service/index'; +import { JobLogService } from '../service/index'; import { ErrorHandler } from '../error-handler/index'; import { toPromise } from '../utils'; +const supportSet: string[] = ["replication", "scan"]; + @Component({ selector: 'job-log-viewer', template: JOB_LOG_VIEWER_TEMPLATE, @@ -25,12 +27,32 @@ import { toPromise } from '../utils'; }) export class JobLogViewerComponent { + _jobType: string = "replication"; + opened: boolean = false; log: string = ''; onGoing: boolean = true; + @Input() + get jobType(): string { + return this._jobType; + } + set jobType(v: string) { + if (supportSet.find((t: string) => t === v)) { + this._jobType = v; + } + } + + get title(): string { + if(this.jobType === "scan"){ + return "VULNERABILITY.JOB_LOG_VIEWER"; + } + + return "REPLICATION.JOB_LOG_VIEWER"; + } + constructor( - private replicationService: ReplicationService, + private jobLogService: JobLogService, private errorHandler: ErrorHandler ) { } @@ -47,7 +69,7 @@ export class JobLogViewerComponent { load(jobId: number | string): void { this.onGoing = true; - toPromise(this.replicationService.getJobLog(jobId)) + toPromise(this.jobLogService.getJobLog(this.jobType, jobId)) .then((log: string) => { this.onGoing = false; this.log = log; diff --git a/src/ui_ng/lib/src/replication/replication.component.spec.ts b/src/ui_ng/lib/src/replication/replication.component.spec.ts index 9967f2c8e..0f9323b60 100644 --- a/src/ui_ng/lib/src/replication/replication.component.spec.ts +++ b/src/ui_ng/lib/src/replication/replication.component.spec.ts @@ -19,6 +19,7 @@ import { SERVICE_CONFIG, IServiceConfig } from '../service.config'; import { ReplicationService, ReplicationDefaultService } from '../service/replication.service'; import { EndpointService, EndpointDefaultService } from '../service/endpoint.service'; import { JobLogViewerComponent } from '../job-log-viewer/job-log-viewer.component'; +import { JobLogService, JobLogDefaultService } from '../service/index'; describe('Replication Component (inline template)', ()=>{ @@ -183,7 +184,8 @@ describe('Replication Component (inline template)', ()=>{ ErrorHandler, { provide: SERVICE_CONFIG, useValue: config }, { provide: ReplicationService, useClass: ReplicationDefaultService }, - { provide: EndpointService, useClass: EndpointDefaultService } + { provide: EndpointService, useClass: EndpointDefaultService }, + { provide: JobLogService, useClass: JobLogDefaultService } ] }); })); diff --git a/src/ui_ng/lib/src/repository-stackview/repository-stackview.component.spec.ts b/src/ui_ng/lib/src/repository-stackview/repository-stackview.component.spec.ts index 3f3b0224e..8d16be171 100644 --- a/src/ui_ng/lib/src/repository-stackview/repository-stackview.component.spec.ts +++ b/src/ui_ng/lib/src/repository-stackview/repository-stackview.component.spec.ts @@ -17,6 +17,7 @@ import { SystemInfoService, SystemInfoDefaultService } from '../service/system-i import { VULNERABILITY_DIRECTIVES } from '../vulnerability-scanning/index'; import { PUSH_IMAGE_BUTTON_DIRECTIVES } from '../push-image/index'; import { INLINE_ALERT_DIRECTIVES } from '../inline-alert/index'; +import { JobLogViewerComponent } from '../job-log-viewer/index'; import { click } from '../utils'; @@ -101,7 +102,8 @@ describe('RepositoryComponentStackview (inline template)', () => { FilterComponent, VULNERABILITY_DIRECTIVES, PUSH_IMAGE_BUTTON_DIRECTIVES, - INLINE_ALERT_DIRECTIVES + INLINE_ALERT_DIRECTIVES, + JobLogViewerComponent ], providers: [ ErrorHandler, diff --git a/src/ui_ng/lib/src/service.config.ts b/src/ui_ng/lib/src/service.config.ts index 07f3bf009..268f46648 100644 --- a/src/ui_ng/lib/src/service.config.ts +++ b/src/ui_ng/lib/src/service.config.ts @@ -180,4 +180,12 @@ export interface IServiceConfig { * @memberOf IServiceConfig */ configurationEndpoint?: string; + + /** + * The base endpoint of scan job service. + * + * @type {string} + * @memberof IServiceConfig + */ + scanJobEndpoint?: string; } \ No newline at end of file diff --git a/src/ui_ng/lib/src/service/index.ts b/src/ui_ng/lib/src/service/index.ts index 2b10f7ebd..98bf0cb06 100644 --- a/src/ui_ng/lib/src/service/index.ts +++ b/src/ui_ng/lib/src/service/index.ts @@ -7,4 +7,5 @@ export * from './repository.service'; export * from './tag.service'; export * from './RequestQueryParams'; export * from './scanning.service'; -export * from './configuration.service'; \ No newline at end of file +export * from './configuration.service'; +export * from './job-log.service'; \ No newline at end of file diff --git a/src/ui_ng/lib/src/service/interface.ts b/src/ui_ng/lib/src/service/interface.ts index 9482df0b8..17ecdf1be 100644 --- a/src/ui_ng/lib/src/service/interface.ts +++ b/src/ui_ng/lib/src/service/interface.ts @@ -168,6 +168,7 @@ export interface SystemInfo { has_ca_root?: boolean; harbor_version?: string; clair_vulnerability_status?: ClairDBStatus; + next_scan_all?: number; } /** diff --git a/src/ui_ng/lib/src/service/job-log.service.spec.ts b/src/ui_ng/lib/src/service/job-log.service.spec.ts new file mode 100644 index 000000000..9ee59dbf5 --- /dev/null +++ b/src/ui_ng/lib/src/service/job-log.service.spec.ts @@ -0,0 +1,39 @@ +import { TestBed, inject } from '@angular/core/testing'; + +import { JobLogService, JobLogDefaultService } from './job-log.service'; +import { SharedModule } from '../shared/shared.module'; +import { SERVICE_CONFIG, IServiceConfig } from '../service.config'; + +describe('JobLogService', () => { + const mockConfig: IServiceConfig = { + replicationJobEndpoint: "/api/jobs/replication/testing", + scanJobEndpoint: "/api/jobs/scan/testing" + }; + + let config: IServiceConfig; + + beforeEach(() => { + TestBed.configureTestingModule({ + imports: [ + SharedModule + ], + providers: [ + JobLogDefaultService, + { + provide: JobLogService, + useClass: JobLogDefaultService + }, { + provide: SERVICE_CONFIG, + useValue: mockConfig + }] + }); + + config = TestBed.get(SERVICE_CONFIG); + }); + + it('should be initialized', inject([JobLogDefaultService], (service: JobLogService) => { + expect(service).toBeTruthy(); + expect(config.replicationJobEndpoint).toEqual("/api/jobs/replication/testing"); + expect(config.scanJobEndpoint).toEqual("/api/jobs/scan/testing"); + })); +}); diff --git a/src/ui_ng/lib/src/service/job-log.service.ts b/src/ui_ng/lib/src/service/job-log.service.ts new file mode 100644 index 000000000..f35f253ca --- /dev/null +++ b/src/ui_ng/lib/src/service/job-log.service.ts @@ -0,0 +1,84 @@ +import { Observable } from 'rxjs/Observable'; +import { RequestQueryParams } from './RequestQueryParams'; +import { ReplicationJob, ReplicationRule } from './interface'; +import { Injectable, Inject } from "@angular/core"; +import 'rxjs/add/observable/of'; +import { Http, RequestOptions } from '@angular/http'; +import { SERVICE_CONFIG, IServiceConfig } from '../service.config'; +import { buildHttpRequestOptions, HTTP_JSON_OPTIONS } from '../utils'; + +/** + * Define the service methods to handle the job log related things. + * + * @export + * @abstract + * @class JobLogService + */ +export abstract class JobLogService { + + /** + * Get the log of the specified job + * + * @abstract + * @param {string} jobType + * @param {(number | string)} jobId + * @returns {(Observable | Promise | string)} + * @memberof JobLogService + */ + abstract getJobLog(jobType: string, jobId: number | string): Observable | Promise | string; +} + +/** + * Implement default service for job log service. + * + * @export + * @class JobLogDefaultService + * @extends {ReplicationService} + */ +@Injectable() +export class JobLogDefaultService extends JobLogService { + _replicationJobBaseUrl: string; + _scanningJobBaseUrl: string; + _supportedJobTypes: string[]; + + constructor( + private http: Http, + @Inject(SERVICE_CONFIG) config: IServiceConfig + ) { + super(); + this._replicationJobBaseUrl = config.replicationJobEndpoint ? + config.replicationJobEndpoint : '/api/jobs/replication'; + this._scanningJobBaseUrl = config.scanJobEndpoint ? config.scanJobEndpoint : "/api/jobs/scan"; + this._supportedJobTypes = ["replication", "scan"]; + } + + _getJobLog(logUrl: string): Observable | Promise | string { + return this.http.get(logUrl).toPromise() + .then(response => response.text()) + .catch(error => Promise.reject(error)); + } + + _isSupportedJobType(jobType: string): boolean { + if (this._supportedJobTypes.find((t: string) => t === jobType)) { + return true; + } + + return false; + } + + public getJobLog(jobType: string, jobId: number | string): Observable | Promise | string { + if (!this._isSupportedJobType(jobType)) { + return Promise.reject("Unsupport job type: " + jobType); + } + if (!jobId || jobId <= 0) { + return Promise.reject('Bad argument'); + } + + let logUrl: string = `${this._replicationJobBaseUrl}/${jobId}/log`; + if (jobType === "scan") { + logUrl = `${this._scanningJobBaseUrl}/${jobId}/log`; + } + + return this._getJobLog(logUrl); + } +} \ No newline at end of file diff --git a/src/ui_ng/lib/src/tag/tag.component.css.ts b/src/ui_ng/lib/src/tag/tag.component.css.ts index 32d854d01..80335b50b 100644 --- a/src/ui_ng/lib/src/tag/tag.component.css.ts +++ b/src/ui_ng/lib/src/tag/tag.component.css.ts @@ -14,7 +14,6 @@ export const TAG_STYLE = ` :host >>> .datagrid { margin: 0; - border-bottom: none; } :host >>> .datagrid-placeholder { diff --git a/src/ui_ng/lib/src/tag/tag.component.spec.ts b/src/ui_ng/lib/src/tag/tag.component.spec.ts index d63279c5a..2f14b4824 100644 --- a/src/ui_ng/lib/src/tag/tag.component.spec.ts +++ b/src/ui_ng/lib/src/tag/tag.component.spec.ts @@ -17,6 +17,8 @@ import { FILTER_DIRECTIVES } from '../filter/index' import { Observable, Subscription } from 'rxjs/Rx'; import { ChannelService } from '../channel/index'; +import { JobLogViewerComponent } from '../job-log-viewer/index'; + describe('TagComponent (inline template)', () => { let comp: TagComponent; @@ -49,7 +51,8 @@ describe('TagComponent (inline template)', () => { TagComponent, ConfirmationDialogComponent, VULNERABILITY_DIRECTIVES, - FILTER_DIRECTIVES + FILTER_DIRECTIVES, + JobLogViewerComponent ], providers: [ ErrorHandler, diff --git a/src/ui_ng/lib/src/vulnerability-scanning/result-bar-chart.component.spec.ts b/src/ui_ng/lib/src/vulnerability-scanning/result-bar-chart.component.spec.ts index f796f628f..7af25b335 100644 --- a/src/ui_ng/lib/src/vulnerability-scanning/result-bar-chart.component.spec.ts +++ b/src/ui_ng/lib/src/vulnerability-scanning/result-bar-chart.component.spec.ts @@ -8,13 +8,16 @@ import { ScanningResultService, ScanningResultDefaultService, TagService, - TagDefaultService + TagDefaultService, + JobLogService, + JobLogDefaultService } from '../service/index'; import { SERVICE_CONFIG, IServiceConfig } from '../service.config'; import { ErrorHandler } from '../error-handler/index'; import { SharedModule } from '../shared/shared.module'; import { VULNERABILITY_SCAN_STATUS } from '../utils'; import { ChannelService } from '../channel/index'; +import { JobLogViewerComponent } from '../job-log-viewer/index'; describe('ResultBarChartComponent (inline template)', () => { let component: ResultBarChartComponent; @@ -52,13 +55,15 @@ describe('ResultBarChartComponent (inline template)', () => { ], declarations: [ ResultBarChartComponent, - ResultTipComponent], + ResultTipComponent, + JobLogViewerComponent], providers: [ ErrorHandler, ChannelService, { provide: SERVICE_CONFIG, useValue: testConfig }, { provide: TagService, useValue: TagDefaultService }, - { provide: ScanningResultService, useValue: ScanningResultDefaultService } + { provide: ScanningResultService, useValue: ScanningResultDefaultService }, + { provide: JobLogService, useValue: JobLogDefaultService} ] }); diff --git a/src/ui_ng/lib/src/vulnerability-scanning/result-bar-chart.component.ts b/src/ui_ng/lib/src/vulnerability-scanning/result-bar-chart.component.ts index 479c42eff..b51254a31 100644 --- a/src/ui_ng/lib/src/vulnerability-scanning/result-bar-chart.component.ts +++ b/src/ui_ng/lib/src/vulnerability-scanning/result-bar-chart.component.ts @@ -4,7 +4,8 @@ import { OnInit, OnDestroy, ChangeDetectionStrategy, - ChangeDetectorRef + ChangeDetectorRef, + ViewChild } from '@angular/core'; import { SCANNING_STYLES } from './scanning.css'; @@ -21,6 +22,7 @@ import { ErrorHandler } from '../error-handler/index'; import { toPromise } from '../utils'; import { Observable, Subscription } from 'rxjs/Rx'; import { ChannelService } from '../channel/index'; +import { JobLogViewerComponent } from '../job-log-viewer/index'; const STATE_CHECK_INTERVAL: number = 2000;//2s const RETRY_TIMES: number = 3; @@ -39,6 +41,9 @@ export class ResultBarChartComponent implements OnInit, OnDestroy { stateCheckTimer: Subscription; timerHandler: any; + @ViewChild("scanningLogViewer") + scanningJobLogViewer: JobLogViewerComponent; + constructor( private tagService: TagService, private scanningService: ScanningResultService, @@ -188,4 +193,11 @@ export class ResultBarChartComponent implements OnInit, OnDestroy { } }, duration); } + + //Check error log + viewLog(): void { + if (this.summary && this.summary.job_id) { + this.scanningJobLogViewer.open(this.summary.job_id); + } + } } diff --git a/src/ui_ng/lib/src/vulnerability-scanning/scanning.css.ts b/src/ui_ng/lib/src/vulnerability-scanning/scanning.css.ts index d9a280c22..5b4f9422a 100644 --- a/src/ui_ng/lib/src/vulnerability-scanning/scanning.css.ts +++ b/src/ui_ng/lib/src/vulnerability-scanning/scanning.css.ts @@ -20,6 +20,7 @@ export const SCANNING_STYLES: string = ` position: relative; top: 1px; margin-left: -5px; + cursor: pointer; } .scanning-button { diff --git a/src/ui_ng/lib/src/vulnerability-scanning/scanning.html.ts b/src/ui_ng/lib/src/vulnerability-scanning/scanning.html.ts index 1f607279b..aace726df 100644 --- a/src/ui_ng/lib/src/vulnerability-scanning/scanning.html.ts +++ b/src/ui_ng/lib/src/vulnerability-scanning/scanning.html.ts @@ -96,8 +96,10 @@ export const BAR_CHART_COMPONENT_HTML: string = ` {{'VULNERABILITY.STATE.QUEUED' | translate}}
- - {{'VULNERABILITY.STATE.ERROR' | translate}} + + + {{'VULNERABILITY.STATE.ERROR' | translate}} +
{{'VULNERABILITY.STATE.SCANNING' | translate}}
@@ -110,5 +112,6 @@ export const BAR_CHART_COMPONENT_HTML: string = ` {{'VULNERABILITY.STATE.UNKNOWN' | translate}}
+
`; \ No newline at end of file diff --git a/src/ui_ng/package.json b/src/ui_ng/package.json index 0dab15a1b..d64f0a6f0 100644 --- a/src/ui_ng/package.json +++ b/src/ui_ng/package.json @@ -31,7 +31,7 @@ "clarity-icons": "^0.9.8", "clarity-ui": "^0.9.8", "core-js": "^2.4.1", - "harbor-ui": "0.3.24", + "harbor-ui": "0.3.42", "intl": "^1.2.5", "mutationobserver-shim": "^0.3.2", "ngx-cookie": "^1.0.0", diff --git a/src/ui_ng/src/app/app-config.ts b/src/ui_ng/src/app/app-config.ts index ce5ac5167..0702cba12 100644 --- a/src/ui_ng/src/app/app-config.ts +++ b/src/ui_ng/src/app/app-config.ts @@ -30,6 +30,7 @@ export class AppConfig { overall_last_update: 0, details: [] }; + this.next_scan_all = 0; } with_notary: boolean; @@ -43,4 +44,5 @@ export class AppConfig { has_ca_root: boolean; harbor_version: string; clair_vulnerability_status?: ClairDBStatus; + next_scan_all: number; } \ No newline at end of file diff --git a/src/ui_ng/src/app/config/config.component.html b/src/ui_ng/src/app/config/config.component.html index 8895e863d..cdc9f58c4 100644 --- a/src/ui_ng/src/app/config/config.component.html +++ b/src/ui_ng/src/app/config/config.component.html @@ -32,7 +32,7 @@
- +
diff --git a/src/ui_ng/src/app/config/config.component.ts b/src/ui_ng/src/app/config/config.component.ts index d1c8dcd6b..85cff6aae 100644 --- a/src/ui_ng/src/app/config/config.component.ts +++ b/src/ui_ng/src/app/config/config.component.ts @@ -85,10 +85,6 @@ export class ConfigurationComponent implements OnInit, OnDestroy { return this.appConfigService.getConfig().with_clair; } - public get clairDB(): ClairDBStatus { - return this.appConfigService.getConfig().clair_vulnerability_status; - } - isCurrentTabLink(tabId: string): boolean { return this.currentTabId === tabId; } diff --git a/src/ui_ng/src/app/repository/tag-repository/tag-repository.component.html b/src/ui_ng/src/app/repository/tag-repository/tag-repository.component.html index 0ef68de50..3305a4db7 100644 --- a/src/ui_ng/src/app/repository/tag-repository/tag-repository.component.html +++ b/src/ui_ng/src/app/repository/tag-repository/tag-repository.component.html @@ -1,5 +1,5 @@ \ No newline at end of file diff --git a/src/ui_ng/src/app/repository/tag-repository/tag-repository.component.ts b/src/ui_ng/src/app/repository/tag-repository/tag-repository.component.ts index c2d4b1eea..4286bf6de 100644 --- a/src/ui_ng/src/app/repository/tag-repository/tag-repository.component.ts +++ b/src/ui_ng/src/app/repository/tag-repository/tag-repository.component.ts @@ -29,8 +29,6 @@ export class TagRepositoryComponent implements OnInit { repoName: string; hasProjectAdminRole: boolean = false; registryUrl: string; - withNotary: boolean; - hasSignedIn: boolean; constructor( private route: ActivatedRoute, @@ -40,7 +38,6 @@ export class TagRepositoryComponent implements OnInit { } ngOnInit() { - this.hasSignedIn = (this.session.getCurrentUser() !== null); let resolverData = this.route.snapshot.data; if (resolverData) { this.hasProjectAdminRole = (resolverData['projectResolver']).has_project_admin_role; @@ -49,7 +46,18 @@ export class TagRepositoryComponent implements OnInit { this.repoName = this.route.snapshot.params['repo']; this.registryUrl = this.appConfigService.getConfig().registry_url; - this.withNotary = this.appConfigService.getConfig().with_notary; + } + + get withNotary(): boolean { + return this.appConfigService.getConfig().with_notary; + } + + get withClair(): boolean { + return this.appConfigService.getConfig().with_clair; + } + + get hasSignedIn(): boolean { + return this.session.getCurrentUser() !== null; } watchTagClickEvt(tagEvt: TagClickEvent): void { diff --git a/src/ui_ng/src/i18n/lang/en-us-lang.json b/src/ui_ng/src/i18n/lang/en-us-lang.json index 6745c5e56..23f8c3095 100644 --- a/src/ui_ng/src/i18n/lang/en-us-lang.json +++ b/src/ui_ng/src/i18n/lang/en-us-lang.json @@ -416,7 +416,8 @@ "DAILY_POLICY": "Daily At", "REFRESH_POLICY": "Upon Refresh", "DB_REFRESH_TIME": "Database updated on", - "DB_NOT_READY": "Vulnerability database might not be fully ready!" + "DB_NOT_READY": "Vulnerability database might not be fully ready!", + "NEXT_SCAN": "Available after" }, "TEST_MAIL_SUCCESS": "Connection to mail server is verified.", "TEST_LDAP_SUCCESS": "Connection to LDAP server is verified.", @@ -464,7 +465,7 @@ "STATE": { "STOPPED": "Not Scanned", "QUEUED": "Queued", - "ERROR": "Error", + "ERROR": "View Log", "SCANNING": "Scanning", "UNKNOWN": "Unknown" }, @@ -496,7 +497,8 @@ "PLACEHOLDER": "Filter Vulnerabilities", "PACKAGE": "Package with", "PACKAGES": "Packages with", - "SCAN_NOW": "Scan" + "SCAN_NOW": "Scan", + "JOB_LOG_VIEWER": "View Scanning Job Log" }, "PUSH_IMAGE": { "TITLE": "Push Image", diff --git a/src/ui_ng/src/i18n/lang/es-es-lang.json b/src/ui_ng/src/i18n/lang/es-es-lang.json index 84e423c4b..9be60dcbb 100644 --- a/src/ui_ng/src/i18n/lang/es-es-lang.json +++ b/src/ui_ng/src/i18n/lang/es-es-lang.json @@ -417,7 +417,8 @@ "DAILY_POLICY": "Daily At", "REFRESH_POLICY": "Upon Refresh", "DB_REFRESH_TIME": "Database updated on", - "DB_NOT_READY": "Vulnerability database might not be fully ready!" + "DB_NOT_READY": "Vulnerability database might not be fully ready!", + "NEXT_SCAN": "Available after" }, "TEST_MAIL_SUCCESS": "La conexión al servidor de correo ha sido verificada.", "TEST_LDAP_SUCCESS": "La conexión al servidor LDAP ha sido verificada.", @@ -463,7 +464,7 @@ "STATE": { "STOPPED": "Not Scanned", "QUEUED": "Queued", - "ERROR": "Error", + "ERROR": "View Log", "SCANNING": "Scanning", "UNKNOWN": "Unknown" }, @@ -495,7 +496,8 @@ "PLACEHOLDER": "Filter Vulnerabilities", "PACKAGE": "Package with", "PACKAGES": "Packages with", - "SCAN_NOW": "Scan" + "SCAN_NOW": "Scan", + "JOB_LOG_VIEWER": "View Scanning Job Log" }, "PUSH_IMAGE": { "TITLE": "Push Image", diff --git a/src/ui_ng/src/i18n/lang/zh-cn-lang.json b/src/ui_ng/src/i18n/lang/zh-cn-lang.json index e7a33e58b..e979c19b6 100644 --- a/src/ui_ng/src/i18n/lang/zh-cn-lang.json +++ b/src/ui_ng/src/i18n/lang/zh-cn-lang.json @@ -420,7 +420,8 @@ "DAILY_POLICY": "每日定时", "REFRESH_POLICY": "缺陷库刷新后", "DB_REFRESH_TIME": "数据库更新于", - "DB_NOT_READY": "缺陷数据库可能没有完全准备好!" + "DB_NOT_READY": "缺陷数据库可能没有完全准备好!", + "NEXT_SCAN": "下次可用时间" }, "TEST_MAIL_SUCCESS": "邮件服务器的连通正常。", "TEST_LDAP_SUCCESS": "LDAP服务器的连通正常。", @@ -468,7 +469,7 @@ "STATE": { "STOPPED": "未扫描", "QUEUED": "已入队列", - "ERROR": "错误", + "ERROR": "查看日志", "SCANNING": "扫描中", "UNKNOWN": "未知" }, @@ -500,7 +501,8 @@ "PLACEHOLDER": "过滤缺陷", "PACKAGE": "个组件有", "PACKAGES": "个组件有", - "SCAN_NOW": "扫描" + "SCAN_NOW": "扫描", + "JOB_LOG_VIEWER": "查看扫描日志" }, "PUSH_IMAGE": { "TITLE": "推送镜像", From 598f2b43dbb0169853f3c69de0f3b1b96d12a22f Mon Sep 17 00:00:00 2001 From: Steven Zou Date: Mon, 24 Jul 2017 13:49:43 +0800 Subject: [PATCH 09/21] add missing i18n message keys --- .../list-project-ro/list-project-ro.component.html | 5 ++--- .../list-repository-ro.component.html | 5 ++--- src/ui_ng/src/i18n/lang/en-us-lang.json | 12 ++++++++---- src/ui_ng/src/i18n/lang/es-es-lang.json | 12 ++++++++---- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/ui_ng/src/app/shared/list-project-ro/list-project-ro.component.html b/src/ui_ng/src/app/shared/list-project-ro/list-project-ro.component.html index c8df1a62c..2d1bdc9db 100644 --- a/src/ui_ng/src/app/shared/list-project-ro/list-project-ro.component.html +++ b/src/ui_ng/src/app/shared/list-project-ro/list-project-ro.component.html @@ -10,9 +10,8 @@ {{p.creation_time | date: 'short'}} - {{pagination.firstItem + 1}} - {{pagination.lastItem +1 }} {{'PROJECT.OF' | translate}} - {{pagination.totalItems }} {{'PROJECT.ITEMS' | translate}} - + {{pagination.firstItem + 1}} - {{pagination.lastItem +1 }} {{'PROJECT.OF' | translate}} {{pagination.totalItems }} {{'PROJECT.ITEMS' | translate}} + diff --git a/src/ui_ng/src/app/shared/list-repository-ro/list-repository-ro.component.html b/src/ui_ng/src/app/shared/list-repository-ro/list-repository-ro.component.html index 2871d8c51..d4b622465 100644 --- a/src/ui_ng/src/app/shared/list-repository-ro/list-repository-ro.component.html +++ b/src/ui_ng/src/app/shared/list-repository-ro/list-repository-ro.component.html @@ -8,8 +8,7 @@ {{r.pull_count}} - {{pagination.firstItem + 1}} - {{pagination.lastItem +1 }} {{'REPOSITORY.OF' | translate}} - {{pagination.totalItems }} {{'REPOSITORY.ITEMS' | translate}} - + {{pagination.firstItem + 1}} - {{pagination.lastItem +1 }} {{'REPOSITORY.OF' | translate}} {{pagination.totalItems }} {{'REPOSITORY.ITEMS' | translate}} + \ No newline at end of file diff --git a/src/ui_ng/src/i18n/lang/en-us-lang.json b/src/ui_ng/src/i18n/lang/en-us-lang.json index 23f8c3095..4d9bfb203 100644 --- a/src/ui_ng/src/i18n/lang/en-us-lang.json +++ b/src/ui_ng/src/i18n/lang/en-us-lang.json @@ -117,7 +117,8 @@ "DELETION_TITLE": "Confirm user deletion", "DELETION_SUMMARY": "Do you want to delete user {{param}}?", "DELETE_SUCCESS": "User deleted successfully.", - "ITEMS": "items" + "ITEMS": "items", + "OF": "of" }, "PROJECT": { "PROJECTS": "Projects", @@ -152,7 +153,8 @@ "DELETED_SUCCESS": "Deleted project successfully.", "TOGGLED_SUCCESS": "Toggled project successfully.", "FAILED_TO_DELETE_PROJECT": "Project contains repositories or replication rules cannot be deleted.", - "INLINE_HELP_PUBLIC": "When a project is set to public, anyone has read permission to the repositories under this project, and the user does not need to run \"docker login\" before pulling images under this project." + "INLINE_HELP_PUBLIC": "When a project is set to public, anyone has read permission to the repositories under this project, and the user does not need to run \"docker login\" before pulling images under this project.", + "OF": "of" }, "PROJECT_DETAIL": { "REPOSITORIES": "Repositories", @@ -182,7 +184,8 @@ "DELETION_SUMMARY": "Do you want to delete project member {{param}}?", "ADDED_SUCCESS": "Added member successfully.", "DELETED_SUCCESS": "Deleted member successfully.", - "SWITCHED_SUCCESS": "Switched member role successfully." + "SWITCHED_SUCCESS": "Switched member role successfully.", + "OF": "of" }, "AUDIT_LOG": { "USERNAME": "Username", @@ -201,7 +204,8 @@ "SIMPLE": "Simple", "ITEMS": "items", "FILTER_PLACEHOLDER": "Filter Logs", - "INVALID_DATE": "Invalid date." + "INVALID_DATE": "Invalid date.", + "OF": "of" }, "REPLICATION": { "REPLICATION_RULE": "Replication Rule", diff --git a/src/ui_ng/src/i18n/lang/es-es-lang.json b/src/ui_ng/src/i18n/lang/es-es-lang.json index 9be60dcbb..6b79c19bb 100644 --- a/src/ui_ng/src/i18n/lang/es-es-lang.json +++ b/src/ui_ng/src/i18n/lang/es-es-lang.json @@ -117,7 +117,8 @@ "DELETION_TITLE": "Confirmar eliminación de usuario", "DELETION_SUMMARY": "¿Quiere eliminar el usuario {{param}}?", "DELETE_SUCCESS": "Usuario eliminado satisfactoriamente.", - "ITEMS": "elementos" + "ITEMS": "elementos", + "OF": "of" }, "PROJECT": { "PROJECTS": "Proyectos", @@ -152,7 +153,8 @@ "DELETED_SUCCESS": "Proyecto eliminado satisfactoriamente.", "TOGGLED_SUCCESS": "Proyecto alternado satisfactoriamente.", "FAILED_TO_DELETE_PROJECT": "Los proyectos que contienen repositorios o reglas de replicación no pueden eliminarse.", - "INLINE_HELP_PUBLIC": "Cuando un proyecto se marca como público, todo el mundo tiene permisos de lectura sobre los repositorio de dicho proyecto, y no hace falta hacer \"docker login\" antes de subir imágenes a ellos." + "INLINE_HELP_PUBLIC": "Cuando un proyecto se marca como público, todo el mundo tiene permisos de lectura sobre los repositorio de dicho proyecto, y no hace falta hacer \"docker login\" antes de subir imágenes a ellos.", + "OF": "of" }, "PROJECT_DETAIL": { "REPOSITORIES": "Repositorios", @@ -182,7 +184,8 @@ "DELETION_SUMMARY": "¿Quiere eliminar el miembro {{param}} del proyecto?", "ADDED_SUCCESS": "Miembro añadido satisfactoriamente.", "DELETED_SUCCESS": "Miembro eliminado satisfactoriamente", - "SWITCHED_SUCCESS": "Rol del miembro cambiado satisfactoriamente." + "SWITCHED_SUCCESS": "Rol del miembro cambiado satisfactoriamente.", + "OF": "of" }, "AUDIT_LOG": { "USERNAME": "Nombre de usuario", @@ -201,7 +204,8 @@ "SIMPLE": "Simple", "ITEMS": "elementos", "FILTER_PLACEHOLDER": "Filtrar logs", - "INVALID_DATE": "Fecha invalida." + "INVALID_DATE": "Fecha invalida.", + "OF": "of" }, "REPLICATION": { "REPLICATION_RULE": "Reglas de Replicación", From 686b477775e6964c50f58fe2fb7f2e3482220e63 Mon Sep 17 00:00:00 2001 From: Yan Date: Mon, 24 Jul 2017 02:19:32 -0700 Subject: [PATCH 10/21] update registry to 2.6.2 (#2851) rm dockerfile update add comments --- Makefile | 2 +- make/docker-compose.tpl | 2 +- make/photon/registry/Dockerfile | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 make/photon/registry/Dockerfile diff --git a/Makefile b/Makefile index c5a46a9fe..e02021485 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ REGISTRYSERVER= REGISTRYPROJECTNAME=vmware DEVFLAG=true NOTARYFLAG=false -REGISTRYVERSION=2.6.1-photon +REGISTRYVERSION=2.6.2-photon NGINXVERSION=1.11.13 PHOTONVERSION=1.0 NOTARYVERSION=server-0.5.0 diff --git a/make/docker-compose.tpl b/make/docker-compose.tpl index e22a5793d..2878d2b9f 100644 --- a/make/docker-compose.tpl +++ b/make/docker-compose.tpl @@ -11,7 +11,7 @@ services: networks: - harbor registry: - image: vmware/registry:2.6.1-photon + image: vmware/registry:2.6.2-photon container_name: registry restart: always volumes: diff --git a/make/photon/registry/Dockerfile b/make/photon/registry/Dockerfile new file mode 100644 index 000000000..a4100c4d1 --- /dev/null +++ b/make/photon/registry/Dockerfile @@ -0,0 +1,15 @@ +FROM library/photon:1.0 + +MAINTAINER wangyan@vmware.com + +# The original script in the docker offical registry image. +COPY entrypoint.sh / +RUN chmod u+x /entrypoint.sh + +COPY registry /usr/bin +RUN chmod u+x /usr/bin/registry + +VOLUME ["/var/lib/registry"] +EXPOSE 5000 +ENTRYPOINT ["/entrypoint.sh"] +CMD ["/etc/docker/registry/config.yml"] From fd473cbaa007d80aba2a4767f5a12d71cec1d2c2 Mon Sep 17 00:00:00 2001 From: Yan Date: Mon, 24 Jul 2017 19:48:12 -0700 Subject: [PATCH 11/21] update DB Migrator according latest change. (#2763) * update DB Migrator according latest change. update format update add import update return the rc from container update update to false * add user_id --- tools/migration/changelog.md | 5 +- tools/migration/db_meta.py | 12 +- tools/migration/export | 109 ++++++++++++++++++ tools/migration/import | 88 ++++++++++++++ .../migration_harbor/versions/1_2_0.py | 5 +- tools/migration/run.sh | 10 +- 6 files changed, 223 insertions(+), 6 deletions(-) create mode 100755 tools/migration/export create mode 100755 tools/migration/import diff --git a/tools/migration/changelog.md b/tools/migration/changelog.md index 20ea6d8e2..cfcb21912 100644 --- a/tools/migration/changelog.md +++ b/tools/migration/changelog.md @@ -45,4 +45,7 @@ Changelog for harbor database schema - delete foreign key (user_id) references user(user_id)from table `access_log` - delete foreign key (project_id) references project(project_id)from table `access_log` - add column `username` varchar (32) to table `access_log` - - alter column `realname` on table `user`: varchar(20)->varchar(255) \ No newline at end of file + - alter column `realname` on table `user`: varchar(20)->varchar(255) + - create table `img_scan_job` + - create table `img_scan_overview` + - create table `clair_vuln_timestamp` \ No newline at end of file diff --git a/tools/migration/db_meta.py b/tools/migration/db_meta.py index ac7d45bf2..248667fe7 100644 --- a/tools/migration/db_meta.py +++ b/tools/migration/db_meta.py @@ -169,10 +169,18 @@ class ImageScanJob(Base): class ImageScanOverview(Base): __tablename__ = "img_scan_overview" + id = sa.Column(sa.Integer, nullable=False, primary_key=True) + image_digest = sa.Column(sa.String(128), nullable=False) scan_job_id = sa.Column(sa.Integer, nullable=False) - image_digest = sa.Column(sa.String(128), nullable=False, primary_key=True) severity = sa.Column(sa.Integer, nullable=False, server_default=sa.text("'0'")) components_overview = sa.Column(sa.String(2048)) details_key = sa.Column(sa.String(128)) creation_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP")) - update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP")) \ No newline at end of file + update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP")) + +class ClairVulnTimestamp(Base): + __tablename__ = "clair_vuln_timestamp" + + id = sa.Column(sa.Integer, nullable=False, primary_key=True) + namespace = sa.Column(sa.String(128), nullable=False, unique=True) + last_update = sa.Column(mysql.TIMESTAMP) diff --git a/tools/migration/export b/tools/migration/export new file mode 100755 index 000000000..76f746193 --- /dev/null +++ b/tools/migration/export @@ -0,0 +1,109 @@ +#!/usr/bin/python +import json +import fileinput +from optparse import OptionParser +import os +import MySQLdb +import sys + +class Parameters(object): + def __init__(self): + self.dbuser = '' + self.dbpwd = '' + self.exportpath = '' + self.init_from_input() + + @staticmethod + def parse_input(): + usage = "usage: %prog [options] " + parser = OptionParser(usage) + parser.add_option("-u", "--dbuser", dest="dbuser", help="db user") + parser.add_option("-p", "--dbpwd", dest="dbpwd", help="db password") + parser.add_option("-o", "--exportpath", dest="exportpath", help="the path of exported json file") + + (options, args) = parser.parse_args() + return (options.dbuser, options.dbpwd, options.exportpath) + + def init_from_input(self): + (self.dbuser, self.dbpwd, self.exportpath) = Parameters.parse_input() + +class Project: + def __init__(self, project_id, name, public): + self.project_id = project_id + self.project_name = name + if public == 0: + self.public = "false" + elif public == 1: + self.public = "true" + else: + self.public = "false" + +class HarborUtil: + def __init__(self, dbuser, dbpwd): + self.serverName = 'localhost' + self.user = dbuser + self.password = dbpwd + self.port = '3306' + self.subDB = 'registry' + self.db = None + self.cursor = None + + def connect(self): + try: + self.db = MySQLdb.connect(host=self.serverName, user=self.user, + passwd=self.password, db=self.subDB) + self.cursor = self.db.cursor() + except Exception, e: + raise Exception(e) + + def close(self): + try: + self.cursor.close() + self.db.close() + except Exception, e: + print str(e) + + def get_projects(self): + projects = [] + try: + query = "SELECT project_id, name, public from registry.project where deleted=0" + self.cursor.execute(query) + self.cursor.fetchall() + for result in self.cursor: + projects.append(Project(int(result[0]), result[1], result[2])) + return projects + except Exception, e: + raise Exception(e) + +def delfile(src): + if not os.path.exists(src): + return + try: + os.remove(src) + except Exception, e: + raise Exception("unable to delete file: %s, error: %s" % (src, str(e))) + +def main(): + commandline_input = Parameters() + harbor = HarborUtil(commandline_input.dbuser, commandline_input.dbpwd) + + try: + harbor.connect() + projects = harbor.get_projects() + if len(projects) == 0: + return + + harbor_projects_json = commandline_input.exportpath + '/harbor_projects.json' + delfile(harbor_projects_json) + + with open(harbor_projects_json, 'w') as outfile: + json.dump({'projects': [project.__dict__ for project in projects]}, outfile, sort_keys=True, indent=4) + + except Exception, e: + print e + sys.exit(1) + finally: + harbor.close() + +if __name__ == '__main__': + main() diff --git a/tools/migration/import b/tools/migration/import new file mode 100755 index 000000000..af97830ec --- /dev/null +++ b/tools/migration/import @@ -0,0 +1,88 @@ +#!/usr/bin/python +import json +from optparse import OptionParser +import os +import urllib2 +import sys +import logging +import logging.config + +logging.basicConfig(filename="import_project.log", level=logging.INFO) +logger = logging.getLogger() + +class Parameters(object): + def __init__(self): + self.admiral_endpoint = '' + self.admiral_token = '' + self.projectsfile = '' + self.init_from_input() + + @staticmethod + def parse_input(): + usage = "usage: %prog [options] " + parser = OptionParser(usage) + parser.add_option("-a", "--admiralendpoint", dest="admiral_endpoint", help="admiral endpoint") + parser.add_option("-t", "--token", dest="admiral_token", help="admiral token") + parser.add_option("-f", "--projectsfile", dest="projectsfile", help="the path of exported json file") + + (options, args) = parser.parse_args() + return (options.admiral_endpoint, options.admiral_token, options.projectsfile) + + def init_from_input(self): + (self.admiral_endpoint, self.admiral_token, self.projectsfile) = Parameters.parse_input() + +class Project: + def __init__(self, name, public): + self.project_name = name + self.public = public + +class Admiral: + def __init__(self, admiral_url, token): + self.admiral_url = admiral_url + '/projects' + self.token = token + + def __import_project(self, project, retry=True): + project_data = json.dumps({ "name": project.project_name, "isPublic": project.public, + "customProperties": {"__enableContentTrust": False, "__preventVulnerableImagesFromRunning":False, + "__preventVulnerableImagesFromRunningSeverity":"high", "__automaticallyScanImagesOnPush":False }}) + data_len = len(project_data) + request = urllib2.Request(self.admiral_url, project_data) + request.add_header('x-xenon-auth-token', self.token) + request.add_header('Content-Type', 'application/json') + request.add_header('Content-Length', data_len) + + try: + urllib2.urlopen(request) + except Exception, e: + if not retry: + logger.error("failed to import project: %s, admiral_endpoint: %s, error: %s " % (project.project_name, self.admiral_url, str(e))) + return + self.__import_project(project, False) + + def import_project(self, projects): + for project in projects: + self.__import_project(project) + +def main(): + commandline_input = Parameters() + admiral = Admiral(commandline_input.admiral_endpoint, commandline_input.admiral_token) + + try: + if not os.path.exists(commandline_input.projectsfile): + raise Exception('Error: %s does not exist' % commandline_input.projectsfile) + + with open(commandline_input.projectsfile, 'r') as project_data_file: + project_data = json.load(project_data_file) + + projects_import_list = [] + for item in project_data['projects']: + projects_import_list.append(Project(item['project_name'], item['public'])) + + admiral.import_project(projects_import_list) + + except Exception, e: + logger.error("failed to import project, admiral_endpoint: %s, error: %s " % (commandline_input.admiral_endpoint, str(e))) + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/tools/migration/migration_harbor/versions/1_2_0.py b/tools/migration/migration_harbor/versions/1_2_0.py index 5c41509fd..a00654dcf 100644 --- a/tools/migration/migration_harbor/versions/1_2_0.py +++ b/tools/migration/migration_harbor/versions/1_2_0.py @@ -59,10 +59,11 @@ def upgrade(): op.drop_column("access_log", "user_id") op.drop_column("repository", "owner_id") - #create tables: img_scan_job, img_scan_overview + #create tables: img_scan_job, img_scan_overview, clair_vuln_timestamp ImageScanJob.__table__.create(bind) ImageScanOverview.__table__.create(bind) - + ClairVulnTimestamp.__table__.create(bind) + def downgrade(): """ Downgrade has been disabled. diff --git a/tools/migration/run.sh b/tools/migration/run.sh index 82fc09d87..be1935064 100755 --- a/tools/migration/run.sh +++ b/tools/migration/run.sh @@ -31,7 +31,7 @@ if [[ ( $1 = "up" || $1 = "upgrade" ) && ${SKIP_CONFIRM} != "y" ]]; then case $ans in [Yy]* ) ;; - [Nn]* ) + [Nn]* ) exit 0 ;; * ) echo "illegal answer: $ans. Upgrade abort!!" @@ -97,6 +97,14 @@ backup) mysqldump $DBCNF --add-drop-database --databases registry > ./backup/registry.sql echo "Backup performed." ;; +export) + echo "Performing export..." + ./export --dbuser ${DB_USR} --dbpwd ${DB_PWD} --exportpath ${EXPORTPATH} + rc="$?" + echo "Export performed." + echo $rc + exit $rc + ;; restore) echo "Performing restore..." mysql $DBCNF < ./backup/registry.sql From 7c2699953d6ebfddae31b1bcb3d175984f744dba Mon Sep 17 00:00:00 2001 From: Steven Zou Date: Tue, 25 Jul 2017 19:47:05 +0800 Subject: [PATCH 12/21] Fix config change watching issue --- src/common/notifier/config_watcher.go | 7 ++++--- src/common/notifier/notifier.go | 2 ++ src/common/scheduler/policy/alternate_policy.go | 2 ++ src/common/scheduler/watcher.go | 1 + 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/common/notifier/config_watcher.go b/src/common/notifier/config_watcher.go index bd86ef231..8eab3b88e 100644 --- a/src/common/notifier/config_watcher.go +++ b/src/common/notifier/config_watcher.go @@ -2,7 +2,6 @@ package notifier import ( "errors" - "reflect" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils" @@ -27,8 +26,10 @@ func WatchConfigChanges(cfg map[string]interface{}) error { } if t, yes := policyCfg.Parm["daily_time"]; yes { - if reflect.TypeOf(t).Kind() == reflect.Int { - policyNotification.DailyTime = (int64)(t.(int)) + if dt, success := t.(float64); success { + policyNotification.DailyTime = (int64)(dt) + } else { + return errors.New("Invalid daily_time type") } } diff --git a/src/common/notifier/notifier.go b/src/common/notifier/notifier.go index f93df9462..d68b9a997 100644 --- a/src/common/notifier/notifier.go +++ b/src/common/notifier/notifier.go @@ -200,6 +200,8 @@ func (nw *NotificationWatcher) Notify(notification Notification) error { if err := hd.Handle(notification.Value); err != nil { //Currently, we just log the error log.Errorf("Error occurred when triggerring handler %s of topic %s: %s\n", reflect.TypeOf(hd).String(), notification.Topic, err.Error()) + } else { + log.Infof("Handle notification with topic '%s': %#v\n", notification.Topic, notification.Value) } }() }(h, handlerChan) diff --git a/src/common/scheduler/policy/alternate_policy.go b/src/common/scheduler/policy/alternate_policy.go index 8d6ac3859..d57a103b4 100644 --- a/src/common/scheduler/policy/alternate_policy.go +++ b/src/common/scheduler/policy/alternate_policy.go @@ -5,6 +5,7 @@ import ( "time" "github.com/vmware/harbor/src/common/scheduler/task" + "github.com/vmware/harbor/src/common/utils/log" ) //AlternatePolicyConfiguration store the related configurations for alternate policy. @@ -125,6 +126,7 @@ func (alp *AlternatePolicy) Evaluate() (<-chan bool, error) { } if diff > 0 { //Wait for a while. + log.Infof("Waiting for %d seconds after comparing offset %d and utc time %d\n", diff, alp.config.OffsetTime, utcTime) select { case <-time.After(time.Duration(diff) * time.Second): case <-alp.terminator: diff --git a/src/common/scheduler/watcher.go b/src/common/scheduler/watcher.go index 9827ab37c..23067dab5 100644 --- a/src/common/scheduler/watcher.go +++ b/src/common/scheduler/watcher.go @@ -70,6 +70,7 @@ func (wc *Watcher) Start() { select { case <-evalChan: { + log.Infof("Receive evaluation signal from policy '%s'\n", pl.Name()) //Start to run the attached tasks. for _, t := range pl.Tasks() { go func(tk task.Task) { From 97b334c3c04f022a4f35fbc51f5356801b8938ce Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Tue, 25 Jul 2017 21:12:03 +0800 Subject: [PATCH 13/21] fix #1953 --- src/jobservice/job/workerpool.go | 15 +++++++++------ src/jobservice/main.go | 8 ++++++-- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/jobservice/job/workerpool.go b/src/jobservice/job/workerpool.go index 082abeef8..910c55f4d 100644 --- a/src/jobservice/job/workerpool.go +++ b/src/jobservice/job/workerpool.go @@ -16,6 +16,7 @@ package job import ( "fmt" + "sync" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" @@ -33,6 +34,9 @@ type workerPool struct { // WorkerPools is a map contains workerpools for different types of jobs. var WorkerPools map[Type]*workerPool +// For WorkerPools initialization. +var once sync.Once + //TODO: remove the hard code? const maxScanWorker = 3 @@ -118,16 +122,15 @@ func NewWorker(id int, t Type, wp *workerPool) *Worker { // InitWorkerPools create worker pools for different types of jobs. func InitWorkerPools() error { - if len(WorkerPools) > 0 { - return fmt.Errorf("The WorkerPool map has been initialised") - } maxRepWorker, err := config.MaxJobWorkers() if err != nil { return err } - WorkerPools = make(map[Type]*workerPool) - WorkerPools[ReplicationType] = createWorkerPool(maxRepWorker, ReplicationType) - WorkerPools[ScanType] = createWorkerPool(maxScanWorker, ScanType) + once.Do(func() { + WorkerPools = make(map[Type]*workerPool) + WorkerPools[ReplicationType] = createWorkerPool(maxRepWorker, ReplicationType) + WorkerPools[ScanType] = createWorkerPool(maxScanWorker, ScanType) + }) return nil } diff --git a/src/jobservice/main.go b/src/jobservice/main.go index e62a7a0e6..e6e9c1546 100644 --- a/src/jobservice/main.go +++ b/src/jobservice/main.go @@ -42,7 +42,9 @@ func main() { } initRouters() - job.InitWorkerPools() + if err := job.InitWorkerPools(); err != nil { + log.Fatalf("Failed to initialize worker pools, error: %v", err) + } go job.Dispatch() resumeJobs() beego.Run() @@ -71,6 +73,8 @@ func init() { configPath := os.Getenv("CONFIG_PATH") if len(configPath) != 0 { log.Infof("Config path: %s", configPath) - beego.LoadAppConfig("ini", configPath) + if err := beego.LoadAppConfig("ini", configPath); err != nil { + log.Fatalf("Failed to load config file: %s, error: %v", configPath, err) + } } } From 30bd67748eb5924acdc92b6f68dfeeb27322bf07 Mon Sep 17 00:00:00 2001 From: pengpengshui Date: Wed, 26 Jul 2017 03:51:47 +0800 Subject: [PATCH 14/21] fix #2853 about replication rule link issue --- .../total-replication-page.component.html | 2 +- .../total-replication-page.component.ts | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.html b/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.html index e4c28b783..b93ec2ca3 100644 --- a/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.html +++ b/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.html @@ -1,3 +1,3 @@
- +
\ No newline at end of file diff --git a/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.ts b/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.ts index a6662673e..fc4417eff 100644 --- a/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.ts +++ b/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.ts @@ -12,10 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. import { Component } from '@angular/core'; +import {ReplicationRule} from "../../../../lib/src/service/interface"; +import {Router} from "@angular/router"; @Component({ selector: 'total-replication', templateUrl: 'total-replication-page.component.html' }) export class TotalReplicationPageComponent { + + rule: ReplicationRule[]; + constructor(private router: Router){} + customRedirect(rule: ReplicationRule): void { + if (rule) { + this.router.navigateByUrl(`/harbor/projects/${rule.project_id}/replications`); + } + } } \ No newline at end of file From 9c6c71ed7203efc5feb6a8d6f0c7e57ba2de5a8b Mon Sep 17 00:00:00 2001 From: pengpengshui Date: Wed, 26 Jul 2017 04:16:21 +0800 Subject: [PATCH 15/21] modify code about replication rule link issue --- .../total-replication/total-replication-page.component.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.ts b/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.ts index fc4417eff..873330bc2 100644 --- a/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.ts +++ b/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.ts @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. import { Component } from '@angular/core'; -import {ReplicationRule} from "../../../../lib/src/service/interface"; + import {Router} from "@angular/router"; +import {ReplicationRule} from "harbor-ui"; @Component({ selector: 'total-replication', @@ -25,7 +26,7 @@ export class TotalReplicationPageComponent { constructor(private router: Router){} customRedirect(rule: ReplicationRule): void { if (rule) { - this.router.navigateByUrl(`/harbor/projects/${rule.project_id}/replications`); + this.router.navigate(["harbor", "projects", rule.project_id, "replications"]); } } } \ No newline at end of file From e4c21dc5de619c33e47aec2359253c64d8b66c0a Mon Sep 17 00:00:00 2001 From: pengpengshui Date: Wed, 26 Jul 2017 05:20:28 +0800 Subject: [PATCH 16/21] change code style again about replication rule link issue --- .../total-replication/total-replication-page.component.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.ts b/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.ts index 873330bc2..9716c07d7 100644 --- a/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.ts +++ b/src/ui_ng/src/app/replication/total-replication/total-replication-page.component.ts @@ -13,7 +13,7 @@ // limitations under the License. import { Component } from '@angular/core'; -import {Router} from "@angular/router"; +import {Router,ActivatedRoute} from "@angular/router"; import {ReplicationRule} from "harbor-ui"; @Component({ @@ -22,11 +22,11 @@ import {ReplicationRule} from "harbor-ui"; }) export class TotalReplicationPageComponent { - rule: ReplicationRule[]; - constructor(private router: Router){} + constructor(private router: Router, + private activeRoute: ActivatedRoute){} customRedirect(rule: ReplicationRule): void { if (rule) { - this.router.navigate(["harbor", "projects", rule.project_id, "replications"]); + this.router.navigate(['../../projects', rule.project_id, "replications"], { relativeTo: this.activeRoute }); } } } \ No newline at end of file From 1fd68529aef8bb0db3d30f8c73bb801af3b3a436 Mon Sep 17 00:00:00 2001 From: pengpengshui Date: Wed, 26 Jul 2017 10:44:06 +0800 Subject: [PATCH 17/21] fix #2839 about project log has no logs --- src/ui_ng/lib/src/log/recent-log.template.ts | 2 +- src/ui_ng/src/app/log/audit-log.component.html | 2 +- src/ui_ng/src/app/log/audit-log.component.ts | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/ui_ng/lib/src/log/recent-log.template.ts b/src/ui_ng/lib/src/log/recent-log.template.ts index 1f787d41e..e1f2aa7ed 100644 --- a/src/ui_ng/lib/src/log/recent-log.template.ts +++ b/src/ui_ng/lib/src/log/recent-log.template.ts @@ -32,7 +32,7 @@ export const LOG_TEMPLATE: string = ` {{pagination.firstItem + 1}} - {{pagination.lastItem + 1}} - of {{pagination.totalItems}} {{'AUDIT_LOG.ITEMS' | translate}} + {{'AUDIT_LOG.OF' | translate}} {{pagination.totalItems}} {{'AUDIT_LOG.ITEMS' | translate}} diff --git a/src/ui_ng/src/app/log/audit-log.component.html b/src/ui_ng/src/app/log/audit-log.component.html index c2d79ec6b..b51bd9dc6 100644 --- a/src/ui_ng/src/app/log/audit-log.component.html +++ b/src/ui_ng/src/app/log/audit-log.component.html @@ -58,7 +58,7 @@ {{pagination.firstItem + 1}} - {{pagination.lastItem +1 }} {{'AUDIT_LOG.OF' | translate}} {{pagination.totalItems }} {{'AUDIT_LOG.ITEMS' | translate}} - + diff --git a/src/ui_ng/src/app/log/audit-log.component.ts b/src/ui_ng/src/app/log/audit-log.component.ts index b7b8a27e5..32159858c 100644 --- a/src/ui_ng/src/app/log/audit-log.component.ts +++ b/src/ui_ng/src/app/log/audit-log.component.ts @@ -69,6 +69,7 @@ export class AuditLogComponent implements OnInit { pageOffset: number = 1; pageSize: number = 15; totalRecordCount: number; + currentPage: number; totalPage: number; @ViewChild('fromTime') fromTimeInput: NgModel; @@ -96,14 +97,14 @@ export class AuditLogComponent implements OnInit { retrieve(state?: State): void { if(state) { - this.queryParam.page = state.page.to + 1; + this.queryParam.page = Math.ceil((state.page.to + 1) / this.pageSize); + this.currentPage = this.queryParam.page; } this.auditLogService .listAuditLogs(this.queryParam) .subscribe( response=>{ - this.totalRecordCount = response.headers.get('x-total-count'); - this.totalPage = Math.ceil(this.totalRecordCount / this.pageSize); + this.totalRecordCount =parseInt(response.headers.get('x-total-count')); this.auditLogs = response.json(); }, error=>{ From 10c857346419419016de42bcb44ae9a05cdfd1f3 Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Mon, 24 Jul 2017 19:50:35 +0800 Subject: [PATCH 18/21] provide API for scanning images under a projet --- src/common/dao/repository.go | 11 +++--- src/ui/api/repository.go | 48 +++++++++++++++++-------- src/ui/api/utils.go | 2 +- src/ui/utils/utils.go | 69 +++++++++++++++++++++++------------- 4 files changed, 84 insertions(+), 46 deletions(-) diff --git a/src/common/dao/repository.go b/src/common/dao/repository.go index d5a61cf4c..fc6f75fbf 100644 --- a/src/common/dao/repository.go +++ b/src/common/dao/repository.go @@ -48,9 +48,9 @@ func GetRepositoryByName(name string) (*models.RepoRecord, error) { } // GetAllRepositories ... -func GetAllRepositories() ([]models.RepoRecord, error) { +func GetAllRepositories() ([]*models.RepoRecord, error) { o := GetOrmer() - var repos []models.RepoRecord + var repos []*models.RepoRecord _, err := o.QueryTable("repository"). OrderBy("Name").All(&repos) return repos, err @@ -160,9 +160,10 @@ func GetRepositoriesByProject(projectID int64, name string, if len(name) != 0 { qs = qs.Filter("Name__contains", name) } - - _, err := qs.Limit(limit). - Offset(offset).All(&repositories) + if limit > 0 { + qs = qs.Limit(limit).Offset(offset) + } + _, err := qs.All(&repositories) return repositories, err } diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index 038824d6f..a730d302c 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -19,6 +19,7 @@ import ( "fmt" "io/ioutil" "net/http" + "strconv" "time" "github.com/docker/distribution/manifest/schema1" @@ -741,23 +742,40 @@ func (ra *RepositoryAPI) ScanAll() { ra.HandleUnauthorized() return } - if !ra.SecurityCtx.IsSysAdmin() { - ra.HandleForbidden(ra.SecurityCtx.GetUsername()) - return - } + projectIDStr := ra.GetString("project_id") + if len(projectIDStr) > 0 { //scan images under the project only. + pid, err := strconv.ParseInt(projectIDStr, 10, 64) + if err != nil || pid <= 0 { + ra.HandleBadRequest(fmt.Sprintf("Invalid project_id %s", projectIDStr)) + return + } + if !ra.SecurityCtx.HasAllPerm(pid) { + ra.HandleForbidden(ra.SecurityCtx.GetUsername()) + return + } + if err := uiutils.ScanImagesByProjectID(pid); err != nil { + log.Errorf("Failed triggering scan images in project: %d, error: %v", pid, err) + ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err)) + return + } + } else { //scan all images in Harbor + if !ra.SecurityCtx.IsSysAdmin() { + ra.HandleForbidden(ra.SecurityCtx.GetUsername()) + return + } + if !utils.ScanAllMarker().Check() { + log.Warningf("There is a scan all scheduled at: %v, the request will not be processed.", utils.ScanAllMarker().Next()) + ra.RenderError(http.StatusPreconditionFailed, "Unable handle frequent scan all requests") + return + } - if !utils.ScanAllMarker().Check() { - log.Warningf("There is a scan all scheduled at: %v, the request will not be processed.", utils.ScanAllMarker().Next()) - ra.RenderError(http.StatusPreconditionFailed, "Unable handle frequent scan all requests") - return + if err := uiutils.ScanAllImages(); err != nil { + log.Errorf("Failed triggering scan all images, error: %v", err) + ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err)) + return + } + utils.ScanAllMarker().Mark() } - - if err := uiutils.ScanAllImages(); err != nil { - log.Errorf("Failed triggering scan all images, error: %v", err) - ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err)) - return - } - utils.ScanAllMarker().Mark() ra.Ctx.ResponseWriter.WriteHeader(http.StatusAccepted) } diff --git a/src/ui/api/utils.go b/src/ui/api/utils.go index a639057f9..8d78729b4 100644 --- a/src/ui/api/utils.go +++ b/src/ui/api/utils.go @@ -175,7 +175,7 @@ func SyncRegistry(pm projectmanager.ProjectManager) error { return err } - var repoRecordsInDB []models.RepoRecord + var repoRecordsInDB []*models.RepoRecord repoRecordsInDB, err = dao.GetAllRepositories() if err != nil { log.Errorf("error occurred while getting all registories. %v", err) diff --git a/src/ui/utils/utils.go b/src/ui/utils/utils.go index 025c452d5..2d324de23 100644 --- a/src/ui/utils/utils.go +++ b/src/ui/utils/utils.go @@ -30,7 +30,7 @@ import ( "net/http" ) -// ScanAllImages scans all images of Harbor by submiting jobs to jobservice, the whole process will move one if failed to subit any job of a single image. +// ScanAllImages scans all images of Harbor by submiting jobs to jobservice, the whole process will move on if failed to submit any job of a single image. func ScanAllImages() error { regURL, err := config.RegistryURL() if err != nil { @@ -42,33 +42,52 @@ func ScanAllImages() error { log.Errorf("Failed to list all repositories, error: %v", err) return err } - log.Infof("Rescanning all images.") + log.Infof("Scanning all images on Harbor.") - go func() { - var repoClient *registry.Repository - var err error - var tags []string - for _, r := range repos { - repoClient, err = NewRepositoryClientForUI(regURL, true, "harbor-ui", r.Name, "pull") - if err != nil { - log.Errorf("Failed to initialize client for repository: %s, error: %v, skip scanning", r.Name, err) - continue - } - tags, err = repoClient.ListTag() - if err != nil { - log.Errorf("Failed to get tags for repository: %s, error: %v, skip scanning.", r.Name, err) - continue - } - for _, t := range tags { - if err = TriggerImageScan(r.Name, t); err != nil { - log.Errorf("Failed to scan image with repository: %s, tag: %s, error: %v.", r.Name, t, err) - } else { - log.Debugf("Triggered scan for image with repository: %s, tag: %s", r.Name, t) - } + go scanRepos(repos, regURL) + return nil +} + +// ScanImagesByProjectID scans all images under a projet, the whole process will move on if failed to submit any job of a single image. +func ScanImagesByProjectID(id int64) error { + regURL, err := config.RegistryURL() + if err != nil { + log.Errorf("Failed to load registry url") + return err + } + repos, err := dao.GetRepositoriesByProject(id, "", 0, 0) + if err != nil { + log.Errorf("Failed list repositories in project %d, error: %v", id, err) + return err + } + log.Infof("Scanning all images in project: %d ", id) + go scanRepos(repos, regURL) + return nil +} + +func scanRepos(repos []*models.RepoRecord, regURL string) { + var repoClient *registry.Repository + var err error + var tags []string + for _, r := range repos { + repoClient, err = NewRepositoryClientForUI(regURL, true, "harbor-ui", r.Name, "pull") + if err != nil { + log.Errorf("Failed to initialize client for repository: %s, error: %v, skip scanning", r.Name, err) + continue + } + tags, err = repoClient.ListTag() + if err != nil { + log.Errorf("Failed to get tags for repository: %s, error: %v, skip scanning.", r.Name, err) + continue + } + for _, t := range tags { + if err = TriggerImageScan(r.Name, t); err != nil { + log.Errorf("Failed to scan image with repository: %s, tag: %s, error: %v.", r.Name, t, err) + } else { + log.Debugf("Triggered scan for image with repository: %s, tag: %s", r.Name, t) } } - }() - return nil + } } // RequestAsUI is a shortcut to make a request attach UI secret and send the request. From 7244134e502ff92d396cb51c7758ad4c56b54860 Mon Sep 17 00:00:00 2001 From: Steven Zou Date: Wed, 26 Jul 2017 15:51:37 +0800 Subject: [PATCH 19/21] fix issue 2858 --- src/ui_ng/lib/src/tag/tag.component.ts | 1 + src/ui_ng/package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ui_ng/lib/src/tag/tag.component.ts b/src/ui_ng/lib/src/tag/tag.component.ts index 7e67be10a..caf46bf3f 100644 --- a/src/ui_ng/lib/src/tag/tag.component.ts +++ b/src/ui_ng/lib/src/tag/tag.component.ts @@ -241,6 +241,7 @@ export class TagComponent implements OnInit { //Whether show the 'scan now' menu canScanNow(t: Tag): boolean { if (!this.withClair) { return false; } + if (!this.hasProjectAdminRole) { return false; } let st: string = this.scanStatus(t); return st !== VULNERABILITY_SCAN_STATUS.pending && diff --git a/src/ui_ng/package.json b/src/ui_ng/package.json index d64f0a6f0..7792f6e8c 100644 --- a/src/ui_ng/package.json +++ b/src/ui_ng/package.json @@ -31,7 +31,7 @@ "clarity-icons": "^0.9.8", "clarity-ui": "^0.9.8", "core-js": "^2.4.1", - "harbor-ui": "0.3.42", + "harbor-ui": "0.3.47", "intl": "^1.2.5", "mutationobserver-shim": "^0.3.2", "ngx-cookie": "^1.0.0", From ec7b5ccfea91fc9ece6d2ed88c32ee3eeb2d237a Mon Sep 17 00:00:00 2001 From: Steven Zou Date: Wed, 26 Jul 2017 16:55:20 +0800 Subject: [PATCH 20/21] fix issue #2832 --- .../result-tip.component.ts | 18 +++++++++++++++--- .../src/vulnerability-scanning/scanning.css.ts | 7 +++++-- .../vulnerability-scanning/scanning.html.ts | 12 ++++++------ src/ui_ng/package.json | 2 +- src/ui_ng/src/i18n/lang/en-us-lang.json | 8 ++++---- src/ui_ng/src/i18n/lang/es-es-lang.json | 8 ++++---- src/ui_ng/src/i18n/lang/zh-cn-lang.json | 14 +++++++------- 7 files changed, 42 insertions(+), 27 deletions(-) diff --git a/src/ui_ng/lib/src/vulnerability-scanning/result-tip.component.ts b/src/ui_ng/lib/src/vulnerability-scanning/result-tip.component.ts index b01fdbd9a..570fe9b12 100644 --- a/src/ui_ng/lib/src/vulnerability-scanning/result-tip.component.ts +++ b/src/ui_ng/lib/src/vulnerability-scanning/result-tip.component.ts @@ -66,9 +66,21 @@ export class ResultTipComponent implements OnInit { } }); } - this.translate.get('VULNERABILITY.CHART.TOOLTIPS_TITLE', - { totalVulnerability: this.packagesWithVul, totalPackages: this.totalPackages }) - .subscribe((res: string) => this._tipTitle = res); + this.translate.get(this.packageText(this.totalPackages)).subscribe((p1: string) => { + this.translate.get(this.packageText(this.packagesWithVul)).subscribe((p2: string) => { + this.translate.get(this.unitText(this.packagesWithVul)).subscribe((vul: string) => { + this.translate.get('VULNERABILITY.CHART.TOOLTIPS_TITLE', + { + totalVulnerability: this.packagesWithVul, + totalPackages: this.totalPackages, + package: p1, + packageExt: p2, + vulnerability: vul + }) + .subscribe((res: string) => this._tipTitle = res); + }); + }); + }); } tipWidth(severity: VulnerabilitySeverity): string { diff --git a/src/ui_ng/lib/src/vulnerability-scanning/scanning.css.ts b/src/ui_ng/lib/src/vulnerability-scanning/scanning.css.ts index 5b4f9422a..2d0a87492 100644 --- a/src/ui_ng/lib/src/vulnerability-scanning/scanning.css.ts +++ b/src/ui_ng/lib/src/vulnerability-scanning/scanning.css.ts @@ -46,10 +46,10 @@ export const SCANNING_STYLES: string = ` background-color: red; } .bar-block-medium { - background-color: orange; + background-color: yellow; } .bar-block-low { - background-color: yellow; + background-color: orange; } .bar-block-none { background-color: green; @@ -84,4 +84,7 @@ export const SCANNING_STYLES: string = ` .refresh-btn:hover { color: #007CBB; } +.tip-icon-low { + color: orange; +} `; \ No newline at end of file diff --git a/src/ui_ng/lib/src/vulnerability-scanning/scanning.html.ts b/src/ui_ng/lib/src/vulnerability-scanning/scanning.html.ts index aace726df..d1cb9c9c4 100644 --- a/src/ui_ng/lib/src/vulnerability-scanning/scanning.html.ts +++ b/src/ui_ng/lib/src/vulnerability-scanning/scanning.html.ts @@ -13,23 +13,23 @@ export const TIP_COMPONENT_HTML: string = `
- {{highCount}} {{packageText(highCount) | translate }} {{'VULNERABILITY.SEVERITY.HIGH' | translate }} + {{highCount}} {{'VULNERABILITY.SEVERITY.HIGH' | translate }}
- {{mediumCount}} {{packageText(mediumCount) | translate }} {{'VULNERABILITY.SEVERITY.MEDIUM' | translate }} + {{mediumCount}} {{'VULNERABILITY.SEVERITY.MEDIUM' | translate }}
- - {{lowCount}} {{packageText(lowCount) | translate }} {{'VULNERABILITY.SEVERITY.LOW' | translate }} + + {{lowCount}} {{'VULNERABILITY.SEVERITY.LOW' | translate }}
- {{unknownCount}} {{packageText(unknownCount) | translate }} {{'VULNERABILITY.SEVERITY.UNKNOWN' | translate }} + {{unknownCount}} {{'VULNERABILITY.SEVERITY.UNKNOWN' | translate }}
- {{noneCount}} {{packageText(noneCount) | translate }} {{'VULNERABILITY.SEVERITY.NONE' | translate }} + {{noneCount}} {{'VULNERABILITY.SEVERITY.NONE' | translate }}
diff --git a/src/ui_ng/package.json b/src/ui_ng/package.json index 7792f6e8c..99107159a 100644 --- a/src/ui_ng/package.json +++ b/src/ui_ng/package.json @@ -31,7 +31,7 @@ "clarity-icons": "^0.9.8", "clarity-ui": "^0.9.8", "core-js": "^2.4.1", - "harbor-ui": "0.3.47", + "harbor-ui": "0.3.49", "intl": "^1.2.5", "mutationobserver-shim": "^0.3.2", "ngx-cookie": "^1.0.0", diff --git a/src/ui_ng/src/i18n/lang/en-us-lang.json b/src/ui_ng/src/i18n/lang/en-us-lang.json index 4d9bfb203..dfc1e9bec 100644 --- a/src/ui_ng/src/i18n/lang/en-us-lang.json +++ b/src/ui_ng/src/i18n/lang/en-us-lang.json @@ -485,8 +485,8 @@ "FOOT_OF": "of" }, "CHART": { - "SCANNING_TIME": "Scan completed", - "TOOLTIPS_TITLE": "This tag has {{totalVulnerability}} package(s) with vulnerabilities across all {{totalPackages}} package(s)." + "SCANNING_TIME": "Scan completed datetime", + "TOOLTIPS_TITLE": "This image includes {{totalPackages}} {{package}} with {{vulnerability}} in {{totalVulnerability}} of the {{packageExt}}." }, "SEVERITY": { "HIGH": "high", @@ -499,8 +499,8 @@ "SINGULAR": "Vulnerability", "PLURAL": "Vulnerabilities", "PLACEHOLDER": "Filter Vulnerabilities", - "PACKAGE": "Package with", - "PACKAGES": "Packages with", + "PACKAGE": "package", + "PACKAGES": "packages", "SCAN_NOW": "Scan", "JOB_LOG_VIEWER": "View Scanning Job Log" }, diff --git a/src/ui_ng/src/i18n/lang/es-es-lang.json b/src/ui_ng/src/i18n/lang/es-es-lang.json index 6b79c19bb..6d5a328d5 100644 --- a/src/ui_ng/src/i18n/lang/es-es-lang.json +++ b/src/ui_ng/src/i18n/lang/es-es-lang.json @@ -484,8 +484,8 @@ "FOOT_OF": "of" }, "CHART": { - "SCANNING_TIME": "Scan completed", - "TOOLTIPS_TITLE": "This tag has {{totalVulnerability}} package(s) with vulnerabilities across all {{totalPackages}} package(s)." + "SCANNING_TIME": "Scan completed datetime", + "TOOLTIPS_TITLE": "This image includes {{totalPackages}} {{package}} with {{vulnerability}} in {{totalVulnerability}} of the {{packageExt}}." }, "SEVERITY": { "HIGH": "high", @@ -498,8 +498,8 @@ "SINGULAR": "Vulnerability", "PLURAL": "Vulnerabilities", "PLACEHOLDER": "Filter Vulnerabilities", - "PACKAGE": "Package with", - "PACKAGES": "Packages with", + "PACKAGE": "package", + "PACKAGES": "packages", "SCAN_NOW": "Scan", "JOB_LOG_VIEWER": "View Scanning Job Log" }, diff --git a/src/ui_ng/src/i18n/lang/zh-cn-lang.json b/src/ui_ng/src/i18n/lang/zh-cn-lang.json index e979c19b6..60e7a03bf 100644 --- a/src/ui_ng/src/i18n/lang/zh-cn-lang.json +++ b/src/ui_ng/src/i18n/lang/zh-cn-lang.json @@ -477,7 +477,7 @@ "PLACEHOLDER": "没有扫描结果！", "COLUMN_ID": "缺陷码", "COLUMN_SEVERITY": "严重度", - "COLUMN_PACKAGE": "组", + "COLUMN_PACKAGE": "组件", "COLUMN_VERSION": "当前版本", "COLUMN_FIXED": "修复版本", "COLUMN_DESCRIPTION": "简介", @@ -486,7 +486,7 @@ }, "CHART": { "SCANNING_TIME": "扫描完成", - "TOOLTIPS_TITLE": "在此镜像的{{totalPackages}}包中扫描出{{totalVulnerability}}个有缺陷的包。" + "TOOLTIPS_TITLE": "在此镜像总共包含{{totalPackages}}{{package}},其中{{totalVulnerability}}个{{packageExt}}含有{{vulnerability}}。" }, "SEVERITY": { "HIGH": "严重", @@ -496,11 +496,11 @@ "UNKNOWN": "未知", "NONE": "无" }, - "SINGULAR": "缺陷", - "PLURAL": "缺陷", - "PLACEHOLDER": "过滤缺陷", - "PACKAGE": "个组件有", - "PACKAGES": "个组件有", + "SINGULAR": "漏洞", + "PLURAL": "漏洞", + "PLACEHOLDER": "过滤漏洞", + "PACKAGE": "组件", + "PACKAGES": "组件", "SCAN_NOW": "扫描", "JOB_LOG_VIEWER": "查看扫描日志" }, From f3c1caeb3cc10fc5475a68ac9191c70ad1e49159 Mon Sep 17 00:00:00 2001 From: Steven Zou Date: Wed, 26 Jul 2017 17:58:43 +0800 Subject: [PATCH 21/21] fix issue #2811 --- .../vulnerability-config.component.ts | 14 ++++++++++++-- src/ui_ng/package.json | 2 +- src/ui_ng/src/i18n/lang/en-us-lang.json | 1 + src/ui_ng/src/i18n/lang/es-es-lang.json | 1 + src/ui_ng/src/i18n/lang/zh-cn-lang.json | 3 ++- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/ui_ng/lib/src/config/vulnerability/vulnerability-config.component.ts b/src/ui_ng/lib/src/config/vulnerability/vulnerability-config.component.ts index 625bffff4..6a724670f 100644 --- a/src/ui_ng/lib/src/config/vulnerability/vulnerability-config.component.ts +++ b/src/ui_ng/lib/src/config/vulnerability/vulnerability-config.component.ts @@ -55,7 +55,7 @@ export class VulnerabilityConfigComponent implements OnInit { get scanAvailable(): boolean { let dt: Date = new Date(); - return !this.onSubmitting && (this.nextScanTime <= 0 || dt.getTime() > ((this.nextScanTime + 30) * 1000)); + return !this.onSubmitting && (this.nextScanTime <= 0 || dt.getTime() > ((this.nextScanTime + 300) * 1000)); } get nextScanTimestamp(): Date { @@ -256,6 +256,10 @@ export class VulnerabilityConfigComponent implements OnInit { return;//Aoid duplicated submitting } + if(!this.scanAvailable) { + return; //Aoid page hacking + } + this.onSubmitting = true; toPromise(this.scanningService.startScanningAll()) .then(() => { @@ -271,7 +275,13 @@ export class VulnerabilityConfigComponent implements OnInit { }); }) .catch(error => { - this.errorHandler.error(error); + if (error && error.status && error.status === 412) { + this.translate.get("CONFIG.SCANNING.TRIGGER_SCAN_ALL_FAIL", { error: '' + error }).subscribe((res: string) => { + this.errorHandler.error(res); + }); + } else { + this.errorHandler.error(error); + } this.onSubmitting = false; }); } diff --git a/src/ui_ng/package.json b/src/ui_ng/package.json index 99107159a..5a27ca495 100644 --- a/src/ui_ng/package.json +++ b/src/ui_ng/package.json @@ -31,7 +31,7 @@ "clarity-icons": "^0.9.8", "clarity-ui": "^0.9.8", "core-js": "^2.4.1", - "harbor-ui": "0.3.49", + "harbor-ui": "0.3.54", "intl": "^1.2.5", "mutationobserver-shim": "^0.3.2", "ngx-cookie": "^1.0.0", diff --git a/src/ui_ng/src/i18n/lang/en-us-lang.json b/src/ui_ng/src/i18n/lang/en-us-lang.json index dfc1e9bec..dc6d95ae7 100644 --- a/src/ui_ng/src/i18n/lang/en-us-lang.json +++ b/src/ui_ng/src/i18n/lang/en-us-lang.json @@ -413,6 +413,7 @@ }, "SCANNING": { "TRIGGER_SCAN_ALL_SUCCESS": "Trigger scan all successfully!", + "TRIGGER_SCAN_ALL_FAIL": "Failed to trigger scan all with error: {{error}", "TITLE": "Vulnerability Scanning", "SCAN_ALL": "Scan All", "SCAN_NOW": "SCAN NOW", diff --git a/src/ui_ng/src/i18n/lang/es-es-lang.json b/src/ui_ng/src/i18n/lang/es-es-lang.json index 6d5a328d5..729ee939b 100644 --- a/src/ui_ng/src/i18n/lang/es-es-lang.json +++ b/src/ui_ng/src/i18n/lang/es-es-lang.json @@ -414,6 +414,7 @@ }, "SCANNING": { "TRIGGER_SCAN_ALL_SUCCESS": "Trigger scan all successfully!", + "TRIGGER_SCAN_ALL_FAIL": "Failed to trigger scan all with error: {{error}", "TITLE": "Vulnerability Scanning", "SCAN_ALL": "Scan All", "SCAN_NOW": "SCAN NOW", diff --git a/src/ui_ng/src/i18n/lang/zh-cn-lang.json b/src/ui_ng/src/i18n/lang/zh-cn-lang.json index 60e7a03bf..27fcc42ca 100644 --- a/src/ui_ng/src/i18n/lang/zh-cn-lang.json +++ b/src/ui_ng/src/i18n/lang/zh-cn-lang.json @@ -412,7 +412,8 @@ "SCOPE": "LDAP搜索范围" }, "SCANNING": { - "TRIGGER_SCAN_ALL_SUCCESS": "成功启动扫描所有镜像任务!", + "TRIGGER_SCAN_ALL_SUCCESS": "启动扫描所有镜像任务成功!", + "TRIGGER_SCAN_ALL_FAIL": "启动扫描所有镜像任务失败:{{error}", "TITLE": "缺陷扫描", "SCAN_ALL": "扫描所有", "SCAN_NOW": "开始扫描",