diff --git a/.travis.yml b/.travis.yml index 2183b06e5..be4508a97 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ env: - POSTGRESQL_PWD: root123 - POSTGRESQL_DATABASE: registry - ADMINSERVER_URL: http://127.0.0.1:8888 - - DOCKER_COMPOSE_VERSION: 1.22.0 + - DOCKER_COMPOSE_VERSION: 1.23.0 - HARBOR_ADMIN: admin - HARBOR_ADMIN_PASSWD: Harbor12345 - CORE_SECRET: tempString diff --git a/Makefile b/Makefile index 9178e4198..79e5584aa 100644 --- a/Makefile +++ b/Makefile @@ -100,7 +100,7 @@ PREPARE_VERSION_NAME=versions REGISTRYVERSION=v2.7.1-patch-2819 NGINXVERSION=$(VERSIONTAG) NOTARYVERSION=v0.6.1 -CLAIRVERSION=v2.0.8 +CLAIRVERSION=v2.0.9 CLAIRDBVERSION=$(VERSIONTAG) MIGRATORVERSION=$(VERSIONTAG) REDISVERSION=$(VERSIONTAG) diff --git a/README.md b/README.md index 7e0264fa4..2d9e562e1 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Harbor is hosted by the [Cloud Native Computing Foundation](https://cncf.io) (CN **System requirements:** -**On a Linux host:** docker 17.06.0-ce+ and docker-compose 1.18.0+ . +**On a Linux host:** docker 17.06.0-ce+ and docker-compose 1.23.0+ . Download binaries of **[Harbor release ](https://github.com/vmware/harbor/releases)** and follow **[Installation & Configuration Guide](docs/installation_guide.md)** to install Harbor. diff --git a/docs/compile_guide.md b/docs/compile_guide.md index 30743b1d1..0dfd33fa0 100644 --- a/docs/compile_guide.md +++ b/docs/compile_guide.md @@ -4,14 +4,13 @@ This guide provides instructions for developers to build and run Harbor from sou ## Step 1: Prepare for a build environment for Harbor -Harbor is deployed as several Docker containers and most of the code is written in Go language. The build environment requires Python, Docker, Docker Compose and golang development environment. Please install the below prerequisites: +Harbor is deployed as several Docker containers and most of the code is written in Go language. The build environment requires Docker, Docker Compose and golang development environment. Please install the below prerequisites: Software | Required Version ----------------------|-------------------------- docker | 17.05 + -docker-compose | 1.11.0 + -python | 2.7 + +docker-compose | 1.23.0 + git | 1.9.1 + make | 3.81 + golang* | 1.7.3 + diff --git a/docs/installation_guide.md b/docs/installation_guide.md index 687beb094..9d0084799 100644 --- a/docs/installation_guide.md +++ b/docs/installation_guide.md @@ -31,7 +31,7 @@ Harbor is deployed as several Docker containers, and, therefore, can be deployed |Software|Version|Description| |---|---|---| |Docker engine|version 17.06.0-ce+ or higher|For installation instructions, please refer to: [docker engine doc](https://docs.docker.com/engine/installation/)| -|Docker Compose|version 1.18.0 or higher|For installation instructions, please refer to: [docker compose doc](https://docs.docker.com/compose/install/)| +|Docker Compose|version 1.23.0 or higher|For installation instructions, please refer to: [docker compose doc](https://docs.docker.com/compose/install/)| |Openssl|latest is preferred|Generate certificate and keys for Harbor| ### Network ports diff --git a/docs/manage_role_by_ldap_group.md b/docs/manage_role_by_ldap_group.md index 2e4bbc658..51e62e6b0 100644 --- a/docs/manage_role_by_ldap_group.md +++ b/docs/manage_role_by_ldap_group.md @@ -4,7 +4,7 @@ This guide provides instructions to manage roles by LDAP/AD group. You can impor ## Prerequisite -1. Harbor's auth_mode is ldap_auth and **[basic LDAP configure paremters](https://github.com/vmware/harbor/blob/master/docs/installation_guide.md#optional-parameters)** are configured. +1. Harbor's auth_mode is ldap_auth and **[basic LDAP configure parameters](https://github.com/vmware/harbor/blob/master/docs/installation_guide.md#optional-parameters)** are configured. 1. Memberof overlay This feature requires the LDAP/AD server enabled the feature **memberof overlay**. diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 01071390b..84b6665fc 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -122,8 +122,6 @@ paths: responses: '200': description: Project name exists. - '401': - description: User need to log in first. '404': description: Project name does not exist. '500': @@ -333,10 +331,10 @@ paths: description: Illegal format of provided ID value. '401': description: User need to log in first. - '404': - description: Project ID does not exist. '403': description: User does not have permission to get summary of the project. + '404': + description: Project ID does not exist. '500': description: Unexpected internal errors. '/projects/{project_id}/metadatas': @@ -1263,11 +1261,16 @@ paths: type: string required: true description: Relevant repository name. - - name: label_ids + - name: label_id in: query type: string required: false - description: A list of comma separated label IDs. + description: A label ID. + - name: detail + in: query + type: boolean + required: false + description: Bool value indicating whether return detailed information of the tag, such as vulnerability scan info, if set to false, only tag name is returned. tags: - Products responses: @@ -2380,10 +2383,10 @@ paths: $ref: '#/definitions/Namespace' '401': description: User need to login first. - '404': - description: No registry found. '403': description: User has no privilege for the operation. + '404': + description: No registry found. '500': description: Unexpected internal errors. /internal/syncregistry: @@ -2404,6 +2407,20 @@ paths: $ref: '#/responses/UnsupportedMediaType' '500': description: Unexpected internal errors. + /internal/syncquota: + post: + summary: Sync quota from registry/chart to DB. + description: | + This endpoint is for syncing quota usage of registry/chart with database. + tags: + - Products + responses: + '200': + description: Sync repositories successfully. + '401': + description: User need to log in first. + '403': + description: User does not have permission of system admin role. /systeminfo: get: summary: Get general system info @@ -3684,7 +3701,7 @@ paths: description: Unexpected internal errors. '/projects/{project_id}/webhook/policies': get: - sumary: List project webhook policies. + summary: List project webhook policies. description: | This endpoint returns webhook policies of a project. parameters: @@ -3712,7 +3729,7 @@ paths: '500': description: Unexpected internal errors. post: - sumary: Create project webhook policy. + summary: Create project webhook policy. description: | This endpoint create a webhook policy if the project does not have one. parameters: @@ -3757,7 +3774,7 @@ paths: in: path description: The id of webhook policy. required: true - type: int64 + type: integer format: int64 tags: - Products @@ -3791,7 +3808,7 @@ paths: in: path description: The id of webhook policy. required: true - type: int64 + type: integer format: int64 - name: policy in: body @@ -3829,7 +3846,7 @@ paths: in: path description: The id of webhook policy. required: true - type: int64 + type: integer format: int64 tags: - Products @@ -3908,7 +3925,7 @@ paths: description: Internal server errors. '/projects/{project_id}/webhook/jobs': get: - sumary: List project webhook jobs + summary: List project webhook jobs description: | This endpoint returns webhook jobs of a project. parameters: @@ -4023,6 +4040,9 @@ definitions: metadata: description: The metadata of the project. $ref: '#/definitions/ProjectMetadata' + cve_whitelist: + description: The CVE whitelist of the project. + $ref: '#/definitions/CVEWhitelist' count_limit: type: integer format: int64 @@ -4083,16 +4103,20 @@ definitions: description: 'The public status of the project. The valid values are "true", "false".' enable_content_trust: type: string - description: 'Whether content trust is enabled or not. If it is enabled, user cann''t pull unsigned images from this project. The valid values are "true", "false".' + description: 'Whether content trust is enabled or not. If it is enabled, user can''t pull unsigned images from this project. The valid values are "true", "false".' prevent_vul: type: string description: 'Whether prevent the vulnerable images from running. The valid values are "true", "false".' severity: type: string - description: 'If the vulnerability is high than severity defined here, the images cann''t be pulled. The valid values are "negligible", "low", "medium", "high", "critical".' + description: 'If the vulnerability is high than severity defined here, the images can''t be pulled. The valid values are "negligible", "low", "medium", "high", "critical".' auto_scan: type: string description: 'Whether scan images automatically when pushing. The valid values are "true", "false".' + reuse_sys_cve_whitelist: + type: string + description: 'Whether this project reuse the system level CVE whitelist as the whitelist of its own. The valid values are "true", "false". + If it is set to "true" the actual whitelist associate with this project, if any, will be ignored.' ProjectSummary: type: object properties: @@ -4841,6 +4865,9 @@ definitions: project_creation_restriction: type: string description: This attribute restricts what users have the permission to create project. It can be "everyone" or "adminonly". + quota_per_project_enable: + type: boolean + description: This attribute indicates whether quota per project enabled in harbor read_only: type: boolean description: '''docker push'' is prohibited by Harbor if you set it to true. ' @@ -4938,6 +4965,9 @@ definitions: project_creation_restriction: $ref: '#/definitions/StringConfigItem' description: This attribute restricts what users have the permission to create project. It can be "everyone" or "adminonly". + quota_per_project_enable: + $ref: '#/definitions/BoolConfigItem' + description: This attribute indicates whether quota per project enabled in harbor read_only: $ref: '#/definitions/BoolConfigItem' description: '''docker push'' is prohibited by Harbor if you set it to true. ' @@ -5349,7 +5379,9 @@ definitions: properties: type: type: string - description: The schedule type. The valid values are hourly, daily, weekly, custom and None. 'None' means to cancel the schedule. + description: | + The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manually' and 'None'. + 'Manually' means to trigger it right away and 'None' means to cancel the schedule. cron: type: string description: A cron expression, a time-based job scheduler. @@ -5724,7 +5756,7 @@ definitions: description: The webhook job ID. policy_id: type: integer - fromat: int64 + format: int64 description: The webhook policy ID. event_type: type: string diff --git a/make/harbor.yml b/make/harbor.yml index 347ef0c8c..f2bc1c8c4 100644 --- a/make/harbor.yml +++ b/make/harbor.yml @@ -30,6 +30,11 @@ harbor_admin_password: Harbor12345 database: # The password for the root user of Harbor DB. Change this before any production use. password: root123 + # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained. + max_idle_conns: 50 + # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections. + # Note: the default number of connections is 100 for postgres. + max_open_conns: 100 # The default data volume data_volume: /data @@ -50,18 +55,12 @@ data_volume: /data # disabled: false # Clair configuration -clair: +clair: # The interval of clair updaters, the unit is hour, set to 0 to disable the updaters. updaters_interval: 12 - # Config http proxy for Clair, e.g. http://my.proxy.com:3128 - # Clair doesn't need to connect to harbor internal components via http proxy. - http_proxy: - https_proxy: - no_proxy: 127.0.0.1,localhost,core,registry - jobservice: - # Maximum number of job workers in job service + # Maximum number of job workers in job service max_job_workers: 10 notification: @@ -80,8 +79,8 @@ log: local: # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. rotate_count: 50 - # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. - # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G + # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. + # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G # are all valid. rotate_size: 200M # The directory on your host that store log @@ -97,7 +96,7 @@ log: # port: 5140 #This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! -_version: 1.8.0 +_version: 1.9.0 # Uncomment external_database if using external database. # external_database: @@ -143,3 +142,20 @@ _version: 1.8.0 # Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert. # uaa: # ca_file: /path/to/ca + +# Global proxy +# Config http proxy for components, e.g. http://my.proxy.com:3128 +# Components doesn't need to connect to each others via http proxy. +# Remove component from `components` array if want disable proxy +# for it. If you want use proxy for replication, MUST enable proxy +# for core and jobservice, and set `http_proxy` and `https_proxy`. +# Add domain to the `no_proxy` field, when you want disable proxy +# for some special registry. +proxy: + http_proxy: + https_proxy: + no_proxy: 127.0.0.1,localhost,.local,.internal,log,db,redis,nginx,core,portal,postgresql,jobservice,registry,registryctl,clair + components: + - core + - jobservice + - clair diff --git a/make/install.sh b/make/install.sh index f8acf0f2d..9fa16038b 100755 --- a/make/install.sh +++ b/make/install.sh @@ -117,7 +117,7 @@ function check_docker { function check_dockercompose { if ! docker-compose --version &> /dev/null then - error "Need to install docker-compose(1.18.0+) by yourself first and run this script again." + error "Need to install docker-compose(1.23.0+) by yourself first and run this script again." exit 1 fi @@ -129,9 +129,9 @@ function check_dockercompose { docker_compose_version_part2=${BASH_REMATCH[3]} # the version of docker-compose does not meet the requirement - if [ "$docker_compose_version_part1" -lt 1 ] || ([ "$docker_compose_version_part1" -eq 1 ] && [ "$docker_compose_version_part2" -lt 18 ]) + if [ "$docker_compose_version_part1" -lt 1 ] || ([ "$docker_compose_version_part1" -eq 1 ] && [ "$docker_compose_version_part2" -lt 23 ]) then - error "Need to upgrade docker-compose package to 1.18.0+." + error "Need to upgrade docker-compose package to 1.23.0+." exit 1 else note "docker-compose version: $docker_compose_version" diff --git a/make/migrations/postgresql/0001_initial_schema.up.sql b/make/migrations/postgresql/0001_initial_schema.up.sql index bccd7f4cb..e3f2bb903 100644 --- a/make/migrations/postgresql/0001_initial_schema.up.sql +++ b/make/migrations/postgresql/0001_initial_schema.up.sql @@ -56,9 +56,9 @@ $$; CREATE TRIGGER harbor_user_update_time_at_modtime BEFORE UPDATE ON harbor_user FOR EACH ROW EXECUTE PROCEDURE update_update_time_at_column(); -insert into harbor_user (username, email, password, realname, comment, deleted, sysadmin_flag, creation_time, update_time) values -('admin', 'admin@example.com', '', 'system admin', 'admin user',false, true, NOW(), NOW()), -('anonymous', 'anonymous@example.com', '', 'anonymous user', 'anonymous user', true, false, NOW(), NOW()); +insert into harbor_user (username, password, realname, comment, deleted, sysadmin_flag, creation_time, update_time) values +('admin', '', 'system admin', 'admin user',false, true, NOW(), NOW()), +('anonymous', '', 'anonymous user', 'anonymous user', true, false, NOW(), NOW()); create table project ( project_id SERIAL PRIMARY KEY NOT NULL, diff --git a/make/migrations/postgresql/0010_1.9.0_schema.up.sql b/make/migrations/postgresql/0010_1.9.0_schema.up.sql index 7fd3a4241..80725fbe4 100644 --- a/make/migrations/postgresql/0010_1.9.0_schema.up.sql +++ b/make/migrations/postgresql/0010_1.9.0_schema.up.sql @@ -86,6 +86,7 @@ CREATE TABLE quota_usage UNIQUE (reference, reference_id) ); +/* only set quota and usage for 'library', and let the sync quota handling others. */ INSERT INTO quota (reference, reference_id, hard, creation_time, update_time) SELECT 'project', CAST(project_id AS VARCHAR), @@ -93,7 +94,7 @@ SELECT 'project', NOW(), NOW() FROM project -WHERE deleted = 'f'; +WHERE name = 'library' and deleted = 'f'; INSERT INTO quota_usage (id, reference, reference_id, used, creation_time, update_time) SELECT id, @@ -131,6 +132,8 @@ create table retention_task repository varchar(255), job_id varchar(64), status varchar(32), + status_code integer, + status_revision integer, start_time timestamp default CURRENT_TIMESTAMP, end_time timestamp default CURRENT_TIMESTAMP, total integer, diff --git a/make/photon/core/Dockerfile b/make/photon/core/Dockerfile index 7eaa4191c..d585a98ee 100644 --- a/make/photon/core/Dockerfile +++ b/make/photon/core/Dockerfile @@ -1,6 +1,6 @@ FROM photon:2.0 -RUN tdnf install sudo -y >> /dev/null\ +RUN tdnf install sudo tzdata -y >> /dev/null \ && tdnf clean all \ && groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor \ && mkdir /harbor/ diff --git a/make/photon/jobservice/Dockerfile b/make/photon/jobservice/Dockerfile index eddb8e65b..1ee9277dd 100644 --- a/make/photon/jobservice/Dockerfile +++ b/make/photon/jobservice/Dockerfile @@ -1,6 +1,6 @@ FROM photon:2.0 -RUN tdnf install sudo -y >> /dev/null\ +RUN tdnf install sudo tzdata -y >> /dev/null \ && tdnf clean all \ && groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor diff --git a/make/photon/portal/Dockerfile b/make/photon/portal/Dockerfile index f6adb9bf1..9f71410f7 100644 --- a/make/photon/portal/Dockerfile +++ b/make/photon/portal/Dockerfile @@ -1,7 +1,8 @@ FROM node:10.15.0 as nodeportal COPY src/portal /portal_src -COPY ./docs/swagger.yaml /portal_src +COPY ./docs/swagger.yaml /portal_src +COPY ./LICENSE /portal_src WORKDIR /build_dir @@ -21,6 +22,7 @@ FROM photon:2.0 COPY --from=nodeportal /build_dir/dist /usr/share/nginx/html COPY --from=nodeportal /build_dir/swagger.yaml /usr/share/nginx/html COPY --from=nodeportal /build_dir/swagger.json /usr/share/nginx/html +COPY --from=nodeportal /build_dir/LICENSE /usr/share/nginx/html COPY make/photon/portal/nginx.conf /etc/nginx/nginx.conf diff --git a/make/photon/prepare/g.py b/make/photon/prepare/g.py index f0eab0675..229f61a54 100644 --- a/make/photon/prepare/g.py +++ b/make/photon/prepare/g.py @@ -12,11 +12,12 @@ REDIS_UID = 999 REDIS_GID = 999 ## Global variable +host_root_dir = '/hostfs' + base_dir = '/harbor_make' templates_dir = "/usr/src/app/templates" config_dir = '/config' data_dir = '/data' - secret_dir = '/secret' secret_key_dir='/secret/keys' diff --git a/make/photon/prepare/templates/clair/clair_env.jinja b/make/photon/prepare/templates/clair/clair_env.jinja index 038f1a130..3825ca8fb 100644 --- a/make/photon/prepare/templates/clair/clair_env.jinja +++ b/make/photon/prepare/templates/clair/clair_env.jinja @@ -1,3 +1,3 @@ -http_proxy={{clair_http_proxy}} -https_proxy={{clair_https_proxy}} -no_proxy={{clair_no_proxy}} +HTTP_PROXY={{clair_http_proxy}} +HTTPS_PROXY={{clair_https_proxy}} +NO_PROXY={{clair_no_proxy}} diff --git a/make/photon/prepare/templates/core/env.jinja b/make/photon/prepare/templates/core/env.jinja index bc29a505d..d6413678e 100644 --- a/make/photon/prepare/templates/core/env.jinja +++ b/make/photon/prepare/templates/core/env.jinja @@ -15,6 +15,8 @@ POSTGRESQL_USERNAME={{harbor_db_username}} POSTGRESQL_PASSWORD={{harbor_db_password}} POSTGRESQL_DATABASE={{harbor_db_name}} POSTGRESQL_SSLMODE={{harbor_db_sslmode}} +POSTGRESQL_MAX_IDLE_CONNS={{harbor_db_max_idle_conns}} +POSTGRESQL_MAX_OPEN_CONNS={{harbor_db_max_open_conns}} REGISTRY_URL={{registry_url}} TOKEN_SERVICE_URL={{token_service_url}} HARBOR_ADMIN_PASSWORD={{harbor_admin_password}} @@ -41,3 +43,7 @@ RELOAD_KEY={{reload_key}} CHART_REPOSITORY_URL={{chart_repository_url}} REGISTRY_CONTROLLER_URL={{registry_controller_url}} WITH_CHARTMUSEUM={{with_chartmuseum}} + +HTTP_PROXY={{core_http_proxy}} +HTTPS_PROXY={{core_https_proxy}} +NO_PROXY={{core_no_proxy}} diff --git a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja index 9e70cc8de..cb6785766 100644 --- a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja +++ b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja @@ -276,12 +276,7 @@ services: volumes: - ./common/config/nginx:/etc/nginx:z {% if protocol == 'https' %} - - type: bind - source: {{cert_key_path}} - target: /etc/cert/server.key - - type: bind - source: {{cert_path}} - target: /etc/cert/server.crt + - {{data_volume}}/secret/cert:/etc/cert:z {% endif %} networks: - harbor diff --git a/make/photon/prepare/templates/jobservice/env.jinja b/make/photon/prepare/templates/jobservice/env.jinja index d9e32c521..c38534f02 100644 --- a/make/photon/prepare/templates/jobservice/env.jinja +++ b/make/photon/prepare/templates/jobservice/env.jinja @@ -2,3 +2,7 @@ CORE_SECRET={{core_secret}} JOBSERVICE_SECRET={{jobservice_secret}} CORE_URL={{core_url}} JOBSERVICE_WEBHOOK_JOB_MAX_RETRY={{notification_webhook_job_max_retry}} + +HTTP_PROXY={{jobservice_http_proxy}} +HTTPS_PROXY={{jobservice_https_proxy}} +NO_PROXY={{jobservice_no_proxy}} diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index c57856845..df14a53de 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -112,6 +112,11 @@ def parse_yaml_config(config_file_path): config_dict['harbor_db_username'] = 'postgres' config_dict['harbor_db_password'] = db_configs.get("password") or '' config_dict['harbor_db_sslmode'] = 'disable' + + default_max_idle_conns = 2 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns + default_max_open_conns = 0 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxOpenConns + config_dict['harbor_db_max_idle_conns'] = db_configs.get("max_idle_conns") or default_max_idle_conns + config_dict['harbor_db_max_open_conns'] = db_configs.get("max_open_conns") or default_max_open_conns # clari db config_dict['clair_db_host'] = 'postgresql' config_dict['clair_db_port'] = 5432 @@ -171,13 +176,18 @@ def parse_yaml_config(config_file_path): if storage_config.get('redirect'): config_dict['storage_redirect_disabled'] = storage_config['redirect']['disabled'] + # Global proxy configs + proxy_config = configs.get('proxy') or {} + proxy_components = proxy_config.get('components') or [] + for proxy_component in proxy_components: + config_dict[proxy_component + '_http_proxy'] = proxy_config.get('http_proxy') or '' + config_dict[proxy_component + '_https_proxy'] = proxy_config.get('https_proxy') or '' + config_dict[proxy_component + '_no_proxy'] = proxy_config.get('no_proxy') or '127.0.0.1,localhost,core,registry' + # Clair configs, optional clair_configs = configs.get("clair") or {} config_dict['clair_db'] = 'postgres' config_dict['clair_updaters_interval'] = clair_configs.get("updaters_interval") or 12 - config_dict['clair_http_proxy'] = clair_configs.get('http_proxy') or '' - config_dict['clair_https_proxy'] = clair_configs.get('https_proxy') or '' - config_dict['clair_no_proxy'] = clair_configs.get('no_proxy') or '127.0.0.1,localhost,core,registry' # Chart configs chart_configs = configs.get("chart") or {} @@ -286,4 +296,4 @@ def parse_yaml_config(config_file_path): # UAA configs config_dict['uaa'] = configs.get('uaa') or {} - return config_dict \ No newline at end of file + return config_dict diff --git a/make/photon/prepare/utils/docker_compose.py b/make/photon/prepare/utils/docker_compose.py index 6f46a951a..648d6b979 100644 --- a/make/photon/prepare/utils/docker_compose.py +++ b/make/photon/prepare/utils/docker_compose.py @@ -13,7 +13,7 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum): VERSION_TAG = versions.get('VERSION_TAG') or 'dev' REGISTRY_VERSION = versions.get('REGISTRY_VERSION') or 'v2.7.1' NOTARY_VERSION = versions.get('NOTARY_VERSION') or 'v0.6.1' - CLAIR_VERSION = versions.get('CLAIR_VERSION') or 'v2.0.7' + CLAIR_VERSION = versions.get('CLAIR_VERSION') or 'v2.0.9' CHARTMUSEUM_VERSION = versions.get('CHARTMUSEUM_VERSION') or 'v0.9.0' rendering_variables = { diff --git a/make/photon/prepare/utils/nginx.py b/make/photon/prepare/utils/nginx.py index 74fc3deab..0d1117448 100644 --- a/make/photon/prepare/utils/nginx.py +++ b/make/photon/prepare/utils/nginx.py @@ -2,11 +2,13 @@ import os, shutil from fnmatch import fnmatch from pathlib import Path -from g import config_dir, templates_dir, DEFAULT_GID, DEFAULT_UID +from g import config_dir, templates_dir, host_root_dir, DEFAULT_GID, DEFAULT_UID, data_dir from utils.misc import prepare_dir, mark_file from utils.jinja import render_jinja from utils.cert import SSL_CERT_KEY_PATH, SSL_CERT_PATH +host_ngx_real_cert_dir = Path(os.path.join(data_dir, 'secret', 'cert')) + nginx_conf = os.path.join(config_dir, "nginx", "nginx.conf") nginx_confd_dir = os.path.join(config_dir, "nginx", "conf.d") nginx_https_conf_template = os.path.join(templates_dir, "nginx", "nginx.https.conf.jinja") @@ -20,8 +22,38 @@ def prepare_nginx(config_dict): prepare_dir(nginx_confd_dir, uid=DEFAULT_UID, gid=DEFAULT_GID) render_nginx_template(config_dict) + +def prepare_nginx_certs(cert_key_path, cert_path): + """ + Prepare the certs file with proper ownership + 1. Remove nginx cert files in secret dir + 2. Copy cert files on host filesystem to secret dir + 3. Change the permission to 644 and ownership to 10000:10000 + """ + host_ngx_cert_key_path = Path(os.path.join(host_root_dir, cert_key_path.lstrip('/'))) + host_ngx_cert_path = Path(os.path.join(host_root_dir, cert_path.lstrip('/'))) + + if host_ngx_real_cert_dir.exists() and host_ngx_real_cert_dir.is_dir(): + shutil.rmtree(host_ngx_real_cert_dir) + + os.makedirs(host_ngx_real_cert_dir, mode=0o755) + real_key_path = os.path.join(host_ngx_real_cert_dir, 'server.key') + real_crt_path = os.path.join(host_ngx_real_cert_dir, 'server.crt') + shutil.copy2(host_ngx_cert_key_path, real_key_path) + shutil.copy2(host_ngx_cert_path, real_crt_path) + + os.chown(host_ngx_real_cert_dir, uid=DEFAULT_UID, gid=DEFAULT_GID) + mark_file(real_key_path, uid=DEFAULT_UID, gid=DEFAULT_GID) + mark_file(real_crt_path, uid=DEFAULT_UID, gid=DEFAULT_GID) + + def render_nginx_template(config_dict): - if config_dict['protocol'] == "https": + """ + 1. render nginx config file through protocol + 2. copy additional configs to cert.d dir + """ + if config_dict['protocol'] == 'https': + prepare_nginx_certs(config_dict['cert_key_path'], config_dict['cert_path']) render_jinja( nginx_https_conf_template, nginx_conf, @@ -30,12 +62,7 @@ def render_nginx_template(config_dict): ssl_cert=SSL_CERT_PATH, ssl_cert_key=SSL_CERT_KEY_PATH) location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTPS - cert_dir = Path(os.path.join(config_dir, 'cert')) - ssl_key_path = Path(os.path.join(cert_dir, 'server.key')) - ssl_crt_path = Path(os.path.join(cert_dir, 'server.crt')) - cert_dir.mkdir(parents=True, exist_ok=True) - ssl_key_path.touch() - ssl_crt_path.touch() + else: render_jinja( nginx_http_conf_template, @@ -45,22 +72,23 @@ def render_nginx_template(config_dict): location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTP copy_nginx_location_configs_if_exist(nginx_template_ext_dir, nginx_confd_dir, location_file_pattern) -def add_additional_location_config(src, dst): - """ - These conf files is used for user that wanna add additional customized locations to harbor proxy - :params src: source of the file - :params dst: destination file path - """ - if not os.path.isfile(src): - return - print("Copying nginx configuration file {src} to {dst}".format( - src=src, dst=dst)) - shutil.copy2(src, dst) - mark_file(dst, mode=0o644) def copy_nginx_location_configs_if_exist(src_config_dir, dst_config_dir, filename_pattern): if not os.path.exists(src_config_dir): return + + def add_additional_location_config(src, dst): + """ + These conf files is used for user that wanna add additional customized locations to harbor proxy + :params src: source of the file + :params dst: destination file path + """ + if not os.path.isfile(src): + return + print("Copying nginx configuration file {src} to {dst}".format(src=src, dst=dst)) + shutil.copy2(src, dst) + mark_file(dst, mode=0o644) + map(lambda filename: add_additional_location_config( os.path.join(src_config_dir, filename), os.path.join(dst_config_dir, filename)), diff --git a/make/photon/prepare/utils/registry.py b/make/photon/prepare/utils/registry.py index bd42f183d..2a3512d9b 100644 --- a/make/photon/prepare/utils/registry.py +++ b/make/photon/prepare/utils/registry.py @@ -9,6 +9,13 @@ registry_config_dir = os.path.join(config_dir, "registry") registry_config_template_path = os.path.join(templates_dir, "registry", "config.yml.jinja") registry_conf = os.path.join(config_dir, "registry", "config.yml") +levels_map = { + 'debug': 'debug', + 'info': 'info', + 'warning': 'warn', + 'error': 'error', + 'fatal': 'fatal' +} def prepare_registry(config_dict): prepare_dir(registry_config_dir) @@ -22,6 +29,7 @@ def prepare_registry(config_dict): registry_conf, uid=DEFAULT_UID, gid=DEFAULT_GID, + level=levels_map[config_dict['log_level']], storage_provider_info=storage_provider_info, **config_dict) diff --git a/make/prepare b/make/prepare index 28d570c92..c628f46a3 100755 --- a/make/prepare +++ b/make/prepare @@ -1,8 +1,8 @@ #!/bin/bash set +e -# If compling source code this dir is harbor's make dir -# If install harbor via pacakge, this dir is harbor's root dir +# If compiling source code this dir is harbor's make dir. +# If installing harbor via pacakge, this dir is harbor's root dir. if [[ -n "$HARBOR_BUNDLE_DIR" ]]; then harbor_prepare_path=$HARBOR_BUNDLE_DIR else @@ -50,6 +50,7 @@ docker run --rm -v $input_dir:/input:z \ -v $harbor_prepare_path:/compose_location:z \ -v $config_dir:/config:z \ -v $secret_dir:/secret:z \ + -v /:/hostfs:z \ goharbor/prepare:dev $@ echo "Clean up the input dir" diff --git a/src/common/config/manager.go b/src/common/config/manager.go index 0df6eaa47..3886f160f 100644 --- a/src/common/config/manager.go +++ b/src/common/config/manager.go @@ -210,12 +210,14 @@ func (c *CfgManager) GetDatabaseCfg() *models.Database { return &models.Database{ Type: c.Get(common.DatabaseType).GetString(), PostGreSQL: &models.PostGreSQL{ - Host: c.Get(common.PostGreSQLHOST).GetString(), - Port: c.Get(common.PostGreSQLPort).GetInt(), - Username: c.Get(common.PostGreSQLUsername).GetString(), - Password: c.Get(common.PostGreSQLPassword).GetString(), - Database: c.Get(common.PostGreSQLDatabase).GetString(), - SSLMode: c.Get(common.PostGreSQLSSLMode).GetString(), + Host: c.Get(common.PostGreSQLHOST).GetString(), + Port: c.Get(common.PostGreSQLPort).GetInt(), + Username: c.Get(common.PostGreSQLUsername).GetString(), + Password: c.Get(common.PostGreSQLPassword).GetString(), + Database: c.Get(common.PostGreSQLDatabase).GetString(), + SSLMode: c.Get(common.PostGreSQLSSLMode).GetString(), + MaxIdleConns: c.Get(common.PostGreSQLMaxIdleConns).GetInt(), + MaxOpenConns: c.Get(common.PostGreSQLMaxOpenConns).GetInt(), }, } } diff --git a/src/common/config/metadata/metadatalist.go b/src/common/config/metadata/metadatalist.go index 3aa42f619..7106a38c6 100644 --- a/src/common/config/metadata/metadatalist.go +++ b/src/common/config/metadata/metadatalist.go @@ -116,6 +116,8 @@ var ( {Name: common.PostGreSQLPort, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_PORT", DefaultValue: "5432", ItemType: &PortType{}, Editable: false}, {Name: common.PostGreSQLSSLMode, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_SSLMODE", DefaultValue: "disable", ItemType: &StringType{}, Editable: false}, {Name: common.PostGreSQLUsername, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_USERNAME", DefaultValue: "postgres", ItemType: &StringType{}, Editable: false}, + {Name: common.PostGreSQLMaxIdleConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_IDLE_CONNS", DefaultValue: "2", ItemType: &IntType{}, Editable: false}, + {Name: common.PostGreSQLMaxOpenConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_OPEN_CONNS", DefaultValue: "0", ItemType: &IntType{}, Editable: false}, {Name: common.ProjectCreationRestriction, Scope: UserScope, Group: BasicGroup, EnvKey: "PROJECT_CREATION_RESTRICTION", DefaultValue: common.ProCrtRestrEveryone, ItemType: &ProjectCreationRestrictionType{}, Editable: false}, {Name: common.ReadOnly, Scope: UserScope, Group: BasicGroup, EnvKey: "READ_ONLY", DefaultValue: "false", ItemType: &BoolType{}, Editable: false}, @@ -151,6 +153,7 @@ var ( {Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "43200", ItemType: &IntType{}, Editable: true}, {Name: common.NotificationEnable, Scope: UserScope, Group: BasicGroup, EnvKey: "NOTIFICATION_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true}, + {Name: common.QuotaPerProjectEnable, Scope: UserScope, Group: QuotaGroup, EnvKey: "QUOTA_PER_PROJECT_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true}, {Name: common.CountPerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "COUNT_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true}, {Name: common.StoragePerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "STORAGE_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true}, } diff --git a/src/common/const.go b/src/common/const.go index b7582439e..d6722ce07 100755 --- a/src/common/const.go +++ b/src/common/const.go @@ -53,6 +53,8 @@ const ( PostGreSQLPassword = "postgresql_password" PostGreSQLDatabase = "postgresql_database" PostGreSQLSSLMode = "postgresql_sslmode" + PostGreSQLMaxIdleConns = "postgresql_max_idle_conns" + PostGreSQLMaxOpenConns = "postgresql_max_open_conns" SelfRegistration = "self_registration" CoreURL = "core_url" CoreLocalURL = "core_local_url" @@ -147,7 +149,9 @@ const ( // Global notification enable configuration NotificationEnable = "notification_enable" + // Quota setting items for project - CountPerProject = "count_per_project" - StoragePerProject = "storage_per_project" + QuotaPerProjectEnable = "quota_per_project_enable" + CountPerProject = "count_per_project" + StoragePerProject = "storage_per_project" ) diff --git a/src/common/dao/artifact.go b/src/common/dao/artifact.go index bac77d74b..34663b5cd 100644 --- a/src/common/dao/artifact.go +++ b/src/common/dao/artifact.go @@ -58,6 +58,7 @@ func UpdateArtifactPullTime(af *models.Artifact) error { // DeleteArtifact ... func DeleteArtifact(id int64) error { + _, err := GetOrmer().QueryTable(&models.Artifact{}).Filter("ID", id).Delete() return err } diff --git a/src/common/dao/base.go b/src/common/dao/base.go index 253b02692..43ded29ef 100644 --- a/src/common/dao/base.go +++ b/src/common/dao/base.go @@ -121,12 +121,16 @@ func getDatabase(database *models.Database) (db Database, err error) { switch database.Type { case "", "postgresql": - db = NewPGSQL(database.PostGreSQL.Host, + db = NewPGSQL( + database.PostGreSQL.Host, strconv.Itoa(database.PostGreSQL.Port), database.PostGreSQL.Username, database.PostGreSQL.Password, database.PostGreSQL.Database, - database.PostGreSQL.SSLMode) + database.PostGreSQL.SSLMode, + database.PostGreSQL.MaxIdleConns, + database.PostGreSQL.MaxOpenConns, + ) default: err = fmt.Errorf("invalid database: %s", database.Type) } @@ -139,6 +143,8 @@ var once sync.Once // GetOrmer :set ormer singleton func GetOrmer() orm.Ormer { once.Do(func() { + // override the default value(1000) to return all records when setting no limit + orm.DefaultRowsLimit = -1 globalOrm = orm.NewOrm() }) return globalOrm diff --git a/src/common/dao/blob.go b/src/common/dao/blob.go index b8cbd4065..66e402567 100644 --- a/src/common/dao/blob.go +++ b/src/common/dao/blob.go @@ -78,10 +78,15 @@ func GetBlobsByArtifact(artifactDigest string) ([]*models.Blob, error) { // GetExclusiveBlobs returns layers of repository:tag which are not shared with other repositories in the project func GetExclusiveBlobs(projectID int64, repository, digest string) ([]*models.Blob, error) { + var exclusive []*models.Blob + blobs, err := GetBlobsByArtifact(digest) if err != nil { return nil, err } + if len(blobs) == 0 { + return exclusive, nil + } sql := fmt.Sprintf(` SELECT @@ -103,13 +108,11 @@ FROM ) ) AS a LEFT JOIN artifact_blob b ON a.digest = b.digest_af - AND b.digest_blob IN (%s)`, paramPlaceholder(len(blobs)-1)) + AND b.digest_blob IN (%s)`, ParamPlaceholderForIn(len(blobs))) params := []interface{}{projectID, repository, projectID, digest} for _, blob := range blobs { - if blob.Digest != digest { - params = append(params, blob.Digest) - } + params = append(params, blob.Digest) } var rows []struct { @@ -125,9 +128,8 @@ FROM shared[row.Digest] = true } - var exclusive []*models.Blob for _, blob := range blobs { - if blob.Digest != digest && !shared[blob.Digest] { + if !shared[blob.Digest] { exclusive = append(exclusive, blob) } } diff --git a/src/common/dao/blob_test.go b/src/common/dao/blob_test.go index 26dc5e492..9b7e8c077 100644 --- a/src/common/dao/blob_test.go +++ b/src/common/dao/blob_test.go @@ -133,30 +133,32 @@ func (suite *GetExclusiveBlobsSuite) mustPrepareImage(projectID int64, projectNa func (suite *GetExclusiveBlobsSuite) TestInSameRepository() { withProject(func(projectID int64, projectName string) { + digest1 := digest.FromString(utils.GenerateRandomString()).String() digest2 := digest.FromString(utils.GenerateRandomString()).String() digest3 := digest.FromString(utils.GenerateRandomString()).String() manifest1 := suite.mustPrepareImage(projectID, projectName, "mysql", "latest", digest1, digest2) if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest1); suite.Nil(err) { - suite.Len(blobs, 2) + suite.Len(blobs, 3) } manifest2 := suite.mustPrepareImage(projectID, projectName, "mysql", "8.0", digest1, digest2) if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest2); suite.Nil(err) { - suite.Len(blobs, 2) + suite.Len(blobs, 3) } manifest3 := suite.mustPrepareImage(projectID, projectName, "mysql", "dev", digest1, digest2, digest3) if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest1); suite.Nil(err) { - suite.Len(blobs, 0) + suite.Len(blobs, 1) + suite.Equal(manifest1, blobs[0].Digest) } if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest2); suite.Nil(err) { - suite.Len(blobs, 0) + suite.Len(blobs, 1) + suite.Equal(manifest2, blobs[0].Digest) } if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest3); suite.Nil(err) { - suite.Len(blobs, 1) - suite.Equal(digest3, blobs[0].Digest) + suite.Len(blobs, 2) } }) } @@ -169,7 +171,7 @@ func (suite *GetExclusiveBlobsSuite) TestInDifferentRepositories() { manifest1 := suite.mustPrepareImage(projectID, projectName, "mysql", "latest", digest1, digest2) if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest1); suite.Nil(err) { - suite.Len(blobs, 2) + suite.Len(blobs, 3) } manifest2 := suite.mustPrepareImage(projectID, projectName, "mariadb", "latest", digest1, digest2) @@ -188,8 +190,7 @@ func (suite *GetExclusiveBlobsSuite) TestInDifferentRepositories() { suite.Len(blobs, 0) } if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest3); suite.Nil(err) { - suite.Len(blobs, 1) - suite.Equal(digest3, blobs[0].Digest) + suite.Len(blobs, 2) } }) } @@ -201,16 +202,16 @@ func (suite *GetExclusiveBlobsSuite) TestInDifferentProjects() { manifest1 := suite.mustPrepareImage(projectID, projectName, "mysql", "latest", digest1, digest2) if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest1); suite.Nil(err) { - suite.Len(blobs, 2) + suite.Len(blobs, 3) } withProject(func(id int64, name string) { manifest2 := suite.mustPrepareImage(id, name, "mysql", "latest", digest1, digest2) if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest1); suite.Nil(err) { - suite.Len(blobs, 2) + suite.Len(blobs, 3) } if blobs, err := GetExclusiveBlobs(id, name+"/mysql", manifest2); suite.Nil(err) { - suite.Len(blobs, 2) + suite.Len(blobs, 3) } }) diff --git a/src/common/dao/cve_whitelist.go b/src/common/dao/cve_whitelist.go index 25c1c9b98..645a1c076 100644 --- a/src/common/dao/cve_whitelist.go +++ b/src/common/dao/cve_whitelist.go @@ -21,6 +21,14 @@ import ( "github.com/goharbor/harbor/src/common/utils/log" ) +// CreateCVEWhitelist creates the CVE whitelist +func CreateCVEWhitelist(l models.CVEWhitelist) (int64, error) { + o := GetOrmer() + itemsBytes, _ := json.Marshal(l.Items) + l.ItemsText = string(itemsBytes) + return o.Insert(&l) +} + // UpdateCVEWhitelist Updates the vulnerability white list to DB func UpdateCVEWhitelist(l models.CVEWhitelist) (int64, error) { o := GetOrmer() @@ -30,23 +38,6 @@ func UpdateCVEWhitelist(l models.CVEWhitelist) (int64, error) { return id, err } -// GetSysCVEWhitelist Gets the system level vulnerability white list from DB -func GetSysCVEWhitelist() (*models.CVEWhitelist, error) { - return GetCVEWhitelist(0) -} - -// UpdateSysCVEWhitelist updates the system level CVE whitelist -/* -func UpdateSysCVEWhitelist(l models.CVEWhitelist) error { - if l.ProjectID != 0 { - return fmt.Errorf("system level CVE whitelist cannot set project ID") - } - l.ProjectID = -1 - _, err := UpdateCVEWhitelist(l) - return err -} -*/ - // GetCVEWhitelist Gets the CVE whitelist of the project based on the project ID in parameter func GetCVEWhitelist(pid int64) (*models.CVEWhitelist, error) { o := GetOrmer() @@ -58,8 +49,7 @@ func GetCVEWhitelist(pid int64) (*models.CVEWhitelist, error) { return nil, fmt.Errorf("failed to get CVE whitelist for project %d, error: %v", pid, err) } if len(r) == 0 { - log.Infof("No CVE whitelist found for project %d, returning empty list.", pid) - return &models.CVEWhitelist{ProjectID: pid, Items: []models.CVEWhitelistItem{}}, nil + return nil, nil } else if len(r) > 1 { log.Infof("Multiple CVE whitelists found for project %d, length: %d, returning first element.", pid, len(r)) } diff --git a/src/common/dao/cve_whitelist_test.go b/src/common/dao/cve_whitelist_test.go index 35af2f294..099409de5 100644 --- a/src/common/dao/cve_whitelist_test.go +++ b/src/common/dao/cve_whitelist_test.go @@ -23,12 +23,9 @@ import ( func TestUpdateAndGetCVEWhitelist(t *testing.T) { require.Nil(t, ClearTable("cve_whitelist")) - l, err := GetSysCVEWhitelist() - assert.Nil(t, err) - assert.Equal(t, models.CVEWhitelist{ProjectID: 0, Items: []models.CVEWhitelistItem{}}, *l) l2, err := GetCVEWhitelist(5) assert.Nil(t, err) - assert.Equal(t, models.CVEWhitelist{ProjectID: 5, Items: []models.CVEWhitelistItem{}}, *l2) + assert.Nil(t, l2) longList := []models.CVEWhitelistItem{} for i := 0; i < 50; i++ { @@ -46,15 +43,6 @@ func TestUpdateAndGetCVEWhitelist(t *testing.T) { assert.Equal(t, longList, out1.Items) assert.Equal(t, e, *out1.ExpiresAt) - in2 := models.CVEWhitelist{ProjectID: 3, Items: []models.CVEWhitelistItem{}} - _, err = UpdateCVEWhitelist(in2) - require.Nil(t, err) - // assert.Equal(t, int64(1), n2) - out2, err := GetCVEWhitelist(3) - require.Nil(t, err) - assert.Equal(t, int64(3), out2.ProjectID) - assert.Equal(t, []models.CVEWhitelistItem{}, out2.Items) - sysCVEs := []models.CVEWhitelistItem{ {CVEID: "CVE-2019-10164"}, {CVEID: "CVE-2017-12345"}, @@ -62,11 +50,6 @@ func TestUpdateAndGetCVEWhitelist(t *testing.T) { in3 := models.CVEWhitelist{Items: sysCVEs} _, err = UpdateCVEWhitelist(in3) require.Nil(t, err) - // assert.Equal(t, int64(1), n3) - sysList, err := GetSysCVEWhitelist() - require.Nil(t, err) - assert.Equal(t, int64(0), sysList.ProjectID) - assert.Equal(t, sysCVEs, sysList.Items) - // require.Nil(t, ClearTable("cve_whitelist")) + require.Nil(t, ClearTable("cve_whitelist")) } diff --git a/src/common/dao/pgsql.go b/src/common/dao/pgsql.go index e1b3da6cb..bf98c6b08 100644 --- a/src/common/dao/pgsql.go +++ b/src/common/dao/pgsql.go @@ -31,12 +31,14 @@ import ( const defaultMigrationPath = "migrations/postgresql/" type pgsql struct { - host string - port string - usr string - pwd string - database string - sslmode string + host string + port string + usr string + pwd string + database string + sslmode string + maxIdleConns int + maxOpenConns int } // Name returns the name of PostgreSQL @@ -51,17 +53,19 @@ func (p *pgsql) String() string { } // NewPGSQL returns an instance of postgres -func NewPGSQL(host string, port string, usr string, pwd string, database string, sslmode string) Database { +func NewPGSQL(host string, port string, usr string, pwd string, database string, sslmode string, maxIdleConns int, maxOpenConns int) Database { if len(sslmode) == 0 { sslmode = "disable" } return &pgsql{ - host: host, - port: port, - usr: usr, - pwd: pwd, - database: database, - sslmode: sslmode, + host: host, + port: port, + usr: usr, + pwd: pwd, + database: database, + sslmode: sslmode, + maxIdleConns: maxIdleConns, + maxOpenConns: maxOpenConns, } } @@ -82,7 +86,7 @@ func (p *pgsql) Register(alias ...string) error { info := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s", p.host, p.port, p.usr, p.pwd, p.database, p.sslmode) - return orm.RegisterDataBase(an, "postgres", info) + return orm.RegisterDataBase(an, "postgres", info, p.maxIdleConns, p.maxOpenConns) } // UpgradeSchema calls migrate tool to upgrade schema to the latest based on the SQL scripts. diff --git a/src/common/dao/pro_meta.go b/src/common/dao/pro_meta.go index d4a9c4e6f..a6593e2ef 100644 --- a/src/common/dao/pro_meta.go +++ b/src/common/dao/pro_meta.go @@ -44,7 +44,7 @@ func DeleteProjectMetadata(projectID int64, name ...string) error { params = append(params, projectID) if len(name) > 0 { - sql += fmt.Sprintf(` and name in ( %s )`, paramPlaceholder(len(name))) + sql += fmt.Sprintf(` and name in ( %s )`, ParamPlaceholderForIn(len(name))) params = append(params, name) } @@ -74,7 +74,7 @@ func GetProjectMetadata(projectID int64, name ...string) ([]*models.ProjectMetad params = append(params, projectID) if len(name) > 0 { - sql += fmt.Sprintf(` and name in ( %s )`, paramPlaceholder(len(name))) + sql += fmt.Sprintf(` and name in ( %s )`, ParamPlaceholderForIn(len(name))) params = append(params, name) } @@ -82,7 +82,9 @@ func GetProjectMetadata(projectID int64, name ...string) ([]*models.ProjectMetad return proMetas, err } -func paramPlaceholder(n int) string { +// ParamPlaceholderForIn returns a string that contains placeholders for sql keyword "in" +// e.g. n=3, returns "?,?,?" +func ParamPlaceholderForIn(n int) string { placeholders := []string{} for i := 0; i < n; i++ { placeholders = append(placeholders, "?") diff --git a/src/common/dao/project.go b/src/common/dao/project.go index b3066bcf1..e027ec221 100644 --- a/src/common/dao/project.go +++ b/src/common/dao/project.go @@ -167,9 +167,10 @@ func GetGroupProjects(groupIDs []int, query *models.ProjectQueryParam) ([]*model from project p left join project_member pm on p.project_id = pm.project_id left join user_group ug on ug.id = pm.entity_id and pm.entity_type = 'g' - where ug.id in ( %s ) order by name`, + where ug.id in ( %s )`, sql, groupIDCondition) } + sql = sql + ` order by name` sqlStr, queryParams := CreatePagination(query, sql, params) log.Debugf("query sql:%v", sql) var projects []*models.Project @@ -259,7 +260,7 @@ func projectQueryConditions(query *models.ProjectQueryParam) (string, []interfac } if len(query.ProjectIDs) > 0 { sql += fmt.Sprintf(` and p.project_id in ( %s )`, - paramPlaceholder(len(query.ProjectIDs))) + ParamPlaceholderForIn(len(query.ProjectIDs))) params = append(params, query.ProjectIDs) } return sql, params diff --git a/src/common/dao/project_blob.go b/src/common/dao/project_blob.go index 9111cdf9c..bb4b59a10 100644 --- a/src/common/dao/project_blob.go +++ b/src/common/dao/project_blob.go @@ -64,7 +64,7 @@ func RemoveBlobsFromProject(projectID int64, blobs ...*models.Blob) error { return nil } - sql := fmt.Sprintf(`DELETE FROM project_blob WHERE blob_id IN (%s)`, paramPlaceholder(len(blobIDs))) + sql := fmt.Sprintf(`DELETE FROM project_blob WHERE blob_id IN (%s)`, ParamPlaceholderForIn(len(blobIDs))) _, err := GetOrmer().Raw(sql, blobIDs).Exec() return err @@ -89,7 +89,7 @@ func GetBlobsNotInProject(projectID int64, blobDigests ...string) ([]*models.Blo } sql := fmt.Sprintf("SELECT * FROM blob WHERE id NOT IN (SELECT blob_id FROM project_blob WHERE project_id = ?) AND digest IN (%s)", - paramPlaceholder(len(blobDigests))) + ParamPlaceholderForIn(len(blobDigests))) params := []interface{}{projectID} for _, digest := range blobDigests { @@ -103,3 +103,34 @@ func GetBlobsNotInProject(projectID int64, blobDigests ...string) ([]*models.Blo return blobs, nil } + +// CountSizeOfProject ... +func CountSizeOfProject(pid int64) (int64, error) { + var blobs []models.Blob + + sql := ` +SELECT + DISTINCT bb.digest, + bb.id, + bb.content_type, + bb.size, + bb.creation_time +FROM artifact af +JOIN artifact_blob afnb + ON af.digest = afnb.digest_af +JOIN BLOB bb + ON afnb.digest_blob = bb.digest +WHERE af.project_id = ? +` + _, err := GetOrmer().Raw(sql, pid).QueryRows(&blobs) + if err != nil { + return 0, err + } + + var size int64 + for _, blob := range blobs { + size += blob.Size + } + + return size, err +} diff --git a/src/common/dao/project_blob_test.go b/src/common/dao/project_blob_test.go index 071bfdd3d..dec4c5fab 100644 --- a/src/common/dao/project_blob_test.go +++ b/src/common/dao/project_blob_test.go @@ -38,3 +38,161 @@ func TestHasBlobInProject(t *testing.T) { require.Nil(t, err) assert.True(t, has) } + +func TestCountSizeOfProject(t *testing.T) { + _, err := AddBlob(&models.Blob{ + Digest: "CountSizeOfProject_blob1", + Size: 101, + }) + require.Nil(t, err) + + _, err = AddBlob(&models.Blob{ + Digest: "CountSizeOfProject_blob2", + Size: 202, + }) + require.Nil(t, err) + + _, err = AddBlob(&models.Blob{ + Digest: "CountSizeOfProject_blob3", + Size: 303, + }) + require.Nil(t, err) + + pid1, err := AddProject(models.Project{ + Name: "CountSizeOfProject_project1", + OwnerID: 1, + }) + require.Nil(t, err) + + af := &models.Artifact{ + PID: pid1, + Repo: "hello-world", + Tag: "v1", + Digest: "CountSizeOfProject_af1", + Kind: "image", + } + + // add + _, err = AddArtifact(af) + require.Nil(t, err) + + afnb1 := &models.ArtifactAndBlob{ + DigestAF: "CountSizeOfProject_af1", + DigestBlob: "CountSizeOfProject_blob1", + } + afnb2 := &models.ArtifactAndBlob{ + DigestAF: "CountSizeOfProject_af1", + DigestBlob: "CountSizeOfProject_blob2", + } + afnb3 := &models.ArtifactAndBlob{ + DigestAF: "CountSizeOfProject_af1", + DigestBlob: "CountSizeOfProject_blob3", + } + + var afnbs []*models.ArtifactAndBlob + afnbs = append(afnbs, afnb1) + afnbs = append(afnbs, afnb2) + afnbs = append(afnbs, afnb3) + + // add + err = AddArtifactNBlobs(afnbs) + require.Nil(t, err) + + pSize, err := CountSizeOfProject(pid1) + assert.Equal(t, pSize, int64(606)) +} + +func TestCountSizeOfProjectDupdigest(t *testing.T) { + _, err := AddBlob(&models.Blob{ + Digest: "CountSizeOfProject_blob11", + Size: 101, + }) + require.Nil(t, err) + _, err = AddBlob(&models.Blob{ + Digest: "CountSizeOfProject_blob22", + Size: 202, + }) + require.Nil(t, err) + _, err = AddBlob(&models.Blob{ + Digest: "CountSizeOfProject_blob33", + Size: 303, + }) + require.Nil(t, err) + _, err = AddBlob(&models.Blob{ + Digest: "CountSizeOfProject_blob44", + Size: 404, + }) + require.Nil(t, err) + + pid1, err := AddProject(models.Project{ + Name: "CountSizeOfProject_project11", + OwnerID: 1, + }) + require.Nil(t, err) + + // add af1 into project + af1 := &models.Artifact{ + PID: pid1, + Repo: "hello-world", + Tag: "v1", + Digest: "CountSizeOfProject_af11", + Kind: "image", + } + _, err = AddArtifact(af1) + require.Nil(t, err) + afnb11 := &models.ArtifactAndBlob{ + DigestAF: "CountSizeOfProject_af11", + DigestBlob: "CountSizeOfProject_blob11", + } + afnb12 := &models.ArtifactAndBlob{ + DigestAF: "CountSizeOfProject_af11", + DigestBlob: "CountSizeOfProject_blob22", + } + afnb13 := &models.ArtifactAndBlob{ + DigestAF: "CountSizeOfProject_af11", + DigestBlob: "CountSizeOfProject_blob33", + } + var afnbs1 []*models.ArtifactAndBlob + afnbs1 = append(afnbs1, afnb11) + afnbs1 = append(afnbs1, afnb12) + afnbs1 = append(afnbs1, afnb13) + err = AddArtifactNBlobs(afnbs1) + require.Nil(t, err) + + // add af2 into project + af2 := &models.Artifact{ + PID: pid1, + Repo: "hello-world", + Tag: "v2", + Digest: "CountSizeOfProject_af22", + Kind: "image", + } + _, err = AddArtifact(af2) + require.Nil(t, err) + afnb21 := &models.ArtifactAndBlob{ + DigestAF: "CountSizeOfProject_af22", + DigestBlob: "CountSizeOfProject_blob11", + } + afnb22 := &models.ArtifactAndBlob{ + DigestAF: "CountSizeOfProject_af22", + DigestBlob: "CountSizeOfProject_blob22", + } + afnb23 := &models.ArtifactAndBlob{ + DigestAF: "CountSizeOfProject_af22", + DigestBlob: "CountSizeOfProject_blob33", + } + afnb24 := &models.ArtifactAndBlob{ + DigestAF: "CountSizeOfProject_af22", + DigestBlob: "CountSizeOfProject_blob44", + } + var afnbs2 []*models.ArtifactAndBlob + afnbs2 = append(afnbs2, afnb21) + afnbs2 = append(afnbs2, afnb22) + afnbs2 = append(afnbs2, afnb23) + afnbs2 = append(afnbs2, afnb24) + err = AddArtifactNBlobs(afnbs2) + require.Nil(t, err) + + pSize, err := CountSizeOfProject(pid1) + assert.Equal(t, pSize, int64(1010)) +} diff --git a/src/common/dao/quota.go b/src/common/dao/quota.go index 6cf130d3d..c86c53797 100644 --- a/src/common/dao/quota.go +++ b/src/common/dao/quota.go @@ -193,7 +193,7 @@ func quotaQueryConditions(query ...*models.QuotaQuery) (string, []interface{}) { } if len(q.ReferenceIDs) != 0 { - sql += fmt.Sprintf(`AND a.reference_id IN (%s) `, paramPlaceholder(len(q.ReferenceIDs))) + sql += fmt.Sprintf(`AND a.reference_id IN (%s) `, ParamPlaceholderForIn(len(q.ReferenceIDs))) params = append(params, q.ReferenceIDs) } diff --git a/src/common/dao/quota_usage.go b/src/common/dao/quota_usage.go index 8e2f7ca48..d8b55db9b 100644 --- a/src/common/dao/quota_usage.go +++ b/src/common/dao/quota_usage.go @@ -111,7 +111,7 @@ func quotaUsageQueryConditions(query ...*models.QuotaUsageQuery) (string, []inte params = append(params, q.ReferenceID) } if len(q.ReferenceIDs) != 0 { - sql += fmt.Sprintf(`and reference_id in (%s) `, paramPlaceholder(len(q.ReferenceIDs))) + sql += fmt.Sprintf(`and reference_id in (%s) `, ParamPlaceholderForIn(len(q.ReferenceIDs))) params = append(params, q.ReferenceIDs) } diff --git a/src/common/dao/repository.go b/src/common/dao/repository.go index c05a46899..abb859525 100644 --- a/src/common/dao/repository.go +++ b/src/common/dao/repository.go @@ -178,7 +178,7 @@ func repositoryQueryConditions(query ...*models.RepositoryQuery) (string, []inte if len(q.ProjectIDs) > 0 { sql += fmt.Sprintf(`and r.project_id in ( %s ) `, - paramPlaceholder(len(q.ProjectIDs))) + ParamPlaceholderForIn(len(q.ProjectIDs))) params = append(params, q.ProjectIDs) } diff --git a/src/common/dao/user.go b/src/common/dao/user.go index 9349c3477..535887b1e 100644 --- a/src/common/dao/user.go +++ b/src/common/dao/user.go @@ -117,12 +117,18 @@ func ListUsers(query *models.UserQuery) ([]models.User, error) { } func userQueryConditions(query *models.UserQuery) orm.QuerySeter { - qs := GetOrmer().QueryTable(&models.User{}). - Filter("deleted", 0). - Filter("user_id__gt", 1) + qs := GetOrmer().QueryTable(&models.User{}).Filter("deleted", 0) if query == nil { - return qs + // Exclude admin account, see https://github.com/goharbor/harbor/issues/2527 + return qs.Filter("user_id__gt", 1) + } + + if len(query.UserIDs) > 0 { + qs = qs.Filter("user_id__in", query.UserIDs) + } else { + // Exclude admin account when not filter by UserIDs, see https://github.com/goharbor/harbor/issues/2527 + qs = qs.Filter("user_id__gt", 1) } if len(query.Username) > 0 { @@ -202,7 +208,7 @@ func DeleteUser(userID int) error { name := fmt.Sprintf("%s#%d", user.Username, user.UserID) email := fmt.Sprintf("%s#%d", user.Email, user.UserID) - _, err = o.Raw(`update harbor_user + _, err = o.Raw(`update harbor_user set deleted = true, username = ?, email = ? where user_id = ?`, name, email, userID).Exec() return err @@ -234,6 +240,14 @@ func OnBoardUser(u *models.User) error { } if created { u.UserID = int(id) + // current orm framework doesn't support to fetch a pointer or sql.NullString with QueryRow + // https://github.com/astaxie/beego/issues/3767 + if len(u.Email) == 0 { + _, err = o.Raw("update harbor_user set email = null where user_id = ? ", id).Exec() + if err != nil { + return err + } + } } else { existing, err := GetUser(*u) if err != nil { diff --git a/src/common/dao/user_test.go b/src/common/dao/user_test.go index ff48b27ec..2b3029c17 100644 --- a/src/common/dao/user_test.go +++ b/src/common/dao/user_test.go @@ -90,3 +90,23 @@ func TestOnBoardUser(t *testing.T) { assert.True(u.UserID == id) CleanUser(int64(id)) } +func TestOnBoardUser_EmptyEmail(t *testing.T) { + assert := assert.New(t) + u := &models.User{ + Username: "empty_email", + Password: "password1", + Realname: "empty_email", + } + err := OnBoardUser(u) + assert.Nil(err) + id := u.UserID + assert.True(id > 0) + err = OnBoardUser(u) + assert.Nil(err) + assert.True(u.UserID == id) + assert.Equal("", u.Email) + + user, err := GetUser(models.User{Username: "empty_email"}) + assert.Equal("", user.Email) + CleanUser(int64(id)) +} diff --git a/src/common/models/config.go b/src/common/models/config.go index b8c7a0e6b..dfd13d4bb 100644 --- a/src/common/models/config.go +++ b/src/common/models/config.go @@ -45,12 +45,14 @@ type SQLite struct { // PostGreSQL ... type PostGreSQL struct { - Host string `json:"host"` - Port int `json:"port"` - Username string `json:"username"` - Password string `json:"password,omitempty"` - Database string `json:"database"` - SSLMode string `json:"sslmode"` + Host string `json:"host"` + Port int `json:"port"` + Username string `json:"username"` + Password string `json:"password,omitempty"` + Database string `json:"database"` + SSLMode string `json:"sslmode"` + MaxIdleConns int `json:"max_idle_conns"` + MaxOpenConns int `json:"max_open_conns"` } // Email ... diff --git a/src/common/models/user.go b/src/common/models/user.go index 77fac1a83..c4299869f 100644 --- a/src/common/models/user.go +++ b/src/common/models/user.go @@ -46,6 +46,7 @@ type User struct { // UserQuery ... type UserQuery struct { + UserIDs []int Username string Email string Pagination *Pagination diff --git a/src/common/quota/driver/project/driver.go b/src/common/quota/driver/project/driver.go index 8fafded6c..58de3aa2a 100644 --- a/src/common/quota/driver/project/driver.go +++ b/src/common/quota/driver/project/driver.go @@ -55,11 +55,23 @@ func getProjectsBatchFn(ctx context.Context, keys dataloader.Keys) []*dataloader return handleError(err) } + var ownerIDs []int var projectsMap = make(map[int64]*models.Project, len(projectIDs)) for _, project := range projects { + ownerIDs = append(ownerIDs, project.OwnerID) projectsMap[project.ProjectID] = project } + owners, err := dao.ListUsers(&models.UserQuery{UserIDs: ownerIDs}) + if err != nil { + return handleError(err) + } + + var ownersMap = make(map[int]*models.User, len(owners)) + for i, owner := range owners { + ownersMap[owner.UserID] = &owners[i] + } + var results []*dataloader.Result for _, projectID := range projectIDs { project, ok := projectsMap[projectID] @@ -67,6 +79,11 @@ func getProjectsBatchFn(ctx context.Context, keys dataloader.Keys) []*dataloader return handleError(fmt.Errorf("project not found, "+"project_id: %d", projectID)) } + owner, ok := ownersMap[project.OwnerID] + if ok { + project.OwnerName = owner.Username + } + result := dataloader.Result{ Data: project, Error: nil, diff --git a/src/common/quota/driver/project/driver_test.go b/src/common/quota/driver/project/driver_test.go index 992af0ae9..b3812a097 100644 --- a/src/common/quota/driver/project/driver_test.go +++ b/src/common/quota/driver/project/driver_test.go @@ -41,7 +41,7 @@ func (suite *DriverSuite) TestLoad() { obj := dr.RefObject{ "id": int64(1), "name": "library", - "owner_name": "", + "owner_name": "admin", } suite.Equal(obj, ref) diff --git a/src/common/quota/errors.go b/src/common/quota/errors.go new file mode 100644 index 000000000..c828734dd --- /dev/null +++ b/src/common/quota/errors.go @@ -0,0 +1,111 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quota + +import ( + "fmt" + "strings" + + "github.com/goharbor/harbor/src/pkg/types" +) + +// Errors contains all happened errors +type Errors []error + +// GetErrors gets all errors that have occurred and returns a slice of errors (Error type) +func (errs Errors) GetErrors() []error { + return errs +} + +// Add adds an error to a given slice of errors +func (errs Errors) Add(newErrors ...error) Errors { + for _, err := range newErrors { + if err == nil { + continue + } + + if errors, ok := err.(Errors); ok { + errs = errs.Add(errors...) + } else { + ok = true + for _, e := range errs { + if err == e { + ok = false + } + } + if ok { + errs = append(errs, err) + } + } + } + + return errs +} + +// Error takes a slice of all errors that have occurred and returns it as a formatted string +func (errs Errors) Error() string { + var errors = []string{} + for _, e := range errs { + errors = append(errors, e.Error()) + } + return strings.Join(errors, "; ") +} + +// ResourceOverflow ... +type ResourceOverflow struct { + Resource types.ResourceName + HardLimit int64 + CurrentUsed int64 + NewUsed int64 +} + +func (e *ResourceOverflow) Error() string { + resource := e.Resource + var ( + op string + delta int64 + ) + + if e.NewUsed > e.CurrentUsed { + op = "add" + delta = e.NewUsed - e.CurrentUsed + } else { + op = "subtract" + delta = e.CurrentUsed - e.NewUsed + } + + return fmt.Sprintf("%s %s of %s resource overflow the hard limit, current usage is %s and hard limit is %s", + op, resource.FormatValue(delta), resource, + resource.FormatValue(e.CurrentUsed), resource.FormatValue(e.HardLimit)) +} + +// NewResourceOverflowError ... +func NewResourceOverflowError(resource types.ResourceName, hardLimit, currentUsed, newUsed int64) error { + return &ResourceOverflow{Resource: resource, HardLimit: hardLimit, CurrentUsed: currentUsed, NewUsed: newUsed} +} + +// ResourceNotFound ... +type ResourceNotFound struct { + Resource types.ResourceName +} + +func (e *ResourceNotFound) Error() string { + return fmt.Sprintf("resource %s not found", e.Resource) +} + +// NewResourceNotFoundError ... +func NewResourceNotFoundError(resource types.ResourceName) error { + return &ResourceNotFound{Resource: resource} +} diff --git a/src/common/quota/manager.go b/src/common/quota/manager.go index 43d70777b..558334e6d 100644 --- a/src/common/quota/manager.go +++ b/src/common/quota/manager.go @@ -110,7 +110,8 @@ func (m *Manager) getUsageForUpdate(o orm.Ormer) (*models.QuotaUsage, error) { } func (m *Manager) updateUsage(o orm.Ormer, resources types.ResourceList, - calculate func(types.ResourceList, types.ResourceList) types.ResourceList) error { + calculate func(types.ResourceList, types.ResourceList) types.ResourceList, + skipOverflow bool) error { quota, err := m.getQuotaForUpdate(o) if err != nil { @@ -131,7 +132,13 @@ func (m *Manager) updateUsage(o orm.Ormer, resources types.ResourceList, } newUsed := calculate(used, resources) - if err := isSafe(hardLimits, newUsed); err != nil { + + // ensure that new used is never negative + if negativeUsed := types.IsNegative(newUsed); len(negativeUsed) > 0 { + return fmt.Errorf("quota usage is negative for resource(s): %s", prettyPrintResourceNames(negativeUsed)) + } + + if err := isSafe(hardLimits, used, newUsed, skipOverflow); err != nil { return err } @@ -176,27 +183,87 @@ func (m *Manager) DeleteQuota() error { // UpdateQuota update the quota resource spec func (m *Manager) UpdateQuota(hardLimits types.ResourceList) error { + o := dao.GetOrmer() if err := m.driver.Validate(hardLimits); err != nil { return err } sql := `UPDATE quota SET hard = ? WHERE reference = ? AND reference_id = ?` - _, err := dao.GetOrmer().Raw(sql, hardLimits.String(), m.reference, m.referenceID).Exec() + _, err := o.Raw(sql, hardLimits.String(), m.reference, m.referenceID).Exec() return err } +// SetResourceUsage sets the usage per resource name +func (m *Manager) SetResourceUsage(resource types.ResourceName, value int64) error { + o := dao.GetOrmer() + + sql := fmt.Sprintf("UPDATE quota_usage SET used = jsonb_set(used, '{%s}', to_jsonb(%d::int), true) WHERE reference = ? AND reference_id = ?", resource, value) + _, err := o.Raw(sql, m.reference, m.referenceID).Exec() + + return err +} + +// EnsureQuota ensures the reference has quota and usage, +// if non-existent, will create new quota and usage. +// if existent, update the quota and usage. +func (m *Manager) EnsureQuota(usages types.ResourceList) error { + query := &models.QuotaQuery{ + Reference: m.reference, + ReferenceID: m.referenceID, + } + quotas, err := dao.ListQuotas(query) + if err != nil { + return err + } + + // non-existent: create quota and usage + defaultHardLimit := m.driver.HardLimits() + if len(quotas) == 0 { + _, err := m.NewQuota(defaultHardLimit, usages) + if err != nil { + return err + } + return nil + } + + // existent + used := usages + quotaUsed, err := types.NewResourceList(quotas[0].Used) + if err != nil { + return err + } + if types.Equals(quotaUsed, used) { + return nil + } + dao.WithTransaction(func(o orm.Ormer) error { + usage, err := m.getUsageForUpdate(o) + if err != nil { + return err + } + usage.Used = used.String() + usage.UpdateTime = time.Now() + _, err = o.Update(usage) + if err != nil { + return err + } + return nil + }) + + return nil +} + // AddResources add resources to usage func (m *Manager) AddResources(resources types.ResourceList) error { return dao.WithTransaction(func(o orm.Ormer) error { - return m.updateUsage(o, resources, types.Add) + return m.updateUsage(o, resources, types.Add, false) }) } // SubtractResources subtract resources from usage func (m *Manager) SubtractResources(resources types.ResourceList) error { return dao.WithTransaction(func(o orm.Ormer) error { - return m.updateUsage(o, resources, types.Subtract) + return m.updateUsage(o, resources, types.Subtract, true) }) } diff --git a/src/common/quota/manager_test.go b/src/common/quota/manager_test.go index 7de96d998..6ce705575 100644 --- a/src/common/quota/manager_test.go +++ b/src/common/quota/manager_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/quota/driver" "github.com/goharbor/harbor/src/common/quota/driver/mocks" "github.com/goharbor/harbor/src/pkg/types" @@ -131,6 +132,66 @@ func (suite *ManagerSuite) TestUpdateQuota() { } } +func (suite *ManagerSuite) TestSetResourceUsage() { + mgr := suite.quotaManager() + id, _ := mgr.NewQuota(hardLimits) + + if err := mgr.SetResourceUsage(types.ResourceCount, 123); suite.Nil(err) { + quota, _ := dao.GetQuota(id) + suite.Equal(hardLimits, mustResourceList(quota.Hard)) + + usage, _ := dao.GetQuotaUsage(id) + suite.Equal(types.ResourceList{types.ResourceCount: 123, types.ResourceStorage: 0}, mustResourceList(usage.Used)) + } + + if err := mgr.SetResourceUsage(types.ResourceStorage, 234); suite.Nil(err) { + usage, _ := dao.GetQuotaUsage(id) + suite.Equal(types.ResourceList{types.ResourceCount: 123, types.ResourceStorage: 234}, mustResourceList(usage.Used)) + } +} + +func (suite *ManagerSuite) TestEnsureQuota() { + // non-existent + nonExistRefID := "3" + mgr := suite.quotaManager(nonExistRefID) + infinite := types.ResourceList{types.ResourceCount: -1, types.ResourceStorage: -1} + usage := types.ResourceList{types.ResourceCount: 10, types.ResourceStorage: 10} + err := mgr.EnsureQuota(usage) + suite.Nil(err) + query := &models.QuotaQuery{ + Reference: reference, + ReferenceID: nonExistRefID, + } + quotas, err := dao.ListQuotas(query) + suite.Nil(err) + suite.Equal(usage, mustResourceList(quotas[0].Used)) + suite.Equal(infinite, mustResourceList(quotas[0].Hard)) + + // existent + existRefID := "4" + mgr = suite.quotaManager(existRefID) + used := types.ResourceList{types.ResourceCount: 11, types.ResourceStorage: 11} + if id, err := mgr.NewQuota(hardLimits, used); suite.Nil(err) { + quota, _ := dao.GetQuota(id) + suite.Equal(hardLimits, mustResourceList(quota.Hard)) + + usage, _ := dao.GetQuotaUsage(id) + suite.Equal(used, mustResourceList(usage.Used)) + } + + usage2 := types.ResourceList{types.ResourceCount: 12, types.ResourceStorage: 12} + err = mgr.EnsureQuota(usage2) + suite.Nil(err) + query2 := &models.QuotaQuery{ + Reference: reference, + ReferenceID: existRefID, + } + quotas2, err := dao.ListQuotas(query2) + suite.Equal(usage2, mustResourceList(quotas2[0].Used)) + suite.Equal(hardLimits, mustResourceList(quotas2[0].Hard)) + +} + func (suite *ManagerSuite) TestQuotaAutoCreation() { for i := 0; i < 10; i++ { mgr := suite.quotaManager(fmt.Sprintf("%d", i)) @@ -157,7 +218,11 @@ func (suite *ManagerSuite) TestAddResources() { } if err := mgr.AddResources(types.ResourceList{types.ResourceStorage: 10000}); suite.Error(err) { - suite.True(IsUnsafeError(err)) + if errs, ok := err.(Errors); suite.True(ok) { + for _, err := range errs { + suite.IsType(&ResourceOverflow{}, err) + } + } } } diff --git a/src/common/quota/util.go b/src/common/quota/util.go index 33f3ce0a3..e57e05170 100644 --- a/src/common/quota/util.go +++ b/src/common/quota/util.go @@ -15,48 +15,43 @@ package quota import ( - "fmt" + "sort" + "strings" "github.com/goharbor/harbor/src/pkg/types" ) -type unsafe struct { - message string -} +func isSafe(hardLimits types.ResourceList, currentUsed types.ResourceList, newUsed types.ResourceList, skipOverflow bool) error { + var errs Errors -func (err *unsafe) Error() string { - return err.message -} - -func newUnsafe(message string) error { - return &unsafe{message: message} -} - -// IsUnsafeError returns true when the err is unsafe error -func IsUnsafeError(err error) bool { - _, ok := err.(*unsafe) - return ok -} - -func isSafe(hardLimits types.ResourceList, used types.ResourceList) error { - for key, value := range used { - if value < 0 { - return newUnsafe(fmt.Sprintf("bad used value: %d", value)) + for resource, value := range newUsed { + hardLimit, found := hardLimits[resource] + if !found { + errs = errs.Add(NewResourceNotFoundError(resource)) + continue } - if hard, found := hardLimits[key]; found { - if hard == types.UNLIMITED { - continue - } - - if value > hard { - return newUnsafe(fmt.Sprintf("over the quota: used %d but only hard %d", value, hard)) - } - } else { - return newUnsafe(fmt.Sprintf("hard limit not found: %s", key)) + if hardLimit == types.UNLIMITED || value == currentUsed[resource] { + continue } + if value > hardLimit && !skipOverflow { + errs = errs.Add(NewResourceOverflowError(resource, hardLimit, currentUsed[resource], value)) + } + } + + if len(errs) > 0 { + return errs } return nil } + +func prettyPrintResourceNames(a []types.ResourceName) string { + values := []string{} + for _, value := range a { + values = append(values, string(value)) + } + sort.Strings(values) + return strings.Join(values, ",") +} diff --git a/src/common/quota/util_test.go b/src/common/quota/util_test.go index 806ae56af..44432348d 100644 --- a/src/common/quota/util_test.go +++ b/src/common/quota/util_test.go @@ -15,45 +15,17 @@ package quota import ( - "errors" "testing" "github.com/goharbor/harbor/src/pkg/types" ) -func TestIsUnsafeError(t *testing.T) { +func Test_isSafe(t *testing.T) { type args struct { - err error - } - tests := []struct { - name string - args args - want bool - }{ - { - "is unsafe error", - args{err: newUnsafe("unsafe")}, - true, - }, - { - "is not unsafe error", - args{err: errors.New("unsafe")}, - false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := IsUnsafeError(tt.args.err); got != tt.want { - t.Errorf("IsUnsafeError() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_checkQuotas(t *testing.T) { - type args struct { - hardLimits types.ResourceList - used types.ResourceList + hardLimits types.ResourceList + currentUsed types.ResourceList + newUsed types.ResourceList + skipOverflow bool } tests := []struct { name string @@ -62,33 +34,58 @@ func Test_checkQuotas(t *testing.T) { }{ { "unlimited", - args{hardLimits: types.ResourceList{types.ResourceStorage: types.UNLIMITED}, used: types.ResourceList{types.ResourceStorage: 1000}}, + args{ + types.ResourceList{types.ResourceStorage: types.UNLIMITED}, + types.ResourceList{types.ResourceStorage: 1000}, + types.ResourceList{types.ResourceStorage: 1000}, + false, + }, false, }, { "ok", - args{hardLimits: types.ResourceList{types.ResourceStorage: 100}, used: types.ResourceList{types.ResourceStorage: 1}}, + args{ + types.ResourceList{types.ResourceStorage: 100}, + types.ResourceList{types.ResourceStorage: 10}, + types.ResourceList{types.ResourceStorage: 1}, + false, + }, false, }, { - "bad used value", - args{hardLimits: types.ResourceList{types.ResourceStorage: 100}, used: types.ResourceList{types.ResourceStorage: -1}}, + "over the hard limit", + args{ + types.ResourceList{types.ResourceStorage: 100}, + types.ResourceList{types.ResourceStorage: 0}, + types.ResourceList{types.ResourceStorage: 200}, + false, + }, true, }, { - "over the hard limit", - args{hardLimits: types.ResourceList{types.ResourceStorage: 100}, used: types.ResourceList{types.ResourceStorage: 200}}, - true, + "skip overflow", + args{ + types.ResourceList{types.ResourceStorage: 100}, + types.ResourceList{types.ResourceStorage: 0}, + types.ResourceList{types.ResourceStorage: 200}, + true, + }, + false, }, { "hard limit not found", - args{hardLimits: types.ResourceList{types.ResourceStorage: 100}, used: types.ResourceList{types.ResourceCount: 1}}, + args{ + types.ResourceList{types.ResourceStorage: 100}, + types.ResourceList{types.ResourceCount: 0}, + types.ResourceList{types.ResourceCount: 1}, + false, + }, true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if err := isSafe(tt.args.hardLimits, tt.args.used); (err != nil) != tt.wantErr { + if err := isSafe(tt.args.hardLimits, tt.args.currentUsed, tt.args.newUsed, tt.args.skipOverflow); (err != nil) != tt.wantErr { t.Errorf("isSafe() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/src/common/rbac/namespace.go b/src/common/rbac/namespace.go index 7f4f0f6a3..e1f123b10 100644 --- a/src/common/rbac/namespace.go +++ b/src/common/rbac/namespace.go @@ -31,8 +31,8 @@ type Namespace interface { } type projectNamespace struct { - projectIDOrName interface{} - isPublic bool + projectID int64 + isPublic bool } func (ns *projectNamespace) Kind() string { @@ -40,11 +40,11 @@ func (ns *projectNamespace) Kind() string { } func (ns *projectNamespace) Resource(subresources ...Resource) Resource { - return Resource(fmt.Sprintf("/project/%v", ns.projectIDOrName)).Subresource(subresources...) + return Resource(fmt.Sprintf("/project/%d", ns.projectID)).Subresource(subresources...) } func (ns *projectNamespace) Identity() interface{} { - return ns.projectIDOrName + return ns.projectID } func (ns *projectNamespace) IsPublic() bool { @@ -52,10 +52,10 @@ func (ns *projectNamespace) IsPublic() bool { } // NewProjectNamespace returns namespace for project -func NewProjectNamespace(projectIDOrName interface{}, isPublic ...bool) Namespace { +func NewProjectNamespace(projectID int64, isPublic ...bool) Namespace { isPublicNamespace := false if len(isPublic) > 0 { isPublicNamespace = isPublic[0] } - return &projectNamespace{projectIDOrName: projectIDOrName, isPublic: isPublicNamespace} + return &projectNamespace{projectID: projectID, isPublic: isPublicNamespace} } diff --git a/src/common/rbac/namespace_test.go b/src/common/rbac/namespace_test.go index 5fddad0e4..97a8bfbc2 100644 --- a/src/common/rbac/namespace_test.go +++ b/src/common/rbac/namespace_test.go @@ -27,7 +27,7 @@ type ProjectNamespaceTestSuite struct { func (suite *ProjectNamespaceTestSuite) TestResource() { var namespace Namespace - namespace = &projectNamespace{projectIDOrName: int64(1)} + namespace = &projectNamespace{projectID: int64(1)} suite.Equal(namespace.Resource(Resource("image")), Resource("/project/1/image")) } @@ -35,9 +35,6 @@ func (suite *ProjectNamespaceTestSuite) TestResource() { func (suite *ProjectNamespaceTestSuite) TestIdentity() { namespace, _ := Resource("/project/1/image").GetNamespace() suite.Equal(namespace.Identity(), int64(1)) - - namespace, _ = Resource("/project/library/image").GetNamespace() - suite.Equal(namespace.Identity(), "library") } func TestProjectNamespaceTestSuite(t *testing.T) { diff --git a/src/common/rbac/parser.go b/src/common/rbac/parser.go index bb65943e6..e4db275cc 100644 --- a/src/common/rbac/parser.go +++ b/src/common/rbac/parser.go @@ -37,14 +37,10 @@ func projectNamespaceParser(resource Resource) (Namespace, error) { return nil, errors.New("not support resource") } - var projectIDOrName interface{} - - id, err := strconv.ParseInt(matches[1], 10, 64) - if err == nil { - projectIDOrName = id - } else { - projectIDOrName = matches[1] + projectID, err := strconv.ParseInt(matches[1], 10, 64) + if err != nil { + return nil, err } - return &projectNamespace{projectIDOrName: projectIDOrName}, nil + return &projectNamespace{projectID: projectID}, nil } diff --git a/src/common/rbac/parser_test.go b/src/common/rbac/parser_test.go index cf23d517b..2a56a5025 100644 --- a/src/common/rbac/parser_test.go +++ b/src/common/rbac/parser_test.go @@ -26,7 +26,7 @@ type ProjectParserTestSuite struct { func (suite *ProjectParserTestSuite) TestParse() { namespace, err := projectNamespaceParser(Resource("/project/1/image")) - suite.Equal(namespace, &projectNamespace{projectIDOrName: int64(1)}) + suite.Equal(namespace, &projectNamespace{projectID: 1}) suite.Nil(err) namespace, err = projectNamespaceParser(Resource("/fake/1/image")) diff --git a/src/common/rbac/project/visitor_test.go b/src/common/rbac/project/visitor_test.go index 32fa78df6..921f81c1b 100644 --- a/src/common/rbac/project/visitor_test.go +++ b/src/common/rbac/project/visitor_test.go @@ -50,8 +50,8 @@ type VisitorTestSuite struct { } func (suite *VisitorTestSuite) TestGetPolicies() { - namespace := rbac.NewProjectNamespace("library", false) - publicNamespace := rbac.NewProjectNamespace("library", true) + namespace := rbac.NewProjectNamespace(1, false) + publicNamespace := rbac.NewProjectNamespace(1, true) anonymous := NewUser(anonymousCtx, namespace) suite.Nil(anonymous.GetPolicies()) @@ -73,7 +73,7 @@ func (suite *VisitorTestSuite) TestGetPolicies() { } func (suite *VisitorTestSuite) TestGetRoles() { - namespace := rbac.NewProjectNamespace("library", false) + namespace := rbac.NewProjectNamespace(1, false) anonymous := NewUser(anonymousCtx, namespace) suite.Nil(anonymous.GetRoles()) diff --git a/src/common/security/admiral/context.go b/src/common/security/admiral/context.go index 962a6dafb..fcc0a069f 100644 --- a/src/common/security/admiral/context.go +++ b/src/common/security/admiral/context.go @@ -75,10 +75,10 @@ func (s *SecurityContext) Can(action rbac.Action, resource rbac.Resource) bool { if err == nil { switch ns.Kind() { case "project": - projectIDOrName := ns.Identity() - isPublicProject, _ := s.pm.IsPublic(projectIDOrName) - projectNamespace := rbac.NewProjectNamespace(projectIDOrName, isPublicProject) - user := project.NewUser(s, projectNamespace, s.GetProjectRoles(projectIDOrName)...) + projectID := ns.Identity().(int64) + isPublicProject, _ := s.pm.IsPublic(projectID) + projectNamespace := rbac.NewProjectNamespace(projectID, isPublicProject) + user := project.NewUser(s, projectNamespace, s.GetProjectRoles(projectID)...) return rbac.HasPermission(user, resource, action) } } diff --git a/src/common/security/local/context.go b/src/common/security/local/context.go index 907521e2f..b246ae2e8 100644 --- a/src/common/security/local/context.go +++ b/src/common/security/local/context.go @@ -72,10 +72,10 @@ func (s *SecurityContext) Can(action rbac.Action, resource rbac.Resource) bool { if err == nil { switch ns.Kind() { case "project": - projectIDOrName := ns.Identity() - isPublicProject, _ := s.pm.IsPublic(projectIDOrName) - projectNamespace := rbac.NewProjectNamespace(projectIDOrName, isPublicProject) - user := project.NewUser(s, projectNamespace, s.GetProjectRoles(projectIDOrName)...) + projectID := ns.Identity().(int64) + isPublicProject, _ := s.pm.IsPublic(projectID) + projectNamespace := rbac.NewProjectNamespace(projectID, isPublicProject) + user := project.NewUser(s, projectNamespace, s.GetProjectRoles(projectID)...) return rbac.HasPermission(user, resource, action) } } diff --git a/src/common/security/local/context_test.go b/src/common/security/local/context_test.go index ffbb51885..45cd14450 100644 --- a/src/common/security/local/context_test.go +++ b/src/common/security/local/context_test.go @@ -176,12 +176,12 @@ func TestHasPullPerm(t *testing.T) { // public project ctx := NewSecurityContext(nil, pm) - resource := rbac.NewProjectNamespace("library").Resource(rbac.ResourceRepository) + resource := rbac.NewProjectNamespace(1).Resource(rbac.ResourceRepository) assert.True(t, ctx.Can(rbac.ActionPull, resource)) // private project, unauthenticated ctx = NewSecurityContext(nil, pm) - resource = rbac.NewProjectNamespace(private.Name).Resource(rbac.ResourceRepository) + resource = rbac.NewProjectNamespace(private.ProjectID).Resource(rbac.ResourceRepository) assert.False(t, ctx.Can(rbac.ActionPull, resource)) // private project, authenticated, has no perm @@ -203,7 +203,7 @@ func TestHasPullPerm(t *testing.T) { } func TestHasPushPerm(t *testing.T) { - resource := rbac.NewProjectNamespace(private.Name).Resource(rbac.ResourceRepository) + resource := rbac.NewProjectNamespace(private.ProjectID).Resource(rbac.ResourceRepository) // unauthenticated ctx := NewSecurityContext(nil, pm) @@ -226,7 +226,7 @@ func TestHasPushPerm(t *testing.T) { } func TestHasPushPullPerm(t *testing.T) { - resource := rbac.NewProjectNamespace(private.Name).Resource(rbac.ResourceRepository) + resource := rbac.NewProjectNamespace(private.ProjectID).Resource(rbac.ResourceRepository) // unauthenticated ctx := NewSecurityContext(nil, pm) @@ -265,7 +265,7 @@ func TestHasPushPullPermWithGroup(t *testing.T) { developer.GroupIDs = []int{userGroups[0].ID} - resource := rbac.NewProjectNamespace(project.Name).Resource(rbac.ResourceRepository) + resource := rbac.NewProjectNamespace(project.ProjectID).Resource(rbac.ResourceRepository) ctx := NewSecurityContext(developer, pm) assert.True(t, ctx.Can(rbac.ActionPush, resource)) diff --git a/src/common/security/robot/context.go b/src/common/security/robot/context.go index 49d80ef35..8fc622fe0 100644 --- a/src/common/security/robot/context.go +++ b/src/common/security/robot/context.go @@ -76,9 +76,9 @@ func (s *SecurityContext) Can(action rbac.Action, resource rbac.Resource) bool { if err == nil { switch ns.Kind() { case "project": - projectIDOrName := ns.Identity() - isPublicProject, _ := s.pm.IsPublic(projectIDOrName) - projectNamespace := rbac.NewProjectNamespace(projectIDOrName, isPublicProject) + projectID := ns.Identity().(int64) + isPublicProject, _ := s.pm.IsPublic(projectID) + projectNamespace := rbac.NewProjectNamespace(projectID, isPublicProject) robot := NewRobot(s.GetUsername(), projectNamespace, s.policy) return rbac.HasPermission(robot, resource, action) } diff --git a/src/common/security/robot/context_test.go b/src/common/security/robot/context_test.go index e9fb2ce8f..36a8a5316 100644 --- a/src/common/security/robot/context_test.go +++ b/src/common/security/robot/context_test.go @@ -15,6 +15,7 @@ package robot import ( + "fmt" "os" "strconv" "testing" @@ -136,7 +137,7 @@ func TestIsSolutionUser(t *testing.T) { func TestHasPullPerm(t *testing.T) { policies := []*rbac.Policy{ { - Resource: "/project/testrobot/repository", + Resource: rbac.Resource(fmt.Sprintf("/project/%d/repository", private.ProjectID)), Action: rbac.ActionPull, }, } @@ -146,14 +147,14 @@ func TestHasPullPerm(t *testing.T) { } ctx := NewSecurityContext(robot, pm, policies) - resource := rbac.NewProjectNamespace(private.Name).Resource(rbac.ResourceRepository) + resource := rbac.NewProjectNamespace(private.ProjectID).Resource(rbac.ResourceRepository) assert.True(t, ctx.Can(rbac.ActionPull, resource)) } func TestHasPushPerm(t *testing.T) { policies := []*rbac.Policy{ { - Resource: "/project/testrobot/repository", + Resource: rbac.Resource(fmt.Sprintf("/project/%d/repository", private.ProjectID)), Action: rbac.ActionPush, }, } @@ -163,18 +164,18 @@ func TestHasPushPerm(t *testing.T) { } ctx := NewSecurityContext(robot, pm, policies) - resource := rbac.NewProjectNamespace(private.Name).Resource(rbac.ResourceRepository) + resource := rbac.NewProjectNamespace(private.ProjectID).Resource(rbac.ResourceRepository) assert.True(t, ctx.Can(rbac.ActionPush, resource)) } func TestHasPushPullPerm(t *testing.T) { policies := []*rbac.Policy{ { - Resource: "/project/testrobot/repository", + Resource: rbac.Resource(fmt.Sprintf("/project/%d/repository", private.ProjectID)), Action: rbac.ActionPush, }, { - Resource: "/project/testrobot/repository", + Resource: rbac.Resource(fmt.Sprintf("/project/%d/repository", private.ProjectID)), Action: rbac.ActionPull, }, } @@ -184,7 +185,7 @@ func TestHasPushPullPerm(t *testing.T) { } ctx := NewSecurityContext(robot, pm, policies) - resource := rbac.NewProjectNamespace(private.Name).Resource(rbac.ResourceRepository) + resource := rbac.NewProjectNamespace(private.ProjectID).Resource(rbac.ResourceRepository) assert.True(t, ctx.Can(rbac.ActionPush, resource) && ctx.Can(rbac.ActionPull, resource)) } diff --git a/src/common/security/robot/robot_test.go b/src/common/security/robot/robot_test.go index 62acbe11f..ba89fac41 100644 --- a/src/common/security/robot/robot_test.go +++ b/src/common/security/robot/robot_test.go @@ -1,9 +1,24 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package robot import ( + "testing" + "github.com/goharbor/harbor/src/common/rbac" "github.com/stretchr/testify/assert" - "testing" ) func TestGetPolicies(t *testing.T) { @@ -17,7 +32,7 @@ func TestGetPolicies(t *testing.T) { robot := robot{ username: "test", - namespace: rbac.NewProjectNamespace("library", false), + namespace: rbac.NewProjectNamespace(1, false), policy: policies, } diff --git a/src/common/utils/ldap/ldap.go b/src/common/utils/ldap/ldap.go index e7c453376..512af7618 100644 --- a/src/common/utils/ldap/ldap.go +++ b/src/common/utils/ldap/ldap.go @@ -220,6 +220,27 @@ func (session *Session) SearchUser(username string) ([]models.LdapUser, error) { } u.GroupDNList = groupDNList } + + log.Debugf("Searching for nested groups") + nestedGroupDNList := []string{} + nestedGroupFilter := createNestedGroupFilter(ldapEntry.DN) + result, err := session.SearchLdap(nestedGroupFilter) + if err != nil { + return nil, err + } + + for _, groupEntry := range result.Entries { + if !contains(u.GroupDNList, groupEntry.DN) { + nestedGroupDNList = append(nestedGroupDNList, strings.TrimSpace(groupEntry.DN)) + log.Debugf("Found group %v", groupEntry.DN) + } else { + log.Debugf("%v is already in GroupDNList", groupEntry.DN) + } + } + + u.GroupDNList = append(u.GroupDNList, nestedGroupDNList...) + log.Debugf("Done searching for nested groups") + u.DN = ldapEntry.DN ldapUsers = append(ldapUsers, u) @@ -330,13 +351,13 @@ func (session *Session) createUserFilter(username string) string { filterTag = goldap.EscapeFilter(username) } - ldapFilter := session.ldapConfig.LdapFilter + ldapFilter := normalizeFilter(session.ldapConfig.LdapFilter) ldapUID := session.ldapConfig.LdapUID if ldapFilter == "" { ldapFilter = "(" + ldapUID + "=" + filterTag + ")" } else { - ldapFilter = "(&" + ldapFilter + "(" + ldapUID + "=" + filterTag + "))" + ldapFilter = "(&(" + ldapFilter + ")(" + ldapUID + "=" + filterTag + "))" } log.Debug("ldap filter :", ldapFilter) @@ -404,6 +425,7 @@ func createGroupSearchFilter(oldFilter, groupName, groupNameAttribute string) st filter := "" groupName = goldap.EscapeFilter(groupName) groupNameAttribute = goldap.EscapeFilter(groupNameAttribute) + oldFilter = normalizeFilter(oldFilter) if len(oldFilter) == 0 { if len(groupName) == 0 { filter = groupNameAttribute + "=*" @@ -419,3 +441,26 @@ func createGroupSearchFilter(oldFilter, groupName, groupNameAttribute string) st } return filter } + +func createNestedGroupFilter(userDN string) string { + filter := "" + filter = "(&(objectClass=group)(member:1.2.840.113556.1.4.1941:=" + userDN + "))" + return filter +} + +func contains(s []string, e string) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} + +// normalizeFilter - remove '(' and ')' in ldap filter +func normalizeFilter(filter string) string { + norFilter := strings.TrimSpace(filter) + norFilter = strings.TrimPrefix(norFilter, "(") + norFilter = strings.TrimSuffix(norFilter, ")") + return norFilter +} diff --git a/src/common/utils/ldap/ldap_test.go b/src/common/utils/ldap/ldap_test.go index ed80fd17a..e7b3344a6 100644 --- a/src/common/utils/ldap/ldap_test.go +++ b/src/common/utils/ldap/ldap_test.go @@ -369,3 +369,25 @@ func TestSession_SearchGroupByDN(t *testing.T) { }) } } + +func TestNormalizeFilter(t *testing.T) { + type args struct { + filter string + } + tests := []struct { + name string + args args + want string + }{ + {"normal test", args{"(objectclass=user)"}, "objectclass=user"}, + {"with space", args{" (objectclass=user) "}, "objectclass=user"}, + {"nothing", args{"objectclass=user"}, "objectclass=user"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := normalizeFilter(tt.args.filter); got != tt.want { + t.Errorf("normalizeFilter() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/src/common/utils/registry/repository.go b/src/common/utils/registry/repository.go index f7304499c..7a4a1c6c7 100644 --- a/src/common/utils/registry/repository.go +++ b/src/common/utils/registry/repository.go @@ -25,11 +25,9 @@ import ( "sort" "strconv" "strings" - // "time" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" - commonhttp "github.com/goharbor/harbor/src/common/http" "github.com/goharbor/harbor/src/common/utils" ) @@ -407,6 +405,7 @@ func (r *Repository) monolithicBlobUpload(location, digest string, size int64, d if err != nil { return err } + req.ContentLength = size resp, err := r.client.Do(req) if err != nil { diff --git a/src/core/api/base.go b/src/core/api/base.go index 7c4dfddf4..195f7f9c8 100644 --- a/src/core/api/base.go +++ b/src/core/api/base.go @@ -15,22 +15,24 @@ package api import ( + "encoding/json" "errors" - - "github.com/goharbor/harbor/src/pkg/retention" - "github.com/goharbor/harbor/src/pkg/scheduler" - + "fmt" "net/http" "github.com/ghodss/yaml" "github.com/goharbor/harbor/src/common/api" + "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/security" + "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/filter" "github.com/goharbor/harbor/src/core/promgr" "github.com/goharbor/harbor/src/pkg/project" "github.com/goharbor/harbor/src/pkg/repository" + "github.com/goharbor/harbor/src/pkg/retention" + "github.com/goharbor/harbor/src/pkg/scheduler" ) const ( @@ -47,6 +49,10 @@ var ( retentionController retention.APIController ) +var ( + errNotFound = errors.New("not found") +) + // BaseController ... type BaseController struct { api.BaseAPI @@ -77,6 +83,71 @@ func (b *BaseController) Prepare() { b.ProjectMgr = pm } +// RequireAuthenticated returns true when the request is authenticated +// otherwise send Unauthorized response and returns false +func (b *BaseController) RequireAuthenticated() bool { + if !b.SecurityCtx.IsAuthenticated() { + b.SendUnAuthorizedError(errors.New("Unauthorized")) + return false + } + + return true +} + +// HasProjectPermission returns true when the request has action permission on project subresource +func (b *BaseController) HasProjectPermission(projectIDOrName interface{}, action rbac.Action, subresource ...rbac.Resource) (bool, error) { + projectID, projectName, err := utils.ParseProjectIDOrName(projectIDOrName) + if err != nil { + return false, err + } + + if projectName != "" { + project, err := b.ProjectMgr.Get(projectName) + if err != nil { + return false, err + } + if project == nil { + return false, errNotFound + } + + projectID = project.ProjectID + } + + resource := rbac.NewProjectNamespace(projectID).Resource(subresource...) + if !b.SecurityCtx.Can(action, resource) { + return false, nil + } + + return true, nil +} + +// RequireProjectAccess returns true when the request has action access on project subresource +// otherwise send UnAuthorized or Forbidden response and returns false +func (b *BaseController) RequireProjectAccess(projectIDOrName interface{}, action rbac.Action, subresource ...rbac.Resource) bool { + hasPermission, err := b.HasProjectPermission(projectIDOrName, action, subresource...) + if err != nil { + if err == errNotFound { + b.SendNotFoundError(fmt.Errorf("project %v not found", projectIDOrName)) + } else { + b.SendInternalServerError(err) + } + + return false + } + + if !hasPermission { + if !b.SecurityCtx.IsAuthenticated() { + b.SendUnAuthorizedError(errors.New("UnAuthorized")) + } else { + b.SendForbiddenError(errors.New(b.SecurityCtx.GetUsername())) + } + + return false + } + + return true +} + // WriteJSONData writes the JSON data to the client. func (b *BaseController) WriteJSONData(object interface{}) { b.Data["json"] = object @@ -121,12 +192,16 @@ func Init() error { retentionController = retention.NewAPIController(retentionMgr, projectMgr, repositoryMgr, retentionScheduler, retentionLauncher) callbackFun := func(p interface{}) error { - r, ok := p.(retention.TriggerParam) - if ok { - _, err := retentionController.TriggerRetentionExec(r.PolicyID, r.Trigger, false) - return err + str, ok := p.(string) + if !ok { + return fmt.Errorf("the type of param %v isn't string", p) } - return errors.New("bad retention callback param") + param := &retention.TriggerParam{} + if err := json.Unmarshal([]byte(str), param); err != nil { + return fmt.Errorf("failed to unmarshal the param: %v", err) + } + _, err := retentionController.TriggerRetentionExec(param.PolicyID, param.Trigger, false) + return err } err := scheduler.Register(retention.SchedulerCallback, callbackFun) diff --git a/src/core/api/chart_label.go b/src/core/api/chart_label.go index 02f5a98ee..e70254e77 100644 --- a/src/core/api/chart_label.go +++ b/src/core/api/chart_label.go @@ -58,14 +58,7 @@ func (cla *ChartLabelAPI) Prepare() { } func (cla *ChartLabelAPI) requireAccess(action rbac.Action) bool { - resource := rbac.NewProjectNamespace(cla.project.ProjectID).Resource(rbac.ResourceHelmChartVersionLabel) - - if !cla.SecurityCtx.Can(action, resource) { - cla.SendForbiddenError(errors.New(cla.SecurityCtx.GetUsername())) - return false - } - - return true + return cla.RequireProjectAccess(cla.project.ProjectID, action, rbac.ResourceHelmChartVersionLabel) } // MarkLabel handles the request of marking label to chart. diff --git a/src/core/api/chart_repository.go b/src/core/api/chart_repository.go index 5a67acdb8..7c45d36bc 100755 --- a/src/core/api/chart_repository.go +++ b/src/core/api/chart_repository.go @@ -105,19 +105,8 @@ func (cra *ChartRepositoryAPI) requireAccess(action rbac.Action, subresource ... if len(subresource) == 0 { subresource = append(subresource, rbac.ResourceHelmChart) } - resource := rbac.NewProjectNamespace(cra.namespace).Resource(subresource...) - if !cra.SecurityCtx.Can(action, resource) { - if !cra.SecurityCtx.IsAuthenticated() { - cra.SendUnAuthorizedError(errors.New("Unauthorized")) - } else { - cra.SendForbiddenError(errors.New(cra.SecurityCtx.GetUsername())) - } - - return false - } - - return true + return cra.RequireProjectAccess(cra.namespace, action, subresource...) } // GetHealthStatus handles GET /api/chartrepo/health diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index 5357a6579..b6ed840b2 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -35,6 +35,7 @@ import ( testutils "github.com/goharbor/harbor/src/common/utils/test" api_models "github.com/goharbor/harbor/src/core/api/models" apimodels "github.com/goharbor/harbor/src/core/api/models" + quota "github.com/goharbor/harbor/src/core/api/quota" _ "github.com/goharbor/harbor/src/core/auth/db" _ "github.com/goharbor/harbor/src/core/auth/ldap" "github.com/goharbor/harbor/src/core/config" @@ -202,11 +203,18 @@ func init() { beego.Router("/api/quotas", quotaAPIType, "get:List") beego.Router("/api/quotas/:id([0-9]+)", quotaAPIType, "get:Get;put:Put") + beego.Router("/api/internal/switchquota", &InternalAPI{}, "put:SwitchQuota") + beego.Router("/api/internal/syncquota", &InternalAPI{}, "post:SyncQuota") + // syncRegistry if err := SyncRegistry(config.GlobalProjectMgr); err != nil { log.Fatalf("failed to sync repositories from registry: %v", err) } + if err := quota.Sync(config.GlobalProjectMgr, false); err != nil { + log.Fatalf("failed to sync quota from backend: %v", err) + } + // Init user Info admin = &usrInfo{adminName, adminPwd} unknownUsr = &usrInfo{"unknown", "unknown"} diff --git a/src/core/api/health.go b/src/core/api/health.go index 6d5a890d1..0d4ef2cac 100644 --- a/src/core/api/health.go +++ b/src/core/api/health.go @@ -34,8 +34,9 @@ import ( ) var ( - timeout = 60 * time.Second - healthCheckerRegistry = map[string]health.Checker{} + timeout = 60 * time.Second + // HealthCheckerRegistry ... + HealthCheckerRegistry = map[string]health.Checker{} ) type overallHealthStatus struct { @@ -67,11 +68,11 @@ type HealthAPI struct { func (h *HealthAPI) CheckHealth() { var isHealthy healthy = true components := []*componentHealthStatus{} - c := make(chan *componentHealthStatus, len(healthCheckerRegistry)) - for name, checker := range healthCheckerRegistry { + c := make(chan *componentHealthStatus, len(HealthCheckerRegistry)) + for name, checker := range HealthCheckerRegistry { go check(name, checker, timeout, c) } - for i := 0; i < len(healthCheckerRegistry); i++ { + for i := 0; i < len(HealthCheckerRegistry); i++ { componentStatus := <-c if len(componentStatus.Error) != 0 { isHealthy = false @@ -290,21 +291,21 @@ func redisHealthChecker() health.Checker { } func registerHealthCheckers() { - healthCheckerRegistry["core"] = coreHealthChecker() - healthCheckerRegistry["portal"] = portalHealthChecker() - healthCheckerRegistry["jobservice"] = jobserviceHealthChecker() - healthCheckerRegistry["registry"] = registryHealthChecker() - healthCheckerRegistry["registryctl"] = registryCtlHealthChecker() - healthCheckerRegistry["database"] = databaseHealthChecker() - healthCheckerRegistry["redis"] = redisHealthChecker() + HealthCheckerRegistry["core"] = coreHealthChecker() + HealthCheckerRegistry["portal"] = portalHealthChecker() + HealthCheckerRegistry["jobservice"] = jobserviceHealthChecker() + HealthCheckerRegistry["registry"] = registryHealthChecker() + HealthCheckerRegistry["registryctl"] = registryCtlHealthChecker() + HealthCheckerRegistry["database"] = databaseHealthChecker() + HealthCheckerRegistry["redis"] = redisHealthChecker() if config.WithChartMuseum() { - healthCheckerRegistry["chartmuseum"] = chartmuseumHealthChecker() + HealthCheckerRegistry["chartmuseum"] = chartmuseumHealthChecker() } if config.WithClair() { - healthCheckerRegistry["clair"] = clairHealthChecker() + HealthCheckerRegistry["clair"] = clairHealthChecker() } if config.WithNotary() { - healthCheckerRegistry["notary"] = notaryHealthChecker() + HealthCheckerRegistry["notary"] = notaryHealthChecker() } } diff --git a/src/core/api/health_test.go b/src/core/api/health_test.go index 8426a74b1..c98d021b5 100644 --- a/src/core/api/health_test.go +++ b/src/core/api/health_test.go @@ -92,9 +92,9 @@ func fakeHealthChecker(healthy bool) health.Checker { } func TestCheckHealth(t *testing.T) { // component01: healthy, component02: healthy => status: healthy - healthCheckerRegistry = map[string]health.Checker{} - healthCheckerRegistry["component01"] = fakeHealthChecker(true) - healthCheckerRegistry["component02"] = fakeHealthChecker(true) + HealthCheckerRegistry = map[string]health.Checker{} + HealthCheckerRegistry["component01"] = fakeHealthChecker(true) + HealthCheckerRegistry["component02"] = fakeHealthChecker(true) status := map[string]interface{}{} err := handleAndParse(&testingRequest{ method: http.MethodGet, @@ -104,9 +104,9 @@ func TestCheckHealth(t *testing.T) { assert.Equal(t, "healthy", status["status"].(string)) // component01: healthy, component02: unhealthy => status: unhealthy - healthCheckerRegistry = map[string]health.Checker{} - healthCheckerRegistry["component01"] = fakeHealthChecker(true) - healthCheckerRegistry["component02"] = fakeHealthChecker(false) + HealthCheckerRegistry = map[string]health.Checker{} + HealthCheckerRegistry["component01"] = fakeHealthChecker(true) + HealthCheckerRegistry["component02"] = fakeHealthChecker(false) status = map[string]interface{}{} err = handleAndParse(&testingRequest{ method: http.MethodGet, @@ -128,7 +128,7 @@ func TestDatabaseHealthChecker(t *testing.T) { } func TestRegisterHealthCheckers(t *testing.T) { - healthCheckerRegistry = map[string]health.Checker{} + HealthCheckerRegistry = map[string]health.Checker{} registerHealthCheckers() - assert.NotNil(t, healthCheckerRegistry["core"]) + assert.NotNil(t, HealthCheckerRegistry["core"]) } diff --git a/src/core/api/internal.go b/src/core/api/internal.go index 71f1f317e..06e6c45a2 100644 --- a/src/core/api/internal.go +++ b/src/core/api/internal.go @@ -15,12 +15,21 @@ package api import ( - "errors" - + "fmt" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" + common_quota "github.com/goharbor/harbor/src/common/quota" "github.com/goharbor/harbor/src/common/utils/log" + + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/pkg/errors" + "strconv" + + quota "github.com/goharbor/harbor/src/core/api/quota" + + comcfg "github.com/goharbor/harbor/src/common/config" ) // InternalAPI handles request of harbor admin... @@ -69,3 +78,103 @@ func (ia *InternalAPI) RenameAdmin() { log.Debugf("The super user has been renamed to: %s", newName) ia.DestroySession() } + +// QuotaSwitcher ... +type QuotaSwitcher struct { + Enabled bool +} + +// SwitchQuota ... +func (ia *InternalAPI) SwitchQuota() { + var req QuotaSwitcher + if err := ia.DecodeJSONReq(&req); err != nil { + ia.SendBadRequestError(err) + return + } + // quota per project from disable to enable, it needs to update the quota usage bases on the DB records. + if !config.QuotaPerProjectEnable() && req.Enabled { + if err := ia.ensureQuota(); err != nil { + ia.SendInternalServerError(err) + return + } + } + defer func() { + config.GetCfgManager().Set(common.QuotaPerProjectEnable, req.Enabled) + config.GetCfgManager().Save() + }() + return +} + +func (ia *InternalAPI) ensureQuota() error { + projects, err := dao.GetProjects(nil) + if err != nil { + return err + } + for _, project := range projects { + pSize, err := dao.CountSizeOfProject(project.ProjectID) + if err != nil { + logger.Warningf("error happen on counting size of project:%d , error:%v, just skip it.", project.ProjectID, err) + continue + } + afQuery := &models.ArtifactQuery{ + PID: project.ProjectID, + } + afs, err := dao.ListArtifacts(afQuery) + if err != nil { + logger.Warningf("error happen on counting number of project:%d , error:%v, just skip it.", project.ProjectID, err) + continue + } + pCount := int64(len(afs)) + + // it needs to append the chart count + if config.WithChartMuseum() { + count, err := chartController.GetCountOfCharts([]string{project.Name}) + if err != nil { + err = errors.Wrap(err, fmt.Sprintf("get chart count of project %d failed", project.ProjectID)) + logger.Error(err) + continue + } + pCount = pCount + int64(count) + } + + quotaMgr, err := common_quota.NewManager("project", strconv.FormatInt(project.ProjectID, 10)) + if err != nil { + logger.Errorf("Error occurred when to new quota manager %v, just skip it.", err) + continue + } + used := common_quota.ResourceList{ + common_quota.ResourceStorage: pSize, + common_quota.ResourceCount: pCount, + } + if err := quotaMgr.EnsureQuota(used); err != nil { + logger.Errorf("cannot ensure quota for the project: %d, err: %v, just skip it.", project.ProjectID, err) + continue + } + } + return nil +} + +// SyncQuota ... +func (ia *InternalAPI) SyncQuota() { + cur := config.ReadOnly() + cfgMgr := comcfg.NewDBCfgManager() + if cur != true { + cfgMgr.Set(common.ReadOnly, true) + } + // For api call, to avoid the timeout, it should be asynchronous + go func() { + defer func() { + if cur != true { + cfgMgr.Set(common.ReadOnly, false) + } + }() + log.Info("start to sync quota(API), the system will be set to ReadOnly and back it normal once it done.") + err := quota.Sync(ia.ProjectMgr, false) + if err != nil { + log.Errorf("fail to sync quota(API), but with error: %v, please try to do it again.", err) + return + } + log.Info("success to sync quota(API).") + }() + return +} diff --git a/src/core/api/internal_test.go b/src/core/api/internal_test.go new file mode 100644 index 000000000..02903a98b --- /dev/null +++ b/src/core/api/internal_test.go @@ -0,0 +1,89 @@ +// Copyright 2018 Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "net/http" + "testing" +) + +// cannot verify the real scenario here +func TestSwitchQuota(t *testing.T) { + cases := []*codeCheckingCase{ + // 401 + { + request: &testingRequest{ + method: http.MethodPut, + url: "/api/internal/switchquota", + }, + code: http.StatusUnauthorized, + }, + // 200 + { + request: &testingRequest{ + method: http.MethodPut, + url: "/api/internal/switchquota", + credential: sysAdmin, + bodyJSON: &QuotaSwitcher{ + Enabled: true, + }, + }, + code: http.StatusOK, + }, + // 403 + { + request: &testingRequest{ + url: "/api/internal/switchquota", + method: http.MethodPut, + credential: nonSysAdmin, + }, + code: http.StatusForbidden, + }, + } + runCodeCheckingCases(t, cases...) +} + +// cannot verify the real scenario here +func TestSyncQuota(t *testing.T) { + cases := []*codeCheckingCase{ + // 401 + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/internal/syncquota", + }, + code: http.StatusUnauthorized, + }, + // 200 + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/internal/syncquota", + credential: sysAdmin, + }, + code: http.StatusOK, + }, + // 403 + { + request: &testingRequest{ + url: "/api/internal/syncquota", + method: http.MethodPost, + credential: nonSysAdmin, + }, + code: http.StatusForbidden, + }, + } + runCodeCheckingCases(t, cases...) +} diff --git a/src/core/api/label.go b/src/core/api/label.go index e91001f0f..eb2aaaf88 100644 --- a/src/core/api/label.go +++ b/src/core/api/label.go @@ -78,8 +78,7 @@ func (l *LabelAPI) requireAccess(label *models.Label, action rbac.Action, subres if len(subresources) == 0 { subresources = append(subresources, rbac.ResourceLabel) } - resource := rbac.NewProjectNamespace(label.ProjectID).Resource(subresources...) - hasPermission = l.SecurityCtx.Can(action, resource) + hasPermission, _ = l.HasProjectPermission(label.ProjectID, action, subresources...) } if !hasPermission { @@ -203,13 +202,7 @@ func (l *LabelAPI) List() { return } - resource := rbac.NewProjectNamespace(projectID).Resource(rbac.ResourceLabel) - if !l.SecurityCtx.Can(rbac.ActionList, resource) { - if !l.SecurityCtx.IsAuthenticated() { - l.SendUnAuthorizedError(errors.New("UnAuthorized")) - return - } - l.SendForbiddenError(errors.New(l.SecurityCtx.GetUsername())) + if !l.RequireProjectAccess(projectID, rbac.ActionList, rbac.ResourceLabel) { return } query.ProjectID = projectID diff --git a/src/core/api/metadata.go b/src/core/api/metadata.go index 5dd34b8c0..20cdd35dd 100644 --- a/src/core/api/metadata.go +++ b/src/core/api/metadata.go @@ -22,6 +22,7 @@ import ( "strings" "errors" + "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/utils/log" @@ -90,18 +91,7 @@ func (m *MetadataAPI) Prepare() { } func (m *MetadataAPI) requireAccess(action rbac.Action) bool { - resource := rbac.NewProjectNamespace(m.project.ProjectID).Resource(rbac.ResourceMetadata) - - if !m.SecurityCtx.Can(action, resource) { - if !m.SecurityCtx.IsAuthenticated() { - m.SendUnAuthorizedError(errors.New("Unauthorized")) - } else { - m.SendForbiddenError(errors.New(m.SecurityCtx.GetUsername())) - } - return false - } - - return true + return m.RequireProjectAccess(m.project.ProjectID, action, rbac.ResourceMetadata) } // Get ... diff --git a/src/core/api/notification_job.go b/src/core/api/notification_job.go index 775c9fc9f..ed1da61d6 100755 --- a/src/core/api/notification_job.go +++ b/src/core/api/notification_job.go @@ -93,16 +93,5 @@ func (w *NotificationJobAPI) validateRBAC(action rbac.Action, projectID int64) b return true } - project, err := w.ProjectMgr.Get(projectID) - if err != nil { - w.ParseAndHandleError(fmt.Sprintf("failed to get project %d", projectID), err) - return false - } - - resource := rbac.NewProjectNamespace(project.ProjectID).Resource(rbac.ResourceNotificationPolicy) - if !w.SecurityCtx.Can(action, resource) { - w.SendForbiddenError(errors.New(w.SecurityCtx.GetUsername())) - return false - } - return true + return w.RequireProjectAccess(projectID, action, rbac.ResourceNotificationPolicy) } diff --git a/src/core/api/notification_policy.go b/src/core/api/notification_policy.go index c7acdbea2..597e3e4f1 100755 --- a/src/core/api/notification_policy.go +++ b/src/core/api/notification_policy.go @@ -283,18 +283,7 @@ func (w *NotificationPolicyAPI) validateRBAC(action rbac.Action, projectID int64 return true } - project, err := w.ProjectMgr.Get(projectID) - if err != nil { - w.ParseAndHandleError(fmt.Sprintf("failed to get project %d", projectID), err) - return false - } - - resource := rbac.NewProjectNamespace(project.ProjectID).Resource(rbac.ResourceNotificationPolicy) - if !w.SecurityCtx.Can(action, resource) { - w.SendForbiddenError(errors.New(w.SecurityCtx.GetUsername())) - return false - } - return true + return w.RequireProjectAccess(projectID, action, rbac.ResourceNotificationPolicy) } func (w *NotificationPolicyAPI) validateTargets(policy *models.NotificationPolicy) bool { diff --git a/src/core/api/project.go b/src/core/api/project.go index 4a71dd316..77285453c 100644 --- a/src/core/api/project.go +++ b/src/core/api/project.go @@ -86,20 +86,8 @@ func (p *ProjectAPI) requireAccess(action rbac.Action, subresource ...rbac.Resou if len(subresource) == 0 { subresource = append(subresource, rbac.ResourceSelf) } - resource := rbac.NewProjectNamespace(p.project.ProjectID).Resource(subresource...) - if !p.SecurityCtx.Can(action, resource) { - if !p.SecurityCtx.IsAuthenticated() { - p.SendUnAuthorizedError(errors.New("Unauthorized")) - - } else { - p.SendForbiddenError(errors.New(p.SecurityCtx.GetUsername())) - } - - return false - } - - return true + return p.RequireProjectAccess(p.project.ProjectID, action, subresource...) } // Post ... @@ -139,23 +127,26 @@ func (p *ProjectAPI) Post() { return } - setting, err := config.QuotaSetting() - if err != nil { - log.Errorf("failed to get quota setting: %v", err) - p.SendInternalServerError(fmt.Errorf("failed to get quota setting: %v", err)) - return - } + var hardLimits types.ResourceList + if config.QuotaPerProjectEnable() { + setting, err := config.QuotaSetting() + if err != nil { + log.Errorf("failed to get quota setting: %v", err) + p.SendInternalServerError(fmt.Errorf("failed to get quota setting: %v", err)) + return + } - if !p.SecurityCtx.IsSysAdmin() { - pro.CountLimit = &setting.CountPerProject - pro.StorageLimit = &setting.StoragePerProject - } + if !p.SecurityCtx.IsSysAdmin() { + pro.CountLimit = &setting.CountPerProject + pro.StorageLimit = &setting.StoragePerProject + } - hardLimits, err := projectQuotaHardLimits(pro, setting) - if err != nil { - log.Errorf("Invalid project request, error: %v", err) - p.SendBadRequestError(fmt.Errorf("invalid request: %v", err)) - return + hardLimits, err = projectQuotaHardLimits(pro, setting) + if err != nil { + log.Errorf("Invalid project request, error: %v", err) + p.SendBadRequestError(fmt.Errorf("invalid request: %v", err)) + return + } } exist, err := p.ProjectMgr.Exists(pro.Name) @@ -212,14 +203,16 @@ func (p *ProjectAPI) Post() { return } - quotaMgr, err := quota.NewManager("project", strconv.FormatInt(projectID, 10)) - if err != nil { - p.SendInternalServerError(fmt.Errorf("failed to get quota manager: %v", err)) - return - } - if _, err := quotaMgr.NewQuota(hardLimits); err != nil { - p.SendInternalServerError(fmt.Errorf("failed to create quota for project: %v", err)) - return + if config.QuotaPerProjectEnable() { + quotaMgr, err := quota.NewManager("project", strconv.FormatInt(projectID, 10)) + if err != nil { + p.SendInternalServerError(fmt.Errorf("failed to get quota manager: %v", err)) + return + } + if _, err := quotaMgr.NewQuota(hardLimits); err != nil { + p.SendInternalServerError(fmt.Errorf("failed to create quota for project: %v", err)) + return + } } go func() { @@ -653,6 +646,11 @@ func projectQuotaHardLimits(req *models.ProjectRequest, setting *models.QuotaSet } func getProjectQuotaSummary(projectID int64, summary *models.ProjectSummary) { + if !config.QuotaPerProjectEnable() { + log.Debug("Quota per project disabled") + return + } + quotas, err := dao.ListQuotas(&models.QuotaQuery{Reference: "project", ReferenceID: strconv.FormatInt(projectID, 10)}) if err != nil { log.Debugf("failed to get quota for project: %d", projectID) diff --git a/src/core/api/project_test.go b/src/core/api/project_test.go index 2ff65b2fa..2c2d3d8fe 100644 --- a/src/core/api/project_test.go +++ b/src/core/api/project_test.go @@ -172,7 +172,7 @@ func TestListProjects(t *testing.T) { }() // ----------------------------case 1 : Response Code=200----------------------------// - fmt.Println("case 1: respose code:200") + fmt.Println("case 1: response code:200") httpStatusCode, result, err := apiTest.ProjectsGet( &apilib.ProjectQuery{ Name: addProject.ProjectName, @@ -263,7 +263,7 @@ func TestProGetByID(t *testing.T) { }() // ----------------------------case 1 : Response Code=200----------------------------// - fmt.Println("case 1: respose code:200") + fmt.Println("case 1: response code:200") httpStatusCode, result, err := apiTest.ProjectsGetByPID(projectID) if err != nil { t.Error("Error while search project by proID", err.Error()) @@ -295,7 +295,7 @@ func TestDeleteProject(t *testing.T) { } // --------------------------case 2: Response Code=200---------------------------------// - fmt.Println("case2: respose code:200") + fmt.Println("case2: response code:200") httpStatusCode, err = apiTest.ProjectsDelete(*admin, projectID) if err != nil { t.Error("Error while delete project", err.Error()) @@ -335,7 +335,7 @@ func TestProHead(t *testing.T) { apiTest := newHarborAPI() // ----------------------------case 1 : Response Code=200----------------------------// - fmt.Println("case 1: respose code:200") + fmt.Println("case 1: response code:200") httpStatusCode, err := apiTest.ProjectsHead(*admin, "library") if err != nil { t.Error("Error while search project by proName", err.Error()) @@ -345,7 +345,7 @@ func TestProHead(t *testing.T) { } // ----------------------------case 2 : Response Code=404:Project name does not exist.----------------------------// - fmt.Println("case 2: respose code:404,Project name does not exist.") + fmt.Println("case 2: response code:404,Project name does not exist.") httpStatusCode, err = apiTest.ProjectsHead(*admin, "libra") if err != nil { t.Error("Error while search project by proName", err.Error()) @@ -369,22 +369,22 @@ func TestPut(t *testing.T) { }, } - fmt.Println("case 1: respose code:200") + fmt.Println("case 1: response code:200") code, err := apiTest.ProjectsPut(*admin, "1", project) require.Nil(t, err) assert.Equal(int(200), code) - fmt.Println("case 2: respose code:401, User need to log in first.") + fmt.Println("case 2: response code:401, User need to log in first.") code, err = apiTest.ProjectsPut(*unknownUsr, "1", project) require.Nil(t, err) assert.Equal(int(401), code) - fmt.Println("case 3: respose code:400, Invalid project id") + fmt.Println("case 3: response code:400, Invalid project id") code, err = apiTest.ProjectsPut(*admin, "cc", project) require.Nil(t, err) assert.Equal(int(400), code) - fmt.Println("case 4: respose code:404, Not found the project") + fmt.Println("case 4: response code:404, Not found the project") code, err = apiTest.ProjectsPut(*admin, "1234", project) require.Nil(t, err) assert.Equal(int(404), code) @@ -407,7 +407,7 @@ func TestProjectLogsFilter(t *testing.T) { } // -------------------case1: Response Code=200------------------------------// - fmt.Println("case 1: respose code:200") + fmt.Println("case 1: response code:200") projectID := "1" httpStatusCode, _, err := apiTest.ProjectLogs(*admin, projectID, query) if err != nil { @@ -417,7 +417,7 @@ func TestProjectLogsFilter(t *testing.T) { assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") } // -------------------case2: Response Code=401:User need to log in first.------------------------------// - fmt.Println("case 2: respose code:401:User need to log in first.") + fmt.Println("case 2: response code:401:User need to log in first.") projectID = "1" httpStatusCode, _, err = apiTest.ProjectLogs(*unknownUsr, projectID, query) if err != nil { @@ -427,7 +427,7 @@ func TestProjectLogsFilter(t *testing.T) { assert.Equal(int(401), httpStatusCode, "httpStatusCode should be 401") } // -------------------case3: Response Code=404:Project does not exist.-------------------------// - fmt.Println("case 3: respose code:404:Illegal format of provided ID value.") + fmt.Println("case 3: response code:404:Illegal format of provided ID value.") projectID = "11111" httpStatusCode, _, err = apiTest.ProjectLogs(*admin, projectID, query) if err != nil { @@ -498,7 +498,7 @@ func TestProjectSummary(t *testing.T) { }() // ----------------------------case 1 : Response Code=200----------------------------// - fmt.Println("case 1: respose code:200") + fmt.Println("case 1: response code:200") httpStatusCode, summary, err := apiTest.ProjectSummary(*admin, fmt.Sprintf("%d", projectID)) if err != nil { t.Error("Error while search project by proName", err.Error()) diff --git a/src/core/api/projectmember.go b/src/core/api/projectmember.go index 5495ac26a..c836016f7 100644 --- a/src/core/api/projectmember.go +++ b/src/core/api/projectmember.go @@ -99,19 +99,7 @@ func (pma *ProjectMemberAPI) Prepare() { } func (pma *ProjectMemberAPI) requireAccess(action rbac.Action) bool { - resource := rbac.NewProjectNamespace(pma.project.ProjectID).Resource(rbac.ResourceMember) - - if !pma.SecurityCtx.Can(action, resource) { - if !pma.SecurityCtx.IsAuthenticated() { - pma.SendUnAuthorizedError(errors.New("Unauthorized")) - } else { - pma.SendForbiddenError(errors.New(pma.SecurityCtx.GetUsername())) - } - - return false - } - - return true + return pma.RequireProjectAccess(pma.project.ProjectID, action, rbac.ResourceMember) } // Get ... diff --git a/src/core/api/quota/chart/chart.go b/src/core/api/quota/chart/chart.go new file mode 100644 index 000000000..f3ebc1f11 --- /dev/null +++ b/src/core/api/quota/chart/chart.go @@ -0,0 +1,226 @@ +// Copyright 2018 Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chart + +import ( + "fmt" + "github.com/goharbor/harbor/src/chartserver" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + common_quota "github.com/goharbor/harbor/src/common/quota" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/api" + quota "github.com/goharbor/harbor/src/core/api/quota" + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/core/promgr" + "github.com/pkg/errors" + "net/url" + "strings" + "sync" +) + +// Migrator ... +type Migrator struct { + pm promgr.ProjectManager +} + +// NewChartMigrator returns a new RegistryMigrator. +func NewChartMigrator(pm promgr.ProjectManager) quota.QuotaMigrator { + migrator := Migrator{ + pm: pm, + } + return &migrator +} + +var ( + controller *chartserver.Controller + controllerErr error + controllerOnce sync.Once +) + +// Ping ... +func (rm *Migrator) Ping() error { + return api.HealthCheckerRegistry["chartmuseum"].Check() +} + +// Dump ... +// Depends on DB to dump chart data, as chart cannot get all of namespaces. +func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) { + var ( + projects []quota.ProjectInfo + wg sync.WaitGroup + err error + ) + + all, err := dao.GetProjects(nil) + if err != nil { + return nil, err + } + + wg.Add(len(all)) + errChan := make(chan error, 1) + infoChan := make(chan interface{}) + done := make(chan bool, 1) + + go func() { + defer func() { + done <- true + }() + + for { + select { + case result := <-infoChan: + if result == nil { + return + } + project, ok := result.(quota.ProjectInfo) + if ok { + projects = append(projects, project) + } + + case e := <-errChan: + if err == nil { + err = errors.Wrap(e, "quota sync error on getting info of project") + } else { + err = errors.Wrap(e, err.Error()) + } + } + } + }() + + for _, project := range all { + go func(project *models.Project) { + defer wg.Done() + + var repos []quota.RepoData + ctr, err := chartController() + if err != nil { + errChan <- err + return + } + + chartInfo, err := ctr.ListCharts(project.Name) + if err != nil { + errChan <- err + return + } + + // repo + for _, chart := range chartInfo { + var afs []*models.Artifact + chartVersions, err := ctr.GetChart(project.Name, chart.Name) + if err != nil { + errChan <- err + continue + } + for _, chart := range chartVersions { + af := &models.Artifact{ + PID: project.ProjectID, + Repo: chart.Name, + Tag: chart.Version, + Digest: chart.Digest, + Kind: "Chart", + } + afs = append(afs, af) + } + repoData := quota.RepoData{ + Name: project.Name, + Afs: afs, + } + repos = append(repos, repoData) + } + + projectInfo := quota.ProjectInfo{ + Name: project.Name, + Repos: repos, + } + + infoChan <- projectInfo + }(project) + } + + wg.Wait() + close(infoChan) + + <-done + + if err != nil { + return nil, err + } + + return projects, nil +} + +// Usage ... +// Chart will not cover size. +func (rm *Migrator) Usage(projects []quota.ProjectInfo) ([]quota.ProjectUsage, error) { + var pros []quota.ProjectUsage + for _, project := range projects { + var count int64 + // usage count + for _, repo := range project.Repos { + count = count + int64(len(repo.Afs)) + } + proUsage := quota.ProjectUsage{ + Project: project.Name, + Used: common_quota.ResourceList{ + common_quota.ResourceCount: count, + common_quota.ResourceStorage: 0, + }, + } + pros = append(pros, proUsage) + } + return pros, nil + +} + +// Persist ... +// Chart will not persist data into db. +func (rm *Migrator) Persist(projects []quota.ProjectInfo) error { + return nil +} + +func chartController() (*chartserver.Controller, error) { + controllerOnce.Do(func() { + addr, err := config.GetChartMuseumEndpoint() + if err != nil { + controllerErr = fmt.Errorf("failed to get the endpoint URL of chart storage server: %s", err.Error()) + return + } + + addr = strings.TrimSuffix(addr, "/") + url, err := url.Parse(addr) + if err != nil { + controllerErr = errors.New("endpoint URL of chart storage server is malformed") + return + } + + ctr, err := chartserver.NewController(url) + if err != nil { + controllerErr = errors.New("failed to initialize chart API controller") + } + + controller = ctr + + log.Debugf("Chart storage server is set to %s", url.String()) + log.Info("API controller for chart repository server is successfully initialized") + }) + + return controller, controllerErr +} + +func init() { + quota.Register("chart", NewChartMigrator) +} diff --git a/src/core/api/quota/migrator.go b/src/core/api/quota/migrator.go new file mode 100644 index 000000000..bfd2fc164 --- /dev/null +++ b/src/core/api/quota/migrator.go @@ -0,0 +1,173 @@ +// Copyright 2018 Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/quota" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/core/promgr" + "github.com/goharbor/harbor/src/pkg/types" + "strconv" +) + +// QuotaMigrator ... +type QuotaMigrator interface { + // Ping validates and wait for backend service ready. + Ping() error + + // Dump exports all data from backend service, registry, chartmuseum + Dump() ([]ProjectInfo, error) + + // Usage computes the quota usage of all the projects + Usage([]ProjectInfo) ([]ProjectUsage, error) + + // Persist record the data to DB, artifact, artifact_blob and blob tabel. + Persist([]ProjectInfo) error +} + +// ProjectInfo ... +type ProjectInfo struct { + Name string + Repos []RepoData +} + +// RepoData ... +type RepoData struct { + Name string + Afs []*models.Artifact + Afnbs []*models.ArtifactAndBlob + Blobs []*models.Blob +} + +// ProjectUsage ... +type ProjectUsage struct { + Project string + Used quota.ResourceList +} + +// Instance ... +type Instance func(promgr.ProjectManager) QuotaMigrator + +var adapters = make(map[string]Instance) + +// Register ... +func Register(name string, adapter Instance) { + if adapter == nil { + panic("quota: Register adapter is nil") + } + if _, ok := adapters[name]; ok { + panic("quota: Register called twice for adapter " + name) + } + adapters[name] = adapter +} + +// Sync ... +func Sync(pm promgr.ProjectManager, populate bool) error { + totalUsage := make(map[string][]ProjectUsage) + for name, instanceFunc := range adapters { + if !config.WithChartMuseum() { + if name == "chart" { + continue + } + } + adapter := instanceFunc(pm) + if err := adapter.Ping(); err != nil { + return err + } + data, err := adapter.Dump() + if err != nil { + return err + } + usage, err := adapter.Usage(data) + if err != nil { + return err + } + totalUsage[name] = usage + if populate { + if err := adapter.Persist(data); err != nil { + return err + } + } + } + merged := mergeUsage(totalUsage) + if err := ensureQuota(merged); err != nil { + return err + } + return nil +} + +// mergeUsage merges the usage of adapters +func mergeUsage(total map[string][]ProjectUsage) []ProjectUsage { + if !config.WithChartMuseum() { + return total["registry"] + } + regUsgs := total["registry"] + chartUsgs := total["chart"] + + var mergedUsage []ProjectUsage + temp := make(map[string]quota.ResourceList) + + for _, regUsg := range regUsgs { + _, exist := temp[regUsg.Project] + if !exist { + temp[regUsg.Project] = regUsg.Used + mergedUsage = append(mergedUsage, ProjectUsage{ + Project: regUsg.Project, + Used: regUsg.Used, + }) + } + } + for _, chartUsg := range chartUsgs { + var usedTemp quota.ResourceList + _, exist := temp[chartUsg.Project] + if !exist { + usedTemp = chartUsg.Used + } else { + usedTemp = types.Add(temp[chartUsg.Project], chartUsg.Used) + } + temp[chartUsg.Project] = usedTemp + mergedUsage = append(mergedUsage, ProjectUsage{ + Project: chartUsg.Project, + Used: usedTemp, + }) + } + return mergedUsage +} + +// ensureQuota updates the quota and quota usage in the data base. +func ensureQuota(usages []ProjectUsage) error { + var pid int64 + for _, usage := range usages { + project, err := dao.GetProjectByName(usage.Project) + if err != nil { + log.Error(err) + return err + } + pid = project.ProjectID + quotaMgr, err := quota.NewManager("project", strconv.FormatInt(pid, 10)) + if err != nil { + log.Errorf("Error occurred when to new quota manager %v", err) + return err + } + if err := quotaMgr.EnsureQuota(usage.Used); err != nil { + log.Errorf("cannot ensure quota for the project: %d, err: %v", pid, err) + return err + } + } + return nil +} diff --git a/src/core/api/quota/registry/registry.go b/src/core/api/quota/registry/registry.go new file mode 100644 index 000000000..e9c2608e4 --- /dev/null +++ b/src/core/api/quota/registry/registry.go @@ -0,0 +1,436 @@ +// Copyright 2018 Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/distribution/manifest/schema2" + + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + common_quota "github.com/goharbor/harbor/src/common/quota" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/common/utils/registry" + "github.com/goharbor/harbor/src/core/api" + quota "github.com/goharbor/harbor/src/core/api/quota" + "github.com/goharbor/harbor/src/core/promgr" + coreutils "github.com/goharbor/harbor/src/core/utils" + "github.com/pkg/errors" + "strings" + "sync" + "time" +) + +// Migrator ... +type Migrator struct { + pm promgr.ProjectManager +} + +// NewRegistryMigrator returns a new Migrator. +func NewRegistryMigrator(pm promgr.ProjectManager) quota.QuotaMigrator { + migrator := Migrator{ + pm: pm, + } + return &migrator +} + +// Ping ... +func (rm *Migrator) Ping() error { + return api.HealthCheckerRegistry["registry"].Check() +} + +// Dump ... +func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) { + var ( + projects []quota.ProjectInfo + wg sync.WaitGroup + err error + ) + + reposInRegistry, err := api.Catalog() + if err != nil { + return nil, err + } + + // repoMap : map[project_name : []repo list] + repoMap := make(map[string][]string) + for _, item := range reposInRegistry { + projectName := strings.Split(item, "/")[0] + pro, err := rm.pm.Get(projectName) + if err != nil { + log.Errorf("failed to get project %s: %v", projectName, err) + continue + } + if pro == nil { + continue + } + _, exist := repoMap[pro.Name] + if !exist { + repoMap[pro.Name] = []string{item} + } else { + repos := repoMap[pro.Name] + repos = append(repos, item) + repoMap[pro.Name] = repos + } + } + + wg.Add(len(repoMap)) + errChan := make(chan error, 1) + infoChan := make(chan interface{}) + done := make(chan bool, 1) + + go func() { + defer func() { + done <- true + }() + + for { + select { + case result := <-infoChan: + if result == nil { + return + } + project, ok := result.(quota.ProjectInfo) + if ok { + projects = append(projects, project) + } + + case e := <-errChan: + if err == nil { + err = errors.Wrap(e, "quota sync error on getting info of project") + } else { + err = errors.Wrap(e, err.Error()) + } + } + } + }() + + for project, repos := range repoMap { + go func(project string, repos []string) { + defer wg.Done() + info, err := infoOfProject(project, repos) + if err != nil { + errChan <- err + return + } + infoChan <- info + }(project, repos) + } + + wg.Wait() + close(infoChan) + + // wait for all of project info + <-done + + if err != nil { + return nil, err + } + + return projects, nil +} + +// Usage ... +// registry needs to merge the shard blobs of different repositories. +func (rm *Migrator) Usage(projects []quota.ProjectInfo) ([]quota.ProjectUsage, error) { + var pros []quota.ProjectUsage + + for _, project := range projects { + var size, count int64 + var blobs = make(map[string]int64) + + // usage count + for _, repo := range project.Repos { + count = count + int64(len(repo.Afs)) + // Because that there are some shared blobs between repositories, it needs to remove the duplicate items. + for _, blob := range repo.Blobs { + _, exist := blobs[blob.Digest] + if !exist { + blobs[blob.Digest] = blob.Size + } + } + } + // size + for _, item := range blobs { + size = size + item + } + + proUsage := quota.ProjectUsage{ + Project: project.Name, + Used: common_quota.ResourceList{ + common_quota.ResourceCount: count, + common_quota.ResourceStorage: size, + }, + } + pros = append(pros, proUsage) + } + + return pros, nil +} + +// Persist ... +func (rm *Migrator) Persist(projects []quota.ProjectInfo) error { + for _, project := range projects { + for _, repo := range project.Repos { + if err := persistAf(repo.Afs); err != nil { + return err + } + if err := persistAfnbs(repo.Afnbs); err != nil { + return err + } + if err := persistBlob(repo.Blobs); err != nil { + return err + } + } + } + if err := persistPB(projects); err != nil { + return err + } + return nil +} + +func persistAf(afs []*models.Artifact) error { + if len(afs) != 0 { + for _, af := range afs { + _, err := dao.AddArtifact(af) + if err != nil { + if err == dao.ErrDupRows { + continue + } + log.Error(err) + return err + } + } + } + return nil +} + +func persistAfnbs(afnbs []*models.ArtifactAndBlob) error { + if len(afnbs) != 0 { + for _, afnb := range afnbs { + _, err := dao.AddArtifactNBlob(afnb) + if err != nil { + if err == dao.ErrDupRows { + continue + } + log.Error(err) + return err + } + } + } + return nil +} + +func persistBlob(blobs []*models.Blob) error { + if len(blobs) != 0 { + for _, blob := range blobs { + _, err := dao.AddBlob(blob) + if err != nil { + if err == dao.ErrDupRows { + continue + } + log.Error(err) + return err + } + } + } + return nil +} + +func persistPB(projects []quota.ProjectInfo) error { + for _, project := range projects { + var blobs = make(map[string]int64) + var blobsOfPro []*models.Blob + for _, repo := range project.Repos { + for _, blob := range repo.Blobs { + _, exist := blobs[blob.Digest] + if exist { + continue + } + blobs[blob.Digest] = blob.Size + blobInDB, err := dao.GetBlob(blob.Digest) + if err != nil { + log.Error(err) + return err + } + if blobInDB != nil { + blobsOfPro = append(blobsOfPro, blobInDB) + } + } + } + pro, err := dao.GetProjectByName(project.Name) + if err != nil { + log.Error(err) + return err + } + _, err = dao.AddBlobsToProject(pro.ProjectID, blobsOfPro...) + if err != nil { + log.Error(err) + return err + } + } + return nil +} + +func infoOfProject(project string, repoList []string) (quota.ProjectInfo, error) { + var ( + repos []quota.RepoData + wg sync.WaitGroup + err error + ) + wg.Add(len(repoList)) + + errChan := make(chan error, 1) + infoChan := make(chan interface{}) + done := make(chan bool, 1) + + pro, err := dao.GetProjectByName(project) + if err != nil { + log.Error(err) + return quota.ProjectInfo{}, err + } + + go func() { + defer func() { + done <- true + }() + + for { + select { + case result := <-infoChan: + if result == nil { + return + } + repoData, ok := result.(quota.RepoData) + if ok { + repos = append(repos, repoData) + } + + case e := <-errChan: + if err == nil { + err = errors.Wrap(e, "quota sync error on getting info of repo") + } else { + err = errors.Wrap(e, err.Error()) + } + } + } + }() + + for _, repo := range repoList { + go func(pid int64, repo string) { + defer func() { + wg.Done() + }() + info, err := infoOfRepo(pid, repo) + if err != nil { + errChan <- err + return + } + infoChan <- info + }(pro.ProjectID, repo) + } + + wg.Wait() + close(infoChan) + + <-done + + if err != nil { + return quota.ProjectInfo{}, err + } + + return quota.ProjectInfo{ + Name: project, + Repos: repos, + }, nil +} + +func infoOfRepo(pid int64, repo string) (quota.RepoData, error) { + repoClient, err := coreutils.NewRepositoryClientForUI("harbor-core", repo) + if err != nil { + return quota.RepoData{}, err + } + tags, err := repoClient.ListTag() + if err != nil { + return quota.RepoData{}, err + } + var afnbs []*models.ArtifactAndBlob + var afs []*models.Artifact + var blobs []*models.Blob + + for _, tag := range tags { + _, mediaType, payload, err := repoClient.PullManifest(tag, []string{ + schema1.MediaTypeManifest, + schema1.MediaTypeSignedManifest, + schema2.MediaTypeManifest, + }) + if err != nil { + log.Error(err) + return quota.RepoData{}, err + } + manifest, desc, err := registry.UnMarshal(mediaType, payload) + if err != nil { + log.Error(err) + return quota.RepoData{}, err + } + // self + afnb := &models.ArtifactAndBlob{ + DigestAF: desc.Digest.String(), + DigestBlob: desc.Digest.String(), + } + afnbs = append(afnbs, afnb) + // add manifest as a blob. + blob := &models.Blob{ + Digest: desc.Digest.String(), + ContentType: desc.MediaType, + Size: desc.Size, + CreationTime: time.Now(), + } + blobs = append(blobs, blob) + for _, layer := range manifest.References() { + afnb := &models.ArtifactAndBlob{ + DigestAF: desc.Digest.String(), + DigestBlob: layer.Digest.String(), + } + afnbs = append(afnbs, afnb) + blob := &models.Blob{ + Digest: layer.Digest.String(), + ContentType: layer.MediaType, + Size: layer.Size, + CreationTime: time.Now(), + } + blobs = append(blobs, blob) + } + af := &models.Artifact{ + PID: pid, + Repo: strings.Split(repo, "/")[1], + Tag: tag, + Digest: desc.Digest.String(), + Kind: "Docker-Image", + CreationTime: time.Now(), + } + afs = append(afs, af) + } + return quota.RepoData{ + Name: repo, + Afs: afs, + Afnbs: afnbs, + Blobs: blobs, + }, nil +} + +func init() { + quota.Register("registry", NewRegistryMigrator) +} diff --git a/src/core/api/repository.go b/src/core/api/repository.go index bea194509..b7219dd59 100755 --- a/src/core/api/repository.go +++ b/src/core/api/repository.go @@ -111,13 +111,7 @@ func (ra *RepositoryAPI) Get() { return } - resource := rbac.NewProjectNamespace(projectID).Resource(rbac.ResourceRepository) - if !ra.SecurityCtx.Can(rbac.ActionList, resource) { - if !ra.SecurityCtx.IsAuthenticated() { - ra.SendUnAuthorizedError(errors.New("Unauthorized")) - return - } - ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername())) + if !ra.RequireProjectAccess(projectID, rbac.ActionList, rbac.ResourceRepository) { return } @@ -228,14 +222,8 @@ func (ra *RepositoryAPI) Delete() { return } - if !ra.SecurityCtx.IsAuthenticated() { - ra.SendUnAuthorizedError(errors.New("UnAuthorized")) - return - } - - resource := rbac.NewProjectNamespace(project.ProjectID).Resource(rbac.ResourceRepository) - if !ra.SecurityCtx.Can(rbac.ActionDelete, resource) { - ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername())) + if !ra.RequireAuthenticated() || + !ra.RequireProjectAccess(project.ProjectID, rbac.ActionDelete, rbac.ResourceRepository) { return } @@ -403,14 +391,9 @@ func (ra *RepositoryAPI) GetTag() { ra.SendNotFoundError(fmt.Errorf("resource: %s:%s not found", repository, tag)) return } - project, _ := utils.ParseRepository(repository) - resource := rbac.NewProjectNamespace(project).Resource(rbac.ResourceRepositoryTag) - if !ra.SecurityCtx.Can(rbac.ActionRead, resource) { - if !ra.SecurityCtx.IsAuthenticated() { - ra.SendUnAuthorizedError(errors.New("UnAuthorized")) - return - } - ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername())) + + projectName, _ := utils.ParseRepository(repository) + if !ra.RequireProjectAccess(projectName, rbac.ActionRead, rbac.ResourceRepositoryTag) { return } @@ -503,16 +486,14 @@ func (ra *RepositoryAPI) Retag() { } // Check whether user has read permission to source project - srcResource := rbac.NewProjectNamespace(srcImage.Project).Resource(rbac.ResourceRepository) - if !ra.SecurityCtx.Can(rbac.ActionPull, srcResource) { + if hasPermission, _ := ra.HasProjectPermission(srcImage.Project, rbac.ActionPull, rbac.ResourceRepository); !hasPermission { log.Errorf("user has no read permission to project '%s'", srcImage.Project) ra.SendForbiddenError(fmt.Errorf("%s has no read permission to project %s", ra.SecurityCtx.GetUsername(), srcImage.Project)) return } // Check whether user has write permission to target project - destResource := rbac.NewProjectNamespace(project).Resource(rbac.ResourceRepository) - if !ra.SecurityCtx.Can(rbac.ActionPush, destResource) { + if hasPermission, _ := ra.HasProjectPermission(project, rbac.ActionPush, rbac.ResourceRepository); !hasPermission { log.Errorf("user has no write permission to project '%s'", project) ra.SendForbiddenError(fmt.Errorf("%s has no write permission to project %s", ra.SecurityCtx.GetUsername(), project)) return @@ -550,13 +531,7 @@ func (ra *RepositoryAPI) GetTags() { return } - resource := rbac.NewProjectNamespace(projectName).Resource(rbac.ResourceRepositoryTag) - if !ra.SecurityCtx.Can(rbac.ActionList, resource) { - if !ra.SecurityCtx.IsAuthenticated() { - ra.SendUnAuthorizedError(errors.New("UnAuthorized")) - return - } - ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername())) + if !ra.RequireProjectAccess(projectName, rbac.ActionList, rbac.ResourceRepositoryTag) { return } @@ -585,7 +560,12 @@ func (ra *RepositoryAPI) GetTags() { } labeledTags := map[string]struct{}{} for _, rl := range rls { - labeledTags[strings.Split(rl.ResourceName, ":")[1]] = struct{}{} + strs := strings.SplitN(rl.ResourceName, ":", 2) + // the "rls" may contain images which don't belong to the repository + if strs[0] != repoName { + continue + } + labeledTags[strs[1]] = struct{}{} } ts := []string{} for _, tag := range tags { @@ -596,11 +576,31 @@ func (ra *RepositoryAPI) GetTags() { tags = ts } + detail, err := ra.GetBool("detail", true) + if !detail && err == nil { + ra.Data["json"] = simpleTags(tags) + ra.ServeJSON() + return + } + ra.Data["json"] = assembleTagsInParallel(client, repoName, tags, ra.SecurityCtx.GetUsername()) ra.ServeJSON() } +func simpleTags(tags []string) []*models.TagResp { + var tagsResp []*models.TagResp + for _, tag := range tags { + tagsResp = append(tagsResp, &models.TagResp{ + TagDetail: models.TagDetail{ + Name: tag, + }, + }) + } + + return tagsResp +} + // get config, signature and scan overview and assemble them into one // struct for each tag in tags func assembleTagsInParallel(client *registry.Repository, repository string, @@ -791,14 +791,7 @@ func (ra *RepositoryAPI) GetManifests() { return } - resource := rbac.NewProjectNamespace(projectName).Resource(rbac.ResourceRepositoryTagManifest) - if !ra.SecurityCtx.Can(rbac.ActionRead, resource) { - if !ra.SecurityCtx.IsAuthenticated() { - ra.SendUnAuthorizedError(errors.New("Unauthorized")) - return - } - - ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername())) + if !ra.RequireProjectAccess(projectName, rbac.ActionRead, rbac.ResourceRepositoryTagManifest) { return } @@ -919,10 +912,8 @@ func (ra *RepositoryAPI) Put() { return } - project, _ := utils.ParseRepository(name) - resource := rbac.NewProjectNamespace(project).Resource(rbac.ResourceRepository) - if !ra.SecurityCtx.Can(rbac.ActionUpdate, resource) { - ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername())) + projectName, _ := utils.ParseRepository(name) + if !ra.RequireProjectAccess(projectName, rbac.ActionUpdate, rbac.ResourceRepository) { return } @@ -958,13 +949,7 @@ func (ra *RepositoryAPI) GetSignatures() { return } - resource := rbac.NewProjectNamespace(projectName).Resource(rbac.ResourceRepository) - if !ra.SecurityCtx.Can(rbac.ActionRead, resource) { - if !ra.SecurityCtx.IsAuthenticated() { - ra.SendUnAuthorizedError(errors.New("Unauthorized")) - return - } - ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername())) + if !ra.RequireProjectAccess(projectName, rbac.ActionRead, rbac.ResourceRepository) { return } @@ -1004,9 +989,7 @@ func (ra *RepositoryAPI) ScanImage() { return } - resource := rbac.NewProjectNamespace(projectName).Resource(rbac.ResourceRepositoryTagScanJob) - if !ra.SecurityCtx.Can(rbac.ActionCreate, resource) { - ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername())) + if !ra.RequireProjectAccess(projectName, rbac.ActionCreate, rbac.ResourceRepositoryTagScanJob) { return } err = coreutils.TriggerImageScan(repoName, tag) @@ -1035,15 +1018,9 @@ func (ra *RepositoryAPI) VulnerabilityDetails() { ra.SendNotFoundError(fmt.Errorf("resource: %s:%s not found", repository, tag)) return } - project, _ := utils.ParseRepository(repository) - resource := rbac.NewProjectNamespace(project).Resource(rbac.ResourceRepositoryTagVulnerability) - if !ra.SecurityCtx.Can(rbac.ActionList, resource) { - if !ra.SecurityCtx.IsAuthenticated() { - ra.SendUnAuthorizedError(errors.New("Unauthorized")) - return - } - ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername())) + projectName, _ := utils.ParseRepository(repository) + if !ra.RequireProjectAccess(projectName, rbac.ActionList, rbac.ResourceRepositoryTagVulnerability) { return } res, err := scan.VulnListByDigest(digest) diff --git a/src/core/api/repository_label.go b/src/core/api/repository_label.go index 53c606507..3d565b0da 100644 --- a/src/core/api/repository_label.go +++ b/src/core/api/repository_label.go @@ -91,19 +91,8 @@ func (r *RepositoryLabelAPI) requireAccess(action rbac.Action, subresource ...rb if len(subresource) == 0 { subresource = append(subresource, rbac.ResourceRepositoryLabel) } - resource := rbac.NewProjectNamespace(r.repository.ProjectID).Resource(rbac.ResourceRepositoryLabel) - if !r.SecurityCtx.Can(action, resource) { - if !r.SecurityCtx.IsAuthenticated() { - r.SendUnAuthorizedError(errors.New("UnAuthorized")) - } else { - r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername())) - } - - return false - } - - return true + return r.RequireProjectAccess(r.repository.ProjectID, action, subresource...) } func (r *RepositoryLabelAPI) isValidLabelReq() bool { diff --git a/src/core/api/retention.go b/src/core/api/retention.go index 885c0de70..f5d7d026f 100644 --- a/src/core/api/retention.go +++ b/src/core/api/retention.go @@ -67,25 +67,19 @@ func (r *RetentionAPI) GetMetadatas() { ] }, { - "rule_template": "nothing", - "display_text": "none", - "action": "retain", - "params": [] - }, - { - "rule_template": "always", - "display_text": "always", - "action": "retain", - "params": [ - { - "type": "int", - "unit": "COUNT", - "required": true - } - ] - }, + "rule_template": "nDaysSinceLastPush", + "display_text": "pushed within the last # days", + "action": "retain", + "params": [ + { + "type": "int", + "unit": "DAYS", + "required": true + } + ] + }, { - "rule_template": "dayspl", + "rule_template": "nDaysSinceLastPull", "display_text": "pulled within the last # days", "action": "retain", "params": [ @@ -97,17 +91,11 @@ func (r *RetentionAPI) GetMetadatas() { ] }, { - "rule_template": "daysps", - "display_text": "pushed within the last # days", - "action": "retain", - "params": [ - { - "type": "int", - "unit": "DAYS", - "required": true - } - ] - } + "rule_template": "always", + "display_text": "always", + "action": "retain", + "params": [] + } ], "scope_selectors": [ { @@ -120,14 +108,6 @@ func (r *RetentionAPI) GetMetadatas() { } ], "tag_selectors": [ - { - "display_text": "Labels", - "kind": "label", - "decorations": [ - "withLabels", - "withoutLabels" - ] - }, { "display_text": "Tags", "kind": "doublestar", @@ -244,7 +224,7 @@ func (r *RetentionAPI) checkRuleConflict(p *policy.Metadata) error { if old, exists := temp[string(bs)]; exists { return fmt.Errorf("rule %d is conflict with rule %d", n, old) } - temp[string(bs)] = tid + temp[string(bs)] = n rule.ID = tid } return nil @@ -424,8 +404,7 @@ func (r *RetentionAPI) requireAccess(p *policy.Metadata, action rbac.Action, sub if len(subresources) == 0 { subresources = append(subresources, rbac.ResourceTagRetention) } - resource := rbac.NewProjectNamespace(p.Scope.Reference).Resource(subresources...) - hasPermission = r.SecurityCtx.Can(action, resource) + hasPermission, _ = r.HasProjectPermission(p.Scope.Reference, action, subresources...) default: hasPermission = r.SecurityCtx.IsSysAdmin() } diff --git a/src/core/api/robot.go b/src/core/api/robot.go index be49983a4..d098c4059 100644 --- a/src/core/api/robot.go +++ b/src/core/api/robot.go @@ -17,16 +17,16 @@ package api import ( "errors" "fmt" + "net/http" + "strconv" + "time" + "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/token" - "net/http" - "strconv" - "github.com/goharbor/harbor/src/core/config" - "time" ) // RobotAPI ... @@ -91,13 +91,7 @@ func (r *RobotAPI) Prepare() { } func (r *RobotAPI) requireAccess(action rbac.Action) bool { - resource := rbac.NewProjectNamespace(r.project.ProjectID).Resource(rbac.ResourceRobot) - if !r.SecurityCtx.Can(action, resource) { - r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername())) - return false - } - - return true + return r.RequireProjectAccess(r.project.ProjectID, action, rbac.ResourceRobot) } // Post ... diff --git a/src/core/api/scan_job.go b/src/core/api/scan_job.go index 446611ccf..7cc38d61e 100644 --- a/src/core/api/scan_job.go +++ b/src/core/api/scan_job.go @@ -55,12 +55,10 @@ func (sj *ScanJobAPI) Prepare() { sj.SendInternalServerError(errors.New("Failed to get Job data")) return } - projectName := strings.SplitN(data.Repository, "/", 2)[0] - resource := rbac.NewProjectNamespace(projectName).Resource(rbac.ResourceRepositoryTagScanJob) - if !sj.SecurityCtx.Can(rbac.ActionRead, resource) { + projectName := strings.SplitN(data.Repository, "/", 2)[0] + if !sj.RequireProjectAccess(projectName, rbac.ActionRead, rbac.ResourceRepositoryTagScanJob) { log.Errorf("User does not have read permission for project: %s", projectName) - sj.SendForbiddenError(errors.New(sj.SecurityCtx.GetUsername())) return } sj.projectName = projectName diff --git a/src/core/api/user_test.go b/src/core/api/user_test.go index 0c2bbc519..a666a4d7b 100644 --- a/src/core/api/user_test.go +++ b/src/core/api/user_test.go @@ -612,7 +612,7 @@ func TestUsersCurrentPermissions(t *testing.T) { assert := assert.New(t) apiTest := newHarborAPI() - httpStatusCode, permissions, err := apiTest.UsersGetPermissions("current", "/project/library", *projAdmin) + httpStatusCode, permissions, err := apiTest.UsersGetPermissions("current", "/project/1", *projAdmin) assert.Nil(err) assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") assert.NotEmpty(permissions, "permissions should not be empty") @@ -622,11 +622,11 @@ func TestUsersCurrentPermissions(t *testing.T) { assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") assert.Empty(permissions, "permissions should be empty") - httpStatusCode, _, err = apiTest.UsersGetPermissions(projAdminID, "/project/library", *projAdmin) + httpStatusCode, _, err = apiTest.UsersGetPermissions(projAdminID, "/project/1", *projAdmin) assert.Nil(err) assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") - httpStatusCode, _, err = apiTest.UsersGetPermissions(projDeveloperID, "/project/library", *projAdmin) + httpStatusCode, _, err = apiTest.UsersGetPermissions(projDeveloperID, "/project/1", *projAdmin) assert.Nil(err) assert.Equal(int(403), httpStatusCode, "httpStatusCode should be 403") } diff --git a/src/core/api/utils.go b/src/core/api/utils.go index 4b7305524..4fd20d383 100644 --- a/src/core/api/utils.go +++ b/src/core/api/utils.go @@ -38,7 +38,7 @@ func SyncRegistry(pm promgr.ProjectManager) error { log.Infof("Start syncing repositories from registry to DB... ") - reposInRegistry, err := catalog() + reposInRegistry, err := Catalog() if err != nil { log.Error(err) return err @@ -105,7 +105,8 @@ func SyncRegistry(pm promgr.ProjectManager) error { return nil } -func catalog() ([]string, error) { +// Catalog ... +func Catalog() ([]string, error) { repositories := []string{} rc, err := initRegistryClient() diff --git a/src/core/auth/authproxy/auth.go b/src/core/auth/authproxy/auth.go index f3efae49d..1efe42f3e 100644 --- a/src/core/auth/authproxy/auth.go +++ b/src/core/auth/authproxy/auth.go @@ -211,8 +211,6 @@ func (a *Auth) fillInModel(u *models.User) error { u.Comment = userEntryComment if strings.Contains(u.Username, "@") { u.Email = u.Username - } else { - u.Email = fmt.Sprintf("%s@placeholder.com", u.Username) } return nil } diff --git a/src/core/auth/authproxy/auth_test.go b/src/core/auth/authproxy/auth_test.go index 2dbcdf061..b1fb4ab22 100644 --- a/src/core/auth/authproxy/auth_test.go +++ b/src/core/auth/authproxy/auth_test.go @@ -154,7 +154,7 @@ func TestAuth_PostAuthenticate(t *testing.T) { }, expect: models.User{ Username: "jt", - Email: "jt@placeholder.com", + Email: "", Realname: "jt", Password: pwd, Comment: userEntryComment, diff --git a/src/core/auth/ldap/ldap.go b/src/core/auth/ldap/ldap.go index 15a44da12..c5fd86d29 100644 --- a/src/core/auth/ldap/ldap.go +++ b/src/core/auth/ldap/ldap.go @@ -124,8 +124,6 @@ func (l *Auth) OnBoardUser(u *models.User) error { if u.Email == "" { if strings.Contains(u.Username, "@") { u.Email = u.Username - } else { - u.Email = u.Username + "@placeholder.com" } } u.Password = "12345678AbC" // Password is not kept in local db diff --git a/src/core/auth/ldap/ldap_test.go b/src/core/auth/ldap/ldap_test.go index 498ffee2a..9002bd8bf 100644 --- a/src/core/auth/ldap/ldap_test.go +++ b/src/core/auth/ldap/ldap_test.go @@ -224,7 +224,7 @@ func TestOnBoardUser_02(t *testing.T) { t.Errorf("Failed to onboard user") } - assert.Equal(t, "sample02@placeholder.com", user.Email) + assert.Equal(t, "", user.Email) dao.CleanUser(int64(user.UserID)) } diff --git a/src/core/auth/uaa/uaa.go b/src/core/auth/uaa/uaa.go index b4889302c..8ca250fc1 100644 --- a/src/core/auth/uaa/uaa.go +++ b/src/core/auth/uaa/uaa.go @@ -77,9 +77,8 @@ func fillEmailRealName(user *models.User) { if len(user.Realname) == 0 { user.Realname = user.Username } - if len(user.Email) == 0 { - // TODO: handle the case when user.Username itself is an email address. - user.Email = user.Username + "@uaa.placeholder" + if len(user.Email) == 0 && strings.Contains(user.Username, "@") { + user.Email = user.Username } } diff --git a/src/core/auth/uaa/uaa_test.go b/src/core/auth/uaa/uaa_test.go index 7b0ff9ea9..a62bd7d7d 100644 --- a/src/core/auth/uaa/uaa_test.go +++ b/src/core/auth/uaa/uaa_test.go @@ -110,7 +110,7 @@ func TestOnBoardUser(t *testing.T) { user, _ := dao.GetUser(models.User{Username: "test"}) assert.Equal("test", user.Realname) assert.Equal("test", user.Username) - assert.Equal("test@uaa.placeholder", user.Email) + assert.Equal("", user.Email) err3 := dao.ClearTable(models.UserTable) assert.Nil(err3) } @@ -128,7 +128,7 @@ func TestPostAuthenticate(t *testing.T) { } assert.Nil(err) user, _ := dao.GetUser(models.User{Username: "test"}) - assert.Equal("test@uaa.placeholder", user.Email) + assert.Equal("", user.Email) um2.Email = "newEmail@new.com" um2.Realname = "newName" err2 := auth.PostAuthenticate(um2) @@ -145,7 +145,7 @@ func TestPostAuthenticate(t *testing.T) { assert.Nil(err3) user3, _ := dao.GetUser(models.User{Username: "test"}) assert.Equal(user3.UserID, um3.UserID) - assert.Equal("test@uaa.placeholder", user3.Email) + assert.Equal("", user3.Email) assert.Equal("test", user3.Realname) err4 := dao.ClearTable(models.UserTable) assert.Nil(err4) diff --git a/src/core/config/config.go b/src/core/config/config.go index 57c02bad1..b3808745d 100755 --- a/src/core/config/config.go +++ b/src/core/config/config.go @@ -331,12 +331,14 @@ func Database() (*models.Database, error) { database := &models.Database{} database.Type = cfgMgr.Get(common.DatabaseType).GetString() postgresql := &models.PostGreSQL{ - Host: cfgMgr.Get(common.PostGreSQLHOST).GetString(), - Port: cfgMgr.Get(common.PostGreSQLPort).GetInt(), - Username: cfgMgr.Get(common.PostGreSQLUsername).GetString(), - Password: cfgMgr.Get(common.PostGreSQLPassword).GetString(), - Database: cfgMgr.Get(common.PostGreSQLDatabase).GetString(), - SSLMode: cfgMgr.Get(common.PostGreSQLSSLMode).GetString(), + Host: cfgMgr.Get(common.PostGreSQLHOST).GetString(), + Port: cfgMgr.Get(common.PostGreSQLPort).GetInt(), + Username: cfgMgr.Get(common.PostGreSQLUsername).GetString(), + Password: cfgMgr.Get(common.PostGreSQLPassword).GetString(), + Database: cfgMgr.Get(common.PostGreSQLDatabase).GetString(), + SSLMode: cfgMgr.Get(common.PostGreSQLSSLMode).GetString(), + MaxIdleConns: cfgMgr.Get(common.PostGreSQLMaxIdleConns).GetInt(), + MaxOpenConns: cfgMgr.Get(common.PostGreSQLMaxOpenConns).GetInt(), } database.PostGreSQL = postgresql @@ -520,6 +522,11 @@ func NotificationEnable() bool { return cfgMgr.Get(common.NotificationEnable).GetBool() } +// QuotaPerProjectEnable returns a bool to indicates if quota per project enabled in harbor +func QuotaPerProjectEnable() bool { + return cfgMgr.Get(common.QuotaPerProjectEnable).GetBool() +} + // QuotaSetting returns the setting of quota. func QuotaSetting() (*models.QuotaSetting, error) { if err := cfgMgr.Load(); err != nil { diff --git a/src/core/controllers/oidc.go b/src/core/controllers/oidc.go index 1479b8e5a..903b99954 100644 --- a/src/core/controllers/oidc.go +++ b/src/core/controllers/oidc.go @@ -17,6 +17,9 @@ package controllers import ( "encoding/json" "fmt" + "net/http" + "strings" + "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" @@ -26,8 +29,6 @@ import ( "github.com/goharbor/harbor/src/core/api" "github.com/goharbor/harbor/src/core/config" "github.com/pkg/errors" - "net/http" - "strings" ) const tokenKey = "oidc_token" @@ -189,9 +190,6 @@ func (oc *OIDCController) Onboard() { } email := d.Email - if email == "" { - email = utils.GenerateRandomString() + "@placeholder.com" - } user := models.User{ Username: username, Realname: d.Username, diff --git a/src/core/main.go b/src/core/main.go index 6ea199757..b70c61700 100755 --- a/src/core/main.go +++ b/src/core/main.go @@ -17,16 +17,12 @@ package main import ( "encoding/gob" "fmt" - "os" - "os/signal" - "strconv" - "syscall" - "github.com/astaxie/beego" _ "github.com/astaxie/beego/session/redis" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/job" "github.com/goharbor/harbor/src/common/models" + common_quota "github.com/goharbor/harbor/src/common/quota" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/api" @@ -34,6 +30,15 @@ import ( _ "github.com/goharbor/harbor/src/core/auth/db" _ "github.com/goharbor/harbor/src/core/auth/ldap" _ "github.com/goharbor/harbor/src/core/auth/uaa" + "os" + "os/signal" + "strconv" + "syscall" + + quota "github.com/goharbor/harbor/src/core/api/quota" + _ "github.com/goharbor/harbor/src/core/api/quota/chart" + _ "github.com/goharbor/harbor/src/core/api/quota/registry" + "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/filter" "github.com/goharbor/harbor/src/core/middlewares" @@ -41,6 +46,7 @@ import ( "github.com/goharbor/harbor/src/core/service/token" "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/scheduler" + "github.com/goharbor/harbor/src/pkg/types" "github.com/goharbor/harbor/src/replication" ) @@ -67,13 +73,71 @@ func updateInitPassword(userID int, password string) error { return fmt.Errorf("Failed to update user encrypted password, userID: %d, err: %v", userID, err) } - log.Infof("User id: %d updated its encypted password successfully.", userID) + log.Infof("User id: %d updated its encrypted password successfully.", userID) } else { log.Infof("User id: %d already has its encrypted password.", userID) } return nil } +// Quota migration +func quotaSync() error { + usages, err := dao.ListQuotaUsages() + if err != nil { + log.Errorf("list quota usage error, %v", err) + return err + } + projects, err := dao.GetProjects(nil) + if err != nil { + log.Errorf("list project error, %v", err) + return err + } + + // The condition handles these two cases: + // 1, len(project) > 1 && len(usages) == 1. existing projects without usage, as we do always has 'library' usage in DB. + // 2, migration fails at the phase of inserting usage into DB, and parts of them are inserted successfully. + if len(projects) != len(usages) { + log.Info("Start to sync quota data .....") + if err := quota.Sync(config.GlobalProjectMgr, true); err != nil { + log.Errorf("Fail to sync quota data, %v", err) + return err + } + log.Info("Success to sync quota data .....") + return nil + } + + // Only has one project without usage + zero := common_quota.ResourceList{ + common_quota.ResourceCount: 0, + common_quota.ResourceStorage: 0, + } + if len(projects) == 1 && len(usages) == 1 { + totalRepo, err := dao.GetTotalOfRepositories() + if totalRepo == 0 { + return nil + } + refID, err := strconv.ParseInt(usages[0].ReferenceID, 10, 64) + if err != nil { + log.Error(err) + return err + } + usedRes, err := types.NewResourceList(usages[0].Used) + if err != nil { + log.Error(err) + return err + } + if types.Equals(usedRes, zero) && refID == projects[0].ProjectID { + log.Info("Start to sync quota data .....") + if err := quota.Sync(config.GlobalProjectMgr, true); err != nil { + log.Errorf("Fail to sync quota data, %v", err) + return err + } + log.Info("Success to sync quota data .....") + } + } + return nil +} + func gracefulShutdown(closing chan struct{}) { signals := make(chan os.Signal, 1) signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) @@ -117,7 +181,7 @@ func main() { password, err := config.InitialAdminPassword() if err != nil { - log.Fatalf("failed to get admin's initia password: %v", err) + log.Fatalf("failed to get admin's initial password: %v", err) } if err := updateInitPassword(adminUserID, password); err != nil { log.Error(err) @@ -174,6 +238,9 @@ func main() { log.Fatalf("init proxy error, %v", err) } - // go proxy.StartProxy() + if err := quotaSync(); err != nil { + log.Fatalf("quota migration error, %v", err) + } + beego.Run() } diff --git a/src/core/middlewares/chart/builder.go b/src/core/middlewares/chart/builder.go index 669509ff4..ba54cd2de 100644 --- a/src/core/middlewares/chart/builder.go +++ b/src/core/middlewares/chart/builder.go @@ -21,6 +21,7 @@ import ( "strconv" "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/middlewares/interceptor" "github.com/goharbor/harbor/src/core/middlewares/interceptor/quota" "github.com/goharbor/harbor/src/core/middlewares/util" @@ -69,6 +70,7 @@ func (*chartVersionDeletionBuilder) Build(req *http.Request) (interceptor.Interc } opts := []quota.Option{ + quota.EnforceResources(config.QuotaPerProjectEnable()), quota.WithManager("project", strconv.FormatInt(project.ProjectID, 10)), quota.WithAction(quota.SubtractAction), quota.StatusCode(http.StatusOK), @@ -101,22 +103,26 @@ func (*chartVersionCreationBuilder) Build(req *http.Request) (interceptor.Interc return nil, fmt.Errorf("project %s not found", namespace) } - chart, err := parseChart(req) - if err != nil { - return nil, fmt.Errorf("failed to parse chart from body, error: %v", err) - } - chartName, version := chart.Metadata.Name, chart.Metadata.Version + info, ok := util.ChartVersionInfoFromContext(req.Context()) + if !ok { + chart, err := parseChart(req) + if err != nil { + return nil, fmt.Errorf("failed to parse chart from body, error: %v", err) + } + chartName, version := chart.Metadata.Name, chart.Metadata.Version - info := &util.ChartVersionInfo{ - ProjectID: project.ProjectID, - Namespace: namespace, - ChartName: chartName, - Version: version, + info = &util.ChartVersionInfo{ + ProjectID: project.ProjectID, + Namespace: namespace, + ChartName: chartName, + Version: version, + } + // Chart version info will be used by computeQuotaForUpload + *req = *req.WithContext(util.NewChartVersionInfoContext(req.Context(), info)) } - // Chart version info will be used by computeQuotaForUpload - *req = *req.WithContext(util.NewChartVersionInfoContext(req.Context(), info)) opts := []quota.Option{ + quota.EnforceResources(config.QuotaPerProjectEnable()), quota.WithManager("project", strconv.FormatInt(project.ProjectID, 10)), quota.WithAction(quota.AddAction), quota.StatusCode(http.StatusCreated), diff --git a/src/core/middlewares/chart/handler_test.go b/src/core/middlewares/chart/handler_test.go new file mode 100644 index 000000000..aedf1218e --- /dev/null +++ b/src/core/middlewares/chart/handler_test.go @@ -0,0 +1,137 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chart + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/goharbor/harbor/src/chartserver" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/core/middlewares/util" + "github.com/goharbor/harbor/src/pkg/types" + htesting "github.com/goharbor/harbor/src/testing" + "github.com/stretchr/testify/suite" +) + +func deleteChartVersion(projectName, chartName, version string) { + url := fmt.Sprintf("/api/chartrepo/%s/charts/%s/%s", projectName, chartName, version) + req, _ := http.NewRequest(http.MethodDelete, url, nil) + + next := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + rr := httptest.NewRecorder() + h := New(next) + h.ServeHTTP(util.NewCustomResponseWriter(rr), req) +} + +func uploadChartVersion(projectID int64, projectName, chartName, version string) { + url := fmt.Sprintf("/api/chartrepo/%s/charts/", projectName) + req, _ := http.NewRequest(http.MethodPost, url, nil) + + info := &util.ChartVersionInfo{ + ProjectID: projectID, + Namespace: projectName, + ChartName: chartName, + Version: version, + } + *req = *req.WithContext(util.NewChartVersionInfoContext(req.Context(), info)) + + next := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusCreated) + }) + + rr := httptest.NewRecorder() + h := New(next) + h.ServeHTTP(util.NewCustomResponseWriter(rr), req) +} + +func mockChartController() (*httptest.Server, *chartserver.Controller, error) { + mockServer := httptest.NewServer(htesting.MockChartRepoHandler) + + var oldController, newController *chartserver.Controller + url, err := url.Parse(mockServer.URL) + if err == nil { + newController, err = chartserver.NewController(url) + } + + if err != nil { + mockServer.Close() + return nil, nil, err + } + + chartController() // Init chart controller + + // Override current controller and keep the old one for restoring + oldController = controller + controller = newController + + return mockServer, oldController, nil +} + +type HandlerSuite struct { + htesting.Suite + oldController *chartserver.Controller + mockChartServer *httptest.Server +} + +func (suite *HandlerSuite) SetupTest() { + mockServer, oldController, err := mockChartController() + suite.Nil(err, "Mock chart controller failed") + + suite.oldController = oldController + suite.mockChartServer = mockServer +} + +func (suite *HandlerSuite) TearDownTest() { + for _, table := range []string{ + "quota", "quota_usage", + } { + dao.ClearTable(table) + } + + controller = suite.oldController + suite.mockChartServer.Close() +} + +func (suite *HandlerSuite) TestUpload() { + suite.WithProject(func(projectID int64, projectName string) { + uploadChartVersion(projectID, projectName, "harbor", "0.2.1") + suite.AssertResourceUsage(1, types.ResourceCount, projectID) + + // harbor:0.2.0 exists in repo1, upload it again + uploadChartVersion(projectID, projectName, "harbor", "0.2.0") + suite.AssertResourceUsage(1, types.ResourceCount, projectID) + }, "repo1") +} + +func (suite *HandlerSuite) TestDelete() { + suite.WithProject(func(projectID int64, projectName string) { + uploadChartVersion(projectID, projectName, "harbor", "0.2.1") + suite.AssertResourceUsage(1, types.ResourceCount, projectID) + + deleteChartVersion(projectName, "harbor", "0.2.1") + suite.AssertResourceUsage(0, types.ResourceCount, projectID) + }, "repo1") +} + +func TestRunHandlerSuite(t *testing.T) { + suite.Run(t, new(HandlerSuite)) +} diff --git a/src/core/middlewares/countquota/builder.go b/src/core/middlewares/countquota/builder.go index 5de9a2735..e84d9a454 100644 --- a/src/core/middlewares/countquota/builder.go +++ b/src/core/middlewares/countquota/builder.go @@ -20,6 +20,7 @@ import ( "strconv" "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/middlewares/interceptor" "github.com/goharbor/harbor/src/core/middlewares/interceptor/quota" "github.com/goharbor/harbor/src/core/middlewares/util" @@ -52,6 +53,7 @@ func (*manifestDeletionBuilder) Build(req *http.Request) (interceptor.Intercepto } opts := []quota.Option{ + quota.EnforceResources(config.QuotaPerProjectEnable()), quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)), quota.WithAction(quota.SubtractAction), quota.StatusCode(http.StatusAccepted), @@ -75,7 +77,7 @@ func (*manifestCreationBuilder) Build(req *http.Request) (interceptor.Intercepto info, ok := util.ManifestInfoFromContext(req.Context()) if !ok { var err error - info, err = util.ParseManifestInfo(req) + info, err = util.ParseManifestInfoFromReq(req) if err != nil { return nil, fmt.Errorf("failed to parse manifest, error %v", err) } @@ -85,6 +87,7 @@ func (*manifestCreationBuilder) Build(req *http.Request) (interceptor.Intercepto } opts := []quota.Option{ + quota.EnforceResources(config.QuotaPerProjectEnable()), quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)), quota.WithAction(quota.AddAction), quota.StatusCode(http.StatusCreated), diff --git a/src/core/middlewares/countquota/handler.go b/src/core/middlewares/countquota/handler.go index 1b05a4cf5..7564b1a5e 100644 --- a/src/core/middlewares/countquota/handler.go +++ b/src/core/middlewares/countquota/handler.go @@ -21,6 +21,7 @@ import ( "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/middlewares/interceptor" "github.com/goharbor/harbor/src/core/middlewares/util" + "strings" ) type countQuotaHandler struct { @@ -57,6 +58,10 @@ func (h *countQuotaHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) if err := interceptor.HandleRequest(req); err != nil { log.Warningf("Error occurred when to handle request in count quota handler: %v", err) + if strings.Contains(err.Error(), "resource overflow the hard limit") { + http.Error(rw, util.MarshalError("DENIED", fmt.Sprintf("Not enough quota is available to process the request: %v", err)), http.StatusForbidden) + return + } http.Error(rw, util.MarshalError("InternalError", fmt.Sprintf("Error occurred when to handle request in count quota handler: %v", err)), http.StatusInternalServerError) return diff --git a/src/core/middlewares/countquota/handler_test.go b/src/core/middlewares/countquota/handler_test.go index a25166734..84833e4de 100644 --- a/src/core/middlewares/countquota/handler_test.go +++ b/src/core/middlewares/countquota/handler_test.go @@ -26,6 +26,7 @@ import ( "github.com/docker/distribution" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/middlewares/util" "github.com/goharbor/harbor/src/pkg/types" "github.com/opencontainers/go-digest" @@ -201,12 +202,12 @@ func (suite *HandlerSuite) TestPutManifestFailed() { }() next := func(w http.ResponseWriter, req *http.Request) { - w.WriteHeader(http.StatusInternalServerError) + w.WriteHeader(http.StatusForbidden) } dgt := digest.FromString(randomString(15)).String() code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, next) - suite.Equal(http.StatusInternalServerError, code) + suite.Equal(http.StatusForbidden, code) suite.checkCountUsage(0, projectID) total, err := dao.GetTotalOfArtifacts(&models.ArtifactQuery{Digest: dgt}) @@ -290,6 +291,7 @@ func (suite *HandlerSuite) TestDeleteManifestInMultiProjects() { } func TestMain(m *testing.M) { + config.Init() dao.PrepareTestForPostgresSQL() if result := m.Run(); result != 0 { diff --git a/src/core/middlewares/interceptor/quota/options.go b/src/core/middlewares/interceptor/quota/options.go index ca43c4165..ddf102a74 100644 --- a/src/core/middlewares/interceptor/quota/options.go +++ b/src/core/middlewares/interceptor/quota/options.go @@ -36,6 +36,8 @@ const ( // Options ... type Options struct { + enforceResources *bool + Action Action Manager *quota.Manager MutexKeys []string @@ -48,6 +50,15 @@ type Options struct { OnFinally func(http.ResponseWriter, *http.Request) error } +// EnforceResources ... +func (opts *Options) EnforceResources() bool { + return opts.enforceResources != nil && *opts.enforceResources +} + +func boolPtr(v bool) *bool { + return &v +} + func newOptions(opt ...Option) Options { opts := Options{} @@ -63,9 +74,20 @@ func newOptions(opt ...Option) Options { opts.StatusCode = http.StatusOK } + if opts.enforceResources == nil { + opts.enforceResources = boolPtr(true) + } + return opts } +// EnforceResources sets the interceptor enforceResources +func EnforceResources(enforceResources bool) Option { + return func(o *Options) { + o.enforceResources = boolPtr(enforceResources) + } +} + // WithAction sets the interceptor action func WithAction(a Action) Option { return func(o *Options) { diff --git a/src/core/middlewares/interceptor/quota/quota.go b/src/core/middlewares/interceptor/quota/quota.go index 85c289ff3..1ae530078 100644 --- a/src/core/middlewares/interceptor/quota/quota.go +++ b/src/core/middlewares/interceptor/quota/quota.go @@ -16,7 +16,9 @@ package quota import ( "fmt" + "math/rand" "net/http" + "time" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/redis" @@ -24,6 +26,10 @@ import ( "github.com/goharbor/harbor/src/pkg/types" ) +func init() { + rand.Seed(time.Now().UnixNano()) +} + // New .... func New(opts ...Option) interceptor.Interceptor { options := newOptions(opts...) @@ -49,28 +55,19 @@ func (qi *quotaInterceptor) HandleRequest(req *http.Request) (err error) { } }() - opts := qi.opts - - for _, key := range opts.MutexKeys { - m, err := redis.RequireLock(key) - if err != nil { - return err - } - qi.mutexes = append(qi.mutexes, m) - } - - resources := opts.Resources - if len(resources) == 0 && opts.OnResources != nil { - resources, err = opts.OnResources(req) - if err != nil { - return fmt.Errorf("failed to compute the resources for quota, error: %v", err) - } - } - qi.resources = resources - - err = qi.reserve() + err = qi.requireMutexes() if err != nil { - log.Errorf("Failed to %s resources, error: %v", opts.Action, err) + return + } + + err = qi.computeResources(req) + if err != nil { + return + } + + err = qi.doTry() + if err != nil { + log.Errorf("Failed to %s resources, error: %v", qi.opts.Action, err) } return @@ -89,14 +86,18 @@ func (qi *quotaInterceptor) HandleResponse(w http.ResponseWriter, req *http.Requ switch sr.Status() { case opts.StatusCode: + if err := qi.doConfirm(); err != nil { + log.Errorf("Failed to confirm for resource, error: %v", err) + } + if opts.OnFulfilled != nil { if err := opts.OnFulfilled(w, req); err != nil { log.Errorf("Failed to handle on fulfilled, error: %v", err) } } default: - if err := qi.unreserve(); err != nil { - log.Errorf("Failed to %s resources, error: %v", opts.Action, err) + if err := qi.doCancel(); err != nil { + log.Errorf("Failed to cancel for resource, error: %v", err) } if opts.OnRejected != nil { @@ -113,6 +114,23 @@ func (qi *quotaInterceptor) HandleResponse(w http.ResponseWriter, req *http.Requ } } +func (qi *quotaInterceptor) requireMutexes() error { + if !qi.opts.EnforceResources() { + // Do nothing for locks when quota interceptor not enforce resources + return nil + } + + for _, key := range qi.opts.MutexKeys { + m, err := redis.RequireLock(key) + if err != nil { + return err + } + qi.mutexes = append(qi.mutexes, m) + } + + return nil +} + func (qi *quotaInterceptor) freeMutexes() { for i := len(qi.mutexes) - 1; i >= 0; i-- { if err := redis.FreeLock(qi.mutexes[i]); err != nil { @@ -121,32 +139,84 @@ func (qi *quotaInterceptor) freeMutexes() { } } -func (qi *quotaInterceptor) reserve() error { - if len(qi.resources) == 0 { +func (qi *quotaInterceptor) computeResources(req *http.Request) error { + if !qi.opts.EnforceResources() { + // Do nothing in compute resources when quota interceptor not enforce resources return nil } - switch qi.opts.Action { - case AddAction: - return qi.opts.Manager.AddResources(qi.resources) - case SubtractAction: - return qi.opts.Manager.SubtractResources(qi.resources) + qi.resources = qi.opts.Resources + if len(qi.resources) == 0 && qi.opts.OnResources != nil { + resources, err := qi.opts.OnResources(req) + if err != nil { + return fmt.Errorf("failed to compute the resources for quota, error: %v", err) + } + + qi.resources = resources } return nil } -func (qi *quotaInterceptor) unreserve() error { - if len(qi.resources) == 0 { +func (qi *quotaInterceptor) doTry() error { + if !qi.opts.EnforceResources() { + // Do nothing in try stage when quota interceptor not enforce resources return nil } - switch qi.opts.Action { - case AddAction: - return qi.opts.Manager.SubtractResources(qi.resources) - case SubtractAction: + // Add resources in try stage when it is add action + // And do nothing in confirm stage for add action + if len(qi.resources) != 0 && qi.opts.Action == AddAction { return qi.opts.Manager.AddResources(qi.resources) } return nil } + +func (qi *quotaInterceptor) doConfirm() error { + if !qi.opts.EnforceResources() { + // Do nothing in confirm stage when quota interceptor not enforce resources + return nil + } + + // Subtract resources in confirm stage when it is subtract action + // And do nothing in try stage for subtract action + if len(qi.resources) != 0 && qi.opts.Action == SubtractAction { + return retry(3, 100*time.Millisecond, func() error { + return qi.opts.Manager.SubtractResources(qi.resources) + }) + } + + return nil +} + +func (qi *quotaInterceptor) doCancel() error { + if !qi.opts.EnforceResources() { + // Do nothing in cancel stage when quota interceptor not enforce resources + return nil + } + + // Subtract resources back when process failed for add action + if len(qi.resources) != 0 && qi.opts.Action == AddAction { + return retry(3, 100*time.Millisecond, func() error { + return qi.opts.Manager.SubtractResources(qi.resources) + }) + } + + return nil +} + +func retry(attempts int, sleep time.Duration, f func() error) error { + if err := f(); err != nil { + if attempts--; attempts > 0 { + r := time.Duration(rand.Int63n(int64(sleep))) + sleep = sleep + r/2 + + time.Sleep(sleep) + return retry(attempts, 2*sleep, f) + } + return err + } + + return nil +} diff --git a/src/core/middlewares/sizequota/builder.go b/src/core/middlewares/sizequota/builder.go index 310c6e5bc..37c3c145b 100644 --- a/src/core/middlewares/sizequota/builder.go +++ b/src/core/middlewares/sizequota/builder.go @@ -20,8 +20,8 @@ import ( "strconv" "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/middlewares/interceptor" "github.com/goharbor/harbor/src/core/middlewares/interceptor/quota" "github.com/goharbor/harbor/src/core/middlewares/util" @@ -89,6 +89,7 @@ func (*blobStorageQuotaBuilder) Build(req *http.Request) (interceptor.Intercepto *req = *(req.WithContext(util.NewBlobInfoContext(req.Context(), info))) opts := []quota.Option{ + quota.EnforceResources(config.QuotaPerProjectEnable()), quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)), quota.WithAction(quota.AddAction), quota.StatusCode(http.StatusCreated), // NOTICE: mount blob and blob upload complete both return 201 when success @@ -110,7 +111,7 @@ func (*manifestCreationBuilder) Build(req *http.Request) (interceptor.Intercepto return nil, nil } - info, err := util.ParseManifestInfo(req) + info, err := util.ParseManifestInfoFromReq(req) if err != nil { return nil, err } @@ -119,6 +120,7 @@ func (*manifestCreationBuilder) Build(req *http.Request) (interceptor.Intercepto *req = *req.WithContext(util.NewManifestInfoContext(req.Context(), info)) opts := []quota.Option{ + quota.EnforceResources(config.QuotaPerProjectEnable()), quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)), quota.WithAction(quota.AddAction), quota.StatusCode(http.StatusCreated), @@ -181,6 +183,7 @@ func (*manifestDeletionBuilder) Build(req *http.Request) (interceptor.Intercepto } opts := []quota.Option{ + quota.EnforceResources(config.QuotaPerProjectEnable()), quota.WithManager("project", strconv.FormatInt(info.ProjectID, 10)), quota.WithAction(quota.SubtractAction), quota.StatusCode(http.StatusAccepted), @@ -188,18 +191,6 @@ func (*manifestDeletionBuilder) Build(req *http.Request) (interceptor.Intercepto quota.MutexKeys(mutexKeys...), quota.OnFulfilled(func(http.ResponseWriter, *http.Request) error { blobs := info.ExclusiveBlobs - - total, err := dao.GetTotalOfArtifacts(&models.ArtifactQuery{ - PID: info.ProjectID, - Digest: info.Digest, - }) - if err == nil && total > 0 { - blob, err := dao.GetBlob(info.Digest) - if err == nil { - blobs = append(blobs, blob) - } - } - return dao.RemoveBlobsFromProject(info.ProjectID, blobs...) }), } diff --git a/src/core/middlewares/sizequota/handler.go b/src/core/middlewares/sizequota/handler.go index 68ae4258e..638d560e3 100644 --- a/src/core/middlewares/sizequota/handler.go +++ b/src/core/middlewares/sizequota/handler.go @@ -21,6 +21,7 @@ import ( "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/middlewares/interceptor" "github.com/goharbor/harbor/src/core/middlewares/util" + "strings" ) type sizeQuotaHandler struct { @@ -44,6 +45,7 @@ func New(next http.Handler, builders ...interceptor.Builder) http.Handler { func (h *sizeQuotaHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { interceptor, err := h.getInterceptor(req) if err != nil { + log.Warningf("Error occurred when to handle request in size quota handler: %v", err) http.Error(rw, util.MarshalError("InternalError", fmt.Sprintf("Error occurred when to handle request in size quota handler: %v", err)), http.StatusInternalServerError) return @@ -56,6 +58,10 @@ func (h *sizeQuotaHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) if err := interceptor.HandleRequest(req); err != nil { log.Warningf("Error occurred when to handle request in size quota handler: %v", err) + if strings.Contains(err.Error(), "resource overflow the hard limit") { + http.Error(rw, util.MarshalError("DENIED", fmt.Sprintf("Not enough quota is available to process the request: %v", err)), http.StatusForbidden) + return + } http.Error(rw, util.MarshalError("InternalError", fmt.Sprintf("Error occurred when to handle request in size quota handler: %v", err)), http.StatusInternalServerError) return diff --git a/src/core/middlewares/sizequota/handler_test.go b/src/core/middlewares/sizequota/handler_test.go index cd9ca972f..a8d416394 100644 --- a/src/core/middlewares/sizequota/handler_test.go +++ b/src/core/middlewares/sizequota/handler_test.go @@ -30,8 +30,10 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/manifest" "github.com/docker/distribution/manifest/schema2" + "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/middlewares/countquota" "github.com/goharbor/harbor/src/core/middlewares/util" "github.com/goharbor/harbor/src/pkg/types" @@ -451,10 +453,10 @@ func (suite *HandlerSuite) TestPushImageToDifferentRepositories() { suite.checkStorageUsage(size, projectID) pushImage(projectName, "redis", "latest", manifest) - suite.checkStorageUsage(size+sizeOfManifest(manifest), projectID) + suite.checkStorageUsage(size, projectID) pushImage(projectName, "postgres", "latest", manifest) - suite.checkStorageUsage(size+2*sizeOfManifest(manifest), projectID) + suite.checkStorageUsage(size, projectID) }) } @@ -560,7 +562,7 @@ func (suite *HandlerSuite) TestDeleteManifestInDifferentRepositories() { pushImage(projectName, "redis", "latest", manifest) suite.checkCountUsage(3, projectID) - suite.checkStorageUsage(size+sizeOfManifest(manifest), projectID) + suite.checkStorageUsage(size, projectID) deleteManifest(projectName, "redis", digestOfManifest(manifest)) suite.checkCountUsage(2, projectID) @@ -568,7 +570,7 @@ func (suite *HandlerSuite) TestDeleteManifestInDifferentRepositories() { pushImage(projectName, "redis", "latest", manifest) suite.checkCountUsage(3, projectID) - suite.checkStorageUsage(size+sizeOfManifest(manifest), projectID) + suite.checkStorageUsage(size, projectID) }) } @@ -662,7 +664,40 @@ func (suite *HandlerSuite) TestDeleteImageRace() { }) } +func (suite *HandlerSuite) TestDisableProjectQuota() { + withProject(func(projectID int64, projectName string) { + manifest := makeManifest(1, []int64{2, 3, 4, 5}) + pushImage(projectName, "photon", "latest", manifest) + + quotas, err := dao.ListQuotas(&models.QuotaQuery{ + Reference: "project", + ReferenceID: strconv.FormatInt(projectID, 10), + }) + + suite.Nil(err) + suite.Len(quotas, 1) + }) + + withProject(func(projectID int64, projectName string) { + cfg := config.GetCfgManager() + cfg.Set(common.QuotaPerProjectEnable, false) + defer cfg.Set(common.QuotaPerProjectEnable, true) + + manifest := makeManifest(1, []int64{2, 3, 4, 5}) + pushImage(projectName, "photon", "latest", manifest) + + quotas, err := dao.ListQuotas(&models.QuotaQuery{ + Reference: "project", + ReferenceID: strconv.FormatInt(projectID, 10), + }) + + suite.Nil(err) + suite.Len(quotas, 0) + }) +} + func TestMain(m *testing.M) { + config.Init() dao.PrepareTestForPostgresSQL() if result := m.Run(); result != 0 { diff --git a/src/core/middlewares/sizequota/util.go b/src/core/middlewares/sizequota/util.go index edcf92631..b01c7dd6f 100644 --- a/src/core/middlewares/sizequota/util.go +++ b/src/core/middlewares/sizequota/util.go @@ -146,7 +146,7 @@ func parseBlobInfoFromComplete(req *http.Request) (*util.BlobInfo, error) { func parseBlobInfoFromManifest(req *http.Request) (*util.BlobInfo, error) { info, ok := util.ManifestInfoFromContext(req.Context()) if !ok { - manifest, err := util.ParseManifestInfo(req) + manifest, err := util.ParseManifestInfoFromReq(req) if err != nil { return nil, err } @@ -295,14 +295,7 @@ func computeResourcesForManifestDeletion(req *http.Request) (types.ResourceList, info.ExclusiveBlobs = blobs - blob, err := dao.GetBlob(info.Digest) - if err != nil { - return nil, err - } - - // manifest size will always be released - size := blob.Size - + var size int64 for _, blob := range blobs { size = size + blob.Size } diff --git a/src/core/middlewares/util/util.go b/src/core/middlewares/util/util.go index 7b8d2839e..1c9d86034 100644 --- a/src/core/middlewares/util/util.go +++ b/src/core/middlewares/util/util.go @@ -228,7 +228,6 @@ func (info *ManifestInfo) ManifestExists() (bool, error) { info.manifestExistOnce.Do(func() { total, err := dao.GetTotalOfArtifacts(&models.ArtifactQuery{ PID: info.ProjectID, - Repo: info.Repository, Digest: info.Digest, }) @@ -441,8 +440,8 @@ func NewManifestInfoContext(ctx context.Context, info *ManifestInfo) context.Con return context.WithValue(ctx, manifestInfoKey, info) } -// ParseManifestInfo prase manifest from request -func ParseManifestInfo(req *http.Request) (*ManifestInfo, error) { +// ParseManifestInfoFromReq parse manifest from request +func ParseManifestInfoFromReq(req *http.Request) (*ManifestInfo, error) { match, repository, reference := MatchManifestURL(req) if !match { return nil, fmt.Errorf("not match url %s for manifest", req.URL.Path) @@ -496,7 +495,7 @@ func ParseManifestInfo(req *http.Request) (*ManifestInfo, error) { }, nil } -// ParseManifestInfoFromPath prase manifest from request path +// ParseManifestInfoFromPath parse manifest from request path func ParseManifestInfoFromPath(req *http.Request) (*ManifestInfo, error) { match, repository, reference := MatchManifestURL(req) if !match { diff --git a/src/core/middlewares/util/util_test.go b/src/core/middlewares/util/util_test.go index e02229ad9..2e6c9d609 100644 --- a/src/core/middlewares/util/util_test.go +++ b/src/core/middlewares/util/util_test.go @@ -326,13 +326,13 @@ func TestParseManifestInfo(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := ParseManifestInfo(tt.req()) + got, err := ParseManifestInfoFromReq(tt.req()) if (err != nil) != tt.wantErr { - t.Errorf("ParseManifestInfo() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("ParseManifestInfoFromReq() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("ParseManifestInfo() = %v, want %v", got, tt.want) + t.Errorf("ParseManifestInfoFromReq() = %v, want %v", got, tt.want) } }) } diff --git a/src/core/promgr/promgr.go b/src/core/promgr/promgr.go index 5a0a9bc12..3ac8f6ca8 100644 --- a/src/core/promgr/promgr.go +++ b/src/core/promgr/promgr.go @@ -94,7 +94,7 @@ func (d *defaultProjectManager) Create(project *models.Project) (int64, error) { return 0, err } if d.metaMgrEnabled { - d.whitelistMgr.CreateEmpty(project.ProjectID) + d.whitelistMgr.CreateEmpty(id) if len(project.Metadata) > 0 { if err = d.metaMgr.Add(id, project.Metadata); err != nil { log.Errorf("failed to add metadata for project %s: %v", project.Name, err) diff --git a/src/core/router.go b/src/core/router.go index 04fd1a173..7e01b934e 100755 --- a/src/core/router.go +++ b/src/core/router.go @@ -134,6 +134,8 @@ func initRouters() { beego.Router("/api/internal/syncregistry", &api.InternalAPI{}, "post:SyncRegistry") beego.Router("/api/internal/renameadmin", &api.InternalAPI{}, "post:RenameAdmin") + beego.Router("/api/internal/switchquota", &api.InternalAPI{}, "put:SwitchQuota") + beego.Router("/api/internal/syncquota", &api.InternalAPI{}, "post:SyncQuota") // external service that hosted on harbor process: beego.Router("/service/notifications", ®istry.NotificationHandler{}) diff --git a/src/core/service/notifications/jobs/handler.go b/src/core/service/notifications/jobs/handler.go index 30361eb43..12a7e44fd 100755 --- a/src/core/service/notifications/jobs/handler.go +++ b/src/core/service/notifications/jobs/handler.go @@ -20,11 +20,11 @@ import ( "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/job" - jobmodels "github.com/goharbor/harbor/src/common/job/models" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/api" "github.com/goharbor/harbor/src/core/notifier/event" + jjob "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/retention" "github.com/goharbor/harbor/src/replication" @@ -34,12 +34,11 @@ import ( var statusMap = map[string]string{ job.JobServiceStatusPending: models.JobPending, + job.JobServiceStatusScheduled: models.JobScheduled, job.JobServiceStatusRunning: models.JobRunning, job.JobServiceStatusStopped: models.JobStopped, - job.JobServiceStatusCancelled: models.JobCanceled, job.JobServiceStatusError: models.JobError, job.JobServiceStatusSuccess: models.JobFinished, - job.JobServiceStatusScheduled: models.JobScheduled, } // Handler handles reqeust on /service/notifications/jobs/*, which listens to the webhook of jobservice. @@ -49,6 +48,7 @@ type Handler struct { status string rawStatus string checkIn string + revision int64 } // Prepare ... @@ -61,7 +61,7 @@ func (h *Handler) Prepare() { return } h.id = id - var data jobmodels.JobStatusChange + var data jjob.StatusChange err = json.Unmarshal(h.Ctx.Input.CopyBody(1<<32), &data) if err != nil { log.Errorf("Failed to decode job status change, job ID: %d, error: %v", id, err) @@ -77,6 +77,9 @@ func (h *Handler) Prepare() { } h.status = status h.checkIn = data.CheckIn + if data.Metadata != nil { + h.revision = data.Metadata.Revision + } } // HandleScan handles the webhook of scan job @@ -127,39 +130,36 @@ func (h *Handler) HandleReplicationTask() { // HandleRetentionTask handles the webhook of retention task func (h *Handler) HandleRetentionTask() { - log.Debugf("received retention task status update event: task-%d, status-%s", h.id, h.status) + taskID := h.id + status := h.rawStatus + log.Debugf("received retention task status update event: task-%d, status-%s", taskID, status) mgr := &retention.DefaultManager{} - props := []string{"Status"} - task := &retention.Task{ - ID: h.id, - Status: h.status, - } - if h.status == models.JobFinished || h.status == models.JobError || - h.status == models.JobStopped { - task.EndTime = time.Now() - props = append(props, "EndTime") - } else if h.status == models.JobRunning { - if h.checkIn != "" { - var retainObj struct { - Total int `json:"total"` - Retained int `json:"retained"` - } - if err := json.Unmarshal([]byte(h.checkIn), &retainObj); err != nil { - log.Errorf("failed to resolve checkin of retention task %d: %v", h.id, err) - } else { - if retainObj.Total > 0 { - task.Total = retainObj.Total - props = append(props, "Total") - } - if retainObj.Retained > 0 { - task.Retained = retainObj.Retained - props = append(props, "Retained") - } - } + // handle checkin + if h.checkIn != "" { + var retainObj struct { + Total int `json:"total"` + Retained int `json:"retained"` } + if err := json.Unmarshal([]byte(h.checkIn), &retainObj); err != nil { + log.Errorf("failed to resolve checkin of retention task %d: %v", taskID, err) + return + } + task := &retention.Task{ + ID: taskID, + Total: retainObj.Total, + Retained: retainObj.Retained, + } + if err := mgr.UpdateTask(task, "Total", "Retained"); err != nil { + log.Errorf("failed to update of retention task %d: %v", taskID, err) + h.SendInternalServerError(err) + return + } + return } - if err := mgr.UpdateTask(task, props...); err != nil { - log.Errorf("failed to update the status of retention task %d: %v", h.id, err) + + // handle status updating + if err := mgr.UpdateTaskStatus(taskID, status, h.revision); err != nil { + log.Errorf("failed to update the status of retention task %d: %v", taskID, err) h.SendInternalServerError(err) return } diff --git a/src/core/service/notifications/registry/handler.go b/src/core/service/notifications/registry/handler.go index 3887001ad..66fba6102 100755 --- a/src/core/service/notifications/registry/handler.go +++ b/src/core/service/notifications/registry/handler.go @@ -112,7 +112,7 @@ func (n *NotificationHandler) Post() { }() } - if !coreutils.WaitForManifestReady(repository, tag, 5) { + if !coreutils.WaitForManifestReady(repository, tag, 6) { log.Errorf("Manifest for image %s:%s is not ready, skip the follow up actions.", repository, tag) return } diff --git a/src/core/service/notifications/scheduler/handler.go b/src/core/service/notifications/scheduler/handler.go index a3592f072..b07cfd5b6 100644 --- a/src/core/service/notifications/scheduler/handler.go +++ b/src/core/service/notifications/scheduler/handler.go @@ -21,6 +21,7 @@ import ( "github.com/goharbor/harbor/src/common/job/models" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/api" + "github.com/goharbor/harbor/src/pkg/scheduler" "github.com/goharbor/harbor/src/pkg/scheduler/hook" ) @@ -31,7 +32,13 @@ type Handler struct { // Handle ... func (h *Handler) Handle() { + log.Debugf("received scheduler hook event for schedule %s", h.GetStringFromPath(":id")) + var data models.JobStatusChange + if err := json.Unmarshal(h.Ctx.Input.CopyBody(1<<32), &data); err != nil { + log.Errorf("failed to decode hook event: %v", err) + return + } // status update if len(data.CheckIn) == 0 { schedulerID, err := h.GetInt64FromPath(":id") @@ -43,6 +50,7 @@ func (h *Handler) Handle() { h.SendInternalServerError(fmt.Errorf("failed to update status of job %s: %v", data.JobID, err)) return } + log.Debugf("handle status update hook event for schedule %s completed", h.GetStringFromPath(":id")) return } @@ -53,7 +61,7 @@ func (h *Handler) Handle() { log.Errorf("failed to unmarshal parameters from check in message: %v", err) return } - callbackFuncNameParam, exist := params["callback_func_name"] + callbackFuncNameParam, exist := params[scheduler.JobParamCallbackFunc] if !exist { log.Error("cannot get the parameter \"callback_func_name\" from the check in message") return @@ -63,8 +71,9 @@ func (h *Handler) Handle() { log.Errorf("invalid \"callback_func_name\": %v", callbackFuncName) return } - if err := hook.GlobalController.Run(callbackFuncName, params["params"]); err != nil { + if err := hook.GlobalController.Run(callbackFuncName, params[scheduler.JobParamCallbackFuncParams]); err != nil { log.Errorf("failed to run the callback function %s: %v", callbackFuncName, err) return } + log.Debugf("callback function %s called for schedule %s", callbackFuncName, h.GetStringFromPath(":id")) } diff --git a/src/core/service/token/creator.go b/src/core/service/token/creator.go index fbd229783..feca191e6 100644 --- a/src/core/service/token/creator.go +++ b/src/core/service/token/creator.go @@ -162,17 +162,17 @@ func (rep repositoryFilter) filter(ctx security.Context, pm promgr.ProjectManage projectName := img.namespace permission := "" - exist, err := pm.Exists(projectName) + project, err := pm.Get(projectName) if err != nil { return err } - if !exist { + if project == nil { log.Debugf("project %s does not exist, set empty permission", projectName) a.Actions = []string{} return nil } - resource := rbac.NewProjectNamespace(projectName).Resource(rbac.ResourceRepository) + resource := rbac.NewProjectNamespace(project.ProjectID).Resource(rbac.ResourceRepository) if ctx.Can(rbac.ActionPush, resource) && ctx.Can(rbac.ActionPull, resource) { permission = "RWM" } else if ctx.Can(rbac.ActionPush, resource) { diff --git a/src/core/utils/utils.go b/src/core/utils/utils.go index 7997227a8..e55f8a010 100644 --- a/src/core/utils/utils.go +++ b/src/core/utils/utils.go @@ -62,14 +62,19 @@ func newRepositoryClient(endpoint, username, repository string) (*registry.Repos // WaitForManifestReady implements exponential sleeep to wait until manifest is ready in registry. // This is a workaround for https://github.com/docker/distribution/issues/2625 func WaitForManifestReady(repository string, tag string, maxRetry int) bool { - // The initial wait interval, hard-coded to 50ms - interval := 50 * time.Millisecond + // The initial wait interval, hard-coded to 80ms, interval will be 80ms,200ms,500ms,1.25s,3.124999936s + interval := 80 * time.Millisecond repoClient, err := NewRepositoryClientForUI("harbor-core", repository) if err != nil { log.Errorf("Failed to create repo client.") return false } for i := 0; i < maxRetry; i++ { + if i != 0 { + log.Warningf("manifest for image %s:%s is not ready, retry after %v", repository, tag, interval) + time.Sleep(interval) + interval = time.Duration(int64(float32(interval) * 2.5)) + } _, exist, err := repoClient.ManifestExist(tag) if err != nil { log.Errorf("Unexpected error when checking manifest existence, image: %s:%s, error: %v", repository, tag, err) @@ -78,9 +83,6 @@ func WaitForManifestReady(repository string, tag string, maxRetry int) bool { if exist { return true } - log.Warningf("manifest for image %s:%s is not ready, retry after %v", repository, tag, interval) - time.Sleep(interval) - interval = interval * 2 } return false } diff --git a/src/core/views/404.tpl b/src/core/views/404.tpl index 88213a5d5..e6d0d6f2e 100644 --- a/src/core/views/404.tpl +++ b/src/core/views/404.tpl @@ -67,7 +67,7 @@ a.underline, .underline{ Page Not Found
-

Home

+

Home

diff --git a/src/jobservice/job/impl/gc/job.go b/src/jobservice/job/impl/gc/job.go index fef5bd30a..6c6b5f82b 100644 --- a/src/jobservice/job/impl/gc/job.go +++ b/src/jobservice/job/impl/gc/job.go @@ -22,10 +22,14 @@ import ( "github.com/garyburd/redigo/redis" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/config" + "github.com/goharbor/harbor/src/common/dao" + common_quota "github.com/goharbor/harbor/src/common/quota" "github.com/goharbor/harbor/src/common/registryctl" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/pkg/types" "github.com/goharbor/harbor/src/registryctl/client" + "strconv" ) const ( @@ -88,6 +92,9 @@ func (gc *GarbageCollector) Run(ctx job.Context, params job.Parameters) error { if err := gc.cleanCache(); err != nil { return err } + if err := gc.ensureQuota(); err != nil { + gc.logger.Warningf("failed to align quota data in gc job, with error: %v", err) + } gc.logger.Infof("GC results: status: %t, message: %s, start: %s, end: %s.", gcr.Status, gcr.Msg, gcr.StartTime, gcr.EndTime) gc.logger.Infof("success to run gc in job.") return nil @@ -193,3 +200,27 @@ func delKeys(con redis.Conn, pattern string) error { } return nil } + +func (gc *GarbageCollector) ensureQuota() error { + projects, err := dao.GetProjects(nil) + if err != nil { + return err + } + for _, project := range projects { + pSize, err := dao.CountSizeOfProject(project.ProjectID) + if err != nil { + gc.logger.Warningf("error happen on counting size of project:%d by artifact, error:%v, just skip it.", project.ProjectID, err) + continue + } + quotaMgr, err := common_quota.NewManager("project", strconv.FormatInt(project.ProjectID, 10)) + if err != nil { + gc.logger.Errorf("Error occurred when to new quota manager %v, just skip it.", err) + continue + } + if err := quotaMgr.SetResourceUsage(types.ResourceStorage, pSize); err != nil { + gc.logger.Errorf("cannot ensure quota for the project: %d, err: %v, just skip it.", project.ProjectID, err) + continue + } + } + return nil +} diff --git a/src/jobservice/job/impl/sample/job.go b/src/jobservice/job/impl/sample/job.go index 81101ecce..be860ec0e 100644 --- a/src/jobservice/job/impl/sample/job.go +++ b/src/jobservice/job/impl/sample/job.go @@ -17,6 +17,7 @@ package sample import ( "errors" "fmt" + "os" "strings" "time" @@ -67,6 +68,13 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error { fmt.Printf("Get prop form context: sample=%s\n", v) } + // For failure case + if len(os.Getenv("JOB_FAILED")) > 0 { + <-time.After(3 * time.Second) + logger.Info("Job exit with error because `JOB_FAILED` env is set") + return errors.New("`JOB_FAILED` env is set") + } + ctx.Checkin("progress data: %30") <-time.After(1 * time.Second) ctx.Checkin("progress data: %60") diff --git a/src/jobservice/job/models.go b/src/jobservice/job/models.go index 16c4ad708..1ae70679e 100644 --- a/src/jobservice/job/models.go +++ b/src/jobservice/job/models.go @@ -67,6 +67,7 @@ type StatsInfo struct { UpstreamJobID string `json:"upstream_job_id,omitempty"` // Ref the upstream job if existing NumericPID int64 `json:"numeric_policy_id,omitempty"` // The numeric policy ID of the periodic job Parameters Parameters `json:"parameters,omitempty"` + Revision int64 `json:"revision,omitempty"` // For differentiating the each retry of the same job } // ActionRequest defines for triggering job action like stop/cancel. diff --git a/src/jobservice/job/tracker.go b/src/jobservice/job/tracker.go index 0e70d338a..cbd3b2011 100644 --- a/src/jobservice/job/tracker.go +++ b/src/jobservice/job/tracker.go @@ -17,20 +17,23 @@ package job import ( "context" "encoding/json" + "math/rand" + "strconv" + "time" + "github.com/goharbor/harbor/src/jobservice/common/rds" "github.com/goharbor/harbor/src/jobservice/common/utils" "github.com/goharbor/harbor/src/jobservice/errs" "github.com/goharbor/harbor/src/jobservice/logger" "github.com/gomodule/redigo/redis" "github.com/pkg/errors" - "math/rand" - "strconv" - "time" ) const ( - // Try best to keep the job stats data but anyway clear it after a long time - statDataExpireTime = 180 * 24 * 3600 + // Try best to keep the job stats data but anyway clear it after a reasonable time + statDataExpireTime = 7 * 24 * 3600 + // 1 hour to discard the job stats of success jobs + statDataExpireTimeForSuccess = 3600 ) // Tracker is designed to track the life cycle of the job described by the stats @@ -96,6 +99,9 @@ type Tracker interface { // Switch the status to success Succeed() error + + // Reset the status to `pending` + Reset() error } // basicTracker implements Tracker interface based on redis @@ -233,22 +239,7 @@ func (bt *basicTracker) CheckIn(message string) error { // Expire job stats func (bt *basicTracker) Expire() error { - conn := bt.pool.Get() - defer func() { - _ = conn.Close() - }() - - key := rds.KeyJobStats(bt.namespace, bt.jobID) - num, err := conn.Do("EXPIRE", key, statDataExpireTime) - if err != nil { - return err - } - - if num == 0 { - return errors.Errorf("job stats for expiring %s does not exist", bt.jobID) - } - - return nil + return bt.expire(statDataExpireTime) } // Run job @@ -302,6 +293,13 @@ func (bt *basicTracker) Succeed() error { err := bt.UpdateStatusWithRetry(SuccessStatus) if !errs.IsStatusMismatchError(err) { bt.refresh(SuccessStatus) + + // Expire the stat data of the successful job + if er := bt.expire(statDataExpireTimeForSuccess); er != nil { + // Only logged + logger.Errorf("Expire stat data for the success job `%s` failed with error: %s", bt.jobID, er) + } + if er := bt.fireHookEvent(SuccessStatus); err == nil && er != nil { return er } @@ -361,6 +359,8 @@ func (bt *basicTracker) Save() (err error) { } // Set update timestamp args = append(args, "update_time", time.Now().Unix()) + // Set the first revision + args = append(args, "revision", time.Now().Unix()) // Do it in a transaction err = conn.Send("MULTI") @@ -419,6 +419,29 @@ func (bt *basicTracker) UpdateStatusWithRetry(targetStatus Status) error { return err } +// Reset the job status to `pending` and update the revision. +// Usually for the retry jobs +func (bt *basicTracker) Reset() error { + conn := bt.pool.Get() + defer func() { + closeConn(conn) + }() + + now := time.Now().Unix() + err := bt.Update( + "status", + PendingStatus.String(), + "revision", + now, + ) + if err == nil { + bt.refresh(PendingStatus) + bt.jobStats.Info.Revision = now + } + + return err +} + // Refresh the job stats in mem func (bt *basicTracker) refresh(targetStatus Status, checkIn ...string) { now := time.Now().Unix() @@ -571,20 +594,16 @@ func (bt *basicTracker) retrieve() error { res.Info.RefLink = value break case "enqueue_time": - v, _ := strconv.ParseInt(value, 10, 64) - res.Info.EnqueueTime = v + res.Info.EnqueueTime = parseInt64(value) break case "update_time": - v, _ := strconv.ParseInt(value, 10, 64) - res.Info.UpdateTime = v + res.Info.UpdateTime = parseInt64(value) break case "run_at": - v, _ := strconv.ParseInt(value, 10, 64) - res.Info.RunAt = v + res.Info.RunAt = parseInt64(value) break case "check_in_at": - v, _ := strconv.ParseInt(value, 10, 64) - res.Info.CheckInAt = v + res.Info.CheckInAt = parseInt64(value) break case "check_in": res.Info.CheckIn = value @@ -596,14 +615,12 @@ func (bt *basicTracker) retrieve() error { res.Info.WebHookURL = value break case "die_at": - v, _ := strconv.ParseInt(value, 10, 64) - res.Info.DieAt = v + res.Info.DieAt = parseInt64(value) case "upstream_job_id": res.Info.UpstreamJobID = value break case "numeric_policy_id": - v, _ := strconv.ParseInt(value, 10, 64) - res.Info.NumericPID = v + res.Info.NumericPID = parseInt64(value) break case "parameters": params := make(Parameters) @@ -611,6 +628,9 @@ func (bt *basicTracker) retrieve() error { res.Info.Parameters = params } break + case "revision": + res.Info.Revision = parseInt64(value) + break default: break } @@ -621,6 +641,25 @@ func (bt *basicTracker) retrieve() error { return nil } +func (bt *basicTracker) expire(expireTime int64) error { + conn := bt.pool.Get() + defer func() { + _ = conn.Close() + }() + + key := rds.KeyJobStats(bt.namespace, bt.jobID) + num, err := conn.Do("EXPIRE", key, expireTime) + if err != nil { + return err + } + + if num == 0 { + return errors.Errorf("job stats for expiring %s does not exist", bt.jobID) + } + + return nil +} + func getStatus(conn redis.Conn, key string) (Status, error) { values, err := rds.HmGet(conn, key, "status") if err != nil { @@ -640,3 +679,21 @@ func getStatus(conn redis.Conn, key string) (Status, error) { func setStatus(conn redis.Conn, key string, status Status) error { return rds.HmSet(conn, key, "status", status.String(), "update_time", time.Now().Unix()) } + +func closeConn(conn redis.Conn) { + if conn != nil { + if err := conn.Close(); err != nil { + logger.Errorf("Close redis connection failed with error: %s", err) + } + } +} + +func parseInt64(v string) int64 { + intV, err := strconv.ParseInt(v, 10, 64) + if err != nil { + logger.Errorf("Parse int64 error: %s", err) + return 0 + } + + return intV +} diff --git a/src/jobservice/period/enqueuer.go b/src/jobservice/period/enqueuer.go index 6303ac80e..7de9e54dd 100644 --- a/src/jobservice/period/enqueuer.go +++ b/src/jobservice/period/enqueuer.go @@ -15,11 +15,11 @@ package period import ( + "context" "fmt" "math/rand" "time" - "context" "github.com/gocraft/work" "github.com/goharbor/harbor/src/jobservice/common/rds" "github.com/goharbor/harbor/src/jobservice/common/utils" @@ -175,23 +175,14 @@ func (e *enqueuer) scheduleNextJobs(p *Policy, conn redis.Conn) { e.lastEnqueueErr = err logger.Errorf("Invalid corn spec in periodic policy %s %s: %s", p.JobName, p.ID, err) } else { - if p.JobParameters == nil { - p.JobParameters = make(job.Parameters) - } - - // Clone job parameters - wJobParams := make(job.Parameters) - if p.JobParameters != nil && len(p.JobParameters) > 0 { - for k, v := range p.JobParameters { - wJobParams[k] = v - } - } - // Add extra argument for job running - // Notes: Only for system using - wJobParams[PeriodicExecutionMark] = true for t := schedule.Next(nowTime); t.Before(horizon); t = schedule.Next(t) { epoch := t.Unix() + // Clone parameters + // Add extra argument for job running too. + // Notes: Only for system using + wJobParams := cloneParameters(p.JobParameters, epoch) + // Create an execution (job) based on the periodic job template (policy) j := &work.Job{ Name: p.JobName, @@ -316,3 +307,16 @@ func (e *enqueuer) shouldEnqueue() bool { return false } + +func cloneParameters(params job.Parameters, epoch int64) job.Parameters { + p := make(job.Parameters) + + // Clone parameters to a new param map + for k, v := range params { + p[k] = v + } + + p[PeriodicExecutionMark] = fmt.Sprintf("%d", epoch) + + return p +} diff --git a/src/jobservice/runner/redis.go b/src/jobservice/runner/redis.go index f8409b2d7..69cc94714 100644 --- a/src/jobservice/runner/redis.go +++ b/src/jobservice/runner/redis.go @@ -15,17 +15,18 @@ package runner import ( - "github.com/goharbor/harbor/src/jobservice/job/impl" - "runtime" - "fmt" + "runtime" + "time" + "github.com/gocraft/work" "github.com/goharbor/harbor/src/jobservice/env" "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/jobservice/job/impl" "github.com/goharbor/harbor/src/jobservice/lcm" "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/jobservice/period" "github.com/pkg/errors" - "time" ) // RedisJob is a job wrapper to wrap the job.Interface to the style which can be recognized by the redis worker. @@ -67,8 +68,10 @@ func (rj *RedisJob) Run(j *work.Job) (err error) { // Track the running job now jID := j.ID - if isPeriodicJobExecution(j) { - jID = fmt.Sprintf("%s@%d", j.ID, j.EnqueuedAt) + + // Check if the job is a periodic one as periodic job has its own ID format + if eID, yes := isPeriodicJobExecution(j); yes { + jID = eID } if tracker, err = rj.ctl.Track(jID); err != nil { @@ -85,11 +88,38 @@ func (rj *RedisJob) Run(j *work.Job) (err error) { return } - if job.RunningStatus.Compare(job.Status(tracker.Job().Info.Status)) <= 0 { + // Do operation based on the job status + jStatus := job.Status(tracker.Job().Info.Status) + switch jStatus { + case job.PendingStatus, job.ScheduledStatus: + // do nothing now + break + case job.StoppedStatus: // Probably jobs has been stopped by directly mark status to stopped. // Directly exit and no retry markStopped = bp(true) return nil + case job.ErrorStatus: + if j.FailedAt > 0 && j.Fails > 0 { + // Retry job + // Reset job info + if er := tracker.Reset(); er != nil { + // Log error and return the original error if existing + er = errors.Wrap(er, fmt.Sprintf("retrying job %s:%s failed", j.Name, j.ID)) + logger.Error(er) + + if len(j.LastErr) > 0 { + return errors.New(j.LastErr) + } + + return err + } + + logger.Infof("|*_*| Retrying job %s:%s, revision: %d", j.Name, j.ID, tracker.Job().Info.Revision) + } + break + default: + return errors.Errorf("mismatch status for running job: expected <%s <> got %s", job.RunningStatus.String(), jStatus.String()) } // Defer to switch status @@ -162,7 +192,7 @@ func (rj *RedisJob) Run(j *work.Job) (err error) { // Handle retry rj.retry(runningJob, j) // Handle periodic job execution - if isPeriodicJobExecution(j) { + if _, yes := isPeriodicJobExecution(j); yes { if er := tracker.PeriodicExecutionDone(); er != nil { // Just log it logger.Error(er) @@ -181,14 +211,9 @@ func (rj *RedisJob) retry(j job.Interface, wj *work.Job) { } } -func isPeriodicJobExecution(j *work.Job) bool { - if isPeriodic, ok := j.Args["_job_kind_periodic_"]; ok { - if isPeriodicV, yes := isPeriodic.(bool); yes && isPeriodicV { - return true - } - } - - return false +func isPeriodicJobExecution(j *work.Job) (string, bool) { + epoch, ok := j.Args[period.PeriodicExecutionMark] + return fmt.Sprintf("%s@%s", j.ID, epoch), ok } func bp(b bool) *bool { diff --git a/src/jobservice/worker/cworker/c_worker.go b/src/jobservice/worker/cworker/c_worker.go index 6de36856f..ffa8428f1 100644 --- a/src/jobservice/worker/cworker/c_worker.go +++ b/src/jobservice/worker/cworker/c_worker.go @@ -406,6 +406,7 @@ func (w *basicWorker) registerJob(name string, j interface{}) (err error) { name, work.JobOptions{ MaxFails: theJ.MaxFails(), + SkipDead: true, }, // Use generic handler to handle as we do not accept context with this way. func(job *work.Job) error { diff --git a/src/pkg/retention/controller.go b/src/pkg/retention/controller.go index d156bd8ce..d12ef3996 100644 --- a/src/pkg/retention/controller.go +++ b/src/pkg/retention/controller.go @@ -92,7 +92,7 @@ func (r *DefaultAPIController) GetRetention(id int64) (*policy.Metadata, error) func (r *DefaultAPIController) CreateRetention(p *policy.Metadata) (int64, error) { if p.Trigger.Kind == policy.TriggerKindSchedule { cron, ok := p.Trigger.Settings[policy.TriggerSettingsCron] - if ok { + if ok && len(cron.(string)) > 0 { jobid, err := r.scheduler.Schedule(cron.(string), SchedulerCallback, TriggerParam{ PolicyID: p.ID, Trigger: ExecutionTriggerSchedule, diff --git a/src/pkg/retention/controller_test.go b/src/pkg/retention/controller_test.go index 28202dd71..e8c150d97 100644 --- a/src/pkg/retention/controller_test.go +++ b/src/pkg/retention/controller_test.go @@ -152,12 +152,7 @@ func (s *ControllerTestSuite) TestExecution() { }, TagSelectors: []*rule.Selector{ { - Kind: "label", - Decoration: "with", - Pattern: "latest", - }, - { - Kind: "regularExpression", + Kind: "doublestar", Decoration: "matches", Pattern: "release-[\\d\\.]+", }, @@ -165,7 +160,7 @@ func (s *ControllerTestSuite) TestExecution() { ScopeSelectors: map[string][]*rule.Selector{ "repository": { { - Kind: "regularExpression", + Kind: "doublestar", Decoration: "matches", Pattern: ".+", }, diff --git a/src/pkg/retention/dao/models/retention.go b/src/pkg/retention/dao/models/retention.go index f1c5cce7b..b101a87dc 100644 --- a/src/pkg/retention/dao/models/retention.go +++ b/src/pkg/retention/dao/models/retention.go @@ -49,13 +49,15 @@ type RetentionExecution struct { // RetentionTask ... type RetentionTask struct { - ID int64 `orm:"pk;auto;column(id)"` - ExecutionID int64 `orm:"column(execution_id)"` - Repository string `orm:"column(repository)"` - JobID string `orm:"column(job_id)"` - Status string `orm:"column(status)"` - StartTime time.Time `orm:"column(start_time)"` - EndTime time.Time `orm:"column(end_time)"` - Total int `orm:"column(total)"` - Retained int `orm:"column(retained)"` + ID int64 `orm:"pk;auto;column(id)"` + ExecutionID int64 `orm:"column(execution_id)"` + Repository string `orm:"column(repository)"` + JobID string `orm:"column(job_id)"` + Status string `orm:"column(status)"` + StatusCode int `orm:"column(status_code)"` // For order the different statuses + StatusRevision int64 `orm:"column(status_revision)"` // For differentiating the each retry of the same job + StartTime time.Time `orm:"column(start_time)"` + EndTime time.Time `orm:"column(end_time)"` + Total int `orm:"column(total)"` + Retained int `orm:"column(retained)"` } diff --git a/src/pkg/retention/dao/retention.go b/src/pkg/retention/dao/retention.go index e10cf12e4..2a4e5970d 100644 --- a/src/pkg/retention/dao/retention.go +++ b/src/pkg/retention/dao/retention.go @@ -3,12 +3,14 @@ package dao import ( "errors" "fmt" + "strconv" + "time" + "github.com/astaxie/beego/orm" "github.com/goharbor/harbor/src/common/dao" - jobmodels "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/pkg/retention/dao/models" "github.com/goharbor/harbor/src/pkg/retention/q" - "strconv" ) // CreatePolicy Create Policy @@ -30,9 +32,6 @@ func DeletePolicyAndExec(id int64) error { if _, err := o.Raw("delete from retention_task where execution_id in (select id from retention_execution where policy_id = ?) ", id).Exec(); err != nil { return nil } - if _, err := o.Raw("delete from retention_execution where policy_id = ?", id).Exec(); err != nil { - return err - } if _, err := o.Delete(&models.RetentionExecution{ PolicyID: id, }); err != nil { @@ -72,7 +71,13 @@ func UpdateExecution(e *models.RetentionExecution, cols ...string) error { // DeleteExecution Delete Execution func DeleteExecution(id int64) error { o := dao.GetOrmer() - _, err := o.Delete(&models.RetentionExecution{ + _, err := o.Delete(&models.RetentionTask{ + ExecutionID: id, + }) + if err != nil { + return err + } + _, err = o.Delete(&models.RetentionExecution{ ID: id, }) return err @@ -111,21 +116,17 @@ func fillStatus(exec *models.RetentionExecution) error { } total += v switch k { - case jobmodels.JobScheduled: + case job.ScheduledStatus.String(): running += v - case jobmodels.JobPending: + case job.PendingStatus.String(): running += v - case jobmodels.JobRunning: + case job.RunningStatus.String(): running += v - case jobmodels.JobRetrying: - running += v - case jobmodels.JobFinished: + case job.SuccessStatus.String(): succeed += v - case jobmodels.JobCanceled: + case job.StoppedStatus.String(): stopped += v - case jobmodels.JobStopped: - stopped += v - case jobmodels.JobError: + case job.ErrorStatus.String(): failed += v } } @@ -228,6 +229,29 @@ func UpdateTask(task *models.RetentionTask, cols ...string) error { return err } +// UpdateTaskStatus updates the status of task according to the status code and revision to avoid +// override when running in concurrency +func UpdateTaskStatus(taskID int64, status string, statusCode int, statusRevision int64) error { + params := []interface{}{} + // use raw sql rather than the ORM as the sql generated by ORM isn't a "single" statement + // which means the operation isn't atomic + sql := `update retention_task set status = ?, status_code = ?, status_revision = ?, end_time = ? ` + params = append(params, status, statusCode, statusRevision) + var t time.Time + // when the task is in final status, update the endtime + // when the task re-runs again, the endtime should be cleared + // so set the endtime to null if the task isn't in final status + if IsFinalStatus(status) { + t = time.Now() + } + params = append(params, t) + sql += `where id = ? and + (status_revision = ? and status_code < ? or status_revision < ?) ` + params = append(params, taskID, statusRevision, statusCode, statusRevision) + _, err := dao.GetOrmer().Raw(sql, params).Exec() + return err +} + // DeleteTask deletes the task record specified by ID in database func DeleteTask(id int64) error { _, err := dao.GetOrmer().Delete(&models.RetentionTask{ @@ -276,3 +300,12 @@ func GetTotalOfTasks(executionID int64) (int64, error) { qs = qs.Filter("ExecutionID", executionID) return qs.Count() } + +// IsFinalStatus checks whether the status is a final status +func IsFinalStatus(status string) bool { + if status == job.StoppedStatus.String() || status == job.SuccessStatus.String() || + status == job.ErrorStatus.String() { + return true + } + return false +} diff --git a/src/pkg/retention/dao/retention_test.go b/src/pkg/retention/dao/retention_test.go index df7c757c7..b3fcdc85b 100644 --- a/src/pkg/retention/dao/retention_test.go +++ b/src/pkg/retention/dao/retention_test.go @@ -194,8 +194,12 @@ func TestTask(t *testing.T) { // update task.ID = id - task.Status = "running" - err = UpdateTask(task, "Status") + task.Total = 1 + err = UpdateTask(task, "Total") + require.Nil(t, err) + + // update status + err = UpdateTaskStatus(id, "running", 1, 1) require.Nil(t, err) // list @@ -205,8 +209,11 @@ func TestTask(t *testing.T) { }) require.Nil(t, err) require.Equal(t, 1, len(tasks)) + assert.Equal(t, 1, tasks[0].Total) assert.Equal(t, int64(1), tasks[0].ExecutionID) assert.Equal(t, "running", tasks[0].Status) + assert.Equal(t, 1, tasks[0].StatusCode) + assert.Equal(t, int64(1), tasks[0].StatusRevision) // delete err = DeleteTask(id) diff --git a/src/pkg/retention/job_test.go b/src/pkg/retention/job_test.go index cd9c137f1..cf7155b34 100644 --- a/src/pkg/retention/job_test.go +++ b/src/pkg/retention/job_test.go @@ -30,7 +30,6 @@ import ( "github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestps" "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar" - "github.com/goharbor/harbor/src/pkg/retention/res/selectors/label" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -90,10 +89,6 @@ func (suite *JobTestSuite) TestRunSuccess() { Template: latestps.TemplateID, Parameters: ruleParams, TagSelectors: []*rule.Selector{{ - Kind: label.Kind, - Decoration: label.With, - Pattern: "L3", - }, { Kind: doublestar.Kind, Decoration: doublestar.Matches, Pattern: "**", diff --git a/src/pkg/retention/launcher_test.go b/src/pkg/retention/launcher_test.go index fa94087ca..c63b7bf28 100644 --- a/src/pkg/retention/launcher_test.go +++ b/src/pkg/retention/launcher_test.go @@ -126,6 +126,9 @@ func (f *fakeRetentionManager) CreateTask(task *Task) (int64, error) { func (f *fakeRetentionManager) UpdateTask(task *Task, cols ...string) error { return nil } +func (f *fakeRetentionManager) UpdateTaskStatus(int64, string, int64) error { + return nil +} func (f *fakeRetentionManager) GetTaskLog(taskID int64) ([]byte, error) { return nil, nil } diff --git a/src/pkg/retention/manager.go b/src/pkg/retention/manager.go index ccb2f6339..4d3121f94 100644 --- a/src/pkg/retention/manager.go +++ b/src/pkg/retention/manager.go @@ -21,7 +21,8 @@ import ( "time" "github.com/astaxie/beego/orm" - "github.com/goharbor/harbor/src/common/job" + cjob "github.com/goharbor/harbor/src/common/job" + "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/pkg/retention/dao" "github.com/goharbor/harbor/src/pkg/retention/dao/models" "github.com/goharbor/harbor/src/pkg/retention/policy" @@ -58,6 +59,10 @@ type Manager interface { CreateTask(task *Task) (int64, error) // Update the specified task UpdateTask(task *Task, cols ...string) error + // Update the status of the specified task + // The status is updated only when (the statusRevision > the current revision) + // or (the the statusRevision = the current revision and status > the current status) + UpdateTaskStatus(taskID int64, status string, statusRevision int64) error // Get the task specified by the task ID GetTask(taskID int64) (*Task, error) // Get the log of the specified task @@ -188,14 +193,16 @@ func (d *DefaultManager) CreateTask(task *Task) (int64, error) { return 0, errors.New("nil task") } t := &models.RetentionTask{ - ExecutionID: task.ExecutionID, - Repository: task.Repository, - JobID: task.JobID, - Status: task.Status, - StartTime: task.StartTime, - EndTime: task.EndTime, - Total: task.Total, - Retained: task.Retained, + ExecutionID: task.ExecutionID, + Repository: task.Repository, + JobID: task.JobID, + Status: task.Status, + StatusCode: task.StatusCode, + StatusRevision: task.StatusRevision, + StartTime: task.StartTime, + EndTime: task.EndTime, + Total: task.Total, + Retained: task.Retained, } return dao.CreateTask(t) } @@ -212,15 +219,17 @@ func (d *DefaultManager) ListTasks(query ...*q.TaskQuery) ([]*Task, error) { tasks := make([]*Task, 0) for _, t := range ts { tasks = append(tasks, &Task{ - ID: t.ID, - ExecutionID: t.ExecutionID, - Repository: t.Repository, - JobID: t.JobID, - Status: t.Status, - StartTime: t.StartTime, - EndTime: t.EndTime, - Total: t.Total, - Retained: t.Retained, + ID: t.ID, + ExecutionID: t.ExecutionID, + Repository: t.Repository, + JobID: t.JobID, + Status: t.Status, + StatusCode: t.StatusCode, + StatusRevision: t.StatusRevision, + StartTime: t.StartTime, + EndTime: t.EndTime, + Total: t.Total, + Retained: t.Retained, }) } return tasks, nil @@ -240,18 +249,29 @@ func (d *DefaultManager) UpdateTask(task *Task, cols ...string) error { return fmt.Errorf("invalid task ID: %d", task.ID) } return dao.UpdateTask(&models.RetentionTask{ - ID: task.ID, - ExecutionID: task.ExecutionID, - Repository: task.Repository, - JobID: task.JobID, - Status: task.Status, - StartTime: task.StartTime, - EndTime: task.EndTime, - Total: task.Total, - Retained: task.Retained, + ID: task.ID, + ExecutionID: task.ExecutionID, + Repository: task.Repository, + JobID: task.JobID, + Status: task.Status, + StatusCode: task.StatusCode, + StatusRevision: task.StatusRevision, + StartTime: task.StartTime, + EndTime: task.EndTime, + Total: task.Total, + Retained: task.Retained, }, cols...) } +// UpdateTaskStatus updates the status of the specified task +func (d *DefaultManager) UpdateTaskStatus(taskID int64, status string, statusRevision int64) error { + if taskID <= 0 { + return fmt.Errorf("invalid task ID: %d", taskID) + } + st := job.Status(status) + return dao.UpdateTaskStatus(taskID, status, st.Code(), statusRevision) +} + // GetTask returns the task specified by task ID func (d *DefaultManager) GetTask(taskID int64) (*Task, error) { if taskID <= 0 { @@ -262,15 +282,17 @@ func (d *DefaultManager) GetTask(taskID int64) (*Task, error) { return nil, err } return &Task{ - ID: task.ID, - ExecutionID: task.ExecutionID, - Repository: task.Repository, - JobID: task.JobID, - Status: task.Status, - StartTime: task.StartTime, - EndTime: task.EndTime, - Total: task.Total, - Retained: task.Retained, + ID: task.ID, + ExecutionID: task.ExecutionID, + Repository: task.Repository, + JobID: task.JobID, + Status: task.Status, + StatusCode: task.StatusCode, + StatusRevision: task.StatusRevision, + StartTime: task.StartTime, + EndTime: task.EndTime, + Total: task.Total, + Retained: task.Retained, }, nil } @@ -283,7 +305,7 @@ func (d *DefaultManager) GetTaskLog(taskID int64) ([]byte, error) { if task == nil { return nil, fmt.Errorf("task %d not found", taskID) } - return job.GlobalClient.GetJobLog(task.JobID) + return cjob.GlobalClient.GetJobLog(task.JobID) } // NewManager ... diff --git a/src/pkg/retention/manager_test.go b/src/pkg/retention/manager_test.go index b8310af8c..83e1ab0ef 100644 --- a/src/pkg/retention/manager_test.go +++ b/src/pkg/retention/manager_test.go @@ -6,8 +6,8 @@ import ( "time" "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/job" + jjob "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/pkg/retention/policy" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" "github.com/goharbor/harbor/src/pkg/retention/q" @@ -168,11 +168,16 @@ func TestExecution(t *testing.T) { func TestTask(t *testing.T) { m := NewManager() + err := m.DeleteExecution(1000) + require.Nil(t, err) task := &Task{ - ExecutionID: 1, - JobID: "1", - Status: TaskStatusPending, - StartTime: time.Now(), + ExecutionID: 1000, + JobID: "1", + Status: jjob.PendingStatus.String(), + StatusCode: jjob.PendingStatus.Code(), + StatusRevision: 1, + Total: 0, + StartTime: time.Now(), } // create id, err := m.CreateTask(task) @@ -181,27 +186,39 @@ func TestTask(t *testing.T) { // get tk, err := m.GetTask(id) require.Nil(t, err) - assert.EqualValues(t, 1, tk.ExecutionID) + assert.EqualValues(t, 1000, tk.ExecutionID) // update task.ID = id - task.Status = TaskStatusInProgress - err = m.UpdateTask(task, "Status") + task.Total = 1 + err = m.UpdateTask(task, "Total") + require.Nil(t, err) + + // update status to success which is a final status + err = m.UpdateTaskStatus(id, jjob.SuccessStatus.String(), 1) + require.Nil(t, err) + + // try to update status to running, as the status has already + // been updated to a final status and the stautus revision doesn't change, + // this updating shouldn't take effect + err = m.UpdateTaskStatus(id, jjob.RunningStatus.String(), 1) + require.Nil(t, err) + + // update the revision and try to update status to running again + err = m.UpdateTaskStatus(id, jjob.RunningStatus.String(), 2) require.Nil(t, err) // list tasks, err := m.ListTasks(&q.TaskQuery{ - ExecutionID: 1, - Status: TaskStatusInProgress, + ExecutionID: 1000, }) require.Nil(t, err) require.Equal(t, 1, len(tasks)) - assert.Equal(t, int64(1), tasks[0].ExecutionID) - assert.Equal(t, TaskStatusInProgress, tasks[0].Status) - - task.Status = TaskStatusFailed - err = m.UpdateTask(task, "Status") - require.Nil(t, err) + assert.Equal(t, int64(1000), tasks[0].ExecutionID) + assert.Equal(t, 1, tasks[0].Total) + assert.Equal(t, jjob.RunningStatus.String(), tasks[0].Status) + assert.Equal(t, jjob.RunningStatus.Code(), tasks[0].StatusCode) + assert.Equal(t, int64(2), tasks[0].StatusRevision) // get task log job.GlobalClient = &tjob.MockJobClient{ diff --git a/src/pkg/retention/models.go b/src/pkg/retention/models.go index 6b2daab9a..920b98e7d 100644 --- a/src/pkg/retention/models.go +++ b/src/pkg/retention/models.go @@ -23,12 +23,6 @@ const ( ExecutionStatusFailed string = "Failed" ExecutionStatusStopped string = "Stopped" - TaskStatusPending string = "Pending" - TaskStatusInProgress string = "InProgress" - TaskStatusSucceed string = "Succeed" - TaskStatusFailed string = "Failed" - TaskStatusStopped string = "Stopped" - CandidateKindImage string = "image" CandidateKindChart string = "chart" @@ -49,15 +43,17 @@ type Execution struct { // Task of retention type Task struct { - ID int64 `json:"id"` - ExecutionID int64 `json:"execution_id"` - Repository string `json:"repository"` - JobID string `json:"job_id"` - Status string `json:"status"` - StartTime time.Time `json:"start_time"` - EndTime time.Time `json:"end_time"` - Total int `json:"total"` - Retained int `json:"retained"` + ID int64 `json:"id"` + ExecutionID int64 `json:"execution_id"` + Repository string `json:"repository"` + JobID string `json:"job_id"` + Status string `json:"status"` + StatusCode int `json:"status_code"` + StatusRevision int64 `json:"status_revision"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` + Total int `json:"total"` + Retained int `json:"retained"` } // History of retention diff --git a/src/pkg/retention/policy/models.go b/src/pkg/retention/policy/models.go index 7fd48c205..94a6adff0 100644 --- a/src/pkg/retention/policy/models.go +++ b/src/pkg/retention/policy/models.go @@ -67,7 +67,6 @@ func (m *Metadata) Valid(v *validation.Validation) { _ = v.SetError("Trigger.Settings", "cron in Trigger.Settings is required") } } - } } diff --git a/src/pkg/retention/policy/rule/dayspl/evaluator.go b/src/pkg/retention/policy/rule/dayspl/evaluator.go index 9b2fb34e9..c0fd76256 100644 --- a/src/pkg/retention/policy/rule/dayspl/evaluator.go +++ b/src/pkg/retention/policy/rule/dayspl/evaluator.go @@ -58,12 +58,13 @@ func (e *evaluator) Action() string { func New(params rule.Parameters) rule.Evaluator { if params != nil { if p, ok := params[ParameterN]; ok { - if v, ok := p.(int); ok && v >= 0 { - return &evaluator{n: v} + if v, ok := p.(float64); ok && v >= 0 { + return &evaluator{n: int(v)} } } } - log.Debugf("default parameter %d used for rule %s", DefaultN, TemplateID) + log.Warningf("default parameter %d used for rule %s", DefaultN, TemplateID) + return &evaluator{n: DefaultN} } diff --git a/src/pkg/retention/policy/rule/dayspl/evaluator_test.go b/src/pkg/retention/policy/rule/dayspl/evaluator_test.go index 0c8ba1ec1..a8587ccd8 100644 --- a/src/pkg/retention/policy/rule/dayspl/evaluator_test.go +++ b/src/pkg/retention/policy/rule/dayspl/evaluator_test.go @@ -15,7 +15,7 @@ package dayspl import ( - "strconv" + "fmt" "testing" "time" @@ -36,8 +36,8 @@ func (e *EvaluatorTestSuite) TestNew() { args rule.Parameters expectedN int }{ - {Name: "Valid", args: map[string]rule.Parameter{ParameterN: 5}, expectedN: 5}, - {Name: "Default If Negative", args: map[string]rule.Parameter{ParameterN: -1}, expectedN: DefaultN}, + {Name: "Valid", args: map[string]rule.Parameter{ParameterN: float64(5)}, expectedN: 5}, + {Name: "Default If Negative", args: map[string]rule.Parameter{ParameterN: float64(-1)}, expectedN: DefaultN}, {Name: "Default If Not Set", args: map[string]rule.Parameter{}, expectedN: DefaultN}, {Name: "Default If Wrong Type", args: map[string]rule.Parameter{ParameterN: "foo"}, expectedN: DefaultN}, } @@ -65,7 +65,7 @@ func (e *EvaluatorTestSuite) TestProcess() { } tests := []struct { - n int + n float64 expected int minPullTime int64 }{ @@ -80,7 +80,7 @@ func (e *EvaluatorTestSuite) TestProcess() { } for _, tt := range tests { - e.T().Run(strconv.Itoa(tt.n), func(t *testing.T) { + e.T().Run(fmt.Sprintf("%v", tt.n), func(t *testing.T) { sut := New(map[string]rule.Parameter{ParameterN: tt.n}) result, err := sut.Process(data) diff --git a/src/pkg/retention/policy/rule/daysps/evaluator.go b/src/pkg/retention/policy/rule/daysps/evaluator.go index 2c121dde7..ee4dd436d 100644 --- a/src/pkg/retention/policy/rule/daysps/evaluator.go +++ b/src/pkg/retention/policy/rule/daysps/evaluator.go @@ -58,12 +58,13 @@ func (e *evaluator) Action() string { func New(params rule.Parameters) rule.Evaluator { if params != nil { if p, ok := params[ParameterN]; ok { - if v, ok := p.(int); ok && v >= 0 { - return &evaluator{n: v} + if v, ok := p.(float64); ok && v >= 0 { + return &evaluator{n: int(v)} } } } - log.Debugf("default parameter %d used for rule %s", DefaultN, TemplateID) + log.Warningf("default parameter %d used for rule %s", DefaultN, TemplateID) + return &evaluator{n: DefaultN} } diff --git a/src/pkg/retention/policy/rule/daysps/evaluator_test.go b/src/pkg/retention/policy/rule/daysps/evaluator_test.go index 07c9f9bdd..75287ce4f 100644 --- a/src/pkg/retention/policy/rule/daysps/evaluator_test.go +++ b/src/pkg/retention/policy/rule/daysps/evaluator_test.go @@ -15,7 +15,7 @@ package daysps import ( - "strconv" + "fmt" "testing" "time" @@ -36,8 +36,8 @@ func (e *EvaluatorTestSuite) TestNew() { args rule.Parameters expectedN int }{ - {Name: "Valid", args: map[string]rule.Parameter{ParameterN: 5}, expectedN: 5}, - {Name: "Default If Negative", args: map[string]rule.Parameter{ParameterN: -1}, expectedN: DefaultN}, + {Name: "Valid", args: map[string]rule.Parameter{ParameterN: float64(5)}, expectedN: 5}, + {Name: "Default If Negative", args: map[string]rule.Parameter{ParameterN: float64(-1)}, expectedN: DefaultN}, {Name: "Default If Not Set", args: map[string]rule.Parameter{}, expectedN: DefaultN}, {Name: "Default If Wrong Type", args: map[string]rule.Parameter{ParameterN: "foo"}, expectedN: DefaultN}, } @@ -65,7 +65,7 @@ func (e *EvaluatorTestSuite) TestProcess() { } tests := []struct { - n int + n float64 expected int minPushTime int64 }{ @@ -80,7 +80,7 @@ func (e *EvaluatorTestSuite) TestProcess() { } for _, tt := range tests { - e.T().Run(strconv.Itoa(tt.n), func(t *testing.T) { + e.T().Run(fmt.Sprintf("%v", tt.n), func(t *testing.T) { sut := New(map[string]rule.Parameter{ParameterN: tt.n}) result, err := sut.Process(data) diff --git a/src/pkg/retention/policy/rule/index/index.go b/src/pkg/retention/policy/rule/index/index.go index 40a4cccc0..7360a2cb8 100644 --- a/src/pkg/retention/policy/rule/index/index.go +++ b/src/pkg/retention/policy/rule/index/index.go @@ -26,7 +26,6 @@ import ( "github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestk" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestpl" "github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestps" - "github.com/goharbor/harbor/src/pkg/retention/policy/rule/nothing" "github.com/pkg/errors" ) @@ -122,11 +121,11 @@ func init() { }, lastx.New) // Register nothing - Register(&Metadata{ - TemplateID: nothing.TemplateID, - Action: action.Retain, - Parameters: []*IndexedParam{}, - }, nothing.New) + // Register(&Metadata{ + // TemplateID: nothing.TemplateID, + // Action: action.Retain, + // Parameters: []*IndexedParam{}, + // }, nothing.New) // Register always Register(&Metadata{ diff --git a/src/pkg/retention/policy/rule/index/index_test.go b/src/pkg/retention/policy/rule/index/index_test.go index b55d29f79..fd8268f18 100644 --- a/src/pkg/retention/policy/rule/index/index_test.go +++ b/src/pkg/retention/policy/rule/index/index_test.go @@ -84,7 +84,7 @@ func (suite *IndexTestSuite) TestGet() { // TestIndex tests Index func (suite *IndexTestSuite) TestIndex() { metas := Index() - require.Equal(suite.T(), 9, len(metas)) + require.Equal(suite.T(), 8, len(metas)) assert.Condition(suite.T(), func() bool { for _, m := range metas { if m.TemplateID == "fakeEvaluator" && diff --git a/src/pkg/retention/policy/rule/lastx/evaluator.go b/src/pkg/retention/policy/rule/lastx/evaluator.go index 6d98c5c2d..b466f5eda 100644 --- a/src/pkg/retention/policy/rule/lastx/evaluator.go +++ b/src/pkg/retention/policy/rule/lastx/evaluator.go @@ -59,15 +59,15 @@ func (e *evaluator) Action() string { func New(params rule.Parameters) rule.Evaluator { if params != nil { if param, ok := params[ParameterX]; ok { - if v, ok := param.(int); ok && v >= 0 { + if v, ok := param.(float64); ok && v >= 0 { return &evaluator{ - x: v, + x: int(v), } } } } - log.Debugf("default parameter %d used for rule %s", DefaultX, TemplateID) + log.Warningf("default parameter %d used for rule %s", DefaultX, TemplateID) return &evaluator{ x: DefaultX, diff --git a/src/pkg/retention/policy/rule/lastx/evaluator_test.go b/src/pkg/retention/policy/rule/lastx/evaluator_test.go index eafc30f2f..becd79234 100644 --- a/src/pkg/retention/policy/rule/lastx/evaluator_test.go +++ b/src/pkg/retention/policy/rule/lastx/evaluator_test.go @@ -21,8 +21,8 @@ func (e *EvaluatorTestSuite) TestNew() { args rule.Parameters expectedX int }{ - {Name: "Valid", args: map[string]rule.Parameter{ParameterX: 3}, expectedX: 3}, - {Name: "Default If Negative", args: map[string]rule.Parameter{ParameterX: -3}, expectedX: DefaultX}, + {Name: "Valid", args: map[string]rule.Parameter{ParameterX: float64(3)}, expectedX: 3}, + {Name: "Default If Negative", args: map[string]rule.Parameter{ParameterX: float64(-3)}, expectedX: DefaultX}, {Name: "Default If Not Set", args: map[string]rule.Parameter{}, expectedX: DefaultX}, {Name: "Default If Wrong Type", args: map[string]rule.Parameter{}, expectedX: DefaultX}, } @@ -48,7 +48,7 @@ func (e *EvaluatorTestSuite) TestProcess() { } tests := []struct { - days int + days float64 expected int }{ {days: 0, expected: 0}, @@ -62,7 +62,7 @@ func (e *EvaluatorTestSuite) TestProcess() { } for _, tt := range tests { - e.T().Run(fmt.Sprintf("%d days - should keep %d", tt.days, tt.expected), func(t *testing.T) { + e.T().Run(fmt.Sprintf("%v days - should keep %d", tt.days, tt.expected), func(t *testing.T) { e := New(rule.Parameters{ParameterX: tt.days}) result, err := e.Process(data) diff --git a/src/pkg/retention/policy/rule/latestk/evaluator.go b/src/pkg/retention/policy/rule/latestk/evaluator.go index bb1d246de..f6d73599a 100644 --- a/src/pkg/retention/policy/rule/latestk/evaluator.go +++ b/src/pkg/retention/policy/rule/latestk/evaluator.go @@ -65,9 +65,9 @@ func (e *evaluator) Action() string { func New(params rule.Parameters) rule.Evaluator { if params != nil { if param, ok := params[ParameterK]; ok { - if v, ok := param.(int); ok && v >= 0 { + if v, ok := param.(float64); ok && v >= 0 { return &evaluator{ - k: v, + k: int(v), } } } diff --git a/src/pkg/retention/policy/rule/latestk/evaluator_test.go b/src/pkg/retention/policy/rule/latestk/evaluator_test.go index ab2967f51..24b04fb9e 100644 --- a/src/pkg/retention/policy/rule/latestk/evaluator_test.go +++ b/src/pkg/retention/policy/rule/latestk/evaluator_test.go @@ -15,7 +15,7 @@ package latestk import ( - "strconv" + "fmt" "testing" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" @@ -58,7 +58,7 @@ func (e *EvaluatorTestSuite) TestProcess() { {k: 99, expected: len(e.artifacts)}, } for _, tt := range tests { - e.T().Run(strconv.Itoa(tt.k), func(t *testing.T) { + e.T().Run(fmt.Sprintf("%v", tt.k), func(t *testing.T) { sut := &evaluator{k: tt.k} result, err := sut.Process(e.artifacts) @@ -79,8 +79,8 @@ func (e *EvaluatorTestSuite) TestNew() { params rule.Parameters expectedK int }{ - {name: "Valid", params: rule.Parameters{ParameterK: 5}, expectedK: 5}, - {name: "Default If Negative", params: rule.Parameters{ParameterK: -5}, expectedK: DefaultK}, + {name: "Valid", params: rule.Parameters{ParameterK: float64(5)}, expectedK: 5}, + {name: "Default If Negative", params: rule.Parameters{ParameterK: float64(-5)}, expectedK: DefaultK}, {name: "Default If Wrong Type", params: rule.Parameters{ParameterK: "5"}, expectedK: DefaultK}, {name: "Default If Wrong Key", params: rule.Parameters{"n": 5}, expectedK: DefaultK}, {name: "Default If Empty", params: rule.Parameters{}, expectedK: DefaultK}, diff --git a/src/pkg/retention/policy/rule/latestpl/evaluator.go b/src/pkg/retention/policy/rule/latestpl/evaluator.go index 620790a73..bed7b6e4e 100644 --- a/src/pkg/retention/policy/rule/latestpl/evaluator.go +++ b/src/pkg/retention/policy/rule/latestpl/evaluator.go @@ -59,13 +59,13 @@ func (e *evaluator) Action() string { func New(params rule.Parameters) rule.Evaluator { if params != nil { if p, ok := params[ParameterN]; ok { - if v, ok := p.(int); ok && v >= 0 { - return &evaluator{n: v} + if v, ok := p.(float64); ok && v >= 0 { + return &evaluator{n: int(v)} } } } - log.Debugf("default parameter %d used for rule %s", DefaultN, TemplateID) + log.Warningf("default parameter %d used for rule %s", DefaultN, TemplateID) return &evaluator{n: DefaultN} } diff --git a/src/pkg/retention/policy/rule/latestpl/evaluator_test.go b/src/pkg/retention/policy/rule/latestpl/evaluator_test.go index 443481649..69b0605f5 100644 --- a/src/pkg/retention/policy/rule/latestpl/evaluator_test.go +++ b/src/pkg/retention/policy/rule/latestpl/evaluator_test.go @@ -15,8 +15,8 @@ package latestpl import ( + "fmt" "math/rand" - "strconv" "testing" "github.com/goharbor/harbor/src/pkg/retention/policy/rule" @@ -35,8 +35,8 @@ func (e *EvaluatorTestSuite) TestNew() { args rule.Parameters expectedK int }{ - {Name: "Valid", args: map[string]rule.Parameter{ParameterN: 5}, expectedK: 5}, - {Name: "Default If Negative", args: map[string]rule.Parameter{ParameterN: -1}, expectedK: DefaultN}, + {Name: "Valid", args: map[string]rule.Parameter{ParameterN: float64(5)}, expectedK: 5}, + {Name: "Default If Negative", args: map[string]rule.Parameter{ParameterN: float64(-1)}, expectedK: DefaultN}, {Name: "Default If Not Set", args: map[string]rule.Parameter{}, expectedK: DefaultN}, {Name: "Default If Wrong Type", args: map[string]rule.Parameter{ParameterN: "foo"}, expectedK: DefaultN}, } @@ -57,7 +57,7 @@ func (e *EvaluatorTestSuite) TestProcess() { }) tests := []struct { - n int + n float64 expected int minPullTime int64 }{ @@ -69,7 +69,7 @@ func (e *EvaluatorTestSuite) TestProcess() { } for _, tt := range tests { - e.T().Run(strconv.Itoa(tt.n), func(t *testing.T) { + e.T().Run(fmt.Sprintf("%v", tt.n), func(t *testing.T) { ev := New(map[string]rule.Parameter{ParameterN: tt.n}) result, err := ev.Process(data) diff --git a/src/pkg/retention/policy/rule/latestps/evaluator.go b/src/pkg/retention/policy/rule/latestps/evaluator.go index 8ac090b3f..ac000a302 100644 --- a/src/pkg/retention/policy/rule/latestps/evaluator.go +++ b/src/pkg/retention/policy/rule/latestps/evaluator.go @@ -62,15 +62,15 @@ func (e *evaluator) Action() string { func New(params rule.Parameters) rule.Evaluator { if params != nil { if param, ok := params[ParameterK]; ok { - if v, ok := param.(int); ok && v >= 0 { + if v, ok := param.(float64); ok && v >= 0 { return &evaluator{ - k: v, + k: int(v), } } } } - log.Debugf("default parameter %d used for rule %s", DefaultK, TemplateID) + log.Warningf("default parameter %d used for rule %s", DefaultK, TemplateID) return &evaluator{ k: DefaultK, diff --git a/src/pkg/retention/policy/rule/latestps/evaluator_test.go b/src/pkg/retention/policy/rule/latestps/evaluator_test.go index 7136b69d6..6e303c3c4 100644 --- a/src/pkg/retention/policy/rule/latestps/evaluator_test.go +++ b/src/pkg/retention/policy/rule/latestps/evaluator_test.go @@ -1,8 +1,8 @@ package latestps import ( + "fmt" "math/rand" - "strconv" "testing" "github.com/stretchr/testify/suite" @@ -22,8 +22,8 @@ func (e *EvaluatorTestSuite) TestNew() { args rule.Parameters expectedK int }{ - {Name: "Valid", args: map[string]rule.Parameter{ParameterK: 5}, expectedK: 5}, - {Name: "Default If Negative", args: map[string]rule.Parameter{ParameterK: -1}, expectedK: DefaultK}, + {Name: "Valid", args: map[string]rule.Parameter{ParameterK: float64(5)}, expectedK: 5}, + {Name: "Default If Negative", args: map[string]rule.Parameter{ParameterK: float64(-1)}, expectedK: DefaultK}, {Name: "Default If Not Set", args: map[string]rule.Parameter{}, expectedK: DefaultK}, {Name: "Default If Wrong Type", args: map[string]rule.Parameter{ParameterK: "foo"}, expectedK: DefaultK}, } @@ -44,7 +44,7 @@ func (e *EvaluatorTestSuite) TestProcess() { }) tests := []struct { - k int + k float64 expected int }{ {k: 0, expected: 0}, @@ -55,7 +55,7 @@ func (e *EvaluatorTestSuite) TestProcess() { } for _, tt := range tests { - e.T().Run(strconv.Itoa(tt.k), func(t *testing.T) { + e.T().Run(fmt.Sprintf("%v", tt.k), func(t *testing.T) { e := New(map[string]rule.Parameter{ParameterK: tt.k}) result, err := e.Process(data) diff --git a/src/pkg/retention/policy/rule/models.go b/src/pkg/retention/policy/rule/models.go index 448b10183..4e85872be 100644 --- a/src/pkg/retention/policy/rule/models.go +++ b/src/pkg/retention/policy/rule/models.go @@ -45,11 +45,11 @@ type Metadata struct { // Selector to narrow down the list type Selector struct { // Kind of the selector - // "regularExpression" or "label" - Kind string `json:"kind" valid:"Required"` + // "doublestar" or "label" + Kind string `json:"kind" valid:"Required;Match(doublestar)"` // Decorated the selector - // for "regularExpression" : "matches" and "excludes" + // for "doublestar" : "matching" and "excluding" // for "label" : "with" and "without" Decoration string `json:"decoration" valid:"Required"` diff --git a/src/pkg/retention/res/selectors/index/index.go b/src/pkg/retention/res/selectors/index/index.go index fe00c4f4b..690beef2d 100644 --- a/src/pkg/retention/res/selectors/index/index.go +++ b/src/pkg/retention/res/selectors/index/index.go @@ -17,8 +17,6 @@ package index import ( "sync" - "github.com/goharbor/harbor/src/pkg/retention/res/selectors/label" - "github.com/goharbor/harbor/src/pkg/retention/res" "github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar" "github.com/pkg/errors" @@ -36,7 +34,7 @@ func init() { }, doublestar.New) // Register label selector - Register(label.Kind, []string{label.With, label.Without}, label.New) + // Register(label.Kind, []string{label.With, label.Without}, label.New) } // index for keeping the mapping between selector meta and its implementation diff --git a/src/pkg/scan/whitelist/manager.go b/src/pkg/scan/whitelist/manager.go index 5aa793b9a..d582e3f10 100644 --- a/src/pkg/scan/whitelist/manager.go +++ b/src/pkg/scan/whitelist/manager.go @@ -17,6 +17,7 @@ package whitelist import ( "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/jobservice/logger" ) @@ -41,7 +42,7 @@ func (d *defaultManager) CreateEmpty(projectID int64) error { l := models.CVEWhitelist{ ProjectID: projectID, } - _, err := dao.UpdateCVEWhitelist(l) + _, err := dao.CreateCVEWhitelist(l) if err != nil { logger.Errorf("Failed to create empty CVE whitelist for project: %d, error: %v", projectID, err) } @@ -60,7 +61,12 @@ func (d *defaultManager) Set(projectID int64, list models.CVEWhitelist) error { // Get gets the whitelist for given project func (d *defaultManager) Get(projectID int64) (*models.CVEWhitelist, error) { - return dao.GetCVEWhitelist(projectID) + wl, err := dao.GetCVEWhitelist(projectID) + if wl == nil && err == nil { + log.Debugf("No CVE whitelist found for project %d, returning empty list.", projectID) + return &models.CVEWhitelist{ProjectID: projectID, Items: []models.CVEWhitelistItem{}}, nil + } + return wl, err } // SetSys sets the system level whitelist diff --git a/src/pkg/scan/whitelist/manager_test.go b/src/pkg/scan/whitelist/manager_test.go new file mode 100644 index 000000000..8dbf6da37 --- /dev/null +++ b/src/pkg/scan/whitelist/manager_test.go @@ -0,0 +1,46 @@ +package whitelist + +import ( + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/stretchr/testify/assert" + "os" + "testing" +) + +func TestMain(m *testing.M) { + + // databases := []string{"mysql", "sqlite"} + databases := []string{"postgresql"} + for _, database := range databases { + log.Infof("run test cases for database: %s", database) + + result := 1 + switch database { + case "postgresql": + dao.PrepareTestForPostgresSQL() + default: + log.Fatalf("invalid database: %s", database) + } + + result = m.Run() + + if result != 0 { + os.Exit(result) + } + } +} + +func TestDefaultManager_CreateEmpty(t *testing.T) { + dm := NewDefaultManager() + assert.NoError(t, dm.CreateEmpty(99)) + assert.Error(t, dm.CreateEmpty(99)) +} + +func TestDefaultManager_Get(t *testing.T) { + dm := NewDefaultManager() + // return empty list + l, err := dm.Get(1234) + assert.Nil(t, err) + assert.Empty(t, l.Items) +} diff --git a/src/pkg/scheduler/scheduler.go b/src/pkg/scheduler/scheduler.go index c0fa44d68..6fb7d7e87 100644 --- a/src/pkg/scheduler/scheduler.go +++ b/src/pkg/scheduler/scheduler.go @@ -15,6 +15,7 @@ package scheduler import ( + "encoding/json" "fmt" "net/http" "sync" @@ -29,9 +30,10 @@ import ( "github.com/pkg/errors" ) +// const definitions const ( - jobParamCallbackFunc = "callback_func" - jobParamCallbackFuncParams = "params" + JobParamCallbackFunc = "callback_func" + JobParamCallbackFuncParams = "params" ) var ( @@ -46,6 +48,8 @@ type CallbackFunc func(interface{}) error // Scheduler provides the capability to run a periodic task, a callback function // needs to be registered before using the scheduler +// The "params" is passed to the callback function specified by "callbackFuncName" +// as encoded json string, so the callback function must decode it before using type Scheduler interface { Schedule(cron string, callbackFuncName string, params interface{}) (int64, error) UnSchedule(id int64) error @@ -119,6 +123,15 @@ func (s *scheduler) Schedule(cron string, callbackFuncName string, params interf if err != nil { return 0, err } + // if got error in the following steps, delete the schedule record in database + defer func() { + if err != nil { + e := s.manager.Delete(scheduleID) + if e != nil { + log.Errorf("failed to delete the schedule %d: %v", scheduleID, e) + } + } + }() log.Debugf("the schedule record %d created", scheduleID) // submit scheduler job to Jobservice @@ -126,8 +139,7 @@ func (s *scheduler) Schedule(cron string, callbackFuncName string, params interf jd := &models.JobData{ Name: JobNameScheduler, Parameters: map[string]interface{}{ - jobParamCallbackFunc: callbackFuncName, - jobParamCallbackFuncParams: params, + JobParamCallbackFunc: callbackFuncName, }, Metadata: &models.JobMetadata{ JobKind: job.JobKindPeriodic, @@ -135,15 +147,26 @@ func (s *scheduler) Schedule(cron string, callbackFuncName string, params interf }, StatusHook: statusHookURL, } + if params != nil { + var paramsData []byte + paramsData, err = json.Marshal(params) + if err != nil { + return 0, err + } + jd.Parameters[JobParamCallbackFuncParams] = string(paramsData) + } jobID, err := s.jobserviceClient.SubmitJob(jd) if err != nil { - // if failed to submit to Jobservice, delete the schedule record in database - e := s.manager.Delete(scheduleID) - if e != nil { - log.Errorf("failed to delete the schedule %d: %v", scheduleID, e) - } return 0, err } + // if got error in the following steps, stop the scheduler job + defer func() { + if err != nil { + if e := s.jobserviceClient.PostAction(jobID, job.JobActionStop); e != nil { + log.Errorf("failed to stop the scheduler job %s: %v", jobID, e) + } + } + }() log.Debugf("the scheduler job submitted to Jobservice, job ID: %s", jobID) // populate the job ID for the schedule @@ -152,14 +175,6 @@ func (s *scheduler) Schedule(cron string, callbackFuncName string, params interf JobID: jobID, }, "JobID") if err != nil { - // stop the scheduler job - if e := s.jobserviceClient.PostAction(jobID, job.JobActionStop); e != nil { - log.Errorf("failed to stop the scheduler job %s: %v", jobID, e) - } - // delete the schedule record - if e := s.manager.Delete(scheduleID); e != nil { - log.Errorf("failed to delete the schedule record %d: %v", scheduleID, e) - } return 0, err } @@ -172,7 +187,8 @@ func (s *scheduler) UnSchedule(id int64) error { return err } if schedule == nil { - return fmt.Errorf("the schedule record %d not found", id) + log.Warningf("the schedule record %d not found", id) + return nil } if err = s.jobserviceClient.PostAction(schedule.JobID, job.JobActionStop); err != nil { herr, ok := err.(*chttp.Error) diff --git a/src/pkg/types/format.go b/src/pkg/types/format.go new file mode 100644 index 000000000..cc97f0764 --- /dev/null +++ b/src/pkg/types/format.go @@ -0,0 +1,40 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "fmt" +) + +var ( + resourceValueFormats = map[ResourceName]func(int64) string{ + ResourceStorage: byteCountToDisplaySize, + } +) + +func byteCountToDisplaySize(value int64) string { + const unit = 1024 + if value < unit { + return fmt.Sprintf("%d B", value) + } + + div, exp := int64(unit), 0 + for n := value / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + + return fmt.Sprintf("%.1f %ciB", float64(value)/float64(div), "KMGTPE"[exp]) +} diff --git a/src/pkg/types/format_test.go b/src/pkg/types/format_test.go new file mode 100644 index 000000000..19f6607eb --- /dev/null +++ b/src/pkg/types/format_test.go @@ -0,0 +1,45 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import "testing" + +func Test_byteCountToDisplaySize(t *testing.T) { + type args struct { + value int64 + } + tests := []struct { + name string + args args + want string + }{ + {"100 B", args{100}, "100 B"}, + {"1.0 KiB", args{1024}, "1.0 KiB"}, + {"1.5 KiB", args{1024 * 3 / 2}, "1.5 KiB"}, + {"1.0 MiB", args{1024 * 1024}, "1.0 MiB"}, + {"1.5 MiB", args{1024 * 1024 * 3 / 2}, "1.5 MiB"}, + {"1.0 GiB", args{1024 * 1024 * 1024}, "1.0 GiB"}, + {"1.5 GiB", args{1024 * 1024 * 1024 * 3 / 2}, "1.5 GiB"}, + {"1.0 TiB", args{1024 * 1024 * 1024 * 1024}, "1.0 TiB"}, + {"1.5 TiB", args{1024 * 1024 * 1024 * 1024 * 3 / 2}, "1.5 TiB"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := byteCountToDisplaySize(tt.args.value); got != tt.want { + t.Errorf("byteCountToDisplaySize() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/src/pkg/types/resources.go b/src/pkg/types/resources.go index f30d91934..95a98fdff 100644 --- a/src/pkg/types/resources.go +++ b/src/pkg/types/resources.go @@ -16,6 +16,7 @@ package types import ( "encoding/json" + "strconv" ) const ( @@ -31,6 +32,16 @@ const ( // ResourceName is the name identifying various resources in a ResourceList. type ResourceName string +// FormatValue returns string for the resource value +func (resource ResourceName) FormatValue(value int64) string { + format, ok := resourceValueFormats[resource] + if ok { + return format(value) + } + + return strconv.FormatInt(value, 10) +} + // ResourceList is a set of (resource name, value) pairs. type ResourceList map[ResourceName]int64 @@ -113,3 +124,14 @@ func Zero(a ResourceList) ResourceList { } return result } + +// IsNegative returns the set of resource names that have a negative value. +func IsNegative(a ResourceList) []ResourceName { + results := []ResourceName{} + for k, v := range a { + if v < 0 { + results = append(results, k) + } + } + return results +} diff --git a/src/pkg/types/resources_test.go b/src/pkg/types/resources_test.go index 20f164707..473fa13a3 100644 --- a/src/pkg/types/resources_test.go +++ b/src/pkg/types/resources_test.go @@ -76,6 +76,15 @@ func (suite *ResourcesSuite) TestZero() { suite.Equal(ResourceList{ResourceStorage: 0, ResourceCount: 0}, Zero(res2)) } +func (suite *ResourcesSuite) TestIsNegative() { + suite.Len(IsNegative(ResourceList{ResourceStorage: -100, ResourceCount: 100}), 1) + suite.Contains(IsNegative(ResourceList{ResourceStorage: -100, ResourceCount: 100}), ResourceStorage) + + suite.Len(IsNegative(ResourceList{ResourceStorage: -100, ResourceCount: -100}), 2) + suite.Contains(IsNegative(ResourceList{ResourceStorage: -100, ResourceCount: -100}), ResourceStorage) + suite.Contains(IsNegative(ResourceList{ResourceStorage: -100, ResourceCount: -100}), ResourceCount) +} + func TestRunResourcesSuite(t *testing.T) { suite.Run(t, new(ResourcesSuite)) } diff --git a/src/portal/lib/src/config/gc/gc-history/gc-history.component.ts b/src/portal/lib/src/config/gc/gc-history/gc-history.component.ts index dfcc0c4e8..2a8c55c18 100644 --- a/src/portal/lib/src/config/gc/gc-history/gc-history.component.ts +++ b/src/portal/lib/src/config/gc/gc-history/gc-history.component.ts @@ -4,6 +4,7 @@ import { GcJobViewModel } from "../gcLog"; import { GcViewModelFactory } from "../gc.viewmodel.factory"; import { ErrorHandler } from "../../../error-handler/index"; import { Subscription, timer } from "rxjs"; +import { REFRESH_TIME_DIFFERENCE } from '../../../shared/shared.const'; const JOB_STATUS = { PENDING: "pending", RUNNING: "running" @@ -34,7 +35,7 @@ export class GcHistoryComponent implements OnInit, OnDestroy { this.loading = false; // to avoid some jobs not finished. if (!this.timerDelay) { - this.timerDelay = timer(3000, 3000).subscribe(() => { + this.timerDelay = timer(REFRESH_TIME_DIFFERENCE, REFRESH_TIME_DIFFERENCE).subscribe(() => { let count: number = 0; this.jobs.forEach(job => { if ( diff --git a/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.html b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.html index 98e57589a..c9bd48440 100644 --- a/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.html +++ b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.html @@ -25,8 +25,8 @@ {{'PROJECT.QUOTA_UNLIMIT_TIP' | translate }}
-
+
@@ -60,8 +60,9 @@ {{'PROJECT.QUOTA_UNLIMIT_TIP' | translate }}
-
- +
diff --git a/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.scss b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.scss index 024a058d6..43f9bf3bc 100644 --- a/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.scss +++ b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.scss @@ -1,21 +1,25 @@ - ::ng-deep .modal-dialog { width: 25rem; } + .modal-body { padding-top: 0.8rem; overflow-y: visible; overflow-x: visible; + .clr-form-compact { div.form-group { padding-left: 8.5rem; + .mr-3px { margin-right: 3px; } + .quota-input { width: 2rem; padding-right: 0.8rem; } + .select-div { width: 2.5rem; @@ -51,6 +55,22 @@ width: 9rem; } +::ng-deep { + .progress { + &.warning>progress { + color: orange; + + &::-webkit-progress-value { + background-color: orange; + } + + &::-moz-progress-bar { + background-color: orange; + } + } + } +} + .progress-label { position: absolute; right: -2.3rem; @@ -58,4 +78,7 @@ width: 3.5rem; font-weight: 100; font-size: 10px; + + overflow: hidden; + text-overflow: ellipsis; } \ No newline at end of file diff --git a/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.ts b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.ts index a0e435c91..ca3248b32 100644 --- a/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.ts +++ b/src/portal/lib/src/config/project-quotas/edit-project-quotas/edit-project-quotas.component.ts @@ -6,15 +6,12 @@ import { OnInit, } from '@angular/core'; import { NgForm, Validators } from '@angular/forms'; -import { ActivatedRoute } from "@angular/router"; - -import { TranslateService } from '@ngx-translate/core'; import { InlineAlertComponent } from '../../../inline-alert/inline-alert.component'; -import { QuotaUnits, QuotaUnlimited } from "../../../shared/shared.const"; +import { QuotaUnits, QuotaUnlimited, QUOTA_DANGER_COEFFICIENT, QUOTA_WARNING_COEFFICIENT } from "../../../shared/shared.const"; -import { clone, getSuitableUnit, getByte, GetIntegerAndUnit, validateLimit } from '../../../utils'; +import { clone, getSuitableUnit, getByte, GetIntegerAndUnit, validateCountLimit, validateLimit } from '../../../utils'; import { EditQuotaQuotaInterface, QuotaHardLimitInterface } from '../../../service'; import { distinctUntilChanged } from 'rxjs/operators'; @@ -47,9 +44,9 @@ export class EditProjectQuotasComponent implements OnInit { @ViewChild('quotaForm') currentForm: NgForm; @Output() confirmAction = new EventEmitter(); - constructor( - private translateService: TranslateService, - private route: ActivatedRoute) { } + quotaDangerCoefficient: number = QUOTA_DANGER_COEFFICIENT; + quotaWarningCoefficient: number = QUOTA_WARNING_COEFFICIENT; + constructor() { } ngOnInit() { } @@ -106,10 +103,16 @@ export class EditProjectQuotasComponent implements OnInit { Validators.pattern('(^-1$)|(^([1-9]+)([0-9]+)*$)'), validateLimit(this.currentForm.form.controls['storageUnit']) ]); + this.currentForm.form.controls['count'].setValidators( + [ + Validators.required, + Validators.pattern('(^-1$)|(^([1-9]+)([0-9]+)*$)'), + validateCountLimit() + ]); this.currentForm.form.valueChanges .pipe(distinctUntilChanged((a, b) => JSON.stringify(a) === JSON.stringify(b))) .subscribe((data) => { - ['storage', 'storageUnit'].forEach(fieldName => { + ['storage', 'storageUnit', 'count'].forEach(fieldName => { if (this.currentForm.form.get(fieldName) && this.currentForm.form.get(fieldName).value !== null) { this.currentForm.form.get(fieldName).updateValueAndValidity(); } @@ -134,10 +137,18 @@ export class EditProjectQuotasComponent implements OnInit { } return 0; } - getDangerStyle(limit: number | string, used: number | string, unit?: string) { + isDangerColor(limit: number | string, used: number | string, unit?: string) { if (unit) { - return limit !== QuotaUnlimited ? +used / getByte(+limit, unit) > 0.9 : false; + return limit !== QuotaUnlimited ? +used / getByte(+limit, unit) >= this.quotaDangerCoefficient : false; } - return limit !== QuotaUnlimited ? +used / +limit > 0.9 : false; + return limit !== QuotaUnlimited ? +used / +limit >= this.quotaDangerCoefficient : false; + } + isWarningColor(limit: number | string, used: number | string, unit?: string) { + if (unit) { + return limit !== QuotaUnlimited ? + +used / getByte(+limit, unit) >= this.quotaWarningCoefficient && +used / getByte(+limit, unit) <= this.quotaDangerCoefficient : false; + } + return limit !== QuotaUnlimited ? + +used / +limit >= this.quotaWarningCoefficient && +used / +limit <= this.quotaDangerCoefficient : false; } } diff --git a/src/portal/lib/src/config/project-quotas/project-quotas.component.html b/src/portal/lib/src/config/project-quotas/project-quotas.component.html index 6f9e9cad2..22af1333d 100644 --- a/src/portal/lib/src/config/project-quotas/project-quotas.component.html +++ b/src/portal/lib/src/config/project-quotas/project-quotas.component.html @@ -37,7 +37,9 @@
+ [class.danger]="quota.hard.count!==-1?quota.used.count/quota.hard.count>quotaDangerCoefficient:false" + [class.warning]="quota.hard.count!==-1?quota.used.count/quota.hard.count<=quotaDangerCoefficient &"a.used.count/quota.hard.count>=quotaWarningCoefficient:false" + >
@@ -48,7 +50,9 @@
+ [class.danger]="quota.hard.storage!==-1?quota.used.storage/quota.hard.storage>quotaDangerCoefficient:false" + [class.warning]="quota.hard.storage!==-1?quota.used.storage/quota.hard.storage>=quotaWarningCoefficient&"a.used.storage/quota.hard.storage<=quotaDangerCoefficient:false" + >
diff --git a/src/portal/lib/src/config/project-quotas/project-quotas.component.ts b/src/portal/lib/src/config/project-quotas/project-quotas.component.ts index df5effd32..fa457b03e 100644 --- a/src/portal/lib/src/config/project-quotas/project-quotas.component.ts +++ b/src/portal/lib/src/config/project-quotas/project-quotas.component.ts @@ -8,7 +8,7 @@ import { , getByte, GetIntegerAndUnit } from '../../utils'; import { ErrorHandler } from '../../error-handler/index'; -import { QuotaUnits, QuotaUnlimited } from '../../shared/shared.const'; +import { QuotaUnits, QuotaUnlimited, QUOTA_DANGER_COEFFICIENT, QUOTA_WARNING_COEFFICIENT } from '../../shared/shared.const'; import { EditProjectQuotasComponent } from './edit-project-quotas/edit-project-quotas.component'; import { ConfigurationService @@ -46,6 +46,8 @@ export class ProjectQuotasComponent implements OnChanges { currentPage = 1; totalCount = 0; pageSize = 15; + quotaDangerCoefficient: number = QUOTA_DANGER_COEFFICIENT; + quotaWarningCoefficient: number = QUOTA_WARNING_COEFFICIENT; @Input() get allConfig(): Configuration { return this.config; diff --git a/src/portal/lib/src/config/system/system-settings.component.html b/src/portal/lib/src/config/system/system-settings.component.html index 82f83f024..72b9458f6 100644 --- a/src/portal/lib/src/config/system/system-settings.component.html +++ b/src/portal/lib/src/config/system/system-settings.component.html @@ -95,7 +95,7 @@
-
@@ -103,13 +103,13 @@
- {{'CVE_WHITELIST.HELP'|translate}}
-
diff --git a/src/portal/lib/src/config/system/system-settings.component.scss b/src/portal/lib/src/config/system/system-settings.component.scss index 18a577cc3..5b708737f 100644 --- a/src/portal/lib/src/config/system/system-settings.component.scss +++ b/src/portal/lib/src/config/system/system-settings.component.scss @@ -27,8 +27,6 @@ width: 222px; color: #0079bb; overflow-y: auto; - white-space: nowrap; - li { height: 24px; line-height: 24px; diff --git a/src/portal/lib/src/create-edit-rule/create-edit-rule.component.ts b/src/portal/lib/src/create-edit-rule/create-edit-rule.component.ts index 0508e4c14..2ecf2ffca 100644 --- a/src/portal/lib/src/create-edit-rule/create-edit-rule.component.ts +++ b/src/portal/lib/src/create-edit-rule/create-edit-rule.component.ts @@ -434,7 +434,7 @@ export class CreateEditRuleComponent implements OnInit, OnDestroy { // get supportedFilterLabels labels from supportedFilters this.getLabelListFromAdapter(element); // only when edit replication rule - if (ruleInfo && this.supportedFilterLabels.length) { + if (ruleInfo && ruleInfo.filters && this.supportedFilterLabels.length ) { this.getLabelListFromRuleInfo(ruleInfo); } }); diff --git a/src/portal/lib/src/cron-schedule/cron-schedule.component.html b/src/portal/lib/src/cron-schedule/cron-schedule.component.html index 315560e47..5c19753d9 100644 --- a/src/portal/lib/src/cron-schedule/cron-schedule.component.html +++ b/src/portal/lib/src/cron-schedule/cron-schedule.component.html @@ -17,7 +17,7 @@ {{ "SCHEDULE.CRON" | translate }} : {{ oriCron }}
-
diff --git a/src/portal/lib/src/cron-schedule/cron-schedule.component.ts b/src/portal/lib/src/cron-schedule/cron-schedule.component.ts index ba9abae5b..bf1e764ad 100644 --- a/src/portal/lib/src/cron-schedule/cron-schedule.component.ts +++ b/src/portal/lib/src/cron-schedule/cron-schedule.component.ts @@ -27,6 +27,7 @@ export class CronScheduleComponent implements OnChanges { @Input() originCron: OriginCron; @Input() labelEdit: string; @Input() labelCurrent: string; + @Input() disabled: boolean; dateInvalid: boolean; originScheduleType: string; oriCron: string; diff --git a/src/portal/lib/src/image-name-input/image-name-input.component.ts b/src/portal/lib/src/image-name-input/image-name-input.component.ts index 30bd4e126..1a3170a62 100644 --- a/src/portal/lib/src/image-name-input/image-name-input.component.ts +++ b/src/portal/lib/src/image-name-input/image-name-input.component.ts @@ -52,10 +52,10 @@ export class ImageNameInputComponent implements OnInit, OnDestroy { const prolist: any = this.proService.listProjects(name, undefined); if (prolist.subscribe) { prolist.subscribe(response => { - if (response) { - this.selectedProjectList = response.slice(0, 10); + if (response.body) { + this.selectedProjectList = response.body.slice(0, 10); // if input project name exist in the project list - let exist = response.find((data: any) => data.name === name); + let exist = response.body.find((data: any) => data.name === name); if (!exist) { this.noProjectInfo = "REPLICATION.NO_PROJECT_INFO"; } else { diff --git a/src/portal/lib/src/log/recent-log.component.spec.ts b/src/portal/lib/src/log/recent-log.component.spec.ts index 39425c8b2..bdb7cc4c0 100644 --- a/src/portal/lib/src/log/recent-log.component.spec.ts +++ b/src/portal/lib/src/log/recent-log.component.spec.ts @@ -203,7 +203,7 @@ describe('RecentLogComponent (inline template)', () => { fixture.detectChanges(); expect(component.recentLogs).toBeTruthy(); expect(component.logsCache).toBeTruthy(); - expect(component.recentLogs.length).toEqual(3); + expect(component.recentLogs.length).toEqual(15); }); }); diff --git a/src/portal/lib/src/log/recent-log.component.ts b/src/portal/lib/src/log/recent-log.component.ts index b791e7bfb..8a187c9df 100644 --- a/src/portal/lib/src/log/recent-log.component.ts +++ b/src/portal/lib/src/log/recent-log.component.ts @@ -67,7 +67,8 @@ export class RecentLogComponent implements OnInit { } public doFilter(terms: string): void { - if (!terms) { + // allow search by null characters + if (terms === undefined || terms === null) { return; } this.currentTerm = terms.trim(); diff --git a/src/portal/lib/src/project-policy-config/project-policy-config.component.html b/src/portal/lib/src/project-policy-config/project-policy-config.component.html index 31912edc3..8d5e69f81 100644 --- a/src/portal/lib/src/project-policy-config/project-policy-config.component.html +++ b/src/portal/lib/src/project-policy-config/project-policy-config.component.html @@ -89,12 +89,12 @@
- - @@ -102,10 +102,10 @@
- -
@@ -113,12 +113,12 @@
- + {{'CVE_WHITELIST.HELP'|translate}}
-
@@ -144,17 +144,17 @@
- -
-
diff --git a/src/portal/lib/src/project-policy-config/project-policy-config.component.scss b/src/portal/lib/src/project-policy-config/project-policy-config.component.scss index 4b7cab641..05d4bc5d1 100644 --- a/src/portal/lib/src/project-policy-config/project-policy-config.component.scss +++ b/src/portal/lib/src/project-policy-config/project-policy-config.component.scss @@ -17,8 +17,6 @@ width: 222px; color: #0079bb; overflow-y: auto; - white-space: nowrap; - li { height: 24px; line-height: 24px; diff --git a/src/portal/lib/src/project-policy-config/project-policy-config.component.ts b/src/portal/lib/src/project-policy-config/project-policy-config.component.ts index 58a68227e..a6ab59495 100644 --- a/src/portal/lib/src/project-policy-config/project-policy-config.component.ts +++ b/src/portal/lib/src/project-policy-config/project-policy-config.component.ts @@ -170,6 +170,9 @@ export class ProjectPolicyConfigComponent implements OnInit { if (!response.cve_whitelist['expires_at']) { response.cve_whitelist['expires_at'] = null; } + if (!response.metadata.reuse_sys_cve_whitelist) { + response.metadata.reuse_sys_cve_whitelist = "true"; + } if (response && response.cve_whitelist) { this.projectWhitelist = clone(response.cve_whitelist); this.projectWhitelistOrigin = clone(response.cve_whitelist); diff --git a/src/portal/lib/src/replication/replication-tasks/replication-tasks.component.html b/src/portal/lib/src/replication/replication-tasks/replication-tasks.component.html index d441c799b..633f020e6 100644 --- a/src/portal/lib/src/replication/replication-tasks/replication-tasks.component.html +++ b/src/portal/lib/src/replication/replication-tasks/replication-tasks.component.html @@ -92,7 +92,7 @@
- + {{'REPLICATION.TASK_ID'| translate}} {{'REPLICATION.RESOURCE_TYPE' | translate}} {{'REPLICATION.SOURCE' | translate}} @@ -102,7 +102,7 @@ {{'REPLICATION.CREATION_TIME' | translate}} {{'REPLICATION.END_TIME' | translate}} {{'REPLICATION.LOGS' | translate}} - + {{t.id}} {{t.resource_type}} {{t.src_resource}} @@ -118,8 +118,8 @@ - {{pagination.firstItem + 1}} - {{pagination.lastItem +1 }} {{'REPLICATION.OF' | translate}} {{pagination.totalItems }} {{'REPLICATION.ITEMS' | translate}} - + {{pagination.firstItem + 1}} - {{pagination.lastItem +1 }} {{'REPLICATION.OF' | translate}} {{totalCount }} {{'REPLICATION.ITEMS' | translate}} +
diff --git a/src/portal/lib/src/replication/replication-tasks/replication-tasks.component.ts b/src/portal/lib/src/replication/replication-tasks/replication-tasks.component.ts index a9984d47f..d80320319 100644 --- a/src/portal/lib/src/replication/replication-tasks/replication-tasks.component.ts +++ b/src/portal/lib/src/replication/replication-tasks/replication-tasks.component.ts @@ -6,8 +6,9 @@ import { finalize } from "rxjs/operators"; import { Subscription, timer } from "rxjs"; import { ErrorHandler } from "../../error-handler/error-handler"; import { ReplicationJob, ReplicationTasks, Comparator, ReplicationJobItem, State } from "../../service/interface"; -import { CustomComparator, DEFAULT_PAGE_SIZE, calculatePage, doFiltering, doSorting } from "../../utils"; +import { CustomComparator, DEFAULT_PAGE_SIZE } from "../../utils"; import { RequestQueryParams } from "../../service/RequestQueryParams"; +import { REFRESH_TIME_DIFFERENCE } from '../../shared/shared.const'; const executionStatus = 'InProgress'; @Component({ selector: 'replication-tasks', @@ -18,8 +19,8 @@ export class ReplicationTasksComponent implements OnInit, OnDestroy { isOpenFilterTag: boolean; inProgress: boolean = false; currentPage: number = 1; - selectedRow: []; pageSize: number = DEFAULT_PAGE_SIZE; + totalCount: number; loading = true; searchTask: string; defaultFilter = "resource_type"; @@ -47,7 +48,6 @@ export class ReplicationTasksComponent implements OnInit, OnDestroy { ngOnInit(): void { this.searchTask = ''; this.getExecutionDetail(); - this.clrLoadTasks(); } getExecutionDetail(): void { @@ -67,14 +67,17 @@ export class ReplicationTasksComponent implements OnInit, OnDestroy { clrLoadPage(): void { if (!this.timerDelay) { - this.timerDelay = timer(10000, 10000).subscribe(() => { + this.timerDelay = timer(REFRESH_TIME_DIFFERENCE, REFRESH_TIME_DIFFERENCE).subscribe(() => { let count: number = 0; - if (this.executions['status'] === executionStatus) { - count++; - } + if (this.executions['status'] === executionStatus) { + count++; + } if (count > 0) { this.getExecutionDetail(); - this.clrLoadTasks(); + let state: State = { + page: {} + }; + this.clrLoadTasks(state); } else { this.timerDelay.unsubscribe(); this.timerDelay = null; @@ -136,16 +139,30 @@ export class ReplicationTasksComponent implements OnInit, OnDestroy { } } - clrLoadTasks(): void { - this.loading = true; + clrLoadTasks(state: State): void { + if (!state || !state.page || !this.executionId) { + return; + } + let params: RequestQueryParams = new RequestQueryParams(); + params = params.set('page_size', this.pageSize + '').set('page', this.currentPage + ''); if (this.searchTask && this.searchTask !== "") { params = params.set(this.defaultFilter, this.searchTask); } + + this.loading = true; this.replicationService.getReplicationTasks(this.executionId, params) - .pipe(finalize(() => (this.loading = false))) + .pipe(finalize(() => { + this.loading = false; + })) .subscribe(res => { - this.tasks = res; // Keep the data + if (res.headers) { + let xHeader: string = res.headers.get("X-Total-Count"); + if (xHeader) { + this.totalCount = parseInt(xHeader, 0); + } + } + this.tasks = res.body; // Keep the data }, error => { this.errorHandler.error(error); @@ -162,23 +179,20 @@ export class ReplicationTasksComponent implements OnInit, OnDestroy { // refresh icon refreshTasks(): void { - this.loading = true; this.currentPage = 1; - this.replicationService.getReplicationTasks(this.executionId) - .subscribe(res => { - this.tasks = res; - this.loading = false; - }, - error => { - this.loading = false; - this.errorHandler.error(error); - }); + let state: State = { + page: {} + }; + this.clrLoadTasks(state); } public doSearch(value: string): void { + this.currentPage = 1; this.searchTask = value.trim(); - this.loading = true; - this.clrLoadTasks(); + let state: State = { + page: {} + }; + this.clrLoadTasks(state); } openFilter(isOpen: boolean): void { diff --git a/src/portal/lib/src/replication/replication.component.ts b/src/portal/lib/src/replication/replication.component.ts index e211af9ea..1bdbfe200 100644 --- a/src/portal/lib/src/replication/replication.component.ts +++ b/src/portal/lib/src/replication/replication.component.ts @@ -48,7 +48,8 @@ import { import { ConfirmationTargets, ConfirmationButtons, - ConfirmationState + ConfirmationState, + REFRESH_TIME_DIFFERENCE } from "../shared/shared.const"; import { ConfirmationMessage } from "../confirmation-dialog/confirmation-message"; import { ConfirmationDialogComponent } from "../confirmation-dialog/confirmation-dialog.component"; @@ -214,7 +215,7 @@ export class ReplicationComponent implements OnInit, OnDestroy { this.totalCount = response.metadata.xTotalCount; this.jobs = response.data; if (!this.timerDelay) { - this.timerDelay = timer(10000, 10000).subscribe(() => { + this.timerDelay = timer(REFRESH_TIME_DIFFERENCE, REFRESH_TIME_DIFFERENCE).subscribe(() => { let count: number = 0; this.jobs.forEach(job => { if ( diff --git a/src/portal/lib/src/service/interface.ts b/src/portal/lib/src/service/interface.ts index 5de814f2b..17e739d91 100644 --- a/src/portal/lib/src/service/interface.ts +++ b/src/portal/lib/src/service/interface.ts @@ -66,6 +66,8 @@ export interface Tag extends Base { signature?: string; scan_overview?: VulnerabilitySummary; labels: Label[]; + push_time?: string; + pull_time?: string; } /** diff --git a/src/portal/lib/src/service/replication.service.ts b/src/portal/lib/src/service/replication.service.ts index 135f04b8c..3da31077f 100644 --- a/src/portal/lib/src/service/replication.service.ts +++ b/src/portal/lib/src/service/replication.service.ts @@ -296,8 +296,7 @@ export class ReplicationDefaultService extends ReplicationService { } let url: string = `${this._replicateUrl}/executions/${executionId}/tasks`; return this.http - .get(url, - queryParams ? buildHttpRequestOptions(queryParams) : HTTP_GET_OPTIONS) + .get(url, buildHttpRequestOptionsWithObserveResponse(queryParams)) .pipe(map(response => response as ReplicationTasks) , catchError(error => observableThrowError(error))); } diff --git a/src/portal/lib/src/service/tag.service.ts b/src/portal/lib/src/service/tag.service.ts index 9020200f3..336fa2369 100644 --- a/src/portal/lib/src/service/tag.service.ts +++ b/src/portal/lib/src/service/tag.service.ts @@ -140,7 +140,7 @@ export class TagDefaultService extends TagService { queryParams = queryParams = new RequestQueryParams(); } - queryParams = queryParams.set("detail", "1"); + queryParams = queryParams.set("detail", "true"); let url: string = `${this._baseUrl}/${repositoryName}/tags`; return this.http diff --git a/src/portal/lib/src/shared/shared.const.ts b/src/portal/lib/src/shared/shared.const.ts index 8a310b086..ff1935071 100644 --- a/src/portal/lib/src/shared/shared.const.ts +++ b/src/portal/lib/src/shared/shared.const.ts @@ -90,6 +90,7 @@ export const QuotaUnits = [ ]; export const QuotaUnlimited = -1; export const StorageMultipleConstant = 1024; +export const LimitCount = 100000000; export enum QuotaUnit { TB = "TB", GB = "GB", MB = "MB", KB = "KB", BIT = "Byte" } @@ -122,6 +123,8 @@ export const CONFIG_AUTH_MODE = { OIDC_AUTH: "oidc_auth", UAA_AUTH: "uaa_auth" }; +export const QUOTA_DANGER_COEFFICIENT = 0.9; +export const QUOTA_WARNING_COEFFICIENT = 0.7; export const PROJECT_ROOTS = [ { NAME: "admin", @@ -149,3 +152,4 @@ export enum GroupType { LDAP_TYPE = 1, HTTP_TYPE = 2 } +export const REFRESH_TIME_DIFFERENCE = 10000; diff --git a/src/portal/lib/src/tag/tag.component.html b/src/portal/lib/src/tag/tag.component.html index 362030eb2..9df2ddad1 100644 --- a/src/portal/lib/src/tag/tag.component.html +++ b/src/portal/lib/src/tag/tag.component.html @@ -85,8 +85,8 @@ {{'REPOSITORY.CREATED' | translate}} {{'REPOSITORY.DOCKER_VERSION' | translate}} {{'REPOSITORY.LABELS' | translate}} - {{'REPOSITORY.PULL_TIME' | translate}} - {{'REPOSITORY.PUSH_TIME' | translate}} + {{'REPOSITORY.PUSH_TIME' | translate}} + {{'REPOSITORY.PULL_TIME' | translate}} {{'TAG.PLACEHOLDER' | translate }} @@ -125,8 +125,8 @@
- {{t.pull_time | date: 'short'}} {{t.push_time | date: 'short'}} + {{t.pull_time | date: 'short'}} {{pagination.firstItem + 1}} - {{pagination.lastItem + 1}} {{'REPOSITORY.OF' | translate}} {{pagination.totalItems}} {{'REPOSITORY.ITEMS' | translate}}     diff --git a/src/portal/lib/src/tag/tag.component.ts b/src/portal/lib/src/tag/tag.component.ts index b74f790c8..61dd6ac36 100644 --- a/src/portal/lib/src/tag/tag.component.ts +++ b/src/portal/lib/src/tag/tag.component.ts @@ -66,7 +66,7 @@ export interface LabelState { label: Label; show: boolean; } - +export const AVAILABLE_TIME = '0001-01-01T00:00:00Z'; @Component({ selector: 'hbr-tag', templateUrl: './tag.component.html', @@ -107,6 +107,8 @@ export class TagComponent implements OnInit, AfterViewInit { showlabel: boolean; createdComparator: Comparator = new CustomComparator("created", "date"); + pullComparator: Comparator = new CustomComparator("pull_time", "date"); + pushComparator: Comparator = new CustomComparator("push_time", "date"); loading = false; copyFailed = false; @@ -271,7 +273,10 @@ export class TagComponent implements OnInit, AfterViewInit { // Do filtering and sorting this.tags = doFiltering(tags, state); this.tags = doSorting(this.tags, state); - + this.tags = this.tags.map(tag => { + tag.pull_time = tag.pull_time === AVAILABLE_TIME ? '' : tag.pull_time; + return tag; + }); this.loading = false; }, error => { this.loading = false; @@ -539,7 +544,10 @@ export class TagComponent implements OnInit, AfterViewInit { signatures.push(t.name); } }); - this.tags = items; + this.tags = items.map(tag => { + tag.pull_time = tag.pull_time === AVAILABLE_TIME ? '' : tag.pull_time; + return tag; + }); let signedName: { [key: string]: string[] } = {}; signedName[this.repoName] = signatures; this.signatureOutput.emit(signedName); diff --git a/src/portal/lib/src/utils.ts b/src/portal/lib/src/utils.ts index 7c3ede678..a44f889f2 100644 --- a/src/portal/lib/src/utils.ts +++ b/src/portal/lib/src/utils.ts @@ -4,8 +4,9 @@ import { HttpHeaders } from '@angular/common/http'; import { RequestQueryParams } from './service/RequestQueryParams'; import { DebugElement } from '@angular/core'; import { Comparator, State, HttpOptionInterface, HttpOptionTextInterface, QuotaUnitInterface } from './service/interface'; -import { QuotaUnits, StorageMultipleConstant } from './shared/shared.const'; +import { QuotaUnits, StorageMultipleConstant, LimitCount } from './shared/shared.const'; import { AbstractControl } from "@angular/forms"; + /** * Convert the different async channels to the Promise type. * @@ -504,15 +505,30 @@ export const GetIntegerAndUnit = (hardNumber: number, quotaUnitsDeep: QuotaUnitI } } }; -export const validateLimit = (unitContrl) => { - return (control: AbstractControl) => { - if (getByte(control.value, unitContrl.value) > StorageMultipleConstant * StorageMultipleConstant - * StorageMultipleConstant * StorageMultipleConstant * StorageMultipleConstant) { - return { - error: true - }; - } - return null; - }; + +export const validateCountLimit = () => { + return (control: AbstractControl) => { + if (control.value > LimitCount) { + return { + error: true + }; + } + return null; + }; +}; + +export const validateLimit = unitContrl => { + return (control: AbstractControl) => { + if ( + // 1024TB + getByte(control.value, unitContrl.value) > + Math.pow(StorageMultipleConstant, 5) + ) { + return { + error: true + }; + } + return null; + }; }; diff --git a/src/portal/lib/src/vulnerability-scanning/result-bar-chart.component.ts b/src/portal/lib/src/vulnerability-scanning/result-bar-chart.component.ts index d4648a6ce..947ba031c 100644 --- a/src/portal/lib/src/vulnerability-scanning/result-bar-chart.component.ts +++ b/src/portal/lib/src/vulnerability-scanning/result-bar-chart.component.ts @@ -48,8 +48,12 @@ export class ResultBarChartComponent implements OnInit, OnDestroy { ) { } ngOnInit(): void { - if (this.tagStatus === "running") { - this.scanNow(); + if ((this.tagStatus === VULNERABILITY_SCAN_STATUS.running || this.tagStatus === VULNERABILITY_SCAN_STATUS.pending) + && !this.stateCheckTimer) { + // Avoid duplicated subscribing + this.stateCheckTimer = timer(0, STATE_CHECK_INTERVAL).subscribe(() => { + this.getSummary(); + }); } this.scanSubscription = this.channel.scanCommand$.subscribe((tagId: string) => { let myFullTag: string = this.repoName + "/" + this.tagId; diff --git a/src/portal/lib/src/vulnerability-scanning/result-grid.component.ts b/src/portal/lib/src/vulnerability-scanning/result-grid.component.ts index 9ec18a35a..e9740bed8 100644 --- a/src/portal/lib/src/vulnerability-scanning/result-grid.component.ts +++ b/src/portal/lib/src/vulnerability-scanning/result-grid.component.ts @@ -39,7 +39,9 @@ export class ResultGridComponent implements OnInit { this.scanningService.getVulnerabilityScanningResults(repositoryId, tagId) .subscribe((results: VulnerabilityItem[]) => { this.dataCache = results; - this.scanningResults = this.dataCache.filter((item: VulnerabilityItem) => item.id !== ''); + if (results) { + this.scanningResults = this.dataCache.filter((item: VulnerabilityItem) => item.id !== ''); + } }, error => { this.errorHandler.error(error); }); } diff --git a/src/portal/src/app/app.module.ts b/src/portal/src/app/app.module.ts index 5d0cd92f8..bb21b63ae 100644 --- a/src/portal/src/app/app.module.ts +++ b/src/portal/src/app/app.module.ts @@ -33,14 +33,17 @@ import zh from '@angular/common/locales/zh-Hans'; import es from '@angular/common/locales/es'; import localeFr from '@angular/common/locales/fr'; import localePt from '@angular/common/locales/pt-PT'; +import localeTr from '@angular/common/locales/tr'; import { DevCenterComponent } from './dev-center/dev-center.component'; import { VulnerabilityPageComponent } from './vulnerability-page/vulnerability-page.component'; import { GcPageComponent } from './gc-page/gc-page.component'; import { OidcOnboardModule } from './oidc-onboard/oidc-onboard.module'; +import { LicenseModule } from './license/license.module'; registerLocaleData(zh, 'zh-cn'); registerLocaleData(es, 'es-es'); registerLocaleData(localeFr, 'fr-fr'); registerLocaleData(localePt, 'pt-br'); +registerLocaleData(localeTr, 'tr-tr'); export function initConfig(configService: AppConfigService, skinableService: SkinableConfig) { @@ -70,7 +73,8 @@ export function getCurrentLanguage(translateService: TranslateService) { HarborRoutingModule, ConfigurationModule, DeveloperCenterModule, - OidcOnboardModule + OidcOnboardModule, + LicenseModule ], exports: [ ], diff --git a/src/portal/src/app/base/navigator/navigator.component.html b/src/portal/src/app/base/navigator/navigator.component.html index f7721a66e..7c3cc19af 100644 --- a/src/portal/src/app/base/navigator/navigator.component.html +++ b/src/portal/src/app/base/navigator/navigator.component.html @@ -25,6 +25,7 @@ Español Français Português do Brasil + Türkçe diff --git a/src/portal/src/app/harbor-routing.module.ts b/src/portal/src/app/harbor-routing.module.ts index 0c8e10d35..662b51fe9 100644 --- a/src/portal/src/app/harbor-routing.module.ts +++ b/src/portal/src/app/harbor-routing.module.ts @@ -57,6 +57,7 @@ import { ListChartsComponent } from './project/helm-chart/list-charts.component' import { ListChartVersionsComponent } from './project/helm-chart/list-chart-versions/list-chart-versions.component'; import { HelmChartDetailComponent } from './project/helm-chart/helm-chart-detail/chart-detail.component'; import { OidcOnboardComponent } from './oidc-onboard/oidc-onboard.component'; +import { LicenseComponent } from './license/license.component'; import { SummaryComponent } from './project/summary/summary.component'; import { TagRetentionComponent } from "./project/tag-retention/tag-retention.component"; @@ -73,6 +74,10 @@ const harborRoutes: Routes = [ component: OidcOnboardComponent, canActivate: [OidcGuard, SignInGuard] }, + { + path: 'license', + component: LicenseComponent + }, { path: 'harbor/sign-in', component: SignInComponent, diff --git a/src/portal/src/app/license/license.component.html b/src/portal/src/app/license/license.component.html new file mode 100644 index 000000000..caa00cb9e --- /dev/null +++ b/src/portal/src/app/license/license.component.html @@ -0,0 +1 @@ +
{{licenseContent}}
diff --git a/src/portal/src/app/license/license.component.scss b/src/portal/src/app/license/license.component.scss new file mode 100644 index 000000000..6d00fe18b --- /dev/null +++ b/src/portal/src/app/license/license.component.scss @@ -0,0 +1,8 @@ +.license { + display: block; + font-family: monospace; + word-wrap: break-word; + white-space: pre-wrap; + margin: 1em 0px; + font-size: 1rem; +} \ No newline at end of file diff --git a/src/portal/src/app/license/license.component.spec.ts b/src/portal/src/app/license/license.component.spec.ts new file mode 100644 index 000000000..f1d41ee71 --- /dev/null +++ b/src/portal/src/app/license/license.component.spec.ts @@ -0,0 +1,25 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { LicenseComponent } from './license.component'; + +describe('LicenseComponent', () => { + let component: LicenseComponent; + let fixture: ComponentFixture; + + beforeEach(async(() => { + TestBed.configureTestingModule({ + declarations: [ LicenseComponent ] + }) + .compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(LicenseComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/portal/src/app/license/license.component.ts b/src/portal/src/app/license/license.component.ts new file mode 100644 index 000000000..218b2492a --- /dev/null +++ b/src/portal/src/app/license/license.component.ts @@ -0,0 +1,25 @@ +import { Component, OnInit } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { throwError as observableThrowError } from 'rxjs'; +import { catchError } from 'rxjs/operators'; +import { Title } from '@angular/platform-browser'; +@Component({ + selector: 'app-license', + viewProviders: [Title], + templateUrl: './license.component.html', + styleUrls: ['./license.component.scss'] +}) +export class LicenseComponent implements OnInit { + + constructor( + private http: HttpClient + ) { } + public licenseContent: any; + ngOnInit() { + this.http.get("/LICENSE", { responseType: 'text'}) + .pipe(catchError(error => observableThrowError(error))) + .subscribe(json => { + this.licenseContent = json; + }); + } +} diff --git a/src/portal/src/app/license/license.module.ts b/src/portal/src/app/license/license.module.ts new file mode 100644 index 000000000..543dcddc4 --- /dev/null +++ b/src/portal/src/app/license/license.module.ts @@ -0,0 +1,11 @@ +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { LicenseComponent } from './license.component'; + +@NgModule({ + declarations: [LicenseComponent], + imports: [ + CommonModule + ] +}) +export class LicenseModule { } diff --git a/src/portal/src/app/project/create-project/create-project.component.html b/src/portal/src/app/project/create-project/create-project.component.html index e3fa55024..1d9c8e068 100644 --- a/src/portal/src/app/project/create-project/create-project.component.html +++ b/src/portal/src/app/project/create-project/create-project.component.html @@ -35,11 +35,11 @@
- - +
- - + - diff --git a/src/portal/src/app/project/create-project/create-project.component.ts b/src/portal/src/app/project/create-project/create-project.component.ts index eff57e473..aa662176a 100644 --- a/src/portal/src/app/project/create-project/create-project.component.ts +++ b/src/portal/src/app/project/create-project/create-project.component.ts @@ -34,7 +34,7 @@ import { InlineAlertComponent } from "../../shared/inline-alert/inline-alert.com import { Project } from "../project"; import { ProjectService, QuotaUnits, QuotaHardInterface, QuotaUnlimited, getByte - , GetIntegerAndUnit, clone, StorageMultipleConstant, validateLimit} from "@harbor/ui"; + , GetIntegerAndUnit, clone, StorageMultipleConstant, validateLimit, validateCountLimit} from "@harbor/ui"; import { errorHandler } from '@angular/platform-browser/src/browser'; @Component({ @@ -118,17 +118,23 @@ export class CreateProjectComponent implements OnInit, OnChanges, OnDestroy { this.storageDefaultLimit = this.storageLimit; this.storageDefaultLimitUnit = this.storageLimitUnit; if (this.isSystemAdmin) { - this.currentForm.form.controls['create_project_storage-limit'].setValidators( + this.currentForm.form.controls['create_project_storage_limit'].setValidators( [ Validators.required, Validators.pattern('(^-1$)|(^([1-9]+)([0-9]+)*$)'), - validateLimit(this.currentForm.form.controls['create_project_storage-limit-unit']) + validateLimit(this.currentForm.form.controls['create_project_storage_limit_unit']) ]); + this.currentForm.form.controls['create_project_count_limit'].setValidators( + [ + Validators.required, + Validators.pattern('(^-1$)|(^([1-9]+)([0-9]+)*$)'), + validateCountLimit() + ]); } this.currentForm.form.valueChanges .pipe(distinctUntilChanged((a, b) => JSON.stringify(a) === JSON.stringify(b))) .subscribe((data) => { - ['create_project_storage-limit', 'create_project_storage-limit-unit'].forEach(fieldName => { + ['create_project_storage_limit', 'create_project_storage_limit_unit', 'create_project_count_limit'].forEach(fieldName => { if (this.currentForm.form.get(fieldName) && this.currentForm.form.get(fieldName).value !== null) { this.currentForm.form.get(fieldName).updateValueAndValidity(); } diff --git a/src/portal/src/app/project/robot-account/robot-account.service.ts b/src/portal/src/app/project/robot-account/robot-account.service.ts index 9898b0998..4438a4b27 100644 --- a/src/portal/src/app/project/robot-account/robot-account.service.ts +++ b/src/portal/src/app/project/robot-account/robot-account.service.ts @@ -26,23 +26,23 @@ export class RobotService { ) { } /** addRobotAccount - * projecId + * projectId * robot: Robot * projectName */ - public addRobotAccount(projecId: number, robot: Robot, projectName: string): Observable { + public addRobotAccount(projectId: number, robot: Robot, projectName: string): Observable { let access = []; if (robot.access.isPullImage) { - access.push({ "resource": `/project/${projectName}/repository`, "action": "pull" }); + access.push({ "resource": `/project/${projectId}/repository`, "action": "pull" }); } if (robot.access.isPushOrPullImage) { - access.push({ "resource": `/project/${projectName}/repository`, "action": "push" }); + access.push({ "resource": `/project/${projectId}/repository`, "action": "push" }); } if (robot.access.isPullChart) { - access.push({ "resource": `/project/${projectName}/helm-chart`, "action": "read" }); + access.push({ "resource": `/project/${projectId}/helm-chart`, "action": "read" }); } if (robot.access.isPushChart) { - access.push({ "resource": `/project/${projectName}/helm-chart-version`, "action": "create" }); + access.push({ "resource": `/project/${projectId}/helm-chart-version`, "action": "create" }); } let param = { @@ -51,25 +51,25 @@ export class RobotService { access }; - return this.robotApiRepository.postRobot(projecId, param); + return this.robotApiRepository.postRobot(projectId, param); } - public deleteRobotAccount(projecId, id): Observable { - return this.robotApiRepository.deleteRobot(projecId, id); + public deleteRobotAccount(projectId, id): Observable { + return this.robotApiRepository.deleteRobot(projectId, id); } - public listRobotAccount(projecId): Observable { - return this.robotApiRepository.listRobot(projecId); + public listRobotAccount(projectId): Observable { + return this.robotApiRepository.listRobot(projectId); } - public getRobotAccount(projecId, id): Observable { - return this.robotApiRepository.getRobot(projecId, id); + public getRobotAccount(projectId, id): Observable { + return this.robotApiRepository.getRobot(projectId, id); } - public toggleDisabledAccount(projecId, id, isDisabled): Observable { + public toggleDisabledAccount(projectId, id, isDisabled): Observable { let data = { Disabled: isDisabled }; - return this.robotApiRepository.toggleDisabledAccount(projecId, id, data); + return this.robotApiRepository.toggleDisabledAccount(projectId, id, data); } } diff --git a/src/portal/src/app/project/summary/summary.component.html b/src/portal/src/app/project/summary/summary.component.html index d53588b42..70d640c1a 100644 --- a/src/portal/src/app/project/summary/summary.component.html +++ b/src/portal/src/app/project/summary/summary.component.html @@ -35,7 +35,8 @@
-
+
@@ -58,7 +59,8 @@
+ [class.danger]="summaryInformation?.quota?.hard?.storage!==-1?summaryInformation?.quota?.used?.storage/summaryInformation?.quota?.hard?.storage>quotaDangerCoefficient:false" + [class.warning]="summaryInformation?.quota?.hard?.storage!==-1?summaryInformation?.quota?.used?.storage/summaryInformation?.quota?.hard?.storage<=quotaDangerCoefficient&&summaryInformation?.quota?.used?.storage/summaryInformation?.quota?.hard?.storage>=quotaWarningCoefficient:false"> diff --git a/src/portal/src/app/project/summary/summary.component.scss b/src/portal/src/app/project/summary/summary.component.scss index 3ce73946c..7f53fb84a 100644 --- a/src/portal/src/app/project/summary/summary.component.scss +++ b/src/portal/src/app/project/summary/summary.component.scss @@ -36,4 +36,19 @@ progress { max-height: 0.48rem; } +} +::ng-deep { + .progress { + &.warning>progress { + color: orange; + + &::-webkit-progress-value { + background-color: orange; + } + + &::-moz-progress-bar { + background-color: orange; + } + } + } } \ No newline at end of file diff --git a/src/portal/src/app/project/summary/summary.component.ts b/src/portal/src/app/project/summary/summary.component.ts index 00bdaf146..457d7fbb2 100644 --- a/src/portal/src/app/project/summary/summary.component.ts +++ b/src/portal/src/app/project/summary/summary.component.ts @@ -1,9 +1,9 @@ -import { Component, OnInit, Input } from '@angular/core'; -import { ProjectService, clone, QuotaUnits, getSuitableUnit, ErrorHandler, GetIntegerAndUnit } from '@harbor/ui'; -import { Router, ActivatedRoute } from '@angular/router'; -import { forkJoin } from 'rxjs'; +import { Component, OnInit } from '@angular/core'; +import { ProjectService, clone, QuotaUnits, getSuitableUnit, ErrorHandler, GetIntegerAndUnit + , QUOTA_DANGER_COEFFICIENT, QUOTA_WARNING_COEFFICIENT } from '@harbor/ui'; +import { ActivatedRoute } from '@angular/router'; + import { AppConfigService } from "../../app-config.service"; -export const riskRatio = 0.9; @Component({ selector: 'summary', templateUrl: './summary.component.html', @@ -12,6 +12,8 @@ export const riskRatio = 0.9; export class SummaryComponent implements OnInit { projectId: number; summaryInformation: any; + quotaDangerCoefficient: number = QUOTA_DANGER_COEFFICIENT; + quotaWarningCoefficient: number = QUOTA_WARNING_COEFFICIENT; constructor( private projectService: ProjectService, private errorHandler: ErrorHandler, diff --git a/src/portal/src/app/project/tag-retention/add-rule/add-rule.component.html b/src/portal/src/app/project/tag-retention/add-rule/add-rule.component.html index 397883cab..87a961d79 100644 --- a/src/portal/src/app/project/tag-retention/add-rule/add-rule.component.html +++ b/src/portal/src/app/project/tag-retention/add-rule/add-rule.component.html @@ -63,7 +63,7 @@
@@ -81,32 +81,6 @@
-
-
-
- -
-
-
- -
-
-
-
- -
-
-
-
-
-
- {{'TAG_RETENTION.REP_LABELS' | translate}} -
-
-