mirror of
https://github.com/goharbor/harbor
synced 2024-09-21 13:42:16 +00:00
Merge branch 'master' into official-wehook-events-20190811
This commit is contained in:
commit
d5f87063e4
|
@ -28,7 +28,7 @@ env:
|
||||||
- POSTGRESQL_PWD: root123
|
- POSTGRESQL_PWD: root123
|
||||||
- POSTGRESQL_DATABASE: registry
|
- POSTGRESQL_DATABASE: registry
|
||||||
- ADMINSERVER_URL: http://127.0.0.1:8888
|
- ADMINSERVER_URL: http://127.0.0.1:8888
|
||||||
- DOCKER_COMPOSE_VERSION: 1.22.0
|
- DOCKER_COMPOSE_VERSION: 1.23.0
|
||||||
- HARBOR_ADMIN: admin
|
- HARBOR_ADMIN: admin
|
||||||
- HARBOR_ADMIN_PASSWD: Harbor12345
|
- HARBOR_ADMIN_PASSWD: Harbor12345
|
||||||
- CORE_SECRET: tempString
|
- CORE_SECRET: tempString
|
||||||
|
|
2
Makefile
2
Makefile
|
@ -100,7 +100,7 @@ PREPARE_VERSION_NAME=versions
|
||||||
REGISTRYVERSION=v2.7.1-patch-2819
|
REGISTRYVERSION=v2.7.1-patch-2819
|
||||||
NGINXVERSION=$(VERSIONTAG)
|
NGINXVERSION=$(VERSIONTAG)
|
||||||
NOTARYVERSION=v0.6.1
|
NOTARYVERSION=v0.6.1
|
||||||
CLAIRVERSION=v2.0.8
|
CLAIRVERSION=v2.0.9
|
||||||
CLAIRDBVERSION=$(VERSIONTAG)
|
CLAIRDBVERSION=$(VERSIONTAG)
|
||||||
MIGRATORVERSION=$(VERSIONTAG)
|
MIGRATORVERSION=$(VERSIONTAG)
|
||||||
REDISVERSION=$(VERSIONTAG)
|
REDISVERSION=$(VERSIONTAG)
|
||||||
|
|
|
@ -49,7 +49,7 @@ Harbor is hosted by the [Cloud Native Computing Foundation](https://cncf.io) (CN
|
||||||
|
|
||||||
**System requirements:**
|
**System requirements:**
|
||||||
|
|
||||||
**On a Linux host:** docker 17.06.0-ce+ and docker-compose 1.18.0+ .
|
**On a Linux host:** docker 17.06.0-ce+ and docker-compose 1.23.0+ .
|
||||||
|
|
||||||
Download binaries of **[Harbor release ](https://github.com/vmware/harbor/releases)** and follow **[Installation & Configuration Guide](docs/installation_guide.md)** to install Harbor.
|
Download binaries of **[Harbor release ](https://github.com/vmware/harbor/releases)** and follow **[Installation & Configuration Guide](docs/installation_guide.md)** to install Harbor.
|
||||||
|
|
||||||
|
|
|
@ -4,14 +4,13 @@ This guide provides instructions for developers to build and run Harbor from sou
|
||||||
|
|
||||||
## Step 1: Prepare for a build environment for Harbor
|
## Step 1: Prepare for a build environment for Harbor
|
||||||
|
|
||||||
Harbor is deployed as several Docker containers and most of the code is written in Go language. The build environment requires Python, Docker, Docker Compose and golang development environment. Please install the below prerequisites:
|
Harbor is deployed as several Docker containers and most of the code is written in Go language. The build environment requires Docker, Docker Compose and golang development environment. Please install the below prerequisites:
|
||||||
|
|
||||||
|
|
||||||
Software | Required Version
|
Software | Required Version
|
||||||
----------------------|--------------------------
|
----------------------|--------------------------
|
||||||
docker | 17.05 +
|
docker | 17.05 +
|
||||||
docker-compose | 1.11.0 +
|
docker-compose | 1.23.0 +
|
||||||
python | 2.7 +
|
|
||||||
git | 1.9.1 +
|
git | 1.9.1 +
|
||||||
make | 3.81 +
|
make | 3.81 +
|
||||||
golang* | 1.7.3 +
|
golang* | 1.7.3 +
|
||||||
|
|
|
@ -31,7 +31,7 @@ Harbor is deployed as several Docker containers, and, therefore, can be deployed
|
||||||
|Software|Version|Description|
|
|Software|Version|Description|
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
|Docker engine|version 17.06.0-ce+ or higher|For installation instructions, please refer to: [docker engine doc](https://docs.docker.com/engine/installation/)|
|
|Docker engine|version 17.06.0-ce+ or higher|For installation instructions, please refer to: [docker engine doc](https://docs.docker.com/engine/installation/)|
|
||||||
|Docker Compose|version 1.18.0 or higher|For installation instructions, please refer to: [docker compose doc](https://docs.docker.com/compose/install/)|
|
|Docker Compose|version 1.23.0 or higher|For installation instructions, please refer to: [docker compose doc](https://docs.docker.com/compose/install/)|
|
||||||
|Openssl|latest is preferred|Generate certificate and keys for Harbor|
|
|Openssl|latest is preferred|Generate certificate and keys for Harbor|
|
||||||
|
|
||||||
### Network ports
|
### Network ports
|
||||||
|
|
|
@ -4,7 +4,7 @@ This guide provides instructions to manage roles by LDAP/AD group. You can impor
|
||||||
|
|
||||||
## Prerequisite
|
## Prerequisite
|
||||||
|
|
||||||
1. Harbor's auth_mode is ldap_auth and **[basic LDAP configure paremters](https://github.com/vmware/harbor/blob/master/docs/installation_guide.md#optional-parameters)** are configured.
|
1. Harbor's auth_mode is ldap_auth and **[basic LDAP configure parameters](https://github.com/vmware/harbor/blob/master/docs/installation_guide.md#optional-parameters)** are configured.
|
||||||
1. Memberof overlay
|
1. Memberof overlay
|
||||||
|
|
||||||
This feature requires the LDAP/AD server enabled the feature **memberof overlay**.
|
This feature requires the LDAP/AD server enabled the feature **memberof overlay**.
|
||||||
|
|
|
@ -122,8 +122,6 @@ paths:
|
||||||
responses:
|
responses:
|
||||||
'200':
|
'200':
|
||||||
description: Project name exists.
|
description: Project name exists.
|
||||||
'401':
|
|
||||||
description: User need to log in first.
|
|
||||||
'404':
|
'404':
|
||||||
description: Project name does not exist.
|
description: Project name does not exist.
|
||||||
'500':
|
'500':
|
||||||
|
@ -333,10 +331,10 @@ paths:
|
||||||
description: Illegal format of provided ID value.
|
description: Illegal format of provided ID value.
|
||||||
'401':
|
'401':
|
||||||
description: User need to log in first.
|
description: User need to log in first.
|
||||||
'404':
|
|
||||||
description: Project ID does not exist.
|
|
||||||
'403':
|
'403':
|
||||||
description: User does not have permission to get summary of the project.
|
description: User does not have permission to get summary of the project.
|
||||||
|
'404':
|
||||||
|
description: Project ID does not exist.
|
||||||
'500':
|
'500':
|
||||||
description: Unexpected internal errors.
|
description: Unexpected internal errors.
|
||||||
'/projects/{project_id}/metadatas':
|
'/projects/{project_id}/metadatas':
|
||||||
|
@ -1263,11 +1261,16 @@ paths:
|
||||||
type: string
|
type: string
|
||||||
required: true
|
required: true
|
||||||
description: Relevant repository name.
|
description: Relevant repository name.
|
||||||
- name: label_ids
|
- name: label_id
|
||||||
in: query
|
in: query
|
||||||
type: string
|
type: string
|
||||||
required: false
|
required: false
|
||||||
description: A list of comma separated label IDs.
|
description: A label ID.
|
||||||
|
- name: detail
|
||||||
|
in: query
|
||||||
|
type: boolean
|
||||||
|
required: false
|
||||||
|
description: Bool value indicating whether return detailed information of the tag, such as vulnerability scan info, if set to false, only tag name is returned.
|
||||||
tags:
|
tags:
|
||||||
- Products
|
- Products
|
||||||
responses:
|
responses:
|
||||||
|
@ -2380,10 +2383,10 @@ paths:
|
||||||
$ref: '#/definitions/Namespace'
|
$ref: '#/definitions/Namespace'
|
||||||
'401':
|
'401':
|
||||||
description: User need to login first.
|
description: User need to login first.
|
||||||
'404':
|
|
||||||
description: No registry found.
|
|
||||||
'403':
|
'403':
|
||||||
description: User has no privilege for the operation.
|
description: User has no privilege for the operation.
|
||||||
|
'404':
|
||||||
|
description: No registry found.
|
||||||
'500':
|
'500':
|
||||||
description: Unexpected internal errors.
|
description: Unexpected internal errors.
|
||||||
/internal/syncregistry:
|
/internal/syncregistry:
|
||||||
|
@ -2404,6 +2407,20 @@ paths:
|
||||||
$ref: '#/responses/UnsupportedMediaType'
|
$ref: '#/responses/UnsupportedMediaType'
|
||||||
'500':
|
'500':
|
||||||
description: Unexpected internal errors.
|
description: Unexpected internal errors.
|
||||||
|
/internal/syncquota:
|
||||||
|
post:
|
||||||
|
summary: Sync quota from registry/chart to DB.
|
||||||
|
description: |
|
||||||
|
This endpoint is for syncing quota usage of registry/chart with database.
|
||||||
|
tags:
|
||||||
|
- Products
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: Sync repositories successfully.
|
||||||
|
'401':
|
||||||
|
description: User need to log in first.
|
||||||
|
'403':
|
||||||
|
description: User does not have permission of system admin role.
|
||||||
/systeminfo:
|
/systeminfo:
|
||||||
get:
|
get:
|
||||||
summary: Get general system info
|
summary: Get general system info
|
||||||
|
@ -3684,7 +3701,7 @@ paths:
|
||||||
description: Unexpected internal errors.
|
description: Unexpected internal errors.
|
||||||
'/projects/{project_id}/webhook/policies':
|
'/projects/{project_id}/webhook/policies':
|
||||||
get:
|
get:
|
||||||
sumary: List project webhook policies.
|
summary: List project webhook policies.
|
||||||
description: |
|
description: |
|
||||||
This endpoint returns webhook policies of a project.
|
This endpoint returns webhook policies of a project.
|
||||||
parameters:
|
parameters:
|
||||||
|
@ -3712,7 +3729,7 @@ paths:
|
||||||
'500':
|
'500':
|
||||||
description: Unexpected internal errors.
|
description: Unexpected internal errors.
|
||||||
post:
|
post:
|
||||||
sumary: Create project webhook policy.
|
summary: Create project webhook policy.
|
||||||
description: |
|
description: |
|
||||||
This endpoint create a webhook policy if the project does not have one.
|
This endpoint create a webhook policy if the project does not have one.
|
||||||
parameters:
|
parameters:
|
||||||
|
@ -3757,7 +3774,7 @@ paths:
|
||||||
in: path
|
in: path
|
||||||
description: The id of webhook policy.
|
description: The id of webhook policy.
|
||||||
required: true
|
required: true
|
||||||
type: int64
|
type: integer
|
||||||
format: int64
|
format: int64
|
||||||
tags:
|
tags:
|
||||||
- Products
|
- Products
|
||||||
|
@ -3791,7 +3808,7 @@ paths:
|
||||||
in: path
|
in: path
|
||||||
description: The id of webhook policy.
|
description: The id of webhook policy.
|
||||||
required: true
|
required: true
|
||||||
type: int64
|
type: integer
|
||||||
format: int64
|
format: int64
|
||||||
- name: policy
|
- name: policy
|
||||||
in: body
|
in: body
|
||||||
|
@ -3829,7 +3846,7 @@ paths:
|
||||||
in: path
|
in: path
|
||||||
description: The id of webhook policy.
|
description: The id of webhook policy.
|
||||||
required: true
|
required: true
|
||||||
type: int64
|
type: integer
|
||||||
format: int64
|
format: int64
|
||||||
tags:
|
tags:
|
||||||
- Products
|
- Products
|
||||||
|
@ -3908,7 +3925,7 @@ paths:
|
||||||
description: Internal server errors.
|
description: Internal server errors.
|
||||||
'/projects/{project_id}/webhook/jobs':
|
'/projects/{project_id}/webhook/jobs':
|
||||||
get:
|
get:
|
||||||
sumary: List project webhook jobs
|
summary: List project webhook jobs
|
||||||
description: |
|
description: |
|
||||||
This endpoint returns webhook jobs of a project.
|
This endpoint returns webhook jobs of a project.
|
||||||
parameters:
|
parameters:
|
||||||
|
@ -4023,6 +4040,9 @@ definitions:
|
||||||
metadata:
|
metadata:
|
||||||
description: The metadata of the project.
|
description: The metadata of the project.
|
||||||
$ref: '#/definitions/ProjectMetadata'
|
$ref: '#/definitions/ProjectMetadata'
|
||||||
|
cve_whitelist:
|
||||||
|
description: The CVE whitelist of the project.
|
||||||
|
$ref: '#/definitions/CVEWhitelist'
|
||||||
count_limit:
|
count_limit:
|
||||||
type: integer
|
type: integer
|
||||||
format: int64
|
format: int64
|
||||||
|
@ -4083,16 +4103,20 @@ definitions:
|
||||||
description: 'The public status of the project. The valid values are "true", "false".'
|
description: 'The public status of the project. The valid values are "true", "false".'
|
||||||
enable_content_trust:
|
enable_content_trust:
|
||||||
type: string
|
type: string
|
||||||
description: 'Whether content trust is enabled or not. If it is enabled, user cann''t pull unsigned images from this project. The valid values are "true", "false".'
|
description: 'Whether content trust is enabled or not. If it is enabled, user can''t pull unsigned images from this project. The valid values are "true", "false".'
|
||||||
prevent_vul:
|
prevent_vul:
|
||||||
type: string
|
type: string
|
||||||
description: 'Whether prevent the vulnerable images from running. The valid values are "true", "false".'
|
description: 'Whether prevent the vulnerable images from running. The valid values are "true", "false".'
|
||||||
severity:
|
severity:
|
||||||
type: string
|
type: string
|
||||||
description: 'If the vulnerability is high than severity defined here, the images cann''t be pulled. The valid values are "negligible", "low", "medium", "high", "critical".'
|
description: 'If the vulnerability is high than severity defined here, the images can''t be pulled. The valid values are "negligible", "low", "medium", "high", "critical".'
|
||||||
auto_scan:
|
auto_scan:
|
||||||
type: string
|
type: string
|
||||||
description: 'Whether scan images automatically when pushing. The valid values are "true", "false".'
|
description: 'Whether scan images automatically when pushing. The valid values are "true", "false".'
|
||||||
|
reuse_sys_cve_whitelist:
|
||||||
|
type: string
|
||||||
|
description: 'Whether this project reuse the system level CVE whitelist as the whitelist of its own. The valid values are "true", "false".
|
||||||
|
If it is set to "true" the actual whitelist associate with this project, if any, will be ignored.'
|
||||||
ProjectSummary:
|
ProjectSummary:
|
||||||
type: object
|
type: object
|
||||||
properties:
|
properties:
|
||||||
|
@ -4841,6 +4865,9 @@ definitions:
|
||||||
project_creation_restriction:
|
project_creation_restriction:
|
||||||
type: string
|
type: string
|
||||||
description: This attribute restricts what users have the permission to create project. It can be "everyone" or "adminonly".
|
description: This attribute restricts what users have the permission to create project. It can be "everyone" or "adminonly".
|
||||||
|
quota_per_project_enable:
|
||||||
|
type: boolean
|
||||||
|
description: This attribute indicates whether quota per project enabled in harbor
|
||||||
read_only:
|
read_only:
|
||||||
type: boolean
|
type: boolean
|
||||||
description: '''docker push'' is prohibited by Harbor if you set it to true. '
|
description: '''docker push'' is prohibited by Harbor if you set it to true. '
|
||||||
|
@ -4938,6 +4965,9 @@ definitions:
|
||||||
project_creation_restriction:
|
project_creation_restriction:
|
||||||
$ref: '#/definitions/StringConfigItem'
|
$ref: '#/definitions/StringConfigItem'
|
||||||
description: This attribute restricts what users have the permission to create project. It can be "everyone" or "adminonly".
|
description: This attribute restricts what users have the permission to create project. It can be "everyone" or "adminonly".
|
||||||
|
quota_per_project_enable:
|
||||||
|
$ref: '#/definitions/BoolConfigItem'
|
||||||
|
description: This attribute indicates whether quota per project enabled in harbor
|
||||||
read_only:
|
read_only:
|
||||||
$ref: '#/definitions/BoolConfigItem'
|
$ref: '#/definitions/BoolConfigItem'
|
||||||
description: '''docker push'' is prohibited by Harbor if you set it to true. '
|
description: '''docker push'' is prohibited by Harbor if you set it to true. '
|
||||||
|
@ -5349,7 +5379,9 @@ definitions:
|
||||||
properties:
|
properties:
|
||||||
type:
|
type:
|
||||||
type: string
|
type: string
|
||||||
description: The schedule type. The valid values are hourly, daily, weekly, custom and None. 'None' means to cancel the schedule.
|
description: |
|
||||||
|
The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manually' and 'None'.
|
||||||
|
'Manually' means to trigger it right away and 'None' means to cancel the schedule.
|
||||||
cron:
|
cron:
|
||||||
type: string
|
type: string
|
||||||
description: A cron expression, a time-based job scheduler.
|
description: A cron expression, a time-based job scheduler.
|
||||||
|
@ -5724,7 +5756,7 @@ definitions:
|
||||||
description: The webhook job ID.
|
description: The webhook job ID.
|
||||||
policy_id:
|
policy_id:
|
||||||
type: integer
|
type: integer
|
||||||
fromat: int64
|
format: int64
|
||||||
description: The webhook policy ID.
|
description: The webhook policy ID.
|
||||||
event_type:
|
event_type:
|
||||||
type: string
|
type: string
|
||||||
|
|
|
@ -30,6 +30,11 @@ harbor_admin_password: Harbor12345
|
||||||
database:
|
database:
|
||||||
# The password for the root user of Harbor DB. Change this before any production use.
|
# The password for the root user of Harbor DB. Change this before any production use.
|
||||||
password: root123
|
password: root123
|
||||||
|
# The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
|
||||||
|
max_idle_conns: 50
|
||||||
|
# The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.
|
||||||
|
# Note: the default number of connections is 100 for postgres.
|
||||||
|
max_open_conns: 100
|
||||||
|
|
||||||
# The default data volume
|
# The default data volume
|
||||||
data_volume: /data
|
data_volume: /data
|
||||||
|
@ -54,12 +59,6 @@ clair:
|
||||||
# The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
|
# The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
|
||||||
updaters_interval: 12
|
updaters_interval: 12
|
||||||
|
|
||||||
# Config http proxy for Clair, e.g. http://my.proxy.com:3128
|
|
||||||
# Clair doesn't need to connect to harbor internal components via http proxy.
|
|
||||||
http_proxy:
|
|
||||||
https_proxy:
|
|
||||||
no_proxy: 127.0.0.1,localhost,core,registry
|
|
||||||
|
|
||||||
jobservice:
|
jobservice:
|
||||||
# Maximum number of job workers in job service
|
# Maximum number of job workers in job service
|
||||||
max_job_workers: 10
|
max_job_workers: 10
|
||||||
|
@ -97,7 +96,7 @@ log:
|
||||||
# port: 5140
|
# port: 5140
|
||||||
|
|
||||||
#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
|
#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
|
||||||
_version: 1.8.0
|
_version: 1.9.0
|
||||||
|
|
||||||
# Uncomment external_database if using external database.
|
# Uncomment external_database if using external database.
|
||||||
# external_database:
|
# external_database:
|
||||||
|
@ -143,3 +142,20 @@ _version: 1.8.0
|
||||||
# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
|
# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
|
||||||
# uaa:
|
# uaa:
|
||||||
# ca_file: /path/to/ca
|
# ca_file: /path/to/ca
|
||||||
|
|
||||||
|
# Global proxy
|
||||||
|
# Config http proxy for components, e.g. http://my.proxy.com:3128
|
||||||
|
# Components doesn't need to connect to each others via http proxy.
|
||||||
|
# Remove component from `components` array if want disable proxy
|
||||||
|
# for it. If you want use proxy for replication, MUST enable proxy
|
||||||
|
# for core and jobservice, and set `http_proxy` and `https_proxy`.
|
||||||
|
# Add domain to the `no_proxy` field, when you want disable proxy
|
||||||
|
# for some special registry.
|
||||||
|
proxy:
|
||||||
|
http_proxy:
|
||||||
|
https_proxy:
|
||||||
|
no_proxy: 127.0.0.1,localhost,.local,.internal,log,db,redis,nginx,core,portal,postgresql,jobservice,registry,registryctl,clair
|
||||||
|
components:
|
||||||
|
- core
|
||||||
|
- jobservice
|
||||||
|
- clair
|
||||||
|
|
|
@ -117,7 +117,7 @@ function check_docker {
|
||||||
function check_dockercompose {
|
function check_dockercompose {
|
||||||
if ! docker-compose --version &> /dev/null
|
if ! docker-compose --version &> /dev/null
|
||||||
then
|
then
|
||||||
error "Need to install docker-compose(1.18.0+) by yourself first and run this script again."
|
error "Need to install docker-compose(1.23.0+) by yourself first and run this script again."
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
@ -129,9 +129,9 @@ function check_dockercompose {
|
||||||
docker_compose_version_part2=${BASH_REMATCH[3]}
|
docker_compose_version_part2=${BASH_REMATCH[3]}
|
||||||
|
|
||||||
# the version of docker-compose does not meet the requirement
|
# the version of docker-compose does not meet the requirement
|
||||||
if [ "$docker_compose_version_part1" -lt 1 ] || ([ "$docker_compose_version_part1" -eq 1 ] && [ "$docker_compose_version_part2" -lt 18 ])
|
if [ "$docker_compose_version_part1" -lt 1 ] || ([ "$docker_compose_version_part1" -eq 1 ] && [ "$docker_compose_version_part2" -lt 23 ])
|
||||||
then
|
then
|
||||||
error "Need to upgrade docker-compose package to 1.18.0+."
|
error "Need to upgrade docker-compose package to 1.23.0+."
|
||||||
exit 1
|
exit 1
|
||||||
else
|
else
|
||||||
note "docker-compose version: $docker_compose_version"
|
note "docker-compose version: $docker_compose_version"
|
||||||
|
|
|
@ -56,9 +56,9 @@ $$;
|
||||||
|
|
||||||
CREATE TRIGGER harbor_user_update_time_at_modtime BEFORE UPDATE ON harbor_user FOR EACH ROW EXECUTE PROCEDURE update_update_time_at_column();
|
CREATE TRIGGER harbor_user_update_time_at_modtime BEFORE UPDATE ON harbor_user FOR EACH ROW EXECUTE PROCEDURE update_update_time_at_column();
|
||||||
|
|
||||||
insert into harbor_user (username, email, password, realname, comment, deleted, sysadmin_flag, creation_time, update_time) values
|
insert into harbor_user (username, password, realname, comment, deleted, sysadmin_flag, creation_time, update_time) values
|
||||||
('admin', 'admin@example.com', '', 'system admin', 'admin user',false, true, NOW(), NOW()),
|
('admin', '', 'system admin', 'admin user',false, true, NOW(), NOW()),
|
||||||
('anonymous', 'anonymous@example.com', '', 'anonymous user', 'anonymous user', true, false, NOW(), NOW());
|
('anonymous', '', 'anonymous user', 'anonymous user', true, false, NOW(), NOW());
|
||||||
|
|
||||||
create table project (
|
create table project (
|
||||||
project_id SERIAL PRIMARY KEY NOT NULL,
|
project_id SERIAL PRIMARY KEY NOT NULL,
|
||||||
|
|
|
@ -86,6 +86,7 @@ CREATE TABLE quota_usage
|
||||||
UNIQUE (reference, reference_id)
|
UNIQUE (reference, reference_id)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/* only set quota and usage for 'library', and let the sync quota handling others. */
|
||||||
INSERT INTO quota (reference, reference_id, hard, creation_time, update_time)
|
INSERT INTO quota (reference, reference_id, hard, creation_time, update_time)
|
||||||
SELECT 'project',
|
SELECT 'project',
|
||||||
CAST(project_id AS VARCHAR),
|
CAST(project_id AS VARCHAR),
|
||||||
|
@ -93,7 +94,7 @@ SELECT 'project',
|
||||||
NOW(),
|
NOW(),
|
||||||
NOW()
|
NOW()
|
||||||
FROM project
|
FROM project
|
||||||
WHERE deleted = 'f';
|
WHERE name = 'library' and deleted = 'f';
|
||||||
|
|
||||||
INSERT INTO quota_usage (id, reference, reference_id, used, creation_time, update_time)
|
INSERT INTO quota_usage (id, reference, reference_id, used, creation_time, update_time)
|
||||||
SELECT id,
|
SELECT id,
|
||||||
|
@ -131,6 +132,8 @@ create table retention_task
|
||||||
repository varchar(255),
|
repository varchar(255),
|
||||||
job_id varchar(64),
|
job_id varchar(64),
|
||||||
status varchar(32),
|
status varchar(32),
|
||||||
|
status_code integer,
|
||||||
|
status_revision integer,
|
||||||
start_time timestamp default CURRENT_TIMESTAMP,
|
start_time timestamp default CURRENT_TIMESTAMP,
|
||||||
end_time timestamp default CURRENT_TIMESTAMP,
|
end_time timestamp default CURRENT_TIMESTAMP,
|
||||||
total integer,
|
total integer,
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
FROM photon:2.0
|
FROM photon:2.0
|
||||||
|
|
||||||
RUN tdnf install sudo -y >> /dev/null\
|
RUN tdnf install sudo tzdata -y >> /dev/null \
|
||||||
&& tdnf clean all \
|
&& tdnf clean all \
|
||||||
&& groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor \
|
&& groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor \
|
||||||
&& mkdir /harbor/
|
&& mkdir /harbor/
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
FROM photon:2.0
|
FROM photon:2.0
|
||||||
|
|
||||||
RUN tdnf install sudo -y >> /dev/null\
|
RUN tdnf install sudo tzdata -y >> /dev/null \
|
||||||
&& tdnf clean all \
|
&& tdnf clean all \
|
||||||
&& groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor
|
&& groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,8 @@
|
||||||
FROM node:10.15.0 as nodeportal
|
FROM node:10.15.0 as nodeportal
|
||||||
|
|
||||||
COPY src/portal /portal_src
|
COPY src/portal /portal_src
|
||||||
COPY ./docs/swagger.yaml /portal_src
|
COPY ./docs/swagger.yaml /portal_src
|
||||||
|
COPY ./LICENSE /portal_src
|
||||||
|
|
||||||
WORKDIR /build_dir
|
WORKDIR /build_dir
|
||||||
|
|
||||||
|
@ -21,6 +22,7 @@ FROM photon:2.0
|
||||||
COPY --from=nodeportal /build_dir/dist /usr/share/nginx/html
|
COPY --from=nodeportal /build_dir/dist /usr/share/nginx/html
|
||||||
COPY --from=nodeportal /build_dir/swagger.yaml /usr/share/nginx/html
|
COPY --from=nodeportal /build_dir/swagger.yaml /usr/share/nginx/html
|
||||||
COPY --from=nodeportal /build_dir/swagger.json /usr/share/nginx/html
|
COPY --from=nodeportal /build_dir/swagger.json /usr/share/nginx/html
|
||||||
|
COPY --from=nodeportal /build_dir/LICENSE /usr/share/nginx/html
|
||||||
|
|
||||||
COPY make/photon/portal/nginx.conf /etc/nginx/nginx.conf
|
COPY make/photon/portal/nginx.conf /etc/nginx/nginx.conf
|
||||||
|
|
||||||
|
|
|
@ -12,11 +12,12 @@ REDIS_UID = 999
|
||||||
REDIS_GID = 999
|
REDIS_GID = 999
|
||||||
|
|
||||||
## Global variable
|
## Global variable
|
||||||
|
host_root_dir = '/hostfs'
|
||||||
|
|
||||||
base_dir = '/harbor_make'
|
base_dir = '/harbor_make'
|
||||||
templates_dir = "/usr/src/app/templates"
|
templates_dir = "/usr/src/app/templates"
|
||||||
config_dir = '/config'
|
config_dir = '/config'
|
||||||
data_dir = '/data'
|
data_dir = '/data'
|
||||||
|
|
||||||
secret_dir = '/secret'
|
secret_dir = '/secret'
|
||||||
secret_key_dir='/secret/keys'
|
secret_key_dir='/secret/keys'
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,3 @@
|
||||||
http_proxy={{clair_http_proxy}}
|
HTTP_PROXY={{clair_http_proxy}}
|
||||||
https_proxy={{clair_https_proxy}}
|
HTTPS_PROXY={{clair_https_proxy}}
|
||||||
no_proxy={{clair_no_proxy}}
|
NO_PROXY={{clair_no_proxy}}
|
||||||
|
|
|
@ -15,6 +15,8 @@ POSTGRESQL_USERNAME={{harbor_db_username}}
|
||||||
POSTGRESQL_PASSWORD={{harbor_db_password}}
|
POSTGRESQL_PASSWORD={{harbor_db_password}}
|
||||||
POSTGRESQL_DATABASE={{harbor_db_name}}
|
POSTGRESQL_DATABASE={{harbor_db_name}}
|
||||||
POSTGRESQL_SSLMODE={{harbor_db_sslmode}}
|
POSTGRESQL_SSLMODE={{harbor_db_sslmode}}
|
||||||
|
POSTGRESQL_MAX_IDLE_CONNS={{harbor_db_max_idle_conns}}
|
||||||
|
POSTGRESQL_MAX_OPEN_CONNS={{harbor_db_max_open_conns}}
|
||||||
REGISTRY_URL={{registry_url}}
|
REGISTRY_URL={{registry_url}}
|
||||||
TOKEN_SERVICE_URL={{token_service_url}}
|
TOKEN_SERVICE_URL={{token_service_url}}
|
||||||
HARBOR_ADMIN_PASSWORD={{harbor_admin_password}}
|
HARBOR_ADMIN_PASSWORD={{harbor_admin_password}}
|
||||||
|
@ -41,3 +43,7 @@ RELOAD_KEY={{reload_key}}
|
||||||
CHART_REPOSITORY_URL={{chart_repository_url}}
|
CHART_REPOSITORY_URL={{chart_repository_url}}
|
||||||
REGISTRY_CONTROLLER_URL={{registry_controller_url}}
|
REGISTRY_CONTROLLER_URL={{registry_controller_url}}
|
||||||
WITH_CHARTMUSEUM={{with_chartmuseum}}
|
WITH_CHARTMUSEUM={{with_chartmuseum}}
|
||||||
|
|
||||||
|
HTTP_PROXY={{core_http_proxy}}
|
||||||
|
HTTPS_PROXY={{core_https_proxy}}
|
||||||
|
NO_PROXY={{core_no_proxy}}
|
||||||
|
|
|
@ -276,12 +276,7 @@ services:
|
||||||
volumes:
|
volumes:
|
||||||
- ./common/config/nginx:/etc/nginx:z
|
- ./common/config/nginx:/etc/nginx:z
|
||||||
{% if protocol == 'https' %}
|
{% if protocol == 'https' %}
|
||||||
- type: bind
|
- {{data_volume}}/secret/cert:/etc/cert:z
|
||||||
source: {{cert_key_path}}
|
|
||||||
target: /etc/cert/server.key
|
|
||||||
- type: bind
|
|
||||||
source: {{cert_path}}
|
|
||||||
target: /etc/cert/server.crt
|
|
||||||
{% endif %}
|
{% endif %}
|
||||||
networks:
|
networks:
|
||||||
- harbor
|
- harbor
|
||||||
|
|
|
@ -2,3 +2,7 @@ CORE_SECRET={{core_secret}}
|
||||||
JOBSERVICE_SECRET={{jobservice_secret}}
|
JOBSERVICE_SECRET={{jobservice_secret}}
|
||||||
CORE_URL={{core_url}}
|
CORE_URL={{core_url}}
|
||||||
JOBSERVICE_WEBHOOK_JOB_MAX_RETRY={{notification_webhook_job_max_retry}}
|
JOBSERVICE_WEBHOOK_JOB_MAX_RETRY={{notification_webhook_job_max_retry}}
|
||||||
|
|
||||||
|
HTTP_PROXY={{jobservice_http_proxy}}
|
||||||
|
HTTPS_PROXY={{jobservice_https_proxy}}
|
||||||
|
NO_PROXY={{jobservice_no_proxy}}
|
||||||
|
|
|
@ -112,6 +112,11 @@ def parse_yaml_config(config_file_path):
|
||||||
config_dict['harbor_db_username'] = 'postgres'
|
config_dict['harbor_db_username'] = 'postgres'
|
||||||
config_dict['harbor_db_password'] = db_configs.get("password") or ''
|
config_dict['harbor_db_password'] = db_configs.get("password") or ''
|
||||||
config_dict['harbor_db_sslmode'] = 'disable'
|
config_dict['harbor_db_sslmode'] = 'disable'
|
||||||
|
|
||||||
|
default_max_idle_conns = 2 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns
|
||||||
|
default_max_open_conns = 0 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxOpenConns
|
||||||
|
config_dict['harbor_db_max_idle_conns'] = db_configs.get("max_idle_conns") or default_max_idle_conns
|
||||||
|
config_dict['harbor_db_max_open_conns'] = db_configs.get("max_open_conns") or default_max_open_conns
|
||||||
# clari db
|
# clari db
|
||||||
config_dict['clair_db_host'] = 'postgresql'
|
config_dict['clair_db_host'] = 'postgresql'
|
||||||
config_dict['clair_db_port'] = 5432
|
config_dict['clair_db_port'] = 5432
|
||||||
|
@ -171,13 +176,18 @@ def parse_yaml_config(config_file_path):
|
||||||
if storage_config.get('redirect'):
|
if storage_config.get('redirect'):
|
||||||
config_dict['storage_redirect_disabled'] = storage_config['redirect']['disabled']
|
config_dict['storage_redirect_disabled'] = storage_config['redirect']['disabled']
|
||||||
|
|
||||||
|
# Global proxy configs
|
||||||
|
proxy_config = configs.get('proxy') or {}
|
||||||
|
proxy_components = proxy_config.get('components') or []
|
||||||
|
for proxy_component in proxy_components:
|
||||||
|
config_dict[proxy_component + '_http_proxy'] = proxy_config.get('http_proxy') or ''
|
||||||
|
config_dict[proxy_component + '_https_proxy'] = proxy_config.get('https_proxy') or ''
|
||||||
|
config_dict[proxy_component + '_no_proxy'] = proxy_config.get('no_proxy') or '127.0.0.1,localhost,core,registry'
|
||||||
|
|
||||||
# Clair configs, optional
|
# Clair configs, optional
|
||||||
clair_configs = configs.get("clair") or {}
|
clair_configs = configs.get("clair") or {}
|
||||||
config_dict['clair_db'] = 'postgres'
|
config_dict['clair_db'] = 'postgres'
|
||||||
config_dict['clair_updaters_interval'] = clair_configs.get("updaters_interval") or 12
|
config_dict['clair_updaters_interval'] = clair_configs.get("updaters_interval") or 12
|
||||||
config_dict['clair_http_proxy'] = clair_configs.get('http_proxy') or ''
|
|
||||||
config_dict['clair_https_proxy'] = clair_configs.get('https_proxy') or ''
|
|
||||||
config_dict['clair_no_proxy'] = clair_configs.get('no_proxy') or '127.0.0.1,localhost,core,registry'
|
|
||||||
|
|
||||||
# Chart configs
|
# Chart configs
|
||||||
chart_configs = configs.get("chart") or {}
|
chart_configs = configs.get("chart") or {}
|
||||||
|
|
|
@ -13,7 +13,7 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):
|
||||||
VERSION_TAG = versions.get('VERSION_TAG') or 'dev'
|
VERSION_TAG = versions.get('VERSION_TAG') or 'dev'
|
||||||
REGISTRY_VERSION = versions.get('REGISTRY_VERSION') or 'v2.7.1'
|
REGISTRY_VERSION = versions.get('REGISTRY_VERSION') or 'v2.7.1'
|
||||||
NOTARY_VERSION = versions.get('NOTARY_VERSION') or 'v0.6.1'
|
NOTARY_VERSION = versions.get('NOTARY_VERSION') or 'v0.6.1'
|
||||||
CLAIR_VERSION = versions.get('CLAIR_VERSION') or 'v2.0.7'
|
CLAIR_VERSION = versions.get('CLAIR_VERSION') or 'v2.0.9'
|
||||||
CHARTMUSEUM_VERSION = versions.get('CHARTMUSEUM_VERSION') or 'v0.9.0'
|
CHARTMUSEUM_VERSION = versions.get('CHARTMUSEUM_VERSION') or 'v0.9.0'
|
||||||
|
|
||||||
rendering_variables = {
|
rendering_variables = {
|
||||||
|
|
|
@ -2,11 +2,13 @@ import os, shutil
|
||||||
from fnmatch import fnmatch
|
from fnmatch import fnmatch
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from g import config_dir, templates_dir, DEFAULT_GID, DEFAULT_UID
|
from g import config_dir, templates_dir, host_root_dir, DEFAULT_GID, DEFAULT_UID, data_dir
|
||||||
from utils.misc import prepare_dir, mark_file
|
from utils.misc import prepare_dir, mark_file
|
||||||
from utils.jinja import render_jinja
|
from utils.jinja import render_jinja
|
||||||
from utils.cert import SSL_CERT_KEY_PATH, SSL_CERT_PATH
|
from utils.cert import SSL_CERT_KEY_PATH, SSL_CERT_PATH
|
||||||
|
|
||||||
|
host_ngx_real_cert_dir = Path(os.path.join(data_dir, 'secret', 'cert'))
|
||||||
|
|
||||||
nginx_conf = os.path.join(config_dir, "nginx", "nginx.conf")
|
nginx_conf = os.path.join(config_dir, "nginx", "nginx.conf")
|
||||||
nginx_confd_dir = os.path.join(config_dir, "nginx", "conf.d")
|
nginx_confd_dir = os.path.join(config_dir, "nginx", "conf.d")
|
||||||
nginx_https_conf_template = os.path.join(templates_dir, "nginx", "nginx.https.conf.jinja")
|
nginx_https_conf_template = os.path.join(templates_dir, "nginx", "nginx.https.conf.jinja")
|
||||||
|
@ -20,8 +22,38 @@ def prepare_nginx(config_dict):
|
||||||
prepare_dir(nginx_confd_dir, uid=DEFAULT_UID, gid=DEFAULT_GID)
|
prepare_dir(nginx_confd_dir, uid=DEFAULT_UID, gid=DEFAULT_GID)
|
||||||
render_nginx_template(config_dict)
|
render_nginx_template(config_dict)
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_nginx_certs(cert_key_path, cert_path):
|
||||||
|
"""
|
||||||
|
Prepare the certs file with proper ownership
|
||||||
|
1. Remove nginx cert files in secret dir
|
||||||
|
2. Copy cert files on host filesystem to secret dir
|
||||||
|
3. Change the permission to 644 and ownership to 10000:10000
|
||||||
|
"""
|
||||||
|
host_ngx_cert_key_path = Path(os.path.join(host_root_dir, cert_key_path.lstrip('/')))
|
||||||
|
host_ngx_cert_path = Path(os.path.join(host_root_dir, cert_path.lstrip('/')))
|
||||||
|
|
||||||
|
if host_ngx_real_cert_dir.exists() and host_ngx_real_cert_dir.is_dir():
|
||||||
|
shutil.rmtree(host_ngx_real_cert_dir)
|
||||||
|
|
||||||
|
os.makedirs(host_ngx_real_cert_dir, mode=0o755)
|
||||||
|
real_key_path = os.path.join(host_ngx_real_cert_dir, 'server.key')
|
||||||
|
real_crt_path = os.path.join(host_ngx_real_cert_dir, 'server.crt')
|
||||||
|
shutil.copy2(host_ngx_cert_key_path, real_key_path)
|
||||||
|
shutil.copy2(host_ngx_cert_path, real_crt_path)
|
||||||
|
|
||||||
|
os.chown(host_ngx_real_cert_dir, uid=DEFAULT_UID, gid=DEFAULT_GID)
|
||||||
|
mark_file(real_key_path, uid=DEFAULT_UID, gid=DEFAULT_GID)
|
||||||
|
mark_file(real_crt_path, uid=DEFAULT_UID, gid=DEFAULT_GID)
|
||||||
|
|
||||||
|
|
||||||
def render_nginx_template(config_dict):
|
def render_nginx_template(config_dict):
|
||||||
if config_dict['protocol'] == "https":
|
"""
|
||||||
|
1. render nginx config file through protocol
|
||||||
|
2. copy additional configs to cert.d dir
|
||||||
|
"""
|
||||||
|
if config_dict['protocol'] == 'https':
|
||||||
|
prepare_nginx_certs(config_dict['cert_key_path'], config_dict['cert_path'])
|
||||||
render_jinja(
|
render_jinja(
|
||||||
nginx_https_conf_template,
|
nginx_https_conf_template,
|
||||||
nginx_conf,
|
nginx_conf,
|
||||||
|
@ -30,12 +62,7 @@ def render_nginx_template(config_dict):
|
||||||
ssl_cert=SSL_CERT_PATH,
|
ssl_cert=SSL_CERT_PATH,
|
||||||
ssl_cert_key=SSL_CERT_KEY_PATH)
|
ssl_cert_key=SSL_CERT_KEY_PATH)
|
||||||
location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTPS
|
location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTPS
|
||||||
cert_dir = Path(os.path.join(config_dir, 'cert'))
|
|
||||||
ssl_key_path = Path(os.path.join(cert_dir, 'server.key'))
|
|
||||||
ssl_crt_path = Path(os.path.join(cert_dir, 'server.crt'))
|
|
||||||
cert_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
ssl_key_path.touch()
|
|
||||||
ssl_crt_path.touch()
|
|
||||||
else:
|
else:
|
||||||
render_jinja(
|
render_jinja(
|
||||||
nginx_http_conf_template,
|
nginx_http_conf_template,
|
||||||
|
@ -45,22 +72,23 @@ def render_nginx_template(config_dict):
|
||||||
location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTP
|
location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTP
|
||||||
copy_nginx_location_configs_if_exist(nginx_template_ext_dir, nginx_confd_dir, location_file_pattern)
|
copy_nginx_location_configs_if_exist(nginx_template_ext_dir, nginx_confd_dir, location_file_pattern)
|
||||||
|
|
||||||
def add_additional_location_config(src, dst):
|
|
||||||
"""
|
|
||||||
These conf files is used for user that wanna add additional customized locations to harbor proxy
|
|
||||||
:params src: source of the file
|
|
||||||
:params dst: destination file path
|
|
||||||
"""
|
|
||||||
if not os.path.isfile(src):
|
|
||||||
return
|
|
||||||
print("Copying nginx configuration file {src} to {dst}".format(
|
|
||||||
src=src, dst=dst))
|
|
||||||
shutil.copy2(src, dst)
|
|
||||||
mark_file(dst, mode=0o644)
|
|
||||||
|
|
||||||
def copy_nginx_location_configs_if_exist(src_config_dir, dst_config_dir, filename_pattern):
|
def copy_nginx_location_configs_if_exist(src_config_dir, dst_config_dir, filename_pattern):
|
||||||
if not os.path.exists(src_config_dir):
|
if not os.path.exists(src_config_dir):
|
||||||
return
|
return
|
||||||
|
|
||||||
|
def add_additional_location_config(src, dst):
|
||||||
|
"""
|
||||||
|
These conf files is used for user that wanna add additional customized locations to harbor proxy
|
||||||
|
:params src: source of the file
|
||||||
|
:params dst: destination file path
|
||||||
|
"""
|
||||||
|
if not os.path.isfile(src):
|
||||||
|
return
|
||||||
|
print("Copying nginx configuration file {src} to {dst}".format(src=src, dst=dst))
|
||||||
|
shutil.copy2(src, dst)
|
||||||
|
mark_file(dst, mode=0o644)
|
||||||
|
|
||||||
map(lambda filename: add_additional_location_config(
|
map(lambda filename: add_additional_location_config(
|
||||||
os.path.join(src_config_dir, filename),
|
os.path.join(src_config_dir, filename),
|
||||||
os.path.join(dst_config_dir, filename)),
|
os.path.join(dst_config_dir, filename)),
|
||||||
|
|
|
@ -9,6 +9,13 @@ registry_config_dir = os.path.join(config_dir, "registry")
|
||||||
registry_config_template_path = os.path.join(templates_dir, "registry", "config.yml.jinja")
|
registry_config_template_path = os.path.join(templates_dir, "registry", "config.yml.jinja")
|
||||||
registry_conf = os.path.join(config_dir, "registry", "config.yml")
|
registry_conf = os.path.join(config_dir, "registry", "config.yml")
|
||||||
|
|
||||||
|
levels_map = {
|
||||||
|
'debug': 'debug',
|
||||||
|
'info': 'info',
|
||||||
|
'warning': 'warn',
|
||||||
|
'error': 'error',
|
||||||
|
'fatal': 'fatal'
|
||||||
|
}
|
||||||
|
|
||||||
def prepare_registry(config_dict):
|
def prepare_registry(config_dict):
|
||||||
prepare_dir(registry_config_dir)
|
prepare_dir(registry_config_dir)
|
||||||
|
@ -22,6 +29,7 @@ def prepare_registry(config_dict):
|
||||||
registry_conf,
|
registry_conf,
|
||||||
uid=DEFAULT_UID,
|
uid=DEFAULT_UID,
|
||||||
gid=DEFAULT_GID,
|
gid=DEFAULT_GID,
|
||||||
|
level=levels_map[config_dict['log_level']],
|
||||||
storage_provider_info=storage_provider_info,
|
storage_provider_info=storage_provider_info,
|
||||||
**config_dict)
|
**config_dict)
|
||||||
|
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
set +e
|
set +e
|
||||||
|
|
||||||
# If compling source code this dir is harbor's make dir
|
# If compiling source code this dir is harbor's make dir.
|
||||||
# If install harbor via pacakge, this dir is harbor's root dir
|
# If installing harbor via pacakge, this dir is harbor's root dir.
|
||||||
if [[ -n "$HARBOR_BUNDLE_DIR" ]]; then
|
if [[ -n "$HARBOR_BUNDLE_DIR" ]]; then
|
||||||
harbor_prepare_path=$HARBOR_BUNDLE_DIR
|
harbor_prepare_path=$HARBOR_BUNDLE_DIR
|
||||||
else
|
else
|
||||||
|
@ -50,6 +50,7 @@ docker run --rm -v $input_dir:/input:z \
|
||||||
-v $harbor_prepare_path:/compose_location:z \
|
-v $harbor_prepare_path:/compose_location:z \
|
||||||
-v $config_dir:/config:z \
|
-v $config_dir:/config:z \
|
||||||
-v $secret_dir:/secret:z \
|
-v $secret_dir:/secret:z \
|
||||||
|
-v /:/hostfs:z \
|
||||||
goharbor/prepare:dev $@
|
goharbor/prepare:dev $@
|
||||||
|
|
||||||
echo "Clean up the input dir"
|
echo "Clean up the input dir"
|
||||||
|
|
|
@ -210,12 +210,14 @@ func (c *CfgManager) GetDatabaseCfg() *models.Database {
|
||||||
return &models.Database{
|
return &models.Database{
|
||||||
Type: c.Get(common.DatabaseType).GetString(),
|
Type: c.Get(common.DatabaseType).GetString(),
|
||||||
PostGreSQL: &models.PostGreSQL{
|
PostGreSQL: &models.PostGreSQL{
|
||||||
Host: c.Get(common.PostGreSQLHOST).GetString(),
|
Host: c.Get(common.PostGreSQLHOST).GetString(),
|
||||||
Port: c.Get(common.PostGreSQLPort).GetInt(),
|
Port: c.Get(common.PostGreSQLPort).GetInt(),
|
||||||
Username: c.Get(common.PostGreSQLUsername).GetString(),
|
Username: c.Get(common.PostGreSQLUsername).GetString(),
|
||||||
Password: c.Get(common.PostGreSQLPassword).GetString(),
|
Password: c.Get(common.PostGreSQLPassword).GetString(),
|
||||||
Database: c.Get(common.PostGreSQLDatabase).GetString(),
|
Database: c.Get(common.PostGreSQLDatabase).GetString(),
|
||||||
SSLMode: c.Get(common.PostGreSQLSSLMode).GetString(),
|
SSLMode: c.Get(common.PostGreSQLSSLMode).GetString(),
|
||||||
|
MaxIdleConns: c.Get(common.PostGreSQLMaxIdleConns).GetInt(),
|
||||||
|
MaxOpenConns: c.Get(common.PostGreSQLMaxOpenConns).GetInt(),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -116,6 +116,8 @@ var (
|
||||||
{Name: common.PostGreSQLPort, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_PORT", DefaultValue: "5432", ItemType: &PortType{}, Editable: false},
|
{Name: common.PostGreSQLPort, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_PORT", DefaultValue: "5432", ItemType: &PortType{}, Editable: false},
|
||||||
{Name: common.PostGreSQLSSLMode, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_SSLMODE", DefaultValue: "disable", ItemType: &StringType{}, Editable: false},
|
{Name: common.PostGreSQLSSLMode, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_SSLMODE", DefaultValue: "disable", ItemType: &StringType{}, Editable: false},
|
||||||
{Name: common.PostGreSQLUsername, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_USERNAME", DefaultValue: "postgres", ItemType: &StringType{}, Editable: false},
|
{Name: common.PostGreSQLUsername, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_USERNAME", DefaultValue: "postgres", ItemType: &StringType{}, Editable: false},
|
||||||
|
{Name: common.PostGreSQLMaxIdleConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_IDLE_CONNS", DefaultValue: "2", ItemType: &IntType{}, Editable: false},
|
||||||
|
{Name: common.PostGreSQLMaxOpenConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_OPEN_CONNS", DefaultValue: "0", ItemType: &IntType{}, Editable: false},
|
||||||
|
|
||||||
{Name: common.ProjectCreationRestriction, Scope: UserScope, Group: BasicGroup, EnvKey: "PROJECT_CREATION_RESTRICTION", DefaultValue: common.ProCrtRestrEveryone, ItemType: &ProjectCreationRestrictionType{}, Editable: false},
|
{Name: common.ProjectCreationRestriction, Scope: UserScope, Group: BasicGroup, EnvKey: "PROJECT_CREATION_RESTRICTION", DefaultValue: common.ProCrtRestrEveryone, ItemType: &ProjectCreationRestrictionType{}, Editable: false},
|
||||||
{Name: common.ReadOnly, Scope: UserScope, Group: BasicGroup, EnvKey: "READ_ONLY", DefaultValue: "false", ItemType: &BoolType{}, Editable: false},
|
{Name: common.ReadOnly, Scope: UserScope, Group: BasicGroup, EnvKey: "READ_ONLY", DefaultValue: "false", ItemType: &BoolType{}, Editable: false},
|
||||||
|
@ -151,6 +153,7 @@ var (
|
||||||
{Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "43200", ItemType: &IntType{}, Editable: true},
|
{Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "43200", ItemType: &IntType{}, Editable: true},
|
||||||
{Name: common.NotificationEnable, Scope: UserScope, Group: BasicGroup, EnvKey: "NOTIFICATION_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true},
|
{Name: common.NotificationEnable, Scope: UserScope, Group: BasicGroup, EnvKey: "NOTIFICATION_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true},
|
||||||
|
|
||||||
|
{Name: common.QuotaPerProjectEnable, Scope: UserScope, Group: QuotaGroup, EnvKey: "QUOTA_PER_PROJECT_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true},
|
||||||
{Name: common.CountPerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "COUNT_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true},
|
{Name: common.CountPerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "COUNT_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true},
|
||||||
{Name: common.StoragePerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "STORAGE_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true},
|
{Name: common.StoragePerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "STORAGE_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true},
|
||||||
}
|
}
|
||||||
|
|
|
@ -53,6 +53,8 @@ const (
|
||||||
PostGreSQLPassword = "postgresql_password"
|
PostGreSQLPassword = "postgresql_password"
|
||||||
PostGreSQLDatabase = "postgresql_database"
|
PostGreSQLDatabase = "postgresql_database"
|
||||||
PostGreSQLSSLMode = "postgresql_sslmode"
|
PostGreSQLSSLMode = "postgresql_sslmode"
|
||||||
|
PostGreSQLMaxIdleConns = "postgresql_max_idle_conns"
|
||||||
|
PostGreSQLMaxOpenConns = "postgresql_max_open_conns"
|
||||||
SelfRegistration = "self_registration"
|
SelfRegistration = "self_registration"
|
||||||
CoreURL = "core_url"
|
CoreURL = "core_url"
|
||||||
CoreLocalURL = "core_local_url"
|
CoreLocalURL = "core_local_url"
|
||||||
|
@ -147,7 +149,9 @@ const (
|
||||||
|
|
||||||
// Global notification enable configuration
|
// Global notification enable configuration
|
||||||
NotificationEnable = "notification_enable"
|
NotificationEnable = "notification_enable"
|
||||||
|
|
||||||
// Quota setting items for project
|
// Quota setting items for project
|
||||||
CountPerProject = "count_per_project"
|
QuotaPerProjectEnable = "quota_per_project_enable"
|
||||||
StoragePerProject = "storage_per_project"
|
CountPerProject = "count_per_project"
|
||||||
|
StoragePerProject = "storage_per_project"
|
||||||
)
|
)
|
||||||
|
|
|
@ -58,6 +58,7 @@ func UpdateArtifactPullTime(af *models.Artifact) error {
|
||||||
|
|
||||||
// DeleteArtifact ...
|
// DeleteArtifact ...
|
||||||
func DeleteArtifact(id int64) error {
|
func DeleteArtifact(id int64) error {
|
||||||
|
|
||||||
_, err := GetOrmer().QueryTable(&models.Artifact{}).Filter("ID", id).Delete()
|
_, err := GetOrmer().QueryTable(&models.Artifact{}).Filter("ID", id).Delete()
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
@ -121,12 +121,16 @@ func getDatabase(database *models.Database) (db Database, err error) {
|
||||||
|
|
||||||
switch database.Type {
|
switch database.Type {
|
||||||
case "", "postgresql":
|
case "", "postgresql":
|
||||||
db = NewPGSQL(database.PostGreSQL.Host,
|
db = NewPGSQL(
|
||||||
|
database.PostGreSQL.Host,
|
||||||
strconv.Itoa(database.PostGreSQL.Port),
|
strconv.Itoa(database.PostGreSQL.Port),
|
||||||
database.PostGreSQL.Username,
|
database.PostGreSQL.Username,
|
||||||
database.PostGreSQL.Password,
|
database.PostGreSQL.Password,
|
||||||
database.PostGreSQL.Database,
|
database.PostGreSQL.Database,
|
||||||
database.PostGreSQL.SSLMode)
|
database.PostGreSQL.SSLMode,
|
||||||
|
database.PostGreSQL.MaxIdleConns,
|
||||||
|
database.PostGreSQL.MaxOpenConns,
|
||||||
|
)
|
||||||
default:
|
default:
|
||||||
err = fmt.Errorf("invalid database: %s", database.Type)
|
err = fmt.Errorf("invalid database: %s", database.Type)
|
||||||
}
|
}
|
||||||
|
@ -139,6 +143,8 @@ var once sync.Once
|
||||||
// GetOrmer :set ormer singleton
|
// GetOrmer :set ormer singleton
|
||||||
func GetOrmer() orm.Ormer {
|
func GetOrmer() orm.Ormer {
|
||||||
once.Do(func() {
|
once.Do(func() {
|
||||||
|
// override the default value(1000) to return all records when setting no limit
|
||||||
|
orm.DefaultRowsLimit = -1
|
||||||
globalOrm = orm.NewOrm()
|
globalOrm = orm.NewOrm()
|
||||||
})
|
})
|
||||||
return globalOrm
|
return globalOrm
|
||||||
|
|
|
@ -78,10 +78,15 @@ func GetBlobsByArtifact(artifactDigest string) ([]*models.Blob, error) {
|
||||||
|
|
||||||
// GetExclusiveBlobs returns layers of repository:tag which are not shared with other repositories in the project
|
// GetExclusiveBlobs returns layers of repository:tag which are not shared with other repositories in the project
|
||||||
func GetExclusiveBlobs(projectID int64, repository, digest string) ([]*models.Blob, error) {
|
func GetExclusiveBlobs(projectID int64, repository, digest string) ([]*models.Blob, error) {
|
||||||
|
var exclusive []*models.Blob
|
||||||
|
|
||||||
blobs, err := GetBlobsByArtifact(digest)
|
blobs, err := GetBlobsByArtifact(digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if len(blobs) == 0 {
|
||||||
|
return exclusive, nil
|
||||||
|
}
|
||||||
|
|
||||||
sql := fmt.Sprintf(`
|
sql := fmt.Sprintf(`
|
||||||
SELECT
|
SELECT
|
||||||
|
@ -103,13 +108,11 @@ FROM
|
||||||
)
|
)
|
||||||
) AS a
|
) AS a
|
||||||
LEFT JOIN artifact_blob b ON a.digest = b.digest_af
|
LEFT JOIN artifact_blob b ON a.digest = b.digest_af
|
||||||
AND b.digest_blob IN (%s)`, paramPlaceholder(len(blobs)-1))
|
AND b.digest_blob IN (%s)`, ParamPlaceholderForIn(len(blobs)))
|
||||||
|
|
||||||
params := []interface{}{projectID, repository, projectID, digest}
|
params := []interface{}{projectID, repository, projectID, digest}
|
||||||
for _, blob := range blobs {
|
for _, blob := range blobs {
|
||||||
if blob.Digest != digest {
|
params = append(params, blob.Digest)
|
||||||
params = append(params, blob.Digest)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var rows []struct {
|
var rows []struct {
|
||||||
|
@ -125,9 +128,8 @@ FROM
|
||||||
shared[row.Digest] = true
|
shared[row.Digest] = true
|
||||||
}
|
}
|
||||||
|
|
||||||
var exclusive []*models.Blob
|
|
||||||
for _, blob := range blobs {
|
for _, blob := range blobs {
|
||||||
if blob.Digest != digest && !shared[blob.Digest] {
|
if !shared[blob.Digest] {
|
||||||
exclusive = append(exclusive, blob)
|
exclusive = append(exclusive, blob)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -133,30 +133,32 @@ func (suite *GetExclusiveBlobsSuite) mustPrepareImage(projectID int64, projectNa
|
||||||
|
|
||||||
func (suite *GetExclusiveBlobsSuite) TestInSameRepository() {
|
func (suite *GetExclusiveBlobsSuite) TestInSameRepository() {
|
||||||
withProject(func(projectID int64, projectName string) {
|
withProject(func(projectID int64, projectName string) {
|
||||||
|
|
||||||
digest1 := digest.FromString(utils.GenerateRandomString()).String()
|
digest1 := digest.FromString(utils.GenerateRandomString()).String()
|
||||||
digest2 := digest.FromString(utils.GenerateRandomString()).String()
|
digest2 := digest.FromString(utils.GenerateRandomString()).String()
|
||||||
digest3 := digest.FromString(utils.GenerateRandomString()).String()
|
digest3 := digest.FromString(utils.GenerateRandomString()).String()
|
||||||
|
|
||||||
manifest1 := suite.mustPrepareImage(projectID, projectName, "mysql", "latest", digest1, digest2)
|
manifest1 := suite.mustPrepareImage(projectID, projectName, "mysql", "latest", digest1, digest2)
|
||||||
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest1); suite.Nil(err) {
|
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest1); suite.Nil(err) {
|
||||||
suite.Len(blobs, 2)
|
suite.Len(blobs, 3)
|
||||||
}
|
}
|
||||||
|
|
||||||
manifest2 := suite.mustPrepareImage(projectID, projectName, "mysql", "8.0", digest1, digest2)
|
manifest2 := suite.mustPrepareImage(projectID, projectName, "mysql", "8.0", digest1, digest2)
|
||||||
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest2); suite.Nil(err) {
|
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest2); suite.Nil(err) {
|
||||||
suite.Len(blobs, 2)
|
suite.Len(blobs, 3)
|
||||||
}
|
}
|
||||||
|
|
||||||
manifest3 := suite.mustPrepareImage(projectID, projectName, "mysql", "dev", digest1, digest2, digest3)
|
manifest3 := suite.mustPrepareImage(projectID, projectName, "mysql", "dev", digest1, digest2, digest3)
|
||||||
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest1); suite.Nil(err) {
|
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest1); suite.Nil(err) {
|
||||||
suite.Len(blobs, 0)
|
suite.Len(blobs, 1)
|
||||||
|
suite.Equal(manifest1, blobs[0].Digest)
|
||||||
}
|
}
|
||||||
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest2); suite.Nil(err) {
|
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest2); suite.Nil(err) {
|
||||||
suite.Len(blobs, 0)
|
suite.Len(blobs, 1)
|
||||||
|
suite.Equal(manifest2, blobs[0].Digest)
|
||||||
}
|
}
|
||||||
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest3); suite.Nil(err) {
|
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest3); suite.Nil(err) {
|
||||||
suite.Len(blobs, 1)
|
suite.Len(blobs, 2)
|
||||||
suite.Equal(digest3, blobs[0].Digest)
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
@ -169,7 +171,7 @@ func (suite *GetExclusiveBlobsSuite) TestInDifferentRepositories() {
|
||||||
|
|
||||||
manifest1 := suite.mustPrepareImage(projectID, projectName, "mysql", "latest", digest1, digest2)
|
manifest1 := suite.mustPrepareImage(projectID, projectName, "mysql", "latest", digest1, digest2)
|
||||||
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest1); suite.Nil(err) {
|
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest1); suite.Nil(err) {
|
||||||
suite.Len(blobs, 2)
|
suite.Len(blobs, 3)
|
||||||
}
|
}
|
||||||
|
|
||||||
manifest2 := suite.mustPrepareImage(projectID, projectName, "mariadb", "latest", digest1, digest2)
|
manifest2 := suite.mustPrepareImage(projectID, projectName, "mariadb", "latest", digest1, digest2)
|
||||||
|
@ -188,8 +190,7 @@ func (suite *GetExclusiveBlobsSuite) TestInDifferentRepositories() {
|
||||||
suite.Len(blobs, 0)
|
suite.Len(blobs, 0)
|
||||||
}
|
}
|
||||||
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest3); suite.Nil(err) {
|
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest3); suite.Nil(err) {
|
||||||
suite.Len(blobs, 1)
|
suite.Len(blobs, 2)
|
||||||
suite.Equal(digest3, blobs[0].Digest)
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
@ -201,16 +202,16 @@ func (suite *GetExclusiveBlobsSuite) TestInDifferentProjects() {
|
||||||
|
|
||||||
manifest1 := suite.mustPrepareImage(projectID, projectName, "mysql", "latest", digest1, digest2)
|
manifest1 := suite.mustPrepareImage(projectID, projectName, "mysql", "latest", digest1, digest2)
|
||||||
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest1); suite.Nil(err) {
|
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest1); suite.Nil(err) {
|
||||||
suite.Len(blobs, 2)
|
suite.Len(blobs, 3)
|
||||||
}
|
}
|
||||||
|
|
||||||
withProject(func(id int64, name string) {
|
withProject(func(id int64, name string) {
|
||||||
manifest2 := suite.mustPrepareImage(id, name, "mysql", "latest", digest1, digest2)
|
manifest2 := suite.mustPrepareImage(id, name, "mysql", "latest", digest1, digest2)
|
||||||
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest1); suite.Nil(err) {
|
if blobs, err := GetExclusiveBlobs(projectID, projectName+"/mysql", manifest1); suite.Nil(err) {
|
||||||
suite.Len(blobs, 2)
|
suite.Len(blobs, 3)
|
||||||
}
|
}
|
||||||
if blobs, err := GetExclusiveBlobs(id, name+"/mysql", manifest2); suite.Nil(err) {
|
if blobs, err := GetExclusiveBlobs(id, name+"/mysql", manifest2); suite.Nil(err) {
|
||||||
suite.Len(blobs, 2)
|
suite.Len(blobs, 3)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
@ -21,6 +21,14 @@ import (
|
||||||
"github.com/goharbor/harbor/src/common/utils/log"
|
"github.com/goharbor/harbor/src/common/utils/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// CreateCVEWhitelist creates the CVE whitelist
|
||||||
|
func CreateCVEWhitelist(l models.CVEWhitelist) (int64, error) {
|
||||||
|
o := GetOrmer()
|
||||||
|
itemsBytes, _ := json.Marshal(l.Items)
|
||||||
|
l.ItemsText = string(itemsBytes)
|
||||||
|
return o.Insert(&l)
|
||||||
|
}
|
||||||
|
|
||||||
// UpdateCVEWhitelist Updates the vulnerability white list to DB
|
// UpdateCVEWhitelist Updates the vulnerability white list to DB
|
||||||
func UpdateCVEWhitelist(l models.CVEWhitelist) (int64, error) {
|
func UpdateCVEWhitelist(l models.CVEWhitelist) (int64, error) {
|
||||||
o := GetOrmer()
|
o := GetOrmer()
|
||||||
|
@ -30,23 +38,6 @@ func UpdateCVEWhitelist(l models.CVEWhitelist) (int64, error) {
|
||||||
return id, err
|
return id, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetSysCVEWhitelist Gets the system level vulnerability white list from DB
|
|
||||||
func GetSysCVEWhitelist() (*models.CVEWhitelist, error) {
|
|
||||||
return GetCVEWhitelist(0)
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpdateSysCVEWhitelist updates the system level CVE whitelist
|
|
||||||
/*
|
|
||||||
func UpdateSysCVEWhitelist(l models.CVEWhitelist) error {
|
|
||||||
if l.ProjectID != 0 {
|
|
||||||
return fmt.Errorf("system level CVE whitelist cannot set project ID")
|
|
||||||
}
|
|
||||||
l.ProjectID = -1
|
|
||||||
_, err := UpdateCVEWhitelist(l)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
||||||
// GetCVEWhitelist Gets the CVE whitelist of the project based on the project ID in parameter
|
// GetCVEWhitelist Gets the CVE whitelist of the project based on the project ID in parameter
|
||||||
func GetCVEWhitelist(pid int64) (*models.CVEWhitelist, error) {
|
func GetCVEWhitelist(pid int64) (*models.CVEWhitelist, error) {
|
||||||
o := GetOrmer()
|
o := GetOrmer()
|
||||||
|
@ -58,8 +49,7 @@ func GetCVEWhitelist(pid int64) (*models.CVEWhitelist, error) {
|
||||||
return nil, fmt.Errorf("failed to get CVE whitelist for project %d, error: %v", pid, err)
|
return nil, fmt.Errorf("failed to get CVE whitelist for project %d, error: %v", pid, err)
|
||||||
}
|
}
|
||||||
if len(r) == 0 {
|
if len(r) == 0 {
|
||||||
log.Infof("No CVE whitelist found for project %d, returning empty list.", pid)
|
return nil, nil
|
||||||
return &models.CVEWhitelist{ProjectID: pid, Items: []models.CVEWhitelistItem{}}, nil
|
|
||||||
} else if len(r) > 1 {
|
} else if len(r) > 1 {
|
||||||
log.Infof("Multiple CVE whitelists found for project %d, length: %d, returning first element.", pid, len(r))
|
log.Infof("Multiple CVE whitelists found for project %d, length: %d, returning first element.", pid, len(r))
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,12 +23,9 @@ import (
|
||||||
|
|
||||||
func TestUpdateAndGetCVEWhitelist(t *testing.T) {
|
func TestUpdateAndGetCVEWhitelist(t *testing.T) {
|
||||||
require.Nil(t, ClearTable("cve_whitelist"))
|
require.Nil(t, ClearTable("cve_whitelist"))
|
||||||
l, err := GetSysCVEWhitelist()
|
|
||||||
assert.Nil(t, err)
|
|
||||||
assert.Equal(t, models.CVEWhitelist{ProjectID: 0, Items: []models.CVEWhitelistItem{}}, *l)
|
|
||||||
l2, err := GetCVEWhitelist(5)
|
l2, err := GetCVEWhitelist(5)
|
||||||
assert.Nil(t, err)
|
assert.Nil(t, err)
|
||||||
assert.Equal(t, models.CVEWhitelist{ProjectID: 5, Items: []models.CVEWhitelistItem{}}, *l2)
|
assert.Nil(t, l2)
|
||||||
|
|
||||||
longList := []models.CVEWhitelistItem{}
|
longList := []models.CVEWhitelistItem{}
|
||||||
for i := 0; i < 50; i++ {
|
for i := 0; i < 50; i++ {
|
||||||
|
@ -46,15 +43,6 @@ func TestUpdateAndGetCVEWhitelist(t *testing.T) {
|
||||||
assert.Equal(t, longList, out1.Items)
|
assert.Equal(t, longList, out1.Items)
|
||||||
assert.Equal(t, e, *out1.ExpiresAt)
|
assert.Equal(t, e, *out1.ExpiresAt)
|
||||||
|
|
||||||
in2 := models.CVEWhitelist{ProjectID: 3, Items: []models.CVEWhitelistItem{}}
|
|
||||||
_, err = UpdateCVEWhitelist(in2)
|
|
||||||
require.Nil(t, err)
|
|
||||||
// assert.Equal(t, int64(1), n2)
|
|
||||||
out2, err := GetCVEWhitelist(3)
|
|
||||||
require.Nil(t, err)
|
|
||||||
assert.Equal(t, int64(3), out2.ProjectID)
|
|
||||||
assert.Equal(t, []models.CVEWhitelistItem{}, out2.Items)
|
|
||||||
|
|
||||||
sysCVEs := []models.CVEWhitelistItem{
|
sysCVEs := []models.CVEWhitelistItem{
|
||||||
{CVEID: "CVE-2019-10164"},
|
{CVEID: "CVE-2019-10164"},
|
||||||
{CVEID: "CVE-2017-12345"},
|
{CVEID: "CVE-2017-12345"},
|
||||||
|
@ -62,11 +50,6 @@ func TestUpdateAndGetCVEWhitelist(t *testing.T) {
|
||||||
in3 := models.CVEWhitelist{Items: sysCVEs}
|
in3 := models.CVEWhitelist{Items: sysCVEs}
|
||||||
_, err = UpdateCVEWhitelist(in3)
|
_, err = UpdateCVEWhitelist(in3)
|
||||||
require.Nil(t, err)
|
require.Nil(t, err)
|
||||||
// assert.Equal(t, int64(1), n3)
|
|
||||||
sysList, err := GetSysCVEWhitelist()
|
|
||||||
require.Nil(t, err)
|
|
||||||
assert.Equal(t, int64(0), sysList.ProjectID)
|
|
||||||
assert.Equal(t, sysCVEs, sysList.Items)
|
|
||||||
|
|
||||||
// require.Nil(t, ClearTable("cve_whitelist"))
|
require.Nil(t, ClearTable("cve_whitelist"))
|
||||||
}
|
}
|
||||||
|
|
|
@ -31,12 +31,14 @@ import (
|
||||||
const defaultMigrationPath = "migrations/postgresql/"
|
const defaultMigrationPath = "migrations/postgresql/"
|
||||||
|
|
||||||
type pgsql struct {
|
type pgsql struct {
|
||||||
host string
|
host string
|
||||||
port string
|
port string
|
||||||
usr string
|
usr string
|
||||||
pwd string
|
pwd string
|
||||||
database string
|
database string
|
||||||
sslmode string
|
sslmode string
|
||||||
|
maxIdleConns int
|
||||||
|
maxOpenConns int
|
||||||
}
|
}
|
||||||
|
|
||||||
// Name returns the name of PostgreSQL
|
// Name returns the name of PostgreSQL
|
||||||
|
@ -51,17 +53,19 @@ func (p *pgsql) String() string {
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewPGSQL returns an instance of postgres
|
// NewPGSQL returns an instance of postgres
|
||||||
func NewPGSQL(host string, port string, usr string, pwd string, database string, sslmode string) Database {
|
func NewPGSQL(host string, port string, usr string, pwd string, database string, sslmode string, maxIdleConns int, maxOpenConns int) Database {
|
||||||
if len(sslmode) == 0 {
|
if len(sslmode) == 0 {
|
||||||
sslmode = "disable"
|
sslmode = "disable"
|
||||||
}
|
}
|
||||||
return &pgsql{
|
return &pgsql{
|
||||||
host: host,
|
host: host,
|
||||||
port: port,
|
port: port,
|
||||||
usr: usr,
|
usr: usr,
|
||||||
pwd: pwd,
|
pwd: pwd,
|
||||||
database: database,
|
database: database,
|
||||||
sslmode: sslmode,
|
sslmode: sslmode,
|
||||||
|
maxIdleConns: maxIdleConns,
|
||||||
|
maxOpenConns: maxOpenConns,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -82,7 +86,7 @@ func (p *pgsql) Register(alias ...string) error {
|
||||||
info := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",
|
info := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",
|
||||||
p.host, p.port, p.usr, p.pwd, p.database, p.sslmode)
|
p.host, p.port, p.usr, p.pwd, p.database, p.sslmode)
|
||||||
|
|
||||||
return orm.RegisterDataBase(an, "postgres", info)
|
return orm.RegisterDataBase(an, "postgres", info, p.maxIdleConns, p.maxOpenConns)
|
||||||
}
|
}
|
||||||
|
|
||||||
// UpgradeSchema calls migrate tool to upgrade schema to the latest based on the SQL scripts.
|
// UpgradeSchema calls migrate tool to upgrade schema to the latest based on the SQL scripts.
|
||||||
|
|
|
@ -44,7 +44,7 @@ func DeleteProjectMetadata(projectID int64, name ...string) error {
|
||||||
params = append(params, projectID)
|
params = append(params, projectID)
|
||||||
|
|
||||||
if len(name) > 0 {
|
if len(name) > 0 {
|
||||||
sql += fmt.Sprintf(` and name in ( %s )`, paramPlaceholder(len(name)))
|
sql += fmt.Sprintf(` and name in ( %s )`, ParamPlaceholderForIn(len(name)))
|
||||||
params = append(params, name)
|
params = append(params, name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -74,7 +74,7 @@ func GetProjectMetadata(projectID int64, name ...string) ([]*models.ProjectMetad
|
||||||
params = append(params, projectID)
|
params = append(params, projectID)
|
||||||
|
|
||||||
if len(name) > 0 {
|
if len(name) > 0 {
|
||||||
sql += fmt.Sprintf(` and name in ( %s )`, paramPlaceholder(len(name)))
|
sql += fmt.Sprintf(` and name in ( %s )`, ParamPlaceholderForIn(len(name)))
|
||||||
params = append(params, name)
|
params = append(params, name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -82,7 +82,9 @@ func GetProjectMetadata(projectID int64, name ...string) ([]*models.ProjectMetad
|
||||||
return proMetas, err
|
return proMetas, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func paramPlaceholder(n int) string {
|
// ParamPlaceholderForIn returns a string that contains placeholders for sql keyword "in"
|
||||||
|
// e.g. n=3, returns "?,?,?"
|
||||||
|
func ParamPlaceholderForIn(n int) string {
|
||||||
placeholders := []string{}
|
placeholders := []string{}
|
||||||
for i := 0; i < n; i++ {
|
for i := 0; i < n; i++ {
|
||||||
placeholders = append(placeholders, "?")
|
placeholders = append(placeholders, "?")
|
||||||
|
|
|
@ -167,9 +167,10 @@ func GetGroupProjects(groupIDs []int, query *models.ProjectQueryParam) ([]*model
|
||||||
from project p
|
from project p
|
||||||
left join project_member pm on p.project_id = pm.project_id
|
left join project_member pm on p.project_id = pm.project_id
|
||||||
left join user_group ug on ug.id = pm.entity_id and pm.entity_type = 'g'
|
left join user_group ug on ug.id = pm.entity_id and pm.entity_type = 'g'
|
||||||
where ug.id in ( %s ) order by name`,
|
where ug.id in ( %s )`,
|
||||||
sql, groupIDCondition)
|
sql, groupIDCondition)
|
||||||
}
|
}
|
||||||
|
sql = sql + ` order by name`
|
||||||
sqlStr, queryParams := CreatePagination(query, sql, params)
|
sqlStr, queryParams := CreatePagination(query, sql, params)
|
||||||
log.Debugf("query sql:%v", sql)
|
log.Debugf("query sql:%v", sql)
|
||||||
var projects []*models.Project
|
var projects []*models.Project
|
||||||
|
@ -259,7 +260,7 @@ func projectQueryConditions(query *models.ProjectQueryParam) (string, []interfac
|
||||||
}
|
}
|
||||||
if len(query.ProjectIDs) > 0 {
|
if len(query.ProjectIDs) > 0 {
|
||||||
sql += fmt.Sprintf(` and p.project_id in ( %s )`,
|
sql += fmt.Sprintf(` and p.project_id in ( %s )`,
|
||||||
paramPlaceholder(len(query.ProjectIDs)))
|
ParamPlaceholderForIn(len(query.ProjectIDs)))
|
||||||
params = append(params, query.ProjectIDs)
|
params = append(params, query.ProjectIDs)
|
||||||
}
|
}
|
||||||
return sql, params
|
return sql, params
|
||||||
|
|
|
@ -64,7 +64,7 @@ func RemoveBlobsFromProject(projectID int64, blobs ...*models.Blob) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
sql := fmt.Sprintf(`DELETE FROM project_blob WHERE blob_id IN (%s)`, paramPlaceholder(len(blobIDs)))
|
sql := fmt.Sprintf(`DELETE FROM project_blob WHERE blob_id IN (%s)`, ParamPlaceholderForIn(len(blobIDs)))
|
||||||
|
|
||||||
_, err := GetOrmer().Raw(sql, blobIDs).Exec()
|
_, err := GetOrmer().Raw(sql, blobIDs).Exec()
|
||||||
return err
|
return err
|
||||||
|
@ -89,7 +89,7 @@ func GetBlobsNotInProject(projectID int64, blobDigests ...string) ([]*models.Blo
|
||||||
}
|
}
|
||||||
|
|
||||||
sql := fmt.Sprintf("SELECT * FROM blob WHERE id NOT IN (SELECT blob_id FROM project_blob WHERE project_id = ?) AND digest IN (%s)",
|
sql := fmt.Sprintf("SELECT * FROM blob WHERE id NOT IN (SELECT blob_id FROM project_blob WHERE project_id = ?) AND digest IN (%s)",
|
||||||
paramPlaceholder(len(blobDigests)))
|
ParamPlaceholderForIn(len(blobDigests)))
|
||||||
|
|
||||||
params := []interface{}{projectID}
|
params := []interface{}{projectID}
|
||||||
for _, digest := range blobDigests {
|
for _, digest := range blobDigests {
|
||||||
|
@ -103,3 +103,34 @@ func GetBlobsNotInProject(projectID int64, blobDigests ...string) ([]*models.Blo
|
||||||
|
|
||||||
return blobs, nil
|
return blobs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CountSizeOfProject ...
|
||||||
|
func CountSizeOfProject(pid int64) (int64, error) {
|
||||||
|
var blobs []models.Blob
|
||||||
|
|
||||||
|
sql := `
|
||||||
|
SELECT
|
||||||
|
DISTINCT bb.digest,
|
||||||
|
bb.id,
|
||||||
|
bb.content_type,
|
||||||
|
bb.size,
|
||||||
|
bb.creation_time
|
||||||
|
FROM artifact af
|
||||||
|
JOIN artifact_blob afnb
|
||||||
|
ON af.digest = afnb.digest_af
|
||||||
|
JOIN BLOB bb
|
||||||
|
ON afnb.digest_blob = bb.digest
|
||||||
|
WHERE af.project_id = ?
|
||||||
|
`
|
||||||
|
_, err := GetOrmer().Raw(sql, pid).QueryRows(&blobs)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var size int64
|
||||||
|
for _, blob := range blobs {
|
||||||
|
size += blob.Size
|
||||||
|
}
|
||||||
|
|
||||||
|
return size, err
|
||||||
|
}
|
||||||
|
|
|
@ -38,3 +38,161 @@ func TestHasBlobInProject(t *testing.T) {
|
||||||
require.Nil(t, err)
|
require.Nil(t, err)
|
||||||
assert.True(t, has)
|
assert.True(t, has)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCountSizeOfProject(t *testing.T) {
|
||||||
|
_, err := AddBlob(&models.Blob{
|
||||||
|
Digest: "CountSizeOfProject_blob1",
|
||||||
|
Size: 101,
|
||||||
|
})
|
||||||
|
require.Nil(t, err)
|
||||||
|
|
||||||
|
_, err = AddBlob(&models.Blob{
|
||||||
|
Digest: "CountSizeOfProject_blob2",
|
||||||
|
Size: 202,
|
||||||
|
})
|
||||||
|
require.Nil(t, err)
|
||||||
|
|
||||||
|
_, err = AddBlob(&models.Blob{
|
||||||
|
Digest: "CountSizeOfProject_blob3",
|
||||||
|
Size: 303,
|
||||||
|
})
|
||||||
|
require.Nil(t, err)
|
||||||
|
|
||||||
|
pid1, err := AddProject(models.Project{
|
||||||
|
Name: "CountSizeOfProject_project1",
|
||||||
|
OwnerID: 1,
|
||||||
|
})
|
||||||
|
require.Nil(t, err)
|
||||||
|
|
||||||
|
af := &models.Artifact{
|
||||||
|
PID: pid1,
|
||||||
|
Repo: "hello-world",
|
||||||
|
Tag: "v1",
|
||||||
|
Digest: "CountSizeOfProject_af1",
|
||||||
|
Kind: "image",
|
||||||
|
}
|
||||||
|
|
||||||
|
// add
|
||||||
|
_, err = AddArtifact(af)
|
||||||
|
require.Nil(t, err)
|
||||||
|
|
||||||
|
afnb1 := &models.ArtifactAndBlob{
|
||||||
|
DigestAF: "CountSizeOfProject_af1",
|
||||||
|
DigestBlob: "CountSizeOfProject_blob1",
|
||||||
|
}
|
||||||
|
afnb2 := &models.ArtifactAndBlob{
|
||||||
|
DigestAF: "CountSizeOfProject_af1",
|
||||||
|
DigestBlob: "CountSizeOfProject_blob2",
|
||||||
|
}
|
||||||
|
afnb3 := &models.ArtifactAndBlob{
|
||||||
|
DigestAF: "CountSizeOfProject_af1",
|
||||||
|
DigestBlob: "CountSizeOfProject_blob3",
|
||||||
|
}
|
||||||
|
|
||||||
|
var afnbs []*models.ArtifactAndBlob
|
||||||
|
afnbs = append(afnbs, afnb1)
|
||||||
|
afnbs = append(afnbs, afnb2)
|
||||||
|
afnbs = append(afnbs, afnb3)
|
||||||
|
|
||||||
|
// add
|
||||||
|
err = AddArtifactNBlobs(afnbs)
|
||||||
|
require.Nil(t, err)
|
||||||
|
|
||||||
|
pSize, err := CountSizeOfProject(pid1)
|
||||||
|
assert.Equal(t, pSize, int64(606))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCountSizeOfProjectDupdigest(t *testing.T) {
|
||||||
|
_, err := AddBlob(&models.Blob{
|
||||||
|
Digest: "CountSizeOfProject_blob11",
|
||||||
|
Size: 101,
|
||||||
|
})
|
||||||
|
require.Nil(t, err)
|
||||||
|
_, err = AddBlob(&models.Blob{
|
||||||
|
Digest: "CountSizeOfProject_blob22",
|
||||||
|
Size: 202,
|
||||||
|
})
|
||||||
|
require.Nil(t, err)
|
||||||
|
_, err = AddBlob(&models.Blob{
|
||||||
|
Digest: "CountSizeOfProject_blob33",
|
||||||
|
Size: 303,
|
||||||
|
})
|
||||||
|
require.Nil(t, err)
|
||||||
|
_, err = AddBlob(&models.Blob{
|
||||||
|
Digest: "CountSizeOfProject_blob44",
|
||||||
|
Size: 404,
|
||||||
|
})
|
||||||
|
require.Nil(t, err)
|
||||||
|
|
||||||
|
pid1, err := AddProject(models.Project{
|
||||||
|
Name: "CountSizeOfProject_project11",
|
||||||
|
OwnerID: 1,
|
||||||
|
})
|
||||||
|
require.Nil(t, err)
|
||||||
|
|
||||||
|
// add af1 into project
|
||||||
|
af1 := &models.Artifact{
|
||||||
|
PID: pid1,
|
||||||
|
Repo: "hello-world",
|
||||||
|
Tag: "v1",
|
||||||
|
Digest: "CountSizeOfProject_af11",
|
||||||
|
Kind: "image",
|
||||||
|
}
|
||||||
|
_, err = AddArtifact(af1)
|
||||||
|
require.Nil(t, err)
|
||||||
|
afnb11 := &models.ArtifactAndBlob{
|
||||||
|
DigestAF: "CountSizeOfProject_af11",
|
||||||
|
DigestBlob: "CountSizeOfProject_blob11",
|
||||||
|
}
|
||||||
|
afnb12 := &models.ArtifactAndBlob{
|
||||||
|
DigestAF: "CountSizeOfProject_af11",
|
||||||
|
DigestBlob: "CountSizeOfProject_blob22",
|
||||||
|
}
|
||||||
|
afnb13 := &models.ArtifactAndBlob{
|
||||||
|
DigestAF: "CountSizeOfProject_af11",
|
||||||
|
DigestBlob: "CountSizeOfProject_blob33",
|
||||||
|
}
|
||||||
|
var afnbs1 []*models.ArtifactAndBlob
|
||||||
|
afnbs1 = append(afnbs1, afnb11)
|
||||||
|
afnbs1 = append(afnbs1, afnb12)
|
||||||
|
afnbs1 = append(afnbs1, afnb13)
|
||||||
|
err = AddArtifactNBlobs(afnbs1)
|
||||||
|
require.Nil(t, err)
|
||||||
|
|
||||||
|
// add af2 into project
|
||||||
|
af2 := &models.Artifact{
|
||||||
|
PID: pid1,
|
||||||
|
Repo: "hello-world",
|
||||||
|
Tag: "v2",
|
||||||
|
Digest: "CountSizeOfProject_af22",
|
||||||
|
Kind: "image",
|
||||||
|
}
|
||||||
|
_, err = AddArtifact(af2)
|
||||||
|
require.Nil(t, err)
|
||||||
|
afnb21 := &models.ArtifactAndBlob{
|
||||||
|
DigestAF: "CountSizeOfProject_af22",
|
||||||
|
DigestBlob: "CountSizeOfProject_blob11",
|
||||||
|
}
|
||||||
|
afnb22 := &models.ArtifactAndBlob{
|
||||||
|
DigestAF: "CountSizeOfProject_af22",
|
||||||
|
DigestBlob: "CountSizeOfProject_blob22",
|
||||||
|
}
|
||||||
|
afnb23 := &models.ArtifactAndBlob{
|
||||||
|
DigestAF: "CountSizeOfProject_af22",
|
||||||
|
DigestBlob: "CountSizeOfProject_blob33",
|
||||||
|
}
|
||||||
|
afnb24 := &models.ArtifactAndBlob{
|
||||||
|
DigestAF: "CountSizeOfProject_af22",
|
||||||
|
DigestBlob: "CountSizeOfProject_blob44",
|
||||||
|
}
|
||||||
|
var afnbs2 []*models.ArtifactAndBlob
|
||||||
|
afnbs2 = append(afnbs2, afnb21)
|
||||||
|
afnbs2 = append(afnbs2, afnb22)
|
||||||
|
afnbs2 = append(afnbs2, afnb23)
|
||||||
|
afnbs2 = append(afnbs2, afnb24)
|
||||||
|
err = AddArtifactNBlobs(afnbs2)
|
||||||
|
require.Nil(t, err)
|
||||||
|
|
||||||
|
pSize, err := CountSizeOfProject(pid1)
|
||||||
|
assert.Equal(t, pSize, int64(1010))
|
||||||
|
}
|
||||||
|
|
|
@ -193,7 +193,7 @@ func quotaQueryConditions(query ...*models.QuotaQuery) (string, []interface{}) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(q.ReferenceIDs) != 0 {
|
if len(q.ReferenceIDs) != 0 {
|
||||||
sql += fmt.Sprintf(`AND a.reference_id IN (%s) `, paramPlaceholder(len(q.ReferenceIDs)))
|
sql += fmt.Sprintf(`AND a.reference_id IN (%s) `, ParamPlaceholderForIn(len(q.ReferenceIDs)))
|
||||||
params = append(params, q.ReferenceIDs)
|
params = append(params, q.ReferenceIDs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -111,7 +111,7 @@ func quotaUsageQueryConditions(query ...*models.QuotaUsageQuery) (string, []inte
|
||||||
params = append(params, q.ReferenceID)
|
params = append(params, q.ReferenceID)
|
||||||
}
|
}
|
||||||
if len(q.ReferenceIDs) != 0 {
|
if len(q.ReferenceIDs) != 0 {
|
||||||
sql += fmt.Sprintf(`and reference_id in (%s) `, paramPlaceholder(len(q.ReferenceIDs)))
|
sql += fmt.Sprintf(`and reference_id in (%s) `, ParamPlaceholderForIn(len(q.ReferenceIDs)))
|
||||||
params = append(params, q.ReferenceIDs)
|
params = append(params, q.ReferenceIDs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -178,7 +178,7 @@ func repositoryQueryConditions(query ...*models.RepositoryQuery) (string, []inte
|
||||||
|
|
||||||
if len(q.ProjectIDs) > 0 {
|
if len(q.ProjectIDs) > 0 {
|
||||||
sql += fmt.Sprintf(`and r.project_id in ( %s ) `,
|
sql += fmt.Sprintf(`and r.project_id in ( %s ) `,
|
||||||
paramPlaceholder(len(q.ProjectIDs)))
|
ParamPlaceholderForIn(len(q.ProjectIDs)))
|
||||||
params = append(params, q.ProjectIDs)
|
params = append(params, q.ProjectIDs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -117,12 +117,18 @@ func ListUsers(query *models.UserQuery) ([]models.User, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func userQueryConditions(query *models.UserQuery) orm.QuerySeter {
|
func userQueryConditions(query *models.UserQuery) orm.QuerySeter {
|
||||||
qs := GetOrmer().QueryTable(&models.User{}).
|
qs := GetOrmer().QueryTable(&models.User{}).Filter("deleted", 0)
|
||||||
Filter("deleted", 0).
|
|
||||||
Filter("user_id__gt", 1)
|
|
||||||
|
|
||||||
if query == nil {
|
if query == nil {
|
||||||
return qs
|
// Exclude admin account, see https://github.com/goharbor/harbor/issues/2527
|
||||||
|
return qs.Filter("user_id__gt", 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(query.UserIDs) > 0 {
|
||||||
|
qs = qs.Filter("user_id__in", query.UserIDs)
|
||||||
|
} else {
|
||||||
|
// Exclude admin account when not filter by UserIDs, see https://github.com/goharbor/harbor/issues/2527
|
||||||
|
qs = qs.Filter("user_id__gt", 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(query.Username) > 0 {
|
if len(query.Username) > 0 {
|
||||||
|
@ -234,6 +240,14 @@ func OnBoardUser(u *models.User) error {
|
||||||
}
|
}
|
||||||
if created {
|
if created {
|
||||||
u.UserID = int(id)
|
u.UserID = int(id)
|
||||||
|
// current orm framework doesn't support to fetch a pointer or sql.NullString with QueryRow
|
||||||
|
// https://github.com/astaxie/beego/issues/3767
|
||||||
|
if len(u.Email) == 0 {
|
||||||
|
_, err = o.Raw("update harbor_user set email = null where user_id = ? ", id).Exec()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
existing, err := GetUser(*u)
|
existing, err := GetUser(*u)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -90,3 +90,23 @@ func TestOnBoardUser(t *testing.T) {
|
||||||
assert.True(u.UserID == id)
|
assert.True(u.UserID == id)
|
||||||
CleanUser(int64(id))
|
CleanUser(int64(id))
|
||||||
}
|
}
|
||||||
|
func TestOnBoardUser_EmptyEmail(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
u := &models.User{
|
||||||
|
Username: "empty_email",
|
||||||
|
Password: "password1",
|
||||||
|
Realname: "empty_email",
|
||||||
|
}
|
||||||
|
err := OnBoardUser(u)
|
||||||
|
assert.Nil(err)
|
||||||
|
id := u.UserID
|
||||||
|
assert.True(id > 0)
|
||||||
|
err = OnBoardUser(u)
|
||||||
|
assert.Nil(err)
|
||||||
|
assert.True(u.UserID == id)
|
||||||
|
assert.Equal("", u.Email)
|
||||||
|
|
||||||
|
user, err := GetUser(models.User{Username: "empty_email"})
|
||||||
|
assert.Equal("", user.Email)
|
||||||
|
CleanUser(int64(id))
|
||||||
|
}
|
||||||
|
|
|
@ -45,12 +45,14 @@ type SQLite struct {
|
||||||
|
|
||||||
// PostGreSQL ...
|
// PostGreSQL ...
|
||||||
type PostGreSQL struct {
|
type PostGreSQL struct {
|
||||||
Host string `json:"host"`
|
Host string `json:"host"`
|
||||||
Port int `json:"port"`
|
Port int `json:"port"`
|
||||||
Username string `json:"username"`
|
Username string `json:"username"`
|
||||||
Password string `json:"password,omitempty"`
|
Password string `json:"password,omitempty"`
|
||||||
Database string `json:"database"`
|
Database string `json:"database"`
|
||||||
SSLMode string `json:"sslmode"`
|
SSLMode string `json:"sslmode"`
|
||||||
|
MaxIdleConns int `json:"max_idle_conns"`
|
||||||
|
MaxOpenConns int `json:"max_open_conns"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Email ...
|
// Email ...
|
||||||
|
|
|
@ -46,6 +46,7 @@ type User struct {
|
||||||
|
|
||||||
// UserQuery ...
|
// UserQuery ...
|
||||||
type UserQuery struct {
|
type UserQuery struct {
|
||||||
|
UserIDs []int
|
||||||
Username string
|
Username string
|
||||||
Email string
|
Email string
|
||||||
Pagination *Pagination
|
Pagination *Pagination
|
||||||
|
|
|
@ -55,11 +55,23 @@ func getProjectsBatchFn(ctx context.Context, keys dataloader.Keys) []*dataloader
|
||||||
return handleError(err)
|
return handleError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var ownerIDs []int
|
||||||
var projectsMap = make(map[int64]*models.Project, len(projectIDs))
|
var projectsMap = make(map[int64]*models.Project, len(projectIDs))
|
||||||
for _, project := range projects {
|
for _, project := range projects {
|
||||||
|
ownerIDs = append(ownerIDs, project.OwnerID)
|
||||||
projectsMap[project.ProjectID] = project
|
projectsMap[project.ProjectID] = project
|
||||||
}
|
}
|
||||||
|
|
||||||
|
owners, err := dao.ListUsers(&models.UserQuery{UserIDs: ownerIDs})
|
||||||
|
if err != nil {
|
||||||
|
return handleError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var ownersMap = make(map[int]*models.User, len(owners))
|
||||||
|
for i, owner := range owners {
|
||||||
|
ownersMap[owner.UserID] = &owners[i]
|
||||||
|
}
|
||||||
|
|
||||||
var results []*dataloader.Result
|
var results []*dataloader.Result
|
||||||
for _, projectID := range projectIDs {
|
for _, projectID := range projectIDs {
|
||||||
project, ok := projectsMap[projectID]
|
project, ok := projectsMap[projectID]
|
||||||
|
@ -67,6 +79,11 @@ func getProjectsBatchFn(ctx context.Context, keys dataloader.Keys) []*dataloader
|
||||||
return handleError(fmt.Errorf("project not found, "+"project_id: %d", projectID))
|
return handleError(fmt.Errorf("project not found, "+"project_id: %d", projectID))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
owner, ok := ownersMap[project.OwnerID]
|
||||||
|
if ok {
|
||||||
|
project.OwnerName = owner.Username
|
||||||
|
}
|
||||||
|
|
||||||
result := dataloader.Result{
|
result := dataloader.Result{
|
||||||
Data: project,
|
Data: project,
|
||||||
Error: nil,
|
Error: nil,
|
||||||
|
|
|
@ -41,7 +41,7 @@ func (suite *DriverSuite) TestLoad() {
|
||||||
obj := dr.RefObject{
|
obj := dr.RefObject{
|
||||||
"id": int64(1),
|
"id": int64(1),
|
||||||
"name": "library",
|
"name": "library",
|
||||||
"owner_name": "",
|
"owner_name": "admin",
|
||||||
}
|
}
|
||||||
|
|
||||||
suite.Equal(obj, ref)
|
suite.Equal(obj, ref)
|
||||||
|
|
111
src/common/quota/errors.go
Normal file
111
src/common/quota/errors.go
Normal file
|
@ -0,0 +1,111 @@
|
||||||
|
// Copyright Project Harbor Authors
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package quota
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Errors contains all happened errors
|
||||||
|
type Errors []error
|
||||||
|
|
||||||
|
// GetErrors gets all errors that have occurred and returns a slice of errors (Error type)
|
||||||
|
func (errs Errors) GetErrors() []error {
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add adds an error to a given slice of errors
|
||||||
|
func (errs Errors) Add(newErrors ...error) Errors {
|
||||||
|
for _, err := range newErrors {
|
||||||
|
if err == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if errors, ok := err.(Errors); ok {
|
||||||
|
errs = errs.Add(errors...)
|
||||||
|
} else {
|
||||||
|
ok = true
|
||||||
|
for _, e := range errs {
|
||||||
|
if err == e {
|
||||||
|
ok = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
errs = append(errs, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error takes a slice of all errors that have occurred and returns it as a formatted string
|
||||||
|
func (errs Errors) Error() string {
|
||||||
|
var errors = []string{}
|
||||||
|
for _, e := range errs {
|
||||||
|
errors = append(errors, e.Error())
|
||||||
|
}
|
||||||
|
return strings.Join(errors, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResourceOverflow ...
|
||||||
|
type ResourceOverflow struct {
|
||||||
|
Resource types.ResourceName
|
||||||
|
HardLimit int64
|
||||||
|
CurrentUsed int64
|
||||||
|
NewUsed int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ResourceOverflow) Error() string {
|
||||||
|
resource := e.Resource
|
||||||
|
var (
|
||||||
|
op string
|
||||||
|
delta int64
|
||||||
|
)
|
||||||
|
|
||||||
|
if e.NewUsed > e.CurrentUsed {
|
||||||
|
op = "add"
|
||||||
|
delta = e.NewUsed - e.CurrentUsed
|
||||||
|
} else {
|
||||||
|
op = "subtract"
|
||||||
|
delta = e.CurrentUsed - e.NewUsed
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprintf("%s %s of %s resource overflow the hard limit, current usage is %s and hard limit is %s",
|
||||||
|
op, resource.FormatValue(delta), resource,
|
||||||
|
resource.FormatValue(e.CurrentUsed), resource.FormatValue(e.HardLimit))
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewResourceOverflowError ...
|
||||||
|
func NewResourceOverflowError(resource types.ResourceName, hardLimit, currentUsed, newUsed int64) error {
|
||||||
|
return &ResourceOverflow{Resource: resource, HardLimit: hardLimit, CurrentUsed: currentUsed, NewUsed: newUsed}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResourceNotFound ...
|
||||||
|
type ResourceNotFound struct {
|
||||||
|
Resource types.ResourceName
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ResourceNotFound) Error() string {
|
||||||
|
return fmt.Sprintf("resource %s not found", e.Resource)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewResourceNotFoundError ...
|
||||||
|
func NewResourceNotFoundError(resource types.ResourceName) error {
|
||||||
|
return &ResourceNotFound{Resource: resource}
|
||||||
|
}
|
|
@ -110,7 +110,8 @@ func (m *Manager) getUsageForUpdate(o orm.Ormer) (*models.QuotaUsage, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Manager) updateUsage(o orm.Ormer, resources types.ResourceList,
|
func (m *Manager) updateUsage(o orm.Ormer, resources types.ResourceList,
|
||||||
calculate func(types.ResourceList, types.ResourceList) types.ResourceList) error {
|
calculate func(types.ResourceList, types.ResourceList) types.ResourceList,
|
||||||
|
skipOverflow bool) error {
|
||||||
|
|
||||||
quota, err := m.getQuotaForUpdate(o)
|
quota, err := m.getQuotaForUpdate(o)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -131,7 +132,13 @@ func (m *Manager) updateUsage(o orm.Ormer, resources types.ResourceList,
|
||||||
}
|
}
|
||||||
|
|
||||||
newUsed := calculate(used, resources)
|
newUsed := calculate(used, resources)
|
||||||
if err := isSafe(hardLimits, newUsed); err != nil {
|
|
||||||
|
// ensure that new used is never negative
|
||||||
|
if negativeUsed := types.IsNegative(newUsed); len(negativeUsed) > 0 {
|
||||||
|
return fmt.Errorf("quota usage is negative for resource(s): %s", prettyPrintResourceNames(negativeUsed))
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := isSafe(hardLimits, used, newUsed, skipOverflow); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -176,27 +183,87 @@ func (m *Manager) DeleteQuota() error {
|
||||||
|
|
||||||
// UpdateQuota update the quota resource spec
|
// UpdateQuota update the quota resource spec
|
||||||
func (m *Manager) UpdateQuota(hardLimits types.ResourceList) error {
|
func (m *Manager) UpdateQuota(hardLimits types.ResourceList) error {
|
||||||
|
o := dao.GetOrmer()
|
||||||
if err := m.driver.Validate(hardLimits); err != nil {
|
if err := m.driver.Validate(hardLimits); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
sql := `UPDATE quota SET hard = ? WHERE reference = ? AND reference_id = ?`
|
sql := `UPDATE quota SET hard = ? WHERE reference = ? AND reference_id = ?`
|
||||||
_, err := dao.GetOrmer().Raw(sql, hardLimits.String(), m.reference, m.referenceID).Exec()
|
_, err := o.Raw(sql, hardLimits.String(), m.reference, m.referenceID).Exec()
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetResourceUsage sets the usage per resource name
|
||||||
|
func (m *Manager) SetResourceUsage(resource types.ResourceName, value int64) error {
|
||||||
|
o := dao.GetOrmer()
|
||||||
|
|
||||||
|
sql := fmt.Sprintf("UPDATE quota_usage SET used = jsonb_set(used, '{%s}', to_jsonb(%d::int), true) WHERE reference = ? AND reference_id = ?", resource, value)
|
||||||
|
_, err := o.Raw(sql, m.reference, m.referenceID).Exec()
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnsureQuota ensures the reference has quota and usage,
|
||||||
|
// if non-existent, will create new quota and usage.
|
||||||
|
// if existent, update the quota and usage.
|
||||||
|
func (m *Manager) EnsureQuota(usages types.ResourceList) error {
|
||||||
|
query := &models.QuotaQuery{
|
||||||
|
Reference: m.reference,
|
||||||
|
ReferenceID: m.referenceID,
|
||||||
|
}
|
||||||
|
quotas, err := dao.ListQuotas(query)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// non-existent: create quota and usage
|
||||||
|
defaultHardLimit := m.driver.HardLimits()
|
||||||
|
if len(quotas) == 0 {
|
||||||
|
_, err := m.NewQuota(defaultHardLimit, usages)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// existent
|
||||||
|
used := usages
|
||||||
|
quotaUsed, err := types.NewResourceList(quotas[0].Used)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if types.Equals(quotaUsed, used) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
dao.WithTransaction(func(o orm.Ormer) error {
|
||||||
|
usage, err := m.getUsageForUpdate(o)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
usage.Used = used.String()
|
||||||
|
usage.UpdateTime = time.Now()
|
||||||
|
_, err = o.Update(usage)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// AddResources add resources to usage
|
// AddResources add resources to usage
|
||||||
func (m *Manager) AddResources(resources types.ResourceList) error {
|
func (m *Manager) AddResources(resources types.ResourceList) error {
|
||||||
return dao.WithTransaction(func(o orm.Ormer) error {
|
return dao.WithTransaction(func(o orm.Ormer) error {
|
||||||
return m.updateUsage(o, resources, types.Add)
|
return m.updateUsage(o, resources, types.Add, false)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// SubtractResources subtract resources from usage
|
// SubtractResources subtract resources from usage
|
||||||
func (m *Manager) SubtractResources(resources types.ResourceList) error {
|
func (m *Manager) SubtractResources(resources types.ResourceList) error {
|
||||||
return dao.WithTransaction(func(o orm.Ormer) error {
|
return dao.WithTransaction(func(o orm.Ormer) error {
|
||||||
return m.updateUsage(o, resources, types.Subtract)
|
return m.updateUsage(o, resources, types.Subtract, true)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -21,6 +21,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/common/dao"
|
"github.com/goharbor/harbor/src/common/dao"
|
||||||
|
"github.com/goharbor/harbor/src/common/models"
|
||||||
"github.com/goharbor/harbor/src/common/quota/driver"
|
"github.com/goharbor/harbor/src/common/quota/driver"
|
||||||
"github.com/goharbor/harbor/src/common/quota/driver/mocks"
|
"github.com/goharbor/harbor/src/common/quota/driver/mocks"
|
||||||
"github.com/goharbor/harbor/src/pkg/types"
|
"github.com/goharbor/harbor/src/pkg/types"
|
||||||
|
@ -131,6 +132,66 @@ func (suite *ManagerSuite) TestUpdateQuota() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (suite *ManagerSuite) TestSetResourceUsage() {
|
||||||
|
mgr := suite.quotaManager()
|
||||||
|
id, _ := mgr.NewQuota(hardLimits)
|
||||||
|
|
||||||
|
if err := mgr.SetResourceUsage(types.ResourceCount, 123); suite.Nil(err) {
|
||||||
|
quota, _ := dao.GetQuota(id)
|
||||||
|
suite.Equal(hardLimits, mustResourceList(quota.Hard))
|
||||||
|
|
||||||
|
usage, _ := dao.GetQuotaUsage(id)
|
||||||
|
suite.Equal(types.ResourceList{types.ResourceCount: 123, types.ResourceStorage: 0}, mustResourceList(usage.Used))
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := mgr.SetResourceUsage(types.ResourceStorage, 234); suite.Nil(err) {
|
||||||
|
usage, _ := dao.GetQuotaUsage(id)
|
||||||
|
suite.Equal(types.ResourceList{types.ResourceCount: 123, types.ResourceStorage: 234}, mustResourceList(usage.Used))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *ManagerSuite) TestEnsureQuota() {
|
||||||
|
// non-existent
|
||||||
|
nonExistRefID := "3"
|
||||||
|
mgr := suite.quotaManager(nonExistRefID)
|
||||||
|
infinite := types.ResourceList{types.ResourceCount: -1, types.ResourceStorage: -1}
|
||||||
|
usage := types.ResourceList{types.ResourceCount: 10, types.ResourceStorage: 10}
|
||||||
|
err := mgr.EnsureQuota(usage)
|
||||||
|
suite.Nil(err)
|
||||||
|
query := &models.QuotaQuery{
|
||||||
|
Reference: reference,
|
||||||
|
ReferenceID: nonExistRefID,
|
||||||
|
}
|
||||||
|
quotas, err := dao.ListQuotas(query)
|
||||||
|
suite.Nil(err)
|
||||||
|
suite.Equal(usage, mustResourceList(quotas[0].Used))
|
||||||
|
suite.Equal(infinite, mustResourceList(quotas[0].Hard))
|
||||||
|
|
||||||
|
// existent
|
||||||
|
existRefID := "4"
|
||||||
|
mgr = suite.quotaManager(existRefID)
|
||||||
|
used := types.ResourceList{types.ResourceCount: 11, types.ResourceStorage: 11}
|
||||||
|
if id, err := mgr.NewQuota(hardLimits, used); suite.Nil(err) {
|
||||||
|
quota, _ := dao.GetQuota(id)
|
||||||
|
suite.Equal(hardLimits, mustResourceList(quota.Hard))
|
||||||
|
|
||||||
|
usage, _ := dao.GetQuotaUsage(id)
|
||||||
|
suite.Equal(used, mustResourceList(usage.Used))
|
||||||
|
}
|
||||||
|
|
||||||
|
usage2 := types.ResourceList{types.ResourceCount: 12, types.ResourceStorage: 12}
|
||||||
|
err = mgr.EnsureQuota(usage2)
|
||||||
|
suite.Nil(err)
|
||||||
|
query2 := &models.QuotaQuery{
|
||||||
|
Reference: reference,
|
||||||
|
ReferenceID: existRefID,
|
||||||
|
}
|
||||||
|
quotas2, err := dao.ListQuotas(query2)
|
||||||
|
suite.Equal(usage2, mustResourceList(quotas2[0].Used))
|
||||||
|
suite.Equal(hardLimits, mustResourceList(quotas2[0].Hard))
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
func (suite *ManagerSuite) TestQuotaAutoCreation() {
|
func (suite *ManagerSuite) TestQuotaAutoCreation() {
|
||||||
for i := 0; i < 10; i++ {
|
for i := 0; i < 10; i++ {
|
||||||
mgr := suite.quotaManager(fmt.Sprintf("%d", i))
|
mgr := suite.quotaManager(fmt.Sprintf("%d", i))
|
||||||
|
@ -157,7 +218,11 @@ func (suite *ManagerSuite) TestAddResources() {
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := mgr.AddResources(types.ResourceList{types.ResourceStorage: 10000}); suite.Error(err) {
|
if err := mgr.AddResources(types.ResourceList{types.ResourceStorage: 10000}); suite.Error(err) {
|
||||||
suite.True(IsUnsafeError(err))
|
if errs, ok := err.(Errors); suite.True(ok) {
|
||||||
|
for _, err := range errs {
|
||||||
|
suite.IsType(&ResourceOverflow{}, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -15,48 +15,43 @@
|
||||||
package quota
|
package quota
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/types"
|
"github.com/goharbor/harbor/src/pkg/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
type unsafe struct {
|
func isSafe(hardLimits types.ResourceList, currentUsed types.ResourceList, newUsed types.ResourceList, skipOverflow bool) error {
|
||||||
message string
|
var errs Errors
|
||||||
}
|
|
||||||
|
|
||||||
func (err *unsafe) Error() string {
|
for resource, value := range newUsed {
|
||||||
return err.message
|
hardLimit, found := hardLimits[resource]
|
||||||
}
|
if !found {
|
||||||
|
errs = errs.Add(NewResourceNotFoundError(resource))
|
||||||
func newUnsafe(message string) error {
|
continue
|
||||||
return &unsafe{message: message}
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsUnsafeError returns true when the err is unsafe error
|
|
||||||
func IsUnsafeError(err error) bool {
|
|
||||||
_, ok := err.(*unsafe)
|
|
||||||
return ok
|
|
||||||
}
|
|
||||||
|
|
||||||
func isSafe(hardLimits types.ResourceList, used types.ResourceList) error {
|
|
||||||
for key, value := range used {
|
|
||||||
if value < 0 {
|
|
||||||
return newUnsafe(fmt.Sprintf("bad used value: %d", value))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if hard, found := hardLimits[key]; found {
|
if hardLimit == types.UNLIMITED || value == currentUsed[resource] {
|
||||||
if hard == types.UNLIMITED {
|
continue
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if value > hard {
|
|
||||||
return newUnsafe(fmt.Sprintf("over the quota: used %d but only hard %d", value, hard))
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
return newUnsafe(fmt.Sprintf("hard limit not found: %s", key))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if value > hardLimit && !skipOverflow {
|
||||||
|
errs = errs.Add(NewResourceOverflowError(resource, hardLimit, currentUsed[resource], value))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(errs) > 0 {
|
||||||
|
return errs
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func prettyPrintResourceNames(a []types.ResourceName) string {
|
||||||
|
values := []string{}
|
||||||
|
for _, value := range a {
|
||||||
|
values = append(values, string(value))
|
||||||
|
}
|
||||||
|
sort.Strings(values)
|
||||||
|
return strings.Join(values, ",")
|
||||||
|
}
|
||||||
|
|
|
@ -15,45 +15,17 @@
|
||||||
package quota
|
package quota
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/pkg/types"
|
"github.com/goharbor/harbor/src/pkg/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestIsUnsafeError(t *testing.T) {
|
func Test_isSafe(t *testing.T) {
|
||||||
type args struct {
|
type args struct {
|
||||||
err error
|
hardLimits types.ResourceList
|
||||||
}
|
currentUsed types.ResourceList
|
||||||
tests := []struct {
|
newUsed types.ResourceList
|
||||||
name string
|
skipOverflow bool
|
||||||
args args
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
"is unsafe error",
|
|
||||||
args{err: newUnsafe("unsafe")},
|
|
||||||
true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"is not unsafe error",
|
|
||||||
args{err: errors.New("unsafe")},
|
|
||||||
false,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
if got := IsUnsafeError(tt.args.err); got != tt.want {
|
|
||||||
t.Errorf("IsUnsafeError() = %v, want %v", got, tt.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_checkQuotas(t *testing.T) {
|
|
||||||
type args struct {
|
|
||||||
hardLimits types.ResourceList
|
|
||||||
used types.ResourceList
|
|
||||||
}
|
}
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
@ -62,33 +34,58 @@ func Test_checkQuotas(t *testing.T) {
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
"unlimited",
|
"unlimited",
|
||||||
args{hardLimits: types.ResourceList{types.ResourceStorage: types.UNLIMITED}, used: types.ResourceList{types.ResourceStorage: 1000}},
|
args{
|
||||||
|
types.ResourceList{types.ResourceStorage: types.UNLIMITED},
|
||||||
|
types.ResourceList{types.ResourceStorage: 1000},
|
||||||
|
types.ResourceList{types.ResourceStorage: 1000},
|
||||||
|
false,
|
||||||
|
},
|
||||||
false,
|
false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"ok",
|
"ok",
|
||||||
args{hardLimits: types.ResourceList{types.ResourceStorage: 100}, used: types.ResourceList{types.ResourceStorage: 1}},
|
args{
|
||||||
|
types.ResourceList{types.ResourceStorage: 100},
|
||||||
|
types.ResourceList{types.ResourceStorage: 10},
|
||||||
|
types.ResourceList{types.ResourceStorage: 1},
|
||||||
|
false,
|
||||||
|
},
|
||||||
false,
|
false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"bad used value",
|
"over the hard limit",
|
||||||
args{hardLimits: types.ResourceList{types.ResourceStorage: 100}, used: types.ResourceList{types.ResourceStorage: -1}},
|
args{
|
||||||
|
types.ResourceList{types.ResourceStorage: 100},
|
||||||
|
types.ResourceList{types.ResourceStorage: 0},
|
||||||
|
types.ResourceList{types.ResourceStorage: 200},
|
||||||
|
false,
|
||||||
|
},
|
||||||
true,
|
true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"over the hard limit",
|
"skip overflow",
|
||||||
args{hardLimits: types.ResourceList{types.ResourceStorage: 100}, used: types.ResourceList{types.ResourceStorage: 200}},
|
args{
|
||||||
true,
|
types.ResourceList{types.ResourceStorage: 100},
|
||||||
|
types.ResourceList{types.ResourceStorage: 0},
|
||||||
|
types.ResourceList{types.ResourceStorage: 200},
|
||||||
|
true,
|
||||||
|
},
|
||||||
|
false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"hard limit not found",
|
"hard limit not found",
|
||||||
args{hardLimits: types.ResourceList{types.ResourceStorage: 100}, used: types.ResourceList{types.ResourceCount: 1}},
|
args{
|
||||||
|
types.ResourceList{types.ResourceStorage: 100},
|
||||||
|
types.ResourceList{types.ResourceCount: 0},
|
||||||
|
types.ResourceList{types.ResourceCount: 1},
|
||||||
|
false,
|
||||||
|
},
|
||||||
true,
|
true,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
if err := isSafe(tt.args.hardLimits, tt.args.used); (err != nil) != tt.wantErr {
|
if err := isSafe(tt.args.hardLimits, tt.args.currentUsed, tt.args.newUsed, tt.args.skipOverflow); (err != nil) != tt.wantErr {
|
||||||
t.Errorf("isSafe() error = %v, wantErr %v", err, tt.wantErr)
|
t.Errorf("isSafe() error = %v, wantErr %v", err, tt.wantErr)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
|
@ -31,8 +31,8 @@ type Namespace interface {
|
||||||
}
|
}
|
||||||
|
|
||||||
type projectNamespace struct {
|
type projectNamespace struct {
|
||||||
projectIDOrName interface{}
|
projectID int64
|
||||||
isPublic bool
|
isPublic bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ns *projectNamespace) Kind() string {
|
func (ns *projectNamespace) Kind() string {
|
||||||
|
@ -40,11 +40,11 @@ func (ns *projectNamespace) Kind() string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ns *projectNamespace) Resource(subresources ...Resource) Resource {
|
func (ns *projectNamespace) Resource(subresources ...Resource) Resource {
|
||||||
return Resource(fmt.Sprintf("/project/%v", ns.projectIDOrName)).Subresource(subresources...)
|
return Resource(fmt.Sprintf("/project/%d", ns.projectID)).Subresource(subresources...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ns *projectNamespace) Identity() interface{} {
|
func (ns *projectNamespace) Identity() interface{} {
|
||||||
return ns.projectIDOrName
|
return ns.projectID
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ns *projectNamespace) IsPublic() bool {
|
func (ns *projectNamespace) IsPublic() bool {
|
||||||
|
@ -52,10 +52,10 @@ func (ns *projectNamespace) IsPublic() bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewProjectNamespace returns namespace for project
|
// NewProjectNamespace returns namespace for project
|
||||||
func NewProjectNamespace(projectIDOrName interface{}, isPublic ...bool) Namespace {
|
func NewProjectNamespace(projectID int64, isPublic ...bool) Namespace {
|
||||||
isPublicNamespace := false
|
isPublicNamespace := false
|
||||||
if len(isPublic) > 0 {
|
if len(isPublic) > 0 {
|
||||||
isPublicNamespace = isPublic[0]
|
isPublicNamespace = isPublic[0]
|
||||||
}
|
}
|
||||||
return &projectNamespace{projectIDOrName: projectIDOrName, isPublic: isPublicNamespace}
|
return &projectNamespace{projectID: projectID, isPublic: isPublicNamespace}
|
||||||
}
|
}
|
||||||
|
|
|
@ -27,7 +27,7 @@ type ProjectNamespaceTestSuite struct {
|
||||||
func (suite *ProjectNamespaceTestSuite) TestResource() {
|
func (suite *ProjectNamespaceTestSuite) TestResource() {
|
||||||
var namespace Namespace
|
var namespace Namespace
|
||||||
|
|
||||||
namespace = &projectNamespace{projectIDOrName: int64(1)}
|
namespace = &projectNamespace{projectID: int64(1)}
|
||||||
|
|
||||||
suite.Equal(namespace.Resource(Resource("image")), Resource("/project/1/image"))
|
suite.Equal(namespace.Resource(Resource("image")), Resource("/project/1/image"))
|
||||||
}
|
}
|
||||||
|
@ -35,9 +35,6 @@ func (suite *ProjectNamespaceTestSuite) TestResource() {
|
||||||
func (suite *ProjectNamespaceTestSuite) TestIdentity() {
|
func (suite *ProjectNamespaceTestSuite) TestIdentity() {
|
||||||
namespace, _ := Resource("/project/1/image").GetNamespace()
|
namespace, _ := Resource("/project/1/image").GetNamespace()
|
||||||
suite.Equal(namespace.Identity(), int64(1))
|
suite.Equal(namespace.Identity(), int64(1))
|
||||||
|
|
||||||
namespace, _ = Resource("/project/library/image").GetNamespace()
|
|
||||||
suite.Equal(namespace.Identity(), "library")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProjectNamespaceTestSuite(t *testing.T) {
|
func TestProjectNamespaceTestSuite(t *testing.T) {
|
||||||
|
|
|
@ -37,14 +37,10 @@ func projectNamespaceParser(resource Resource) (Namespace, error) {
|
||||||
return nil, errors.New("not support resource")
|
return nil, errors.New("not support resource")
|
||||||
}
|
}
|
||||||
|
|
||||||
var projectIDOrName interface{}
|
projectID, err := strconv.ParseInt(matches[1], 10, 64)
|
||||||
|
if err != nil {
|
||||||
id, err := strconv.ParseInt(matches[1], 10, 64)
|
return nil, err
|
||||||
if err == nil {
|
|
||||||
projectIDOrName = id
|
|
||||||
} else {
|
|
||||||
projectIDOrName = matches[1]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return &projectNamespace{projectIDOrName: projectIDOrName}, nil
|
return &projectNamespace{projectID: projectID}, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -26,7 +26,7 @@ type ProjectParserTestSuite struct {
|
||||||
|
|
||||||
func (suite *ProjectParserTestSuite) TestParse() {
|
func (suite *ProjectParserTestSuite) TestParse() {
|
||||||
namespace, err := projectNamespaceParser(Resource("/project/1/image"))
|
namespace, err := projectNamespaceParser(Resource("/project/1/image"))
|
||||||
suite.Equal(namespace, &projectNamespace{projectIDOrName: int64(1)})
|
suite.Equal(namespace, &projectNamespace{projectID: 1})
|
||||||
suite.Nil(err)
|
suite.Nil(err)
|
||||||
|
|
||||||
namespace, err = projectNamespaceParser(Resource("/fake/1/image"))
|
namespace, err = projectNamespaceParser(Resource("/fake/1/image"))
|
||||||
|
|
|
@ -50,8 +50,8 @@ type VisitorTestSuite struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *VisitorTestSuite) TestGetPolicies() {
|
func (suite *VisitorTestSuite) TestGetPolicies() {
|
||||||
namespace := rbac.NewProjectNamespace("library", false)
|
namespace := rbac.NewProjectNamespace(1, false)
|
||||||
publicNamespace := rbac.NewProjectNamespace("library", true)
|
publicNamespace := rbac.NewProjectNamespace(1, true)
|
||||||
|
|
||||||
anonymous := NewUser(anonymousCtx, namespace)
|
anonymous := NewUser(anonymousCtx, namespace)
|
||||||
suite.Nil(anonymous.GetPolicies())
|
suite.Nil(anonymous.GetPolicies())
|
||||||
|
@ -73,7 +73,7 @@ func (suite *VisitorTestSuite) TestGetPolicies() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *VisitorTestSuite) TestGetRoles() {
|
func (suite *VisitorTestSuite) TestGetRoles() {
|
||||||
namespace := rbac.NewProjectNamespace("library", false)
|
namespace := rbac.NewProjectNamespace(1, false)
|
||||||
|
|
||||||
anonymous := NewUser(anonymousCtx, namespace)
|
anonymous := NewUser(anonymousCtx, namespace)
|
||||||
suite.Nil(anonymous.GetRoles())
|
suite.Nil(anonymous.GetRoles())
|
||||||
|
|
|
@ -75,10 +75,10 @@ func (s *SecurityContext) Can(action rbac.Action, resource rbac.Resource) bool {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
switch ns.Kind() {
|
switch ns.Kind() {
|
||||||
case "project":
|
case "project":
|
||||||
projectIDOrName := ns.Identity()
|
projectID := ns.Identity().(int64)
|
||||||
isPublicProject, _ := s.pm.IsPublic(projectIDOrName)
|
isPublicProject, _ := s.pm.IsPublic(projectID)
|
||||||
projectNamespace := rbac.NewProjectNamespace(projectIDOrName, isPublicProject)
|
projectNamespace := rbac.NewProjectNamespace(projectID, isPublicProject)
|
||||||
user := project.NewUser(s, projectNamespace, s.GetProjectRoles(projectIDOrName)...)
|
user := project.NewUser(s, projectNamespace, s.GetProjectRoles(projectID)...)
|
||||||
return rbac.HasPermission(user, resource, action)
|
return rbac.HasPermission(user, resource, action)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -72,10 +72,10 @@ func (s *SecurityContext) Can(action rbac.Action, resource rbac.Resource) bool {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
switch ns.Kind() {
|
switch ns.Kind() {
|
||||||
case "project":
|
case "project":
|
||||||
projectIDOrName := ns.Identity()
|
projectID := ns.Identity().(int64)
|
||||||
isPublicProject, _ := s.pm.IsPublic(projectIDOrName)
|
isPublicProject, _ := s.pm.IsPublic(projectID)
|
||||||
projectNamespace := rbac.NewProjectNamespace(projectIDOrName, isPublicProject)
|
projectNamespace := rbac.NewProjectNamespace(projectID, isPublicProject)
|
||||||
user := project.NewUser(s, projectNamespace, s.GetProjectRoles(projectIDOrName)...)
|
user := project.NewUser(s, projectNamespace, s.GetProjectRoles(projectID)...)
|
||||||
return rbac.HasPermission(user, resource, action)
|
return rbac.HasPermission(user, resource, action)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -176,12 +176,12 @@ func TestHasPullPerm(t *testing.T) {
|
||||||
// public project
|
// public project
|
||||||
ctx := NewSecurityContext(nil, pm)
|
ctx := NewSecurityContext(nil, pm)
|
||||||
|
|
||||||
resource := rbac.NewProjectNamespace("library").Resource(rbac.ResourceRepository)
|
resource := rbac.NewProjectNamespace(1).Resource(rbac.ResourceRepository)
|
||||||
assert.True(t, ctx.Can(rbac.ActionPull, resource))
|
assert.True(t, ctx.Can(rbac.ActionPull, resource))
|
||||||
|
|
||||||
// private project, unauthenticated
|
// private project, unauthenticated
|
||||||
ctx = NewSecurityContext(nil, pm)
|
ctx = NewSecurityContext(nil, pm)
|
||||||
resource = rbac.NewProjectNamespace(private.Name).Resource(rbac.ResourceRepository)
|
resource = rbac.NewProjectNamespace(private.ProjectID).Resource(rbac.ResourceRepository)
|
||||||
assert.False(t, ctx.Can(rbac.ActionPull, resource))
|
assert.False(t, ctx.Can(rbac.ActionPull, resource))
|
||||||
|
|
||||||
// private project, authenticated, has no perm
|
// private project, authenticated, has no perm
|
||||||
|
@ -203,7 +203,7 @@ func TestHasPullPerm(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHasPushPerm(t *testing.T) {
|
func TestHasPushPerm(t *testing.T) {
|
||||||
resource := rbac.NewProjectNamespace(private.Name).Resource(rbac.ResourceRepository)
|
resource := rbac.NewProjectNamespace(private.ProjectID).Resource(rbac.ResourceRepository)
|
||||||
|
|
||||||
// unauthenticated
|
// unauthenticated
|
||||||
ctx := NewSecurityContext(nil, pm)
|
ctx := NewSecurityContext(nil, pm)
|
||||||
|
@ -226,7 +226,7 @@ func TestHasPushPerm(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHasPushPullPerm(t *testing.T) {
|
func TestHasPushPullPerm(t *testing.T) {
|
||||||
resource := rbac.NewProjectNamespace(private.Name).Resource(rbac.ResourceRepository)
|
resource := rbac.NewProjectNamespace(private.ProjectID).Resource(rbac.ResourceRepository)
|
||||||
|
|
||||||
// unauthenticated
|
// unauthenticated
|
||||||
ctx := NewSecurityContext(nil, pm)
|
ctx := NewSecurityContext(nil, pm)
|
||||||
|
@ -265,7 +265,7 @@ func TestHasPushPullPermWithGroup(t *testing.T) {
|
||||||
|
|
||||||
developer.GroupIDs = []int{userGroups[0].ID}
|
developer.GroupIDs = []int{userGroups[0].ID}
|
||||||
|
|
||||||
resource := rbac.NewProjectNamespace(project.Name).Resource(rbac.ResourceRepository)
|
resource := rbac.NewProjectNamespace(project.ProjectID).Resource(rbac.ResourceRepository)
|
||||||
|
|
||||||
ctx := NewSecurityContext(developer, pm)
|
ctx := NewSecurityContext(developer, pm)
|
||||||
assert.True(t, ctx.Can(rbac.ActionPush, resource))
|
assert.True(t, ctx.Can(rbac.ActionPush, resource))
|
||||||
|
|
|
@ -76,9 +76,9 @@ func (s *SecurityContext) Can(action rbac.Action, resource rbac.Resource) bool {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
switch ns.Kind() {
|
switch ns.Kind() {
|
||||||
case "project":
|
case "project":
|
||||||
projectIDOrName := ns.Identity()
|
projectID := ns.Identity().(int64)
|
||||||
isPublicProject, _ := s.pm.IsPublic(projectIDOrName)
|
isPublicProject, _ := s.pm.IsPublic(projectID)
|
||||||
projectNamespace := rbac.NewProjectNamespace(projectIDOrName, isPublicProject)
|
projectNamespace := rbac.NewProjectNamespace(projectID, isPublicProject)
|
||||||
robot := NewRobot(s.GetUsername(), projectNamespace, s.policy)
|
robot := NewRobot(s.GetUsername(), projectNamespace, s.policy)
|
||||||
return rbac.HasPermission(robot, resource, action)
|
return rbac.HasPermission(robot, resource, action)
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,6 +15,7 @@
|
||||||
package robot
|
package robot
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
"testing"
|
"testing"
|
||||||
|
@ -136,7 +137,7 @@ func TestIsSolutionUser(t *testing.T) {
|
||||||
func TestHasPullPerm(t *testing.T) {
|
func TestHasPullPerm(t *testing.T) {
|
||||||
policies := []*rbac.Policy{
|
policies := []*rbac.Policy{
|
||||||
{
|
{
|
||||||
Resource: "/project/testrobot/repository",
|
Resource: rbac.Resource(fmt.Sprintf("/project/%d/repository", private.ProjectID)),
|
||||||
Action: rbac.ActionPull,
|
Action: rbac.ActionPull,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -146,14 +147,14 @@ func TestHasPullPerm(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx := NewSecurityContext(robot, pm, policies)
|
ctx := NewSecurityContext(robot, pm, policies)
|
||||||
resource := rbac.NewProjectNamespace(private.Name).Resource(rbac.ResourceRepository)
|
resource := rbac.NewProjectNamespace(private.ProjectID).Resource(rbac.ResourceRepository)
|
||||||
assert.True(t, ctx.Can(rbac.ActionPull, resource))
|
assert.True(t, ctx.Can(rbac.ActionPull, resource))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHasPushPerm(t *testing.T) {
|
func TestHasPushPerm(t *testing.T) {
|
||||||
policies := []*rbac.Policy{
|
policies := []*rbac.Policy{
|
||||||
{
|
{
|
||||||
Resource: "/project/testrobot/repository",
|
Resource: rbac.Resource(fmt.Sprintf("/project/%d/repository", private.ProjectID)),
|
||||||
Action: rbac.ActionPush,
|
Action: rbac.ActionPush,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -163,18 +164,18 @@ func TestHasPushPerm(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx := NewSecurityContext(robot, pm, policies)
|
ctx := NewSecurityContext(robot, pm, policies)
|
||||||
resource := rbac.NewProjectNamespace(private.Name).Resource(rbac.ResourceRepository)
|
resource := rbac.NewProjectNamespace(private.ProjectID).Resource(rbac.ResourceRepository)
|
||||||
assert.True(t, ctx.Can(rbac.ActionPush, resource))
|
assert.True(t, ctx.Can(rbac.ActionPush, resource))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHasPushPullPerm(t *testing.T) {
|
func TestHasPushPullPerm(t *testing.T) {
|
||||||
policies := []*rbac.Policy{
|
policies := []*rbac.Policy{
|
||||||
{
|
{
|
||||||
Resource: "/project/testrobot/repository",
|
Resource: rbac.Resource(fmt.Sprintf("/project/%d/repository", private.ProjectID)),
|
||||||
Action: rbac.ActionPush,
|
Action: rbac.ActionPush,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Resource: "/project/testrobot/repository",
|
Resource: rbac.Resource(fmt.Sprintf("/project/%d/repository", private.ProjectID)),
|
||||||
Action: rbac.ActionPull,
|
Action: rbac.ActionPull,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -184,7 +185,7 @@ func TestHasPushPullPerm(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx := NewSecurityContext(robot, pm, policies)
|
ctx := NewSecurityContext(robot, pm, policies)
|
||||||
resource := rbac.NewProjectNamespace(private.Name).Resource(rbac.ResourceRepository)
|
resource := rbac.NewProjectNamespace(private.ProjectID).Resource(rbac.ResourceRepository)
|
||||||
assert.True(t, ctx.Can(rbac.ActionPush, resource) && ctx.Can(rbac.ActionPull, resource))
|
assert.True(t, ctx.Can(rbac.ActionPush, resource) && ctx.Can(rbac.ActionPull, resource))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,9 +1,24 @@
|
||||||
|
// Copyright Project Harbor Authors
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
package robot
|
package robot
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/common/rbac"
|
"github.com/goharbor/harbor/src/common/rbac"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"testing"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestGetPolicies(t *testing.T) {
|
func TestGetPolicies(t *testing.T) {
|
||||||
|
@ -17,7 +32,7 @@ func TestGetPolicies(t *testing.T) {
|
||||||
|
|
||||||
robot := robot{
|
robot := robot{
|
||||||
username: "test",
|
username: "test",
|
||||||
namespace: rbac.NewProjectNamespace("library", false),
|
namespace: rbac.NewProjectNamespace(1, false),
|
||||||
policy: policies,
|
policy: policies,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -220,6 +220,27 @@ func (session *Session) SearchUser(username string) ([]models.LdapUser, error) {
|
||||||
}
|
}
|
||||||
u.GroupDNList = groupDNList
|
u.GroupDNList = groupDNList
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Debugf("Searching for nested groups")
|
||||||
|
nestedGroupDNList := []string{}
|
||||||
|
nestedGroupFilter := createNestedGroupFilter(ldapEntry.DN)
|
||||||
|
result, err := session.SearchLdap(nestedGroupFilter)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, groupEntry := range result.Entries {
|
||||||
|
if !contains(u.GroupDNList, groupEntry.DN) {
|
||||||
|
nestedGroupDNList = append(nestedGroupDNList, strings.TrimSpace(groupEntry.DN))
|
||||||
|
log.Debugf("Found group %v", groupEntry.DN)
|
||||||
|
} else {
|
||||||
|
log.Debugf("%v is already in GroupDNList", groupEntry.DN)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
u.GroupDNList = append(u.GroupDNList, nestedGroupDNList...)
|
||||||
|
log.Debugf("Done searching for nested groups")
|
||||||
|
|
||||||
u.DN = ldapEntry.DN
|
u.DN = ldapEntry.DN
|
||||||
ldapUsers = append(ldapUsers, u)
|
ldapUsers = append(ldapUsers, u)
|
||||||
|
|
||||||
|
@ -330,13 +351,13 @@ func (session *Session) createUserFilter(username string) string {
|
||||||
filterTag = goldap.EscapeFilter(username)
|
filterTag = goldap.EscapeFilter(username)
|
||||||
}
|
}
|
||||||
|
|
||||||
ldapFilter := session.ldapConfig.LdapFilter
|
ldapFilter := normalizeFilter(session.ldapConfig.LdapFilter)
|
||||||
ldapUID := session.ldapConfig.LdapUID
|
ldapUID := session.ldapConfig.LdapUID
|
||||||
|
|
||||||
if ldapFilter == "" {
|
if ldapFilter == "" {
|
||||||
ldapFilter = "(" + ldapUID + "=" + filterTag + ")"
|
ldapFilter = "(" + ldapUID + "=" + filterTag + ")"
|
||||||
} else {
|
} else {
|
||||||
ldapFilter = "(&" + ldapFilter + "(" + ldapUID + "=" + filterTag + "))"
|
ldapFilter = "(&(" + ldapFilter + ")(" + ldapUID + "=" + filterTag + "))"
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("ldap filter :", ldapFilter)
|
log.Debug("ldap filter :", ldapFilter)
|
||||||
|
@ -404,6 +425,7 @@ func createGroupSearchFilter(oldFilter, groupName, groupNameAttribute string) st
|
||||||
filter := ""
|
filter := ""
|
||||||
groupName = goldap.EscapeFilter(groupName)
|
groupName = goldap.EscapeFilter(groupName)
|
||||||
groupNameAttribute = goldap.EscapeFilter(groupNameAttribute)
|
groupNameAttribute = goldap.EscapeFilter(groupNameAttribute)
|
||||||
|
oldFilter = normalizeFilter(oldFilter)
|
||||||
if len(oldFilter) == 0 {
|
if len(oldFilter) == 0 {
|
||||||
if len(groupName) == 0 {
|
if len(groupName) == 0 {
|
||||||
filter = groupNameAttribute + "=*"
|
filter = groupNameAttribute + "=*"
|
||||||
|
@ -419,3 +441,26 @@ func createGroupSearchFilter(oldFilter, groupName, groupNameAttribute string) st
|
||||||
}
|
}
|
||||||
return filter
|
return filter
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func createNestedGroupFilter(userDN string) string {
|
||||||
|
filter := ""
|
||||||
|
filter = "(&(objectClass=group)(member:1.2.840.113556.1.4.1941:=" + userDN + "))"
|
||||||
|
return filter
|
||||||
|
}
|
||||||
|
|
||||||
|
func contains(s []string, e string) bool {
|
||||||
|
for _, a := range s {
|
||||||
|
if a == e {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizeFilter - remove '(' and ')' in ldap filter
|
||||||
|
func normalizeFilter(filter string) string {
|
||||||
|
norFilter := strings.TrimSpace(filter)
|
||||||
|
norFilter = strings.TrimPrefix(norFilter, "(")
|
||||||
|
norFilter = strings.TrimSuffix(norFilter, ")")
|
||||||
|
return norFilter
|
||||||
|
}
|
||||||
|
|
|
@ -369,3 +369,25 @@ func TestSession_SearchGroupByDN(t *testing.T) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNormalizeFilter(t *testing.T) {
|
||||||
|
type args struct {
|
||||||
|
filter string
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
args args
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"normal test", args{"(objectclass=user)"}, "objectclass=user"},
|
||||||
|
{"with space", args{" (objectclass=user) "}, "objectclass=user"},
|
||||||
|
{"nothing", args{"objectclass=user"}, "objectclass=user"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := normalizeFilter(tt.args.filter); got != tt.want {
|
||||||
|
t.Errorf("normalizeFilter() = %v, want %v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -25,11 +25,9 @@ import (
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
// "time"
|
|
||||||
|
|
||||||
"github.com/docker/distribution/manifest/schema1"
|
"github.com/docker/distribution/manifest/schema1"
|
||||||
"github.com/docker/distribution/manifest/schema2"
|
"github.com/docker/distribution/manifest/schema2"
|
||||||
|
|
||||||
commonhttp "github.com/goharbor/harbor/src/common/http"
|
commonhttp "github.com/goharbor/harbor/src/common/http"
|
||||||
"github.com/goharbor/harbor/src/common/utils"
|
"github.com/goharbor/harbor/src/common/utils"
|
||||||
)
|
)
|
||||||
|
@ -407,6 +405,7 @@ func (r *Repository) monolithicBlobUpload(location, digest string, size int64, d
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
req.ContentLength = size
|
||||||
|
|
||||||
resp, err := r.client.Do(req)
|
resp, err := r.client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -15,22 +15,24 @@
|
||||||
package api
|
package api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"github.com/goharbor/harbor/src/pkg/retention"
|
|
||||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
|
||||||
|
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/ghodss/yaml"
|
"github.com/ghodss/yaml"
|
||||||
"github.com/goharbor/harbor/src/common/api"
|
"github.com/goharbor/harbor/src/common/api"
|
||||||
|
"github.com/goharbor/harbor/src/common/rbac"
|
||||||
"github.com/goharbor/harbor/src/common/security"
|
"github.com/goharbor/harbor/src/common/security"
|
||||||
|
"github.com/goharbor/harbor/src/common/utils"
|
||||||
"github.com/goharbor/harbor/src/common/utils/log"
|
"github.com/goharbor/harbor/src/common/utils/log"
|
||||||
"github.com/goharbor/harbor/src/core/config"
|
"github.com/goharbor/harbor/src/core/config"
|
||||||
"github.com/goharbor/harbor/src/core/filter"
|
"github.com/goharbor/harbor/src/core/filter"
|
||||||
"github.com/goharbor/harbor/src/core/promgr"
|
"github.com/goharbor/harbor/src/core/promgr"
|
||||||
"github.com/goharbor/harbor/src/pkg/project"
|
"github.com/goharbor/harbor/src/pkg/project"
|
||||||
"github.com/goharbor/harbor/src/pkg/repository"
|
"github.com/goharbor/harbor/src/pkg/repository"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/retention"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -47,6 +49,10 @@ var (
|
||||||
retentionController retention.APIController
|
retentionController retention.APIController
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
errNotFound = errors.New("not found")
|
||||||
|
)
|
||||||
|
|
||||||
// BaseController ...
|
// BaseController ...
|
||||||
type BaseController struct {
|
type BaseController struct {
|
||||||
api.BaseAPI
|
api.BaseAPI
|
||||||
|
@ -77,6 +83,71 @@ func (b *BaseController) Prepare() {
|
||||||
b.ProjectMgr = pm
|
b.ProjectMgr = pm
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RequireAuthenticated returns true when the request is authenticated
|
||||||
|
// otherwise send Unauthorized response and returns false
|
||||||
|
func (b *BaseController) RequireAuthenticated() bool {
|
||||||
|
if !b.SecurityCtx.IsAuthenticated() {
|
||||||
|
b.SendUnAuthorizedError(errors.New("Unauthorized"))
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// HasProjectPermission returns true when the request has action permission on project subresource
|
||||||
|
func (b *BaseController) HasProjectPermission(projectIDOrName interface{}, action rbac.Action, subresource ...rbac.Resource) (bool, error) {
|
||||||
|
projectID, projectName, err := utils.ParseProjectIDOrName(projectIDOrName)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if projectName != "" {
|
||||||
|
project, err := b.ProjectMgr.Get(projectName)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if project == nil {
|
||||||
|
return false, errNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
projectID = project.ProjectID
|
||||||
|
}
|
||||||
|
|
||||||
|
resource := rbac.NewProjectNamespace(projectID).Resource(subresource...)
|
||||||
|
if !b.SecurityCtx.Can(action, resource) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequireProjectAccess returns true when the request has action access on project subresource
|
||||||
|
// otherwise send UnAuthorized or Forbidden response and returns false
|
||||||
|
func (b *BaseController) RequireProjectAccess(projectIDOrName interface{}, action rbac.Action, subresource ...rbac.Resource) bool {
|
||||||
|
hasPermission, err := b.HasProjectPermission(projectIDOrName, action, subresource...)
|
||||||
|
if err != nil {
|
||||||
|
if err == errNotFound {
|
||||||
|
b.SendNotFoundError(fmt.Errorf("project %v not found", projectIDOrName))
|
||||||
|
} else {
|
||||||
|
b.SendInternalServerError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if !hasPermission {
|
||||||
|
if !b.SecurityCtx.IsAuthenticated() {
|
||||||
|
b.SendUnAuthorizedError(errors.New("UnAuthorized"))
|
||||||
|
} else {
|
||||||
|
b.SendForbiddenError(errors.New(b.SecurityCtx.GetUsername()))
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
// WriteJSONData writes the JSON data to the client.
|
// WriteJSONData writes the JSON data to the client.
|
||||||
func (b *BaseController) WriteJSONData(object interface{}) {
|
func (b *BaseController) WriteJSONData(object interface{}) {
|
||||||
b.Data["json"] = object
|
b.Data["json"] = object
|
||||||
|
@ -121,12 +192,16 @@ func Init() error {
|
||||||
retentionController = retention.NewAPIController(retentionMgr, projectMgr, repositoryMgr, retentionScheduler, retentionLauncher)
|
retentionController = retention.NewAPIController(retentionMgr, projectMgr, repositoryMgr, retentionScheduler, retentionLauncher)
|
||||||
|
|
||||||
callbackFun := func(p interface{}) error {
|
callbackFun := func(p interface{}) error {
|
||||||
r, ok := p.(retention.TriggerParam)
|
str, ok := p.(string)
|
||||||
if ok {
|
if !ok {
|
||||||
_, err := retentionController.TriggerRetentionExec(r.PolicyID, r.Trigger, false)
|
return fmt.Errorf("the type of param %v isn't string", p)
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
return errors.New("bad retention callback param")
|
param := &retention.TriggerParam{}
|
||||||
|
if err := json.Unmarshal([]byte(str), param); err != nil {
|
||||||
|
return fmt.Errorf("failed to unmarshal the param: %v", err)
|
||||||
|
}
|
||||||
|
_, err := retentionController.TriggerRetentionExec(param.PolicyID, param.Trigger, false)
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
err := scheduler.Register(retention.SchedulerCallback, callbackFun)
|
err := scheduler.Register(retention.SchedulerCallback, callbackFun)
|
||||||
|
|
||||||
|
|
|
@ -58,14 +58,7 @@ func (cla *ChartLabelAPI) Prepare() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cla *ChartLabelAPI) requireAccess(action rbac.Action) bool {
|
func (cla *ChartLabelAPI) requireAccess(action rbac.Action) bool {
|
||||||
resource := rbac.NewProjectNamespace(cla.project.ProjectID).Resource(rbac.ResourceHelmChartVersionLabel)
|
return cla.RequireProjectAccess(cla.project.ProjectID, action, rbac.ResourceHelmChartVersionLabel)
|
||||||
|
|
||||||
if !cla.SecurityCtx.Can(action, resource) {
|
|
||||||
cla.SendForbiddenError(errors.New(cla.SecurityCtx.GetUsername()))
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// MarkLabel handles the request of marking label to chart.
|
// MarkLabel handles the request of marking label to chart.
|
||||||
|
|
|
@ -105,19 +105,8 @@ func (cra *ChartRepositoryAPI) requireAccess(action rbac.Action, subresource ...
|
||||||
if len(subresource) == 0 {
|
if len(subresource) == 0 {
|
||||||
subresource = append(subresource, rbac.ResourceHelmChart)
|
subresource = append(subresource, rbac.ResourceHelmChart)
|
||||||
}
|
}
|
||||||
resource := rbac.NewProjectNamespace(cra.namespace).Resource(subresource...)
|
|
||||||
|
|
||||||
if !cra.SecurityCtx.Can(action, resource) {
|
return cra.RequireProjectAccess(cra.namespace, action, subresource...)
|
||||||
if !cra.SecurityCtx.IsAuthenticated() {
|
|
||||||
cra.SendUnAuthorizedError(errors.New("Unauthorized"))
|
|
||||||
} else {
|
|
||||||
cra.SendForbiddenError(errors.New(cra.SecurityCtx.GetUsername()))
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetHealthStatus handles GET /api/chartrepo/health
|
// GetHealthStatus handles GET /api/chartrepo/health
|
||||||
|
|
|
@ -35,6 +35,7 @@ import (
|
||||||
testutils "github.com/goharbor/harbor/src/common/utils/test"
|
testutils "github.com/goharbor/harbor/src/common/utils/test"
|
||||||
api_models "github.com/goharbor/harbor/src/core/api/models"
|
api_models "github.com/goharbor/harbor/src/core/api/models"
|
||||||
apimodels "github.com/goharbor/harbor/src/core/api/models"
|
apimodels "github.com/goharbor/harbor/src/core/api/models"
|
||||||
|
quota "github.com/goharbor/harbor/src/core/api/quota"
|
||||||
_ "github.com/goharbor/harbor/src/core/auth/db"
|
_ "github.com/goharbor/harbor/src/core/auth/db"
|
||||||
_ "github.com/goharbor/harbor/src/core/auth/ldap"
|
_ "github.com/goharbor/harbor/src/core/auth/ldap"
|
||||||
"github.com/goharbor/harbor/src/core/config"
|
"github.com/goharbor/harbor/src/core/config"
|
||||||
|
@ -202,11 +203,18 @@ func init() {
|
||||||
beego.Router("/api/quotas", quotaAPIType, "get:List")
|
beego.Router("/api/quotas", quotaAPIType, "get:List")
|
||||||
beego.Router("/api/quotas/:id([0-9]+)", quotaAPIType, "get:Get;put:Put")
|
beego.Router("/api/quotas/:id([0-9]+)", quotaAPIType, "get:Get;put:Put")
|
||||||
|
|
||||||
|
beego.Router("/api/internal/switchquota", &InternalAPI{}, "put:SwitchQuota")
|
||||||
|
beego.Router("/api/internal/syncquota", &InternalAPI{}, "post:SyncQuota")
|
||||||
|
|
||||||
// syncRegistry
|
// syncRegistry
|
||||||
if err := SyncRegistry(config.GlobalProjectMgr); err != nil {
|
if err := SyncRegistry(config.GlobalProjectMgr); err != nil {
|
||||||
log.Fatalf("failed to sync repositories from registry: %v", err)
|
log.Fatalf("failed to sync repositories from registry: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := quota.Sync(config.GlobalProjectMgr, false); err != nil {
|
||||||
|
log.Fatalf("failed to sync quota from backend: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
// Init user Info
|
// Init user Info
|
||||||
admin = &usrInfo{adminName, adminPwd}
|
admin = &usrInfo{adminName, adminPwd}
|
||||||
unknownUsr = &usrInfo{"unknown", "unknown"}
|
unknownUsr = &usrInfo{"unknown", "unknown"}
|
||||||
|
|
|
@ -34,8 +34,9 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
timeout = 60 * time.Second
|
timeout = 60 * time.Second
|
||||||
healthCheckerRegistry = map[string]health.Checker{}
|
// HealthCheckerRegistry ...
|
||||||
|
HealthCheckerRegistry = map[string]health.Checker{}
|
||||||
)
|
)
|
||||||
|
|
||||||
type overallHealthStatus struct {
|
type overallHealthStatus struct {
|
||||||
|
@ -67,11 +68,11 @@ type HealthAPI struct {
|
||||||
func (h *HealthAPI) CheckHealth() {
|
func (h *HealthAPI) CheckHealth() {
|
||||||
var isHealthy healthy = true
|
var isHealthy healthy = true
|
||||||
components := []*componentHealthStatus{}
|
components := []*componentHealthStatus{}
|
||||||
c := make(chan *componentHealthStatus, len(healthCheckerRegistry))
|
c := make(chan *componentHealthStatus, len(HealthCheckerRegistry))
|
||||||
for name, checker := range healthCheckerRegistry {
|
for name, checker := range HealthCheckerRegistry {
|
||||||
go check(name, checker, timeout, c)
|
go check(name, checker, timeout, c)
|
||||||
}
|
}
|
||||||
for i := 0; i < len(healthCheckerRegistry); i++ {
|
for i := 0; i < len(HealthCheckerRegistry); i++ {
|
||||||
componentStatus := <-c
|
componentStatus := <-c
|
||||||
if len(componentStatus.Error) != 0 {
|
if len(componentStatus.Error) != 0 {
|
||||||
isHealthy = false
|
isHealthy = false
|
||||||
|
@ -290,21 +291,21 @@ func redisHealthChecker() health.Checker {
|
||||||
}
|
}
|
||||||
|
|
||||||
func registerHealthCheckers() {
|
func registerHealthCheckers() {
|
||||||
healthCheckerRegistry["core"] = coreHealthChecker()
|
HealthCheckerRegistry["core"] = coreHealthChecker()
|
||||||
healthCheckerRegistry["portal"] = portalHealthChecker()
|
HealthCheckerRegistry["portal"] = portalHealthChecker()
|
||||||
healthCheckerRegistry["jobservice"] = jobserviceHealthChecker()
|
HealthCheckerRegistry["jobservice"] = jobserviceHealthChecker()
|
||||||
healthCheckerRegistry["registry"] = registryHealthChecker()
|
HealthCheckerRegistry["registry"] = registryHealthChecker()
|
||||||
healthCheckerRegistry["registryctl"] = registryCtlHealthChecker()
|
HealthCheckerRegistry["registryctl"] = registryCtlHealthChecker()
|
||||||
healthCheckerRegistry["database"] = databaseHealthChecker()
|
HealthCheckerRegistry["database"] = databaseHealthChecker()
|
||||||
healthCheckerRegistry["redis"] = redisHealthChecker()
|
HealthCheckerRegistry["redis"] = redisHealthChecker()
|
||||||
if config.WithChartMuseum() {
|
if config.WithChartMuseum() {
|
||||||
healthCheckerRegistry["chartmuseum"] = chartmuseumHealthChecker()
|
HealthCheckerRegistry["chartmuseum"] = chartmuseumHealthChecker()
|
||||||
}
|
}
|
||||||
if config.WithClair() {
|
if config.WithClair() {
|
||||||
healthCheckerRegistry["clair"] = clairHealthChecker()
|
HealthCheckerRegistry["clair"] = clairHealthChecker()
|
||||||
}
|
}
|
||||||
if config.WithNotary() {
|
if config.WithNotary() {
|
||||||
healthCheckerRegistry["notary"] = notaryHealthChecker()
|
HealthCheckerRegistry["notary"] = notaryHealthChecker()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -92,9 +92,9 @@ func fakeHealthChecker(healthy bool) health.Checker {
|
||||||
}
|
}
|
||||||
func TestCheckHealth(t *testing.T) {
|
func TestCheckHealth(t *testing.T) {
|
||||||
// component01: healthy, component02: healthy => status: healthy
|
// component01: healthy, component02: healthy => status: healthy
|
||||||
healthCheckerRegistry = map[string]health.Checker{}
|
HealthCheckerRegistry = map[string]health.Checker{}
|
||||||
healthCheckerRegistry["component01"] = fakeHealthChecker(true)
|
HealthCheckerRegistry["component01"] = fakeHealthChecker(true)
|
||||||
healthCheckerRegistry["component02"] = fakeHealthChecker(true)
|
HealthCheckerRegistry["component02"] = fakeHealthChecker(true)
|
||||||
status := map[string]interface{}{}
|
status := map[string]interface{}{}
|
||||||
err := handleAndParse(&testingRequest{
|
err := handleAndParse(&testingRequest{
|
||||||
method: http.MethodGet,
|
method: http.MethodGet,
|
||||||
|
@ -104,9 +104,9 @@ func TestCheckHealth(t *testing.T) {
|
||||||
assert.Equal(t, "healthy", status["status"].(string))
|
assert.Equal(t, "healthy", status["status"].(string))
|
||||||
|
|
||||||
// component01: healthy, component02: unhealthy => status: unhealthy
|
// component01: healthy, component02: unhealthy => status: unhealthy
|
||||||
healthCheckerRegistry = map[string]health.Checker{}
|
HealthCheckerRegistry = map[string]health.Checker{}
|
||||||
healthCheckerRegistry["component01"] = fakeHealthChecker(true)
|
HealthCheckerRegistry["component01"] = fakeHealthChecker(true)
|
||||||
healthCheckerRegistry["component02"] = fakeHealthChecker(false)
|
HealthCheckerRegistry["component02"] = fakeHealthChecker(false)
|
||||||
status = map[string]interface{}{}
|
status = map[string]interface{}{}
|
||||||
err = handleAndParse(&testingRequest{
|
err = handleAndParse(&testingRequest{
|
||||||
method: http.MethodGet,
|
method: http.MethodGet,
|
||||||
|
@ -128,7 +128,7 @@ func TestDatabaseHealthChecker(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisterHealthCheckers(t *testing.T) {
|
func TestRegisterHealthCheckers(t *testing.T) {
|
||||||
healthCheckerRegistry = map[string]health.Checker{}
|
HealthCheckerRegistry = map[string]health.Checker{}
|
||||||
registerHealthCheckers()
|
registerHealthCheckers()
|
||||||
assert.NotNil(t, healthCheckerRegistry["core"])
|
assert.NotNil(t, HealthCheckerRegistry["core"])
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,12 +15,21 @@
|
||||||
package api
|
package api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"fmt"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/common"
|
"github.com/goharbor/harbor/src/common"
|
||||||
"github.com/goharbor/harbor/src/common/dao"
|
"github.com/goharbor/harbor/src/common/dao"
|
||||||
"github.com/goharbor/harbor/src/common/models"
|
"github.com/goharbor/harbor/src/common/models"
|
||||||
|
common_quota "github.com/goharbor/harbor/src/common/quota"
|
||||||
"github.com/goharbor/harbor/src/common/utils/log"
|
"github.com/goharbor/harbor/src/common/utils/log"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/core/config"
|
||||||
|
"github.com/goharbor/harbor/src/jobservice/logger"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
quota "github.com/goharbor/harbor/src/core/api/quota"
|
||||||
|
|
||||||
|
comcfg "github.com/goharbor/harbor/src/common/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// InternalAPI handles request of harbor admin...
|
// InternalAPI handles request of harbor admin...
|
||||||
|
@ -69,3 +78,103 @@ func (ia *InternalAPI) RenameAdmin() {
|
||||||
log.Debugf("The super user has been renamed to: %s", newName)
|
log.Debugf("The super user has been renamed to: %s", newName)
|
||||||
ia.DestroySession()
|
ia.DestroySession()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// QuotaSwitcher ...
|
||||||
|
type QuotaSwitcher struct {
|
||||||
|
Enabled bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// SwitchQuota ...
|
||||||
|
func (ia *InternalAPI) SwitchQuota() {
|
||||||
|
var req QuotaSwitcher
|
||||||
|
if err := ia.DecodeJSONReq(&req); err != nil {
|
||||||
|
ia.SendBadRequestError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// quota per project from disable to enable, it needs to update the quota usage bases on the DB records.
|
||||||
|
if !config.QuotaPerProjectEnable() && req.Enabled {
|
||||||
|
if err := ia.ensureQuota(); err != nil {
|
||||||
|
ia.SendInternalServerError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
config.GetCfgManager().Set(common.QuotaPerProjectEnable, req.Enabled)
|
||||||
|
config.GetCfgManager().Save()
|
||||||
|
}()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ia *InternalAPI) ensureQuota() error {
|
||||||
|
projects, err := dao.GetProjects(nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, project := range projects {
|
||||||
|
pSize, err := dao.CountSizeOfProject(project.ProjectID)
|
||||||
|
if err != nil {
|
||||||
|
logger.Warningf("error happen on counting size of project:%d , error:%v, just skip it.", project.ProjectID, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
afQuery := &models.ArtifactQuery{
|
||||||
|
PID: project.ProjectID,
|
||||||
|
}
|
||||||
|
afs, err := dao.ListArtifacts(afQuery)
|
||||||
|
if err != nil {
|
||||||
|
logger.Warningf("error happen on counting number of project:%d , error:%v, just skip it.", project.ProjectID, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pCount := int64(len(afs))
|
||||||
|
|
||||||
|
// it needs to append the chart count
|
||||||
|
if config.WithChartMuseum() {
|
||||||
|
count, err := chartController.GetCountOfCharts([]string{project.Name})
|
||||||
|
if err != nil {
|
||||||
|
err = errors.Wrap(err, fmt.Sprintf("get chart count of project %d failed", project.ProjectID))
|
||||||
|
logger.Error(err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pCount = pCount + int64(count)
|
||||||
|
}
|
||||||
|
|
||||||
|
quotaMgr, err := common_quota.NewManager("project", strconv.FormatInt(project.ProjectID, 10))
|
||||||
|
if err != nil {
|
||||||
|
logger.Errorf("Error occurred when to new quota manager %v, just skip it.", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
used := common_quota.ResourceList{
|
||||||
|
common_quota.ResourceStorage: pSize,
|
||||||
|
common_quota.ResourceCount: pCount,
|
||||||
|
}
|
||||||
|
if err := quotaMgr.EnsureQuota(used); err != nil {
|
||||||
|
logger.Errorf("cannot ensure quota for the project: %d, err: %v, just skip it.", project.ProjectID, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SyncQuota ...
|
||||||
|
func (ia *InternalAPI) SyncQuota() {
|
||||||
|
cur := config.ReadOnly()
|
||||||
|
cfgMgr := comcfg.NewDBCfgManager()
|
||||||
|
if cur != true {
|
||||||
|
cfgMgr.Set(common.ReadOnly, true)
|
||||||
|
}
|
||||||
|
// For api call, to avoid the timeout, it should be asynchronous
|
||||||
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
if cur != true {
|
||||||
|
cfgMgr.Set(common.ReadOnly, false)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
log.Info("start to sync quota(API), the system will be set to ReadOnly and back it normal once it done.")
|
||||||
|
err := quota.Sync(ia.ProjectMgr, false)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("fail to sync quota(API), but with error: %v, please try to do it again.", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
log.Info("success to sync quota(API).")
|
||||||
|
}()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
89
src/core/api/internal_test.go
Normal file
89
src/core/api/internal_test.go
Normal file
|
@ -0,0 +1,89 @@
|
||||||
|
// Copyright 2018 Project Harbor Authors
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// cannot verify the real scenario here
|
||||||
|
func TestSwitchQuota(t *testing.T) {
|
||||||
|
cases := []*codeCheckingCase{
|
||||||
|
// 401
|
||||||
|
{
|
||||||
|
request: &testingRequest{
|
||||||
|
method: http.MethodPut,
|
||||||
|
url: "/api/internal/switchquota",
|
||||||
|
},
|
||||||
|
code: http.StatusUnauthorized,
|
||||||
|
},
|
||||||
|
// 200
|
||||||
|
{
|
||||||
|
request: &testingRequest{
|
||||||
|
method: http.MethodPut,
|
||||||
|
url: "/api/internal/switchquota",
|
||||||
|
credential: sysAdmin,
|
||||||
|
bodyJSON: &QuotaSwitcher{
|
||||||
|
Enabled: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
code: http.StatusOK,
|
||||||
|
},
|
||||||
|
// 403
|
||||||
|
{
|
||||||
|
request: &testingRequest{
|
||||||
|
url: "/api/internal/switchquota",
|
||||||
|
method: http.MethodPut,
|
||||||
|
credential: nonSysAdmin,
|
||||||
|
},
|
||||||
|
code: http.StatusForbidden,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
runCodeCheckingCases(t, cases...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// cannot verify the real scenario here
|
||||||
|
func TestSyncQuota(t *testing.T) {
|
||||||
|
cases := []*codeCheckingCase{
|
||||||
|
// 401
|
||||||
|
{
|
||||||
|
request: &testingRequest{
|
||||||
|
method: http.MethodPost,
|
||||||
|
url: "/api/internal/syncquota",
|
||||||
|
},
|
||||||
|
code: http.StatusUnauthorized,
|
||||||
|
},
|
||||||
|
// 200
|
||||||
|
{
|
||||||
|
request: &testingRequest{
|
||||||
|
method: http.MethodPost,
|
||||||
|
url: "/api/internal/syncquota",
|
||||||
|
credential: sysAdmin,
|
||||||
|
},
|
||||||
|
code: http.StatusOK,
|
||||||
|
},
|
||||||
|
// 403
|
||||||
|
{
|
||||||
|
request: &testingRequest{
|
||||||
|
url: "/api/internal/syncquota",
|
||||||
|
method: http.MethodPost,
|
||||||
|
credential: nonSysAdmin,
|
||||||
|
},
|
||||||
|
code: http.StatusForbidden,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
runCodeCheckingCases(t, cases...)
|
||||||
|
}
|
|
@ -78,8 +78,7 @@ func (l *LabelAPI) requireAccess(label *models.Label, action rbac.Action, subres
|
||||||
if len(subresources) == 0 {
|
if len(subresources) == 0 {
|
||||||
subresources = append(subresources, rbac.ResourceLabel)
|
subresources = append(subresources, rbac.ResourceLabel)
|
||||||
}
|
}
|
||||||
resource := rbac.NewProjectNamespace(label.ProjectID).Resource(subresources...)
|
hasPermission, _ = l.HasProjectPermission(label.ProjectID, action, subresources...)
|
||||||
hasPermission = l.SecurityCtx.Can(action, resource)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if !hasPermission {
|
if !hasPermission {
|
||||||
|
@ -203,13 +202,7 @@ func (l *LabelAPI) List() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
resource := rbac.NewProjectNamespace(projectID).Resource(rbac.ResourceLabel)
|
if !l.RequireProjectAccess(projectID, rbac.ActionList, rbac.ResourceLabel) {
|
||||||
if !l.SecurityCtx.Can(rbac.ActionList, resource) {
|
|
||||||
if !l.SecurityCtx.IsAuthenticated() {
|
|
||||||
l.SendUnAuthorizedError(errors.New("UnAuthorized"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
l.SendForbiddenError(errors.New(l.SecurityCtx.GetUsername()))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
query.ProjectID = projectID
|
query.ProjectID = projectID
|
||||||
|
|
|
@ -22,6 +22,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/common/models"
|
"github.com/goharbor/harbor/src/common/models"
|
||||||
"github.com/goharbor/harbor/src/common/rbac"
|
"github.com/goharbor/harbor/src/common/rbac"
|
||||||
"github.com/goharbor/harbor/src/common/utils/log"
|
"github.com/goharbor/harbor/src/common/utils/log"
|
||||||
|
@ -90,18 +91,7 @@ func (m *MetadataAPI) Prepare() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *MetadataAPI) requireAccess(action rbac.Action) bool {
|
func (m *MetadataAPI) requireAccess(action rbac.Action) bool {
|
||||||
resource := rbac.NewProjectNamespace(m.project.ProjectID).Resource(rbac.ResourceMetadata)
|
return m.RequireProjectAccess(m.project.ProjectID, action, rbac.ResourceMetadata)
|
||||||
|
|
||||||
if !m.SecurityCtx.Can(action, resource) {
|
|
||||||
if !m.SecurityCtx.IsAuthenticated() {
|
|
||||||
m.SendUnAuthorizedError(errors.New("Unauthorized"))
|
|
||||||
} else {
|
|
||||||
m.SendForbiddenError(errors.New(m.SecurityCtx.GetUsername()))
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get ...
|
// Get ...
|
||||||
|
|
|
@ -93,16 +93,5 @@ func (w *NotificationJobAPI) validateRBAC(action rbac.Action, projectID int64) b
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
project, err := w.ProjectMgr.Get(projectID)
|
return w.RequireProjectAccess(projectID, action, rbac.ResourceNotificationPolicy)
|
||||||
if err != nil {
|
|
||||||
w.ParseAndHandleError(fmt.Sprintf("failed to get project %d", projectID), err)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
resource := rbac.NewProjectNamespace(project.ProjectID).Resource(rbac.ResourceNotificationPolicy)
|
|
||||||
if !w.SecurityCtx.Can(action, resource) {
|
|
||||||
w.SendForbiddenError(errors.New(w.SecurityCtx.GetUsername()))
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -283,18 +283,7 @@ func (w *NotificationPolicyAPI) validateRBAC(action rbac.Action, projectID int64
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
project, err := w.ProjectMgr.Get(projectID)
|
return w.RequireProjectAccess(projectID, action, rbac.ResourceNotificationPolicy)
|
||||||
if err != nil {
|
|
||||||
w.ParseAndHandleError(fmt.Sprintf("failed to get project %d", projectID), err)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
resource := rbac.NewProjectNamespace(project.ProjectID).Resource(rbac.ResourceNotificationPolicy)
|
|
||||||
if !w.SecurityCtx.Can(action, resource) {
|
|
||||||
w.SendForbiddenError(errors.New(w.SecurityCtx.GetUsername()))
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *NotificationPolicyAPI) validateTargets(policy *models.NotificationPolicy) bool {
|
func (w *NotificationPolicyAPI) validateTargets(policy *models.NotificationPolicy) bool {
|
||||||
|
|
|
@ -86,20 +86,8 @@ func (p *ProjectAPI) requireAccess(action rbac.Action, subresource ...rbac.Resou
|
||||||
if len(subresource) == 0 {
|
if len(subresource) == 0 {
|
||||||
subresource = append(subresource, rbac.ResourceSelf)
|
subresource = append(subresource, rbac.ResourceSelf)
|
||||||
}
|
}
|
||||||
resource := rbac.NewProjectNamespace(p.project.ProjectID).Resource(subresource...)
|
|
||||||
|
|
||||||
if !p.SecurityCtx.Can(action, resource) {
|
return p.RequireProjectAccess(p.project.ProjectID, action, subresource...)
|
||||||
if !p.SecurityCtx.IsAuthenticated() {
|
|
||||||
p.SendUnAuthorizedError(errors.New("Unauthorized"))
|
|
||||||
|
|
||||||
} else {
|
|
||||||
p.SendForbiddenError(errors.New(p.SecurityCtx.GetUsername()))
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Post ...
|
// Post ...
|
||||||
|
@ -139,23 +127,26 @@ func (p *ProjectAPI) Post() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
setting, err := config.QuotaSetting()
|
var hardLimits types.ResourceList
|
||||||
if err != nil {
|
if config.QuotaPerProjectEnable() {
|
||||||
log.Errorf("failed to get quota setting: %v", err)
|
setting, err := config.QuotaSetting()
|
||||||
p.SendInternalServerError(fmt.Errorf("failed to get quota setting: %v", err))
|
if err != nil {
|
||||||
return
|
log.Errorf("failed to get quota setting: %v", err)
|
||||||
}
|
p.SendInternalServerError(fmt.Errorf("failed to get quota setting: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if !p.SecurityCtx.IsSysAdmin() {
|
if !p.SecurityCtx.IsSysAdmin() {
|
||||||
pro.CountLimit = &setting.CountPerProject
|
pro.CountLimit = &setting.CountPerProject
|
||||||
pro.StorageLimit = &setting.StoragePerProject
|
pro.StorageLimit = &setting.StoragePerProject
|
||||||
}
|
}
|
||||||
|
|
||||||
hardLimits, err := projectQuotaHardLimits(pro, setting)
|
hardLimits, err = projectQuotaHardLimits(pro, setting)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("Invalid project request, error: %v", err)
|
log.Errorf("Invalid project request, error: %v", err)
|
||||||
p.SendBadRequestError(fmt.Errorf("invalid request: %v", err))
|
p.SendBadRequestError(fmt.Errorf("invalid request: %v", err))
|
||||||
return
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
exist, err := p.ProjectMgr.Exists(pro.Name)
|
exist, err := p.ProjectMgr.Exists(pro.Name)
|
||||||
|
@ -212,14 +203,16 @@ func (p *ProjectAPI) Post() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
quotaMgr, err := quota.NewManager("project", strconv.FormatInt(projectID, 10))
|
if config.QuotaPerProjectEnable() {
|
||||||
if err != nil {
|
quotaMgr, err := quota.NewManager("project", strconv.FormatInt(projectID, 10))
|
||||||
p.SendInternalServerError(fmt.Errorf("failed to get quota manager: %v", err))
|
if err != nil {
|
||||||
return
|
p.SendInternalServerError(fmt.Errorf("failed to get quota manager: %v", err))
|
||||||
}
|
return
|
||||||
if _, err := quotaMgr.NewQuota(hardLimits); err != nil {
|
}
|
||||||
p.SendInternalServerError(fmt.Errorf("failed to create quota for project: %v", err))
|
if _, err := quotaMgr.NewQuota(hardLimits); err != nil {
|
||||||
return
|
p.SendInternalServerError(fmt.Errorf("failed to create quota for project: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
|
@ -653,6 +646,11 @@ func projectQuotaHardLimits(req *models.ProjectRequest, setting *models.QuotaSet
|
||||||
}
|
}
|
||||||
|
|
||||||
func getProjectQuotaSummary(projectID int64, summary *models.ProjectSummary) {
|
func getProjectQuotaSummary(projectID int64, summary *models.ProjectSummary) {
|
||||||
|
if !config.QuotaPerProjectEnable() {
|
||||||
|
log.Debug("Quota per project disabled")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
quotas, err := dao.ListQuotas(&models.QuotaQuery{Reference: "project", ReferenceID: strconv.FormatInt(projectID, 10)})
|
quotas, err := dao.ListQuotas(&models.QuotaQuery{Reference: "project", ReferenceID: strconv.FormatInt(projectID, 10)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debugf("failed to get quota for project: %d", projectID)
|
log.Debugf("failed to get quota for project: %d", projectID)
|
||||||
|
|
|
@ -172,7 +172,7 @@ func TestListProjects(t *testing.T) {
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// ----------------------------case 1 : Response Code=200----------------------------//
|
// ----------------------------case 1 : Response Code=200----------------------------//
|
||||||
fmt.Println("case 1: respose code:200")
|
fmt.Println("case 1: response code:200")
|
||||||
httpStatusCode, result, err := apiTest.ProjectsGet(
|
httpStatusCode, result, err := apiTest.ProjectsGet(
|
||||||
&apilib.ProjectQuery{
|
&apilib.ProjectQuery{
|
||||||
Name: addProject.ProjectName,
|
Name: addProject.ProjectName,
|
||||||
|
@ -263,7 +263,7 @@ func TestProGetByID(t *testing.T) {
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// ----------------------------case 1 : Response Code=200----------------------------//
|
// ----------------------------case 1 : Response Code=200----------------------------//
|
||||||
fmt.Println("case 1: respose code:200")
|
fmt.Println("case 1: response code:200")
|
||||||
httpStatusCode, result, err := apiTest.ProjectsGetByPID(projectID)
|
httpStatusCode, result, err := apiTest.ProjectsGetByPID(projectID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error("Error while search project by proID", err.Error())
|
t.Error("Error while search project by proID", err.Error())
|
||||||
|
@ -295,7 +295,7 @@ func TestDeleteProject(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// --------------------------case 2: Response Code=200---------------------------------//
|
// --------------------------case 2: Response Code=200---------------------------------//
|
||||||
fmt.Println("case2: respose code:200")
|
fmt.Println("case2: response code:200")
|
||||||
httpStatusCode, err = apiTest.ProjectsDelete(*admin, projectID)
|
httpStatusCode, err = apiTest.ProjectsDelete(*admin, projectID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error("Error while delete project", err.Error())
|
t.Error("Error while delete project", err.Error())
|
||||||
|
@ -335,7 +335,7 @@ func TestProHead(t *testing.T) {
|
||||||
apiTest := newHarborAPI()
|
apiTest := newHarborAPI()
|
||||||
|
|
||||||
// ----------------------------case 1 : Response Code=200----------------------------//
|
// ----------------------------case 1 : Response Code=200----------------------------//
|
||||||
fmt.Println("case 1: respose code:200")
|
fmt.Println("case 1: response code:200")
|
||||||
httpStatusCode, err := apiTest.ProjectsHead(*admin, "library")
|
httpStatusCode, err := apiTest.ProjectsHead(*admin, "library")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error("Error while search project by proName", err.Error())
|
t.Error("Error while search project by proName", err.Error())
|
||||||
|
@ -345,7 +345,7 @@ func TestProHead(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// ----------------------------case 2 : Response Code=404:Project name does not exist.----------------------------//
|
// ----------------------------case 2 : Response Code=404:Project name does not exist.----------------------------//
|
||||||
fmt.Println("case 2: respose code:404,Project name does not exist.")
|
fmt.Println("case 2: response code:404,Project name does not exist.")
|
||||||
httpStatusCode, err = apiTest.ProjectsHead(*admin, "libra")
|
httpStatusCode, err = apiTest.ProjectsHead(*admin, "libra")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error("Error while search project by proName", err.Error())
|
t.Error("Error while search project by proName", err.Error())
|
||||||
|
@ -369,22 +369,22 @@ func TestPut(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println("case 1: respose code:200")
|
fmt.Println("case 1: response code:200")
|
||||||
code, err := apiTest.ProjectsPut(*admin, "1", project)
|
code, err := apiTest.ProjectsPut(*admin, "1", project)
|
||||||
require.Nil(t, err)
|
require.Nil(t, err)
|
||||||
assert.Equal(int(200), code)
|
assert.Equal(int(200), code)
|
||||||
|
|
||||||
fmt.Println("case 2: respose code:401, User need to log in first.")
|
fmt.Println("case 2: response code:401, User need to log in first.")
|
||||||
code, err = apiTest.ProjectsPut(*unknownUsr, "1", project)
|
code, err = apiTest.ProjectsPut(*unknownUsr, "1", project)
|
||||||
require.Nil(t, err)
|
require.Nil(t, err)
|
||||||
assert.Equal(int(401), code)
|
assert.Equal(int(401), code)
|
||||||
|
|
||||||
fmt.Println("case 3: respose code:400, Invalid project id")
|
fmt.Println("case 3: response code:400, Invalid project id")
|
||||||
code, err = apiTest.ProjectsPut(*admin, "cc", project)
|
code, err = apiTest.ProjectsPut(*admin, "cc", project)
|
||||||
require.Nil(t, err)
|
require.Nil(t, err)
|
||||||
assert.Equal(int(400), code)
|
assert.Equal(int(400), code)
|
||||||
|
|
||||||
fmt.Println("case 4: respose code:404, Not found the project")
|
fmt.Println("case 4: response code:404, Not found the project")
|
||||||
code, err = apiTest.ProjectsPut(*admin, "1234", project)
|
code, err = apiTest.ProjectsPut(*admin, "1234", project)
|
||||||
require.Nil(t, err)
|
require.Nil(t, err)
|
||||||
assert.Equal(int(404), code)
|
assert.Equal(int(404), code)
|
||||||
|
@ -407,7 +407,7 @@ func TestProjectLogsFilter(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// -------------------case1: Response Code=200------------------------------//
|
// -------------------case1: Response Code=200------------------------------//
|
||||||
fmt.Println("case 1: respose code:200")
|
fmt.Println("case 1: response code:200")
|
||||||
projectID := "1"
|
projectID := "1"
|
||||||
httpStatusCode, _, err := apiTest.ProjectLogs(*admin, projectID, query)
|
httpStatusCode, _, err := apiTest.ProjectLogs(*admin, projectID, query)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -417,7 +417,7 @@ func TestProjectLogsFilter(t *testing.T) {
|
||||||
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
|
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
|
||||||
}
|
}
|
||||||
// -------------------case2: Response Code=401:User need to log in first.------------------------------//
|
// -------------------case2: Response Code=401:User need to log in first.------------------------------//
|
||||||
fmt.Println("case 2: respose code:401:User need to log in first.")
|
fmt.Println("case 2: response code:401:User need to log in first.")
|
||||||
projectID = "1"
|
projectID = "1"
|
||||||
httpStatusCode, _, err = apiTest.ProjectLogs(*unknownUsr, projectID, query)
|
httpStatusCode, _, err = apiTest.ProjectLogs(*unknownUsr, projectID, query)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -427,7 +427,7 @@ func TestProjectLogsFilter(t *testing.T) {
|
||||||
assert.Equal(int(401), httpStatusCode, "httpStatusCode should be 401")
|
assert.Equal(int(401), httpStatusCode, "httpStatusCode should be 401")
|
||||||
}
|
}
|
||||||
// -------------------case3: Response Code=404:Project does not exist.-------------------------//
|
// -------------------case3: Response Code=404:Project does not exist.-------------------------//
|
||||||
fmt.Println("case 3: respose code:404:Illegal format of provided ID value.")
|
fmt.Println("case 3: response code:404:Illegal format of provided ID value.")
|
||||||
projectID = "11111"
|
projectID = "11111"
|
||||||
httpStatusCode, _, err = apiTest.ProjectLogs(*admin, projectID, query)
|
httpStatusCode, _, err = apiTest.ProjectLogs(*admin, projectID, query)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -498,7 +498,7 @@ func TestProjectSummary(t *testing.T) {
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// ----------------------------case 1 : Response Code=200----------------------------//
|
// ----------------------------case 1 : Response Code=200----------------------------//
|
||||||
fmt.Println("case 1: respose code:200")
|
fmt.Println("case 1: response code:200")
|
||||||
httpStatusCode, summary, err := apiTest.ProjectSummary(*admin, fmt.Sprintf("%d", projectID))
|
httpStatusCode, summary, err := apiTest.ProjectSummary(*admin, fmt.Sprintf("%d", projectID))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error("Error while search project by proName", err.Error())
|
t.Error("Error while search project by proName", err.Error())
|
||||||
|
|
|
@ -99,19 +99,7 @@ func (pma *ProjectMemberAPI) Prepare() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (pma *ProjectMemberAPI) requireAccess(action rbac.Action) bool {
|
func (pma *ProjectMemberAPI) requireAccess(action rbac.Action) bool {
|
||||||
resource := rbac.NewProjectNamespace(pma.project.ProjectID).Resource(rbac.ResourceMember)
|
return pma.RequireProjectAccess(pma.project.ProjectID, action, rbac.ResourceMember)
|
||||||
|
|
||||||
if !pma.SecurityCtx.Can(action, resource) {
|
|
||||||
if !pma.SecurityCtx.IsAuthenticated() {
|
|
||||||
pma.SendUnAuthorizedError(errors.New("Unauthorized"))
|
|
||||||
} else {
|
|
||||||
pma.SendForbiddenError(errors.New(pma.SecurityCtx.GetUsername()))
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get ...
|
// Get ...
|
||||||
|
|
226
src/core/api/quota/chart/chart.go
Normal file
226
src/core/api/quota/chart/chart.go
Normal file
|
@ -0,0 +1,226 @@
|
||||||
|
// Copyright 2018 Project Harbor Authors
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package chart
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"github.com/goharbor/harbor/src/chartserver"
|
||||||
|
"github.com/goharbor/harbor/src/common/dao"
|
||||||
|
"github.com/goharbor/harbor/src/common/models"
|
||||||
|
common_quota "github.com/goharbor/harbor/src/common/quota"
|
||||||
|
"github.com/goharbor/harbor/src/common/utils/log"
|
||||||
|
"github.com/goharbor/harbor/src/core/api"
|
||||||
|
quota "github.com/goharbor/harbor/src/core/api/quota"
|
||||||
|
"github.com/goharbor/harbor/src/core/config"
|
||||||
|
"github.com/goharbor/harbor/src/core/promgr"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Migrator ...
|
||||||
|
type Migrator struct {
|
||||||
|
pm promgr.ProjectManager
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewChartMigrator returns a new RegistryMigrator.
|
||||||
|
func NewChartMigrator(pm promgr.ProjectManager) quota.QuotaMigrator {
|
||||||
|
migrator := Migrator{
|
||||||
|
pm: pm,
|
||||||
|
}
|
||||||
|
return &migrator
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
controller *chartserver.Controller
|
||||||
|
controllerErr error
|
||||||
|
controllerOnce sync.Once
|
||||||
|
)
|
||||||
|
|
||||||
|
// Ping ...
|
||||||
|
func (rm *Migrator) Ping() error {
|
||||||
|
return api.HealthCheckerRegistry["chartmuseum"].Check()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dump ...
|
||||||
|
// Depends on DB to dump chart data, as chart cannot get all of namespaces.
|
||||||
|
func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
|
||||||
|
var (
|
||||||
|
projects []quota.ProjectInfo
|
||||||
|
wg sync.WaitGroup
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
|
||||||
|
all, err := dao.GetProjects(nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
wg.Add(len(all))
|
||||||
|
errChan := make(chan error, 1)
|
||||||
|
infoChan := make(chan interface{})
|
||||||
|
done := make(chan bool, 1)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
done <- true
|
||||||
|
}()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case result := <-infoChan:
|
||||||
|
if result == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
project, ok := result.(quota.ProjectInfo)
|
||||||
|
if ok {
|
||||||
|
projects = append(projects, project)
|
||||||
|
}
|
||||||
|
|
||||||
|
case e := <-errChan:
|
||||||
|
if err == nil {
|
||||||
|
err = errors.Wrap(e, "quota sync error on getting info of project")
|
||||||
|
} else {
|
||||||
|
err = errors.Wrap(e, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
for _, project := range all {
|
||||||
|
go func(project *models.Project) {
|
||||||
|
defer wg.Done()
|
||||||
|
|
||||||
|
var repos []quota.RepoData
|
||||||
|
ctr, err := chartController()
|
||||||
|
if err != nil {
|
||||||
|
errChan <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
chartInfo, err := ctr.ListCharts(project.Name)
|
||||||
|
if err != nil {
|
||||||
|
errChan <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// repo
|
||||||
|
for _, chart := range chartInfo {
|
||||||
|
var afs []*models.Artifact
|
||||||
|
chartVersions, err := ctr.GetChart(project.Name, chart.Name)
|
||||||
|
if err != nil {
|
||||||
|
errChan <- err
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, chart := range chartVersions {
|
||||||
|
af := &models.Artifact{
|
||||||
|
PID: project.ProjectID,
|
||||||
|
Repo: chart.Name,
|
||||||
|
Tag: chart.Version,
|
||||||
|
Digest: chart.Digest,
|
||||||
|
Kind: "Chart",
|
||||||
|
}
|
||||||
|
afs = append(afs, af)
|
||||||
|
}
|
||||||
|
repoData := quota.RepoData{
|
||||||
|
Name: project.Name,
|
||||||
|
Afs: afs,
|
||||||
|
}
|
||||||
|
repos = append(repos, repoData)
|
||||||
|
}
|
||||||
|
|
||||||
|
projectInfo := quota.ProjectInfo{
|
||||||
|
Name: project.Name,
|
||||||
|
Repos: repos,
|
||||||
|
}
|
||||||
|
|
||||||
|
infoChan <- projectInfo
|
||||||
|
}(project)
|
||||||
|
}
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
close(infoChan)
|
||||||
|
|
||||||
|
<-done
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return projects, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Usage ...
|
||||||
|
// Chart will not cover size.
|
||||||
|
func (rm *Migrator) Usage(projects []quota.ProjectInfo) ([]quota.ProjectUsage, error) {
|
||||||
|
var pros []quota.ProjectUsage
|
||||||
|
for _, project := range projects {
|
||||||
|
var count int64
|
||||||
|
// usage count
|
||||||
|
for _, repo := range project.Repos {
|
||||||
|
count = count + int64(len(repo.Afs))
|
||||||
|
}
|
||||||
|
proUsage := quota.ProjectUsage{
|
||||||
|
Project: project.Name,
|
||||||
|
Used: common_quota.ResourceList{
|
||||||
|
common_quota.ResourceCount: count,
|
||||||
|
common_quota.ResourceStorage: 0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
pros = append(pros, proUsage)
|
||||||
|
}
|
||||||
|
return pros, nil
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// Persist ...
|
||||||
|
// Chart will not persist data into db.
|
||||||
|
func (rm *Migrator) Persist(projects []quota.ProjectInfo) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func chartController() (*chartserver.Controller, error) {
|
||||||
|
controllerOnce.Do(func() {
|
||||||
|
addr, err := config.GetChartMuseumEndpoint()
|
||||||
|
if err != nil {
|
||||||
|
controllerErr = fmt.Errorf("failed to get the endpoint URL of chart storage server: %s", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
addr = strings.TrimSuffix(addr, "/")
|
||||||
|
url, err := url.Parse(addr)
|
||||||
|
if err != nil {
|
||||||
|
controllerErr = errors.New("endpoint URL of chart storage server is malformed")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ctr, err := chartserver.NewController(url)
|
||||||
|
if err != nil {
|
||||||
|
controllerErr = errors.New("failed to initialize chart API controller")
|
||||||
|
}
|
||||||
|
|
||||||
|
controller = ctr
|
||||||
|
|
||||||
|
log.Debugf("Chart storage server is set to %s", url.String())
|
||||||
|
log.Info("API controller for chart repository server is successfully initialized")
|
||||||
|
})
|
||||||
|
|
||||||
|
return controller, controllerErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
quota.Register("chart", NewChartMigrator)
|
||||||
|
}
|
173
src/core/api/quota/migrator.go
Normal file
173
src/core/api/quota/migrator.go
Normal file
|
@ -0,0 +1,173 @@
|
||||||
|
// Copyright 2018 Project Harbor Authors
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/goharbor/harbor/src/common/dao"
|
||||||
|
"github.com/goharbor/harbor/src/common/models"
|
||||||
|
"github.com/goharbor/harbor/src/common/quota"
|
||||||
|
"github.com/goharbor/harbor/src/common/utils/log"
|
||||||
|
"github.com/goharbor/harbor/src/core/config"
|
||||||
|
"github.com/goharbor/harbor/src/core/promgr"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/types"
|
||||||
|
"strconv"
|
||||||
|
)
|
||||||
|
|
||||||
|
// QuotaMigrator ...
|
||||||
|
type QuotaMigrator interface {
|
||||||
|
// Ping validates and wait for backend service ready.
|
||||||
|
Ping() error
|
||||||
|
|
||||||
|
// Dump exports all data from backend service, registry, chartmuseum
|
||||||
|
Dump() ([]ProjectInfo, error)
|
||||||
|
|
||||||
|
// Usage computes the quota usage of all the projects
|
||||||
|
Usage([]ProjectInfo) ([]ProjectUsage, error)
|
||||||
|
|
||||||
|
// Persist record the data to DB, artifact, artifact_blob and blob tabel.
|
||||||
|
Persist([]ProjectInfo) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProjectInfo ...
|
||||||
|
type ProjectInfo struct {
|
||||||
|
Name string
|
||||||
|
Repos []RepoData
|
||||||
|
}
|
||||||
|
|
||||||
|
// RepoData ...
|
||||||
|
type RepoData struct {
|
||||||
|
Name string
|
||||||
|
Afs []*models.Artifact
|
||||||
|
Afnbs []*models.ArtifactAndBlob
|
||||||
|
Blobs []*models.Blob
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProjectUsage ...
|
||||||
|
type ProjectUsage struct {
|
||||||
|
Project string
|
||||||
|
Used quota.ResourceList
|
||||||
|
}
|
||||||
|
|
||||||
|
// Instance ...
|
||||||
|
type Instance func(promgr.ProjectManager) QuotaMigrator
|
||||||
|
|
||||||
|
var adapters = make(map[string]Instance)
|
||||||
|
|
||||||
|
// Register ...
|
||||||
|
func Register(name string, adapter Instance) {
|
||||||
|
if adapter == nil {
|
||||||
|
panic("quota: Register adapter is nil")
|
||||||
|
}
|
||||||
|
if _, ok := adapters[name]; ok {
|
||||||
|
panic("quota: Register called twice for adapter " + name)
|
||||||
|
}
|
||||||
|
adapters[name] = adapter
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sync ...
|
||||||
|
func Sync(pm promgr.ProjectManager, populate bool) error {
|
||||||
|
totalUsage := make(map[string][]ProjectUsage)
|
||||||
|
for name, instanceFunc := range adapters {
|
||||||
|
if !config.WithChartMuseum() {
|
||||||
|
if name == "chart" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
adapter := instanceFunc(pm)
|
||||||
|
if err := adapter.Ping(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
data, err := adapter.Dump()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
usage, err := adapter.Usage(data)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
totalUsage[name] = usage
|
||||||
|
if populate {
|
||||||
|
if err := adapter.Persist(data); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
merged := mergeUsage(totalUsage)
|
||||||
|
if err := ensureQuota(merged); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeUsage merges the usage of adapters
|
||||||
|
func mergeUsage(total map[string][]ProjectUsage) []ProjectUsage {
|
||||||
|
if !config.WithChartMuseum() {
|
||||||
|
return total["registry"]
|
||||||
|
}
|
||||||
|
regUsgs := total["registry"]
|
||||||
|
chartUsgs := total["chart"]
|
||||||
|
|
||||||
|
var mergedUsage []ProjectUsage
|
||||||
|
temp := make(map[string]quota.ResourceList)
|
||||||
|
|
||||||
|
for _, regUsg := range regUsgs {
|
||||||
|
_, exist := temp[regUsg.Project]
|
||||||
|
if !exist {
|
||||||
|
temp[regUsg.Project] = regUsg.Used
|
||||||
|
mergedUsage = append(mergedUsage, ProjectUsage{
|
||||||
|
Project: regUsg.Project,
|
||||||
|
Used: regUsg.Used,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, chartUsg := range chartUsgs {
|
||||||
|
var usedTemp quota.ResourceList
|
||||||
|
_, exist := temp[chartUsg.Project]
|
||||||
|
if !exist {
|
||||||
|
usedTemp = chartUsg.Used
|
||||||
|
} else {
|
||||||
|
usedTemp = types.Add(temp[chartUsg.Project], chartUsg.Used)
|
||||||
|
}
|
||||||
|
temp[chartUsg.Project] = usedTemp
|
||||||
|
mergedUsage = append(mergedUsage, ProjectUsage{
|
||||||
|
Project: chartUsg.Project,
|
||||||
|
Used: usedTemp,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return mergedUsage
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensureQuota updates the quota and quota usage in the data base.
|
||||||
|
func ensureQuota(usages []ProjectUsage) error {
|
||||||
|
var pid int64
|
||||||
|
for _, usage := range usages {
|
||||||
|
project, err := dao.GetProjectByName(usage.Project)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
pid = project.ProjectID
|
||||||
|
quotaMgr, err := quota.NewManager("project", strconv.FormatInt(pid, 10))
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Error occurred when to new quota manager %v", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := quotaMgr.EnsureQuota(usage.Used); err != nil {
|
||||||
|
log.Errorf("cannot ensure quota for the project: %d, err: %v", pid, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
436
src/core/api/quota/registry/registry.go
Normal file
436
src/core/api/quota/registry/registry.go
Normal file
|
@ -0,0 +1,436 @@
|
||||||
|
// Copyright 2018 Project Harbor Authors
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package registry
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/docker/distribution/manifest/schema1"
|
||||||
|
"github.com/docker/distribution/manifest/schema2"
|
||||||
|
|
||||||
|
"github.com/goharbor/harbor/src/common/dao"
|
||||||
|
"github.com/goharbor/harbor/src/common/models"
|
||||||
|
common_quota "github.com/goharbor/harbor/src/common/quota"
|
||||||
|
"github.com/goharbor/harbor/src/common/utils/log"
|
||||||
|
"github.com/goharbor/harbor/src/common/utils/registry"
|
||||||
|
"github.com/goharbor/harbor/src/core/api"
|
||||||
|
quota "github.com/goharbor/harbor/src/core/api/quota"
|
||||||
|
"github.com/goharbor/harbor/src/core/promgr"
|
||||||
|
coreutils "github.com/goharbor/harbor/src/core/utils"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Migrator ...
|
||||||
|
type Migrator struct {
|
||||||
|
pm promgr.ProjectManager
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRegistryMigrator returns a new Migrator.
|
||||||
|
func NewRegistryMigrator(pm promgr.ProjectManager) quota.QuotaMigrator {
|
||||||
|
migrator := Migrator{
|
||||||
|
pm: pm,
|
||||||
|
}
|
||||||
|
return &migrator
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ping ...
|
||||||
|
func (rm *Migrator) Ping() error {
|
||||||
|
return api.HealthCheckerRegistry["registry"].Check()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dump ...
|
||||||
|
func (rm *Migrator) Dump() ([]quota.ProjectInfo, error) {
|
||||||
|
var (
|
||||||
|
projects []quota.ProjectInfo
|
||||||
|
wg sync.WaitGroup
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
|
||||||
|
reposInRegistry, err := api.Catalog()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// repoMap : map[project_name : []repo list]
|
||||||
|
repoMap := make(map[string][]string)
|
||||||
|
for _, item := range reposInRegistry {
|
||||||
|
projectName := strings.Split(item, "/")[0]
|
||||||
|
pro, err := rm.pm.Get(projectName)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("failed to get project %s: %v", projectName, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if pro == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_, exist := repoMap[pro.Name]
|
||||||
|
if !exist {
|
||||||
|
repoMap[pro.Name] = []string{item}
|
||||||
|
} else {
|
||||||
|
repos := repoMap[pro.Name]
|
||||||
|
repos = append(repos, item)
|
||||||
|
repoMap[pro.Name] = repos
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
wg.Add(len(repoMap))
|
||||||
|
errChan := make(chan error, 1)
|
||||||
|
infoChan := make(chan interface{})
|
||||||
|
done := make(chan bool, 1)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
done <- true
|
||||||
|
}()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case result := <-infoChan:
|
||||||
|
if result == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
project, ok := result.(quota.ProjectInfo)
|
||||||
|
if ok {
|
||||||
|
projects = append(projects, project)
|
||||||
|
}
|
||||||
|
|
||||||
|
case e := <-errChan:
|
||||||
|
if err == nil {
|
||||||
|
err = errors.Wrap(e, "quota sync error on getting info of project")
|
||||||
|
} else {
|
||||||
|
err = errors.Wrap(e, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
for project, repos := range repoMap {
|
||||||
|
go func(project string, repos []string) {
|
||||||
|
defer wg.Done()
|
||||||
|
info, err := infoOfProject(project, repos)
|
||||||
|
if err != nil {
|
||||||
|
errChan <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
infoChan <- info
|
||||||
|
}(project, repos)
|
||||||
|
}
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
close(infoChan)
|
||||||
|
|
||||||
|
// wait for all of project info
|
||||||
|
<-done
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return projects, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Usage ...
|
||||||
|
// registry needs to merge the shard blobs of different repositories.
|
||||||
|
func (rm *Migrator) Usage(projects []quota.ProjectInfo) ([]quota.ProjectUsage, error) {
|
||||||
|
var pros []quota.ProjectUsage
|
||||||
|
|
||||||
|
for _, project := range projects {
|
||||||
|
var size, count int64
|
||||||
|
var blobs = make(map[string]int64)
|
||||||
|
|
||||||
|
// usage count
|
||||||
|
for _, repo := range project.Repos {
|
||||||
|
count = count + int64(len(repo.Afs))
|
||||||
|
// Because that there are some shared blobs between repositories, it needs to remove the duplicate items.
|
||||||
|
for _, blob := range repo.Blobs {
|
||||||
|
_, exist := blobs[blob.Digest]
|
||||||
|
if !exist {
|
||||||
|
blobs[blob.Digest] = blob.Size
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// size
|
||||||
|
for _, item := range blobs {
|
||||||
|
size = size + item
|
||||||
|
}
|
||||||
|
|
||||||
|
proUsage := quota.ProjectUsage{
|
||||||
|
Project: project.Name,
|
||||||
|
Used: common_quota.ResourceList{
|
||||||
|
common_quota.ResourceCount: count,
|
||||||
|
common_quota.ResourceStorage: size,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
pros = append(pros, proUsage)
|
||||||
|
}
|
||||||
|
|
||||||
|
return pros, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Persist ...
|
||||||
|
func (rm *Migrator) Persist(projects []quota.ProjectInfo) error {
|
||||||
|
for _, project := range projects {
|
||||||
|
for _, repo := range project.Repos {
|
||||||
|
if err := persistAf(repo.Afs); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := persistAfnbs(repo.Afnbs); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := persistBlob(repo.Blobs); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := persistPB(projects); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func persistAf(afs []*models.Artifact) error {
|
||||||
|
if len(afs) != 0 {
|
||||||
|
for _, af := range afs {
|
||||||
|
_, err := dao.AddArtifact(af)
|
||||||
|
if err != nil {
|
||||||
|
if err == dao.ErrDupRows {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.Error(err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func persistAfnbs(afnbs []*models.ArtifactAndBlob) error {
|
||||||
|
if len(afnbs) != 0 {
|
||||||
|
for _, afnb := range afnbs {
|
||||||
|
_, err := dao.AddArtifactNBlob(afnb)
|
||||||
|
if err != nil {
|
||||||
|
if err == dao.ErrDupRows {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.Error(err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func persistBlob(blobs []*models.Blob) error {
|
||||||
|
if len(blobs) != 0 {
|
||||||
|
for _, blob := range blobs {
|
||||||
|
_, err := dao.AddBlob(blob)
|
||||||
|
if err != nil {
|
||||||
|
if err == dao.ErrDupRows {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.Error(err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func persistPB(projects []quota.ProjectInfo) error {
|
||||||
|
for _, project := range projects {
|
||||||
|
var blobs = make(map[string]int64)
|
||||||
|
var blobsOfPro []*models.Blob
|
||||||
|
for _, repo := range project.Repos {
|
||||||
|
for _, blob := range repo.Blobs {
|
||||||
|
_, exist := blobs[blob.Digest]
|
||||||
|
if exist {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
blobs[blob.Digest] = blob.Size
|
||||||
|
blobInDB, err := dao.GetBlob(blob.Digest)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if blobInDB != nil {
|
||||||
|
blobsOfPro = append(blobsOfPro, blobInDB)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pro, err := dao.GetProjectByName(project.Name)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = dao.AddBlobsToProject(pro.ProjectID, blobsOfPro...)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func infoOfProject(project string, repoList []string) (quota.ProjectInfo, error) {
|
||||||
|
var (
|
||||||
|
repos []quota.RepoData
|
||||||
|
wg sync.WaitGroup
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
wg.Add(len(repoList))
|
||||||
|
|
||||||
|
errChan := make(chan error, 1)
|
||||||
|
infoChan := make(chan interface{})
|
||||||
|
done := make(chan bool, 1)
|
||||||
|
|
||||||
|
pro, err := dao.GetProjectByName(project)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err)
|
||||||
|
return quota.ProjectInfo{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
done <- true
|
||||||
|
}()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case result := <-infoChan:
|
||||||
|
if result == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
repoData, ok := result.(quota.RepoData)
|
||||||
|
if ok {
|
||||||
|
repos = append(repos, repoData)
|
||||||
|
}
|
||||||
|
|
||||||
|
case e := <-errChan:
|
||||||
|
if err == nil {
|
||||||
|
err = errors.Wrap(e, "quota sync error on getting info of repo")
|
||||||
|
} else {
|
||||||
|
err = errors.Wrap(e, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
for _, repo := range repoList {
|
||||||
|
go func(pid int64, repo string) {
|
||||||
|
defer func() {
|
||||||
|
wg.Done()
|
||||||
|
}()
|
||||||
|
info, err := infoOfRepo(pid, repo)
|
||||||
|
if err != nil {
|
||||||
|
errChan <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
infoChan <- info
|
||||||
|
}(pro.ProjectID, repo)
|
||||||
|
}
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
close(infoChan)
|
||||||
|
|
||||||
|
<-done
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return quota.ProjectInfo{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return quota.ProjectInfo{
|
||||||
|
Name: project,
|
||||||
|
Repos: repos,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func infoOfRepo(pid int64, repo string) (quota.RepoData, error) {
|
||||||
|
repoClient, err := coreutils.NewRepositoryClientForUI("harbor-core", repo)
|
||||||
|
if err != nil {
|
||||||
|
return quota.RepoData{}, err
|
||||||
|
}
|
||||||
|
tags, err := repoClient.ListTag()
|
||||||
|
if err != nil {
|
||||||
|
return quota.RepoData{}, err
|
||||||
|
}
|
||||||
|
var afnbs []*models.ArtifactAndBlob
|
||||||
|
var afs []*models.Artifact
|
||||||
|
var blobs []*models.Blob
|
||||||
|
|
||||||
|
for _, tag := range tags {
|
||||||
|
_, mediaType, payload, err := repoClient.PullManifest(tag, []string{
|
||||||
|
schema1.MediaTypeManifest,
|
||||||
|
schema1.MediaTypeSignedManifest,
|
||||||
|
schema2.MediaTypeManifest,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err)
|
||||||
|
return quota.RepoData{}, err
|
||||||
|
}
|
||||||
|
manifest, desc, err := registry.UnMarshal(mediaType, payload)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err)
|
||||||
|
return quota.RepoData{}, err
|
||||||
|
}
|
||||||
|
// self
|
||||||
|
afnb := &models.ArtifactAndBlob{
|
||||||
|
DigestAF: desc.Digest.String(),
|
||||||
|
DigestBlob: desc.Digest.String(),
|
||||||
|
}
|
||||||
|
afnbs = append(afnbs, afnb)
|
||||||
|
// add manifest as a blob.
|
||||||
|
blob := &models.Blob{
|
||||||
|
Digest: desc.Digest.String(),
|
||||||
|
ContentType: desc.MediaType,
|
||||||
|
Size: desc.Size,
|
||||||
|
CreationTime: time.Now(),
|
||||||
|
}
|
||||||
|
blobs = append(blobs, blob)
|
||||||
|
for _, layer := range manifest.References() {
|
||||||
|
afnb := &models.ArtifactAndBlob{
|
||||||
|
DigestAF: desc.Digest.String(),
|
||||||
|
DigestBlob: layer.Digest.String(),
|
||||||
|
}
|
||||||
|
afnbs = append(afnbs, afnb)
|
||||||
|
blob := &models.Blob{
|
||||||
|
Digest: layer.Digest.String(),
|
||||||
|
ContentType: layer.MediaType,
|
||||||
|
Size: layer.Size,
|
||||||
|
CreationTime: time.Now(),
|
||||||
|
}
|
||||||
|
blobs = append(blobs, blob)
|
||||||
|
}
|
||||||
|
af := &models.Artifact{
|
||||||
|
PID: pid,
|
||||||
|
Repo: strings.Split(repo, "/")[1],
|
||||||
|
Tag: tag,
|
||||||
|
Digest: desc.Digest.String(),
|
||||||
|
Kind: "Docker-Image",
|
||||||
|
CreationTime: time.Now(),
|
||||||
|
}
|
||||||
|
afs = append(afs, af)
|
||||||
|
}
|
||||||
|
return quota.RepoData{
|
||||||
|
Name: repo,
|
||||||
|
Afs: afs,
|
||||||
|
Afnbs: afnbs,
|
||||||
|
Blobs: blobs,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
quota.Register("registry", NewRegistryMigrator)
|
||||||
|
}
|
|
@ -111,13 +111,7 @@ func (ra *RepositoryAPI) Get() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
resource := rbac.NewProjectNamespace(projectID).Resource(rbac.ResourceRepository)
|
if !ra.RequireProjectAccess(projectID, rbac.ActionList, rbac.ResourceRepository) {
|
||||||
if !ra.SecurityCtx.Can(rbac.ActionList, resource) {
|
|
||||||
if !ra.SecurityCtx.IsAuthenticated() {
|
|
||||||
ra.SendUnAuthorizedError(errors.New("Unauthorized"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername()))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -228,14 +222,8 @@ func (ra *RepositoryAPI) Delete() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if !ra.SecurityCtx.IsAuthenticated() {
|
if !ra.RequireAuthenticated() ||
|
||||||
ra.SendUnAuthorizedError(errors.New("UnAuthorized"))
|
!ra.RequireProjectAccess(project.ProjectID, rbac.ActionDelete, rbac.ResourceRepository) {
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
resource := rbac.NewProjectNamespace(project.ProjectID).Resource(rbac.ResourceRepository)
|
|
||||||
if !ra.SecurityCtx.Can(rbac.ActionDelete, resource) {
|
|
||||||
ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername()))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -403,14 +391,9 @@ func (ra *RepositoryAPI) GetTag() {
|
||||||
ra.SendNotFoundError(fmt.Errorf("resource: %s:%s not found", repository, tag))
|
ra.SendNotFoundError(fmt.Errorf("resource: %s:%s not found", repository, tag))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
project, _ := utils.ParseRepository(repository)
|
|
||||||
resource := rbac.NewProjectNamespace(project).Resource(rbac.ResourceRepositoryTag)
|
projectName, _ := utils.ParseRepository(repository)
|
||||||
if !ra.SecurityCtx.Can(rbac.ActionRead, resource) {
|
if !ra.RequireProjectAccess(projectName, rbac.ActionRead, rbac.ResourceRepositoryTag) {
|
||||||
if !ra.SecurityCtx.IsAuthenticated() {
|
|
||||||
ra.SendUnAuthorizedError(errors.New("UnAuthorized"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername()))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -503,16 +486,14 @@ func (ra *RepositoryAPI) Retag() {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check whether user has read permission to source project
|
// Check whether user has read permission to source project
|
||||||
srcResource := rbac.NewProjectNamespace(srcImage.Project).Resource(rbac.ResourceRepository)
|
if hasPermission, _ := ra.HasProjectPermission(srcImage.Project, rbac.ActionPull, rbac.ResourceRepository); !hasPermission {
|
||||||
if !ra.SecurityCtx.Can(rbac.ActionPull, srcResource) {
|
|
||||||
log.Errorf("user has no read permission to project '%s'", srcImage.Project)
|
log.Errorf("user has no read permission to project '%s'", srcImage.Project)
|
||||||
ra.SendForbiddenError(fmt.Errorf("%s has no read permission to project %s", ra.SecurityCtx.GetUsername(), srcImage.Project))
|
ra.SendForbiddenError(fmt.Errorf("%s has no read permission to project %s", ra.SecurityCtx.GetUsername(), srcImage.Project))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check whether user has write permission to target project
|
// Check whether user has write permission to target project
|
||||||
destResource := rbac.NewProjectNamespace(project).Resource(rbac.ResourceRepository)
|
if hasPermission, _ := ra.HasProjectPermission(project, rbac.ActionPush, rbac.ResourceRepository); !hasPermission {
|
||||||
if !ra.SecurityCtx.Can(rbac.ActionPush, destResource) {
|
|
||||||
log.Errorf("user has no write permission to project '%s'", project)
|
log.Errorf("user has no write permission to project '%s'", project)
|
||||||
ra.SendForbiddenError(fmt.Errorf("%s has no write permission to project %s", ra.SecurityCtx.GetUsername(), project))
|
ra.SendForbiddenError(fmt.Errorf("%s has no write permission to project %s", ra.SecurityCtx.GetUsername(), project))
|
||||||
return
|
return
|
||||||
|
@ -550,13 +531,7 @@ func (ra *RepositoryAPI) GetTags() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
resource := rbac.NewProjectNamespace(projectName).Resource(rbac.ResourceRepositoryTag)
|
if !ra.RequireProjectAccess(projectName, rbac.ActionList, rbac.ResourceRepositoryTag) {
|
||||||
if !ra.SecurityCtx.Can(rbac.ActionList, resource) {
|
|
||||||
if !ra.SecurityCtx.IsAuthenticated() {
|
|
||||||
ra.SendUnAuthorizedError(errors.New("UnAuthorized"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername()))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -585,7 +560,12 @@ func (ra *RepositoryAPI) GetTags() {
|
||||||
}
|
}
|
||||||
labeledTags := map[string]struct{}{}
|
labeledTags := map[string]struct{}{}
|
||||||
for _, rl := range rls {
|
for _, rl := range rls {
|
||||||
labeledTags[strings.Split(rl.ResourceName, ":")[1]] = struct{}{}
|
strs := strings.SplitN(rl.ResourceName, ":", 2)
|
||||||
|
// the "rls" may contain images which don't belong to the repository
|
||||||
|
if strs[0] != repoName {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
labeledTags[strs[1]] = struct{}{}
|
||||||
}
|
}
|
||||||
ts := []string{}
|
ts := []string{}
|
||||||
for _, tag := range tags {
|
for _, tag := range tags {
|
||||||
|
@ -596,11 +576,31 @@ func (ra *RepositoryAPI) GetTags() {
|
||||||
tags = ts
|
tags = ts
|
||||||
}
|
}
|
||||||
|
|
||||||
|
detail, err := ra.GetBool("detail", true)
|
||||||
|
if !detail && err == nil {
|
||||||
|
ra.Data["json"] = simpleTags(tags)
|
||||||
|
ra.ServeJSON()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
ra.Data["json"] = assembleTagsInParallel(client, repoName, tags,
|
ra.Data["json"] = assembleTagsInParallel(client, repoName, tags,
|
||||||
ra.SecurityCtx.GetUsername())
|
ra.SecurityCtx.GetUsername())
|
||||||
ra.ServeJSON()
|
ra.ServeJSON()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func simpleTags(tags []string) []*models.TagResp {
|
||||||
|
var tagsResp []*models.TagResp
|
||||||
|
for _, tag := range tags {
|
||||||
|
tagsResp = append(tagsResp, &models.TagResp{
|
||||||
|
TagDetail: models.TagDetail{
|
||||||
|
Name: tag,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return tagsResp
|
||||||
|
}
|
||||||
|
|
||||||
// get config, signature and scan overview and assemble them into one
|
// get config, signature and scan overview and assemble them into one
|
||||||
// struct for each tag in tags
|
// struct for each tag in tags
|
||||||
func assembleTagsInParallel(client *registry.Repository, repository string,
|
func assembleTagsInParallel(client *registry.Repository, repository string,
|
||||||
|
@ -791,14 +791,7 @@ func (ra *RepositoryAPI) GetManifests() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
resource := rbac.NewProjectNamespace(projectName).Resource(rbac.ResourceRepositoryTagManifest)
|
if !ra.RequireProjectAccess(projectName, rbac.ActionRead, rbac.ResourceRepositoryTagManifest) {
|
||||||
if !ra.SecurityCtx.Can(rbac.ActionRead, resource) {
|
|
||||||
if !ra.SecurityCtx.IsAuthenticated() {
|
|
||||||
ra.SendUnAuthorizedError(errors.New("Unauthorized"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername()))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -919,10 +912,8 @@ func (ra *RepositoryAPI) Put() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
project, _ := utils.ParseRepository(name)
|
projectName, _ := utils.ParseRepository(name)
|
||||||
resource := rbac.NewProjectNamespace(project).Resource(rbac.ResourceRepository)
|
if !ra.RequireProjectAccess(projectName, rbac.ActionUpdate, rbac.ResourceRepository) {
|
||||||
if !ra.SecurityCtx.Can(rbac.ActionUpdate, resource) {
|
|
||||||
ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername()))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -958,13 +949,7 @@ func (ra *RepositoryAPI) GetSignatures() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
resource := rbac.NewProjectNamespace(projectName).Resource(rbac.ResourceRepository)
|
if !ra.RequireProjectAccess(projectName, rbac.ActionRead, rbac.ResourceRepository) {
|
||||||
if !ra.SecurityCtx.Can(rbac.ActionRead, resource) {
|
|
||||||
if !ra.SecurityCtx.IsAuthenticated() {
|
|
||||||
ra.SendUnAuthorizedError(errors.New("Unauthorized"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername()))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1004,9 +989,7 @@ func (ra *RepositoryAPI) ScanImage() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
resource := rbac.NewProjectNamespace(projectName).Resource(rbac.ResourceRepositoryTagScanJob)
|
if !ra.RequireProjectAccess(projectName, rbac.ActionCreate, rbac.ResourceRepositoryTagScanJob) {
|
||||||
if !ra.SecurityCtx.Can(rbac.ActionCreate, resource) {
|
|
||||||
ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername()))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
err = coreutils.TriggerImageScan(repoName, tag)
|
err = coreutils.TriggerImageScan(repoName, tag)
|
||||||
|
@ -1035,15 +1018,9 @@ func (ra *RepositoryAPI) VulnerabilityDetails() {
|
||||||
ra.SendNotFoundError(fmt.Errorf("resource: %s:%s not found", repository, tag))
|
ra.SendNotFoundError(fmt.Errorf("resource: %s:%s not found", repository, tag))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
project, _ := utils.ParseRepository(repository)
|
|
||||||
|
|
||||||
resource := rbac.NewProjectNamespace(project).Resource(rbac.ResourceRepositoryTagVulnerability)
|
projectName, _ := utils.ParseRepository(repository)
|
||||||
if !ra.SecurityCtx.Can(rbac.ActionList, resource) {
|
if !ra.RequireProjectAccess(projectName, rbac.ActionList, rbac.ResourceRepositoryTagVulnerability) {
|
||||||
if !ra.SecurityCtx.IsAuthenticated() {
|
|
||||||
ra.SendUnAuthorizedError(errors.New("Unauthorized"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername()))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
res, err := scan.VulnListByDigest(digest)
|
res, err := scan.VulnListByDigest(digest)
|
||||||
|
|
|
@ -91,19 +91,8 @@ func (r *RepositoryLabelAPI) requireAccess(action rbac.Action, subresource ...rb
|
||||||
if len(subresource) == 0 {
|
if len(subresource) == 0 {
|
||||||
subresource = append(subresource, rbac.ResourceRepositoryLabel)
|
subresource = append(subresource, rbac.ResourceRepositoryLabel)
|
||||||
}
|
}
|
||||||
resource := rbac.NewProjectNamespace(r.repository.ProjectID).Resource(rbac.ResourceRepositoryLabel)
|
|
||||||
|
|
||||||
if !r.SecurityCtx.Can(action, resource) {
|
return r.RequireProjectAccess(r.repository.ProjectID, action, subresource...)
|
||||||
if !r.SecurityCtx.IsAuthenticated() {
|
|
||||||
r.SendUnAuthorizedError(errors.New("UnAuthorized"))
|
|
||||||
} else {
|
|
||||||
r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername()))
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *RepositoryLabelAPI) isValidLabelReq() bool {
|
func (r *RepositoryLabelAPI) isValidLabelReq() bool {
|
||||||
|
|
|
@ -67,25 +67,19 @@ func (r *RetentionAPI) GetMetadatas() {
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"rule_template": "nothing",
|
"rule_template": "nDaysSinceLastPush",
|
||||||
"display_text": "none",
|
"display_text": "pushed within the last # days",
|
||||||
"action": "retain",
|
"action": "retain",
|
||||||
"params": []
|
"params": [
|
||||||
},
|
{
|
||||||
{
|
"type": "int",
|
||||||
"rule_template": "always",
|
"unit": "DAYS",
|
||||||
"display_text": "always",
|
"required": true
|
||||||
"action": "retain",
|
}
|
||||||
"params": [
|
]
|
||||||
{
|
},
|
||||||
"type": "int",
|
|
||||||
"unit": "COUNT",
|
|
||||||
"required": true
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"rule_template": "dayspl",
|
"rule_template": "nDaysSinceLastPull",
|
||||||
"display_text": "pulled within the last # days",
|
"display_text": "pulled within the last # days",
|
||||||
"action": "retain",
|
"action": "retain",
|
||||||
"params": [
|
"params": [
|
||||||
|
@ -97,17 +91,11 @@ func (r *RetentionAPI) GetMetadatas() {
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"rule_template": "daysps",
|
"rule_template": "always",
|
||||||
"display_text": "pushed within the last # days",
|
"display_text": "always",
|
||||||
"action": "retain",
|
"action": "retain",
|
||||||
"params": [
|
"params": []
|
||||||
{
|
}
|
||||||
"type": "int",
|
|
||||||
"unit": "DAYS",
|
|
||||||
"required": true
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
],
|
||||||
"scope_selectors": [
|
"scope_selectors": [
|
||||||
{
|
{
|
||||||
|
@ -120,14 +108,6 @@ func (r *RetentionAPI) GetMetadatas() {
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"tag_selectors": [
|
"tag_selectors": [
|
||||||
{
|
|
||||||
"display_text": "Labels",
|
|
||||||
"kind": "label",
|
|
||||||
"decorations": [
|
|
||||||
"withLabels",
|
|
||||||
"withoutLabels"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"display_text": "Tags",
|
"display_text": "Tags",
|
||||||
"kind": "doublestar",
|
"kind": "doublestar",
|
||||||
|
@ -244,7 +224,7 @@ func (r *RetentionAPI) checkRuleConflict(p *policy.Metadata) error {
|
||||||
if old, exists := temp[string(bs)]; exists {
|
if old, exists := temp[string(bs)]; exists {
|
||||||
return fmt.Errorf("rule %d is conflict with rule %d", n, old)
|
return fmt.Errorf("rule %d is conflict with rule %d", n, old)
|
||||||
}
|
}
|
||||||
temp[string(bs)] = tid
|
temp[string(bs)] = n
|
||||||
rule.ID = tid
|
rule.ID = tid
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
@ -424,8 +404,7 @@ func (r *RetentionAPI) requireAccess(p *policy.Metadata, action rbac.Action, sub
|
||||||
if len(subresources) == 0 {
|
if len(subresources) == 0 {
|
||||||
subresources = append(subresources, rbac.ResourceTagRetention)
|
subresources = append(subresources, rbac.ResourceTagRetention)
|
||||||
}
|
}
|
||||||
resource := rbac.NewProjectNamespace(p.Scope.Reference).Resource(subresources...)
|
hasPermission, _ = r.HasProjectPermission(p.Scope.Reference, action, subresources...)
|
||||||
hasPermission = r.SecurityCtx.Can(action, resource)
|
|
||||||
default:
|
default:
|
||||||
hasPermission = r.SecurityCtx.IsSysAdmin()
|
hasPermission = r.SecurityCtx.IsSysAdmin()
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,16 +17,16 @@ package api
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/common"
|
"github.com/goharbor/harbor/src/common"
|
||||||
"github.com/goharbor/harbor/src/common/dao"
|
"github.com/goharbor/harbor/src/common/dao"
|
||||||
"github.com/goharbor/harbor/src/common/models"
|
"github.com/goharbor/harbor/src/common/models"
|
||||||
"github.com/goharbor/harbor/src/common/rbac"
|
"github.com/goharbor/harbor/src/common/rbac"
|
||||||
"github.com/goharbor/harbor/src/common/token"
|
"github.com/goharbor/harbor/src/common/token"
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/core/config"
|
"github.com/goharbor/harbor/src/core/config"
|
||||||
"time"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// RobotAPI ...
|
// RobotAPI ...
|
||||||
|
@ -91,13 +91,7 @@ func (r *RobotAPI) Prepare() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *RobotAPI) requireAccess(action rbac.Action) bool {
|
func (r *RobotAPI) requireAccess(action rbac.Action) bool {
|
||||||
resource := rbac.NewProjectNamespace(r.project.ProjectID).Resource(rbac.ResourceRobot)
|
return r.RequireProjectAccess(r.project.ProjectID, action, rbac.ResourceRobot)
|
||||||
if !r.SecurityCtx.Can(action, resource) {
|
|
||||||
r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername()))
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Post ...
|
// Post ...
|
||||||
|
|
|
@ -55,12 +55,10 @@ func (sj *ScanJobAPI) Prepare() {
|
||||||
sj.SendInternalServerError(errors.New("Failed to get Job data"))
|
sj.SendInternalServerError(errors.New("Failed to get Job data"))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
projectName := strings.SplitN(data.Repository, "/", 2)[0]
|
|
||||||
|
|
||||||
resource := rbac.NewProjectNamespace(projectName).Resource(rbac.ResourceRepositoryTagScanJob)
|
projectName := strings.SplitN(data.Repository, "/", 2)[0]
|
||||||
if !sj.SecurityCtx.Can(rbac.ActionRead, resource) {
|
if !sj.RequireProjectAccess(projectName, rbac.ActionRead, rbac.ResourceRepositoryTagScanJob) {
|
||||||
log.Errorf("User does not have read permission for project: %s", projectName)
|
log.Errorf("User does not have read permission for project: %s", projectName)
|
||||||
sj.SendForbiddenError(errors.New(sj.SecurityCtx.GetUsername()))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
sj.projectName = projectName
|
sj.projectName = projectName
|
||||||
|
|
|
@ -612,7 +612,7 @@ func TestUsersCurrentPermissions(t *testing.T) {
|
||||||
assert := assert.New(t)
|
assert := assert.New(t)
|
||||||
apiTest := newHarborAPI()
|
apiTest := newHarborAPI()
|
||||||
|
|
||||||
httpStatusCode, permissions, err := apiTest.UsersGetPermissions("current", "/project/library", *projAdmin)
|
httpStatusCode, permissions, err := apiTest.UsersGetPermissions("current", "/project/1", *projAdmin)
|
||||||
assert.Nil(err)
|
assert.Nil(err)
|
||||||
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
|
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
|
||||||
assert.NotEmpty(permissions, "permissions should not be empty")
|
assert.NotEmpty(permissions, "permissions should not be empty")
|
||||||
|
@ -622,11 +622,11 @@ func TestUsersCurrentPermissions(t *testing.T) {
|
||||||
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
|
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
|
||||||
assert.Empty(permissions, "permissions should be empty")
|
assert.Empty(permissions, "permissions should be empty")
|
||||||
|
|
||||||
httpStatusCode, _, err = apiTest.UsersGetPermissions(projAdminID, "/project/library", *projAdmin)
|
httpStatusCode, _, err = apiTest.UsersGetPermissions(projAdminID, "/project/1", *projAdmin)
|
||||||
assert.Nil(err)
|
assert.Nil(err)
|
||||||
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
|
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
|
||||||
|
|
||||||
httpStatusCode, _, err = apiTest.UsersGetPermissions(projDeveloperID, "/project/library", *projAdmin)
|
httpStatusCode, _, err = apiTest.UsersGetPermissions(projDeveloperID, "/project/1", *projAdmin)
|
||||||
assert.Nil(err)
|
assert.Nil(err)
|
||||||
assert.Equal(int(403), httpStatusCode, "httpStatusCode should be 403")
|
assert.Equal(int(403), httpStatusCode, "httpStatusCode should be 403")
|
||||||
}
|
}
|
||||||
|
|
|
@ -38,7 +38,7 @@ func SyncRegistry(pm promgr.ProjectManager) error {
|
||||||
|
|
||||||
log.Infof("Start syncing repositories from registry to DB... ")
|
log.Infof("Start syncing repositories from registry to DB... ")
|
||||||
|
|
||||||
reposInRegistry, err := catalog()
|
reposInRegistry, err := Catalog()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(err)
|
log.Error(err)
|
||||||
return err
|
return err
|
||||||
|
@ -105,7 +105,8 @@ func SyncRegistry(pm promgr.ProjectManager) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func catalog() ([]string, error) {
|
// Catalog ...
|
||||||
|
func Catalog() ([]string, error) {
|
||||||
repositories := []string{}
|
repositories := []string{}
|
||||||
|
|
||||||
rc, err := initRegistryClient()
|
rc, err := initRegistryClient()
|
||||||
|
|
|
@ -211,8 +211,6 @@ func (a *Auth) fillInModel(u *models.User) error {
|
||||||
u.Comment = userEntryComment
|
u.Comment = userEntryComment
|
||||||
if strings.Contains(u.Username, "@") {
|
if strings.Contains(u.Username, "@") {
|
||||||
u.Email = u.Username
|
u.Email = u.Username
|
||||||
} else {
|
|
||||||
u.Email = fmt.Sprintf("%s@placeholder.com", u.Username)
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -154,7 +154,7 @@ func TestAuth_PostAuthenticate(t *testing.T) {
|
||||||
},
|
},
|
||||||
expect: models.User{
|
expect: models.User{
|
||||||
Username: "jt",
|
Username: "jt",
|
||||||
Email: "jt@placeholder.com",
|
Email: "",
|
||||||
Realname: "jt",
|
Realname: "jt",
|
||||||
Password: pwd,
|
Password: pwd,
|
||||||
Comment: userEntryComment,
|
Comment: userEntryComment,
|
||||||
|
|
|
@ -124,8 +124,6 @@ func (l *Auth) OnBoardUser(u *models.User) error {
|
||||||
if u.Email == "" {
|
if u.Email == "" {
|
||||||
if strings.Contains(u.Username, "@") {
|
if strings.Contains(u.Username, "@") {
|
||||||
u.Email = u.Username
|
u.Email = u.Username
|
||||||
} else {
|
|
||||||
u.Email = u.Username + "@placeholder.com"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
u.Password = "12345678AbC" // Password is not kept in local db
|
u.Password = "12345678AbC" // Password is not kept in local db
|
||||||
|
|
|
@ -224,7 +224,7 @@ func TestOnBoardUser_02(t *testing.T) {
|
||||||
t.Errorf("Failed to onboard user")
|
t.Errorf("Failed to onboard user")
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.Equal(t, "sample02@placeholder.com", user.Email)
|
assert.Equal(t, "", user.Email)
|
||||||
dao.CleanUser(int64(user.UserID))
|
dao.CleanUser(int64(user.UserID))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -77,9 +77,8 @@ func fillEmailRealName(user *models.User) {
|
||||||
if len(user.Realname) == 0 {
|
if len(user.Realname) == 0 {
|
||||||
user.Realname = user.Username
|
user.Realname = user.Username
|
||||||
}
|
}
|
||||||
if len(user.Email) == 0 {
|
if len(user.Email) == 0 && strings.Contains(user.Username, "@") {
|
||||||
// TODO: handle the case when user.Username itself is an email address.
|
user.Email = user.Username
|
||||||
user.Email = user.Username + "@uaa.placeholder"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -110,7 +110,7 @@ func TestOnBoardUser(t *testing.T) {
|
||||||
user, _ := dao.GetUser(models.User{Username: "test"})
|
user, _ := dao.GetUser(models.User{Username: "test"})
|
||||||
assert.Equal("test", user.Realname)
|
assert.Equal("test", user.Realname)
|
||||||
assert.Equal("test", user.Username)
|
assert.Equal("test", user.Username)
|
||||||
assert.Equal("test@uaa.placeholder", user.Email)
|
assert.Equal("", user.Email)
|
||||||
err3 := dao.ClearTable(models.UserTable)
|
err3 := dao.ClearTable(models.UserTable)
|
||||||
assert.Nil(err3)
|
assert.Nil(err3)
|
||||||
}
|
}
|
||||||
|
@ -128,7 +128,7 @@ func TestPostAuthenticate(t *testing.T) {
|
||||||
}
|
}
|
||||||
assert.Nil(err)
|
assert.Nil(err)
|
||||||
user, _ := dao.GetUser(models.User{Username: "test"})
|
user, _ := dao.GetUser(models.User{Username: "test"})
|
||||||
assert.Equal("test@uaa.placeholder", user.Email)
|
assert.Equal("", user.Email)
|
||||||
um2.Email = "newEmail@new.com"
|
um2.Email = "newEmail@new.com"
|
||||||
um2.Realname = "newName"
|
um2.Realname = "newName"
|
||||||
err2 := auth.PostAuthenticate(um2)
|
err2 := auth.PostAuthenticate(um2)
|
||||||
|
@ -145,7 +145,7 @@ func TestPostAuthenticate(t *testing.T) {
|
||||||
assert.Nil(err3)
|
assert.Nil(err3)
|
||||||
user3, _ := dao.GetUser(models.User{Username: "test"})
|
user3, _ := dao.GetUser(models.User{Username: "test"})
|
||||||
assert.Equal(user3.UserID, um3.UserID)
|
assert.Equal(user3.UserID, um3.UserID)
|
||||||
assert.Equal("test@uaa.placeholder", user3.Email)
|
assert.Equal("", user3.Email)
|
||||||
assert.Equal("test", user3.Realname)
|
assert.Equal("test", user3.Realname)
|
||||||
err4 := dao.ClearTable(models.UserTable)
|
err4 := dao.ClearTable(models.UserTable)
|
||||||
assert.Nil(err4)
|
assert.Nil(err4)
|
||||||
|
|
|
@ -331,12 +331,14 @@ func Database() (*models.Database, error) {
|
||||||
database := &models.Database{}
|
database := &models.Database{}
|
||||||
database.Type = cfgMgr.Get(common.DatabaseType).GetString()
|
database.Type = cfgMgr.Get(common.DatabaseType).GetString()
|
||||||
postgresql := &models.PostGreSQL{
|
postgresql := &models.PostGreSQL{
|
||||||
Host: cfgMgr.Get(common.PostGreSQLHOST).GetString(),
|
Host: cfgMgr.Get(common.PostGreSQLHOST).GetString(),
|
||||||
Port: cfgMgr.Get(common.PostGreSQLPort).GetInt(),
|
Port: cfgMgr.Get(common.PostGreSQLPort).GetInt(),
|
||||||
Username: cfgMgr.Get(common.PostGreSQLUsername).GetString(),
|
Username: cfgMgr.Get(common.PostGreSQLUsername).GetString(),
|
||||||
Password: cfgMgr.Get(common.PostGreSQLPassword).GetString(),
|
Password: cfgMgr.Get(common.PostGreSQLPassword).GetString(),
|
||||||
Database: cfgMgr.Get(common.PostGreSQLDatabase).GetString(),
|
Database: cfgMgr.Get(common.PostGreSQLDatabase).GetString(),
|
||||||
SSLMode: cfgMgr.Get(common.PostGreSQLSSLMode).GetString(),
|
SSLMode: cfgMgr.Get(common.PostGreSQLSSLMode).GetString(),
|
||||||
|
MaxIdleConns: cfgMgr.Get(common.PostGreSQLMaxIdleConns).GetInt(),
|
||||||
|
MaxOpenConns: cfgMgr.Get(common.PostGreSQLMaxOpenConns).GetInt(),
|
||||||
}
|
}
|
||||||
database.PostGreSQL = postgresql
|
database.PostGreSQL = postgresql
|
||||||
|
|
||||||
|
@ -520,6 +522,11 @@ func NotificationEnable() bool {
|
||||||
return cfgMgr.Get(common.NotificationEnable).GetBool()
|
return cfgMgr.Get(common.NotificationEnable).GetBool()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// QuotaPerProjectEnable returns a bool to indicates if quota per project enabled in harbor
|
||||||
|
func QuotaPerProjectEnable() bool {
|
||||||
|
return cfgMgr.Get(common.QuotaPerProjectEnable).GetBool()
|
||||||
|
}
|
||||||
|
|
||||||
// QuotaSetting returns the setting of quota.
|
// QuotaSetting returns the setting of quota.
|
||||||
func QuotaSetting() (*models.QuotaSetting, error) {
|
func QuotaSetting() (*models.QuotaSetting, error) {
|
||||||
if err := cfgMgr.Load(); err != nil {
|
if err := cfgMgr.Load(); err != nil {
|
||||||
|
|
|
@ -17,6 +17,9 @@ package controllers
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/common"
|
"github.com/goharbor/harbor/src/common"
|
||||||
"github.com/goharbor/harbor/src/common/dao"
|
"github.com/goharbor/harbor/src/common/dao"
|
||||||
"github.com/goharbor/harbor/src/common/models"
|
"github.com/goharbor/harbor/src/common/models"
|
||||||
|
@ -26,8 +29,6 @@ import (
|
||||||
"github.com/goharbor/harbor/src/core/api"
|
"github.com/goharbor/harbor/src/core/api"
|
||||||
"github.com/goharbor/harbor/src/core/config"
|
"github.com/goharbor/harbor/src/core/config"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const tokenKey = "oidc_token"
|
const tokenKey = "oidc_token"
|
||||||
|
@ -189,9 +190,6 @@ func (oc *OIDCController) Onboard() {
|
||||||
}
|
}
|
||||||
|
|
||||||
email := d.Email
|
email := d.Email
|
||||||
if email == "" {
|
|
||||||
email = utils.GenerateRandomString() + "@placeholder.com"
|
|
||||||
}
|
|
||||||
user := models.User{
|
user := models.User{
|
||||||
Username: username,
|
Username: username,
|
||||||
Realname: d.Username,
|
Realname: d.Username,
|
||||||
|
|
|
@ -17,16 +17,12 @@ package main
|
||||||
import (
|
import (
|
||||||
"encoding/gob"
|
"encoding/gob"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"os/signal"
|
|
||||||
"strconv"
|
|
||||||
"syscall"
|
|
||||||
|
|
||||||
"github.com/astaxie/beego"
|
"github.com/astaxie/beego"
|
||||||
_ "github.com/astaxie/beego/session/redis"
|
_ "github.com/astaxie/beego/session/redis"
|
||||||
"github.com/goharbor/harbor/src/common/dao"
|
"github.com/goharbor/harbor/src/common/dao"
|
||||||
"github.com/goharbor/harbor/src/common/job"
|
"github.com/goharbor/harbor/src/common/job"
|
||||||
"github.com/goharbor/harbor/src/common/models"
|
"github.com/goharbor/harbor/src/common/models"
|
||||||
|
common_quota "github.com/goharbor/harbor/src/common/quota"
|
||||||
"github.com/goharbor/harbor/src/common/utils"
|
"github.com/goharbor/harbor/src/common/utils"
|
||||||
"github.com/goharbor/harbor/src/common/utils/log"
|
"github.com/goharbor/harbor/src/common/utils/log"
|
||||||
"github.com/goharbor/harbor/src/core/api"
|
"github.com/goharbor/harbor/src/core/api"
|
||||||
|
@ -34,6 +30,15 @@ import (
|
||||||
_ "github.com/goharbor/harbor/src/core/auth/db"
|
_ "github.com/goharbor/harbor/src/core/auth/db"
|
||||||
_ "github.com/goharbor/harbor/src/core/auth/ldap"
|
_ "github.com/goharbor/harbor/src/core/auth/ldap"
|
||||||
_ "github.com/goharbor/harbor/src/core/auth/uaa"
|
_ "github.com/goharbor/harbor/src/core/auth/uaa"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"strconv"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
quota "github.com/goharbor/harbor/src/core/api/quota"
|
||||||
|
_ "github.com/goharbor/harbor/src/core/api/quota/chart"
|
||||||
|
_ "github.com/goharbor/harbor/src/core/api/quota/registry"
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/core/config"
|
"github.com/goharbor/harbor/src/core/config"
|
||||||
"github.com/goharbor/harbor/src/core/filter"
|
"github.com/goharbor/harbor/src/core/filter"
|
||||||
"github.com/goharbor/harbor/src/core/middlewares"
|
"github.com/goharbor/harbor/src/core/middlewares"
|
||||||
|
@ -41,6 +46,7 @@ import (
|
||||||
"github.com/goharbor/harbor/src/core/service/token"
|
"github.com/goharbor/harbor/src/core/service/token"
|
||||||
"github.com/goharbor/harbor/src/pkg/notification"
|
"github.com/goharbor/harbor/src/pkg/notification"
|
||||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/types"
|
||||||
"github.com/goharbor/harbor/src/replication"
|
"github.com/goharbor/harbor/src/replication"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -67,13 +73,71 @@ func updateInitPassword(userID int, password string) error {
|
||||||
return fmt.Errorf("Failed to update user encrypted password, userID: %d, err: %v", userID, err)
|
return fmt.Errorf("Failed to update user encrypted password, userID: %d, err: %v", userID, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infof("User id: %d updated its encypted password successfully.", userID)
|
log.Infof("User id: %d updated its encrypted password successfully.", userID)
|
||||||
} else {
|
} else {
|
||||||
log.Infof("User id: %d already has its encrypted password.", userID)
|
log.Infof("User id: %d already has its encrypted password.", userID)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Quota migration
|
||||||
|
func quotaSync() error {
|
||||||
|
usages, err := dao.ListQuotaUsages()
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("list quota usage error, %v", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
projects, err := dao.GetProjects(nil)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("list project error, %v", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// The condition handles these two cases:
|
||||||
|
// 1, len(project) > 1 && len(usages) == 1. existing projects without usage, as we do always has 'library' usage in DB.
|
||||||
|
// 2, migration fails at the phase of inserting usage into DB, and parts of them are inserted successfully.
|
||||||
|
if len(projects) != len(usages) {
|
||||||
|
log.Info("Start to sync quota data .....")
|
||||||
|
if err := quota.Sync(config.GlobalProjectMgr, true); err != nil {
|
||||||
|
log.Errorf("Fail to sync quota data, %v", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Info("Success to sync quota data .....")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only has one project without usage
|
||||||
|
zero := common_quota.ResourceList{
|
||||||
|
common_quota.ResourceCount: 0,
|
||||||
|
common_quota.ResourceStorage: 0,
|
||||||
|
}
|
||||||
|
if len(projects) == 1 && len(usages) == 1 {
|
||||||
|
totalRepo, err := dao.GetTotalOfRepositories()
|
||||||
|
if totalRepo == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
refID, err := strconv.ParseInt(usages[0].ReferenceID, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
usedRes, err := types.NewResourceList(usages[0].Used)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if types.Equals(usedRes, zero) && refID == projects[0].ProjectID {
|
||||||
|
log.Info("Start to sync quota data .....")
|
||||||
|
if err := quota.Sync(config.GlobalProjectMgr, true); err != nil {
|
||||||
|
log.Errorf("Fail to sync quota data, %v", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Info("Success to sync quota data .....")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func gracefulShutdown(closing chan struct{}) {
|
func gracefulShutdown(closing chan struct{}) {
|
||||||
signals := make(chan os.Signal, 1)
|
signals := make(chan os.Signal, 1)
|
||||||
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
|
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
|
||||||
|
@ -117,7 +181,7 @@ func main() {
|
||||||
|
|
||||||
password, err := config.InitialAdminPassword()
|
password, err := config.InitialAdminPassword()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("failed to get admin's initia password: %v", err)
|
log.Fatalf("failed to get admin's initial password: %v", err)
|
||||||
}
|
}
|
||||||
if err := updateInitPassword(adminUserID, password); err != nil {
|
if err := updateInitPassword(adminUserID, password); err != nil {
|
||||||
log.Error(err)
|
log.Error(err)
|
||||||
|
@ -174,6 +238,9 @@ func main() {
|
||||||
log.Fatalf("init proxy error, %v", err)
|
log.Fatalf("init proxy error, %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// go proxy.StartProxy()
|
if err := quotaSync(); err != nil {
|
||||||
|
log.Fatalf("quota migration error, %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
beego.Run()
|
beego.Run()
|
||||||
}
|
}
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user