upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor synced 2025-04-15 21:04:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Clean up Clair in prepare script

Browse Source 
Signed-off-by: DQ <dengq@vmware.com>
This commit is contained in:
DQ 2020-10-21 17:19:02 +08:00
parent 8a584aff89
commit 0c9faea294
15 changed files with 9 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
make/photon/prepare/commands/prepare.py
View File

@ -15,8 +15,6 @@ from utils.registry_ctl import prepare_registry_ctl
from utils.core import prepare_core
from utils.notary import prepare_notary
from utils.log import prepare_log_configs
from utils.clair import prepare_clair
from utils.clair_adapter import prepare_clair_adapter
from utils.chart import prepare_chartmuseum
from utils.docker_compose import prepare_docker_compose
from utils.nginx import prepare_nginx, nginx_confd_dir
@ -30,13 +28,12 @@ old_private_key_pem_path, old_crt_path)
@click.command()
@click.option('--conf', default=input_config_path, help="the path of Harbor configuration file")
@click.option('--with-notary', is_flag=True, help="the Harbor instance is to be deployed with notary")
@click.option('--with-clair', is_flag=True, help="the Harbor instance is to be deployed with clair")
@click.option('--with-trivy', is_flag=True, help="the Harbor instance is to be deployed with Trivy")
@click.option('--with-chartmuseum', is_flag=True, help="the Harbor instance is to be deployed with chart repository supporting")
def prepare(conf, with_notary, with_clair, with_trivy, with_chartmuseum):
def prepare(conf, with_notary, with_trivy, with_chartmuseum):
delfile(config_dir)
config_dict = parse_yaml_config(conf, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
config_dict = parse_yaml_config(conf, with_notary=with_notary, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
try:
validate(config_dict, notary_mode=with_notary)
except Exception as e:
@ -47,7 +44,7 @@ def prepare(conf, with_notary, with_clair, with_trivy, with_chartmuseum):
prepare_portal(config_dict)
prepare_log_configs(config_dict)
prepare_nginx(config_dict)
prepare_core(config_dict, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
prepare_core(config_dict, with_notary=with_notary, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
prepare_registry(config_dict)
prepare_registry_ctl(config_dict)
prepare_db(config_dict)
@ -68,14 +65,10 @@ def prepare(conf, with_notary, with_clair, with_trivy, with_chartmuseum):
if with_notary:
prepare_notary(config_dict, nginx_confd_dir, SSL_CERT_PATH, SSL_CERT_KEY_PATH)
if with_clair:
prepare_clair(config_dict)
prepare_clair_adapter(config_dict)
if with_trivy:
prepare_trivy_adapter(config_dict)
if with_chartmuseum:
prepare_chartmuseum(config_dict)
prepare_docker_compose(config_dict, with_clair, with_trivy, with_notary, with_chartmuseum)
prepare_docker_compose(config_dict, with_trivy, with_notary, with_chartmuseum)

2
make/photon/prepare/g.py
View File

@ -57,10 +57,8 @@ INTERNAL_NO_PROXY_DN = {
'jobservice',
'registry',
'registryctl',
'clair',
'chartmuseum',
'notary-server',
'notary-signer',
'clair-adapter',
'trivy-adapter',
}

7
make/photon/prepare/models.py
View File

@ -19,11 +19,6 @@ class InternalTLS:
'portal.crt', 'portal.key'
}
clair_certs_filename = {
'clair_adapter.crt', 'clair_adapter.key',
'clair.crt', 'clair.key'
}
trivy_certs_filename = {
'trivy_adapter.crt', 'trivy_adapter.key',
}
@ -49,8 +44,6 @@ class InternalTLS:
self.tls_dir = tls_dir
if self.enabled:
self.required_filenames = self.harbor_certs_filename
if kwargs.get('with_clair'):
self.required_filenames.update(self.clair_certs_filename)
if kwargs.get('with_notary'):
self.required_filenames.update(self.notary_certs_filename)
if kwargs.get('with_chartmuseum'):

21
make/photon/prepare/scripts/gencert.sh
View File

@ -77,27 +77,6 @@ openssl req -new \
openssl x509 -req -days $DAYS -sha256 -in registryctl.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out registryctl.crt
# generate clair_adapter key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout clair_adapter.key \
-out clair_adapter.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=clair-adapter"
# sign clair_adapter csr with CA certificate and key
openssl x509 -req -days $DAYS -sha256 -in clair_adapter.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out clair_adapter.crt
# generate clair key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout clair.key \
-out clair.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=clair"
# sign clair csr with CA certificate and key
openssl x509 -req -days $DAYS -sha256 -in clair.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out clair.crt
# generate trivy_adapter key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout trivy_adapter.key \

10
make/photon/prepare/templates/clair-adapter/env.jinja
View File

@ -1,10 +0,0 @@
SCANNER_LOG_LEVEL={{log_level}}
SCANNER_CLAIR_URL={{clair_url}}
SCANNER_CLAIR_DATABASE_URL=postgresql://{{clair_db_username}}:{{clair_db_password}}@{{clair_db_host}}:{{clair_db_port}}/{{clair_db_name}}?sslmode={{clair_db_sslmode}}
SCANNER_STORE_REDIS_URL={{redis_url_clair}}
{%if internal_tls.enabled %}
SCANNER_API_SERVER_ADDR=:8443
SCANNER_API_SERVER_TLS_CERTIFICATE=/etc/harbor/ssl/clair_adapter.crt
SCANNER_API_SERVER_TLS_KEY=/etc/harbor/ssl/clair_adapter.key
{% endif %}

3
make/photon/prepare/templates/clair/clair_env.jinja
View File

@ -1,3 +0,0 @@
HTTP_PROXY={{clair_http_proxy}}
HTTPS_PROXY={{clair_https_proxy}}
NO_PROXY={{clair_no_proxy}}

19
make/photon/prepare/templates/clair/config.yaml.jinja
View File

@ -1,19 +0,0 @@
clair:
database:
type: pgsql
options:
source: postgresql://{{clair_db_username}}:{{clair_db_password}}@{{clair_db_host}}:{{clair_db_port}}/{{clair_db_name}}?sslmode={{clair_db_sslmode}}
# Number of elements kept in the cache
# Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database.
cachesize: 16384
api:
# API server port
port: 6060
healthport: 6061
# Deadline before an API request will respond with a 503
timeout: 300s
updater:
interval: {{clair_updaters_interval}}h

1
make/photon/prepare/templates/clair/postgres_env.jinja
View File

@ -1 +0,0 @@
POSTGRES_PASSWORD={{clair_db_password}}

7
make/photon/prepare/templates/clair/postgresql-init.d/README.md
View File

@ -1,7 +0,0 @@
This folder used to run some initial sql for clair if needed.
Just put the sql file in this directory and then start the
clair .
both .sql and .gz format supported

106
make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja
View File

@ -61,9 +61,6 @@ services:
{% endif %}
networks:
- harbor
{% if with_clair %}
- harbor-clair
{% endif %}
dns_search: .
depends_on:
- log
@ -136,11 +133,6 @@ services:
harbor-notary:
aliases:
- harbor-db
{% endif %}
{% if with_clair %}
harbor-clair:
aliases:
- harbor-db
{% endif %}
dns_search: .
env_file:
@ -198,11 +190,6 @@ services:
{% if with_notary %}
harbor-notary:
{% endif %}
{% if with_clair %}
harbor-clair:
aliases:
- harbor-core
{% endif %}
{% if with_chartmuseum %}
harbor-chartmuseum:
aliases:
@ -287,9 +274,6 @@ services:
{% endif %}
networks:
- harbor
{% if with_clair %}
- harbor-clair
{% endif %}
dns_search: .
depends_on:
- core
@ -317,11 +301,6 @@ services:
harbor-chartmuseum:
aliases:
- redis
{% endif %}
{% if with_clair %}
harbor-clair:
aliases:
- redis
{% endif %}
dns_search: .
depends_on:
@ -468,87 +447,6 @@ services:
syslog-address: "tcp://127.0.0.1:1514"
tag: "notary-signer"
{% endif %}
{% if with_clair %}
clair:
networks:
- harbor-clair
container_name: clair
image: goharbor/clair-photon:{{clair_version}}
restart: always
cap_drop:
- ALL
cap_add:
- DAC_OVERRIDE
- SETGID
- SETUID
cpu_quota: 50000
dns_search: .
depends_on:
- log
{% if external_database == False %}
- postgresql
{% endif %}
volumes:
- type: bind
source: ./common/config/clair/config.yaml
target: /etc/clair/config.yaml
- type: bind
source: ./common/config/shared/trust-certificates
target: /harbor_cust_cert
{%if internal_tls.enabled %}
- type: bind
source: {{internal_tls.clair_crt_path}}
target: /etc/harbor/ssl/clair.crt
- type: bind
source: {{internal_tls.clair_key_path}}
target: /etc/harbor/ssl/clair.key
{% endif %}
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "clair"
env_file:
./common/config/clair/clair_env
clair-adapter:
networks:
- harbor-clair
container_name: clair-adapter
image: goharbor/clair-adapter-photon:{{clair_adapter_version}}
restart: always
cap_drop:
- ALL
cap_add:
- DAC_OVERRIDE
- SETGID
- SETUID
cpu_quota: 50000
dns_search: .
depends_on:
- clair
{% if external_redis == False %}
- redis
{% endif %}
volumes:
- type: bind
source: ./common/config/shared/trust-certificates
target: /harbor_cust_cert
{%if internal_tls.enabled %}
- type: bind
source: {{internal_tls.clair_adapter_crt_path}}
target: /etc/harbor/ssl/clair_adapter.crt
- type: bind
source: {{internal_tls.clair_adapter_key_path}}
target: /etc/harbor/ssl/clair_adapter.key
{% endif %}
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "clair-adapter"
env_file:
./common/config/clair-adapter/env
{% endif %}
{% if with_trivy %}
trivy-adapter:
container_name: trivy-adapter
@ -643,10 +541,6 @@ networks:
notary-sig:
external: false
{% endif %}
{% if with_clair %}
harbor-clair:
external: false
{% endif %}
{% if with_chartmuseum %}
harbor-chartmuseum:
external: false

43
make/photon/prepare/utils/clair.py
View File

@ -1,43 +0,0 @@
import os, shutil
from g import templates_dir, config_dir, DEFAULT_UID, DEFAULT_GID
from .jinja import render_jinja
from .misc import prepare_dir
clair_template_dir = os.path.join(templates_dir, "clair")
def prepare_clair(config_dict):
clair_config_dir = prepare_dir(config_dir, "clair")
if os.path.exists(os.path.join(clair_config_dir, "postgresql-init.d")):
print("Copying offline data file for clair DB")
shutil.rmtree(os.path.join(clair_config_dir, "postgresql-init.d"))
shutil.copytree(os.path.join(clair_template_dir, "postgresql-init.d"), os.path.join(clair_config_dir, "postgresql-init.d"))
postgres_env_path = os.path.join(clair_config_dir, "postgres_env")
postgres_env_template = os.path.join(clair_template_dir, "postgres_env.jinja")
clair_config_path = os.path.join(clair_config_dir, "config.yaml")
clair_config_template = os.path.join(clair_template_dir, "config.yaml.jinja")
clair_env_path = os.path.join(clair_config_dir, "clair_env")
clair_env_template = os.path.join(clair_template_dir, "clair_env.jinja")
render_jinja(
postgres_env_template,
postgres_env_path,
**config_dict)
render_jinja(
clair_config_template,
clair_config_path,
uid=DEFAULT_UID,
gid=DEFAULT_GID,
**config_dict)
# config http proxy for Clair
render_jinja(
clair_env_template,
clair_env_path,
**config_dict)

18
make/photon/prepare/utils/clair_adapter.py
View File

@ -1,18 +0,0 @@
import os
from g import templates_dir, config_dir
from .jinja import render_jinja
from .misc import prepare_dir
clair_adapter_template_dir = os.path.join(templates_dir, "clair-adapter")
def prepare_clair_adapter(config_dict):
clair_adapter_config_dir = prepare_dir(config_dir, "clair-adapter")
clair_adapter_env_path = os.path.join(clair_adapter_config_dir, "env")
clair_adapter_env_template = os.path.join(clair_adapter_template_dir, "env.jinja")
render_jinja(
clair_adapter_env_template,
clair_adapter_env_path,
**config_dict)

43
make/photon/prepare/utils/configs.py
View File

@ -85,7 +85,7 @@ def parse_versions():
return versions
def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, with_chartmuseum):
def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseum):
'''
:param configs: config_parser object
:returns: dict of configs
@ -102,8 +102,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
'core_local_url': 'http://127.0.0.1:8080',
'token_service_url': 'http://core:8080/service/token',
'jobservice_url': 'http://jobservice:8080',
'clair_url': 'http://clair:6060',
'clair_adapter_url': 'http://clair-adapter:8080',
'trivy_adapter_url': 'http://trivy-adapter:8080',
'notary_url': 'http://notary-server:4443',
'chart_repository_url': 'http://chartmuseum:9999'
@ -149,15 +147,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
config_dict['harbor_db_max_idle_conns'] = db_configs.get("max_idle_conns") or default_db_max_idle_conns
config_dict['harbor_db_max_open_conns'] = db_configs.get("max_open_conns") or default_db_max_open_conns
if with_clair:
# clair db
config_dict['clair_db_host'] = 'postgresql'
config_dict['clair_db_port'] = 5432
config_dict['clair_db_name'] = 'postgres'
config_dict['clair_db_username'] = 'postgres'
config_dict['clair_db_password'] = db_configs.get("password") or ''
config_dict['clair_db_sslmode'] = 'disable'
if with_notary:
# notary signer
config_dict['notary_signer_db_host'] = 'postgresql'
@ -224,12 +213,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
config_dict[proxy_component + '_https_proxy'] = proxy_config.get('https_proxy') or ''
config_dict[proxy_component + '_no_proxy'] = ','.join(all_no_proxy)
# Clair configs, optional
clair_configs = configs.get("clair") or {}
config_dict['clair_db'] = 'postgres'
updaters_interval = clair_configs.get("updaters_interval", None)
config_dict['clair_updaters_interval'] = 12 if updaters_interval is None else updaters_interval
# Trivy configs, optional
trivy_configs = configs.get("trivy") or {}
config_dict['trivy_github_token'] = trivy_configs.get("github_token") or ''
@ -292,14 +275,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
config_dict['harbor_db_max_idle_conns'] = external_db_configs['harbor'].get("max_idle_conns") or default_db_max_idle_conns
config_dict['harbor_db_max_open_conns'] = external_db_configs['harbor'].get("max_open_conns") or default_db_max_open_conns
if with_clair:
# clair db
config_dict['clair_db_host'] = external_db_configs['clair']['host']
config_dict['clair_db_port'] = external_db_configs['clair']['port']
config_dict['clair_db_name'] = external_db_configs['clair']['db_name']
config_dict['clair_db_username'] = external_db_configs['clair']['username']
config_dict['clair_db_password'] = external_db_configs['clair']['password']
config_dict['clair_db_sslmode'] = external_db_configs['clair']['ssl_mode']
if with_notary:
# notary signer
config_dict['notary_signer_db_host'] = external_db_configs['notary_signer']['host']
@ -319,7 +294,7 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
config_dict['external_database'] = False
# update redis configs
config_dict.update(get_redis_configs(configs.get("external_redis", None), with_clair, with_trivy))
config_dict.update(get_redis_configs(configs.get("external_redis", None), with_trivy))
# auto generated secret string for core
config_dict['core_secret'] = generate_random_string(16)
@ -339,7 +314,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
internal_tls_config['dir'],
configs['data_volume'],
with_notary=with_notary,
with_clair=with_clair,
with_trivy=with_trivy,
with_chartmuseum=with_chartmuseum,
external_database=config_dict['external_database'])
@ -361,7 +335,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
config_dict['core_local_url'] = 'https://core:8443'
config_dict['token_service_url'] = 'https://core:8443/service/token'
config_dict['jobservice_url'] = 'https://jobservice:8443'
config_dict['clair_adapter_url'] = 'https://clair-adapter:8443'
config_dict['trivy_adapter_url'] = 'https://trivy-adapter:8443'
# config_dict['notary_url'] = 'http://notary-server:4443'
config_dict['chart_repository_url'] = 'https://chartmuseum:9443'
@ -404,7 +377,7 @@ def get_redis_url_param(redis=None):
return ""
def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
def get_redis_configs(external_redis=None, with_trivy=True):
"""Returns configs for redis
>>> get_redis_configs()['external_redis']
@ -413,8 +386,6 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
'redis://redis:6379/1'
>>> get_redis_configs()['redis_url_js']
'redis://redis:6379/2'
>>> get_redis_configs()['redis_url_clair']
'redis://redis:6379/4'
>>> get_redis_configs()['trivy_redis_url']
'redis://redis:6379/5'
@ -433,13 +404,9 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
'redis://anonymous:pass@localhost:6379/1'
>>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['redis_url_js']
'redis://anonymous:pass@localhost:6379/2'
>>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['redis_url_clair']
'redis://anonymous:pass@localhost:6379/4'
>>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['trivy_redis_url']
'redis://anonymous:pass@localhost:6379/5'
>>> 'redis_url_clair' not in get_redis_configs(with_clair=False)
True
>>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False)
True
"""
@ -454,7 +421,6 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
'registry_db_index': 1,
'jobservice_db_index': 2,
'chartmuseum_db_index': 3,
'clair_db_index': 4,
'trivy_db_index': 5,
'idle_timeout_seconds': 30,
}
@ -467,9 +433,6 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis)
configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis)
if with_clair:
configs['redis_url_clair'] = get_redis_url(redis['clair_db_index'], redis)
if with_trivy:
configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis)

3
make/photon/prepare/utils/core.py
View File

@ -13,7 +13,7 @@ core_conf = os.path.join(config_dir, "core", "app.conf")
ca_download_dir = os.path.join(data_dir, 'ca_download')
def prepare_core(config_dict, with_notary, with_clair, with_trivy, with_chartmuseum):
def prepare_core(config_dict, with_notary, with_trivy, with_chartmuseum):
prepare_dir(ca_download_dir, uid=DEFAULT_UID, gid=DEFAULT_GID)
prepare_dir(core_config_dir)
# Render Core
@ -29,7 +29,6 @@ def prepare_core(config_dict, with_notary, with_clair, with_trivy, with_chartmus
core_conf_env,
chart_cache_driver=chart_cache_driver,
with_notary=with_notary,
with_clair=with_clair,
with_trivy=with_trivy,
with_chartmuseum=with_chartmuseum,
csrf_key=generate_random_string(32),

5
make/photon/prepare/utils/docker_compose.py
View File

@ -8,7 +8,7 @@ docker_compose_template_path = os.path.join(templates_dir, 'docker_compose', 'do
docker_compose_yml_path = '/compose_location/docker-compose.yml'
# render docker-compose
def prepare_docker_compose(configs, with_clair, with_trivy, with_notary, with_chartmuseum):
def prepare_docker_compose(configs, with_trivy, with_notary, with_chartmuseum):
versions = parse_versions()
VERSION_TAG = versions.get('VERSION_TAG') or 'dev'
@ -17,8 +17,6 @@ def prepare_docker_compose(configs, with_clair, with_trivy, with_notary, with_ch
'reg_version': VERSION_TAG,
'redis_version': VERSION_TAG,
'notary_version': VERSION_TAG,
'clair_version': VERSION_TAG,
'clair_adapter_version': VERSION_TAG,
'trivy_adapter_version': VERSION_TAG,
'chartmuseum_version': VERSION_TAG,
'data_volume': configs['data_volume'],
@ -28,7 +26,6 @@ def prepare_docker_compose(configs, with_clair, with_trivy, with_notary, with_ch
'external_redis': configs['external_redis'],
'external_database': configs['external_database'],
'with_notary': with_notary,
'with_clair': with_clair,
'with_trivy': with_trivy,
'with_chartmuseum': with_chartmuseum
}