From d328e2586e9993220b66068023199edf08fa4532 Mon Sep 17 00:00:00 2001 From: yixingj Date: Wed, 27 Sep 2017 12:56:57 +0800 Subject: [PATCH] HA installation script Add --ha options when install Harbor. Currently it does nothing. --- Makefile | 10 +- make/common/templates/registry/config_ha.yml | 45 +++++++ make/common/templates/ui/env | 1 + make/ha/docker-compose.tpl | 121 ++++++++++++++++++ .../keepalived_active_active.conf | 80 ++++++++++++ make/ha/sample/active_standby/check_harbor.sh | 7 + .../keepalived_active_standby.conf | 36 ++++++ make/harbor.cfg | 2 + make/install.sh | 24 +++- make/prepare | 112 ++++++++++++++-- 10 files changed, 421 insertions(+), 17 deletions(-) create mode 100644 make/common/templates/registry/config_ha.yml create mode 100644 make/ha/docker-compose.tpl create mode 100644 make/ha/sample/active_active/keepalived_active_active.conf create mode 100644 make/ha/sample/active_standby/check_harbor.sh create mode 100644 make/ha/sample/active_standby/keepalived_active_standby.conf diff --git a/Makefile b/Makefile index efd0c3394..345a23a9b 100644 --- a/Makefile +++ b/Makefile @@ -230,13 +230,14 @@ PACKAGE_OFFLINE_PARA=-zcvf harbor-offline-installer-$(GITTAGVERSION).tgz \ $(HARBORPKG)/prepare $(HARBORPKG)/NOTICE \ $(HARBORPKG)/upgrade $(HARBORPKG)/harbor_1_1_0_template \ $(HARBORPKG)/LICENSE $(HARBORPKG)/install.sh \ - $(HARBORPKG)/harbor.cfg $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) + $(HARBORPKG)/harbor.cfg $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) \ + $(HARBORPKG)/ha PACKAGE_ONLINE_PARA=-zcvf harbor-online-installer-$(GITTAGVERSION).tgz \ $(HARBORPKG)/common/templates $(HARBORPKG)/prepare \ $(HARBORPKG)/LICENSE $(HARBORPKG)/NOTICE \ $(HARBORPKG)/upgrade $(HARBORPKG)/harbor_1_1_0_template \ $(HARBORPKG)/install.sh $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) \ - $(HARBORPKG)/harbor.cfg + $(HARBORPKG)/harbor.cfg $(HARBORPKG)/ha DOCKERCOMPOSE_LIST=-f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME) ifeq ($(NOTARYFLAG), true) @@ -327,7 +328,9 @@ build: build_$(BASEIMAGE) modify_composefile: @echo "preparing docker-compose file..." @cp $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSETPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME) + @cp $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSETPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSEFILENAME) @$(SEDCMD) -i 's/__version__/$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME) + @$(SEDCMD) -i 's/__version__/$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSEFILENAME) modify_sourcefiles: @echo "change mode of source files." @@ -345,6 +348,8 @@ package_online: modify_composefile @if [ -n "$(REGISTRYSERVER)" ] ; then \ $(SEDCMD) -i 's/image\: vmware/image\: $(REGISTRYSERVER)\/$(REGISTRYPROJECTNAME)/' \ $(HARBORPKG)/docker-compose.yml ; \ + $(SEDCMD) -i 's/image\: vmware/image\: $(REGISTRYSERVER)\/$(REGISTRYPROJECTNAME)/' \ + $(HARBORPKG)/ha/docker-compose.yml ; \ fi @cp LICENSE $(HARBORPKG)/LICENSE @cp NOTICE $(HARBORPKG)/NOTICE @@ -363,6 +368,7 @@ package_offline: compile build modify_sourcefiles modify_composefile @cp NOTICE $(HARBORPKG)/NOTICE @cp tools/migration/migration_cfg/upgrade $(HARBORPKG)/upgrade @cp tools/migration/migration_cfg/harbor_1_1_0_template $(HARBORPKG)/harbor_1_1_0_template + @cp $(HARBORPKG)/common/db/registry.sql $(HARBORPKG)/ha/ @echo "pulling nginx and registry..." @$(DOCKERPULL) vmware/registry:$(REGISTRYVERSION) diff --git a/make/common/templates/registry/config_ha.yml b/make/common/templates/registry/config_ha.yml new file mode 100644 index 000000000..bedf6f482 --- /dev/null +++ b/make/common/templates/registry/config_ha.yml @@ -0,0 +1,45 @@ +version: 0.1 +log: + level: debug + fields: + service: registry +storage: + cache: + layerinfo: redis + Place_holder_for_Storage_configureation + maintenance: + uploadpurging: + enabled: false + delete: + enabled: true +redis: + addr: $redis_url + db: 0 + dialtimeout: 10ms + readtimeout: 10ms + writetimeout: 10ms + pool: + maxidle: 16 + maxactive: 64 + idletimeout: 300s + +http: + addr: :5000 + secret: placeholder + debug: + addr: localhost:5001 +auth: + token: + issuer: harbor-token-issuer + realm: $ui_url/service/token + rootcertbundle: /etc/registry/root.crt + service: harbor-registry + +notifications: + endpoints: + - name: harbor + disabled: false + url: http://ui:8080/service/notifications + timeout: 3000ms + threshold: 5 + backoff: 1s diff --git a/make/common/templates/ui/env b/make/common/templates/ui/env index 1fffc2db1..3a7000135 100644 --- a/make/common/templates/ui/env +++ b/make/common/templates/ui/env @@ -5,3 +5,4 @@ JOBSERVICE_SECRET=$jobservice_secret GODEBUG=netdns=cgo ADMINSERVER_URL=http://adminserver:8080 UAA_CA_ROOT=/etc/ui/certificates/uaa_ca.pem +_REDIS_URL=$redis_url diff --git a/make/ha/docker-compose.tpl b/make/ha/docker-compose.tpl new file mode 100644 index 000000000..4adddfb40 --- /dev/null +++ b/make/ha/docker-compose.tpl @@ -0,0 +1,121 @@ +version: '2' +services: + log: + image: vmware/harbor-log:__version__ + container_name: harbor-log + restart: always + volumes: + - /var/log/harbor/:/var/log/docker/:z + - ./common/config/log/:/etc/logrotate.d/:z + ports: + - 127.0.0.1:1514:10514 + networks: + - harbor + registry: + image: vmware/registry:2.6.2-photon + container_name: registry + restart: always + volumes: + - /data/registry:/storage:z + - ./common/config/registry/:/etc/registry/:z + networks: + - harbor + environment: + - GODEBUG=netdns=cgo + command: + ["serve", "/etc/registry/config.yml"] + depends_on: + - log + logging: + driver: "syslog" + options: + syslog-address: "tcp://127.0.0.1:1514" + tag: "registry" + adminserver: + image: vmware/harbor-adminserver:__version__ + container_name: harbor-adminserver + env_file: + - ./common/config/adminserver/env + restart: always + volumes: + - /data/config/:/etc/adminserver/config/:z + - /data/secretkey:/etc/adminserver/key:z + - /data/:/data/:z + networks: + - harbor + depends_on: + - log + logging: + driver: "syslog" + options: + syslog-address: "tcp://127.0.0.1:1514" + tag: "adminserver" + ui: + image: vmware/harbor-ui:__version__ + container_name: harbor-ui + env_file: + - ./common/config/ui/env + restart: always + volumes: + - ./common/config/ui/app.conf:/etc/ui/app.conf:z + - ./common/config/ui/private_key.pem:/etc/ui/private_key.pem:z + - ./common/config/ui/certificates/:/etc/ui/certifates/ + - /data/secretkey:/etc/ui/key:z + - /data/ca_download/:/etc/ui/ca/:z + - /data/psc/:/etc/ui/token/:z + networks: + - harbor + depends_on: + - log + - adminserver + - registry + logging: + driver: "syslog" + options: + syslog-address: "tcp://127.0.0.1:1514" + tag: "ui" + jobservice: + image: vmware/harbor-jobservice:__version__ + container_name: harbor-jobservice + env_file: + - ./common/config/jobservice/env + restart: always + volumes: + - /data/job_logs:/var/log/jobs:z + - ./common/config/jobservice/app.conf:/etc/jobservice/app.conf:z + - /data/secretkey:/etc/jobservice/key:z + networks: + - harbor + depends_on: + - ui + - adminserver + logging: + driver: "syslog" + options: + syslog-address: "tcp://127.0.0.1:1514" + tag: "jobservice" + proxy: + image: vmware/nginx-photon:1.11.13 + container_name: nginx + restart: always + volumes: + - ./common/config/nginx:/etc/nginx:z + networks: + - harbor + ports: + - 80:80 + - 443:443 + - 4443:4443 + depends_on: + - registry + - ui + - log + logging: + driver: "syslog" + options: + syslog-address: "tcp://127.0.0.1:1514" + tag: "proxy" +networks: + harbor: + external: false + diff --git a/make/ha/sample/active_active/keepalived_active_active.conf b/make/ha/sample/active_active/keepalived_active_active.conf new file mode 100644 index 000000000..1dc7a9d6f --- /dev/null +++ b/make/ha/sample/active_active/keepalived_active_active.conf @@ -0,0 +1,80 @@ +global_defs { + router_id haborlb +} +vrrp_sync_groups VG1 { + group { + VI_1 + } +} +#Please change to ens160 to the interface name on you loadbalancer hosts. +vrrp_instance VI_1 { + interface ens160 + + track_interface { + ens160 + } + + state MASTER + virtual_router_id 51 + priority 10 + + virtual_ipaddress { + VIP/32 + } + advert_int 1 + authentication { + auth_type PASS + auth_pass d0cker + } + +} +#Please change VIP, harbor_node1_ip, harbor_node2_ip to real ip address +virtual_server VIP 80 { + delay_loop 15 + lb_algo rr + lb_kind DR + protocol TCP + nat_mask 255.255.255.0 + persistence_timeout 10 + + real_server harbor_node1_ip 80 { + weight 10 + TCP_CHECK { + connect_timeout 3 + connect_port 80 + } + } + + real_server harbor_node2_ip 80 { + weight 10 + TCP_CHECK { + connect_timeout 3 + connect_port 80 + } + } +} +#Please uncomment the follow when harbor running under https +#virtual_server VIP 443 { +# delay_loop 15 +# lb_algo rr +# lb_kind DR +# protocol TCP +# nat_mask 255.255.255.0 +# persistence_timeout 10 +# +# real_server harbor_node1_ip 443 { +# weight 10 +# TCP_CHECK { +# connect_timeout 3 +# connect_port 443 +# } +# } +# +# real_server harbor_node2_ip 443 { +# weight 10 +# TCP_CHECK { +# connect_timeout 3 +# connect_port 443 +# } +# } +#} diff --git a/make/ha/sample/active_standby/check_harbor.sh b/make/ha/sample/active_standby/check_harbor.sh new file mode 100644 index 000000000..0218581dd --- /dev/null +++ b/make/ha/sample/active_standby/check_harbor.sh @@ -0,0 +1,7 @@ +#!/bin/bash +http_code = `curl -s -o /dev/null -w "%{http_code}" 127.0.0.1` +if [ $http_code == 200 ] || [ $http_code == 301 ] ; then + exit 0 +else + exit 1 +fi diff --git a/make/ha/sample/active_standby/keepalived_active_standby.conf b/make/ha/sample/active_standby/keepalived_active_standby.conf new file mode 100644 index 000000000..6c92a843a --- /dev/null +++ b/make/ha/sample/active_standby/keepalived_active_standby.conf @@ -0,0 +1,36 @@ +global_defs { + router_id haborlb +} +vrrp_script check_harbor { + script "/usr/local/bin/check_harbor.sh" + interval 15 + fail 5 + rise 2 +} +vrrp_sync_groups VG1 { + group { + VI_1 + } +} +#Please change to ens160 to the interface name on you loadbalancer hosts. +vrrp_instance VI_1 { + interface ens160 + + track_interface { + ens160 + } + + state MASTER + virtual_router_id 51 + priority 10 + + virtual_ipaddress { + VIP/32 + } + advert_int 1 + authentication { + auth_type PASS + auth_pass d0cker + } + +} diff --git a/make/harbor.cfg b/make/harbor.cfg index f926c0429..8895943ca 100644 --- a/make/harbor.cfg +++ b/make/harbor.cfg @@ -116,6 +116,8 @@ db_port = 3306 #The user name of mysql database db_user = root +#The redis server address +redis_url = #************************END INITIAL PROPERTIES************************ #The following attributes only need to be set when auth mode is uaa_auth uaa_endpoint = uaa.mydomain.org diff --git a/make/install.sh b/make/install.sh index d435cc8d5..f80805618 100755 --- a/make/install.sh +++ b/make/install.sh @@ -58,7 +58,8 @@ item=0 with_notary=$false # clair is not enabled by default with_clair=$false - +# HA mode is not enabled by default +harbor_ha=$false while [ $# -gt 0 ]; do case $1 in --help) @@ -67,7 +68,9 @@ while [ $# -gt 0 ]; do --with-notary) with_notary=true;; --with-clair) - with_clair=true;; + with_clair=true;; + --ha) + harbor_ha=true;; *) note "$usage" exit 1;; @@ -158,24 +161,28 @@ then sed "s/^hostname = .*/hostname = $host/g" -i ./harbor.cfg fi prepare_para= -if [ $with_notary ] +if [ $with_notary ] && [ ! $harbor_ha ] then prepare_para="${prepare_para} --with-notary" fi -if [ $with_clair ] +if [ $with_clair ] && [ ! $harbor_ha ] then prepare_para="${prepare_para} --with-clair" fi +if [ $harbor_ha ] +then + prepare_para="${prepare_para} --ha" +fi ./prepare $prepare_para echo "" h2 "[Step $item]: checking existing instance of Harbor ..."; let item+=1 docker_compose_list='-f docker-compose.yml' -if [ $with_notary ] +if [ $with_notary ] && [ ! $harbor_ha ] then docker_compose_list="${docker_compose_list} -f docker-compose.notary.yml" fi -if [ $with_clair ] +if [ $with_clair ] && [ ! $harbor_ha ] then docker_compose_list="${docker_compose_list} -f docker-compose.clair.yml" fi @@ -188,6 +195,11 @@ fi echo "" h2 "[Step $item]: starting Harbor ..." +if [ $harbor_ha ] +then + mv docker-compose.yml docker-compose.yml.bak + cp ha/docker-compose.yml docker-compose.yml +fi docker-compose $docker_compose_list up -d protocol=http diff --git a/make/prepare b/make/prepare index 5fb7c3380..65487bc9d 100755 --- a/make/prepare +++ b/make/prepare @@ -20,6 +20,27 @@ if sys.version_info[:3][0] == 3: import io as StringIO def validate(conf, args): + if args.ha_mode: + db_host = rcp.get("configuration", "db_host") + if db_host == "mysql": + raise Exception("Error: In HA mode, db_host in harbor.cfg needs to point to an external DB address") + registry_config_path = os.path.join(templates_dir,"registry","config_ha.yml") + if check_storage_config(registry_config_path): + raise Exception("Error: In HA model shared storage configuration is required registry, refer HA installation guide for detail.") + redis_url = rcp.get("configuration", "redis_url") + if redis_url is None or len(redis_url) < 1: + raise Exception("Error: In HA mode redis is required redis_url need to point to an redis cluster") + if args.notary_mode or args.clair_mode: + raise Exception("Error: HA mode doesn't support clair and notary currently") + cert_path = rcp.get("configuration", "ssl_cert") + cert_key_path = rcp.get("configuration", "ssl_cert_key") + shared_cert_key = os.path.join(base_dir, "ha", os.path.basename(cert_key_path)) + shared_cert_path = os.path.join(base_dir, "ha", os.path.basename(cert_path)) + if os.path.isfile(shared_cert_key): + shutil.copy2(shared_cert_key, cert_key_path) + if os.path.isfile(shared_cert_path): + shutil.copy2(shared_cert_path, cert_path) + protocol = rcp.get("configuration", "ui_url_protocol") if protocol != "https" and args.notary_mode: raise Exception("Error: the protocol must be https when Harbor is deployed with Notary") @@ -39,6 +60,63 @@ def validate(conf, args): if project_creation != "everyone" and project_creation != "adminonly": raise Exception("Error invalid value for project_creation_restriction: %s" % project_creation) +def prepare_ha(conf, args): + #files under ha folder will have high prority + protocol = rcp.get("configuration", "ui_url_protocol") + if protocol == "https": + #copy nginx certificate + cert_path = rcp.get("configuration", "ssl_cert") + cert_key_path = rcp.get("configuration", "ssl_cert_key") + shared_cert_key = os.path.join(base_dir, "ha", os.path.basename(cert_key_path)) + shared_cert_path = os.path.join(base_dir, "ha", os.path.basename(cert_path)) + if os.path.isfile(shared_cert_key): + shutil.copy2(shared_cert_key, cert_key_path) + else: + if os.path.isfile(cert_key_path): + shutil.copy2(cert_key_path, shared_cert_key) + if os.path.isfile(shared_cert_path): + shutil.copy2(shared_cert_path, cert_path) + else: + if os.path.isfile(cert_path): + shutil.copy2(cert_path, shared_cert_path) + #check if ca exsit + cert_ca_path = "/data/ca_download/ca.crt" + shared_ca_path = os.path.join(base_dir, "ha", os.path.basename(cert_ca_path)) + if os.path.isfile(shared_ca_path): + shutil.copy2(shared_ca_path, cert_ca_path) + else: + if os.path.isfile(cert_ca_path): + shutil.copy2(cert_ca_path, shared_ca_path) + #check root.crt and priviate_key.pem + private_key_pem = os.path.join(config_dir, "ui", "private_key.pem") + root_crt = os.path.join(config_dir, "registry", "root.crt") + shared_private_key_pem = os.path.join(base_dir, "ha", "private_key.pem") + shared_root_crt = os.path.join(base_dir, "ha", "root.crt") + if os.path.isfile(shared_private_key_pem): + shutil.copy2(shared_private_key_pem, private_key_pem) + else: + if os.path.isfile(private_key_pem): + shutil.copy2(private_key_pem, shared_private_key_pem) + if os.path.isfile(shared_root_crt): + shutil.copy2(shared_root_crt, root_crt) + else: + if os.path.isfile(root_crt): + shutil.copy2(root_crt, shared_root_crt) + #secretkey + shared_secret_key = os.path.join(base_dir, "ha", "secretkey") + secretkey_path = rcp.get("configuration", "secretkey_path") + secret_key = os.path.join(secretkey_path, "secretkey") + if os.path.isfile(shared_secret_key): + shutil.copy2(shared_secret_key, secret_key) + else: + if os.path.isfile(secret_key): + shutil.copy2(secret_key, shared_secret_key) + +def check_storage_config(path): + if 'Place_holder_for_Storage_configureation' in open(path).read(): + return True + return False + def get_secret_key(path): secret_key = _get_secret(path, "secretkey") if len(secret_key) != 16: @@ -96,6 +174,7 @@ parser = argparse.ArgumentParser() parser.add_argument('--conf', dest='cfgfile', default=base_dir+'/harbor.cfg',type=str,help="the path of Harbor configuration file") parser.add_argument('--with-notary', dest='notary_mode', default=False, action='store_true', help="the Harbor instance is to be deployed with notary") parser.add_argument('--with-clair', dest='clair_mode', default=False, action='store_true', help="the Harbor instance is to be deployed with clair") +parser.add_argument('--ha', dest='ha_mode', default=False, action='store_true', help="the Harbor instance is to be deployed in HA mode") args = parser.parse_args() delfile(config_dir) @@ -106,7 +185,6 @@ conf.write(open(args.cfgfile).read()) conf.seek(0, os.SEEK_SET) rcp = ConfigParser.RawConfigParser() rcp.readfp(conf) - validate(rcp, args) hostname = rcp.get("configuration", "hostname") @@ -141,8 +219,8 @@ ldap_scope = rcp.get("configuration", "ldap_scope") ldap_timeout = rcp.get("configuration", "ldap_timeout") db_password = rcp.get("configuration", "db_password") db_host = rcp.get("configuration", "db_host") -db_port = rcp.get("configuration", "db_port") db_user = rcp.get("configuration", "db_user") +db_port = rcp.get("configuration", "db_port") self_registration = rcp.get("configuration", "self_registration") if protocol == "https": cert_path = rcp.get("configuration", "ssl_cert") @@ -161,9 +239,15 @@ uaa_endpoint = rcp.get("configuration", "uaa_endpoint") uaa_clientid = rcp.get("configuration", "uaa_clientid") uaa_clientsecret = rcp.get("configuration", "uaa_clientsecret") uaa_ca_root = rcp.get("configuration", "uaa_ca_root") + secret_key = get_secret_key(secretkey_path) log_rotate_count = rcp.get("configuration", "log_rotate_count") log_rotate_size = rcp.get("configuration", "log_rotate_size") + +if rcp.has_option("configuration", "redis_url"): + redis_url = rcp.get("configuration", "redis_url") +else: + redis_url = "" ######## ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16)) @@ -229,8 +313,8 @@ render(os.path.join(templates_dir, "adminserver", "env"), ldap_timeout=ldap_timeout, db_password=db_password, db_host=db_host, - db_port=db_port, db_user=db_user, + db_port=db_port, email_host=email_host, email_port=email_port, email_usr=email_usr, @@ -258,12 +342,19 @@ render(os.path.join(templates_dir, "ui", "env"), ui_conf_env, ui_secret=ui_secret, jobservice_secret=jobservice_secret, + redis_url = redis_url ) - -render(os.path.join(templates_dir, "registry", - "config.yml"), - registry_conf, - ui_url=ui_url) +if args.ha_mode: + render(os.path.join(templates_dir, "registry", + "config_ha.yml"), + registry_conf, + ui_url=ui_url, + redis_url=redis_url) +else: + render(os.path.join(templates_dir, "registry", + "config.yml"), + registry_conf, + ui_url=ui_url) render(os.path.join(templates_dir, "db", "env"), db_conf_env, @@ -345,7 +436,7 @@ else: shutil.copyfile(os.path.join(templates_dir, "ui", "private_key.pem"), os.path.join(ui_config_dir, "private_key.pem")) print("Copied configuration file: %s" % registry_config_dir + "root.crt") shutil.copyfile(os.path.join(templates_dir, "registry", "root.crt"), os.path.join(registry_config_dir, "root.crt")) - + if args.notary_mode: notary_config_dir = prep_conf_dir(config_dir, "notary") notary_temp_dir = os.path.join(templates_dir, "notary") @@ -413,6 +504,9 @@ if args.clair_mode: clair_conf = os.path.join(clair_config_dir, "config.yaml") render(os.path.join(clair_temp_dir, "config.yaml"), clair_conf, password = pg_password) +if args.ha_mode: + prepare_ha(rcp, args) + FNULL.close() print("The configuration files are ready, please use docker-compose to start the service.")