HA installation script

Add --ha options when install Harbor.

Currently it does nothing.

Makefile

@@ -230,13 +230,14 @@ PACKAGE_OFFLINE_PARA=-zcvf harbor-offline-installer-$(GITTAGVERSION).tgz \
 				  $(HARBORPKG)/prepare $(HARBORPKG)/NOTICE \
 				  $(HARBORPKG)/upgrade $(HARBORPKG)/harbor_1_1_0_template \
 				  $(HARBORPKG)/LICENSE $(HARBORPKG)/install.sh \
-				  $(HARBORPKG)/harbor.cfg $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME)
+				  $(HARBORPKG)/harbor.cfg $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) \
+				  $(HARBORPKG)/ha
 PACKAGE_ONLINE_PARA=-zcvf harbor-online-installer-$(GITTAGVERSION).tgz \
 		          $(HARBORPKG)/common/templates $(HARBORPKG)/prepare \
 				  $(HARBORPKG)/LICENSE $(HARBORPKG)/NOTICE \
 				  $(HARBORPKG)/upgrade $(HARBORPKG)/harbor_1_1_0_template \
 				  $(HARBORPKG)/install.sh $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) \
-				  $(HARBORPKG)/harbor.cfg
+				  $(HARBORPKG)/harbor.cfg $(HARBORPKG)/ha
 DOCKERCOMPOSE_LIST=-f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
 
 ifeq ($(NOTARYFLAG), true)
@@ -327,7 +328,9 @@ build: build_$(BASEIMAGE)
 modify_composefile:
 	@echo "preparing docker-compose file..."
 	@cp $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSETPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
+	@cp $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSETPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSEFILENAME)
 	@$(SEDCMD) -i 's/__version__/$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
+	@$(SEDCMD) -i 's/__version__/$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSEFILENAME)
 
 modify_sourcefiles:
 	@echo "change mode of source files."
@@ -345,6 +348,8 @@ package_online: modify_composefile
 	@if [ -n "$(REGISTRYSERVER)" ] ; then \
 		$(SEDCMD) -i 's/image\: vmware/image\: $(REGISTRYSERVER)\/$(REGISTRYPROJECTNAME)/' \
 		$(HARBORPKG)/docker-compose.yml ; \
+		$(SEDCMD) -i 's/image\: vmware/image\: $(REGISTRYSERVER)\/$(REGISTRYPROJECTNAME)/' \
+		$(HARBORPKG)/ha/docker-compose.yml ; \
 	fi
 	@cp LICENSE $(HARBORPKG)/LICENSE
 	@cp NOTICE $(HARBORPKG)/NOTICE
@@ -363,6 +368,7 @@ package_offline: compile build modify_sourcefiles modify_composefile
 	@cp NOTICE $(HARBORPKG)/NOTICE
 	@cp tools/migration/migration_cfg/upgrade $(HARBORPKG)/upgrade
 	@cp tools/migration/migration_cfg/harbor_1_1_0_template $(HARBORPKG)/harbor_1_1_0_template
+	@cp $(HARBORPKG)/common/db/registry.sql $(HARBORPKG)/ha/
 
 	@echo "pulling nginx and registry..."
 	@$(DOCKERPULL) vmware/registry:$(REGISTRYVERSION)

make/common/templates/registry/config_ha.yml

@@ -0,0 +1,45 @@
+version: 0.1
+log:
+  level: debug
+  fields:
+    service: registry
+storage:
+    cache:
+        layerinfo: redis
+    Place_holder_for_Storage_configureation
+    maintenance:
+        uploadpurging:
+            enabled: false
+    delete:
+        enabled: true
+redis:
+  addr: $redis_url
+  db: 0
+  dialtimeout: 10ms
+  readtimeout: 10ms
+  writetimeout: 10ms
+  pool:
+    maxidle: 16
+    maxactive: 64
+    idletimeout: 300s
+
+http:
+    addr: :5000
+    secret: placeholder
+    debug:
+        addr: localhost:5001
+auth:
+  token:
+    issuer: harbor-token-issuer
+    realm: $ui_url/service/token
+    rootcertbundle: /etc/registry/root.crt
+    service: harbor-registry
+
+notifications:
+  endpoints:
+      - name: harbor
+        disabled: false
+        url: http://ui:8080/service/notifications
+        timeout: 3000ms
+        threshold: 5
+        backoff: 1s

make/common/templates/ui/env

@@ -5,3 +5,4 @@ JOBSERVICE_SECRET=$jobservice_secret
 GODEBUG=netdns=cgo
 ADMINSERVER_URL=http://adminserver:8080
 UAA_CA_ROOT=/etc/ui/certificates/uaa_ca.pem
+_REDIS_URL=$redis_url

make/ha/docker-compose.tpl

@@ -0,0 +1,121 @@
+version: '2'
+services:
+  log:
+    image: vmware/harbor-log:__version__
+    container_name: harbor-log 
+    restart: always
+    volumes:
+      - /var/log/harbor/:/var/log/docker/:z
+      - ./common/config/log/:/etc/logrotate.d/:z
+    ports:
+      - 127.0.0.1:1514:10514
+    networks:
+      - harbor
+  registry:
+    image: vmware/registry:2.6.2-photon
+    container_name: registry
+    restart: always
+    volumes:
+      - /data/registry:/storage:z
+      - ./common/config/registry/:/etc/registry/:z
+    networks:
+      - harbor
+    environment:
+      - GODEBUG=netdns=cgo
+    command:
+      ["serve", "/etc/registry/config.yml"]
+    depends_on:
+      - log
+    logging:
+      driver: "syslog"
+      options:  
+        syslog-address: "tcp://127.0.0.1:1514"
+        tag: "registry"
+  adminserver:
+    image: vmware/harbor-adminserver:__version__
+    container_name: harbor-adminserver
+    env_file:
+      - ./common/config/adminserver/env
+    restart: always
+    volumes:
+      - /data/config/:/etc/adminserver/config/:z
+      - /data/secretkey:/etc/adminserver/key:z
+      - /data/:/data/:z
+    networks:
+      - harbor
+    depends_on:
+      - log
+    logging:
+      driver: "syslog"
+      options:  
+        syslog-address: "tcp://127.0.0.1:1514"
+        tag: "adminserver"
+  ui:
+    image: vmware/harbor-ui:__version__
+    container_name: harbor-ui
+    env_file:
+      - ./common/config/ui/env
+    restart: always
+    volumes:
+      - ./common/config/ui/app.conf:/etc/ui/app.conf:z
+      - ./common/config/ui/private_key.pem:/etc/ui/private_key.pem:z
+      - ./common/config/ui/certificates/:/etc/ui/certifates/
+      - /data/secretkey:/etc/ui/key:z
+      - /data/ca_download/:/etc/ui/ca/:z
+      - /data/psc/:/etc/ui/token/:z
+    networks:
+      - harbor
+    depends_on:
+      - log
+      - adminserver
+      - registry
+    logging:
+      driver: "syslog"
+      options:  
+        syslog-address: "tcp://127.0.0.1:1514"
+        tag: "ui"
+  jobservice:
+    image: vmware/harbor-jobservice:__version__
+    container_name: harbor-jobservice
+    env_file:
+      - ./common/config/jobservice/env
+    restart: always
+    volumes:
+      - /data/job_logs:/var/log/jobs:z
+      - ./common/config/jobservice/app.conf:/etc/jobservice/app.conf:z
+      - /data/secretkey:/etc/jobservice/key:z
+    networks:
+      - harbor
+    depends_on:
+      - ui
+      - adminserver
+    logging:
+      driver: "syslog"
+      options:  
+        syslog-address: "tcp://127.0.0.1:1514"
+        tag: "jobservice"
+  proxy:
+    image: vmware/nginx-photon:1.11.13
+    container_name: nginx
+    restart: always
+    volumes:
+      - ./common/config/nginx:/etc/nginx:z
+    networks:
+      - harbor
+    ports:
+      - 80:80
+      - 443:443
+      - 4443:4443
+    depends_on:
+      - registry
+      - ui
+      - log
+    logging:
+      driver: "syslog"
+      options:  
+        syslog-address: "tcp://127.0.0.1:1514"
+        tag: "proxy"
+networks:
+  harbor:
+    external: false
+

make/ha/sample/active_active/keepalived_active_active.conf

@@ -0,0 +1,80 @@
+global_defs {
+  router_id haborlb
+}
+vrrp_sync_groups VG1 {
+  group {
+    VI_1
+  }
+}
+#Please change to ens160 to the interface name on you loadbalancer hosts.
+vrrp_instance VI_1 {
+  interface ens160
+
+  track_interface {
+    ens160
+  }
+
+  state MASTER
+  virtual_router_id 51
+  priority 10
+
+  virtual_ipaddress {
+    VIP/32
+  }
+  advert_int 1
+  authentication {
+    auth_type PASS
+    auth_pass d0cker
+  }
+
+}
+#Please change VIP, harbor_node1_ip, harbor_node2_ip to real ip address
+virtual_server VIP 80 {
+  delay_loop 15
+  lb_algo rr
+  lb_kind DR
+  protocol TCP
+  nat_mask 255.255.255.0
+  persistence_timeout 10
+
+  real_server harbor_node1_ip 80 {
+    weight 10
+    TCP_CHECK {
+     connect_timeout 3
+     connect_port 80
+    }
+  }
+
+  real_server harbor_node2_ip 80 {
+    weight 10
+    TCP_CHECK {
+      connect_timeout 3
+      connect_port 80
+    }
+  }
+}
+#Please uncomment the follow when harbor running under https
+#virtual_server VIP 443 {
+#  delay_loop 15
+#  lb_algo rr
+#  lb_kind DR
+#  protocol TCP
+#  nat_mask 255.255.255.0
+#  persistence_timeout 10
+#
+#  real_server harbor_node1_ip 443 {
+#    weight 10
+#    TCP_CHECK {
+#     connect_timeout 3
+#     connect_port 443
+#    }
+#  }
+#
+#  real_server harbor_node2_ip 443 {
+#    weight 10
+#    TCP_CHECK {
+#      connect_timeout 3
+#      connect_port 443
+#    }
+#  }
+#}

make/ha/sample/active_standby/check_harbor.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+http_code = `curl -s -o /dev/null -w "%{http_code}" 127.0.0.1`
+if [ $http_code == 200 ] || [ $http_code == 301 ] ; then
+    exit 0
+else
+    exit 1
+fi

make/ha/sample/active_standby/keepalived_active_standby.conf

@@ -0,0 +1,36 @@
+global_defs {
+  router_id haborlb
+}
+vrrp_script check_harbor {
+  script "/usr/local/bin/check_harbor.sh"
+  interval 15
+  fail 5
+  rise 2
+}
+vrrp_sync_groups VG1 {
+  group {
+    VI_1
+  }
+}
+#Please change to ens160 to the interface name on you loadbalancer hosts.
+vrrp_instance VI_1 {
+  interface ens160
+
+  track_interface {
+    ens160
+  }
+
+  state MASTER
+  virtual_router_id 51
+  priority 10
+
+  virtual_ipaddress {
+    VIP/32
+  }
+  advert_int 1
+  authentication {
+    auth_type PASS
+    auth_pass d0cker
+  }
+
+}

make/harbor.cfg

@@ -116,6 +116,8 @@ db_port = 3306
 
 #The user name of mysql database
 db_user = root
+#The redis server address
+redis_url =
 #************************END INITIAL PROPERTIES************************
 #The following attributes only need to be set when auth mode is uaa_auth
 uaa_endpoint = uaa.mydomain.org

make/install.sh

@@ -58,7 +58,8 @@ item=0
 with_notary=$false
 # clair is not enabled by default
 with_clair=$false
-
+# HA mode is not enabled by default
+harbor_ha=$false
 while [ $# -gt 0 ]; do
         case $1 in
             --help)
@@ -67,7 +68,9 @@ while [ $# -gt 0 ]; do
             --with-notary)
             with_notary=true;;
             --with-clair)
-            with_clair=true;;	 					 	
+            with_clair=true;;
+            --ha)
+            harbor_ha=true;;
             *)
             note "$usage"
             exit 1;;
@@ -158,24 +161,28 @@ then
 	sed "s/^hostname = .*/hostname = $host/g" -i ./harbor.cfg
 fi
 prepare_para=
-if [ $with_notary ]
+if [ $with_notary ] && [ ! $harbor_ha ]
 then
 	prepare_para="${prepare_para} --with-notary"
 fi
-if [ $with_clair ]
+if [ $with_clair ] && [ ! $harbor_ha ]
 then
 	prepare_para="${prepare_para} --with-clair"
 fi
+if [ $harbor_ha ]
+then
+    prepare_para="${prepare_para} --ha"
+fi
 ./prepare $prepare_para
 echo ""
 
 h2 "[Step $item]: checking existing instance of Harbor ..."; let item+=1
 docker_compose_list='-f docker-compose.yml'
-if [ $with_notary ]
+if [ $with_notary ] && [ ! $harbor_ha ]
 then
 	docker_compose_list="${docker_compose_list} -f docker-compose.notary.yml"
 fi
-if [ $with_clair ]
+if [ $with_clair ] && [ ! $harbor_ha ]
 then
 	docker_compose_list="${docker_compose_list} -f docker-compose.clair.yml"
 fi
@@ -188,6 +195,11 @@ fi
 echo ""
 
 h2 "[Step $item]: starting Harbor ..."
+if [ $harbor_ha ]
+then
+    mv docker-compose.yml docker-compose.yml.bak 
+    cp ha/docker-compose.yml docker-compose.yml
+fi
 docker-compose $docker_compose_list up -d
 
 protocol=http

make/prepare

@@ -20,6 +20,27 @@ if sys.version_info[:3][0] == 3:
     import io as StringIO
 
 def validate(conf, args): 
+    if args.ha_mode:
+        db_host = rcp.get("configuration", "db_host")
+        if db_host == "mysql":
+            raise Exception("Error: In HA mode, db_host in harbor.cfg needs to point to an external DB address")
+        registry_config_path = os.path.join(templates_dir,"registry","config_ha.yml")
+        if check_storage_config(registry_config_path):
+            raise Exception("Error: In HA model shared storage configuration is required registry, refer HA installation guide for detail.")
+        redis_url = rcp.get("configuration", "redis_url")
+        if redis_url is None or len(redis_url) < 1:
+            raise Exception("Error: In HA mode redis is required redis_url need to point to an redis cluster")
+        if args.notary_mode or args.clair_mode:
+            raise Exception("Error: HA mode doesn't support clair and notary currently")
+        cert_path = rcp.get("configuration", "ssl_cert")
+        cert_key_path = rcp.get("configuration", "ssl_cert_key")
+        shared_cert_key = os.path.join(base_dir, "ha", os.path.basename(cert_key_path))
+        shared_cert_path = os.path.join(base_dir, "ha", os.path.basename(cert_path))
+        if os.path.isfile(shared_cert_key):
+            shutil.copy2(shared_cert_key, cert_key_path)
+        if os.path.isfile(shared_cert_path):
+            shutil.copy2(shared_cert_path, cert_path)
+
     protocol = rcp.get("configuration", "ui_url_protocol")
     if protocol != "https" and args.notary_mode:
         raise Exception("Error: the protocol must be https when Harbor is deployed with Notary")
@@ -39,6 +60,63 @@ def validate(conf, args):
     if project_creation != "everyone" and project_creation != "adminonly":
         raise Exception("Error invalid value for project_creation_restriction: %s" % project_creation)
 
+def prepare_ha(conf, args):
+    #files under ha folder will have high prority
+    protocol = rcp.get("configuration", "ui_url_protocol")
+    if protocol == "https":
+        #copy nginx certificate
+        cert_path = rcp.get("configuration", "ssl_cert")
+        cert_key_path = rcp.get("configuration", "ssl_cert_key")
+        shared_cert_key = os.path.join(base_dir, "ha", os.path.basename(cert_key_path))
+        shared_cert_path = os.path.join(base_dir, "ha", os.path.basename(cert_path))
+        if os.path.isfile(shared_cert_key):
+            shutil.copy2(shared_cert_key, cert_key_path)
+        else: 
+            if os.path.isfile(cert_key_path):
+                shutil.copy2(cert_key_path, shared_cert_key)
+        if os.path.isfile(shared_cert_path):
+            shutil.copy2(shared_cert_path, cert_path)
+        else:
+            if os.path.isfile(cert_path):
+                shutil.copy2(cert_path, shared_cert_path)
+        #check if ca exsit
+        cert_ca_path = "/data/ca_download/ca.crt"
+        shared_ca_path = os.path.join(base_dir, "ha", os.path.basename(cert_ca_path))
+        if os.path.isfile(shared_ca_path):
+            shutil.copy2(shared_ca_path, cert_ca_path)
+        else:
+            if os.path.isfile(cert_ca_path):
+                shutil.copy2(cert_ca_path, shared_ca_path)
+    #check root.crt and priviate_key.pem
+    private_key_pem = os.path.join(config_dir, "ui", "private_key.pem")
+    root_crt = os.path.join(config_dir, "registry", "root.crt")
+    shared_private_key_pem = os.path.join(base_dir, "ha", "private_key.pem")
+    shared_root_crt = os.path.join(base_dir, "ha", "root.crt")
+    if os.path.isfile(shared_private_key_pem):
+        shutil.copy2(shared_private_key_pem, private_key_pem)
+    else:
+        if os.path.isfile(private_key_pem):
+            shutil.copy2(private_key_pem, shared_private_key_pem)
+    if os.path.isfile(shared_root_crt):
+        shutil.copy2(shared_root_crt, root_crt)
+    else:
+        if os.path.isfile(root_crt):
+            shutil.copy2(root_crt, shared_root_crt)
+    #secretkey
+    shared_secret_key = os.path.join(base_dir, "ha", "secretkey")
+    secretkey_path = rcp.get("configuration", "secretkey_path") 
+    secret_key = os.path.join(secretkey_path, "secretkey")
+    if os.path.isfile(shared_secret_key):
+        shutil.copy2(shared_secret_key, secret_key)
+    else:
+        if os.path.isfile(secret_key):
+            shutil.copy2(secret_key, shared_secret_key)
+
+def check_storage_config(path):
+    if 'Place_holder_for_Storage_configureation' in open(path).read():
+        return True
+    return False
+
 def get_secret_key(path):
     secret_key = _get_secret(path, "secretkey") 
     if len(secret_key) != 16:
@@ -96,6 +174,7 @@ parser = argparse.ArgumentParser()
 parser.add_argument('--conf', dest='cfgfile', default=base_dir+'/harbor.cfg',type=str,help="the path of Harbor configuration file")
 parser.add_argument('--with-notary', dest='notary_mode', default=False, action='store_true', help="the Harbor instance is to be deployed with notary")
 parser.add_argument('--with-clair', dest='clair_mode', default=False, action='store_true', help="the Harbor instance is to be deployed with clair")
+parser.add_argument('--ha', dest='ha_mode', default=False, action='store_true', help="the Harbor instance is to be deployed in HA mode")
 args = parser.parse_args()
 
 delfile(config_dir)
@@ -106,7 +185,6 @@ conf.write(open(args.cfgfile).read())
 conf.seek(0, os.SEEK_SET)
 rcp = ConfigParser.RawConfigParser()
 rcp.readfp(conf)
-
 validate(rcp, args)
 
 hostname = rcp.get("configuration", "hostname")
@@ -141,8 +219,8 @@ ldap_scope = rcp.get("configuration", "ldap_scope")
 ldap_timeout = rcp.get("configuration", "ldap_timeout")
 db_password = rcp.get("configuration", "db_password")
 db_host = rcp.get("configuration", "db_host")
-db_port = rcp.get("configuration", "db_port")
 db_user = rcp.get("configuration", "db_user")
+db_port = rcp.get("configuration", "db_port")
 self_registration = rcp.get("configuration", "self_registration")
 if protocol == "https":
     cert_path = rcp.get("configuration", "ssl_cert")
@@ -161,9 +239,15 @@ uaa_endpoint = rcp.get("configuration", "uaa_endpoint")
 uaa_clientid = rcp.get("configuration", "uaa_clientid")
 uaa_clientsecret = rcp.get("configuration", "uaa_clientsecret")
 uaa_ca_root = rcp.get("configuration", "uaa_ca_root")
+
 secret_key = get_secret_key(secretkey_path)
 log_rotate_count = rcp.get("configuration", "log_rotate_count")
 log_rotate_size = rcp.get("configuration", "log_rotate_size")
+
+if rcp.has_option("configuration", "redis_url"):
+    redis_url = rcp.get("configuration", "redis_url")
+else:
+    redis_url = ""
 ########
 
 ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16))  
@@ -229,8 +313,8 @@ render(os.path.join(templates_dir, "adminserver", "env"),
         ldap_timeout=ldap_timeout,
         db_password=db_password,
         db_host=db_host,
-        db_port=db_port,
         db_user=db_user,
+        db_port=db_port,
         email_host=email_host,
         email_port=email_port,
         email_usr=email_usr,
@@ -258,12 +342,19 @@ render(os.path.join(templates_dir, "ui", "env"),
         ui_conf_env, 
         ui_secret=ui_secret,
         jobservice_secret=jobservice_secret,
+        redis_url = redis_url
         )
-
+if args.ha_mode:
-render(os.path.join(templates_dir, "registry", 
+    render(os.path.join(templates_dir, "registry",
-		"config.yml"),
+		"config_ha.yml"),
-        registry_conf,
+                registry_conf,
-        ui_url=ui_url)
+                ui_url=ui_url,
+                redis_url=redis_url)
+else:
+    render(os.path.join(templates_dir, "registry",
+                "config.yml"),
+                registry_conf,
+                ui_url=ui_url)
 
 render(os.path.join(templates_dir, "db", "env"),
         db_conf_env,
@@ -345,7 +436,7 @@ else:
     shutil.copyfile(os.path.join(templates_dir, "ui", "private_key.pem"), os.path.join(ui_config_dir, "private_key.pem"))
     print("Copied configuration file: %s" % registry_config_dir + "root.crt")
     shutil.copyfile(os.path.join(templates_dir, "registry", "root.crt"), os.path.join(registry_config_dir, "root.crt"))
-	
+
 if args.notary_mode:
     notary_config_dir = prep_conf_dir(config_dir, "notary")
     notary_temp_dir = os.path.join(templates_dir, "notary") 
@@ -413,6 +504,9 @@ if args.clair_mode:
     clair_conf = os.path.join(clair_config_dir, "config.yaml")
     render(os.path.join(clair_temp_dir, "config.yaml"), clair_conf, password = pg_password)
 
+if args.ha_mode:
+    prepare_ha(rcp, args)
+
 FNULL.close()
 print("The configuration files are ready, please use docker-compose to start the service.")