Enhance: refactor the mount dirs and workflow of generate cert

mount a temp dir input for all input files and configs
generated secrets file stored in data volumns keys dir
certs file stored in data volumns nginx dir

Signed-off-by: Qian Deng <dengq@vmware.com>

Makefile

@@ -150,7 +150,7 @@ MIGRATEPATCHBINARYNAME=migrate-patch
 
 # configfile
 CONFIGPATH=$(MAKEPATH)
-INSIDE_CONFIGPATH=/harbor_make
+INSIDE_CONFIGPATH=/compose_location
 CONFIGFILE=harbor.yml
 
 # prepare parameters

make/harbor.yml

@@ -48,32 +48,11 @@ log:
 #only take effect in the first boot, the subsequent changes of these properties 
 #should be performed on web ui
 
-#************************BEGIN INITIAL PROPERTIES************************
-
 ##The initial password of Harbor admin, only works for the first time when Harbor starts. 
 #It has no effect after the first launch of Harbor.
 #Change the admin password from UI after launching Harbor.
 harbor_admin_password: Harbor12345
 
-##By default the auth mode is db_auth, i.e. the credentials are stored in a local database.
-#Set it to ldap_auth if you want to verify a user's credentials against an LDAP server.
-auth_mode: db_auth
-
-#A user's DN who has the permission to search the LDAP/AD server. 
-
-#The base DN from which to look up a user in LDAP/AD
-ldap_basedn: ou=people,dc=mydomain,dc=com
-
-#The attribute used to name a LDAP/AD group, it could be cn, name
-ldap_group_gid: cn
-
-#The flag to control what users have permission to create projects
-#The default value "everyone" allows everyone to creates a project. 
-#Set to "adminonly" so that only admin user can create project.
-project_creation_restriction: everyone
-
-#************************END INITIAL PROPERTIES************************
-
 ## Harbor DB configuration
 database:
   #The address of the Harbor database. Only need to change when using external db.

make/photon/prepare/g.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 ## Const
 DEFAULT_UID = 10000
@@ -7,14 +8,18 @@ DEFAULT_GID = 10000
 ## Global variable
 base_dir = '/harbor_make'
 templates_dir = "/usr/src/app/templates"
-config_dir = os.path.join(base_dir, "common/config")
+config_dir = '/config'
-config_file_path = os.path.join(base_dir, 'harbor.yml')
 
-private_key_pem_template = os.path.join(templates_dir, "core", "private_key.pem")
+secret_dir = '/secret'
-root_cert_path_template = os.path.join(templates_dir, "registry", "root.crt")
+secret_key_dir='/secret/keys'
+
+private_key_pem_path = Path('/secret/core/private_key.pem')
+root_crt_path = Path('/secret/registry/root.crt')
+
+config_file_path = '/compose_location/harbor.yml'
 
 cert_dir = os.path.join(config_dir, "nginx", "cert")
 core_cert_dir = os.path.join(config_dir, "core", "certificates")
-private_key_pem = os.path.join(config_dir, "core", "private_key.pem")
+
-root_crt = os.path.join(config_dir, "registry", "root.crt")
+registry_custom_ca_bundle_storage_path = Path('/secret/common/custom-ca-bundle.crt')
-registry_custom_ca_bundle_config = os.path.join(config_dir, "custom-ca-bundle.crt")
+registry_custom_ca_bundle_storage_input_path = Path('/input/common/custom-ca-bundle.crt')

make/photon/prepare/main.py

@@ -2,7 +2,7 @@ import click
 
 from utils.misc import delfile
 from utils.configs import validate, parse_yaml_config
-from utils.cert import prepare_ca, SSL_CERT_KEY_PATH, SSL_CERT_PATH, get_secret_key
+from utils.cert import prepare_ca, SSL_CERT_KEY_PATH, SSL_CERT_PATH, get_secret_key, copy_ssl_cert, copy_secret_keys
 from utils.db import prepare_db
 from utils.jobservice import prepare_job_service
 from utils.registry import prepare_registry
@@ -15,8 +15,8 @@ from utils.clair import prepare_clair
 from utils.chart import prepare_chartmuseum
 from utils.docker_compose import prepare_docker_compose
 from utils.nginx import prepare_nginx, nginx_confd_dir
-from g import (config_dir, private_key_pem_template, config_file_path, core_cert_dir, private_key_pem,
+from g import (config_dir, config_file_path, core_cert_dir, private_key_pem_path, root_crt_path, 
-root_crt, root_cert_path_template, registry_custom_ca_bundle_config)
+registry_custom_ca_bundle_storage_path, registry_custom_ca_bundle_storage_input_path, secret_key_dir)
 
 # Main function
 @click.command()
@@ -38,19 +38,18 @@ def main(conf, with_notary, with_clair, with_chartmuseum):
     prepare_db(config_dict)
     prepare_job_service(config_dict)
 
-    get_secret_key(config_dict['secretkey_path'])
+    copy_secret_keys()
-    if config_dict['auth_mode'] == "uaa_auth":
+    get_secret_key(secret_key_dir)
-        prepare_uaa_cert_file(config_dict['uaa_ca_cert'], core_cert_dir)
+
+    if config_dict['protocol'] == 'https':
+        copy_ssl_cert()
 
     #  If Customized cert enabled
     prepare_ca(
-        customize_crt=config_dict['customize_crt'],
+        private_key_pem_path=private_key_pem_path,
-        private_key_pem_path=private_key_pem,
+        root_crt_path=root_crt_path,
-        private_key_pem_template=private_key_pem_template,
+        registry_custom_ca_bundle_config=registry_custom_ca_bundle_storage_input_path,
-        root_crt_path=root_crt,
+        registry_custom_ca_bundle_storage_path=registry_custom_ca_bundle_storage_path)
-        root_cert_template_path=root_cert_path_template,
-        registry_custom_ca_bundle_path=config_dict['registry_custom_ca_bundle_path'],
-        registry_custom_ca_bundle_config=registry_custom_ca_bundle_config)
 
     if with_notary:
         prepare_notary(config_dict, nginx_confd_dir, SSL_CERT_PATH, SSL_CERT_KEY_PATH)

make/photon/prepare/templates/core/config_env.jinja

@@ -1,7 +1,6 @@
 PORT=8080
 LOG_LEVEL=info
 EXT_ENDPOINT={{public_url}}
-AUTH_MODE={{auth_mode}}
 SELF_REGISTRATION={{self_registration}}
 LDAP_URL={{ldap_url}}
 LDAP_SEARCH_DN={{ldap_searchdn}}

make/photon/prepare/templates/core/private_key.pem

@@ -1,51 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIJKAIBAAKCAgEAtpMvyv153iSmwm6TrFpUOzsIGBEDbGtOOEZMEm08D8IC2n1G
-d6/XOZ5FxPAD6gIpE0EAcMojY5O0Hl4CDoyV3e/iKcBqFOgYtpogNtan7yT5J8gw
-KsPbU/8nBkK75GOq56nfvq4t9GVAclIDtHbuvmlh6O2n+fxtR0M9LbuotbSBdXYU
-hzXqiSsMclBvLyIk/z327VP5l0nUNOzPuKIwQjuxYKDkvq1oGy98oVlE6wl0ldh2
-ZYZLGAYbVhqBVUT1Un/PYqi9Nofa2RI5n1WOkUJQp87vb+PUPFhVOdvH/oAzV6/b
-9dzyhA5paDM06lj2gsg9hQWxCgbFh1x39c6pSI8hmVe6x2d4tAtSyOm3Qwz+zO2l
-bPDvkY8Svh5nxUYObrNreoO8wHr8MC6TGUQLnUt/RfdVKe5fYPFl6VYqJP/L3LDn
-Xj771nFq6PKiYbhBwJw3TM49gpKNS/Of70TP2m7nVlyuyMdE5T1j3xyXNkixXqqn
-JuSMqX/3Bmm0On9KEbemwn7KRYF/bqc50+RcGUdKNcOkN6vuMVZei4GbxALnVqac
-s+/UQAiQP4212UO7iZFwMaCNJ3r/b4GOlyalI1yEA4odoZov7k5zVOzHu8O6QmCj
-3R5TVOudpGiUh+lumRRpNqxDgjngLljvaWU6ttyIbjnAwCjnJoppZM2lkRkCAwEA
-AQKCAgAvsvCPlf2a3fR7Y6xNISRUfS22K+u7DaXX6fXB8qv4afWY45Xfex89vG35
-78L2Bi55C0h0LztjrpkmPeVHq88TtrJduhl88M5UFpxH93jUb9JwZErBQX4xyb2G
-UzUHjEqAT89W3+a9rR5TP74cDd59/MZJtp1mIF7keVqochi3sDsKVxkx4hIuWALe
-csk5hTApRyUWCBRzRCSe1yfF0wnMpA/JcP+SGXfTcmqbNNlelo/Q/kaga59+3UmT
-C0Wy41s8fIvP+MnGT2QLxkkrqYyfwrWTweqoTtuKEIHjpdnwUcoYJKfQ6jKp8aH0
-STyP5UIyFOKNuFjyh6ZfoPbuT1nGW+YKlUnK4hQ9N/GE0oMoecTaHTbqM+psQvbj
-6+CG/1ukA5ZTQyogNyuOApArFBQ+RRmVudPKA3JYygIhwctuB2oItsVEOEZMELCn
-g2aVFAVXGfGRDXvpa8oxs3Pc6RJEp/3tON6+w7cMCx0lwN/Jk2Ie6RgTzUycT3k6
-MoTQJRoO6/ZHcx3hTut/CfnrWiltyAUZOsefLuLg+Pwf9GHhOycLRI6gHfgSwdIV
-S77UbbELWdscVr1EoPIasUm1uYWBBcFRTturRW+GHJ8TZX+mcWSBcWwBhp15LjEl
-tJf+9U6lWMOSB2LvT+vFmR0M9q56fo7UeKFIR7mo7/GpiVu5AQKCAQEA6Qs7G9mw
-N/JZOSeQO6xIQakC+sKApPyXO58fa7WQzri+l2UrLNp0DEQfZCujqDgwys6OOzR/
-xg8ZKQWVoad08Ind3ZwoJgnLn6QLENOcE6PpWxA/JjnVGP4JrXCYR98cP0sf9jEI
-xkR1qT50GbeqU3RDFliI4kGRvbZ8cekzuWppfQcjstSBPdvuxqAcUVmTnTw83nvD
-FmBbhlLiEgI3iKtJ97UB7480ivnWnOuusduk7FO4jF3hkrOa+YRidinTCi8JBo0Y
-jx4Ci3Y5x6nvwkXhKzXapd7YmPNisUc5xA7/a+W71cyC0IKUwRc/8pYWLL3R3CpR
-YiV8gf6gwzOckQKCAQEAyI9CSNoAQH4zpS8B9PF8zILqEEuun8m1f5JB3hQnfWzm
-7uz/zg6I0TkcCE0AJVSKPHQm1V9+TRbF9+DiOWHEYYzPmK8h63SIufaWxZPqai4E
-PUj6eQWykBUVJ96n6/AW0JHRZ+WrJ5RXBqCLuY7NP6wDhORrCJjBwaGMohNpbKPS
-H3QewsoxCh+CEXKdKyy+/yU/f4E89PlHapkW1/bDJ5u7puSD+KvmiDDIXSBncdOO
-uFT8n+XH5IwgjdXFSDim15rQ8jD2l2xLcwKboTpx5GeRl8oB1VGm0fUbBn1dvGPG
-4WfHGyrp9VNZtP160WoHr+vRVPqvHNkoeAlCfEwQCQKCAQBN1dtzLN0HgqE8TrOE
-ysEDdTCykj4nXNoiJr522hi4gsndhQPLolb6NdKKQW0S5Vmekyi8K4e1nhtYMS5N
-5MFRCasZtmtOcR0af87WWucZRDjPmniNCunaxBZ1YFLsRl+H4E6Xir8UgY8O7PYY
-FNkFsKIrl3x4nU/RHl8oKKyG9Dyxbq4Er6dPAuMYYiezIAkGjjUCVjHNindnQM2T
-GDx2IEe/PSydV6ZD+LguhyU88FCAQmI0N7L8rZJIXmgIcWW0VAterceTHYHaFK2t
-u1uB9pcDOKSDnA+Z3kiLT2/CxQOYhQ2clgbnH4YRi/Nm0awsW2X5dATklAKm5GXL
-bLSRAoIBAQClaNnPQdTBXBR2IN3pSZ2XAkXPKMwdxvtk+phOc6raHA4eceLL7FrU
-y9gd1HvRTfcwws8gXcDKDYU62gNaNhMELWEt2QsNqS/2x7Qzwbms1sTyUpUZaSSL
-BohLOKyfv4ThgdIGcXoGi6Z2tcRnRqpq4BCK8uR/05TBgN5+8amaS0ZKYLfaCW4G
-nlPk1fVgHWhtAChtnYZLuKg494fKmB7+NMfAbmmVlxjrq+gkPkxyqXvk9Vrg+V8y
-VIuozu0Fkouv+GRpyw4ldtCHS1hV0eEK8ow2dwmqCMygDxm58X10mYn2b2PcOTl5
-9sNerUw1GNC8O66K+rGgBk4FKgXmg8kZAoIBABBcuisK250fXAfjAWXGqIMs2+Di
-vqAdT041SNZEOJSGNFsLJbhd/3TtCLf29PN/YXtnvBmC37rqryTsqjSbx/YT2Jbr
-Bk3jOr9JVbmcoSubXl8d/uzf7IGs91qaCgBwPZHgeH+kK13FCLexz+U9zYMZ78fF
-/yO82CpoekT+rcl1jzYn43b6gIklHABQU1uCD6MMyMhJ9Op2WmbDk3X+py359jMc
-+Cr2zfzdHAIVff2dOV3OL+ZHEWbwtnn3htKUdOmjoTJrciFx0xNZJS5Q7QYHMONj
-yPqbajyhopiN01aBQpCSGF1F1uRpWeIjTrAZPbrwLl9YSYXz0AT05QeFEFk=
------END RSA PRIVATE KEY-----

make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja

@@ -32,7 +32,10 @@ services:
     volumes:
       - {{data_volume}}/registry:/storage:z
       - ./common/config/registry/:/etc/registry/:z
-      - ./common/config/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z
+      - {{data_volume}}/secret/registry/root.crt:/etc/registry/root.crt:z
+{%if registry_custom_ca_bundle_storage_path %}
+      - {{data_volume}}/secret/common/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z
+{% endif %}
     networks:
       - harbor
 {% if with_clair %}
@@ -121,9 +124,9 @@ services:
       - SETUID
     volumes:
       - ./common/config/core/app.conf:/etc/core/app.conf:z
-      - ./common/config/core/private_key.pem:/etc/core/private_key.pem:z
       - ./common/config/core/certificates/:/etc/core/certificates/:z
-      - {{secretkey_path}}/secretkey:/etc/core/key:z
+      - {{data_volume}}/secret/core/private_key.pem:/etc/core/private_key.pem:z
+      - {{data_volume}}/secret/keys/secretkey:/etc/core/key:z
       - {{data_volume}}/ca_download/:/etc/core/ca/:z
       - {{data_volume}}/psc/:/etc/core/token/:z
       - {{data_volume}}/:/data/:z
@@ -243,8 +246,10 @@ services:
       - NET_BIND_SERVICE
     volumes:
       - ./common/config/nginx:/etc/nginx:z
-      - {{cert_key_path}}:/etc/nginx/cert/server.key
+{% if protocol == 'https' %}
-      - {{cert_path}}:/etc/nginx/cert/server.crt
+      - {{data_volume}}/secret/nginx/server.key:/etc/nginx/cert/server.key
+      - {{data_volume}}/secret/nginx/server.crt:/etc/nginx/cert/server.crt
+{% endif %}
     networks:
       - harbor
 {% if with_notary %}
@@ -328,7 +333,9 @@ services:
       - postgresql
     volumes:
       - ./common/config/clair/config.yaml:/etc/clair/config.yaml:z
-      - ./common/config/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z
+{%if registry_custom_ca_bundle_storage_path %}
+      - {{data_volume}}/secret/common/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z
+{% endif %}
     logging:
       driver: "syslog"
       options:  
@@ -357,7 +364,9 @@ services:
     volumes:
       - {{data_volume}}/chart_storage:/chart_storage:z
       - ./common/config/chartserver:/etc/chartserver:z
-      - ./common/config/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z
+{%if registry_custom_ca_bundle_storage_path %}
+      - {{data_volume}}/secret/common/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z
+{% endif %}
     logging:
       driver: "syslog"
       options:  

make/photon/prepare/templates/registry/root.crt

@@ -1,35 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIGBzCCA++gAwIBAgIJAKB8CNqCxhr7MA0GCSqGSIb3DQEBCwUAMIGZMQswCQYD
-VQQGEwJDTjEOMAwGA1UECAwFU3RhdGUxCzAJBgNVBAcMAkNOMRUwEwYDVQQKDAxv
-cmdhbml6YXRpb24xHDAaBgNVBAsME29yZ2FuaXphdGlvbmFsIHVuaXQxFDASBgNV
-BAMMC2V4YW1wbGUuY29tMSIwIAYJKoZIhvcNAQkBFhNleGFtcGxlQGV4YW1wbGUu
-Y29tMB4XDTE2MDUxNjAyNDY1NVoXDTI2MDUxNDAyNDY1NVowgZkxCzAJBgNVBAYT
-AkNOMQ4wDAYDVQQIDAVTdGF0ZTELMAkGA1UEBwwCQ04xFTATBgNVBAoMDG9yZ2Fu
-aXphdGlvbjEcMBoGA1UECwwTb3JnYW5pemF0aW9uYWwgdW5pdDEUMBIGA1UEAwwL
-ZXhhbXBsZS5jb20xIjAgBgkqhkiG9w0BCQEWE2V4YW1wbGVAZXhhbXBsZS5jb20w
-ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC2ky/K/XneJKbCbpOsWlQ7
-OwgYEQNsa044RkwSbTwPwgLafUZ3r9c5nkXE8APqAikTQQBwyiNjk7QeXgIOjJXd
-7+IpwGoU6Bi2miA21qfvJPknyDAqw9tT/ycGQrvkY6rnqd++ri30ZUByUgO0du6+
-aWHo7af5/G1HQz0tu6i1tIF1dhSHNeqJKwxyUG8vIiT/PfbtU/mXSdQ07M+4ojBC
-O7FgoOS+rWgbL3yhWUTrCXSV2HZlhksYBhtWGoFVRPVSf89iqL02h9rZEjmfVY6R
-QlCnzu9v49Q8WFU528f+gDNXr9v13PKEDmloMzTqWPaCyD2FBbEKBsWHXHf1zqlI
-jyGZV7rHZ3i0C1LI6bdDDP7M7aVs8O+RjxK+HmfFRg5us2t6g7zAevwwLpMZRAud
-S39F91Up7l9g8WXpViok/8vcsOdePvvWcWro8qJhuEHAnDdMzj2Cko1L85/vRM/a
-budWXK7Ix0TlPWPfHJc2SLFeqqcm5Iypf/cGabQ6f0oRt6bCfspFgX9upznT5FwZ
-R0o1w6Q3q+4xVl6LgZvEAudWppyz79RACJA/jbXZQ7uJkXAxoI0nev9vgY6XJqUj
-XIQDih2hmi/uTnNU7Me7w7pCYKPdHlNU652kaJSH6W6ZFGk2rEOCOeAuWO9pZTq2
-3IhuOcDAKOcmimlkzaWRGQIDAQABo1AwTjAdBgNVHQ4EFgQUPJF++WMsv1OJvf7F
-oCew37JTnfQwHwYDVR0jBBgwFoAUPJF++WMsv1OJvf7FoCew37JTnfQwDAYDVR0T
-BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAb5LvqukMxWd5Zajbh3orfYsXmhWn
-UWiwG176+bd3b5xMlG9iLd4vQ11lTZoIhFOfprRQzbizQ8BzR2JBQckpLcy+5hyA
-D3M9vLL37OwA0wT6kxFnd6LtlFaH5gG++huw2ts2PDXFz0jqw+0YE/R8ov2+YdaZ
-aPSEMunmAuEY1TbYWzz4u6PxycxhQzDQ34ZmJZ34Elvw1NYMfPMGTKp34PsxIcgT
-ao5jqb9RMU6JAumfXrOvXRjjl573vX2hgMZzEU6OF2/+uyg95chn6nO1GUQrT2+F
-/1xIqfHfFCm8+jujSDgqfBtGI+2C7No+Dq8LEyEINZe6wSQ81+ryt5jy5SZmAsnj
-V4OsSIwlpR5fLUwrFStVoUWHEKl1DflkYki/cAC1TL0Om+ldJ219kcOnaXDNaq66
-3I75BvRY7/88MYLl4Fgt7sn05Mn3uNPrCrci8d0R1tlXIcwMdCowIHeZdWHX43f7
-NsVk/7VSOxJ343csgaQc+3WxEFK0tBxGO6GP+Xj0XmdVGLhalVBsEhPjnmx+Yyrn
-oMsTA1Yrs88C8ItQn7zuO/30eKNGTnby0gptHiS6sa/c3O083Mpi8y33GPVZDvBl
-l9PfSZT8LG7SvpjsdgdNZlyFvTY4vsB+Vd5Howh7gXYPVXdCs4k7HMyo7zvzliZS
-ekCw9NGLoNqQqnA=
------END CERTIFICATE-----

make/photon/prepare/utils/cert.py

@@ -1,5 +1,6 @@
 # Get or generate private key
 import os, sys, subprocess, shutil
+from pathlib import Path
 from subprocess import DEVNULL
 from functools import wraps
 
@@ -9,6 +10,17 @@ from .misc import generate_random_string
 SSL_CERT_PATH = os.path.join("/etc/nginx/cert", "server.crt")
 SSL_CERT_KEY_PATH = os.path.join("/etc/nginx/cert", "server.key")
 
+input_cert = '/input/nginx/server.crt'
+input_cert_key = '/input/nginx/server.key'
+
+secret_cert_dir = '/secret/nginx'
+secret_cert = '/secret/nginx/server.crt'
+secret_cert_key = '/secret/nginx/server.key'
+
+input_secret_keys_dir = '/input/keys'
+secret_keys_dir = '/secret/keys'
+allowed_secret_key_names = ['defaultalias', 'secretkey']
+
 def _get_secret(folder, filename, length=16):
     key_file = os.path.join(folder, filename)
     if os.path.isfile(key_file):
@@ -38,6 +50,20 @@ def get_alias(path):
     alias = _get_secret(path, "defaultalias", length=8)
     return alias
 
+def copy_secret_keys():
+    if os.path.isdir(secret_cert) and os.path.isdir(input_secret_keys_dir):
+        input_files = os.listdir(input_secret_keys_dir)
+        secret_files = os.listdir(secret_keys_dir)
+        files_need_copy =  [x for x in input_files if (x in allowed_secret_key_names) and (x not in secret_files) ]
+        for f in files_need_copy:
+            shutil.copy(f, secret_keys_dir)
+
+def copy_ssl_cert():
+    if os.path.isfile(input_cert_key) and os.path.isfile(input_cert):
+        os.makedirs(secret_cert_dir, exist_ok=True)
+        shutil.copy(input_cert, secret_cert)
+        shutil.copy(input_cert_key, secret_cert_key)
+
 ## decorator actions
 def stat_decorator(func):
     @wraps(func)
@@ -80,22 +106,23 @@ def openssl_installed():
 
 
 def prepare_ca(
-    customize_crt,
+    private_key_pem_path: Path,
-    private_key_pem_path, private_key_pem_template,
+    root_crt_path: Path,
-    root_crt_path, root_cert_template_path,
+    registry_custom_ca_bundle_config: Path,
-    registry_custom_ca_bundle_path, registry_custom_ca_bundle_config):
+    registry_custom_ca_bundle_storage_path: Path):
+
+    if not (private_key_pem_path.exists() and root_crt_path.exists()):
+
+        private_key_pem_path.parent.mkdir(parents=True, exist_ok=True)
+        root_crt_path.parent.mkdir(parents=True, exist_ok=True)
 
-    if (customize_crt == 'on' or customize_crt == True) and openssl_installed():
         empty_subj = "/"
         create_root_cert(empty_subj, key_path=private_key_pem_path, cert_path=root_crt_path)
         mark_file(private_key_pem_path)
         mark_file(root_crt_path)
-    else:
-        print("Copied configuration file: %s" % private_key_pem_path)
-        shutil.copyfile(private_key_pem_template, private_key_pem_path)
-        print("Copied configuration file: %s" % root_crt_path)
-        shutil.copyfile(root_cert_template_path, root_crt_path)
 
-    if len(registry_custom_ca_bundle_path) > 0 and os.path.isfile(registry_custom_ca_bundle_path):
+    if not registry_custom_ca_bundle_storage_path.exists() and registry_custom_ca_bundle_config.exists():
-        shutil.copyfile(registry_custom_ca_bundle_path, registry_custom_ca_bundle_config)
+        registry_custom_ca_bundle_storage_path.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(registry_custom_ca_bundle_config, registry_custom_ca_bundle_storage_path)
+        mark_file(registry_custom_ca_bundle_storage_path)
         print("Copied custom ca bundle: %s" % registry_custom_ca_bundle_config)

make/photon/prepare/utils/configs.py

@@ -7,17 +7,11 @@ def validate(conf, **kwargs):
         raise Exception(
             "Error: the protocol must be https when Harbor is deployed with Notary")
     if protocol == "https":
-        if not conf.get("cert_path"): ## ssl_path in config
+        if not conf.get("cert_path"):
             raise Exception("Error: The protocol is https but attribute ssl_cert is not set")
         if not conf.get("cert_key_path"):
             raise Exception("Error: The protocol is https but attribute ssl_cert_key is not set")
 
-    # Project validate
-    project_creation = conf.get("project_creation_restriction")
-    if project_creation != "everyone" and project_creation != "adminonly":
-        raise Exception(
-            "Error invalid value for project_creation_restriction: %s" % project_creation)
-
     # Storage validate
     valid_storage_drivers = ["filesystem", "azure", "gcs", "s3", "swift", "oss"]
     storage_provider_name = conf.get("storage_provider_name")

make/photon/prepare/utils/docker_compose.py

@@ -1,6 +1,6 @@
 import os
 
-from g import base_dir, templates_dir
+from g import templates_dir
 from .jinja import render_jinja
 
 
@@ -17,7 +17,7 @@ NGINX_VERSION = VERSION_TAG
 # version of chartmuseum
 
 docker_compose_template_path = os.path.join(templates_dir, 'docker_compose', 'docker-compose.yml.jinja')
-docker_compose_yml_path = os.path.join(base_dir, 'docker-compose.yml')
+docker_compose_yml_path = '/compose_location/docker-compose.yml'
 
 def check_configs(configs):
     pass
@@ -36,10 +36,11 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):
         'log_location': configs['log_location'],
         'cert_key_path': configs['cert_key_path'],
         'cert_path': configs['cert_path'],
+        'protocol': configs['protocol'],
+        'registry_custom_ca_bundle_storage_path': configs['registry_custom_ca_bundle_path'],
         'with_notary': with_notary,
         'with_clair': with_clair,
         'with_chartmuseum': with_chartmuseum
     }
-    rendering_variables['secretkey_path'] = configs['secretkey_path']
 
     render_jinja(docker_compose_template_path, docker_compose_yml_path, **rendering_variables)

make/photon/prepare/utils/notary.py

@@ -1,5 +1,5 @@
 import os, shutil
-from g import base_dir, templates_dir, config_dir, root_crt, DEFAULT_UID, DEFAULT_GID
+from g import base_dir, templates_dir, config_dir, root_crt_path, secret_key_dir,DEFAULT_UID, DEFAULT_GID
 from .cert import openssl_installed, create_cert, create_root_cert, get_alias
 from .jinja import render_jinja
 from .misc import mark_file, prepare_config_dir
@@ -23,7 +23,7 @@ def prepare_env_notary(customize_crt, nginx_config_dir):
     notary_config_dir = prepare_config_dir(config_dir, "notary")
     if (customize_crt == 'on' or customize_crt == True)  and openssl_installed():
         try:
-            temp_cert_dir = os.path.join(base_dir, "cert_tmp")
+            temp_cert_dir = os.path.join('/tmp', "cert_tmp")
             if not os.path.exists(temp_cert_dir):
                 os.makedirs(temp_cert_dir)
             ca_subj = "/C=US/ST=California/L=Palo Alto/O=GoHarbor/OU=Harbor/CN=Self-signed by GoHarbor"
@@ -50,7 +50,7 @@ def prepare_env_notary(customize_crt, nginx_config_dir):
         shutil.copy2(os.path.join(notary_template_dir, "notary-signer.key"), notary_config_dir)
         shutil.copy2(os.path.join(notary_template_dir, "notary-signer-ca.crt"), notary_config_dir)
 
-    shutil.copy2(root_crt, notary_config_dir)
+    shutil.copy2(root_crt_path, notary_config_dir)
     shutil.copy2(
         os.path.join(notary_template_dir, "server_env.jinja"),
         os.path.join(notary_config_dir, "server_env"))
@@ -95,7 +95,7 @@ def prepare_notary(config_dict, nginx_config_dir, ssl_cert_path, ssl_cert_key_pa
         ssl_cert=ssl_cert_path,
         ssl_cert_key=ssl_cert_key_path)
 
-    default_alias = get_alias(config_dict['secretkey_path'])
+    default_alias = get_alias(secret_key_dir)
     render_jinja(
         notary_signer_env_template,
         notary_signer_env_path,

make/prepare

@@ -1,10 +1,49 @@
 #!/bin/bash
 
-host_make_path="$( cd "$(dirname "$0")" ; pwd -P )"
+# If compling source code this dir is harbor's make dir
+# If install harbor via pacakge, this dir is harbor's root dir
+harbor_prepare_path="$( cd "$(dirname "$0")" ; pwd -P )"
 
-echo host make path is set to ${host_make_path}
+echo host make path is set to ${harbor_prepare_path}
-data_path=$(grep '^[^#]*data_volume' ${host_make_path}/harbor.yml | awk '{print $NF}')
+data_path=$(grep '^[^#]*data_volume:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}')
-log_path=$(grep '^[^#]*location' ${host_make_path}/harbor.yml | awk '{print $NF}')
+log_path=$(grep '^[^#]*location:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}')
-secretkey_path=$(grep '^[^#]*secretkey_path' ${host_make_path}/harbor.yml | awk '{print $NF}')
+secretkey_path=$(grep '^[^#]*secretkey_path:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}')
+ssl_cert_path=$(grep '^[^#]*ssl_cert:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}')
+ssl_cert_key_path=$(grep '^[^#]*ssl_cert_key:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}')
+registry_custom_ca_bundle=$(grep '^[^#]*registry_custom_ca_bundle:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}')
+# Create a input dirs
+mkdir -p ${harbor_prepare_path}/input
+input_dir=${harbor_prepare_path}/input
+mkdir -p $input_dir/nginx
+mkdir -p $input_dir/keys
+mkdir -p $input_dir/common
 
-docker run -it --rm -v ${host_make_path}:/harbor_make -v $data_path:/data -v $log_path:/var/log/harbor -v $secretkey_path:$secretkey_path goharbor/prepare:1.7.1 $@
+# Create secret dir
+secret_dir=${data_path}/secret
+config_dir=$harbor_prepare_path/common/config
+
+# Copy nginx config file to input dir
+cp $ssl_cert_path $input_dir/nginx/server.crt
+cp $ssl_cert_key_path $input_dir/nginx/server.key
+
+# Copy secretkey to input dir
+cp -r $secretkey_path $input_dir/keys
+
+# Copy ca bundle to input dir
+if [ -f $registry_custom_ca_bundle ]
+then
+    cp -r $registry_custom_ca_bundle $input_dir/common/custom-ca-bundle.crt
+fi
+
+# Copy harbor.yml to input dir
+cp ${harbor_prepare_path}/harbor.yml $input_dir/harbor.yml
+
+docker run -it --rm -v $input_dir:/input \
+                    -v $harbor_prepare_path:/compose_location \
+                    -v $config_dir:/config \
+                    -v $secret_dir:/secret \
+                    -v $log_path:/var/log/harbor \
+                    goharbor/prepare:1.7.1 $@
+
+# Clean up input dir
+rm -rf ${harbor_prepare_path}/input

tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl

@@ -48,29 +48,10 @@ log:
 #only take effect in the first boot, the subsequent changes of these properties 
 #should be performed on web ui
 
-#************************BEGIN INITIAL PROPERTIES************************
+##The initial password of Harbor admin, only works for the first time when Harbor starts.
-
+#It has no effect after the first launch of Harbor.
-##By default the auth mode is db_auth, i.e. the credentials are stored in a local database.
+#Change the admin password from UI after launching Harbor.
-#Set it to ldap_auth if you want to verify a user's credentials against an LDAP server.
+harbor_admin_password: Harbor12345
-auth_mode: db_auth
-
-#The base DN from which to look up a user in LDAP/AD
-ldap_basedn: ou=people,dc=mydomain,dc=com
-
-#The attribute used to name a LDAP/AD group, it could be cn, name
-ldap_group_gid: cn
-
-#The following attributes only need to be set when auth mode is uaa_auth
-uaa_ca_cert: $uaa_ca_cert
-
-#The flag to control what users have permission to create projects
-#The default value "everyone" allows everyone to creates a project. 
-#Set to "adminonly" so that only admin user can create project.
-project_creation_restriction: everyone
-
-#************************END INITIAL PROPERTIES************************
-
-
 
 database:
   #The address of the Harbor database. Only need to change when using external db.

tools/migration/migrator.py

@@ -71,7 +71,7 @@ class CfgMigrator():
             copyfile(self.cfg_path, self.backup_path+"/harbor.cfg")
             print ("Success to backup harbor.cfg.")
             return True
-        except Exception, e:
+        except Exception as e:
             print ("Back up error: %s" % str(e))
             return False 
 
@@ -83,7 +83,7 @@ class CfgMigrator():
             copyfile(self.backup_path+"/harbor.cfg", self.cfg_path)
             print ("Success to restore harbor.cfg.")
             return True
-        except Exception, e:
+        except Exception as e:
             print ("Restore error: %s" % str(e))
             return False