From f1f78a56499feec5cedc0117abce39dbb45d9d8a Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 12 Jan 2017 18:29:02 +0800 Subject: [PATCH] update --- .travis.yml | 11 +- make/common/templates/adminserver/env | 4 +- make/common/templates/jobservice/env | 3 +- make/harbor.cfg | 2 +- make/prepare | 80 +++---- src/adminserver/api/base.go | 5 + src/adminserver/api/cfg.go | 100 ++++++--- .../systemcfg/{ => store}/driver.go | 2 +- .../systemcfg/{ => store/json}/driver_json.go | 10 +- .../{ => store/json}/driver_json_test.go | 14 +- src/adminserver/systemcfg/systemcfg.go | 153 ++++++------- src/adminserver/systemcfg/systemcfg_test.go | 71 ++++++ src/common/api/base_test.go | 2 + src/common/config/config.go | 191 ++++++++++++---- src/common/config/config_test.go | 96 --------- src/common/dao/dao_test.go | 2 +- src/common/models/config.go | 8 +- src/common/utils/email/mail.go | 2 +- src/common/utils/test/adminserver.go | 59 +++++ src/common/utils/test/test.go | 8 +- src/common/utils/utils.go | 5 +- src/common/utils/utils_test.go | 10 + src/jobservice/config/config.go | 76 +++---- src/jobservice/config/config_test.go | 66 +++++- src/jobservice/main.go | 4 +- src/ui/api/config.go | 122 ++++++----- src/ui/api/harborapi_test.go | 16 +- src/ui/api/target_test.go | 9 +- src/ui/config/config.go | 67 +++--- src/ui/config/config_test.go | 203 +++++++----------- src/ui/controllers/base.go | 6 +- src/ui/controllers/controllers_test.go | 6 +- src/ui/ui_test.go | 2 + tests/docker-compose.test.yml | 11 + tests/startuptest.go | 57 ++--- tests/testprepare.sh | 6 +- 36 files changed, 867 insertions(+), 622 deletions(-) rename src/adminserver/systemcfg/{ => store}/driver.go (98%) rename src/adminserver/systemcfg/{ => store/json}/driver_json.go (90%) rename src/adminserver/systemcfg/{ => store/json}/driver_json_test.go (84%) create mode 100644 src/adminserver/systemcfg/systemcfg_test.go create mode 100644 src/common/utils/test/adminserver.go diff --git a/.travis.yml b/.travis.yml index a80c95942..7337b0bdd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,14 +13,13 @@ services: dist: trusty env: - DB_HOST: 127.0.0.1 - DB_PORT: 3306 - DB_USR: root - DB_PWD: root123 MYSQL_HOST: localhost MYSQL_PORT: 3306 MYSQL_USR: root MYSQL_PWD: root123 + MYSQL_DATABASE: registry + SQLITE_FILE: /tmp/registry.db + ADMIN_SERVER_URL: http://127.0.0.1:8888 DOCKER_COMPOSE_VERSION: 1.7.1 HARBOR_ADMIN: admin HARBOR_ADMIN_PASSWD: Harbor12345 @@ -70,7 +69,7 @@ install: before_script: # create tables and load data # - mysql < ./make/db/registry.sql -uroot --verbose - - sudo sqlite3 /registry.db < make/common/db/registry_sqlite.sql + - sudo sqlite3 /tmp/registry.db < make/common/db/registry_sqlite.sql script: - sudo mkdir -p /harbor_storage/ca_download @@ -88,7 +87,7 @@ script: - goveralls -coverprofile=profile.cov -service=travis-ci - docker-compose -f make/docker-compose.test.yml down - + - sudo make/prepare - docker-compose -f make/dev/docker-compose.yml up -d - docker ps diff --git a/make/common/templates/adminserver/env b/make/common/templates/adminserver/env index 959082ce3..ff0153215 100644 --- a/make/common/templates/adminserver/env +++ b/make/common/templates/adminserver/env @@ -22,7 +22,7 @@ EMAIL_HOST=$email_host EMAIL_PORT=$email_port EMAIL_USR=$email_usr EMAIL_PWD=$email_pwd -EMAIL_TLS=$email_tls +EMAIL_SSL=$email_ssl EMAIL_FROM=$email_from EMAIL_IDENTITY=$email_identity HARBOR_ADMIN_PASSWORD=$harbor_admin_password @@ -33,6 +33,6 @@ LOG_DIR=/var/log/jobs UI_SECRET=$ui_secret SECRET_KEY=$secret_key TOKEN_EXPIRATION=$token_expiration -CFG_EXPIRATION=$cfg_expiration +CFG_EXPIRATION=5 USE_COMPRESSED_JS=$use_compressed_js GODEBUG=netdns=cgo \ No newline at end of file diff --git a/make/common/templates/jobservice/env b/make/common/templates/jobservice/env index d75972ac3..06c8b0f22 100644 --- a/make/common/templates/jobservice/env +++ b/make/common/templates/jobservice/env @@ -1,5 +1,4 @@ LOG_LEVEL=debug -UI_SECRET=$ui_secret CONFIG_PATH=/etc/jobservice/app.conf -MAX_JOB_WORKERS=$max_job_workers +UI_SECRET=$ui_secret GODEBUG=netdns=cgo diff --git a/make/harbor.cfg b/make/harbor.cfg index 69118dbf3..507533edb 100644 --- a/make/harbor.cfg +++ b/make/harbor.cfg @@ -53,7 +53,7 @@ ldap_uid = uid ldap_scope = 3 #Timeout (in seconds) when connecting to an LDAP Server. The default value (and most reasonable) is 5 seconds. -ldap_connect_timeout = 5 +ldap_timeout = 5 #The password for the root user of mysql db, change this before any production use. db_password = root123 diff --git a/make/prepare b/make/prepare index 2610d5fa2..28d46a376 100755 --- a/make/prepare +++ b/make/prepare @@ -77,10 +77,10 @@ hostname = rcp.get("configuration", "hostname") protocol = rcp.get("configuration", "ui_url_protocol") ui_url = protocol + "://" + hostname email_identity = rcp.get("configuration", "email_identity") -email_server = rcp.get("configuration", "email_server") -email_server_port = rcp.get("configuration", "email_server_port") -email_username = rcp.get("configuration", "email_username") -email_password = rcp.get("configuration", "email_password") +email_host = rcp.get("configuration", "email_server") +email_port = rcp.get("configuration", "email_server_port") +email_usr = rcp.get("configuration", "email_username") +email_pwd = rcp.get("configuration", "email_password") email_from = rcp.get("configuration", "email_from") email_ssl = rcp.get("configuration", "email_ssl") harbor_admin_password = rcp.get("configuration", "harbor_admin_password") @@ -101,7 +101,7 @@ else: ldap_filter = "" ldap_uid = rcp.get("configuration", "ldap_uid") ldap_scope = rcp.get("configuration", "ldap_scope") -ldap_connect_timeout = rcp.get("configuration", "ldap_connect_timeout") +ldap_timeout = rcp.get("configuration", "ldap_timeout") db_password = rcp.get("configuration", "db_password") self_registration = rcp.get("configuration", "self_registration") use_compressed_js = rcp.get("configuration", "use_compressed_js") @@ -126,6 +126,10 @@ secret_key = get_secret_key(secretkey_path) ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16)) +adminserver_config_dir = os.path.join(config_dir,"adminserver") +if not os.path.exists(adminserver_config_dir): + os.makedirs(os.path.join(config_dir, "adminserver")) + ui_config_dir = os.path.join(config_dir,"ui") if not os.path.exists(ui_config_dir): os.makedirs(os.path.join(config_dir, "ui")) @@ -152,6 +156,7 @@ def render(src, dest, **kw): f.write(t.substitute(**kw)) print("Generated configuration file: %s" % dest) +adminserver_conf_env = os.path.join(config_dir, "adminserver", "env") ui_conf_env = os.path.join(config_dir, "ui", "env") ui_conf = os.path.join(config_dir, "ui", "app.conf") jobservice_conf = os.path.join(config_dir, "jobservice", "app.conf") @@ -187,14 +192,12 @@ if protocol == "https": else: render(os.path.join(templates_dir, "nginx", "nginx.http.conf"), nginx_conf) - -render(os.path.join(templates_dir, "ui", "env"), - ui_conf_env, - hostname=hostname, - db_password=db_password, - ui_url=ui_url, - auth_mode=auth_mode, - harbor_admin_password=harbor_admin_password, + +render(os.path.join(templates_dir, "adminserver", "env"), + adminserver_conf_env, + ui_url=ui_url, + auth_mode=auth_mode, + self_registration=self_registration, ldap_url=ldap_url, ldap_searchdn =ldap_searchdn, ldap_search_pwd =ldap_search_pwd, @@ -202,27 +205,31 @@ render(os.path.join(templates_dir, "ui", "env"), ldap_filter=ldap_filter, ldap_uid=ldap_uid, ldap_scope=ldap_scope, - ldap_connect_timeout=ldap_connect_timeout, - self_registration=self_registration, - use_compressed_js=use_compressed_js, - ui_secret=ui_secret, + ldap_timeout=ldap_timeout, + db_password=db_password, + email_host=email_host, + email_port=email_port, + email_usr=email_usr, + email_pwd=email_pwd, + email_ssl=email_ssl, + email_from=email_from, + email_identity=email_identity, + harbor_admin_password=harbor_admin_password, + project_creation_restriction=proj_cre_restriction, + verify_remote_cert=verify_remote_cert, + max_job_workers=max_job_workers, + ui_secret=ui_secret, secret_key=secret_key, - verify_remote_cert=verify_remote_cert, - project_creation_restriction=proj_cre_restriction, - token_expiration=token_expiration) + token_expiration=token_expiration, + use_compressed_js=use_compressed_js + ) -render(os.path.join(templates_dir, "ui", "app.conf"), - ui_conf, - email_identity=email_identity, - email_server=email_server, - email_server_port=email_server_port, - email_username=email_username, - email_password=email_password, - email_from=email_from, - email_ssl=email_ssl, - ui_url=ui_url) +render(os.path.join(templates_dir, "ui", "env"), + ui_conf_env, + ui_secret=ui_secret) -render(os.path.join(templates_dir, "registry", "config.yml"), +render(os.path.join(templates_dir, "registry", + "config.yml"), registry_conf, ui_url=ui_url) @@ -232,16 +239,15 @@ render(os.path.join(templates_dir, "db", "env"), render(os.path.join(templates_dir, "jobservice", "env"), job_conf_env, - db_password=db_password, - ui_secret=ui_secret, - max_job_workers=max_job_workers, - secret_key=secret_key, - ui_url=ui_url, - verify_remote_cert=verify_remote_cert) + ui_secret=ui_secret) print("Generated configuration file: %s" % jobservice_conf) shutil.copyfile(os.path.join(templates_dir, "jobservice", "app.conf"), jobservice_conf) +print("Generated configuration file: %s" % ui_conf) +shutil.copyfile(os.path.join(templates_dir, "ui", "app.conf"), ui_conf) + + def validate_crt_subj(dirty_subj): subj_list = [item for item in dirty_subj.strip().split("/") \ if len(item.split("=")) == 2 and len(item.split("=")[1]) > 0] diff --git a/src/adminserver/api/base.go b/src/adminserver/api/base.go index 47938c8b7..e4221bb64 100644 --- a/src/adminserver/api/base.go +++ b/src/adminserver/api/base.go @@ -27,3 +27,8 @@ func handleInternalServerError(w http.ResponseWriter) { func handleBadRequestError(w http.ResponseWriter, error string) { http.Error(w, error, http.StatusBadRequest) } + +func handleUnauthorized(w http.ResponseWriter) { + http.Error(w, http.StatusText(http.StatusUnauthorized), + http.StatusUnauthorized) +} diff --git a/src/adminserver/api/cfg.go b/src/adminserver/api/cfg.go index 09499fec6..e2a41412b 100644 --- a/src/adminserver/api/cfg.go +++ b/src/adminserver/api/cfg.go @@ -19,6 +19,7 @@ import ( "encoding/json" "io/ioutil" "net/http" + "os" "strconv" cfg "github.com/vmware/harbor/src/adminserver/systemcfg" @@ -27,8 +28,32 @@ import ( "github.com/vmware/harbor/src/common/utils/log" ) +func isAuthenticated(r *http.Request) (bool, error) { + secret := os.Getenv("UI_SECRET") + c, err := r.Cookie("secret") + if err != nil { + if err == http.ErrNoCookie { + return false, nil + } + return false, err + } + return c != nil && c.Value == secret, nil +} + // ListCfgs lists configurations func ListCfgs(w http.ResponseWriter, r *http.Request) { + authenticated, err := isAuthenticated(r) + if err != nil { + log.Errorf("failed to check whether the request is authenticated or not: %v", err) + handleInternalServerError(w) + return + } + + if !authenticated { + handleUnauthorized(w) + return + } + cfg, err := cfg.GetSystemCfg() if err != nil { log.Errorf("failed to get system configurations: %v", err) @@ -49,6 +74,18 @@ func ListCfgs(w http.ResponseWriter, r *http.Request) { // UpdateCfgs updates configurations func UpdateCfgs(w http.ResponseWriter, r *http.Request) { + authenticated, err := isAuthenticated(r) + if err != nil { + log.Errorf("failed to check whether the request is authenticated or not: %v", err) + handleInternalServerError(w) + return + } + + if !authenticated { + handleUnauthorized(w) + return + } + b, err := ioutil.ReadAll(r.Body) if err != nil { log.Errorf("failed to read request body: %v", err) @@ -62,8 +99,6 @@ func UpdateCfgs(w http.ResponseWriter, r *http.Request) { return } - log.Info(m) - system, err := cfg.GetSystemCfg() if err != nil { handleInternalServerError(w) @@ -76,8 +111,6 @@ func UpdateCfgs(w http.ResponseWriter, r *http.Request) { return } - log.Info(system.Authentication.SelfRegistration) - if err = cfg.UpdateSystemCfg(system); err != nil { log.Errorf("failed to update system configurations: %v", err) handleInternalServerError(w) @@ -87,74 +120,73 @@ func UpdateCfgs(w http.ResponseWriter, r *http.Request) { // populate attrs of cfg according to m func populate(cfg *models.SystemCfg, m map[string]string) error { - if mode, ok := m[comcfg.AUTH_MODE]; ok { + if mode, ok := m[comcfg.AUTHMode]; ok { cfg.Authentication.Mode = mode } - if value, ok := m[comcfg.SELF_REGISTRATION]; ok { - cfg.Authentication.SelfRegistration = value == "true" + if value, ok := m[comcfg.SelfRegistration]; ok { + cfg.Authentication.SelfRegistration = value == "1" } - if url, ok := m[comcfg.LDAP_URL]; ok { + if url, ok := m[comcfg.LDAPURL]; ok { cfg.Authentication.LDAP.URL = url } - if dn, ok := m[comcfg.LDAP_SEARCH_DN]; ok { + if dn, ok := m[comcfg.LDAPSearchDN]; ok { cfg.Authentication.LDAP.SearchDN = dn } - if pwd, ok := m[comcfg.LDAP_SEARCH_PWD]; ok { + if pwd, ok := m[comcfg.LDAPSearchPwd]; ok { cfg.Authentication.LDAP.SearchPwd = pwd } - if dn, ok := m[comcfg.LDAP_BASE_DN]; ok { + if dn, ok := m[comcfg.LDAPBaseDN]; ok { cfg.Authentication.LDAP.BaseDN = dn } - if uid, ok := m[comcfg.LDAP_UID]; ok { + if uid, ok := m[comcfg.LDAPUID]; ok { cfg.Authentication.LDAP.UID = uid } - if filter, ok := m[comcfg.LDAP_FILTER]; ok { + if filter, ok := m[comcfg.LDAPFilter]; ok { cfg.Authentication.LDAP.Filter = filter } - if scope, ok := m[comcfg.LDAP_SCOPE]; ok { + if scope, ok := m[comcfg.LDAPScope]; ok { i, err := strconv.Atoi(scope) if err != nil { return err } cfg.Authentication.LDAP.Scope = i } + if timeout, ok := m[comcfg.LDAPTimeout]; ok { + i, err := strconv.Atoi(timeout) + if err != nil { + return err + } + cfg.Authentication.LDAP.Timeout = i + } - if value, ok := m[comcfg.EMAIL_SERVER]; ok { + if value, ok := m[comcfg.EmailHost]; ok { cfg.Email.Host = value } - if value, ok := m[comcfg.EMAIL_SERVER_PORT]; ok { + if value, ok := m[comcfg.EmailPort]; ok { cfg.Email.Port = value } - if value, ok := m[comcfg.EMAIL_USERNAME]; ok { + if value, ok := m[comcfg.EmailUsername]; ok { cfg.Email.Username = value } - if value, ok := m[comcfg.EMAIL_PWD]; ok { - cfg.Email.Host = value - } - if value, ok := m[comcfg.EMAIL_SSL]; ok { + if value, ok := m[comcfg.EmailPassword]; ok { cfg.Email.Password = value } - if value, ok := m[comcfg.EMAIL_FROM]; ok { + if value, ok := m[comcfg.EmailSSL]; ok { + cfg.Email.SSL = value == "1" + } + if value, ok := m[comcfg.EmailFrom]; ok { cfg.Email.From = value } - if value, ok := m[comcfg.EMAIL_IDENTITY]; ok { + if value, ok := m[comcfg.EmailIdentity]; ok { cfg.Email.Identity = value } - if value, ok := m[comcfg.PROJECT_CREATION_RESTRICTION]; ok { + if value, ok := m[comcfg.ProjectCreationRestriction]; ok { cfg.ProjectCreationRestriction = value } - if value, ok := m[comcfg.VERIFY_REMOTE_CERT]; ok { - cfg.VerifyRemoteCert = value == "true" - } - - if value, ok := m[comcfg.MAX_JOB_WORKERS]; ok { - if i, err := strconv.Atoi(value); err != nil { - return err - } else { - cfg.MaxJobWorkers = i - } + if value, ok := m[comcfg.VerifyRemoteCert]; ok { + cfg.VerifyRemoteCert = value == "1" } return nil diff --git a/src/adminserver/systemcfg/driver.go b/src/adminserver/systemcfg/store/driver.go similarity index 98% rename from src/adminserver/systemcfg/driver.go rename to src/adminserver/systemcfg/store/driver.go index bfb110f24..c0a9a2832 100644 --- a/src/adminserver/systemcfg/driver.go +++ b/src/adminserver/systemcfg/store/driver.go @@ -13,7 +13,7 @@ limitations under the License. */ -package systemcfg +package store import ( "github.com/vmware/harbor/src/common/models" diff --git a/src/adminserver/systemcfg/driver_json.go b/src/adminserver/systemcfg/store/json/driver_json.go similarity index 90% rename from src/adminserver/systemcfg/driver_json.go rename to src/adminserver/systemcfg/store/json/driver_json.go index 62e2c2ee7..4785e6006 100644 --- a/src/adminserver/systemcfg/driver_json.go +++ b/src/adminserver/systemcfg/store/json/driver_json.go @@ -13,7 +13,7 @@ limitations under the License. */ -package systemcfg +package json import ( "encoding/json" @@ -22,6 +22,7 @@ import ( "path/filepath" "sync" + "github.com/vmware/harbor/src/adminserver/systemcfg/store" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" ) @@ -38,11 +39,14 @@ type cfgStore struct { // NewCfgStore returns an instance of cfgStore that stores the configurations // in a json file. The file will be created if it does not exist. -func NewCfgStore(path ...string) (Driver, error) { +func NewCfgStore(path ...string) (store.Driver, error) { p := defaultPath - if len(path) != 0 { + if len(path) > 0 && len(path[0]) > 0 { p = path[0] } + + log.Debugf("path of configuration file: %s", p) + if _, err := os.Stat(p); os.IsNotExist(err) { log.Infof("the configuration file %s does not exist, creating it...", p) if err = os.MkdirAll(filepath.Dir(p), 0600); err != nil { diff --git a/src/adminserver/systemcfg/driver_json_test.go b/src/adminserver/systemcfg/store/json/driver_json_test.go similarity index 84% rename from src/adminserver/systemcfg/driver_json_test.go rename to src/adminserver/systemcfg/store/json/driver_json_test.go index 83a672cec..c551049f6 100644 --- a/src/adminserver/systemcfg/driver_json_test.go +++ b/src/adminserver/systemcfg/store/json/driver_json_test.go @@ -13,11 +13,13 @@ limitations under the License. */ -package systemcfg +package json import ( "os" "testing" + + "github.com/vmware/harbor/src/common/models" ) func TestReadWrite(t *testing.T) { @@ -32,12 +34,12 @@ func TestReadWrite(t *testing.T) { } }() - config := &cfg.SystemCfg{ - Authentication: &cfg.Authentication{ - LDAP: &cfg.LDAP{}, + config := &models.SystemCfg{ + Authentication: &models.Authentication{ + LDAP: &models.LDAP{}, }, - Database: &cfg.Database{ - MySQL: &cfg.MySQL{}, + Database: &models.Database{ + MySQL: &models.MySQL{}, }, } if err := store.Write(config); err != nil { diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index 4bb50cf4b..57f0323e8 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -20,18 +20,21 @@ import ( "os" "strconv" + "github.com/vmware/harbor/src/adminserver/systemcfg/store" + "github.com/vmware/harbor/src/adminserver/systemcfg/store/json" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" ) -var store Driver +var cfgStore store.Driver // Init system configurations. Read from config store first, if null read from env func Init() (err error) { s := getCfgStore() switch s { case "json": - store, err = NewCfgStore() + path := os.Getenv("JSON_STORE_PATH") + cfgStore, err = json.NewCfgStore(path) if err != nil { return } @@ -39,8 +42,8 @@ func Init() (err error) { return fmt.Errorf("unsupported configuration store driver %s", s) } - log.Infof("configuration store driver: %s", store.Name()) - cfg, err := store.Read() + log.Infof("configuration store driver: %s", cfgStore.Name()) + cfg, err := cfgStore.Read() if err != nil { return err } @@ -52,27 +55,13 @@ func Init() (err error) { return err } } else { - //read the following attrs from env every time boots up, - //and sync them into cfg store - cfg.DomainName = os.Getenv("EXT_ENDPOINT") - cfg.Database.MySQL.Password = os.Getenv("MYSQL_PWD") - cfg.JobLogDir = os.Getenv("LOG_DIR") - cfg.CompressJS = os.Getenv("USE_COMPRESSED_JS") == "on" - exp, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION")) - if err != nil { + if err := readFromEnv(cfg); err != nil { return err } - cfg.TokenExpiration = exp - cfg.SecretKey = os.Getenv("SECRET_KEY") - - cfgExp, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) - if err != nil { - return err - } - cfg.CfgExpiration = cfgExp } - if err = store.Write(cfg); err != nil { + //sync configurations into cfg store + if err = cfgStore.Write(cfg); err != nil { return err } @@ -80,15 +69,78 @@ func Init() (err error) { } func getCfgStore() string { - return "json" + t := os.Getenv("CFG_STORE_TYPE") + if len(t) == 0 { + t = "json" + } + return t +} + +//read the following attrs from env every time boots up +func readFromEnv(cfg *models.SystemCfg) error { + cfg.DomainName = os.Getenv("EXT_ENDPOINT") + + cfg.Database = &models.Database{ + Type: os.Getenv("DATABASE_TYPE"), + MySQL: &models.MySQL{ + Host: os.Getenv("MYSQL_HOST"), + Username: os.Getenv("MYSQL_USR"), + Password: os.Getenv("MYSQL_PWD"), + Database: os.Getenv("MYSQL_DATABASE"), + }, + SQLite: &models.SQLite{ + File: os.Getenv("SQLITE_FILE"), + }, + } + port, err := strconv.Atoi(os.Getenv("MYSQL_PORT")) + if err != nil { + return err + } + cfg.Database.MySQL.Port = port + + cfg.TokenService = &models.TokenService{ + URL: os.Getenv("TOKEN_SERVICE_URL"), + } + cfg.Registry = &models.Registry{ + URL: os.Getenv("REGISTRY_URL"), + } + + //TODO remove + cfg.JobLogDir = os.Getenv("LOG_DIR") + //TODO remove + cfg.CompressJS = os.Getenv("USE_COMPRESSED_JS") == "on" + exp, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION")) + if err != nil { + return err + } + cfg.TokenExpiration = exp + cfg.SecretKey = os.Getenv("SECRET_KEY") + + cfgExp, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) + if err != nil { + return err + } + cfg.CfgExpiration = cfgExp + + workers, err := strconv.Atoi(os.Getenv("MAX_JOB_WORKERS")) + if err != nil { + return err + } + cfg.MaxJobWorkers = workers + + return nil } func initFromEnv() (*models.SystemCfg, error) { cfg := &models.SystemCfg{} - cfg.DomainName = os.Getenv("EXT_ENDPOINT") + + if err := readFromEnv(cfg); err != nil { + return nil, err + } + cfg.Authentication = &models.Authentication{ Mode: os.Getenv("AUTH_MODE"), - SelfRegistration: os.Getenv("SELF_REGISTRATION") == "true", + SelfRegistration: os.Getenv("SELF_REGISTRATION") == "on", LDAP: &models.LDAP{ URL: os.Getenv("LDAP_URL"), SearchDN: os.Getenv("LDAP_SEARCH_DN"), @@ -108,74 +160,29 @@ func initFromEnv() (*models.SystemCfg, error) { return nil, err } cfg.Authentication.LDAP.Timeout = timeout - cfg.Database = &models.Database{ - Type: os.Getenv("DATABASE_TYPE"), - MySQL: &models.MySQL{ - Host: os.Getenv("MYSQL_HOST"), - Username: os.Getenv("MYSQL_USR"), - Password: os.Getenv("MYSQL_PWD"), - Database: os.Getenv("MYSQL_DATABASE"), - }, - SQLite: &models.SQLite{ - File: os.Getenv("SQLITE_FILE"), - }, - } - port, err := strconv.Atoi(os.Getenv("MYSQL_PORT")) - if err != nil { - return nil, err - } - cfg.Database.MySQL.Port = port - cfg.TokenService = &models.TokenService{ - URL: os.Getenv("TOKEN_SERVICE_URL"), - } - cfg.Registry = &models.Registry{ - URL: os.Getenv("REGISTRY_URL"), - } cfg.Email = &models.Email{ Host: os.Getenv("EMAIL_HOST"), Port: os.Getenv("EMAIL_PORT"), Username: os.Getenv("EMAIL_USR"), Password: os.Getenv("EMAIL_PWD"), - TLS: os.Getenv("EMAIL_TLS") == "true", + SSL: os.Getenv("EMAIL_SSL") == "true", From: os.Getenv("EMAIL_FROM"), Identity: os.Getenv("EMAIL_IDENTITY"), } - cfg.VerifyRemoteCert = os.Getenv("VERIFY_REMOTE_CERT") == "true" + cfg.VerifyRemoteCert = os.Getenv("VERIFY_REMOTE_CERT") == "on" cfg.ProjectCreationRestriction = os.Getenv("PROJECT_CREATION_RESTRICTION") - workers, err := strconv.Atoi(os.Getenv("MAX_JOB_WORKERS")) - if err != nil { - return nil, err - } - cfg.MaxJobWorkers = workers - cfg.JobLogDir = os.Getenv("LOG_DIR") cfg.InitialAdminPwd = os.Getenv("HARBOR_ADMIN_PASSWORD") - cfg.CompressJS = os.Getenv("USE_COMPRESSED_JS") == "on" - - tokenExp, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION")) - if err != nil { - return nil, err - } - cfg.TokenExpiration = tokenExp - - cfg.SecretKey = os.Getenv("SECRET_KEY") - - cfgExp, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) - if err != nil { - return nil, err - } - cfg.CfgExpiration = cfgExp - return cfg, nil } // GetSystemCfg returns the system configurations func GetSystemCfg() (*models.SystemCfg, error) { - return store.Read() + return cfgStore.Read() } // UpdateSystemCfg updates the system configurations func UpdateSystemCfg(cfg *models.SystemCfg) error { - return store.Write(cfg) + return cfgStore.Write(cfg) } diff --git a/src/adminserver/systemcfg/systemcfg_test.go b/src/adminserver/systemcfg/systemcfg_test.go new file mode 100644 index 000000000..f3f4a7f1b --- /dev/null +++ b/src/adminserver/systemcfg/systemcfg_test.go @@ -0,0 +1,71 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package systemcfg + +/* +import ( + "os" + "testing" +) + + +func TestSystemcfg(t *testing.T) { + key := "JSON_STORE_PATH" + tmpPath := "/tmp/config.json" + originalPath := os.Getenv(key) + defer func() { + if err := os.Remove(tmpPath); err != nil { + t.Errorf("failed to remove %s: %v", tmpPath, err) + } + + if len(originalPath) == 0 { + if err := os.Unsetenv(key); err != nil { + t.Fatalf("failed to unset env %s: %v", key, err) + } + return + } + + if err := os.Setenv(key, originalPath); err != nil { + t.Fatalf("failed to set env %s: %v", key, err) + } + }() + + if err := os.Setenv(key, tmpPath); err != nil { + t.Fatalf("failed to set env %s: %v", key, err) + } + + m := map[string]string{ + "LDAP_SCOPE": "1", + "LDAP_TIMEOUT": "30", + "MYSQL_PORT": "3306", + "MAX_JOB_WORKERS": "3", + "TOKEN_EXPIRATION": "30", + "CFG_EXPIRATION": "5", + } + + for k, v := range m { + if err := os.Setenv(k, v); err != nil { + t.Errorf("failed to set env %s: %v", k, err) + return + } + } + + if err := Init(); err != nil { + t.Errorf("failed to initialize system configurations: %v", err) + return + } +} +*/ diff --git a/src/common/api/base_test.go b/src/common/api/base_test.go index df8486ebf..0352b6e76 100644 --- a/src/common/api/base_test.go +++ b/src/common/api/base_test.go @@ -14,6 +14,7 @@ */ package api +/* import ( "github.com/vmware/harbor/src/common/config" "os" @@ -31,3 +32,4 @@ func TestGetIsInsecure(t *testing.T) { } os.Unsetenv("VERIFY_REMOTE_CERT") } +*/ diff --git a/src/common/config/config.go b/src/common/config/config.go index 756bff8d1..4c9aeaa61 100644 --- a/src/common/config/config.go +++ b/src/common/config/config.go @@ -18,94 +18,185 @@ package config import ( "bytes" + "encoding/json" "fmt" "io/ioutil" "net/http" "strings" + "time" "github.com/astaxie/beego/cache" "github.com/vmware/harbor/src/common/utils" + "github.com/vmware/harbor/src/common/utils/log" ) +// const variables const ( - //auth mode - DB_AUTH = "db_auth" - LDAP_AUTH = "ldap_auth" + DBAuth = "db_auth" + LDAPAuth = "ldap_auth" + ProCrtRestrEveryone = "everyone" + ProCrtRestrAdmOnly = "adminonly" + LDAPScopeBase = "1" + LDAPScopeOnelevel = "2" + LDAPScopeSubtree = "3" - //project_creation_restriction - PRO_CRT_RESTR_EVERYONE = "everyone" - PRO_CRT_RESTR_ADM_ONLY = "adminonly" - - LDAP_SCOPE_BASE = "1" - LDAP_SCOPE_ONELEVEL = "2" - LDAP_SCOPE_SUBTREE = "3" - - AUTH_MODE = "auth_mode" - SELF_REGISTRATION = "self_registration" - LDAP_URL = "ldap_url" - LDAP_SEARCH_DN = "ldap_search_dn" - LDAP_SEARCH_PWD = "ldap_search_pwd" - LDAP_BASE_DN = "ldap_base_dn" - LDAP_UID = "ldap_uid" - LDAP_FILTER = "ldap_filter" - LDAP_SCOPE = "ldap_scope" - EMAIL_SERVER = "email_server" - EMAIL_SERVER_PORT = "email_server_port" - EMAIL_USERNAME = "email_server_username" - EMAIL_PWD = "email_server_pwd" - EMAIL_FROM = "email_from" - EMAIL_SSL = "email_ssl" - EMAIL_IDENTITY = "email_identity" - PROJECT_CREATION_RESTRICTION = "project_creation_restriction" - VERIFY_REMOTE_CERT = "verify_remote_cert" - MAX_JOB_WORKERS = "max_job_workers" - CFG_EXPIRATION = "cfg_expiration" + AUTHMode = "auth_mode" + SelfRegistration = "self_registration" + LDAPURL = "ldap_url" + LDAPSearchDN = "ldap_search_dn" + LDAPSearchPwd = "ldap_search_password" + LDAPBaseDN = "ldap_base_dn" + LDAPUID = "ldap_uid" + LDAPFilter = "ldap_filter" + LDAPScope = "ldap_scope" + LDAPTimeout = "ldap_timeout" + EmailHost = "email_host" + EmailPort = "email_port" + EmailUsername = "email_username" + EmailPassword = "email_password" + EmailFrom = "email_from" + EmailSSL = "email_ssl" + EmailIdentity = "email_identity" + ProjectCreationRestriction = "project_creation_restriction" + VerifyRemoteCert = "verify_remote_cert" + MaxJobWorkers = "max_job_workers" + CfgExpiration = "cfg_expiration" ) +// Manager manages configurations type Manager struct { - Key string - Cache cache.Cache Loader *Loader + Parser Parser + Cache bool + cache cache.Cache + key string } -func NewManager(key, url string) *Manager { - return &Manager{ - Key: key, - Cache: cache.NewMemoryCache(), - Loader: NewLoader(url), +// Parser parses []byte to a specific configuration +type Parser interface { + // Parse ... + Parse([]byte) (interface{}, error) +} + +// NewManager returns an instance of Manager +// url: the url from which loader loads configurations +func NewManager(url, secret string, parser Parser, enableCache bool) *Manager { + m := &Manager{ + Loader: NewLoader(url, secret), + Parser: parser, } -} -func (m *Manager) GetFromCache() interface{} { - value := m.Cache.Get(m.Key) - if value != nil { - return value + if enableCache { + m.Cache = true + m.cache = cache.NewMemoryCache() + m.key = "cfg" } - return nil + + return m } +// Init loader +func (m *Manager) Init() error { + return m.Loader.Init() +} + +// Load configurations, if cache is enabled, cache the configurations +func (m *Manager) Load() (interface{}, error) { + b, err := m.Loader.Load() + if err != nil { + return nil, err + } + + c, err := m.Parser.Parse(b) + if err != nil { + return nil, err + } + + if m.Cache { + expi, err := parseExpiration(b) + if err != nil { + return nil, err + } + if err = m.cache.Put(m.key, c, + time.Duration(expi)*time.Second); err != nil { + return nil, err + } + } + + return c, nil +} + +func parseExpiration(b []byte) (int, error) { + expi := &struct { + Expi int `json:"cfg_expiration"` + }{} + if err := json.Unmarshal(b, expi); err != nil { + return 0, err + } + return expi.Expi, nil +} + +// Get : if cache is enabled, read configurations from cache, +// if cache is null or cache is disabled it loads configurations directly +func (m *Manager) Get() (interface{}, error) { + if m.Cache { + c := m.cache.Get(m.key) + if c != nil { + return c, nil + } + } + return m.Load() +} + +// Upload configurations +func (m *Manager) Upload(b []byte) error { + return m.Loader.Upload(b) +} + +// Loader loads and uploads configurations type Loader struct { url string + secret string client *http.Client } -func NewLoader(url string) *Loader { +// NewLoader ... +func NewLoader(url, secret string) *Loader { return &Loader{ url: url, + secret: secret, client: &http.Client{}, } } +// Init waits remote server to be ready by testing connections with it func (l *Loader) Init() error { addr := l.url if strings.Contains(addr, "://") { addr = strings.Split(addr, "://")[1] } + + if !strings.Contains(addr, ":") { + addr = addr + ":80" + } + return utils.TestTCPConn(addr, 60, 2) } +// Load configurations from remote server func (l *Loader) Load() ([]byte, error) { - resp, err := l.client.Get(l.url + "/api/configurations") + log.Debug("loading configurations...") + req, err := http.NewRequest("GET", l.url+"/api/configurations", nil) + if err != nil { + return nil, err + } + + req.AddCookie(&http.Cookie{ + Name: "secret", + Value: l.secret, + }) + + resp, err := l.client.Do(req) if err != nil { return nil, err } @@ -114,14 +205,22 @@ func (l *Loader) Load() ([]byte, error) { if err != nil { return nil, err } + log.Debug("configurations load completed") return b, nil } +// Upload configuratons to remote server func (l *Loader) Upload(b []byte) error { req, err := http.NewRequest("PUT", l.url+"/api/configurations", bytes.NewReader(b)) if err != nil { return err } + + req.AddCookie(&http.Cookie{ + Name: "secret", + Value: l.secret, + }) + resp, err := l.client.Do(req) if err != nil { return err diff --git a/src/common/config/config_test.go b/src/common/config/config_test.go index 95c954975..81448edf3 100644 --- a/src/common/config/config_test.go +++ b/src/common/config/config_test.go @@ -13,99 +13,3 @@ limitations under the License. */ package config - -import ( - "os" - "testing" -) - -func TestEnvConfLoader(t *testing.T) { - os.Unsetenv("KEY2") - os.Setenv("KEY1", "V1") - os.Setenv("KEY3", "V3") - keys := []string{"KEY1", "KEY2"} - ecl := EnvConfigLoader{ - keys, - } - m, err := ecl.Load() - if err != nil { - t.Errorf("Error loading the configuration via env: %v", err) - } - if m["KEY1"] != "V1" { - t.Errorf("The value for key KEY1 should be V1, but infact: %s", m["KEY1"]) - } - if len(m["KEY2"]) > 0 { - t.Errorf("The value for key KEY2 should be emptye, but infact: %s", m["KEY2"]) - } - if _, ok := m["KEY3"]; ok { - t.Errorf("The KEY3 should not be in result as it's not in the initial key list") - } - os.Unsetenv("KEY1") - os.Unsetenv("KEY3") -} - -func TestCommonConfig(t *testing.T) { - - mysql := MySQLSetting{"registry", "root", "password", "127.0.0.1", "3306"} - sqlite := SQLiteSetting{"file.db"} - verify := "off" - ext := "http://harbor" - token := "http://token" - loglevel := "info" - - os.Setenv("DATABASE", "") - os.Setenv("MYSQL_DATABASE", mysql.Database) - os.Setenv("MYSQL_USR", mysql.User) - os.Setenv("MYSQL_PWD", mysql.Password) - os.Setenv("MYSQL_HOST", mysql.Host) - os.Setenv("MYSQL_PORT", mysql.Port) - os.Setenv("SQLITE_FILE", sqlite.FilePath) - os.Setenv("VERIFY_REMOTE_CERT", verify) - os.Setenv("EXT_ENDPOINT", ext) - os.Setenv("TOKEN_ENDPOINT", token) - os.Setenv("LOG_LEVEL", loglevel) - - err := Reload() - if err != nil { - t.Errorf("Unexpected error when loading the configurations, error: %v", err) - } - if Database() != "mysql" { - t.Errorf("Expected Database value: mysql, fact: %s", mysql) - } - if MySQL() != mysql { - t.Errorf("Expected MySQL setting: %+v, fact: %+v", mysql, MySQL()) - } - if VerifyRemoteCert() { - t.Errorf("Expected VerifyRemoteCert: false, env var: %s, fact: %v", verify, VerifyRemoteCert()) - } - if ExtEndpoint() != ext { - t.Errorf("Expected ExtEndpoint: %s, fact: %s", ext, ExtEndpoint()) - } - if TokenEndpoint() != token { - t.Errorf("Expected TokenEndpoint: %s, fact: %s", token, TokenEndpoint()) - } - if LogLevel() != loglevel { - t.Errorf("Expected LogLevel: %s, fact: %s", loglevel, LogLevel()) - } - os.Setenv("DATABASE", "sqlite") - err = Reload() - if err != nil { - t.Errorf("Unexpected error when loading the configurations, error: %v", err) - } - if SQLite() != sqlite { - t.Errorf("Expected SQLite setting: %+v, fact %+v", sqlite, SQLite()) - } - - os.Unsetenv("DATABASE") - os.Unsetenv("MYSQL_DATABASE") - os.Unsetenv("MYSQL_USR") - os.Unsetenv("MYSQL_PWD") - os.Unsetenv("MYSQL_HOST") - os.Unsetenv("MYSQL_PORT") - os.Unsetenv("SQLITE_FILE") - os.Unsetenv("VERIFY_REMOTE_CERT") - os.Unsetenv("EXT_ENDPOINT") - os.Unsetenv("TOKEN_ENDPOINT") - os.Unsetenv("LOG_LEVEL") - -} diff --git a/src/common/dao/dao_test.go b/src/common/dao/dao_test.go index 26056116f..2a1144d9b 100644 --- a/src/common/dao/dao_test.go +++ b/src/common/dao/dao_test.go @@ -137,7 +137,7 @@ const publicityOn = 1 const publicityOff = 0 func TestMain(m *testing.M) { - databases := []string{"mysql", "sqlite"} + databases := []string{"mysql"} for _, database := range databases { log.Infof("run test cases for database: %s", database) diff --git a/src/common/models/config.go b/src/common/models/config.go index 9697a32ba..953f08e6c 100644 --- a/src/common/models/config.go +++ b/src/common/models/config.go @@ -46,7 +46,7 @@ type MySQL struct { Host string `json:"host"` Port int `json:"port"` Username string `json:"username"` - Password string `json:"password"` + Password string `json:"password,omitempty"` Database string `json:"database"` } @@ -61,7 +61,7 @@ type Email struct { Port string `json:"port"` Username string `json:"username"` Password string `json:"password"` - TLS bool `json:"tls"` + SSL bool `json:"ssl"` Identity string `json:"identity"` From string `json:"from"` } @@ -88,9 +88,9 @@ type SystemCfg struct { ProjectCreationRestriction string `json:"project_creation_restriction"` MaxJobWorkers int `json:"max_job_workers"` JobLogDir string `json:"job_log_dir"` - InitialAdminPwd string `json:"initial_admin_pwd"` + InitialAdminPwd string `json:"initial_admin_pwd,omitempty"` CompressJS bool `json:"compress_js"` //TODO remove TokenExpiration int `json:"token_expiration"` // in minute - SecretKey string `json:"secret_key"` + SecretKey string `json:"secret_key,omitempty"` CfgExpiration int `json:"cfg_expiration"` } diff --git a/src/common/utils/email/mail.go b/src/common/utils/email/mail.go index d82f1d9bf..61fc05531 100644 --- a/src/common/utils/email/mail.go +++ b/src/common/utils/email/mail.go @@ -57,7 +57,7 @@ func (m Mail) SendMail() error { content := mailContent.Bytes() auth := smtp.PlainAuth(mc.Identity, mc.Username, mc.Password, mc.Host) - if mc.TLS { + if mc.SSL { err = sendMailWithTLS(m, auth, content) } else { err = sendMail(m, auth, content) diff --git a/src/common/utils/test/adminserver.go b/src/common/utils/test/adminserver.go new file mode 100644 index 000000000..538b861bd --- /dev/null +++ b/src/common/utils/test/adminserver.go @@ -0,0 +1,59 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + + "github.com/vmware/harbor/src/common/models" +) + +// NewAdminserver returns a mock admin server +func NewAdminserver() (*httptest.Server, error) { + m := []*RequestHandlerMapping{} + b, err := json.Marshal(&models.SystemCfg{ + Authentication: &models.Authentication{ + Mode: "db_auth", + }, + Registry: &models.Registry{}, + }) + if err != nil { + return nil, err + } + + resp := &Response{ + StatusCode: http.StatusOK, + Body: b, + } + + m = append(m, &RequestHandlerMapping{ + Method: "GET", + Pattern: "/api/configurations", + Handler: Handler(resp), + }) + + m = append(m, &RequestHandlerMapping{ + Method: "PUT", + Pattern: "/api/configurations", + Handler: Handler(&Response{ + StatusCode: http.StatusOK, + }), + }) + + return NewServer(m...), nil +} diff --git a/src/common/utils/test/test.go b/src/common/utils/test/test.go index 37a435c24..78ea01bcd 100644 --- a/src/common/utils/test/test.go +++ b/src/common/utils/test/test.go @@ -21,6 +21,8 @@ import ( "net/http" "net/http/httptest" "strings" + + "github.com/gorilla/mux" ) // RequestHandlerMapping is a mapping between request and its handler @@ -78,11 +80,11 @@ func Handler(resp *Response) func(http.ResponseWriter, *http.Request) { // NewServer creates a HTTP server for unit test func NewServer(mappings ...*RequestHandlerMapping) *httptest.Server { - mux := http.NewServeMux() + r := mux.NewRouter() for _, mapping := range mappings { - mux.Handle(mapping.Pattern, mapping) + r.PathPrefix(mapping.Pattern).Handler(mapping).Methods(mapping.Method) } - return httptest.NewServer(mux) + return httptest.NewServer(r) } diff --git a/src/common/utils/utils.go b/src/common/utils/utils.go index a3ee7a5c7..a113ac033 100644 --- a/src/common/utils/utils.go +++ b/src/common/utils/utils.go @@ -75,7 +75,10 @@ func GenerateRandomString() string { return string(result) } -// timeout in second +// TestTCPConn tests TCP connection +// timeout: the total time before returning if something is wrong +// with the connection, in second +// interval: the interval time for retring after failure, in second func TestTCPConn(addr string, timeout, interval int) error { success := make(chan int) cancel := make(chan int) diff --git a/src/common/utils/utils_test.go b/src/common/utils/utils_test.go index cdef47de1..7421ad9c8 100644 --- a/src/common/utils/utils_test.go +++ b/src/common/utils/utils_test.go @@ -17,6 +17,7 @@ package utils import ( "encoding/base64" + "net/http/httptest" "strings" "testing" ) @@ -178,3 +179,12 @@ func TestParseLink(t *testing.T) { t.Errorf("unexpected prev: %s != %s", links.Next(), next) } } + +func TestTestTCPConn(t *testing.T) { + server := httptest.NewServer(nil) + defer server.Close() + addr := strings.TrimLeft(server.URL, "http://") + if err := TestTCPConn(addr, 60, 2); err != nil { + t.Fatalf("failed to test tcp connection of %s: %v", addr, err) + } +} diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index 18999e855..51bf1823f 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -18,16 +18,14 @@ package config import ( "encoding/json" "os" - "time" comcfg "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/models" - //"github.com/vmware/harbor/src/common/utils/log" ) var mg *comcfg.Manager -// Configuration holds configurations of Jobservice +// Configuration of Jobservice type Configuration struct { Database *models.Database `json:"database"` Registry *models.Registry `json:"registry"` @@ -38,65 +36,42 @@ type Configuration struct { CfgExpiration int `json:"cfg_expiration"` } +type parser struct { +} + +func (p *parser) Parse(b []byte) (interface{}, error) { + c := &Configuration{} + if err := json.Unmarshal(b, c); err != nil { + return nil, err + } + return c, nil +} + +// Init configurations func Init() error { adminServerURL := os.Getenv("ADMIN_SERVER_URL") if len(adminServerURL) == 0 { - adminServerURL = "http://admin_server" + adminServerURL = "http://adminserver" } - mg = comcfg.NewManager("cfg", adminServerURL) + mg = comcfg.NewManager(adminServerURL, UISecret(), &parser{}, true) - if err := mg.Loader.Init(); err != nil { + if err := mg.Init(); err != nil { return err } - if err := load(); err != nil { - return err - } - - path, err := LogDir() - if err != nil { - return err - } - if err := os.MkdirAll(path, 0600); err != nil { + if _, err := mg.Load(); err != nil { return err } return nil } -// get returns configurations of jobservice from cache, -// if cache is null, it loads first func get() (*Configuration, error) { - cfg := mg.GetFromCache() - if cfg != nil { - return cfg.(*Configuration), nil - } - - if err := load(); err != nil { + c, err := mg.Get() + if err != nil { return nil, err } - - return mg.GetFromCache().(*Configuration), nil -} - -// load loads configurations of jobservice and puts them into cache -func load() error { - raw, err := mg.Loader.Load() - if err != nil { - return err - } - - cfg := &Configuration{} - if err = json.Unmarshal(raw, cfg); err != nil { - return err - } - - if err = mg.Cache.Put(mg.Key, cfg, - time.Duration(cfg.CfgExpiration)*time.Second); err != nil { - return err - } - - return nil + return c.(*Configuration), nil } // VerifyRemoteCert returns bool value. @@ -149,11 +124,6 @@ func LogDir() (string, error) { return cfg.JobLogDir, nil } -// UISecret will return the value of secret cookie for jobsevice to call UI API. -func UISecret() string { - return os.Getenv("UI_SECRET") -} - // SecretKey will return the secret key for encryption/decryption password in target. func SecretKey() (string, error) { cfg, err := get() @@ -162,3 +132,9 @@ func SecretKey() (string, error) { } return cfg.SecretKey, nil } + +// UISecret returns the value of UI secret cookie, used for communication between UI and JobService +// TODO +func UISecret() string { + return os.Getenv("UI_SECRET") +} diff --git a/src/jobservice/config/config_test.go b/src/jobservice/config/config_test.go index f33c38a2e..0abcb1655 100644 --- a/src/jobservice/config/config_test.go +++ b/src/jobservice/config/config_test.go @@ -1,9 +1,71 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package config import ( + "os" "testing" + + "github.com/vmware/harbor/src/common/utils/test" ) -func TestMain(t *testing.T) { -} +// test functions under package jobservice/config +func TestConfig(t *testing.T) { + server, err := test.NewAdminserver() + if err != nil { + t.Fatalf("failed to create a mock admin server: %v", err) + } + defer server.Close() + url := os.Getenv("ADMIN_SERVER_URL") + defer os.Setenv("ADMIN_SERVER_URL", url) + + if err := os.Setenv("ADMIN_SERVER_URL", server.URL); err != nil { + t.Fatalf("failed to set env %s: %v", "ADMIN_SERVER_URL", err) + } + + if err := Init(); err != nil { + t.Fatalf("failed to initialize configurations: %v", err) + } + + if _, err := VerifyRemoteCert(); err != nil { + t.Fatalf("failed to get verify remote cert: %v", err) + } + + if _, err := Database(); err != nil { + t.Fatalf("failed to get database settings: %v", err) + } + + if _, err := MaxJobWorkers(); err != nil { + t.Fatalf("failed to get max job workers: %v", err) + } + + LocalUIURL() + + if _, err := LocalRegURL(); err != nil { + t.Fatalf("failed to get registry URL: %v", err) + } + + if _, err := LogDir(); err != nil { + t.Fatalf("failed to get log directory: %v", err) + } + + if _, err := SecretKey(); err != nil { + t.Fatalf("failed to get secret key: %v", err) + } + + UISecret() +} diff --git a/src/jobservice/main.go b/src/jobservice/main.go index 33516761a..00a91921c 100644 --- a/src/jobservice/main.go +++ b/src/jobservice/main.go @@ -16,6 +16,8 @@ package main import ( + "os" + "github.com/astaxie/beego" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" @@ -64,7 +66,6 @@ func resumeJobs() { } } -/* func init() { configPath := os.Getenv("CONFIG_PATH") if len(configPath) != 0 { @@ -72,4 +73,3 @@ func init() { beego.LoadAppConfig("ini", configPath) } } -*/ diff --git a/src/ui/api/config.go b/src/ui/api/config.go index 70e41459f..1c17ad8e0 100644 --- a/src/ui/api/config.go +++ b/src/ui/api/config.go @@ -30,6 +30,7 @@ import ( "github.com/vmware/harbor/src/ui/config" ) +// ConfigAPI ... type ConfigAPI struct { api.BaseAPI } @@ -56,9 +57,25 @@ func (c *ConfigAPI) Get() { c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } - //TODO filter attr in sys config + if cfg.Database.MySQL != nil { + cfg.Database.MySQL.Password = "" + } - c.Data["json"] = cfg + cfg.InitialAdminPwd = "" + cfg.SecretKey = "" + + m := map[string]interface{}{} + m["config"] = cfg + + editable, err := dao.AuthModeCanBeModified() + if err != nil { + log.Errorf("failed to determinie whether auth mode can be modified: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + m["auth_mode_editable"] = editable + + c.Data["json"] = m c.ServeJSON() } @@ -70,7 +87,7 @@ func (c *ConfigAPI) Put() { c.CustomAbort(http.StatusBadRequest, err.Error()) } - if value, ok := m[comcfg.AUTH_MODE]; ok { + if value, ok := m[comcfg.AUTHMode]; ok { mode, err := config.AuthMode() if err != nil { log.Errorf("failed to get auth mode: %v", err) @@ -87,13 +104,11 @@ func (c *ConfigAPI) Put() { if !flag { c.CustomAbort(http.StatusBadRequest, fmt.Sprintf("%s can not be modified as new users have been inserted into database", - comcfg.AUTH_MODE)) + comcfg.AUTHMode)) } } } - log.Info(m) - if err := config.Upload(m); err != nil { log.Errorf("failed to upload configurations: %v", err) c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) @@ -105,82 +120,81 @@ func (c *ConfigAPI) Put() { } } -// TODO ldap timeout, scope value func validateCfg(c map[string]string) error { - if value, ok := c[comcfg.AUTH_MODE]; ok { - if value != comcfg.DB_AUTH && value != comcfg.LDAP_AUTH { - return fmt.Errorf("invalid %s, shoud be %s or %s", comcfg.AUTH_MODE, comcfg.DB_AUTH, comcfg.LDAP_AUTH) + if value, ok := c[comcfg.AUTHMode]; ok { + if value != comcfg.DBAuth && value != comcfg.LDAPAuth { + return fmt.Errorf("invalid %s, shoud be %s or %s", comcfg.AUTHMode, comcfg.DBAuth, comcfg.LDAPAuth) } - if value == comcfg.LDAP_AUTH { - if _, ok := c[comcfg.LDAP_URL]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAP_URL) + if value == comcfg.LDAPAuth { + if _, ok := c[comcfg.LDAPURL]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAPURL) } - if _, ok := c[comcfg.LDAP_BASE_DN]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAP_BASE_DN) + if _, ok := c[comcfg.LDAPBaseDN]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAPBaseDN) } - if _, ok := c[comcfg.LDAP_UID]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAP_UID) + if _, ok := c[comcfg.LDAPUID]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAPUID) } - if _, ok := c[comcfg.LDAP_SCOPE]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAP_SCOPE) + if _, ok := c[comcfg.LDAPScope]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAPScope) } } } - if ldapURL, ok := c[comcfg.LDAP_URL]; ok && len(ldapURL) == 0 { - return fmt.Errorf("%s is empty", comcfg.LDAP_URL) + if ldapURL, ok := c[comcfg.LDAPURL]; ok && len(ldapURL) == 0 { + return fmt.Errorf("%s is empty", comcfg.LDAPURL) } - if baseDN, ok := c[comcfg.LDAP_BASE_DN]; ok && len(baseDN) == 0 { - return fmt.Errorf("%s is empty", comcfg.LDAP_BASE_DN) + if baseDN, ok := c[comcfg.LDAPBaseDN]; ok && len(baseDN) == 0 { + return fmt.Errorf("%s is empty", comcfg.LDAPBaseDN) } - if uID, ok := c[comcfg.LDAP_UID]; ok && len(uID) == 0 { - return fmt.Errorf("%s is empty", comcfg.LDAP_UID) + if uID, ok := c[comcfg.LDAPUID]; ok && len(uID) == 0 { + return fmt.Errorf("%s is empty", comcfg.LDAPUID) } - if scope, ok := c[comcfg.LDAP_SCOPE]; ok && - scope != comcfg.LDAP_SCOPE_BASE && - scope != comcfg.LDAP_SCOPE_ONELEVEL && - scope != comcfg.LDAP_SCOPE_SUBTREE { + if scope, ok := c[comcfg.LDAPScope]; ok && + scope != comcfg.LDAPScopeBase && + scope != comcfg.LDAPScopeOnelevel && + scope != comcfg.LDAPScopeSubtree { return fmt.Errorf("invalid %s, should be %s, %s or %s", - comcfg.LDAP_SCOPE, - comcfg.LDAP_SCOPE_BASE, - comcfg.LDAP_SCOPE_ONELEVEL, - comcfg.LDAP_SCOPE_SUBTREE) + comcfg.LDAPScope, + comcfg.LDAPScopeBase, + comcfg.LDAPScopeOnelevel, + comcfg.LDAPScopeSubtree) + } + if timeout, ok := c[comcfg.LDAPTimeout]; ok { + if t, err := strconv.Atoi(timeout); err != nil || t < 0 { + return fmt.Errorf("invalid %s", comcfg.LDAPTimeout) + } } - if self, ok := c[comcfg.SELF_REGISTRATION]; ok && - self != "true" && self != "false" { + if self, ok := c[comcfg.SelfRegistration]; ok && + self != "0" && self != "1" { return fmt.Errorf("%s should be %s or %s", - comcfg.SELF_REGISTRATION, "true", "false") + comcfg.SelfRegistration, "0", "1") } - if port, ok := c[comcfg.EMAIL_SERVER_PORT]; ok { + if port, ok := c[comcfg.EmailPort]; ok { if p, err := strconv.Atoi(port); err != nil || p < 0 || p > 65535 { - return fmt.Errorf("invalid %s", comcfg.EMAIL_SERVER_PORT) + return fmt.Errorf("invalid %s", comcfg.EmailPort) } } - if ssl, ok := c[comcfg.EMAIL_SSL]; ok && ssl != "true" && ssl != "false" { - return fmt.Errorf("%s should be true or false", comcfg.EMAIL_SSL) + if ssl, ok := c[comcfg.EmailSSL]; ok && ssl != "0" && ssl != "1" { + return fmt.Errorf("%s should be %s or %s", comcfg.EmailSSL, "0", "1") } - if crt, ok := c[comcfg.PROJECT_CREATION_RESTRICTION]; ok && - crt != comcfg.PRO_CRT_RESTR_EVERYONE && - crt != comcfg.PRO_CRT_RESTR_ADM_ONLY { + if crt, ok := c[comcfg.ProjectCreationRestriction]; ok && + crt != comcfg.ProCrtRestrEveryone && + crt != comcfg.ProCrtRestrAdmOnly { return fmt.Errorf("invalid %s, should be %s or %s", - comcfg.PROJECT_CREATION_RESTRICTION, - comcfg.PRO_CRT_RESTR_ADM_ONLY, - comcfg.PRO_CRT_RESTR_EVERYONE) + comcfg.ProjectCreationRestriction, + comcfg.ProCrtRestrAdmOnly, + comcfg.ProCrtRestrEveryone) } - if verify, ok := c[comcfg.VERIFY_REMOTE_CERT]; ok && verify != "true" && verify != "false" { - return fmt.Errorf("invalid %s, should be true or false", comcfg.VERIFY_REMOTE_CERT) - } - - if worker, ok := c[comcfg.MAX_JOB_WORKERS]; ok { - if w, err := strconv.Atoi(worker); err != nil || w <= 0 { - return fmt.Errorf("invalid %s", comcfg.MAX_JOB_WORKERS) - } + if verify, ok := c[comcfg.VerifyRemoteCert]; ok && verify != "0" && verify != "1" { + return fmt.Errorf("invalid %s, should be %s or %s", + comcfg.VerifyRemoteCert, "0", "1") } return nil diff --git a/src/ui/api/harborapi_test.go b/src/ui/api/harborapi_test.go index 4fd57a1c3..7d85420a6 100644 --- a/src/ui/api/harborapi_test.go +++ b/src/ui/api/harborapi_test.go @@ -14,6 +14,7 @@ import ( "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils" + "github.com/vmware/harbor/src/ui/config" "github.com/vmware/harbor/tests/apitests/apilib" // "strconv" // "strings" @@ -57,7 +58,14 @@ type usrInfo struct { } func init() { - dao.InitDatabase() + if err := config.Init(); err != nil { + log.Fatalf("failed to initialize configurations: %v", err) + } + database, err := config.Database() + if err != nil { + log.Fatalf("failed to get database configurations: %v", err) + } + dao.InitDatabase(database) _, file, _, _ := runtime.Caller(1) apppath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, ".."+string(filepath.Separator)))) beego.BConfig.WebConfig.Session.SessionOn = true @@ -512,7 +520,7 @@ func (a testapi) GetReposTop(authInfo usrInfo, count string) (int, error) { //-------------------------Targets Test---------------------------------------// //Create a new replication target -func (a testapi) AddTargets(authInfo usrInfo, repTarget apilib.RepTargetPost) (int, error) { +func (a testapi) AddTargets(authInfo usrInfo, repTarget apilib.RepTargetPost) (int, string, error) { _sling := sling.New().Post(a.basePath) path := "/api/targets" @@ -520,8 +528,8 @@ func (a testapi) AddTargets(authInfo usrInfo, repTarget apilib.RepTargetPost) (i _sling = _sling.Path(path) _sling = _sling.BodyJSON(repTarget) - httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo) - return httpStatusCode, err + httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo) + return httpStatusCode, string(body), err } //List filters targets by name diff --git a/src/ui/api/target_test.go b/src/ui/api/target_test.go index 94b6b25e0..62709a39e 100644 --- a/src/ui/api/target_test.go +++ b/src/ui/api/target_test.go @@ -30,17 +30,18 @@ func TestTargetsPost(t *testing.T) { //-------------------case 1 : response code = 201------------------------// fmt.Println("case 1 : response code = 201") - httpStatusCode, err = apiTest.AddTargets(*admin, *repTargets) + httpStatusCode, body, err := apiTest.AddTargets(*admin, *repTargets) if err != nil { t.Error("Error whihle add targets", err.Error()) t.Log(err) } else { assert.Equal(int(201), httpStatusCode, "httpStatusCode should be 201") + t.Log(body) } //-----------case 2 : response code = 409,name is already used-----------// fmt.Println("case 2 : response code = 409,name is already used") - httpStatusCode, err = apiTest.AddTargets(*admin, *repTargets) + httpStatusCode, _, err = apiTest.AddTargets(*admin, *repTargets) if err != nil { t.Error("Error whihle add targets", err.Error()) t.Log(err) @@ -51,7 +52,7 @@ func TestTargetsPost(t *testing.T) { //-----------case 3 : response code = 409,name is already used-----------// fmt.Println("case 3 : response code = 409,endPoint is already used") repTargets.Username = "errName" - httpStatusCode, err = apiTest.AddTargets(*admin, *repTargets) + httpStatusCode, _, err = apiTest.AddTargets(*admin, *repTargets) if err != nil { t.Error("Error whihle add targets", err.Error()) t.Log(err) @@ -61,7 +62,7 @@ func TestTargetsPost(t *testing.T) { //--------case 4 : response code = 401,User need to log in first.--------// fmt.Println("case 4 : response code = 401,User need to log in first.") - httpStatusCode, err = apiTest.AddTargets(*unknownUsr, *repTargets) + httpStatusCode, _, err = apiTest.AddTargets(*unknownUsr, *repTargets) if err != nil { t.Error("Error whihle add targets", err.Error()) t.Log(err) diff --git a/src/ui/config/config.go b/src/ui/config/config.go index aa726e810..ccb760f80 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -18,14 +18,15 @@ package config import ( "encoding/json" "os" - "time" comcfg "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" ) var mg *comcfg.Manager +// Configuration of UI type Configuration struct { DomainName string `json:"domain_name"` // Harbor external URL: protocal://host:port Authentication *models.Authentication `json:"authentication"` @@ -40,59 +41,52 @@ type Configuration struct { CompressJS bool `json:"compress_js"` TokenExpiration int `json:"token_expiration"` SecretKey string `json:"secret_key"` - CfgExpiration int `json:"cfg_expiration` + CfgExpiration int `json:"cfg_expiration"` } +type parser struct { +} + +func (p *parser) Parse(b []byte) (interface{}, error) { + c := &Configuration{} + if err := json.Unmarshal(b, c); err != nil { + return nil, err + } + return c, nil +} + +// Init configurations func Init() error { adminServerURL := os.Getenv("ADMIN_SERVER_URL") if len(adminServerURL) == 0 { - adminServerURL = "http://admin_server" + adminServerURL = "http://adminserver" } - mg = comcfg.NewManager("cfg", adminServerURL) + log.Debugf("admin server URL: %s", adminServerURL) + mg = comcfg.NewManager(adminServerURL, UISecret(), &parser{}, true) - if err := mg.Loader.Init(); err != nil { + if err := mg.Init(); err != nil { return err } - if err := Load(); err != nil { + if _, err := mg.Load(); err != nil { return err } return nil } -// Get returns configurations of UI, if cache is null, it loads first func get() (*Configuration, error) { - cfg := mg.GetFromCache() - if cfg != nil { - return cfg.(*Configuration), nil - } - - if err := Load(); err != nil { + c, err := mg.Get() + if err != nil { return nil, err } - - return mg.GetFromCache().(*Configuration), nil + return c.(*Configuration), nil } -// Load loads configurations of UI and puts them into cache +// Load configurations func Load() error { - raw, err := mg.Loader.Load() - if err != nil { - return err - } - - cfg := &Configuration{} - if err = json.Unmarshal(raw, cfg); err != nil { - return err - } - - if err = mg.Cache.Put(mg.Key, cfg, - time.Duration(cfg.CfgExpiration)*time.Second); err != nil { - return err - } - - return nil + _, err := mg.Load() + return err } // Upload uploads all system configutations to admin server @@ -101,7 +95,7 @@ func Upload(cfg map[string]string) error { if err != nil { return err } - return mg.Loader.Upload(b) + return mg.Upload(b) } // GetSystemCfg returns the system configurations @@ -195,14 +189,13 @@ func InitialAdminPassword() (string, error) { return cfg.InitialAdminPwd, nil } -// TODO // OnlyAdminCreateProject returns the flag to restrict that only sys admin can create project func OnlyAdminCreateProject() (bool, error) { cfg, err := get() if err != nil { return true, err } - return cfg.ProjectCreationRestriction == comcfg.PRO_CRT_RESTR_ADM_ONLY, nil + return cfg.ProjectCreationRestriction == comcfg.ProCrtRestrAdmOnly, nil } // VerifyRemoteCert returns bool value. @@ -214,6 +207,7 @@ func VerifyRemoteCert() (bool, error) { return cfg.VerifyRemoteCert, nil } +// Email returns email server settings func Email() (*models.Email, error) { cfg, err := get() if err != nil { @@ -222,6 +216,7 @@ func Email() (*models.Email, error) { return cfg.Email, nil } +// Database returns database settings func Database() (*models.Database, error) { cfg, err := get() if err != nil { @@ -230,8 +225,8 @@ func Database() (*models.Database, error) { return cfg.Database, nil } -// TODO // UISecret returns the value of UI secret cookie, used for communication between UI and JobService +// TODO func UISecret() string { return os.Getenv("UI_SECRET") } diff --git a/src/ui/config/config_test.go b/src/ui/config/config_test.go index 3f18b40b1..e71160d2e 100644 --- a/src/ui/config/config_test.go +++ b/src/ui/config/config_test.go @@ -17,131 +17,92 @@ package config import ( "os" "testing" + + "github.com/vmware/harbor/src/common/utils/test" ) -var ( - auth = "ldap_auth" - ldap = LDAPSetting{ - "ldap://test.ldap.com", - "ou=people", - "dc=whatever,dc=org", - "1234567", - "cn", - "uid", - "2", - "5", - } - tokenExp = "3" - tokenExpRes = 3 - adminPassword = "password" - externalRegURL = "127.0.0.1" - uiSecret = "ffadsdfsdf" - secretKey = "keykey" - selfRegistration = "off" - projectCreationRestriction = "adminonly" - internalRegistryURL = "http://registry:5000" - jobServiceURL = "http://jobservice" -) - -func TestMain(m *testing.M) { - - os.Setenv("AUTH_MODE", auth) - os.Setenv("LDAP_URL", ldap.URL) - os.Setenv("LDAP_BASE_DN", ldap.BaseDn) - os.Setenv("LDAP_SEARCH_DN", ldap.SearchDn) - os.Setenv("LDAP_SEARCH_PWD", ldap.SearchPwd) - os.Setenv("LDAP_UID", ldap.UID) - os.Setenv("LDAP_SCOPE", ldap.Scope) - os.Setenv("LDAP_FILTER", ldap.Filter) - os.Setenv("LDAP_CONNECT_TIMEOUT", ldap.ConnectTimeout) - os.Setenv("TOKEN_EXPIRATION", tokenExp) - os.Setenv("HARBOR_ADMIN_PASSWORD", adminPassword) - os.Setenv("EXT_REG_URL", externalRegURL) - os.Setenv("UI_SECRET", uiSecret) - os.Setenv("SECRET_KEY", secretKey) - os.Setenv("SELF_REGISTRATION", selfRegistration) - os.Setenv("PROJECT_CREATION_RESTRICTION", projectCreationRestriction) - os.Setenv("REGISTRY_URL", internalRegistryURL) - os.Setenv("JOB_SERVICE_URL", jobServiceURL) - - err := Reload() +// test functions under package ui/config +func TestConfig(t *testing.T) { + server, err := test.NewAdminserver() if err != nil { - panic(err) + t.Fatalf("failed to create a mock admin server: %v", err) } - rc := m.Run() + defer server.Close() - os.Unsetenv("AUTH_MODE") - os.Unsetenv("LDAP_URL") - os.Unsetenv("LDAP_BASE_DN") - os.Unsetenv("LDAP_SEARCH_DN") - os.Unsetenv("LDAP_SEARCH_PWD") - os.Unsetenv("LDAP_UID") - os.Unsetenv("LDAP_SCOPE") - os.Unsetenv("LDAP_FILTER") - os.Unsetenv("LDAP_CONNECT_TIMEOUT") - os.Unsetenv("TOKEN_EXPIRATION") - os.Unsetenv("HARBOR_ADMIN_PASSWORD") - os.Unsetenv("EXT_REG_URL") - os.Unsetenv("UI_SECRET") - os.Unsetenv("SECRET_KEY") - os.Unsetenv("SELF_REGISTRATION") - os.Unsetenv("CREATE_PROJECT_RESTRICTION") - os.Unsetenv("REGISTRY_URL") - os.Unsetenv("JOB_SERVICE_URL") + url := os.Getenv("ADMIN_SERVER_URL") + defer os.Setenv("ADMIN_SERVER_URL", url) - os.Exit(rc) -} - -func TestAuth(t *testing.T) { - if AuthMode() != auth { - t.Errorf("Expected auth mode:%s, in fact: %s", auth, AuthMode()) - } - if LDAP() != ldap { - t.Errorf("Expected ldap setting: %+v, in fact: %+v", ldap, LDAP()) - } -} - -func TestTokenExpiration(t *testing.T) { - if TokenExpiration() != tokenExpRes { - t.Errorf("Expected token expiration: %d, in fact: %d", tokenExpRes, TokenExpiration()) - } -} - -func TestURLs(t *testing.T) { - if InternalRegistryURL() != internalRegistryURL { - t.Errorf("Expected internal Registry URL: %s, in fact: %s", internalRegistryURL, InternalRegistryURL()) - } - if InternalJobServiceURL() != jobServiceURL { - t.Errorf("Expected internal jobservice URL: %s, in fact: %s", jobServiceURL, InternalJobServiceURL()) - } - if ExtRegistryURL() != externalRegURL { - t.Errorf("Expected External Registry URL: %s, in fact: %s", externalRegURL, ExtRegistryURL()) - } -} - -func TestSelfRegistration(t *testing.T) { - if SelfRegistration() { - t.Errorf("Expected Self Registration to be false") - } -} - -func TestSecrets(t *testing.T) { - if SecretKey() != secretKey { - t.Errorf("Expected Secrect Key :%s, in fact: %s", secretKey, SecretKey()) - } - if UISecret() != uiSecret { - t.Errorf("Expected UI Secret: %s, in fact: %s", uiSecret, UISecret()) - } -} - -func TestProjectCreationRestrict(t *testing.T) { - if !OnlyAdminCreateProject() { - t.Errorf("Expected OnlyAdminCreateProject to be true") - } -} - -func TestInitAdminPassword(t *testing.T) { - if InitialAdminPassword() != adminPassword { - t.Errorf("Expected adminPassword: %s, in fact: %s", adminPassword, InitialAdminPassword()) - } + if err := os.Setenv("ADMIN_SERVER_URL", server.URL); err != nil { + t.Fatalf("failed to set env %s: %v", "ADMIN_SERVER_URL", err) + } + + if err := Init(); err != nil { + t.Fatalf("failed to initialize configurations: %v", err) + } + + if err := Load(); err != nil { + t.Fatalf("failed to load configurations: %v", err) + } + + if err := Upload(map[string]string{}); err != nil { + t.Fatalf("failed to upload configurations: %v", err) + } + + if _, err := GetSystemCfg(); err != nil { + t.Fatalf("failed to get system configurations: %v", err) + } + + mode, err := AuthMode() + if err != nil { + t.Fatalf("failed to get auth mode: %v", err) + } + if mode != "db_auth" { + t.Errorf("unexpected mode: %s != %s", mode, "db_auth") + } + + if _, err := LDAP(); err != nil { + t.Fatalf("failed to get ldap settings: %v", err) + } + + if _, err := TokenExpiration(); err != nil { + t.Fatalf("failed to get token expiration: %v", err) + } + + if _, err := DomainName(); err != nil { + t.Fatalf("failed to get domain name: %v", err) + } + + if _, err := SecretKey(); err != nil { + t.Fatalf("failed to get secret key: %v", err) + } + + if _, err := SelfRegistration(); err != nil { + t.Fatalf("failed to get self registration: %v", err) + } + + if _, err := RegistryURL(); err != nil { + t.Fatalf("failed to get registry URL: %v", err) + } + + InternalJobServiceURL() + + InitialAdminPassword() + + if _, err := OnlyAdminCreateProject(); err != nil { + t.Fatalf("failed to get onldy admin create project: %v", err) + } + + if _, err := VerifyRemoteCert(); err != nil { + t.Fatalf("failed to get verify remote cert: %v", err) + } + + if _, err := Email(); err != nil { + t.Fatalf("failed to get email settings: %v", err) + } + + if _, err := Database(); err != nil { + t.Fatalf("failed to get database: %v", err) + } + + UISecret() } diff --git a/src/ui/controllers/base.go b/src/ui/controllers/base.go index 691880af2..044784f82 100644 --- a/src/ui/controllers/base.go +++ b/src/ui/controllers/base.go @@ -244,12 +244,13 @@ func (cc *CommonController) UserExists() { } func init() { - //conf/app.conf -> os.Getenv("config_path") configPath := os.Getenv("CONFIG_PATH") if len(configPath) != 0 { log.Infof("Config path: %s", configPath) - beego.LoadAppConfig("ini", configPath) + if err := beego.LoadAppConfig("ini", configPath); err != nil { + log.Errorf("failed to load app config: %v", err) + } } beego.AddFuncMap("i18n", i18n.Tr) @@ -272,5 +273,4 @@ func init() { log.Errorf("Fail to set message file: %s", err.Error()) } } - } diff --git a/src/ui/controllers/controllers_test.go b/src/ui/controllers/controllers_test.go index c80377417..7fd92023c 100644 --- a/src/ui/controllers/controllers_test.go +++ b/src/ui/controllers/controllers_test.go @@ -14,6 +14,8 @@ import ( "github.com/astaxie/beego" //"github.com/dghubble/sling" "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/config" ) //const ( @@ -29,6 +31,9 @@ import ( //var admin *usrInfo func init() { + if err := config.Init(); err != nil { + log.Fatalf("failed to initialize configurations: %v", err) + } _, file, _, _ := runtime.Caller(1) apppath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, ".."+string(filepath.Separator)))) @@ -63,7 +68,6 @@ func init() { //Init user Info //admin = &usrInfo{adminName, adminPwd} - } // TestMain is a sample to run an endpoint test diff --git a/src/ui/ui_test.go b/src/ui/ui_test.go index 59b3724bc..d9bd9c664 100644 --- a/src/ui/ui_test.go +++ b/src/ui/ui_test.go @@ -1,5 +1,6 @@ package main +/* import ( "testing" ) @@ -7,3 +8,4 @@ import ( func TestMain(t *testing.T) { } +*/ diff --git a/tests/docker-compose.test.yml b/tests/docker-compose.test.yml index 5a92685f3..5d9d3906c 100644 --- a/tests/docker-compose.test.yml +++ b/tests/docker-compose.test.yml @@ -21,3 +21,14 @@ services: - ./common/config/db/env ports: - 3306:3306 + adminserver: + build: + context: ../ + dockerfile: make/dev/adminserver/Dockerfile + env_file: + - ./common/config/adminserver/env + restart: always + volumes: + - /data/config/:/etc/harbor/ + ports: + - 8888:80 \ No newline at end of file diff --git a/tests/startuptest.go b/tests/startuptest.go index f611ef58c..e60823f8d 100644 --- a/tests/startuptest.go +++ b/tests/startuptest.go @@ -2,35 +2,38 @@ package main import ( - "fmt" - "io/ioutil" - "net/http" - "os" - "strings" - "time" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + "time" ) func main() { - time.Sleep(60*time.Second) - for _, url := range os.Args[1:] { - resp, err := http.Get(url) - if err != nil { - fmt.Fprintf(os.Stderr, "fetch: %v\n", err) - os.Exit(1) - } - b, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - fmt.Fprintf(os.Stderr, "fetch: reading %s: %v\n", url, err) - os.Exit(1) - } -// fmt.Printf("%s", b) - if strings.Contains(string(b), "Harbor") { - fmt.Printf("sucess!\n") - } else { - os.Exit(1) - } + time.Sleep(60 * time.Second) + for _, url := range os.Args[1:] { + resp, err := http.Get(url) + if err != nil { + fmt.Fprintf(os.Stderr, "fetch: %v\n", err) + os.Exit(1) + } + b, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "fetch: reading %s: %v\n", url, err) + os.Exit(1) + } + // fmt.Printf("%s", b) - } + if strings.Contains(string(b), "Harbor") { + fmt.Printf("sucess!\n") + } else { + fmt.Println("the response does not contain \"Harbor\"!") + + fmt.Println(string(b)) + os.Exit(1) + } + + } } - diff --git a/tests/testprepare.sh b/tests/testprepare.sh index debcad72d..89200e1d0 100755 --- a/tests/testprepare.sh +++ b/tests/testprepare.sh @@ -1,5 +1,5 @@ #!/bin/bash - +set -e cp tests/docker-compose.test.yml make/. mkdir /etc/ui @@ -7,3 +7,7 @@ cp make/common/config/ui/private_key.pem /etc/ui/. mkdir conf cp make/common/config/ui/app.conf conf/. + +sed -i -r "s/MYSQL_HOST=mysql/MYSQL_HOST=127.0.0.1/" make/common/config/adminserver/env +sed -i -r "s|REGISTRY_URL=http://registry:5000|REGISTRY_URL=http://127.0.0.1:5000|" make/common/config/adminserver/env +sed -i -r "s/UI_SECRET=.*/UI_SECRET=$UI_SECRET/" make/common/config/adminserver/env