diff --git a/.gitignore b/.gitignore index 7e451d1d5..572a7f08d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,11 @@ harbor make/common/config/* +make/dev/adminserver/harbor_adminserver make/dev/ui/harbor_ui make/dev/jobservice/harbor_jobservice +src/adminserver/adminserver src/ui/ui src/jobservice/jobservice +src/common/dao/dao.test *.pyc jobservice/test diff --git a/.travis.yml b/.travis.yml index a80c95942..7337b0bdd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,14 +13,13 @@ services: dist: trusty env: - DB_HOST: 127.0.0.1 - DB_PORT: 3306 - DB_USR: root - DB_PWD: root123 MYSQL_HOST: localhost MYSQL_PORT: 3306 MYSQL_USR: root MYSQL_PWD: root123 + MYSQL_DATABASE: registry + SQLITE_FILE: /tmp/registry.db + ADMIN_SERVER_URL: http://127.0.0.1:8888 DOCKER_COMPOSE_VERSION: 1.7.1 HARBOR_ADMIN: admin HARBOR_ADMIN_PASSWD: Harbor12345 @@ -70,7 +69,7 @@ install: before_script: # create tables and load data # - mysql < ./make/db/registry.sql -uroot --verbose - - sudo sqlite3 /registry.db < make/common/db/registry_sqlite.sql + - sudo sqlite3 /tmp/registry.db < make/common/db/registry_sqlite.sql script: - sudo mkdir -p /harbor_storage/ca_download @@ -88,7 +87,7 @@ script: - goveralls -coverprofile=profile.cov -service=travis-ci - docker-compose -f make/docker-compose.test.yml down - + - sudo make/prepare - docker-compose -f make/dev/docker-compose.yml up -d - docker ps diff --git a/make/common/templates/adminserver/env b/make/common/templates/adminserver/env new file mode 100644 index 000000000..ff0153215 --- /dev/null +++ b/make/common/templates/adminserver/env @@ -0,0 +1,38 @@ +LOG_LEVEL=debug +EXT_ENDPOINT=$ui_url +AUTH_MODE=$auth_mode +SELF_REGISTRATION=$self_registration +LDAP_URL=$ldap_url +LDAP_SEARCH_DN=$ldap_searchdn +LDAP_SEARCH_PWD=$ldap_search_pwd +LDAP_BASE_DN=$ldap_basedn +LDAP_FILTER=$ldap_filter +LDAP_UID=$ldap_uid +LDAP_SCOPE=$ldap_scope +LDAP_TIMEOUT=$ldap_timeout +DATABASE_TYPE=mysql +MYSQL_HOST=mysql +MYSQL_PORT=3306 +MYSQL_USR=root +MYSQL_PWD=$db_password +MYSQL_DATABASE=registry +REGISTRY_URL=http://registry:5000 +TOKEN_SERVICE_URL=http://ui/service/token +EMAIL_HOST=$email_host +EMAIL_PORT=$email_port +EMAIL_USR=$email_usr +EMAIL_PWD=$email_pwd +EMAIL_SSL=$email_ssl +EMAIL_FROM=$email_from +EMAIL_IDENTITY=$email_identity +HARBOR_ADMIN_PASSWORD=$harbor_admin_password +PROJECT_CREATION_RESTRICTION=$project_creation_restriction +VERIFY_REMOTE_CERT=$verify_remote_cert +MAX_JOB_WORKERS=$max_job_workers +LOG_DIR=/var/log/jobs +UI_SECRET=$ui_secret +SECRET_KEY=$secret_key +TOKEN_EXPIRATION=$token_expiration +CFG_EXPIRATION=5 +USE_COMPRESSED_JS=$use_compressed_js +GODEBUG=netdns=cgo \ No newline at end of file diff --git a/make/common/templates/jobservice/env b/make/common/templates/jobservice/env index c6e9fd736..06c8b0f22 100644 --- a/make/common/templates/jobservice/env +++ b/make/common/templates/jobservice/env @@ -1,15 +1,4 @@ -MYSQL_HOST=mysql -MYSQL_PORT=3306 -MYSQL_USR=root -MYSQL_PWD=$db_password -UI_SECRET=$ui_secret -SECRET_KEY=$secret_key -CONFIG_PATH=/etc/jobservice/app.conf -REGISTRY_URL=http://registry:5000 -VERIFY_REMOTE_CERT=$verify_remote_cert -MAX_JOB_WORKERS=$max_job_workers LOG_LEVEL=debug -LOG_DIR=/var/log/jobs +CONFIG_PATH=/etc/jobservice/app.conf +UI_SECRET=$ui_secret GODEBUG=netdns=cgo -EXT_ENDPOINT=$ui_url -TOKEN_ENDPOINT=http://ui diff --git a/make/common/templates/ui/app.conf b/make/common/templates/ui/app.conf index 3cda6d877..b4090f33c 100644 --- a/make/common/templates/ui/app.conf +++ b/make/common/templates/ui/app.conf @@ -6,13 +6,4 @@ types = en-US|zh-CN names = en-US|zh-CN [dev] -httpport = 80 - -[mail] -identity = $email_identity -host = $email_server -port = $email_server_port -username = $email_username -password = $email_password -from = $email_from -ssl = $email_ssl +httpport = 80 \ No newline at end of file diff --git a/make/common/templates/ui/env b/make/common/templates/ui/env index a15f323b7..fc0d133ab 100644 --- a/make/common/templates/ui/env +++ b/make/common/templates/ui/env @@ -1,30 +1,4 @@ -MYSQL_HOST=mysql -MYSQL_PORT=3306 -MYSQL_USR=root -MYSQL_PWD=$db_password -REGISTRY_URL=http://registry:5000 -JOB_SERVICE_URL=http://jobservice -UI_URL=http://ui -CONFIG_PATH=/etc/ui/app.conf -EXT_REG_URL=$hostname -HARBOR_ADMIN_PASSWORD=$harbor_admin_password -AUTH_MODE=$auth_mode -LDAP_URL=$ldap_url -LDAP_SEARCH_DN=$ldap_searchdn -LDAP_SEARCH_PWD=$ldap_search_pwd -LDAP_BASE_DN=$ldap_basedn -LDAP_FILTER=$ldap_filter -LDAP_UID=$ldap_uid -LDAP_SCOPE=$ldap_scope -LDAP_CONNECT_TIMEOUT=$ldap_connect_timeout -UI_SECRET=$ui_secret -SECRET_KEY=$secret_key -SELF_REGISTRATION=$self_registration -USE_COMPRESSED_JS=$use_compressed_js LOG_LEVEL=debug +CONFIG_PATH=/etc/ui/app.conf +UI_SECRET=$ui_secret GODEBUG=netdns=cgo -EXT_ENDPOINT=$ui_url -TOKEN_ENDPOINT=http://ui -VERIFY_REMOTE_CERT=$verify_remote_cert -TOKEN_EXPIRATION=$token_expiration -PROJECT_CREATION_RESTRICTION=$project_creation_restriction diff --git a/make/dev/adminserver/Dockerfile b/make/dev/adminserver/Dockerfile new file mode 100644 index 000000000..d05f87da2 --- /dev/null +++ b/make/dev/adminserver/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.7.3 + +MAINTAINER yinw@vmware.com + +COPY . /go/src/github.com/vmware/harbor + +WORKDIR /go/src/github.com/vmware/harbor/src/adminserver + +RUN go build -v -a -o /go/bin/harbor_adminserver \ + && chmod u+x /go/bin/harbor_adminserver +WORKDIR /go/bin/ +ENTRYPOINT ["/go/bin/harbor_adminserver"] diff --git a/make/dev/docker-compose.yml b/make/dev/docker-compose.yml index 33c7f2be3..c94a2d875 100644 --- a/make/dev/docker-compose.yml +++ b/make/dev/docker-compose.yml @@ -40,6 +40,22 @@ services: options: syslog-address: "tcp://127.0.0.1:1514" tag: "mysql" + adminserver: + build: + context: ../../ + dockerfile: make/dev/adminserver/Dockerfile + env_file: + - ../common/config/adminserver/env + restart: always + volumes: + - /data/config/:/etc/harbor/ + depends_on: + - log + logging: + driver: "syslog" + options: + syslog-address: "tcp://127.0.0.1:1514" + tag: "adminserver" ui: build: context: ../../ @@ -52,6 +68,8 @@ services: - ../common/config/ui/private_key.pem:/etc/ui/private_key.pem depends_on: - log + - adminserver + - registry logging: driver: "syslog" options: @@ -69,6 +87,7 @@ services: - ../common/config/jobservice/app.conf:/etc/jobservice/app.conf depends_on: - ui + - adminserver logging: driver: "syslog" options: diff --git a/make/docker-compose.tpl b/make/docker-compose.tpl index ac6c33d71..b66eb2698 100644 --- a/make/docker-compose.tpl +++ b/make/docker-compose.tpl @@ -41,6 +41,21 @@ services: options: syslog-address: "tcp://127.0.0.1:1514" tag: "mysql" + adminserver: + image: vmware/harbor-adminserver + container_name: harbor-adminserver + env_file: + - ./common/config/adminserver/env + restart: always + volumes: + - /data/config/:/etc/harbor/ + depends_on: + - log + logging: + driver: "syslog" + options: + syslog-address: "tcp://127.0.0.1:1514" + tag: "adminserver" ui: image: vmware/harbor-ui container_name: harbor-ui @@ -53,6 +68,8 @@ services: - /data:/harbor_storage depends_on: - log + - adminserver + - registry logging: driver: "syslog" options: @@ -69,6 +86,7 @@ services: - ./common/config/jobservice/app.conf:/etc/jobservice/app.conf depends_on: - ui + - adminserver logging: driver: "syslog" options: diff --git a/make/harbor.cfg b/make/harbor.cfg index 69118dbf3..507533edb 100644 --- a/make/harbor.cfg +++ b/make/harbor.cfg @@ -53,7 +53,7 @@ ldap_uid = uid ldap_scope = 3 #Timeout (in seconds) when connecting to an LDAP Server. The default value (and most reasonable) is 5 seconds. -ldap_connect_timeout = 5 +ldap_timeout = 5 #The password for the root user of mysql db, change this before any production use. db_password = root123 diff --git a/make/photon/adminserver/Dockerfile b/make/photon/adminserver/Dockerfile new file mode 100644 index 000000000..cb145275b --- /dev/null +++ b/make/photon/adminserver/Dockerfile @@ -0,0 +1,8 @@ +FROM library/photon:1.0 + +RUN mkdir /harbor/ +COPY ./make/dev/adminserver/harbor_adminserver /harbor/ + +RUN chmod u+x /harbor/harbor_adminserver +WORKDIR /harbor/ +ENTRYPOINT ["/harbor/harbor_adminserver"] diff --git a/make/prepare b/make/prepare index 2610d5fa2..28d46a376 100755 --- a/make/prepare +++ b/make/prepare @@ -77,10 +77,10 @@ hostname = rcp.get("configuration", "hostname") protocol = rcp.get("configuration", "ui_url_protocol") ui_url = protocol + "://" + hostname email_identity = rcp.get("configuration", "email_identity") -email_server = rcp.get("configuration", "email_server") -email_server_port = rcp.get("configuration", "email_server_port") -email_username = rcp.get("configuration", "email_username") -email_password = rcp.get("configuration", "email_password") +email_host = rcp.get("configuration", "email_server") +email_port = rcp.get("configuration", "email_server_port") +email_usr = rcp.get("configuration", "email_username") +email_pwd = rcp.get("configuration", "email_password") email_from = rcp.get("configuration", "email_from") email_ssl = rcp.get("configuration", "email_ssl") harbor_admin_password = rcp.get("configuration", "harbor_admin_password") @@ -101,7 +101,7 @@ else: ldap_filter = "" ldap_uid = rcp.get("configuration", "ldap_uid") ldap_scope = rcp.get("configuration", "ldap_scope") -ldap_connect_timeout = rcp.get("configuration", "ldap_connect_timeout") +ldap_timeout = rcp.get("configuration", "ldap_timeout") db_password = rcp.get("configuration", "db_password") self_registration = rcp.get("configuration", "self_registration") use_compressed_js = rcp.get("configuration", "use_compressed_js") @@ -126,6 +126,10 @@ secret_key = get_secret_key(secretkey_path) ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16)) +adminserver_config_dir = os.path.join(config_dir,"adminserver") +if not os.path.exists(adminserver_config_dir): + os.makedirs(os.path.join(config_dir, "adminserver")) + ui_config_dir = os.path.join(config_dir,"ui") if not os.path.exists(ui_config_dir): os.makedirs(os.path.join(config_dir, "ui")) @@ -152,6 +156,7 @@ def render(src, dest, **kw): f.write(t.substitute(**kw)) print("Generated configuration file: %s" % dest) +adminserver_conf_env = os.path.join(config_dir, "adminserver", "env") ui_conf_env = os.path.join(config_dir, "ui", "env") ui_conf = os.path.join(config_dir, "ui", "app.conf") jobservice_conf = os.path.join(config_dir, "jobservice", "app.conf") @@ -187,14 +192,12 @@ if protocol == "https": else: render(os.path.join(templates_dir, "nginx", "nginx.http.conf"), nginx_conf) - -render(os.path.join(templates_dir, "ui", "env"), - ui_conf_env, - hostname=hostname, - db_password=db_password, - ui_url=ui_url, - auth_mode=auth_mode, - harbor_admin_password=harbor_admin_password, + +render(os.path.join(templates_dir, "adminserver", "env"), + adminserver_conf_env, + ui_url=ui_url, + auth_mode=auth_mode, + self_registration=self_registration, ldap_url=ldap_url, ldap_searchdn =ldap_searchdn, ldap_search_pwd =ldap_search_pwd, @@ -202,27 +205,31 @@ render(os.path.join(templates_dir, "ui", "env"), ldap_filter=ldap_filter, ldap_uid=ldap_uid, ldap_scope=ldap_scope, - ldap_connect_timeout=ldap_connect_timeout, - self_registration=self_registration, - use_compressed_js=use_compressed_js, - ui_secret=ui_secret, + ldap_timeout=ldap_timeout, + db_password=db_password, + email_host=email_host, + email_port=email_port, + email_usr=email_usr, + email_pwd=email_pwd, + email_ssl=email_ssl, + email_from=email_from, + email_identity=email_identity, + harbor_admin_password=harbor_admin_password, + project_creation_restriction=proj_cre_restriction, + verify_remote_cert=verify_remote_cert, + max_job_workers=max_job_workers, + ui_secret=ui_secret, secret_key=secret_key, - verify_remote_cert=verify_remote_cert, - project_creation_restriction=proj_cre_restriction, - token_expiration=token_expiration) + token_expiration=token_expiration, + use_compressed_js=use_compressed_js + ) -render(os.path.join(templates_dir, "ui", "app.conf"), - ui_conf, - email_identity=email_identity, - email_server=email_server, - email_server_port=email_server_port, - email_username=email_username, - email_password=email_password, - email_from=email_from, - email_ssl=email_ssl, - ui_url=ui_url) +render(os.path.join(templates_dir, "ui", "env"), + ui_conf_env, + ui_secret=ui_secret) -render(os.path.join(templates_dir, "registry", "config.yml"), +render(os.path.join(templates_dir, "registry", + "config.yml"), registry_conf, ui_url=ui_url) @@ -232,16 +239,15 @@ render(os.path.join(templates_dir, "db", "env"), render(os.path.join(templates_dir, "jobservice", "env"), job_conf_env, - db_password=db_password, - ui_secret=ui_secret, - max_job_workers=max_job_workers, - secret_key=secret_key, - ui_url=ui_url, - verify_remote_cert=verify_remote_cert) + ui_secret=ui_secret) print("Generated configuration file: %s" % jobservice_conf) shutil.copyfile(os.path.join(templates_dir, "jobservice", "app.conf"), jobservice_conf) +print("Generated configuration file: %s" % ui_conf) +shutil.copyfile(os.path.join(templates_dir, "ui", "app.conf"), ui_conf) + + def validate_crt_subj(dirty_subj): subj_list = [item for item in dirty_subj.strip().split("/") \ if len(item.split("=")) == 2 and len(item.split("=")[1]) > 0] diff --git a/src/adminserver/api/base.go b/src/adminserver/api/base.go new file mode 100644 index 000000000..e4221bb64 --- /dev/null +++ b/src/adminserver/api/base.go @@ -0,0 +1,34 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "net/http" +) + +func handleInternalServerError(w http.ResponseWriter) { + http.Error(w, http.StatusText(http.StatusInternalServerError), + http.StatusInternalServerError) +} + +func handleBadRequestError(w http.ResponseWriter, error string) { + http.Error(w, error, http.StatusBadRequest) +} + +func handleUnauthorized(w http.ResponseWriter) { + http.Error(w, http.StatusText(http.StatusUnauthorized), + http.StatusUnauthorized) +} diff --git a/src/adminserver/api/cfg.go b/src/adminserver/api/cfg.go new file mode 100644 index 000000000..e2a41412b --- /dev/null +++ b/src/adminserver/api/cfg.go @@ -0,0 +1,193 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "os" + "strconv" + + cfg "github.com/vmware/harbor/src/adminserver/systemcfg" + comcfg "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" +) + +func isAuthenticated(r *http.Request) (bool, error) { + secret := os.Getenv("UI_SECRET") + c, err := r.Cookie("secret") + if err != nil { + if err == http.ErrNoCookie { + return false, nil + } + return false, err + } + return c != nil && c.Value == secret, nil +} + +// ListCfgs lists configurations +func ListCfgs(w http.ResponseWriter, r *http.Request) { + authenticated, err := isAuthenticated(r) + if err != nil { + log.Errorf("failed to check whether the request is authenticated or not: %v", err) + handleInternalServerError(w) + return + } + + if !authenticated { + handleUnauthorized(w) + return + } + + cfg, err := cfg.GetSystemCfg() + if err != nil { + log.Errorf("failed to get system configurations: %v", err) + handleInternalServerError(w) + return + } + + b, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + log.Errorf("failed to marshal configurations: %v", err) + handleInternalServerError(w) + return + } + if _, err = w.Write(b); err != nil { + log.Errorf("failed to write response: %v", err) + } +} + +// UpdateCfgs updates configurations +func UpdateCfgs(w http.ResponseWriter, r *http.Request) { + authenticated, err := isAuthenticated(r) + if err != nil { + log.Errorf("failed to check whether the request is authenticated or not: %v", err) + handleInternalServerError(w) + return + } + + if !authenticated { + handleUnauthorized(w) + return + } + + b, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Errorf("failed to read request body: %v", err) + handleInternalServerError(w) + return + } + + m := &map[string]string{} + if err = json.Unmarshal(b, m); err != nil { + handleBadRequestError(w, err.Error()) + return + } + + system, err := cfg.GetSystemCfg() + if err != nil { + handleInternalServerError(w) + return + } + + if err := populate(system, *m); err != nil { + log.Errorf("failed to populate system configurations: %v", err) + handleInternalServerError(w) + return + } + + if err = cfg.UpdateSystemCfg(system); err != nil { + log.Errorf("failed to update system configurations: %v", err) + handleInternalServerError(w) + return + } +} + +// populate attrs of cfg according to m +func populate(cfg *models.SystemCfg, m map[string]string) error { + if mode, ok := m[comcfg.AUTHMode]; ok { + cfg.Authentication.Mode = mode + } + if value, ok := m[comcfg.SelfRegistration]; ok { + cfg.Authentication.SelfRegistration = value == "1" + } + if url, ok := m[comcfg.LDAPURL]; ok { + cfg.Authentication.LDAP.URL = url + } + if dn, ok := m[comcfg.LDAPSearchDN]; ok { + cfg.Authentication.LDAP.SearchDN = dn + } + if pwd, ok := m[comcfg.LDAPSearchPwd]; ok { + cfg.Authentication.LDAP.SearchPwd = pwd + } + if dn, ok := m[comcfg.LDAPBaseDN]; ok { + cfg.Authentication.LDAP.BaseDN = dn + } + if uid, ok := m[comcfg.LDAPUID]; ok { + cfg.Authentication.LDAP.UID = uid + } + if filter, ok := m[comcfg.LDAPFilter]; ok { + cfg.Authentication.LDAP.Filter = filter + } + if scope, ok := m[comcfg.LDAPScope]; ok { + i, err := strconv.Atoi(scope) + if err != nil { + return err + } + cfg.Authentication.LDAP.Scope = i + } + if timeout, ok := m[comcfg.LDAPTimeout]; ok { + i, err := strconv.Atoi(timeout) + if err != nil { + return err + } + cfg.Authentication.LDAP.Timeout = i + } + + if value, ok := m[comcfg.EmailHost]; ok { + cfg.Email.Host = value + } + if value, ok := m[comcfg.EmailPort]; ok { + cfg.Email.Port = value + } + if value, ok := m[comcfg.EmailUsername]; ok { + cfg.Email.Username = value + } + if value, ok := m[comcfg.EmailPassword]; ok { + cfg.Email.Password = value + } + if value, ok := m[comcfg.EmailSSL]; ok { + cfg.Email.SSL = value == "1" + } + if value, ok := m[comcfg.EmailFrom]; ok { + cfg.Email.From = value + } + if value, ok := m[comcfg.EmailIdentity]; ok { + cfg.Email.Identity = value + } + + if value, ok := m[comcfg.ProjectCreationRestriction]; ok { + cfg.ProjectCreationRestriction = value + } + + if value, ok := m[comcfg.VerifyRemoteCert]; ok { + cfg.VerifyRemoteCert = value == "1" + } + + return nil +} diff --git a/src/adminserver/main.go b/src/adminserver/main.go new file mode 100644 index 000000000..d83d95d1a --- /dev/null +++ b/src/adminserver/main.go @@ -0,0 +1,60 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "net/http" + "os" + + cfg "github.com/vmware/harbor/src/adminserver/systemcfg" + "github.com/vmware/harbor/src/common/utils/log" +) + +// Server for admin component +type Server struct { + Port string + Handler http.Handler +} + +// Serve the API +func (s *Server) Serve() error { + server := &http.Server{ + Addr: ":" + s.Port, + Handler: s.Handler, + } + + return server.ListenAndServe() +} + +func main() { + log.Info("initializing system configurations...") + if err := cfg.Init(); err != nil { + log.Fatalf("failed to initialize the system: %v", err) + } + log.Info("system initialization completed") + + port := os.Getenv("PORT") + if len(port) == 0 { + port = "80" + } + server := &Server{ + Port: port, + Handler: newHandler(), + } + if err := server.Serve(); err != nil { + log.Fatal(err) + } +} diff --git a/src/adminserver/router.go b/src/adminserver/router.go new file mode 100644 index 000000000..49083f253 --- /dev/null +++ b/src/adminserver/router.go @@ -0,0 +1,30 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "net/http" + + "github.com/gorilla/mux" + "github.com/vmware/harbor/src/adminserver/api" +) + +func newHandler() http.Handler { + r := mux.NewRouter() + r.HandleFunc("/api/configurations", api.ListCfgs).Methods("GET") + r.HandleFunc("/api/configurations", api.UpdateCfgs).Methods("PUT") + return r +} diff --git a/src/adminserver/systemcfg/store/driver.go b/src/adminserver/systemcfg/store/driver.go new file mode 100644 index 000000000..c0a9a2832 --- /dev/null +++ b/src/adminserver/systemcfg/store/driver.go @@ -0,0 +1,30 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package store + +import ( + "github.com/vmware/harbor/src/common/models" +) + +// Driver defines methods that a configuration store driver must implement +type Driver interface { + // Name returns a human-readable name of the driver + Name() string + // Read reads the configurations from store + Read() (*models.SystemCfg, error) + // Write writes the configurations to store + Write(*models.SystemCfg) error +} diff --git a/src/adminserver/systemcfg/store/json/driver_json.go b/src/adminserver/systemcfg/store/json/driver_json.go new file mode 100644 index 000000000..4785e6006 --- /dev/null +++ b/src/adminserver/systemcfg/store/json/driver_json.go @@ -0,0 +1,108 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package json + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "sync" + + "github.com/vmware/harbor/src/adminserver/systemcfg/store" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" +) + +const ( + // the default path of configuration file + defaultPath = "/etc/harbor/config.json" +) + +type cfgStore struct { + path string // the path of cfg file + sync.RWMutex +} + +// NewCfgStore returns an instance of cfgStore that stores the configurations +// in a json file. The file will be created if it does not exist. +func NewCfgStore(path ...string) (store.Driver, error) { + p := defaultPath + if len(path) > 0 && len(path[0]) > 0 { + p = path[0] + } + + log.Debugf("path of configuration file: %s", p) + + if _, err := os.Stat(p); os.IsNotExist(err) { + log.Infof("the configuration file %s does not exist, creating it...", p) + if err = os.MkdirAll(filepath.Dir(p), 0600); err != nil { + return nil, err + } + if err = ioutil.WriteFile(p, []byte{}, 0600); err != nil { + return nil, err + } + } + + return &cfgStore{ + path: p, + }, nil +} + +// Name ... +func (c *cfgStore) Name() string { + return "JSON" +} + +// Read ... +func (c *cfgStore) Read() (*models.SystemCfg, error) { + c.RLock() + defer c.RUnlock() + + b, err := ioutil.ReadFile(c.path) + if err != nil { + return nil, err + } + + // empty file + if len(b) == 0 { + return nil, nil + } + + config := &models.SystemCfg{} + if err = json.Unmarshal(b, config); err != nil { + return nil, err + } + + return config, nil +} + +// Write ... +func (c *cfgStore) Write(config *models.SystemCfg) error { + b, err := json.MarshalIndent(config, "", " ") + if err != nil { + return err + } + + c.Lock() + defer c.Unlock() + + if err = ioutil.WriteFile(c.path, b, 0600); err != nil { + return err + } + + return nil +} diff --git a/src/adminserver/systemcfg/store/json/driver_json_test.go b/src/adminserver/systemcfg/store/json/driver_json_test.go new file mode 100644 index 000000000..eef48ee36 --- /dev/null +++ b/src/adminserver/systemcfg/store/json/driver_json_test.go @@ -0,0 +1,59 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package json + +import ( + "os" + "testing" + + "github.com/vmware/harbor/src/common/models" +) + +func TestReadWrite(t *testing.T) { + path := "/tmp/config.json" + store, err := NewCfgStore(path) + if err != nil { + t.Fatalf("failed to create json cfg store: %v", err) + } + defer func() { + if err := os.Remove(path); err != nil { + t.Fatalf("failed to remove the json file %s: %v", path, err) + } + }() + + if store.Name() != "JSON" { + t.Errorf("unexpected name: %s != %s", store.Name(), "JSON") + return + } + + config := &models.SystemCfg{ + Authentication: &models.Authentication{ + LDAP: &models.LDAP{}, + }, + Database: &models.Database{ + MySQL: &models.MySQL{}, + }, + } + if err := store.Write(config); err != nil { + t.Errorf("failed to write configurations to json file: %v", err) + return + } + + if _, err = store.Read(); err != nil { + t.Errorf("failed to read configurations from json file: %v", err) + return + } +} diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go new file mode 100644 index 000000000..57f0323e8 --- /dev/null +++ b/src/adminserver/systemcfg/systemcfg.go @@ -0,0 +1,188 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package systemcfg + +import ( + "fmt" + "os" + "strconv" + + "github.com/vmware/harbor/src/adminserver/systemcfg/store" + "github.com/vmware/harbor/src/adminserver/systemcfg/store/json" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" +) + +var cfgStore store.Driver + +// Init system configurations. Read from config store first, if null read from env +func Init() (err error) { + s := getCfgStore() + switch s { + case "json": + path := os.Getenv("JSON_STORE_PATH") + cfgStore, err = json.NewCfgStore(path) + if err != nil { + return + } + default: + return fmt.Errorf("unsupported configuration store driver %s", s) + } + + log.Infof("configuration store driver: %s", cfgStore.Name()) + cfg, err := cfgStore.Read() + if err != nil { + return err + } + + if cfg == nil { + log.Info("configurations read from store driver are null, initializing system from environment variables...") + cfg, err = initFromEnv() + if err != nil { + return err + } + } else { + if err := readFromEnv(cfg); err != nil { + return err + } + } + + //sync configurations into cfg store + if err = cfgStore.Write(cfg); err != nil { + return err + } + + return nil +} + +func getCfgStore() string { + t := os.Getenv("CFG_STORE_TYPE") + if len(t) == 0 { + t = "json" + } + return t +} + +//read the following attrs from env every time boots up +func readFromEnv(cfg *models.SystemCfg) error { + cfg.DomainName = os.Getenv("EXT_ENDPOINT") + + cfg.Database = &models.Database{ + Type: os.Getenv("DATABASE_TYPE"), + MySQL: &models.MySQL{ + Host: os.Getenv("MYSQL_HOST"), + Username: os.Getenv("MYSQL_USR"), + Password: os.Getenv("MYSQL_PWD"), + Database: os.Getenv("MYSQL_DATABASE"), + }, + SQLite: &models.SQLite{ + File: os.Getenv("SQLITE_FILE"), + }, + } + port, err := strconv.Atoi(os.Getenv("MYSQL_PORT")) + if err != nil { + return err + } + cfg.Database.MySQL.Port = port + + cfg.TokenService = &models.TokenService{ + URL: os.Getenv("TOKEN_SERVICE_URL"), + } + cfg.Registry = &models.Registry{ + URL: os.Getenv("REGISTRY_URL"), + } + + //TODO remove + cfg.JobLogDir = os.Getenv("LOG_DIR") + //TODO remove + cfg.CompressJS = os.Getenv("USE_COMPRESSED_JS") == "on" + exp, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION")) + if err != nil { + return err + } + cfg.TokenExpiration = exp + cfg.SecretKey = os.Getenv("SECRET_KEY") + + cfgExp, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) + if err != nil { + return err + } + cfg.CfgExpiration = cfgExp + + workers, err := strconv.Atoi(os.Getenv("MAX_JOB_WORKERS")) + if err != nil { + return err + } + cfg.MaxJobWorkers = workers + + return nil +} + +func initFromEnv() (*models.SystemCfg, error) { + cfg := &models.SystemCfg{} + + if err := readFromEnv(cfg); err != nil { + return nil, err + } + + cfg.Authentication = &models.Authentication{ + Mode: os.Getenv("AUTH_MODE"), + SelfRegistration: os.Getenv("SELF_REGISTRATION") == "on", + LDAP: &models.LDAP{ + URL: os.Getenv("LDAP_URL"), + SearchDN: os.Getenv("LDAP_SEARCH_DN"), + SearchPwd: os.Getenv("LDAP_SEARCH_PWD"), + BaseDN: os.Getenv("LDAP_BASE_DN"), + Filter: os.Getenv("LDAP_FILTER"), + UID: os.Getenv("LDAP_UID"), + }, + } + scope, err := strconv.Atoi(os.Getenv("LDAP_SCOPE")) + if err != nil { + return nil, err + } + cfg.Authentication.LDAP.Scope = scope + timeout, err := strconv.Atoi(os.Getenv("LDAP_TIMEOUT")) + if err != nil { + return nil, err + } + cfg.Authentication.LDAP.Timeout = timeout + + cfg.Email = &models.Email{ + Host: os.Getenv("EMAIL_HOST"), + Port: os.Getenv("EMAIL_PORT"), + Username: os.Getenv("EMAIL_USR"), + Password: os.Getenv("EMAIL_PWD"), + SSL: os.Getenv("EMAIL_SSL") == "true", + From: os.Getenv("EMAIL_FROM"), + Identity: os.Getenv("EMAIL_IDENTITY"), + } + cfg.VerifyRemoteCert = os.Getenv("VERIFY_REMOTE_CERT") == "on" + cfg.ProjectCreationRestriction = os.Getenv("PROJECT_CREATION_RESTRICTION") + + cfg.InitialAdminPwd = os.Getenv("HARBOR_ADMIN_PASSWORD") + return cfg, nil +} + +// GetSystemCfg returns the system configurations +func GetSystemCfg() (*models.SystemCfg, error) { + return cfgStore.Read() +} + +// UpdateSystemCfg updates the system configurations +func UpdateSystemCfg(cfg *models.SystemCfg) error { + return cfgStore.Write(cfg) +} diff --git a/src/adminserver/systemcfg/systemcfg_test.go b/src/adminserver/systemcfg/systemcfg_test.go new file mode 100644 index 000000000..cc28c2b38 --- /dev/null +++ b/src/adminserver/systemcfg/systemcfg_test.go @@ -0,0 +1,103 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package systemcfg + +import ( + "os" + "testing" + + comcfg "github.com/vmware/harbor/src/common/config" +) + +// test functions under adminserver/systemcfg +func TestSystemcfg(t *testing.T) { + key := "JSON_STORE_PATH" + path := "/tmp/config.json" + if _, err := os.Stat(path); err == nil { + if err := os.Remove(path); err != nil { + t.Fatalf("failed to remove %s: %v", path, err) + } + } else if !os.IsNotExist(err) { + t.Fatalf("failed to check the existence of %s: %v", path, err) + } + + if err := os.Setenv(key, path); err != nil { + t.Fatalf("failed to set env %s: %v", key, err) + } + + m := map[string]string{ + "AUTH_MODE": comcfg.DBAuth, + "LDAP_SCOPE": "1", + "LDAP_TIMEOUT": "30", + "MYSQL_PORT": "3306", + "MAX_JOB_WORKERS": "3", + "TOKEN_EXPIRATION": "30", + "CFG_EXPIRATION": "5", + } + + for k, v := range m { + if err := os.Setenv(k, v); err != nil { + t.Fatalf("failed to set env %s: %v", k, err) + } + } + + if err := Init(); err != nil { + t.Errorf("failed to initialize system configurations: %v", err) + return + } + defer func() { + if err := os.Remove(path); err != nil { + t.Fatalf("failed to remove %s: %v", path, err) + } + }() + + // run Init again to make sure it works well when the configuration file + // already exists + if err := Init(); err != nil { + t.Errorf("failed to initialize system configurations: %v", err) + return + } + + cfg, err := GetSystemCfg() + if err != nil { + t.Errorf("failed to get system configurations: %v", err) + return + } + + if cfg.Authentication.Mode != comcfg.DBAuth { + t.Errorf("unexpected auth mode: %s != %s", + cfg.Authentication.Mode, comcfg.DBAuth) + return + } + + cfg.Authentication.Mode = comcfg.LDAPAuth + if err = UpdateSystemCfg(cfg); err != nil { + t.Errorf("failed to update system configurations: %v", err) + return + } + + cfg, err = GetSystemCfg() + if err != nil { + t.Errorf("failed to get system configurations: %v", err) + return + } + + if cfg.Authentication.Mode != comcfg.LDAPAuth { + t.Errorf("unexpected auth mode: %s != %s", + cfg.Authentication.Mode, comcfg.DBAuth) + return + } +} diff --git a/src/common/api/base.go b/src/common/api/base.go index 53d916849..38635f76d 100644 --- a/src/common/api/base.go +++ b/src/common/api/base.go @@ -22,7 +22,6 @@ import ( "strconv" "github.com/astaxie/beego/validation" - "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" @@ -210,8 +209,3 @@ func (b *BaseAPI) GetPaginationParams() (page, pageSize int64) { return page, pageSize } - -// GetIsInsecure ... -func GetIsInsecure() bool { - return !config.VerifyRemoteCert() -} diff --git a/src/common/api/base_test.go b/src/common/api/base_test.go index df8486ebf..386e30e52 100644 --- a/src/common/api/base_test.go +++ b/src/common/api/base_test.go @@ -13,21 +13,3 @@ limitations under the License. */ package api - -import ( - "github.com/vmware/harbor/src/common/config" - "os" - "testing" -) - -func TestGetIsInsecure(t *testing.T) { - os.Setenv("VERIFY_REMOTE_CERT", "off") - err := config.Reload() - if err != nil { - t.Errorf("Failed to load config, error: %v", err) - } - if !GetIsInsecure() { - t.Errorf("GetIsInsecure() should be true when VERIFY_REMOTE_CERT is off, in fact: false") - } - os.Unsetenv("VERIFY_REMOTE_CERT") -} diff --git a/src/common/config/config.go b/src/common/config/config.go index e9c54c27c..4c9aeaa61 100644 --- a/src/common/config/config.go +++ b/src/common/config/config.go @@ -17,162 +17,218 @@ package config import ( + "bytes" + "encoding/json" "fmt" - "os" + "io/ioutil" + "net/http" "strings" + "time" + + "github.com/astaxie/beego/cache" + "github.com/vmware/harbor/src/common/utils" + "github.com/vmware/harbor/src/common/utils/log" ) -// ConfLoader is the interface to load configurations -type ConfLoader interface { - // Load will load configuration from different source into a string map, the values in the map will be parsed in to configurations. - Load() (map[string]string, error) +// const variables +const ( + DBAuth = "db_auth" + LDAPAuth = "ldap_auth" + ProCrtRestrEveryone = "everyone" + ProCrtRestrAdmOnly = "adminonly" + LDAPScopeBase = "1" + LDAPScopeOnelevel = "2" + LDAPScopeSubtree = "3" + + AUTHMode = "auth_mode" + SelfRegistration = "self_registration" + LDAPURL = "ldap_url" + LDAPSearchDN = "ldap_search_dn" + LDAPSearchPwd = "ldap_search_password" + LDAPBaseDN = "ldap_base_dn" + LDAPUID = "ldap_uid" + LDAPFilter = "ldap_filter" + LDAPScope = "ldap_scope" + LDAPTimeout = "ldap_timeout" + EmailHost = "email_host" + EmailPort = "email_port" + EmailUsername = "email_username" + EmailPassword = "email_password" + EmailFrom = "email_from" + EmailSSL = "email_ssl" + EmailIdentity = "email_identity" + ProjectCreationRestriction = "project_creation_restriction" + VerifyRemoteCert = "verify_remote_cert" + MaxJobWorkers = "max_job_workers" + CfgExpiration = "cfg_expiration" +) + +// Manager manages configurations +type Manager struct { + Loader *Loader + Parser Parser + Cache bool + cache cache.Cache + key string } -// EnvConfigLoader loads the config from env vars. -type EnvConfigLoader struct { - Keys []string +// Parser parses []byte to a specific configuration +type Parser interface { + // Parse ... + Parse([]byte) (interface{}, error) } -// Load ... -func (ec *EnvConfigLoader) Load() (map[string]string, error) { - m := make(map[string]string) - for _, k := range ec.Keys { - m[k] = os.Getenv(k) +// NewManager returns an instance of Manager +// url: the url from which loader loads configurations +func NewManager(url, secret string, parser Parser, enableCache bool) *Manager { + m := &Manager{ + Loader: NewLoader(url, secret), + Parser: parser, } - return m, nil + + if enableCache { + m.Cache = true + m.cache = cache.NewMemoryCache() + m.key = "cfg" + } + + return m } -// ConfParser ... -type ConfParser interface { - - //Parse parse the input raw map into a config map - Parse(raw map[string]string, config map[string]interface{}) error +// Init loader +func (m *Manager) Init() error { + return m.Loader.Init() } -// Config wraps a map for the processed configuration values, -// and loader parser to read configuration from external source and process the values. -type Config struct { - Config map[string]interface{} - Loader ConfLoader - Parser ConfParser +// Load configurations, if cache is enabled, cache the configurations +func (m *Manager) Load() (interface{}, error) { + b, err := m.Loader.Load() + if err != nil { + return nil, err + } + + c, err := m.Parser.Parse(b) + if err != nil { + return nil, err + } + + if m.Cache { + expi, err := parseExpiration(b) + if err != nil { + return nil, err + } + if err = m.cache.Put(m.key, c, + time.Duration(expi)*time.Second); err != nil { + return nil, err + } + } + + return c, nil } -// Load reload the configurations -func (conf *Config) Load() error { - rawMap, err := conf.Loader.Load() +func parseExpiration(b []byte) (int, error) { + expi := &struct { + Expi int `json:"cfg_expiration"` + }{} + if err := json.Unmarshal(b, expi); err != nil { + return 0, err + } + return expi.Expi, nil +} + +// Get : if cache is enabled, read configurations from cache, +// if cache is null or cache is disabled it loads configurations directly +func (m *Manager) Get() (interface{}, error) { + if m.Cache { + c := m.cache.Get(m.key) + if c != nil { + return c, nil + } + } + return m.Load() +} + +// Upload configurations +func (m *Manager) Upload(b []byte) error { + return m.Loader.Upload(b) +} + +// Loader loads and uploads configurations +type Loader struct { + url string + secret string + client *http.Client +} + +// NewLoader ... +func NewLoader(url, secret string) *Loader { + return &Loader{ + url: url, + secret: secret, + client: &http.Client{}, + } +} + +// Init waits remote server to be ready by testing connections with it +func (l *Loader) Init() error { + addr := l.url + if strings.Contains(addr, "://") { + addr = strings.Split(addr, "://")[1] + } + + if !strings.Contains(addr, ":") { + addr = addr + ":80" + } + + return utils.TestTCPConn(addr, 60, 2) +} + +// Load configurations from remote server +func (l *Loader) Load() ([]byte, error) { + log.Debug("loading configurations...") + req, err := http.NewRequest("GET", l.url+"/api/configurations", nil) + if err != nil { + return nil, err + } + + req.AddCookie(&http.Cookie{ + Name: "secret", + Value: l.secret, + }) + + resp, err := l.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + log.Debug("configurations load completed") + return b, nil +} + +// Upload configuratons to remote server +func (l *Loader) Upload(b []byte) error { + req, err := http.NewRequest("PUT", l.url+"/api/configurations", bytes.NewReader(b)) if err != nil { return err } - err = conf.Parser.Parse(rawMap, conf.Config) - return err -} -// MySQLSetting wraps the settings of a MySQL DB -type MySQLSetting struct { - Database string - User string - Password string - Host string - Port string -} + req.AddCookie(&http.Cookie{ + Name: "secret", + Value: l.secret, + }) -// SQLiteSetting wraps the settings of a SQLite DB -type SQLiteSetting struct { - FilePath string -} - -type commonParser struct{} - -// Parse parses the db settings, veryfy_remote_cert, ext_endpoint, token_endpoint -func (cp *commonParser) Parse(raw map[string]string, config map[string]interface{}) error { - db := strings.ToLower(raw["DATABASE"]) - if db == "mysql" || db == "" { - db = "mysql" - mySQLDB := raw["MYSQL_DATABASE"] - if len(mySQLDB) == 0 { - mySQLDB = "registry" - } - setting := MySQLSetting{ - mySQLDB, - raw["MYSQL_USR"], - raw["MYSQL_PWD"], - raw["MYSQL_HOST"], - raw["MYSQL_PORT"], - } - config["mysql"] = setting - } else if db == "sqlite" { - f := raw["SQLITE_FILE"] - if len(f) == 0 { - f = "registry.db" - } - setting := SQLiteSetting{ - f, - } - config["sqlite"] = setting - } else { - return fmt.Errorf("Invalid DB: %s", db) + resp, err := l.client.Do(req) + if err != nil { + return err } - config["database"] = db - //By default it's true - config["verify_remote_cert"] = raw["VERIFY_REMOTE_CERT"] != "off" + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected http status code: %d", resp.StatusCode) + } - config["ext_endpoint"] = raw["EXT_ENDPOINT"] - config["token_endpoint"] = raw["TOKEN_ENDPOINT"] - config["log_level"] = raw["LOG_LEVEL"] return nil } - -var commonConfig *Config - -func init() { - commonKeys := []string{"DATABASE", "MYSQL_DATABASE", "MYSQL_USR", "MYSQL_PWD", "MYSQL_HOST", "MYSQL_PORT", "SQLITE_FILE", "VERIFY_REMOTE_CERT", "EXT_ENDPOINT", "TOKEN_ENDPOINT", "LOG_LEVEL"} - commonConfig = &Config{ - Config: make(map[string]interface{}), - Loader: &EnvConfigLoader{Keys: commonKeys}, - Parser: &commonParser{}, - } - if err := commonConfig.Load(); err != nil { - panic(err) - } -} - -// Reload will reload the configuration. -func Reload() error { - return commonConfig.Load() -} - -// Database returns the DB type in configuration. -func Database() string { - return commonConfig.Config["database"].(string) -} - -// MySQL returns the mysql setting in configuration. -func MySQL() MySQLSetting { - return commonConfig.Config["mysql"].(MySQLSetting) -} - -// SQLite returns the SQLite setting -func SQLite() SQLiteSetting { - return commonConfig.Config["sqlite"].(SQLiteSetting) -} - -// VerifyRemoteCert returns bool value. -func VerifyRemoteCert() bool { - return commonConfig.Config["verify_remote_cert"].(bool) -} - -// ExtEndpoint ... -func ExtEndpoint() string { - return commonConfig.Config["ext_endpoint"].(string) -} - -// TokenEndpoint returns the endpoint string of token service, which can be accessed by internal service of Harbor. -func TokenEndpoint() string { - return commonConfig.Config["token_endpoint"].(string) -} - -// LogLevel returns the log level in string format. -func LogLevel() string { - return commonConfig.Config["log_level"].(string) -} diff --git a/src/common/config/config_test.go b/src/common/config/config_test.go index 95c954975..81448edf3 100644 --- a/src/common/config/config_test.go +++ b/src/common/config/config_test.go @@ -13,99 +13,3 @@ limitations under the License. */ package config - -import ( - "os" - "testing" -) - -func TestEnvConfLoader(t *testing.T) { - os.Unsetenv("KEY2") - os.Setenv("KEY1", "V1") - os.Setenv("KEY3", "V3") - keys := []string{"KEY1", "KEY2"} - ecl := EnvConfigLoader{ - keys, - } - m, err := ecl.Load() - if err != nil { - t.Errorf("Error loading the configuration via env: %v", err) - } - if m["KEY1"] != "V1" { - t.Errorf("The value for key KEY1 should be V1, but infact: %s", m["KEY1"]) - } - if len(m["KEY2"]) > 0 { - t.Errorf("The value for key KEY2 should be emptye, but infact: %s", m["KEY2"]) - } - if _, ok := m["KEY3"]; ok { - t.Errorf("The KEY3 should not be in result as it's not in the initial key list") - } - os.Unsetenv("KEY1") - os.Unsetenv("KEY3") -} - -func TestCommonConfig(t *testing.T) { - - mysql := MySQLSetting{"registry", "root", "password", "127.0.0.1", "3306"} - sqlite := SQLiteSetting{"file.db"} - verify := "off" - ext := "http://harbor" - token := "http://token" - loglevel := "info" - - os.Setenv("DATABASE", "") - os.Setenv("MYSQL_DATABASE", mysql.Database) - os.Setenv("MYSQL_USR", mysql.User) - os.Setenv("MYSQL_PWD", mysql.Password) - os.Setenv("MYSQL_HOST", mysql.Host) - os.Setenv("MYSQL_PORT", mysql.Port) - os.Setenv("SQLITE_FILE", sqlite.FilePath) - os.Setenv("VERIFY_REMOTE_CERT", verify) - os.Setenv("EXT_ENDPOINT", ext) - os.Setenv("TOKEN_ENDPOINT", token) - os.Setenv("LOG_LEVEL", loglevel) - - err := Reload() - if err != nil { - t.Errorf("Unexpected error when loading the configurations, error: %v", err) - } - if Database() != "mysql" { - t.Errorf("Expected Database value: mysql, fact: %s", mysql) - } - if MySQL() != mysql { - t.Errorf("Expected MySQL setting: %+v, fact: %+v", mysql, MySQL()) - } - if VerifyRemoteCert() { - t.Errorf("Expected VerifyRemoteCert: false, env var: %s, fact: %v", verify, VerifyRemoteCert()) - } - if ExtEndpoint() != ext { - t.Errorf("Expected ExtEndpoint: %s, fact: %s", ext, ExtEndpoint()) - } - if TokenEndpoint() != token { - t.Errorf("Expected TokenEndpoint: %s, fact: %s", token, TokenEndpoint()) - } - if LogLevel() != loglevel { - t.Errorf("Expected LogLevel: %s, fact: %s", loglevel, LogLevel()) - } - os.Setenv("DATABASE", "sqlite") - err = Reload() - if err != nil { - t.Errorf("Unexpected error when loading the configurations, error: %v", err) - } - if SQLite() != sqlite { - t.Errorf("Expected SQLite setting: %+v, fact %+v", sqlite, SQLite()) - } - - os.Unsetenv("DATABASE") - os.Unsetenv("MYSQL_DATABASE") - os.Unsetenv("MYSQL_USR") - os.Unsetenv("MYSQL_PWD") - os.Unsetenv("MYSQL_HOST") - os.Unsetenv("MYSQL_PORT") - os.Unsetenv("SQLITE_FILE") - os.Unsetenv("VERIFY_REMOTE_CERT") - os.Unsetenv("EXT_ENDPOINT") - os.Unsetenv("TOKEN_ENDPOINT") - os.Unsetenv("LOG_LEVEL") - -} diff --git a/src/common/dao/base.go b/src/common/dao/base.go index 9028ce587..e05f77561 100644 --- a/src/common/dao/base.go +++ b/src/common/dao/base.go @@ -17,11 +17,12 @@ package dao import ( "fmt" + "strconv" "strings" "sync" "github.com/astaxie/beego/orm" - "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" ) @@ -39,27 +40,32 @@ type Database interface { } // InitDatabase initializes the database -func InitDatabase() { - database, err := getDatabase() +func InitDatabase(database *models.Database) error { + db, err := getDatabase(database) if err != nil { - panic(err) + return err } - log.Infof("initializing database: %s", database.String()) - if err := database.Register(); err != nil { - panic(err) + log.Infof("initializing database: %s", db.String()) + if err := db.Register(); err != nil { + return err } + log.Info("initialize database completed") + return nil } -func getDatabase() (db Database, err error) { - switch config.Database() { +func getDatabase(database *models.Database) (db Database, err error) { + switch database.Type { case "", "mysql": - db = NewMySQL(config.MySQL().Host, config.MySQL().Port, config.MySQL().User, - config.MySQL().Password, config.MySQL().Database) + db = NewMySQL(database.MySQL.Host, + strconv.Itoa(database.MySQL.Port), + database.MySQL.Username, + database.MySQL.Password, + database.MySQL.Database) case "sqlite": - db = NewSQLite(config.SQLite().FilePath) + db = NewSQLite(database.SQLite.File) default: - err = fmt.Errorf("invalid database: %s", config.Database()) + err = fmt.Errorf("invalid database: %s", database.Type) } return } diff --git a/src/common/dao/config.go b/src/common/dao/config.go new file mode 100644 index 000000000..eb700c8af --- /dev/null +++ b/src/common/dao/config.go @@ -0,0 +1,32 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package dao + +import ( + "github.com/vmware/harbor/src/common/models" +) + +// AuthModeCanBeModified determines whether auth mode can be +// modified or not. Auth mode can modified when there is only admin +// user in database. +func AuthModeCanBeModified() (bool, error) { + c, err := GetOrmer().QueryTable(&models.User{}).Count() + if err != nil { + return false, err + } + + return c == 1, nil +} diff --git a/src/common/dao/config_test.go b/src/common/dao/config_test.go new file mode 100644 index 000000000..a874a6c6e --- /dev/null +++ b/src/common/dao/config_test.go @@ -0,0 +1,72 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package dao + +import ( + "testing" + + "github.com/vmware/harbor/src/common/models" +) + +func TestAuthModeCanBeModified(t *testing.T) { + c, err := GetOrmer().QueryTable(&models.User{}).Count() + if err != nil { + t.Fatalf("failed to count users: %v", err) + } + + if c == 1 { + flag, err := AuthModeCanBeModified() + if err != nil { + t.Fatalf("failed to determine whether auth mode can be modified: %v", err) + } + if !flag { + t.Errorf("unexpected result: %t != %t", flag, true) + } + + user := models.User{ + Username: "user_for_config_test", + Email: "user_for_config_test@vmware.com", + Password: "P@ssword", + Realname: "user_for_config_test", + } + id, err := Register(user) + if err != nil { + t.Fatalf("failed to register user: %v", err) + } + defer func(id int64) { + if err := deleteUser(id); err != nil { + t.Fatalf("failed to delete user %d: %v", id, err) + } + }(id) + + flag, err = AuthModeCanBeModified() + if err != nil { + t.Fatalf("failed to determine whether auth mode can be modified: %v", err) + } + if flag { + t.Errorf("unexpected result: %t != %t", flag, false) + } + + } else { + flag, err := AuthModeCanBeModified() + if err != nil { + t.Fatalf("failed to determine whether auth mode can be modified: %v", err) + } + if flag { + t.Errorf("unexpected result: %t != %t", flag, false) + } + } +} diff --git a/src/common/dao/dao_test.go b/src/common/dao/dao_test.go index 861bf3ee7..2a1144d9b 100644 --- a/src/common/dao/dao_test.go +++ b/src/common/dao/dao_test.go @@ -17,10 +17,12 @@ package dao import ( "os" + "strconv" "testing" "time" "github.com/astaxie/beego/orm" + //"github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" @@ -42,7 +44,7 @@ func execUpdate(o orm.Ormer, sql string, params ...interface{}) error { func clearUp(username string) { var err error - o := orm.NewOrm() + o := GetOrmer() o.Begin() err = execUpdate(o, `delete @@ -135,7 +137,7 @@ const publicityOn = 1 const publicityOff = 0 func TestMain(m *testing.M) { - databases := []string{"mysql", "sqlite"} + databases := []string{"mysql"} for _, database := range databases { log.Infof("run test cases for database: %s", database) @@ -156,53 +158,63 @@ func TestMain(m *testing.M) { } func testForMySQL(m *testing.M) int { - db := os.Getenv("DATABASE") - defer os.Setenv("DATABASE", db) - - os.Setenv("DATABASE", "mysql") - - dbHost := os.Getenv("DB_HOST") + dbHost := os.Getenv("MYSQL_HOST") if len(dbHost) == 0 { - log.Fatalf("environment variable DB_HOST is not set") + log.Fatalf("environment variable MYSQL_HOST is not set") } - dbUser := os.Getenv("DB_USR") + dbUser := os.Getenv("MYSQL_USR") if len(dbUser) == 0 { - log.Fatalf("environment variable DB_USR is not set") + log.Fatalf("environment variable MYSQL_USR is not set") } - dbPort := os.Getenv("DB_PORT") - if len(dbPort) == 0 { - log.Fatalf("environment variable DB_PORT is not set") + dbPortStr := os.Getenv("MYSQL_PORT") + if len(dbPortStr) == 0 { + log.Fatalf("environment variable MYSQL_PORT is not set") + } + dbPort, err := strconv.Atoi(dbPortStr) + if err != nil { + log.Fatalf("invalid MYSQL_PORT: %v", err) } - dbPassword := os.Getenv("DB_PWD") - log.Infof("DB_HOST: %s, DB_USR: %s, DB_PORT: %s, DB_PWD: %s\n", dbHost, dbUser, dbPort, dbPassword) + dbPassword := os.Getenv("MYSQL_PWD") + dbDatabase := os.Getenv("MYSQL_DATABASE") + if len(dbDatabase) == 0 { + log.Fatalf("environment variable MYSQL_DATABASE is not set") + } - os.Setenv("MYSQL_HOST", dbHost) - os.Setenv("MYSQL_PORT", dbPort) - os.Setenv("MYSQL_USR", dbUser) - os.Setenv("MYSQL_PWD", dbPassword) + database := &models.Database{ + Type: "mysql", + MySQL: &models.MySQL{ + Host: dbHost, + Port: dbPort, + Username: dbUser, + Password: dbPassword, + Database: dbDatabase, + }, + } - return testForAll(m) + log.Infof("MYSQL_HOST: %s, MYSQL_USR: %s, MYSQL_PORT: %s, MYSQL_PWD: %s\n", dbHost, dbUser, dbPort, dbPassword) + + return testForAll(m, database) } func testForSQLite(m *testing.M) int { - db := os.Getenv("DATABASE") - defer os.Setenv("DATABASE", db) - - os.Setenv("DATABASE", "sqlite") - file := os.Getenv("SQLITE_FILE") if len(file) == 0 { - os.Setenv("SQLITE_FILE", "/registry.db") - defer os.Setenv("SQLITE_FILE", "") + log.Fatalf("environment variable SQLITE_FILE is not set") } - return testForAll(m) + database := &models.Database{ + Type: "sqlite", + SQLite: &models.SQLite{ + File: file, + }, + } + + return testForAll(m, database) } -func testForAll(m *testing.M) int { - os.Setenv("AUTH_MODE", "db_auth") - initDatabaseForTest() +func testForAll(m *testing.M, database *models.Database) int { + initDatabaseForTest(database) clearUp(username) return m.Run() @@ -210,8 +222,8 @@ func testForAll(m *testing.M) int { var defaultRegistered = false -func initDatabaseForTest() { - database, err := getDatabase() +func initDatabaseForTest(db *models.Database) { + database, err := getDatabase(db) if err != nil { panic(err) } @@ -226,6 +238,12 @@ func initDatabaseForTest() { if err := database.Register(alias); err != nil { panic(err) } + + if alias != "default" { + if err = globalOrm.Using(alias); err != nil { + log.Fatalf("failed to create new orm: %v", err) + } + } } func TestRegister(t *testing.T) { @@ -980,13 +998,6 @@ func TestGetRecentLogs(t *testing.T) { } } -func TestGetTopRepos(t *testing.T) { - _, err := GetTopRepos(10) - if err != nil { - t.Fatalf("error occured in getting top repos, error: %v", err) - } -} - var targetID, policyID, policyID2, policyID3, jobID, jobID2, jobID3 int64 func TestAddRepTarget(t *testing.T) { diff --git a/src/common/dao/mysql.go b/src/common/dao/mysql.go index a050b054e..8a9c2c304 100644 --- a/src/common/dao/mysql.go +++ b/src/common/dao/mysql.go @@ -16,15 +16,11 @@ package dao import ( - "errors" "fmt" - "net" - - "time" "github.com/astaxie/beego/orm" _ "github.com/go-sql-driver/mysql" //register mysql driver - "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/common/utils" ) type mysql struct { @@ -48,7 +44,8 @@ func NewMySQL(host, port, usr, pwd, database string) Database { // Register registers MySQL as the underlying database used func (m *mysql) Register(alias ...string) error { - if err := m.testConn(m.host, m.port); err != nil { + + if err := utils.TestTCPConn(m.host+":"+m.port, 60, 2); err != nil { return err } @@ -65,30 +62,6 @@ func (m *mysql) Register(alias ...string) error { return orm.RegisterDataBase(an, "mysql", conn) } -func (m *mysql) testConn(host, port string) error { - ch := make(chan int, 1) - go func() { - var err error - var c net.Conn - for { - c, err = net.DialTimeout("tcp", host+":"+port, 20*time.Second) - if err == nil { - c.Close() - ch <- 1 - } else { - log.Errorf("failed to connect to db, retry after 2 seconds :%v", err) - time.Sleep(2 * time.Second) - } - } - }() - select { - case <-ch: - return nil - case <-time.After(60 * time.Second): - return errors.New("failed to connect to database after 60 seconds") - } -} - // Name returns the name of MySQL func (m *mysql) Name() string { return "MySQL" diff --git a/src/common/dao/repository.go b/src/common/dao/repository.go index cda2f5cba..5d9353a0c 100644 --- a/src/common/dao/repository.go +++ b/src/common/dao/repository.go @@ -102,12 +102,22 @@ func GetRepositoryByProjectName(name string) ([]*models.RepoRecord, error) { } //GetTopRepos returns the most popular repositories -func GetTopRepos(count int) ([]models.TopRepo, error) { +func GetTopRepos(userID int, count int) ([]models.TopRepo, error) { topRepos := []models.TopRepo{} + sql := `select r.name, r.pull_count from repository r + inner join project p on r.project_id = p.project_id + where ( + p.deleted = 0 and ( + p.public = 1 or ( + ? <> ? and exists ( + select 1 from project_member pm + where pm.project_id = p.project_id and pm.user_id = ? + )))) + order by r.pull_count desc, r.name limit ?` repositories := []*models.RepoRecord{} - if _, err := GetOrmer().QueryTable(&models.RepoRecord{}). - OrderBy("-PullCount", "Name").Limit(count).All(&repositories); err != nil { + _, err := GetOrmer().Raw(sql, userID, NonExistUserID, userID, count).QueryRows(&repositories) + if err != nil { return topRepos, err } diff --git a/src/common/dao/repository_test.go b/src/common/dao/repository_test.go index 1cc251f60..e8efc7e6b 100644 --- a/src/common/dao/repository_test.go +++ b/src/common/dao/repository_test.go @@ -16,8 +16,11 @@ package dao import ( + "fmt" "testing" + "time" + "github.com/stretchr/testify/require" "github.com/vmware/harbor/src/common/models" ) @@ -160,6 +163,161 @@ func TestGetTotalOfUserRelevantRepositories(t *testing.T) { } } +func TestGetTopRepos(t *testing.T) { + var err error + require := require.New(t) + + require.NoError(GetOrmer().Begin()) + defer func() { + require.NoError(GetOrmer().Rollback()) + }() + + admin, err := GetUser(models.User{Username: "admin"}) + require.NoError(err) + + user := models.User{ + Username: "user", + Password: "user", + Email: "user@test.com", + } + userID, err := Register(user) + require.NoError(err) + user.UserID = int(userID) + + // + // public project with 1 repository + // non-public project with 2 repositories visible by admin + // non-public project with 1 repository visible by admin and user + // deleted public project with 1 repository + // + + project1 := models.Project{ + OwnerID: admin.UserID, + Name: "project1", + CreationTime: time.Now(), + OwnerName: admin.Username, + Public: 0, + } + project1.ProjectID, err = AddProject(project1) + require.NoError(err) + + project2 := models.Project{ + OwnerID: admin.UserID, + Name: "project2", + CreationTime: time.Now(), + OwnerName: admin.Username, + Public: 0, + } + project2.ProjectID, err = AddProject(project2) + require.NoError(err) + + require.NoError(AddProjectMember(project2.ProjectID, user.UserID, models.PROJECTADMIN)) + + err = AddRepository(*repository) + require.NoError(err) + + repository1 := &models.RepoRecord{ + Name: fmt.Sprintf("%v/repository1", project1.Name), + OwnerName: admin.Username, + ProjectName: project1.Name, + } + err = AddRepository(*repository1) + require.NoError(err) + require.NoError(IncreasePullCount(repository1.Name)) + repository1, err = GetRepositoryByName(repository1.Name) + require.NoError(err) + + repository2 := &models.RepoRecord{ + Name: fmt.Sprintf("%v/repository2", project1.Name), + OwnerName: admin.Username, + ProjectName: project1.Name, + } + err = AddRepository(*repository2) + require.NoError(err) + require.NoError(IncreasePullCount(repository2.Name)) + require.NoError(IncreasePullCount(repository2.Name)) + repository2, err = GetRepositoryByName(repository2.Name) + require.NoError(err) + + repository3 := &models.RepoRecord{ + Name: fmt.Sprintf("%v/repository3", project2.Name), + OwnerName: admin.Username, + ProjectName: project2.Name, + } + err = AddRepository(*repository3) + require.NoError(err) + require.NoError(IncreasePullCount(repository3.Name)) + require.NoError(IncreasePullCount(repository3.Name)) + require.NoError(IncreasePullCount(repository3.Name)) + repository3, err = GetRepositoryByName(repository3.Name) + require.NoError(err) + + deletedPublicProject := models.Project{ + OwnerID: admin.UserID, + Name: "public-deleted", + CreationTime: time.Now(), + OwnerName: admin.Username, + Public: 1, + } + deletedPublicProject.ProjectID, err = AddProject(deletedPublicProject) + require.NoError(err) + deletedPublicRepository1 := &models.RepoRecord{ + Name: fmt.Sprintf("%v/repository1", deletedPublicProject.Name), + OwnerName: admin.Username, + ProjectName: deletedPublicProject.Name, + } + err = AddRepository(*deletedPublicRepository1) + require.NoError(err) + DeleteProject(deletedPublicProject.ProjectID) + + var topRepos []models.TopRepo + + // anonymous should retrieve public non-deleted repositories + topRepos, err = GetTopRepos(NonExistUserID, 3) + require.NoError(err) + require.Len(topRepos, 1) + require.Equal(topRepos, []models.TopRepo{ + models.TopRepo{ + RepoName: repository.Name, + AccessCount: repository.PullCount, + }, + }) + + // admin should retrieve all visible repositories limited by count + topRepos, err = GetTopRepos(admin.UserID, 3) + require.NoError(err) + require.Len(topRepos, 3) + require.Equal(topRepos, []models.TopRepo{ + models.TopRepo{ + RepoName: repository3.Name, + AccessCount: repository3.PullCount, + }, + models.TopRepo{ + RepoName: repository2.Name, + AccessCount: repository2.PullCount, + }, + models.TopRepo{ + RepoName: repository1.Name, + AccessCount: repository1.PullCount, + }, + }) + + // user should retrieve all visible repositories + topRepos, err = GetTopRepos(user.UserID, 3) + require.NoError(err) + require.Len(topRepos, 2) + require.Equal(topRepos, []models.TopRepo{ + models.TopRepo{ + RepoName: repository3.Name, + AccessCount: repository3.PullCount, + }, + models.TopRepo{ + RepoName: repository.Name, + AccessCount: repository.PullCount, + }, + }) +} + func addRepository(repository *models.RepoRecord) error { return AddRepository(*repository) } diff --git a/src/common/dao/user_test.go b/src/common/dao/user_test.go index 7d421d848..c227d835e 100644 --- a/src/common/dao/user_test.go +++ b/src/common/dao/user_test.go @@ -38,6 +38,11 @@ func TestDeleteUser(t *testing.T) { if err != nil { t.Fatalf("failed to register user: %v", err) } + defer func(id int64) { + if err := deleteUser(id); err != nil { + t.Fatalf("failed to delete user %d: %v", id, err) + } + }(id) err = DeleteUser(int(id)) if err != nil { @@ -67,3 +72,11 @@ func TestDeleteUser(t *testing.T) { expected) } } + +func deleteUser(id int64) error { + if _, err := GetOrmer().QueryTable(&models.User{}). + Filter("UserID", id).Delete(); err != nil { + return err + } + return nil +} diff --git a/src/common/models/config.go b/src/common/models/config.go new file mode 100644 index 000000000..953f08e6c --- /dev/null +++ b/src/common/models/config.go @@ -0,0 +1,96 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package models + +// Authentication ... +type Authentication struct { + Mode string `json:"mode"` + SelfRegistration bool `json:"self_registration"` + LDAP *LDAP `json:"ldap,omitempty"` +} + +// LDAP ... +type LDAP struct { + URL string `json:"url"` + SearchDN string `json:"search_dn"` + SearchPwd string `json:"search_pwd"` + BaseDN string `json:"base_dn"` + Filter string `json:"filter"` + UID string `json:"uid"` + Scope int `json:"scope"` + Timeout int `json:"timeout"` // in second +} + +// Database ... +type Database struct { + Type string `json:"type"` + MySQL *MySQL `json:"mysql,omitempty"` + SQLite *SQLite `json:"sqlite,omitempty"` +} + +// MySQL ... +type MySQL struct { + Host string `json:"host"` + Port int `json:"port"` + Username string `json:"username"` + Password string `json:"password,omitempty"` + Database string `json:"database"` +} + +// SQLite ... +type SQLite struct { + File string `json:"file"` +} + +// Email ... +type Email struct { + Host string `json:"host"` + Port string `json:"port"` + Username string `json:"username"` + Password string `json:"password"` + SSL bool `json:"ssl"` + Identity string `json:"identity"` + From string `json:"from"` +} + +// Registry ... +type Registry struct { + URL string `json:"url"` +} + +// TokenService ... +type TokenService struct { + URL string `json:"url"` +} + +// SystemCfg holds all configurations of system +type SystemCfg struct { + DomainName string `json:"domain_name"` // Harbor external URL: protocal://host:port + Authentication *Authentication `json:"authentication"` + Database *Database `json:"database"` + TokenService *TokenService `json:"token_service"` + Registry *Registry `json:"registry"` + Email *Email `json:"email"` + VerifyRemoteCert bool `json:"verify_remote_cert"` + ProjectCreationRestriction string `json:"project_creation_restriction"` + MaxJobWorkers int `json:"max_job_workers"` + JobLogDir string `json:"job_log_dir"` + InitialAdminPwd string `json:"initial_admin_pwd,omitempty"` + CompressJS bool `json:"compress_js"` //TODO remove + TokenExpiration int `json:"token_expiration"` // in minute + SecretKey string `json:"secret_key,omitempty"` + CfgExpiration int `json:"cfg_expiration"` +} diff --git a/src/common/utils/mail.go b/src/common/utils/email/mail.go similarity index 89% rename from src/common/utils/mail.go rename to src/common/utils/email/mail.go index c4a364cb8..61fc05531 100644 --- a/src/common/utils/mail.go +++ b/src/common/utils/email/mail.go @@ -13,17 +13,19 @@ limitations under the License. */ -package utils +package email import ( "bytes" "crypto/tls" - "strings" + //"strings" "net/smtp" "text/template" - "github.com/astaxie/beego" + //"github.com/astaxie/beego" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/ui/config" ) // Mail holds information about content of Email @@ -34,24 +36,15 @@ type Mail struct { Message string } -// MailConfig holds information about Email configurations -type MailConfig struct { - Identity string - Host string - Port string - Username string - Password string - TLS bool -} - -var mc MailConfig +var mc models.Email // SendMail sends Email according to the configurations func (m Mail) SendMail() error { - - if mc.Host == "" { - loadConfig() + mc, err := config.Email() + if err != nil { + return err } + mailTemplate, err := template.ParseFiles("views/mail.tpl") if err != nil { return err @@ -64,7 +57,7 @@ func (m Mail) SendMail() error { content := mailContent.Bytes() auth := smtp.PlainAuth(mc.Identity, mc.Username, mc.Password, mc.Host) - if mc.TLS { + if mc.SSL { err = sendMailWithTLS(m, auth, content) } else { err = sendMail(m, auth, content) @@ -123,6 +116,7 @@ func sendMailWithTLS(m Mail, auth smtp.Auth, content []byte) error { return client.Quit() } +/* func loadConfig() { config, err := beego.AppConfig.GetSection("mail") if err != nil { @@ -142,3 +136,4 @@ func loadConfig() { TLS: useTLS, } } +*/ diff --git a/src/common/utils/log/logger.go b/src/common/utils/log/logger.go index dceb70a97..db5d3b4a2 100644 --- a/src/common/utils/log/logger.go +++ b/src/common/utils/log/logger.go @@ -22,8 +22,6 @@ import ( "runtime" "sync" "time" - - "github.com/vmware/harbor/src/common/config" ) var logger = New(os.Stdout, NewTextFormatter(), WarningLevel) @@ -31,7 +29,7 @@ var logger = New(os.Stdout, NewTextFormatter(), WarningLevel) func init() { logger.callDepth = 4 - lvl := config.LogLevel() + lvl := os.Getenv("LOG_LEVEL") if len(lvl) == 0 { logger.SetLevel(InfoLevel) return diff --git a/src/common/utils/registry/auth/tokenauthorizer.go b/src/common/utils/registry/auth/tokenauthorizer.go index 14b230e5a..917e43f38 100644 --- a/src/common/utils/registry/auth/tokenauthorizer.go +++ b/src/common/utils/registry/auth/tokenauthorizer.go @@ -25,7 +25,7 @@ import ( "sync" "time" - "github.com/vmware/harbor/src/common/config" + //"github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/common/utils/registry" registry_error "github.com/vmware/harbor/src/common/utils/registry/error" @@ -234,12 +234,15 @@ func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes [] // 2. the realm field returned by registry is an IP which can not reachable // inside Harbor func tokenURL(realm string) string { - extEndpoint := config.ExtEndpoint() - tokenEndpoint := config.TokenEndpoint() - if len(extEndpoint) != 0 && len(tokenEndpoint) != 0 && - strings.Contains(realm, extEndpoint) { - realm = strings.TrimRight(tokenEndpoint, "/") + "/service/token" - } + //TODO + /* + extEndpoint := config.ExtEndpoint() + tokenEndpoint := config.TokenEndpoint() + if len(extEndpoint) != 0 && len(tokenEndpoint) != 0 && + strings.Contains(realm, extEndpoint) { + realm = strings.TrimRight(tokenEndpoint, "/") + "/service/token" + } + */ return realm } diff --git a/src/common/utils/test/adminserver.go b/src/common/utils/test/adminserver.go new file mode 100644 index 000000000..538b861bd --- /dev/null +++ b/src/common/utils/test/adminserver.go @@ -0,0 +1,59 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + + "github.com/vmware/harbor/src/common/models" +) + +// NewAdminserver returns a mock admin server +func NewAdminserver() (*httptest.Server, error) { + m := []*RequestHandlerMapping{} + b, err := json.Marshal(&models.SystemCfg{ + Authentication: &models.Authentication{ + Mode: "db_auth", + }, + Registry: &models.Registry{}, + }) + if err != nil { + return nil, err + } + + resp := &Response{ + StatusCode: http.StatusOK, + Body: b, + } + + m = append(m, &RequestHandlerMapping{ + Method: "GET", + Pattern: "/api/configurations", + Handler: Handler(resp), + }) + + m = append(m, &RequestHandlerMapping{ + Method: "PUT", + Pattern: "/api/configurations", + Handler: Handler(&Response{ + StatusCode: http.StatusOK, + }), + }) + + return NewServer(m...), nil +} diff --git a/src/common/utils/test/test.go b/src/common/utils/test/test.go index 37a435c24..78ea01bcd 100644 --- a/src/common/utils/test/test.go +++ b/src/common/utils/test/test.go @@ -21,6 +21,8 @@ import ( "net/http" "net/http/httptest" "strings" + + "github.com/gorilla/mux" ) // RequestHandlerMapping is a mapping between request and its handler @@ -78,11 +80,11 @@ func Handler(resp *Response) func(http.ResponseWriter, *http.Request) { // NewServer creates a HTTP server for unit test func NewServer(mappings ...*RequestHandlerMapping) *httptest.Server { - mux := http.NewServeMux() + r := mux.NewRouter() for _, mapping := range mappings { - mux.Handle(mapping.Pattern, mapping) + r.PathPrefix(mapping.Pattern).Handler(mapping).Methods(mapping.Method) } - return httptest.NewServer(mux) + return httptest.NewServer(r) } diff --git a/src/common/utils/utils.go b/src/common/utils/utils.go index 4e930514e..a113ac033 100644 --- a/src/common/utils/utils.go +++ b/src/common/utils/utils.go @@ -16,10 +16,14 @@ package utils import ( + "fmt" "math/rand" + "net" "net/url" "strings" "time" + + "github.com/vmware/harbor/src/common/utils/log" ) // FormatEndpoint formats endpoint @@ -70,3 +74,40 @@ func GenerateRandomString() string { } return string(result) } + +// TestTCPConn tests TCP connection +// timeout: the total time before returning if something is wrong +// with the connection, in second +// interval: the interval time for retring after failure, in second +func TestTCPConn(addr string, timeout, interval int) error { + success := make(chan int) + cancel := make(chan int) + + go func() { + for { + select { + case <-cancel: + break + default: + conn, err := net.DialTimeout("tcp", addr, time.Duration(timeout)*time.Second) + if err != nil { + log.Errorf("failed to connect to tcp://%s, retry after %d seconds :%v", + addr, interval, err) + time.Sleep(time.Duration(interval) * time.Second) + continue + } + conn.Close() + success <- 1 + break + } + } + }() + + select { + case <-success: + return nil + case <-time.After(time.Duration(timeout) * time.Second): + cancel <- 1 + return fmt.Errorf("failed to connect to tcp:%s after %d seconds", addr, timeout) + } +} diff --git a/src/common/utils/utils_test.go b/src/common/utils/utils_test.go index cdef47de1..7421ad9c8 100644 --- a/src/common/utils/utils_test.go +++ b/src/common/utils/utils_test.go @@ -17,6 +17,7 @@ package utils import ( "encoding/base64" + "net/http/httptest" "strings" "testing" ) @@ -178,3 +179,12 @@ func TestParseLink(t *testing.T) { t.Errorf("unexpected prev: %s != %s", links.Next(), next) } } + +func TestTestTCPConn(t *testing.T) { + server := httptest.NewServer(nil) + defer server.Close() + addr := strings.TrimLeft(server.URL, "http://") + if err := TestTCPConn(addr, 60, 2); err != nil { + t.Fatalf("failed to test tcp connection of %s: %v", addr, err) + } +} diff --git a/src/jobservice/api/replication.go b/src/jobservice/api/replication.go index 51807ed39..25aef81e4 100644 --- a/src/jobservice/api/replication.go +++ b/src/jobservice/api/replication.go @@ -25,12 +25,12 @@ import ( "github.com/vmware/harbor/src/common/api" "github.com/vmware/harbor/src/common/dao" - "github.com/vmware/harbor/src/jobservice/job" - "github.com/vmware/harbor/src/jobservice/config" - "github.com/vmware/harbor/src/jobservice/utils" "github.com/vmware/harbor/src/common/models" u "github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" + "github.com/vmware/harbor/src/jobservice/job" + "github.com/vmware/harbor/src/jobservice/utils" ) // ReplicationJob handles /api/replicationJobs /api/replicationJobs/:id/log @@ -171,7 +171,13 @@ func (rj *ReplicationJob) GetLog() { rj.RenderError(http.StatusBadRequest, "Invalid job id") return } - logFile := utils.GetJobLogPath(jid) + logFile, err := utils.GetJobLogPath(jid) + if err != nil { + log.Errorf("failed to get log path of job %s: %v", idStr, err) + rj.RenderError(http.StatusInternalServerError, + http.StatusText(http.StatusInternalServerError)) + return + } rj.Ctx.Output.Download(logFile) } diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index 2e1b2c31d..51bf1823f 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -16,121 +16,125 @@ package config import ( - "fmt" + "encoding/json" "os" - "strconv" - "github.com/astaxie/beego" - "github.com/vmware/harbor/src/common/utils/log" + comcfg "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/models" ) -const defaultMaxWorkers int = 10 +var mg *comcfg.Manager -var maxJobWorkers int -var localUIURL string -var localRegURL string -var logDir string -var uiSecret string -var secretKey string -var verifyRemoteCert string +// Configuration of Jobservice +type Configuration struct { + Database *models.Database `json:"database"` + Registry *models.Registry `json:"registry"` + VerifyRemoteCert bool `json:"verify_remote_cert"` + MaxJobWorkers int `json:"max_job_workers"` + JobLogDir string `json:"job_log_dir"` + SecretKey string `json:"secret_key"` + CfgExpiration int `json:"cfg_expiration"` +} -func init() { - maxWorkersEnv := os.Getenv("MAX_JOB_WORKERS") - maxWorkers64, err := strconv.ParseInt(maxWorkersEnv, 10, 32) - maxJobWorkers = int(maxWorkers64) +type parser struct { +} + +func (p *parser) Parse(b []byte) (interface{}, error) { + c := &Configuration{} + if err := json.Unmarshal(b, c); err != nil { + return nil, err + } + return c, nil +} + +// Init configurations +func Init() error { + adminServerURL := os.Getenv("ADMIN_SERVER_URL") + if len(adminServerURL) == 0 { + adminServerURL = "http://adminserver" + } + mg = comcfg.NewManager(adminServerURL, UISecret(), &parser{}, true) + + if err := mg.Init(); err != nil { + return err + } + + if _, err := mg.Load(); err != nil { + return err + } + + return nil +} + +func get() (*Configuration, error) { + c, err := mg.Get() if err != nil { - log.Warningf("Failed to parse max works setting, error: %v, the default value: %d will be used", err, defaultMaxWorkers) - maxJobWorkers = defaultMaxWorkers + return nil, err } + return c.(*Configuration), nil +} - localRegURL = os.Getenv("REGISTRY_URL") - if len(localRegURL) == 0 { - localRegURL = "http://registry:5000" - } - - localUIURL = os.Getenv("UI_URL") - if len(localUIURL) == 0 { - localUIURL = "http://ui" - } - - logDir = os.Getenv("LOG_DIR") - if len(logDir) == 0 { - logDir = "/var/log" - } - - f, err := os.Open(logDir) - defer f.Close() +// VerifyRemoteCert returns bool value. +func VerifyRemoteCert() (bool, error) { + cfg, err := get() if err != nil { - panic(err) + return true, err } - finfo, err := f.Stat() + return cfg.VerifyRemoteCert, nil +} + +// Database ... +func Database() (*models.Database, error) { + cfg, err := get() if err != nil { - panic(err) + return nil, err } - if !finfo.IsDir() { - panic(fmt.Sprintf("%s is not a direcotry", logDir)) - } - - uiSecret = os.Getenv("UI_SECRET") - if len(uiSecret) == 0 { - panic("UI Secret is not set") - } - - verifyRemoteCert = os.Getenv("VERIFY_REMOTE_CERT") - if len(verifyRemoteCert) == 0 { - verifyRemoteCert = "on" - } - - configPath := os.Getenv("CONFIG_PATH") - if len(configPath) != 0 { - log.Infof("Config path: %s", configPath) - beego.LoadAppConfig("ini", configPath) - } - - secretKey = os.Getenv("SECRET_KEY") - if len(secretKey) != 16 { - panic("The length of secretkey has to be 16 characters!") - } - - log.Debugf("config: maxJobWorkers: %d", maxJobWorkers) - log.Debugf("config: localUIURL: %s", localUIURL) - log.Debugf("config: localRegURL: %s", localRegURL) - log.Debugf("config: verifyRemoteCert: %s", verifyRemoteCert) - log.Debugf("config: logDir: %s", logDir) - log.Debugf("config: uiSecret: ******") + return cfg.Database, nil } // MaxJobWorkers ... -func MaxJobWorkers() int { - return maxJobWorkers +func MaxJobWorkers() (int, error) { + cfg, err := get() + if err != nil { + return 0, err + } + return cfg.MaxJobWorkers, nil } // LocalUIURL returns the local ui url, job service will use this URL to call API hosted on ui process func LocalUIURL() string { - return localUIURL + return "http://ui" } // LocalRegURL returns the local registry url, job service will use this URL to pull image from the registry -func LocalRegURL() string { - return localRegURL +func LocalRegURL() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.Registry.URL, nil } // LogDir returns the absolute path to which the log file will be written -func LogDir() string { - return logDir -} - -// UISecret will return the value of secret cookie for jobsevice to call UI API. -func UISecret() string { - return uiSecret +func LogDir() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.JobLogDir, nil } // SecretKey will return the secret key for encryption/decryption password in target. -func SecretKey() string { - return secretKey +func SecretKey() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.SecretKey, nil } -// VerifyRemoteCert return the flag to tell jobservice whether or not verify the cert of remote registry -func VerifyRemoteCert() bool { - return verifyRemoteCert != "off" +// UISecret returns the value of UI secret cookie, used for communication between UI and JobService +// TODO +func UISecret() string { + return os.Getenv("UI_SECRET") } diff --git a/src/jobservice/config/config_test.go b/src/jobservice/config/config_test.go index f33c38a2e..0abcb1655 100644 --- a/src/jobservice/config/config_test.go +++ b/src/jobservice/config/config_test.go @@ -1,9 +1,71 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package config import ( + "os" "testing" + + "github.com/vmware/harbor/src/common/utils/test" ) -func TestMain(t *testing.T) { -} +// test functions under package jobservice/config +func TestConfig(t *testing.T) { + server, err := test.NewAdminserver() + if err != nil { + t.Fatalf("failed to create a mock admin server: %v", err) + } + defer server.Close() + url := os.Getenv("ADMIN_SERVER_URL") + defer os.Setenv("ADMIN_SERVER_URL", url) + + if err := os.Setenv("ADMIN_SERVER_URL", server.URL); err != nil { + t.Fatalf("failed to set env %s: %v", "ADMIN_SERVER_URL", err) + } + + if err := Init(); err != nil { + t.Fatalf("failed to initialize configurations: %v", err) + } + + if _, err := VerifyRemoteCert(); err != nil { + t.Fatalf("failed to get verify remote cert: %v", err) + } + + if _, err := Database(); err != nil { + t.Fatalf("failed to get database settings: %v", err) + } + + if _, err := MaxJobWorkers(); err != nil { + t.Fatalf("failed to get max job workers: %v", err) + } + + LocalUIURL() + + if _, err := LocalRegURL(); err != nil { + t.Fatalf("failed to get registry URL: %v", err) + } + + if _, err := LogDir(); err != nil { + t.Fatalf("failed to get log directory: %v", err) + } + + if _, err := SecretKey(); err != nil { + t.Fatalf("failed to get secret key: %v", err) + } + + UISecret() +} diff --git a/src/jobservice/job/statemachine.go b/src/jobservice/job/statemachine.go index fd12d5443..b6c9cd47a 100644 --- a/src/jobservice/job/statemachine.go +++ b/src/jobservice/job/statemachine.go @@ -20,12 +20,12 @@ import ( "sync" "github.com/vmware/harbor/src/common/dao" - "github.com/vmware/harbor/src/jobservice/config" - "github.com/vmware/harbor/src/jobservice/replication" - "github.com/vmware/harbor/src/jobservice/utils" "github.com/vmware/harbor/src/common/models" uti "github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" + "github.com/vmware/harbor/src/jobservice/replication" + "github.com/vmware/harbor/src/jobservice/utils" ) // RepJobParm wraps the parm of a job @@ -184,14 +184,17 @@ func (sm *SM) Init() { } // Reset resets the state machine so it will start handling another job. -func (sm *SM) Reset(jid int64) error { +func (sm *SM) Reset(jid int64) (err error) { //To ensure the new jobID is visible to the thread to stop the SM sm.lock.Lock() sm.JobID = jid sm.desiredState = "" sm.lock.Unlock() - sm.Logger = utils.NewLogger(sm.JobID) + sm.Logger, err = utils.NewLogger(sm.JobID) + if err != nil { + return + } //init parms job, err := dao.GetRepJob(sm.JobID) if err != nil { @@ -207,13 +210,22 @@ func (sm *SM) Reset(jid int64) error { if policy == nil { return fmt.Errorf("The policy doesn't exist in DB, policy id:%d", job.PolicyID) } + + regURL, err := config.LocalRegURL() + if err != nil { + return err + } + verify, err := config.VerifyRemoteCert() + if err != nil { + return err + } sm.Parms = &RepJobParm{ - LocalRegURL: config.LocalRegURL(), + LocalRegURL: regURL, Repository: job.Repository, Tags: job.TagList, Enabled: policy.Enabled, Operation: job.Operation, - Insecure: !config.VerifyRemoteCert(), + Insecure: !verify, } if policy.Enabled == 0 { //worker will cancel this job @@ -231,7 +243,11 @@ func (sm *SM) Reset(jid int64) error { pwd := target.Password if len(pwd) != 0 { - pwd, err = uti.ReversibleDecrypt(pwd, config.SecretKey()) + key, err := config.SecretKey() + if err != nil { + return err + } + pwd, err = uti.ReversibleDecrypt(pwd, key) if err != nil { return fmt.Errorf("failed to decrypt password: %v", err) } diff --git a/src/jobservice/job/workerpool.go b/src/jobservice/job/workerpool.go index a1034441a..06301cd69 100644 --- a/src/jobservice/job/workerpool.go +++ b/src/jobservice/job/workerpool.go @@ -17,9 +17,9 @@ package job import ( "github.com/vmware/harbor/src/common/dao" - "github.com/vmware/harbor/src/jobservice/config" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" ) type workerPool struct { @@ -111,17 +111,22 @@ func NewWorker(id int) *Worker { } // InitWorkerPool create workers according to configuration. -func InitWorkerPool() { - WorkerPool = &workerPool{ - workerChan: make(chan *Worker, config.MaxJobWorkers()), - workerList: make([]*Worker, 0, config.MaxJobWorkers()), +func InitWorkerPool() error { + n, err := config.MaxJobWorkers() + if err != nil { + return err } - for i := 0; i < config.MaxJobWorkers(); i++ { + WorkerPool = &workerPool{ + workerChan: make(chan *Worker, n), + workerList: make([]*Worker, 0, n), + } + for i := 0; i < n; i++ { worker := NewWorker(i) WorkerPool.workerList = append(WorkerPool.workerList, worker) worker.Start() log.Debugf("worker %d started", worker.ID) } + return nil } // Dispatch will listen to the jobQueue of job service and try to pick a free worker from the worker pool and assign the job to it. diff --git a/src/jobservice/main.go b/src/jobservice/main.go index 5467d96c6..00a91921c 100644 --- a/src/jobservice/main.go +++ b/src/jobservice/main.go @@ -16,15 +16,32 @@ package main import ( + "os" + "github.com/astaxie/beego" "github.com/vmware/harbor/src/common/dao" - "github.com/vmware/harbor/src/jobservice/job" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" + "github.com/vmware/harbor/src/jobservice/job" ) func main() { - dao.InitDatabase() + log.Info("initializing configurations...") + if err := config.Init(); err != nil { + log.Fatalf("failed to initialize configurations: %v", err) + } + log.Info("configurations initialization completed") + + database, err := config.Database() + if err != nil { + log.Fatalf("failed to get database configurations: %v", err) + } + + if err := dao.InitDatabase(database); err != nil { + log.Fatalf("failed to initialize database: %v", err) + } + initRouters() job.InitWorkerPool() go job.Dispatch() @@ -48,3 +65,11 @@ func resumeJobs() { log.Warningf("Failed to jobs to resume, error: %v", err) } } + +func init() { + configPath := os.Getenv("CONFIG_PATH") + if len(configPath) != 0 { + log.Infof("Config path: %s", configPath) + beego.LoadAppConfig("ini", configPath) + } +} diff --git a/src/jobservice/utils/logger.go b/src/jobservice/utils/logger.go index 6a358285c..7b3b164f4 100644 --- a/src/jobservice/utils/logger.go +++ b/src/jobservice/utils/logger.go @@ -18,16 +18,20 @@ package utils import ( "fmt" - "github.com/vmware/harbor/src/jobservice/config" - "github.com/vmware/harbor/src/common/utils/log" "os" "path/filepath" "strconv" + + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" ) // NewLogger create a logger for a speicified job -func NewLogger(jobID int64) *log.Logger { - logFile := GetJobLogPath(jobID) +func NewLogger(jobID int64) (*log.Logger, error) { + logFile, err := GetJobLogPath(jobID) + if err != nil { + return nil, err + } d := filepath.Dir(logFile) if _, err := os.Stat(d); os.IsNotExist(err) { err := os.MkdirAll(d, 0660) @@ -40,11 +44,11 @@ func NewLogger(jobID int64) *log.Logger { log.Errorf("Failed to open log file %s, the log of job %d will be printed to standard output, the error: %v", logFile, jobID, err) f = os.Stdout } - return log.New(f, log.NewTextFormatter(), log.InfoLevel) + return log.New(f, log.NewTextFormatter(), log.InfoLevel), nil } // GetJobLogPath returns the absolute path in which the job log file is located. -func GetJobLogPath(jobID int64) string { +func GetJobLogPath(jobID int64) (string, error) { f := fmt.Sprintf("job_%d.log", jobID) k := jobID / 1000 p := "" @@ -61,6 +65,10 @@ func GetJobLogPath(jobID int64) string { p = filepath.Join(d, p) } - p = filepath.Join(config.LogDir(), p, f) - return p + base, err := config.LogDir() + if err != nil { + return "", err + } + p = filepath.Join(base, p, f) + return p, nil } diff --git a/src/ui/api/config.go b/src/ui/api/config.go new file mode 100644 index 000000000..1c17ad8e0 --- /dev/null +++ b/src/ui/api/config.go @@ -0,0 +1,259 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "fmt" + "net/http" + "strconv" + //"strings" + + "github.com/vmware/harbor/src/common/api" + comcfg "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/dao" + //"github.com/vmware/harbor/src/common/models" + //"github.com/vmware/harbor/src/common/utils" + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/config" +) + +// ConfigAPI ... +type ConfigAPI struct { + api.BaseAPI +} + +// Prepare validates the user +func (c *ConfigAPI) Prepare() { + userID := c.ValidateUser() + isSysAdmin, err := dao.IsAdminRole(userID) + if err != nil { + log.Errorf("failed to check the role of user: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if !isSysAdmin { + c.CustomAbort(http.StatusForbidden, http.StatusText(http.StatusForbidden)) + } +} + +// Get returns configurations +func (c *ConfigAPI) Get() { + cfg, err := config.GetSystemCfg() + if err != nil { + log.Errorf("failed to get configurations: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if cfg.Database.MySQL != nil { + cfg.Database.MySQL.Password = "" + } + + cfg.InitialAdminPwd = "" + cfg.SecretKey = "" + + m := map[string]interface{}{} + m["config"] = cfg + + editable, err := dao.AuthModeCanBeModified() + if err != nil { + log.Errorf("failed to determinie whether auth mode can be modified: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + m["auth_mode_editable"] = editable + + c.Data["json"] = m + c.ServeJSON() +} + +// Put updates configurations +func (c *ConfigAPI) Put() { + m := map[string]string{} + c.DecodeJSONReq(&m) + if err := validateCfg(m); err != nil { + c.CustomAbort(http.StatusBadRequest, err.Error()) + } + + if value, ok := m[comcfg.AUTHMode]; ok { + mode, err := config.AuthMode() + if err != nil { + log.Errorf("failed to get auth mode: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if mode != value { + flag, err := authModeCanBeModified() + if err != nil { + log.Errorf("failed to determine whether auth mode can be modified: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if !flag { + c.CustomAbort(http.StatusBadRequest, + fmt.Sprintf("%s can not be modified as new users have been inserted into database", + comcfg.AUTHMode)) + } + } + } + + if err := config.Upload(m); err != nil { + log.Errorf("failed to upload configurations: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if err := config.Load(); err != nil { + log.Errorf("failed to load configurations: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } +} + +func validateCfg(c map[string]string) error { + if value, ok := c[comcfg.AUTHMode]; ok { + if value != comcfg.DBAuth && value != comcfg.LDAPAuth { + return fmt.Errorf("invalid %s, shoud be %s or %s", comcfg.AUTHMode, comcfg.DBAuth, comcfg.LDAPAuth) + } + + if value == comcfg.LDAPAuth { + if _, ok := c[comcfg.LDAPURL]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAPURL) + } + if _, ok := c[comcfg.LDAPBaseDN]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAPBaseDN) + } + if _, ok := c[comcfg.LDAPUID]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAPUID) + } + if _, ok := c[comcfg.LDAPScope]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAPScope) + } + } + } + + if ldapURL, ok := c[comcfg.LDAPURL]; ok && len(ldapURL) == 0 { + return fmt.Errorf("%s is empty", comcfg.LDAPURL) + } + if baseDN, ok := c[comcfg.LDAPBaseDN]; ok && len(baseDN) == 0 { + return fmt.Errorf("%s is empty", comcfg.LDAPBaseDN) + } + if uID, ok := c[comcfg.LDAPUID]; ok && len(uID) == 0 { + return fmt.Errorf("%s is empty", comcfg.LDAPUID) + } + if scope, ok := c[comcfg.LDAPScope]; ok && + scope != comcfg.LDAPScopeBase && + scope != comcfg.LDAPScopeOnelevel && + scope != comcfg.LDAPScopeSubtree { + return fmt.Errorf("invalid %s, should be %s, %s or %s", + comcfg.LDAPScope, + comcfg.LDAPScopeBase, + comcfg.LDAPScopeOnelevel, + comcfg.LDAPScopeSubtree) + } + if timeout, ok := c[comcfg.LDAPTimeout]; ok { + if t, err := strconv.Atoi(timeout); err != nil || t < 0 { + return fmt.Errorf("invalid %s", comcfg.LDAPTimeout) + } + } + + if self, ok := c[comcfg.SelfRegistration]; ok && + self != "0" && self != "1" { + return fmt.Errorf("%s should be %s or %s", + comcfg.SelfRegistration, "0", "1") + } + + if port, ok := c[comcfg.EmailPort]; ok { + if p, err := strconv.Atoi(port); err != nil || p < 0 || p > 65535 { + return fmt.Errorf("invalid %s", comcfg.EmailPort) + } + } + + if ssl, ok := c[comcfg.EmailSSL]; ok && ssl != "0" && ssl != "1" { + return fmt.Errorf("%s should be %s or %s", comcfg.EmailSSL, "0", "1") + } + + if crt, ok := c[comcfg.ProjectCreationRestriction]; ok && + crt != comcfg.ProCrtRestrEveryone && + crt != comcfg.ProCrtRestrAdmOnly { + return fmt.Errorf("invalid %s, should be %s or %s", + comcfg.ProjectCreationRestriction, + comcfg.ProCrtRestrAdmOnly, + comcfg.ProCrtRestrEveryone) + } + + if verify, ok := c[comcfg.VerifyRemoteCert]; ok && verify != "0" && verify != "1" { + return fmt.Errorf("invalid %s, should be %s or %s", + comcfg.VerifyRemoteCert, "0", "1") + } + + return nil +} + +/* +func convert() ([]*models.Config, error) { + cfgs := []*models.Config{} + var err error + pwdKeys := []string{config.LDAP_SEARCH_PWD, config.EMAIL_PWD} + for _, pwdKey := range pwdKeys { + if pwd, ok := c[pwdKey]; ok && len(pwd) != 0 { + c[pwdKey], err = utils.ReversibleEncrypt(pwd, ui_cfg.SecretKey()) + if err != nil { + return nil, err + } + } + } + + for _, key := range configKeys { + if value, ok := c[key]; ok { + cfgs = append(cfgs, &models.Config{ + Key: key, + Value: value, + }) + } + } + + return cfgs, nil +} +*/ +/* +//[]*models.Config >> cfgForGet +func convert(cfg *config.Configuration) (map[string]interface{}, error) { + result := map[string]interface{}{} + + for _, config := range configs { + cfg[config.Key] = &value{ + Value: config.Value, + Editable: true, + } + } + + dels := []string{config.LDAP_SEARCH_PWD, config.EMAIL_PWD} + for _, del := range dels { + if _, ok := cfg[del]; ok { + delete(cfg, del) + } + } + + flag, err := authModeCanBeModified() + if err != nil { + return nil, err + } + cfg[config.AUTH_MODE].Editable = flag + + return cfgForGet(cfg), nil +} +*/ +func authModeCanBeModified() (bool, error) { + return dao.AuthModeCanBeModified() +} diff --git a/src/ui/api/harborapi_test.go b/src/ui/api/harborapi_test.go index 979f661d1..ac6dd6819 100644 --- a/src/ui/api/harborapi_test.go +++ b/src/ui/api/harborapi_test.go @@ -14,6 +14,7 @@ import ( "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils" + "github.com/vmware/harbor/src/ui/config" "github.com/vmware/harbor/tests/apitests/apilib" // "strconv" // "strings" @@ -57,7 +58,14 @@ type usrInfo struct { } func init() { - dao.InitDatabase() + if err := config.Init(); err != nil { + log.Fatalf("failed to initialize configurations: %v", err) + } + database, err := config.Database() + if err != nil { + log.Fatalf("failed to get database configurations: %v", err) + } + dao.InitDatabase(database) _, file, _, _ := runtime.Caller(1) apppath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, ".."+string(filepath.Separator)))) beego.BConfig.WebConfig.Session.SessionOn = true @@ -513,7 +521,7 @@ func (a testapi) GetReposTop(authInfo usrInfo, count string) (int, error) { //-------------------------Targets Test---------------------------------------// //Create a new replication target -func (a testapi) AddTargets(authInfo usrInfo, repTarget apilib.RepTargetPost) (int, error) { +func (a testapi) AddTargets(authInfo usrInfo, repTarget apilib.RepTargetPost) (int, string, error) { _sling := sling.New().Post(a.basePath) path := "/api/targets" @@ -521,8 +529,8 @@ func (a testapi) AddTargets(authInfo usrInfo, repTarget apilib.RepTargetPost) (i _sling = _sling.Path(path) _sling = _sling.BodyJSON(repTarget) - httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo) - return httpStatusCode, err + httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo) + return httpStatusCode, string(body), err } //List filters targets by name diff --git a/src/ui/api/project.go b/src/ui/api/project.go index 7ba50fa3b..b28e90600 100644 --- a/src/ui/api/project.go +++ b/src/ui/api/project.go @@ -44,7 +44,8 @@ type projectReq struct { } const projectNameMaxLen int = 30 -const projectNameMinLen int = 4 +const projectNameMinLen int = 2 +const restrictedNameChars = `[a-z0-9]+(?:[._-][a-z0-9]+)*` const dupProjectPattern = `Duplicate entry '\w+' for key 'name'` // Prepare validates the URL and the user @@ -77,7 +78,13 @@ func (p *ProjectAPI) Post() { if err != nil { log.Errorf("Failed to check admin role: %v", err) } - if !isSysAdmin && config.OnlyAdminCreateProject() { + + onlyAdmin, err := config.OnlyAdminCreateProject() + if err != nil { + log.Errorf("failed to determine whether only admin can create projects: %v", err) + p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + if !isSysAdmin && onlyAdmin { log.Errorf("Only sys admin can create project") p.RenderError(http.StatusForbidden, "Only system admin can create project") return @@ -417,9 +424,9 @@ func isProjectAdmin(userID int, pid int64) bool { func validateProjectReq(req projectReq) error { pn := req.ProjectName if isIllegalLength(req.ProjectName, projectNameMinLen, projectNameMaxLen) { - return fmt.Errorf("Project name is illegal in length. (greater than 4 or less than 30)") + return fmt.Errorf("Project name is illegal in length. (greater than 2 or less than 30)") } - validProjectName := regexp.MustCompile(`^[a-z0-9](?:-*[a-z0-9])*(?:[._][a-z0-9](?:-*[a-z0-9])*)*$`) + validProjectName := regexp.MustCompile(`^` + restrictedNameChars + `$`) legal := validProjectName.MatchString(pn) if !legal { return fmt.Errorf("project name is not in lower case or contains illegal characters") diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index 4db0d1eb1..b229c0e96 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -361,11 +361,19 @@ func (ra *RepositoryAPI) GetManifests() { } func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repository, err error) { - endpoint := config.InternalRegistryURL() + endpoint, err := config.RegistryURL() + if err != nil { + return nil, err + } + + verify, err := config.VerifyRemoteCert() + if err != nil { + return nil, err + } username, password, ok := ra.Ctx.Request.BasicAuth() if ok { - return newRepositoryClient(endpoint, api.GetIsInsecure(), username, password, + return newRepositoryClient(endpoint, !verify, username, password, repoName, "repository", repoName, "pull", "push", "*") } @@ -374,7 +382,7 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo return nil, err } - return cache.NewRepositoryClient(endpoint, api.GetIsInsecure(), username, repoName, + return cache.NewRepositoryClient(endpoint, !verify, username, repoName, "repository", repoName, "pull", "push", "*") } @@ -416,7 +424,12 @@ func (ra *RepositoryAPI) GetTopRepos() { ra.CustomAbort(http.StatusBadRequest, "invalid count") } - repos, err := dao.GetTopRepos(count) + userID, _, ok := ra.GetUserIDForRequest() + if !ok { + userID = dao.NonExistUserID + } + + repos, err := dao.GetTopRepos(userID, count) if err != nil { log.Errorf("failed to get top repos: %v", err) ra.CustomAbort(http.StatusInternalServerError, "internal server error") diff --git a/src/ui/api/target.go b/src/ui/api/target.go index abe59a09c..becda414e 100644 --- a/src/ui/api/target.go +++ b/src/ui/api/target.go @@ -41,7 +41,12 @@ type TargetAPI struct { // Prepare validates the user func (t *TargetAPI) Prepare() { - t.secretKey = config.SecretKey() + var err error + t.secretKey, err = config.SecretKey() + if err != nil { + log.Errorf("failed to get secret key: %v", err) + t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } userID := t.ValidateUser() isSysAdmin, err := dao.IsAdminRole(userID) @@ -97,7 +102,12 @@ func (t *TargetAPI) Ping() { password = t.GetString("password") } - registry, err := newRegistryClient(endpoint, api.GetIsInsecure(), username, password, + verify, err := config.VerifyRemoteCert() + if err != nil { + log.Errorf("failed to check whether insecure or not: %v", err) + t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + registry, err := newRegistryClient(endpoint, !verify, username, password, "", "", "") if err != nil { // timeout, dns resolve error, connection refused, etc. diff --git a/src/ui/api/target_test.go b/src/ui/api/target_test.go index 94b6b25e0..62709a39e 100644 --- a/src/ui/api/target_test.go +++ b/src/ui/api/target_test.go @@ -30,17 +30,18 @@ func TestTargetsPost(t *testing.T) { //-------------------case 1 : response code = 201------------------------// fmt.Println("case 1 : response code = 201") - httpStatusCode, err = apiTest.AddTargets(*admin, *repTargets) + httpStatusCode, body, err := apiTest.AddTargets(*admin, *repTargets) if err != nil { t.Error("Error whihle add targets", err.Error()) t.Log(err) } else { assert.Equal(int(201), httpStatusCode, "httpStatusCode should be 201") + t.Log(body) } //-----------case 2 : response code = 409,name is already used-----------// fmt.Println("case 2 : response code = 409,name is already used") - httpStatusCode, err = apiTest.AddTargets(*admin, *repTargets) + httpStatusCode, _, err = apiTest.AddTargets(*admin, *repTargets) if err != nil { t.Error("Error whihle add targets", err.Error()) t.Log(err) @@ -51,7 +52,7 @@ func TestTargetsPost(t *testing.T) { //-----------case 3 : response code = 409,name is already used-----------// fmt.Println("case 3 : response code = 409,endPoint is already used") repTargets.Username = "errName" - httpStatusCode, err = apiTest.AddTargets(*admin, *repTargets) + httpStatusCode, _, err = apiTest.AddTargets(*admin, *repTargets) if err != nil { t.Error("Error whihle add targets", err.Error()) t.Log(err) @@ -61,7 +62,7 @@ func TestTargetsPost(t *testing.T) { //--------case 4 : response code = 401,User need to log in first.--------// fmt.Println("case 4 : response code = 401,User need to log in first.") - httpStatusCode, err = apiTest.AddTargets(*unknownUsr, *repTargets) + httpStatusCode, _, err = apiTest.AddTargets(*unknownUsr, *repTargets) if err != nil { t.Error("Error whihle add targets", err.Error()) t.Log(err) diff --git a/src/ui/api/user.go b/src/ui/api/user.go index 7c1212b45..d5ca84efb 100644 --- a/src/ui/api/user.go +++ b/src/ui/api/user.go @@ -46,10 +46,21 @@ type passwordReq struct { // Prepare validates the URL and parms func (ua *UserAPI) Prepare() { + mode, err := config.AuthMode() + if err != nil { + log.Errorf("failed to get auth mode: %v", err) + ua.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } - ua.AuthMode = config.AuthMode() + ua.AuthMode = mode - ua.SelfRegistration = config.SelfRegistration() + self, err := config.SelfRegistration() + if err != nil { + log.Errorf("failed to get self registration: %v", err) + ua.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + ua.SelfRegistration = self if ua.Ctx.Input.IsPost() { sessionUserID := ua.GetSession("userId") @@ -82,7 +93,6 @@ func (ua *UserAPI) Prepare() { } } - var err error ua.IsAdmin, err = dao.IsAdminRole(ua.currentUserID) if err != nil { log.Errorf("Error occurred in IsAdminRole:%v", err) @@ -234,7 +244,7 @@ func (ua *UserAPI) Delete() { return } - if config.AuthMode() == "ldap_auth" { + if ua.AuthMode == "ldap_auth" { ua.CustomAbort(http.StatusForbidden, "user can not be deleted in LDAP authentication mode") } diff --git a/src/ui/api/utils.go b/src/ui/api/utils.go index 185db1f18..73a11c869 100644 --- a/src/ui/api/utils.go +++ b/src/ui/api/utils.go @@ -20,11 +20,9 @@ import ( "encoding/json" "fmt" "io/ioutil" - "net" "net/http" "sort" "strings" - "time" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" @@ -242,7 +240,7 @@ func addAuthentication(req *http.Request) { // SyncRegistry syncs the repositories of registry with database. func SyncRegistry() error { - log.Debugf("Start syncing repositories from registry to DB... ") + log.Infof("Start syncing repositories from registry to DB... ") reposInRegistry, err := catalog() if err != nil { @@ -304,7 +302,7 @@ func SyncRegistry() error { } } - log.Debugf("Sync repositories from registry to DB is done.") + log.Infof("Sync repositories from registry to DB is done.") return nil } @@ -350,7 +348,10 @@ func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string } // TODO remove the workaround when the bug of registry is fixed - endpoint := config.InternalRegistryURL() + endpoint, err := config.RegistryURL() + if err != nil { + return needsAdd, needsDel, err + } client, err := cache.NewRepositoryClient(endpoint, true, "admin", repoInR, "repository", repoInR) if err != nil { @@ -372,7 +373,10 @@ func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string j++ } else { // TODO remove the workaround when the bug of registry is fixed - endpoint := config.InternalRegistryURL() + endpoint, err := config.RegistryURL() + if err != nil { + return needsAdd, needsDel, err + } client, err := cache.NewRepositoryClient(endpoint, true, "admin", repoInR, "repository", repoInR) if err != nil { @@ -422,32 +426,18 @@ func projectExists(repository string) (bool, error) { } func initRegistryClient() (r *registry.Registry, err error) { - endpoint := config.InternalRegistryURL() - - addr := endpoint - if strings.Contains(endpoint, "/") { - addr = endpoint[strings.LastIndex(endpoint, "/")+1:] + endpoint, err := config.RegistryURL() + if err != nil { + return nil, err } - ch := make(chan int, 1) - go func() { - var err error - var c net.Conn - for { - c, err = net.DialTimeout("tcp", addr, 20*time.Second) - if err == nil { - c.Close() - ch <- 1 - } else { - log.Errorf("failed to connect to registry client, retry after 2 seconds :%v", err) - time.Sleep(2 * time.Second) - } - } - }() - select { - case <-ch: - case <-time.After(60 * time.Second): - panic("Failed to connect to registry client after 60 seconds") + addr := endpoint + if strings.Contains(endpoint, "://") { + addr = strings.Split(endpoint, "://")[1] + } + + if err := utils.TestTCPConn(addr, 60, 2); err != nil { + return nil, err } registryClient, err := cache.NewRegistryClient(endpoint, true, "admin", diff --git a/src/ui/auth/authenticator.go b/src/ui/auth/authenticator.go index 23abdc8f2..d255b60a9 100644 --- a/src/ui/auth/authenticator.go +++ b/src/ui/auth/authenticator.go @@ -50,7 +50,10 @@ func Register(name string, authenticator Authenticator) { // Login authenticates user credentials based on setting. func Login(m models.AuthModel) (*models.User, error) { - var authMode = config.AuthMode() + authMode, err := config.AuthMode() + if err != nil { + return nil, err + } if authMode == "" || m.Principal == "admin" { authMode = "db_auth" } diff --git a/src/ui/auth/ldap/ldap.go b/src/ui/auth/ldap/ldap.go index f5230ba6f..e3928a201 100644 --- a/src/ui/auth/ldap/ldap.go +++ b/src/ui/auth/ldap/ldap.go @@ -16,18 +16,15 @@ package ldap import ( + "crypto/tls" "errors" "fmt" - "strconv" "strings" "time" - "crypto/tls" - - "github.com/vmware/harbor/src/common/utils/log" - "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/auth" "github.com/vmware/harbor/src/ui/config" @@ -41,9 +38,8 @@ const metaChars = "&|!=~*<>()" // Connect checks the LDAP configuration directives, and connects to the LDAP URL // Returns an LDAP connection -func Connect() (*goldap.Conn, error) { - - ldapURL := config.LDAP().URL +func Connect(settings *models.LDAP) (*goldap.Conn, error) { + ldapURL := settings.URL if ldapURL == "" { return nil, errors.New("can not get any available LDAP_URL") } @@ -70,13 +66,10 @@ func Connect() (*goldap.Conn, error) { } // Sets a Dial Timeout for LDAP - cTimeout := config.LDAP().ConnectTimeout - connectTimeout, _ := strconv.Atoi(cTimeout) - goldap.DefaultTimeout = time.Duration(connectTimeout) * time.Second + goldap.DefaultTimeout = time.Duration(settings.Timeout) * time.Second var ldap *goldap.Conn var err error - switch protocol { case "ldap": ldap, err = goldap.Dial("tcp", fmt.Sprintf("%s:%s", host, port)) @@ -104,21 +97,26 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { } } - ldap, err := Connect() + settings, err := config.LDAP() if err != nil { return nil, err } - ldapBaseDn := config.LDAP().BaseDn + ldap, err := Connect(settings) + if err != nil { + return nil, err + } + + ldapBaseDn := settings.BaseDN if ldapBaseDn == "" { return nil, errors.New("can not get any available LDAP_BASE_DN") } log.Debug("baseDn:", ldapBaseDn) - ldapSearchDn := config.LDAP().SearchDn + ldapSearchDn := settings.SearchDN if ldapSearchDn != "" { log.Debug("Search DN: ", ldapSearchDn) - ldapSearchPwd := config.LDAP().SearchPwd + ldapSearchPwd := settings.SearchPwd err = ldap.Bind(ldapSearchDn, ldapSearchPwd) if err != nil { log.Debug("Bind search dn error", err) @@ -126,8 +124,8 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { } } - attrName := config.LDAP().UID - filter := config.LDAP().Filter + attrName := settings.UID + filter := settings.Filter if filter != "" { filter = "(&" + filter + "(" + attrName + "=" + m.Principal + "))" } else { @@ -135,11 +133,11 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { } log.Debug("one or more filter", filter) - ldapScope := config.LDAP().Scope + ldapScope := settings.Scope var scope int - if ldapScope == "1" { + if ldapScope == 1 { scope = goldap.ScopeBaseObject - } else if ldapScope == "2" { + } else if ldapScope == 2 { scope = goldap.ScopeSingleLevel } else { scope = goldap.ScopeWholeSubtree diff --git a/src/ui/config/config.go b/src/ui/config/config.go index dd69504b6..ccb760f80 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -13,145 +13,220 @@ limitations under the License. */ -// Package config provides methods to get configurations required by code in src/ui package config import ( - "strconv" - "strings" + "encoding/json" + "os" - commonConfig "github.com/vmware/harbor/src/common/config" + comcfg "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" ) -// LDAPSetting wraps the setting of an LDAP server -type LDAPSetting struct { - URL string - BaseDn string - SearchDn string - SearchPwd string - UID string - Filter string - Scope string - ConnectTimeout string +var mg *comcfg.Manager + +// Configuration of UI +type Configuration struct { + DomainName string `json:"domain_name"` // Harbor external URL: protocal://host:port + Authentication *models.Authentication `json:"authentication"` + Database *models.Database `json:"database"` + TokenService *models.TokenService `json:"token_service"` + Registry *models.Registry `json:"registry"` + Email *models.Email `json:"email"` + VerifyRemoteCert bool `json:"verify_remote_cert"` + ProjectCreationRestriction string `json:"project_creation_restriction"` + InitialAdminPwd string `json:"initial_admin_pwd"` + //TODO remove + CompressJS bool `json:"compress_js"` + TokenExpiration int `json:"token_expiration"` + SecretKey string `json:"secret_key"` + CfgExpiration int `json:"cfg_expiration"` } -type uiParser struct{} +type parser struct { +} -// Parse parses the auth settings url settings and other configuration consumed by code under src/ui -func (up *uiParser) Parse(raw map[string]string, config map[string]interface{}) error { - mode := raw["AUTH_MODE"] - if mode == "ldap_auth" { - setting := LDAPSetting{ - URL: raw["LDAP_URL"], - BaseDn: raw["LDAP_BASE_DN"], - SearchDn: raw["LDAP_SEARCH_DN"], - SearchPwd: raw["LDAP_SEARCH_PWD"], - UID: raw["LDAP_UID"], - Filter: raw["LDAP_FILTER"], - Scope: raw["LDAP_SCOPE"], - ConnectTimeout: raw["LDAP_CONNECT_TIMEOUT"], - } - config["ldap"] = setting +func (p *parser) Parse(b []byte) (interface{}, error) { + c := &Configuration{} + if err := json.Unmarshal(b, c); err != nil { + return nil, err } - config["auth_mode"] = mode - var tokenExpiration = 30 //minutes - if len(raw["TOKEN_EXPIRATION"]) > 0 { - i, err := strconv.Atoi(raw["TOKEN_EXPIRATION"]) - if err != nil { - log.Warningf("failed to parse token expiration: %v, using default value %d", err, tokenExpiration) - } else if i <= 0 { - log.Warningf("invalid token expiration, using default value: %d minutes", tokenExpiration) - } else { - tokenExpiration = i - } + return c, nil +} + +// Init configurations +func Init() error { + adminServerURL := os.Getenv("ADMIN_SERVER_URL") + if len(adminServerURL) == 0 { + adminServerURL = "http://adminserver" } - config["token_exp"] = tokenExpiration - config["admin_password"] = raw["HARBOR_ADMIN_PASSWORD"] - config["ext_reg_url"] = raw["EXT_REG_URL"] - config["ui_secret"] = raw["UI_SECRET"] - config["secret_key"] = raw["SECRET_KEY"] - config["self_registration"] = raw["SELF_REGISTRATION"] != "off" - config["admin_create_project"] = strings.ToLower(raw["PROJECT_CREATION_RESTRICTION"]) == "adminonly" - registryURL := raw["REGISTRY_URL"] - registryURL = strings.TrimRight(registryURL, "/") - config["internal_registry_url"] = registryURL - jobserviceURL := raw["JOB_SERVICE_URL"] - jobserviceURL = strings.TrimRight(jobserviceURL, "/") - config["internal_jobservice_url"] = jobserviceURL + log.Debugf("admin server URL: %s", adminServerURL) + mg = comcfg.NewManager(adminServerURL, UISecret(), &parser{}, true) + + if err := mg.Init(); err != nil { + return err + } + + if _, err := mg.Load(); err != nil { + return err + } + return nil } -var uiConfig *commonConfig.Config - -func init() { - uiKeys := []string{"AUTH_MODE", "LDAP_URL", "LDAP_BASE_DN", "LDAP_SEARCH_DN", "LDAP_SEARCH_PWD", "LDAP_UID", "LDAP_FILTER", "LDAP_SCOPE", "LDAP_CONNECT_TIMEOUT", "TOKEN_EXPIRATION", "HARBOR_ADMIN_PASSWORD", "EXT_REG_URL", "UI_SECRET", "SECRET_KEY", "SELF_REGISTRATION", "PROJECT_CREATION_RESTRICTION", "REGISTRY_URL", "JOB_SERVICE_URL"} - uiConfig = &commonConfig.Config{ - Config: make(map[string]interface{}), - Loader: &commonConfig.EnvConfigLoader{Keys: uiKeys}, - Parser: &uiParser{}, - } - if err := uiConfig.Load(); err != nil { - panic(err) +func get() (*Configuration, error) { + c, err := mg.Get() + if err != nil { + return nil, err } + return c.(*Configuration), nil } -// Reload ... -func Reload() error { - return uiConfig.Load() +// Load configurations +func Load() error { + _, err := mg.Load() + return err +} + +// Upload uploads all system configutations to admin server +func Upload(cfg map[string]string) error { + b, err := json.Marshal(cfg) + if err != nil { + return err + } + return mg.Upload(b) +} + +// GetSystemCfg returns the system configurations +func GetSystemCfg() (*models.SystemCfg, error) { + raw, err := mg.Loader.Load() + if err != nil { + return nil, err + } + + cfg := &models.SystemCfg{} + if err = json.Unmarshal(raw, cfg); err != nil { + return nil, err + } + return cfg, nil } // AuthMode ... -func AuthMode() string { - return uiConfig.Config["auth_mode"].(string) +func AuthMode() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.Authentication.Mode, nil } // LDAP returns the setting of ldap server -func LDAP() LDAPSetting { - return uiConfig.Config["ldap"].(LDAPSetting) +func LDAP() (*models.LDAP, error) { + cfg, err := get() + if err != nil { + return nil, err + } + return cfg.Authentication.LDAP, nil } // TokenExpiration returns the token expiration time (in minute) -func TokenExpiration() int { - return uiConfig.Config["token_exp"].(int) +func TokenExpiration() (int, error) { + cfg, err := get() + if err != nil { + return 0, err + } + return cfg.TokenExpiration, nil } -// ExtRegistryURL returns the registry URL to exposed to external client -func ExtRegistryURL() string { - return uiConfig.Config["ext_reg_url"].(string) -} - -// UISecret returns the value of UI secret cookie, used for communication between UI and JobService -func UISecret() string { - return uiConfig.Config["ui_secret"].(string) +// DomainName returns the external URL of Harbor: protocal://host:port +func DomainName() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.DomainName, nil } // SecretKey returns the secret key to encrypt the password of target -func SecretKey() string { - return uiConfig.Config["secret_key"].(string) +func SecretKey() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.SecretKey, nil } // SelfRegistration returns the enablement of self registration -func SelfRegistration() bool { - return uiConfig.Config["self_registration"].(bool) +func SelfRegistration() (bool, error) { + cfg, err := get() + if err != nil { + return false, err + } + return cfg.Authentication.SelfRegistration, nil } -// InternalRegistryURL returns registry URL for internal communication between Harbor containers -func InternalRegistryURL() string { - return uiConfig.Config["internal_registry_url"].(string) +// RegistryURL ... +func RegistryURL() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.Registry.URL, nil } // InternalJobServiceURL returns jobservice URL for internal communication between Harbor containers func InternalJobServiceURL() string { - return uiConfig.Config["internal_jobservice_url"].(string) + return "http://jobservice" } // InitialAdminPassword returns the initial password for administrator -func InitialAdminPassword() string { - return uiConfig.Config["admin_password"].(string) +func InitialAdminPassword() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.InitialAdminPwd, nil } // OnlyAdminCreateProject returns the flag to restrict that only sys admin can create project -func OnlyAdminCreateProject() bool { - return uiConfig.Config["admin_create_project"].(bool) +func OnlyAdminCreateProject() (bool, error) { + cfg, err := get() + if err != nil { + return true, err + } + return cfg.ProjectCreationRestriction == comcfg.ProCrtRestrAdmOnly, nil +} + +// VerifyRemoteCert returns bool value. +func VerifyRemoteCert() (bool, error) { + cfg, err := get() + if err != nil { + return true, err + } + return cfg.VerifyRemoteCert, nil +} + +// Email returns email server settings +func Email() (*models.Email, error) { + cfg, err := get() + if err != nil { + return nil, err + } + return cfg.Email, nil +} + +// Database returns database settings +func Database() (*models.Database, error) { + cfg, err := get() + if err != nil { + return nil, err + } + return cfg.Database, nil +} + +// UISecret returns the value of UI secret cookie, used for communication between UI and JobService +// TODO +func UISecret() string { + return os.Getenv("UI_SECRET") } diff --git a/src/ui/config/config_test.go b/src/ui/config/config_test.go index 3f18b40b1..e71160d2e 100644 --- a/src/ui/config/config_test.go +++ b/src/ui/config/config_test.go @@ -17,131 +17,92 @@ package config import ( "os" "testing" + + "github.com/vmware/harbor/src/common/utils/test" ) -var ( - auth = "ldap_auth" - ldap = LDAPSetting{ - "ldap://test.ldap.com", - "ou=people", - "dc=whatever,dc=org", - "1234567", - "cn", - "uid", - "2", - "5", - } - tokenExp = "3" - tokenExpRes = 3 - adminPassword = "password" - externalRegURL = "127.0.0.1" - uiSecret = "ffadsdfsdf" - secretKey = "keykey" - selfRegistration = "off" - projectCreationRestriction = "adminonly" - internalRegistryURL = "http://registry:5000" - jobServiceURL = "http://jobservice" -) - -func TestMain(m *testing.M) { - - os.Setenv("AUTH_MODE", auth) - os.Setenv("LDAP_URL", ldap.URL) - os.Setenv("LDAP_BASE_DN", ldap.BaseDn) - os.Setenv("LDAP_SEARCH_DN", ldap.SearchDn) - os.Setenv("LDAP_SEARCH_PWD", ldap.SearchPwd) - os.Setenv("LDAP_UID", ldap.UID) - os.Setenv("LDAP_SCOPE", ldap.Scope) - os.Setenv("LDAP_FILTER", ldap.Filter) - os.Setenv("LDAP_CONNECT_TIMEOUT", ldap.ConnectTimeout) - os.Setenv("TOKEN_EXPIRATION", tokenExp) - os.Setenv("HARBOR_ADMIN_PASSWORD", adminPassword) - os.Setenv("EXT_REG_URL", externalRegURL) - os.Setenv("UI_SECRET", uiSecret) - os.Setenv("SECRET_KEY", secretKey) - os.Setenv("SELF_REGISTRATION", selfRegistration) - os.Setenv("PROJECT_CREATION_RESTRICTION", projectCreationRestriction) - os.Setenv("REGISTRY_URL", internalRegistryURL) - os.Setenv("JOB_SERVICE_URL", jobServiceURL) - - err := Reload() +// test functions under package ui/config +func TestConfig(t *testing.T) { + server, err := test.NewAdminserver() if err != nil { - panic(err) + t.Fatalf("failed to create a mock admin server: %v", err) } - rc := m.Run() + defer server.Close() - os.Unsetenv("AUTH_MODE") - os.Unsetenv("LDAP_URL") - os.Unsetenv("LDAP_BASE_DN") - os.Unsetenv("LDAP_SEARCH_DN") - os.Unsetenv("LDAP_SEARCH_PWD") - os.Unsetenv("LDAP_UID") - os.Unsetenv("LDAP_SCOPE") - os.Unsetenv("LDAP_FILTER") - os.Unsetenv("LDAP_CONNECT_TIMEOUT") - os.Unsetenv("TOKEN_EXPIRATION") - os.Unsetenv("HARBOR_ADMIN_PASSWORD") - os.Unsetenv("EXT_REG_URL") - os.Unsetenv("UI_SECRET") - os.Unsetenv("SECRET_KEY") - os.Unsetenv("SELF_REGISTRATION") - os.Unsetenv("CREATE_PROJECT_RESTRICTION") - os.Unsetenv("REGISTRY_URL") - os.Unsetenv("JOB_SERVICE_URL") + url := os.Getenv("ADMIN_SERVER_URL") + defer os.Setenv("ADMIN_SERVER_URL", url) - os.Exit(rc) -} - -func TestAuth(t *testing.T) { - if AuthMode() != auth { - t.Errorf("Expected auth mode:%s, in fact: %s", auth, AuthMode()) - } - if LDAP() != ldap { - t.Errorf("Expected ldap setting: %+v, in fact: %+v", ldap, LDAP()) - } -} - -func TestTokenExpiration(t *testing.T) { - if TokenExpiration() != tokenExpRes { - t.Errorf("Expected token expiration: %d, in fact: %d", tokenExpRes, TokenExpiration()) - } -} - -func TestURLs(t *testing.T) { - if InternalRegistryURL() != internalRegistryURL { - t.Errorf("Expected internal Registry URL: %s, in fact: %s", internalRegistryURL, InternalRegistryURL()) - } - if InternalJobServiceURL() != jobServiceURL { - t.Errorf("Expected internal jobservice URL: %s, in fact: %s", jobServiceURL, InternalJobServiceURL()) - } - if ExtRegistryURL() != externalRegURL { - t.Errorf("Expected External Registry URL: %s, in fact: %s", externalRegURL, ExtRegistryURL()) - } -} - -func TestSelfRegistration(t *testing.T) { - if SelfRegistration() { - t.Errorf("Expected Self Registration to be false") - } -} - -func TestSecrets(t *testing.T) { - if SecretKey() != secretKey { - t.Errorf("Expected Secrect Key :%s, in fact: %s", secretKey, SecretKey()) - } - if UISecret() != uiSecret { - t.Errorf("Expected UI Secret: %s, in fact: %s", uiSecret, UISecret()) - } -} - -func TestProjectCreationRestrict(t *testing.T) { - if !OnlyAdminCreateProject() { - t.Errorf("Expected OnlyAdminCreateProject to be true") - } -} - -func TestInitAdminPassword(t *testing.T) { - if InitialAdminPassword() != adminPassword { - t.Errorf("Expected adminPassword: %s, in fact: %s", adminPassword, InitialAdminPassword()) - } + if err := os.Setenv("ADMIN_SERVER_URL", server.URL); err != nil { + t.Fatalf("failed to set env %s: %v", "ADMIN_SERVER_URL", err) + } + + if err := Init(); err != nil { + t.Fatalf("failed to initialize configurations: %v", err) + } + + if err := Load(); err != nil { + t.Fatalf("failed to load configurations: %v", err) + } + + if err := Upload(map[string]string{}); err != nil { + t.Fatalf("failed to upload configurations: %v", err) + } + + if _, err := GetSystemCfg(); err != nil { + t.Fatalf("failed to get system configurations: %v", err) + } + + mode, err := AuthMode() + if err != nil { + t.Fatalf("failed to get auth mode: %v", err) + } + if mode != "db_auth" { + t.Errorf("unexpected mode: %s != %s", mode, "db_auth") + } + + if _, err := LDAP(); err != nil { + t.Fatalf("failed to get ldap settings: %v", err) + } + + if _, err := TokenExpiration(); err != nil { + t.Fatalf("failed to get token expiration: %v", err) + } + + if _, err := DomainName(); err != nil { + t.Fatalf("failed to get domain name: %v", err) + } + + if _, err := SecretKey(); err != nil { + t.Fatalf("failed to get secret key: %v", err) + } + + if _, err := SelfRegistration(); err != nil { + t.Fatalf("failed to get self registration: %v", err) + } + + if _, err := RegistryURL(); err != nil { + t.Fatalf("failed to get registry URL: %v", err) + } + + InternalJobServiceURL() + + InitialAdminPassword() + + if _, err := OnlyAdminCreateProject(); err != nil { + t.Fatalf("failed to get onldy admin create project: %v", err) + } + + if _, err := VerifyRemoteCert(); err != nil { + t.Fatalf("failed to get verify remote cert: %v", err) + } + + if _, err := Email(); err != nil { + t.Fatalf("failed to get email settings: %v", err) + } + + if _, err := Database(); err != nil { + t.Fatalf("failed to get database: %v", err) + } + + UISecret() } diff --git a/src/ui/controllers/base.go b/src/ui/controllers/base.go index 165136a1f..044784f82 100644 --- a/src/ui/controllers/base.go +++ b/src/ui/controllers/base.go @@ -103,7 +103,12 @@ func (b *BaseController) Prepare() { b.Data["CurLang"] = curLang.Name b.Data["RestLangs"] = restLangs - authMode := config.AuthMode() + authMode, err := config.AuthMode() + if err != nil { + log.Errorf("failed to get auth mode: %v", err) + b.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + if authMode == "" { authMode = "db_auth" } @@ -120,9 +125,13 @@ func (b *BaseController) Prepare() { b.UseCompressedJS = false } - b.SelfRegistration = config.SelfRegistration() + b.SelfRegistration, err = config.SelfRegistration() + if err != nil { + log.Errorf("failed to get self registration: %v", err) + b.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } - b.Data["SelfRegistration"] = config.SelfRegistration() + b.Data["SelfRegistration"] = b.SelfRegistration sessionUserID := b.GetSession("userId") if sessionUserID != nil { @@ -235,12 +244,13 @@ func (cc *CommonController) UserExists() { } func init() { - //conf/app.conf -> os.Getenv("config_path") configPath := os.Getenv("CONFIG_PATH") if len(configPath) != 0 { log.Infof("Config path: %s", configPath) - beego.LoadAppConfig("ini", configPath) + if err := beego.LoadAppConfig("ini", configPath); err != nil { + log.Errorf("failed to load app config: %v", err) + } } beego.AddFuncMap("i18n", i18n.Tr) @@ -263,5 +273,4 @@ func init() { log.Errorf("Fail to set message file: %s", err.Error()) } } - } diff --git a/src/ui/controllers/controllers_test.go b/src/ui/controllers/controllers_test.go index c80377417..7fd92023c 100644 --- a/src/ui/controllers/controllers_test.go +++ b/src/ui/controllers/controllers_test.go @@ -14,6 +14,8 @@ import ( "github.com/astaxie/beego" //"github.com/dghubble/sling" "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/config" ) //const ( @@ -29,6 +31,9 @@ import ( //var admin *usrInfo func init() { + if err := config.Init(); err != nil { + log.Fatalf("failed to initialize configurations: %v", err) + } _, file, _, _ := runtime.Caller(1) apppath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, ".."+string(filepath.Separator)))) @@ -63,7 +68,6 @@ func init() { //Init user Info //admin = &usrInfo{adminName, adminPwd} - } // TestMain is a sample to run an endpoint test diff --git a/src/ui/controllers/password.go b/src/ui/controllers/password.go index c3dce0801..786b010a2 100644 --- a/src/ui/controllers/password.go +++ b/src/ui/controllers/password.go @@ -6,12 +6,12 @@ import ( "regexp" "text/template" - "github.com/astaxie/beego" - "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils" + email_util "github.com/vmware/harbor/src/common/utils/email" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/config" ) type messageDetail struct { @@ -49,7 +49,11 @@ func (cc *CommonController) SendEmail() { message := new(bytes.Buffer) - harborURL := config.ExtEndpoint() + harborURL, err := config.DomainName() + if err != nil { + log.Errorf("failed to get domain name: %v", err) + cc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } if harborURL == "" { harborURL = "localhost" } @@ -65,14 +69,14 @@ func (cc *CommonController) SendEmail() { cc.CustomAbort(http.StatusInternalServerError, "internal_error") } - config, err := beego.AppConfig.GetSection("mail") + emailSettings, err := config.Email() if err != nil { - log.Errorf("Can not load app.conf: %v", err) + log.Errorf("failed to get email configurations: %v", err) cc.CustomAbort(http.StatusInternalServerError, "internal_error") } - mail := utils.Mail{ - From: config["from"], + mail := email_util.Mail{ + From: emailSettings.From, To: []string{email}, Subject: cc.Tr("reset_email_subject"), Message: message.String()} diff --git a/src/ui/controllers/project.go b/src/ui/controllers/project.go index 71c7242b0..d2e1ece59 100644 --- a/src/ui/controllers/project.go +++ b/src/ui/controllers/project.go @@ -1,6 +1,8 @@ package controllers import ( + "net/http" + "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/config" @@ -23,6 +25,11 @@ func (pc *ProjectController) Get() { isSysAdmin = false } } - pc.Data["CanCreate"] = !config.OnlyAdminCreateProject() || isSysAdmin + onlyAdmin, err := config.OnlyAdminCreateProject() + if err != nil { + log.Errorf("failed to determine whether only admin can create projects: %v", err) + pc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + pc.Data["CanCreate"] = !onlyAdmin || isSysAdmin pc.Forward("page_title_project", "project.htm") } diff --git a/src/ui/controllers/repository.go b/src/ui/controllers/repository.go index b85ad8782..0fddff097 100644 --- a/src/ui/controllers/repository.go +++ b/src/ui/controllers/repository.go @@ -1,6 +1,10 @@ package controllers import ( + "net/http" + "strings" + + "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/config" ) @@ -11,6 +15,11 @@ type RepositoryController struct { // Get renders repository page func (rc *RepositoryController) Get() { - rc.Data["HarborRegUrl"] = config.ExtRegistryURL() + url, err := config.DomainName() + if err != nil { + log.Errorf("failed to get domain name: %v", err) + rc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + rc.Data["HarborRegUrl"] = strings.Split(url, "://")[1] rc.Forward("page_title_repository", "repository.htm") } diff --git a/src/ui/main.go b/src/ui/main.go index 6c4427279..14770c41b 100644 --- a/src/ui/main.go +++ b/src/ui/main.go @@ -64,7 +64,6 @@ func updateInitPassword(userID int, password string) error { } func main() { - beego.BConfig.WebConfig.Session.SessionOn = true //TODO redisURL := os.Getenv("_REDIS_URL") @@ -72,12 +71,28 @@ func main() { beego.BConfig.WebConfig.Session.SessionProvider = "redis" beego.BConfig.WebConfig.Session.SessionProviderConfig = redisURL } - // beego.AddTemplateExt("htm") - dao.InitDatabase() + log.Info("initializing configurations...") + if err := config.Init(); err != nil { + log.Fatalf("failed to initialize configurations: %v", err) + } + log.Info("configurations initialization completed") - if err := updateInitPassword(adminUserID, config.InitialAdminPassword()); err != nil { + database, err := config.Database() + if err != nil { + log.Fatalf("failed to get database configuration: %v", err) + } + + if err := dao.InitDatabase(database); err != nil { + log.Fatalf("failed to initialize database: %v", err) + } + + password, err := config.InitialAdminPassword() + if err != nil { + log.Fatalf("failed to get admin's initia password: %v", err) + } + if err := updateInitPassword(adminUserID, password); err != nil { log.Error(err) } initRouters() diff --git a/src/ui/router.go b/src/ui/router.go index 55c3f8a73..095218d18 100644 --- a/src/ui/router.go +++ b/src/ui/router.go @@ -84,6 +84,7 @@ func initRouters() { beego.Router("/api/users/:id/sysadmin", &api.UserAPI{}, "put:ToggleUserAdminRole") beego.Router("/api/repositories/top", &api.RepositoryAPI{}, "get:GetTopRepos") beego.Router("/api/logs", &api.LogAPI{}) + beego.Router("/api/configurations", &api.ConfigAPI{}) beego.Router("/api/systeminfo/volumes", &api.SystemInfoAPI{}, "get:GetVolumeInfo") beego.Router("/api/systeminfo/getcert", &api.SystemInfoAPI{}, "get:GetCert") diff --git a/src/ui/service/token/authutils.go b/src/ui/service/token/authutils.go index 1e687365b..3bda33ca4 100644 --- a/src/ui/service/token/authutils.go +++ b/src/ui/service/token/authutils.go @@ -37,13 +37,6 @@ const ( privateKey = "/etc/ui/private_key.pem" ) -var expiration int //minutes - -func init() { - expiration = config.TokenExpiration() - log.Infof("token expiration: %d minutes", expiration) -} - // GetResourceActions ... func GetResourceActions(scopes []string) []*token.ResourceActions { log.Debugf("scopes: %+v", scopes) @@ -91,7 +84,12 @@ func FilterAccess(username string, a *token.ResourceActions) { repoLength := len(repoSplit) if repoLength > 1 { //Only check the permission when the requested image has a namespace, i.e. project var projectName string - registryURL := config.ExtRegistryURL() + registryURL, err := config.DomainName() + if err != nil { + log.Errorf("failed to get domain name: %v", err) + return + } + registryURL = strings.Split(registryURL, "://")[1] if repoSplit[0] == registryURL { projectName = repoSplit[1] log.Infof("Detected Registry URL in Project Name. Assuming this is a notary request and setting Project Name as %s\n", projectName) @@ -153,6 +151,11 @@ func MakeToken(username, service string, access []*token.ResourceActions) (token if err != nil { return "", 0, nil, err } + expiration, err := config.TokenExpiration() + if err != nil { + return "", 0, nil, err + } + tk, expiresIn, issuedAt, err := makeTokenCore(issuer, username, service, expiration, access, pk) if err != nil { return "", 0, nil, err diff --git a/src/ui/ui_test.go b/src/ui/ui_test.go index 59b3724bc..d9bd9c664 100644 --- a/src/ui/ui_test.go +++ b/src/ui/ui_test.go @@ -1,5 +1,6 @@ package main +/* import ( "testing" ) @@ -7,3 +8,4 @@ import ( func TestMain(t *testing.T) { } +*/ diff --git a/tests/docker-compose.test.yml b/tests/docker-compose.test.yml index 63424ad62..0b8499a22 100644 --- a/tests/docker-compose.test.yml +++ b/tests/docker-compose.test.yml @@ -21,6 +21,17 @@ services: - ./common/config/db/env ports: - 3306:3306 + adminserver: + build: + context: ../ + dockerfile: make/dev/adminserver/Dockerfile + env_file: + - ./common/config/adminserver/env + restart: always + volumes: + - /data/config/:/etc/harbor/ + ports: + - 8888:80 ldap: image: osixia/openldap:1.1.7 restart: always diff --git a/tests/startuptest.go b/tests/startuptest.go index f611ef58c..e60823f8d 100644 --- a/tests/startuptest.go +++ b/tests/startuptest.go @@ -2,35 +2,38 @@ package main import ( - "fmt" - "io/ioutil" - "net/http" - "os" - "strings" - "time" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + "time" ) func main() { - time.Sleep(60*time.Second) - for _, url := range os.Args[1:] { - resp, err := http.Get(url) - if err != nil { - fmt.Fprintf(os.Stderr, "fetch: %v\n", err) - os.Exit(1) - } - b, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - fmt.Fprintf(os.Stderr, "fetch: reading %s: %v\n", url, err) - os.Exit(1) - } -// fmt.Printf("%s", b) - if strings.Contains(string(b), "Harbor") { - fmt.Printf("sucess!\n") - } else { - os.Exit(1) - } + time.Sleep(60 * time.Second) + for _, url := range os.Args[1:] { + resp, err := http.Get(url) + if err != nil { + fmt.Fprintf(os.Stderr, "fetch: %v\n", err) + os.Exit(1) + } + b, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "fetch: reading %s: %v\n", url, err) + os.Exit(1) + } + // fmt.Printf("%s", b) - } + if strings.Contains(string(b), "Harbor") { + fmt.Printf("sucess!\n") + } else { + fmt.Println("the response does not contain \"Harbor\"!") + + fmt.Println(string(b)) + os.Exit(1) + } + + } } - diff --git a/tests/testprepare.sh b/tests/testprepare.sh index debcad72d..89200e1d0 100755 --- a/tests/testprepare.sh +++ b/tests/testprepare.sh @@ -1,5 +1,5 @@ #!/bin/bash - +set -e cp tests/docker-compose.test.yml make/. mkdir /etc/ui @@ -7,3 +7,7 @@ cp make/common/config/ui/private_key.pem /etc/ui/. mkdir conf cp make/common/config/ui/app.conf conf/. + +sed -i -r "s/MYSQL_HOST=mysql/MYSQL_HOST=127.0.0.1/" make/common/config/adminserver/env +sed -i -r "s|REGISTRY_URL=http://registry:5000|REGISTRY_URL=http://127.0.0.1:5000|" make/common/config/adminserver/env +sed -i -r "s/UI_SECRET=.*/UI_SECRET=$UI_SECRET/" make/common/config/adminserver/env