From 7fbe811394c3fcdc2fdf2f3d37a3c75a018cb039 Mon Sep 17 00:00:00 2001 From: yhua Date: Mon, 9 Jan 2017 18:24:31 +0800 Subject: [PATCH 01/22] add ldap ping feature --- docs/swagger.yaml | 56 +++++++++++++- src/common/models/ldap.go | 28 +++++++ src/ui/api/ldap.go | 159 ++++++++++++++++++++++++++++++++++++++ src/ui/router.go | 2 + 4 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 src/common/models/ldap.go create mode 100644 src/ui/api/ldap.go diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 644bfa4e2..3b125b55b 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -6,7 +6,7 @@ info: description: These APIs provide services for manipulating Harbor project. version: "0.3.0" # the domain of the service -host: localhost +host: localshot # array of all schemes that your API supports schemes: - http @@ -1347,7 +1347,30 @@ paths: 404: description: Not found the default root certificate. 500: - description: Unexpected internal errors. + description: Unexpected internal errors. + /ldap/ping: + post: + summary: Ping available ldap service. + description: | + This endpoint ping the available ldap service for test related configuration parameters. + parameters: + - name: ldapconf + in: body + description: ldap configuration. + required: true + schema: + $ref: '#/definitions/LdapConf' + tags: + - Products + responses: + 200: + description: Ping ldap service successfully. + 401: + description: Only admin has this authority. + 403: + description: Inviald ldap configuration parameters. + 500: + description: Unexpected internal errors. definitions: Search: type: object @@ -1798,3 +1821,32 @@ definitions: description: The storage of system. items: $ref: '#/definitions/Storage' + LdapConf: + type: object + properties: + ldap_url: + type: string + description: The url of ldap service. + ldap_searchdn: + type: string + description: The search dn of ldap service. + ldap_search_pwd: + type: string + description: The search password of ldap service. + ldap_basedn: + type: string + description: The base dn of ldap service. + ldap_filter: + type: string + description: The serach filter of ldap service. + ldap_uid: + type: string + description: The serach uid of ldap service. + ldap_scope: + type: integer + format: int64 + description: The serach scope of ldap service. + ldap_connect_timeout: + type: integer + format: int64 + description: The connect timeout of ldap service(second). diff --git a/src/common/models/ldap.go b/src/common/models/ldap.go new file mode 100644 index 000000000..f642d1c84 --- /dev/null +++ b/src/common/models/ldap.go @@ -0,0 +1,28 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package models + +// LdapConf holds information about repository that accessed most +type LdapConf struct { + LdapURL string `json:"ldap_url"` + LdapSearchDn string `json:"ldap_searchdn"` + LdapSearchPwd string `json:"ldap_search_pwd"` + LdapBaseDn string `json:"ldap_basedn"` + LdapFilter string `json:"ldap_filter"` + LdapUID string `json:"ldap_uid"` + LdapScope int `json:"ldap_scope"` + LdapConnectTimeout int `json:"ldap_connect_timeout"` +} diff --git a/src/ui/api/ldap.go b/src/ui/api/ldap.go new file mode 100644 index 000000000..0976ac672 --- /dev/null +++ b/src/ui/api/ldap.go @@ -0,0 +1,159 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "crypto/tls" + + "github.com/vmware/harbor/src/common/api" + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" + + goldap "gopkg.in/ldap.v2" +) + +// LdapAPI handles requesst to /api/ldap/ping /api/ldap/search +type LdapAPI struct { + api.BaseAPI +} + +var ldapConfs models.LdapConf + +// Prepare ... +func (l *LdapAPI) Prepare() { + + userID := l.ValidateUser() + isSysAdmin, err := dao.IsAdminRole(userID) + if err != nil { + log.Errorf("error occurred in IsAdminRole: %v", err) + l.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if !isSysAdmin { + l.CustomAbort(http.StatusForbidden, http.StatusText(http.StatusForbidden)) + } +} + +// Ping ... +func (l *LdapAPI) Ping() { + l.DecodeJSONReqAndValidate(&ldapConfs) + + err := validateLdapReq(ldapConfs) + if err != nil { + log.Errorf("Invalid ldap request, error: %v", err) + l.RenderError(http.StatusBadRequest, fmt.Sprintf("invalid ldap request: %v", err)) + return + } + + err = connectTest(ldapConfs) + if err != nil { + log.Errorf("Ldap connect fail, error: %v", err) + l.RenderError(http.StatusBadRequest, fmt.Sprintf("ldap connect fail: %v", err)) + return + } +} + +func validateLdapReq(ldapConfs models.LdapConf) error { + ldapURL := ldapConfs.LdapURL + if ldapURL == "" { + return fmt.Errorf("can not get any available LDAP_URL") + } + log.Debug("ldapURL:", ldapURL) + + ldapConnectTimeout := ldapConfs.LdapConnectTimeout + log.Debug("ldapConnectTimeout:", ldapConnectTimeout) + + return nil + +} + +func connectTest(ldapConfs models.LdapConf) error { + + var ldap *goldap.Conn + var protocol, hostport string + var host, port string + var err error + + ldapURL := ldapConfs.LdapURL + + // This routine keeps compability with the old format used on harbor.cfg + + if strings.Contains(ldapURL, "://") { + splitLdapURL := strings.Split(ldapURL, "://") + protocol, hostport = splitLdapURL[0], splitLdapURL[1] + if !((protocol == "ldap") || (protocol == "ldaps")) { + return fmt.Errorf("unknown ldap protocl") + } + } else { + hostport = ldapURL + protocol = "ldap" + } + + // This tries to detect the used port, if not defined + if strings.Contains(hostport, ":") { + splitHostPort := strings.Split(hostport, ":") + host, port = splitHostPort[0], splitHostPort[1] + _, error := strconv.Atoi(splitHostPort[1]) + if error != nil { + return fmt.Errorf("illegal url format") + } + } else { + host = hostport + switch protocol { + case "ldap": + port = "389" + case "ldaps": + port = "636" + } + } + + // Sets a Dial Timeout for LDAP + connectTimeout := ldapConfs.LdapConnectTimeout + goldap.DefaultTimeout = time.Duration(connectTimeout) * time.Second + + switch protocol { + case "ldap": + ldap, err = goldap.Dial("tcp", fmt.Sprintf("%s:%s", host, port)) + case "ldaps": + ldap, err = goldap.DialTLS("tcp", fmt.Sprintf("%s:%s", host, port), &tls.Config{InsecureSkipVerify: true}) + } + + if err != nil { + return err + } + defer ldap.Close() + + ldapSearchDn := ldapConfs.LdapSearchDn + if ldapSearchDn != "" { + log.Debug("Search DN: ", ldapSearchDn) + ldapSearchPwd := ldapConfs.LdapSearchPwd + err = ldap.Bind(ldapSearchDn, ldapSearchPwd) + if err != nil { + log.Debug("Bind search dn error", err) + return err + } + } + + return nil + +} diff --git a/src/ui/router.go b/src/ui/router.go index 537748b03..55c3f8a73 100644 --- a/src/ui/router.go +++ b/src/ui/router.go @@ -87,6 +87,8 @@ func initRouters() { beego.Router("/api/systeminfo/volumes", &api.SystemInfoAPI{}, "get:GetVolumeInfo") beego.Router("/api/systeminfo/getcert", &api.SystemInfoAPI{}, "get:GetCert") + beego.Router("/api/ldap/ping", &api.LdapAPI{}, "post:Ping") + //external service that hosted on harbor process: beego.Router("/service/notifications", &service.NotificationHandler{}) beego.Router("/service/token", &token.Handler{}) From 84509fbb3ece9d663b243e6e2006e744b60ea4f3 Mon Sep 17 00:00:00 2001 From: kun wang Date: Wed, 11 Jan 2017 04:31:20 -0600 Subject: [PATCH 02/22] regulate some model attribute names. (#1303) LGTM --- src/common/models/ldap.go | 8 ++++---- src/ui/api/ldap.go | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/common/models/ldap.go b/src/common/models/ldap.go index f642d1c84..619d631ce 100644 --- a/src/common/models/ldap.go +++ b/src/common/models/ldap.go @@ -18,11 +18,11 @@ package models // LdapConf holds information about repository that accessed most type LdapConf struct { LdapURL string `json:"ldap_url"` - LdapSearchDn string `json:"ldap_searchdn"` - LdapSearchPwd string `json:"ldap_search_pwd"` - LdapBaseDn string `json:"ldap_basedn"` + LdapSearchDn string `json:"ldap_search_dn"` + LdapSearchPassword string `json:"ldap_search_password"` + LdapBaseDn string `json:"ldap_base_dn"` LdapFilter string `json:"ldap_filter"` LdapUID string `json:"ldap_uid"` LdapScope int `json:"ldap_scope"` - LdapConnectTimeout int `json:"ldap_connect_timeout"` + LdapConnectionTimeout int `json:"ldap_connection_timeout"` } diff --git a/src/ui/api/ldap.go b/src/ui/api/ldap.go index 0976ac672..1a0714445 100644 --- a/src/ui/api/ldap.go +++ b/src/ui/api/ldap.go @@ -80,8 +80,8 @@ func validateLdapReq(ldapConfs models.LdapConf) error { } log.Debug("ldapURL:", ldapURL) - ldapConnectTimeout := ldapConfs.LdapConnectTimeout - log.Debug("ldapConnectTimeout:", ldapConnectTimeout) + ldapConnectionTimeout := ldapConfs.LdapConnectionTimeout + log.Debug("ldapConnectionTimeout:", ldapConnectionTimeout) return nil @@ -128,8 +128,8 @@ func connectTest(ldapConfs models.LdapConf) error { } // Sets a Dial Timeout for LDAP - connectTimeout := ldapConfs.LdapConnectTimeout - goldap.DefaultTimeout = time.Duration(connectTimeout) * time.Second + connectionTimeout := ldapConfs.LdapConnectionTimeout + goldap.DefaultTimeout = time.Duration(connectionTimeout) * time.Second switch protocol { case "ldap": @@ -146,8 +146,8 @@ func connectTest(ldapConfs models.LdapConf) error { ldapSearchDn := ldapConfs.LdapSearchDn if ldapSearchDn != "" { log.Debug("Search DN: ", ldapSearchDn) - ldapSearchPwd := ldapConfs.LdapSearchPwd - err = ldap.Bind(ldapSearchDn, ldapSearchPwd) + ldapSearchPassword := ldapConfs.LdapSearchPassword + err = ldap.Bind(ldapSearchDn, ldapSearchPassword) if err != nil { log.Debug("Bind search dn error", err) return err From b62a95825047f24b7dd8d92d3b0892dd82efa243 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 30 Dec 2016 18:04:01 +0800 Subject: [PATCH 03/22] configure harbor --- .gitignore | 3 + make/common/templates/adminserver/env | 37 +++ make/common/templates/jobservice/env | 16 +- make/common/templates/ui/app.conf | 11 +- make/common/templates/ui/env | 29 +- make/dev/adminserver/Dockerfile | 12 + make/dev/docker-compose.yml | 19 ++ make/docker-compose.tpl | 18 ++ make/photon/adminserver/Dockerfile | 8 + src/adminserver/api/base.go | 29 ++ src/adminserver/api/cfg.go | 161 +++++++++++ src/adminserver/main.go | 60 ++++ src/adminserver/router.go | 30 ++ src/adminserver/systemcfg/driver.go | 30 ++ src/adminserver/systemcfg/driver_json.go | 104 +++++++ src/adminserver/systemcfg/driver_json_test.go | 50 ++++ src/adminserver/systemcfg/systemcfg.go | 176 ++++++++++++ src/common/api/base.go | 10 +- src/common/config/config.go | 227 ++++++--------- src/common/dao/base.go | 32 ++- src/common/dao/config.go | 32 +++ src/common/dao/config_test.go | 236 ++++++++++++++++ src/common/dao/dao_test.go | 84 +++--- src/common/dao/mysql.go | 33 +-- src/common/dao/user_test.go | 13 + src/common/models/config.go | 95 +++++++ src/common/utils/{ => email}/mail.go | 29 +- src/common/utils/log/logger.go | 4 +- .../utils/registry/auth/tokenauthorizer.go | 17 +- src/common/utils/utils.go | 38 +++ src/jobservice/api/replication.go | 14 +- src/jobservice/config/config.go | 182 +++++++----- src/jobservice/job/statemachine.go | 32 ++- src/jobservice/job/workerpool.go | 17 +- src/jobservice/main.go | 29 +- src/jobservice/utils/logger.go | 24 +- src/ui/api/config.go | 245 ++++++++++++++++ src/ui/api/project.go | 8 +- src/ui/api/repository.go | 14 +- src/ui/api/target.go | 14 +- src/ui/api/user.go | 18 +- src/ui/api/utils.go | 50 ++-- src/ui/auth/authenticator.go | 5 +- src/ui/auth/ldap/ldap.go | 24 +- src/ui/config/config.go | 266 ++++++++++++------ src/ui/controllers/base.go | 15 +- src/ui/controllers/password.go | 18 +- src/ui/controllers/project.go | 9 +- src/ui/controllers/repository.go | 11 +- src/ui/main.go | 23 +- src/ui/router.go | 1 + src/ui/service/token/authutils.go | 19 +- 52 files changed, 2119 insertions(+), 562 deletions(-) create mode 100644 make/common/templates/adminserver/env create mode 100644 make/dev/adminserver/Dockerfile create mode 100644 make/photon/adminserver/Dockerfile create mode 100644 src/adminserver/api/base.go create mode 100644 src/adminserver/api/cfg.go create mode 100644 src/adminserver/main.go create mode 100644 src/adminserver/router.go create mode 100644 src/adminserver/systemcfg/driver.go create mode 100644 src/adminserver/systemcfg/driver_json.go create mode 100644 src/adminserver/systemcfg/driver_json_test.go create mode 100644 src/adminserver/systemcfg/systemcfg.go create mode 100644 src/common/dao/config.go create mode 100644 src/common/dao/config_test.go create mode 100644 src/common/models/config.go rename src/common/utils/{ => email}/mail.go (89%) create mode 100644 src/ui/api/config.go diff --git a/.gitignore b/.gitignore index 7e451d1d5..572a7f08d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,11 @@ harbor make/common/config/* +make/dev/adminserver/harbor_adminserver make/dev/ui/harbor_ui make/dev/jobservice/harbor_jobservice +src/adminserver/adminserver src/ui/ui src/jobservice/jobservice +src/common/dao/dao.test *.pyc jobservice/test diff --git a/make/common/templates/adminserver/env b/make/common/templates/adminserver/env new file mode 100644 index 000000000..7c88ba28a --- /dev/null +++ b/make/common/templates/adminserver/env @@ -0,0 +1,37 @@ +LOG_LEVEL=debug +EXT_ENDPOINT=$ui_url +AUTH_MODE=$auth_mode +SELF_REGISTRATION=$self_registration +LDAP_URL=$ldap_url +LDAP_SEARCH_DN=$ldap_searchdn +LDAP_SEARCH_PWD=$ldap_search_pwd +LDAP_BASE_DN=$ldap_basedn +LDAP_FILTER=$ldap_filter +LDAP_UID=$ldap_uid +LDAP_SCOPE=$ldap_scope +DATABASE_TYPE=mysql +MYSQL_HOST=mysql +MYSQL_PORT=3306 +MYSQL_USR=root +MYSQL_PWD=$db_password +MYSQL_DATABASE=registry +REGISTRY_URL=http://registry:5000 +TOKEN_SERVICE_URL=http://ui/service/token +EMAIL_HOST=$email_host +EMAIL_PORT=$email_port +EMAIL_USR=$email_usr +EMAIL_PWD=$email_pwd +EMAIL_TLS=$email_tls +EMAIL_FROM=$email_from +EMAIL_IDENTITY=$email_identity +HARBOR_ADMIN_PASSWORD=$harbor_admin_password +PROJECT_CREATION_RESTRICTION=$project_creation_restriction +VERIFY_REMOTE_CERT=$verify_remote_cert +MAX_JOB_WORKERS=$max_job_workers +LOG_DIR=/var/log/jobs +UI_SECRET=$ui_secret +SECRET_KEY=$secret_key +TOKEN_EXPIRATION=$token_expiration +CFG_EXPIRATION=$cfg_expiration +USE_COMPRESSED_JS=$use_compressed_js +GODEBUG=netdns=cgo \ No newline at end of file diff --git a/make/common/templates/jobservice/env b/make/common/templates/jobservice/env index c6e9fd736..d75972ac3 100644 --- a/make/common/templates/jobservice/env +++ b/make/common/templates/jobservice/env @@ -1,15 +1,5 @@ -MYSQL_HOST=mysql -MYSQL_PORT=3306 -MYSQL_USR=root -MYSQL_PWD=$db_password -UI_SECRET=$ui_secret -SECRET_KEY=$secret_key -CONFIG_PATH=/etc/jobservice/app.conf -REGISTRY_URL=http://registry:5000 -VERIFY_REMOTE_CERT=$verify_remote_cert -MAX_JOB_WORKERS=$max_job_workers LOG_LEVEL=debug -LOG_DIR=/var/log/jobs +UI_SECRET=$ui_secret +CONFIG_PATH=/etc/jobservice/app.conf +MAX_JOB_WORKERS=$max_job_workers GODEBUG=netdns=cgo -EXT_ENDPOINT=$ui_url -TOKEN_ENDPOINT=http://ui diff --git a/make/common/templates/ui/app.conf b/make/common/templates/ui/app.conf index 3cda6d877..b4090f33c 100644 --- a/make/common/templates/ui/app.conf +++ b/make/common/templates/ui/app.conf @@ -6,13 +6,4 @@ types = en-US|zh-CN names = en-US|zh-CN [dev] -httpport = 80 - -[mail] -identity = $email_identity -host = $email_server -port = $email_server_port -username = $email_username -password = $email_password -from = $email_from -ssl = $email_ssl +httpport = 80 \ No newline at end of file diff --git a/make/common/templates/ui/env b/make/common/templates/ui/env index e4feedc2e..fc0d133ab 100644 --- a/make/common/templates/ui/env +++ b/make/common/templates/ui/env @@ -1,29 +1,4 @@ -MYSQL_HOST=mysql -MYSQL_PORT=3306 -MYSQL_USR=root -MYSQL_PWD=$db_password -REGISTRY_URL=http://registry:5000 -JOB_SERVICE_URL=http://jobservice -UI_URL=http://ui -CONFIG_PATH=/etc/ui/app.conf -EXT_REG_URL=$hostname -HARBOR_ADMIN_PASSWORD=$harbor_admin_password -AUTH_MODE=$auth_mode -LDAP_URL=$ldap_url -LDAP_SEARCH_DN=$ldap_searchdn -LDAP_SEARCH_PWD=$ldap_search_pwd -LDAP_BASE_DN=$ldap_basedn -LDAP_FILTER=$ldap_filter -LDAP_UID=$ldap_uid -LDAP_SCOPE=$ldap_scope -UI_SECRET=$ui_secret -SECRET_KEY=$secret_key -SELF_REGISTRATION=$self_registration -USE_COMPRESSED_JS=$use_compressed_js LOG_LEVEL=debug +CONFIG_PATH=/etc/ui/app.conf +UI_SECRET=$ui_secret GODEBUG=netdns=cgo -EXT_ENDPOINT=$ui_url -TOKEN_ENDPOINT=http://ui -VERIFY_REMOTE_CERT=$verify_remote_cert -TOKEN_EXPIRATION=$token_expiration -PROJECT_CREATION_RESTRICTION=$project_creation_restriction diff --git a/make/dev/adminserver/Dockerfile b/make/dev/adminserver/Dockerfile new file mode 100644 index 000000000..d05f87da2 --- /dev/null +++ b/make/dev/adminserver/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.7.3 + +MAINTAINER yinw@vmware.com + +COPY . /go/src/github.com/vmware/harbor + +WORKDIR /go/src/github.com/vmware/harbor/src/adminserver + +RUN go build -v -a -o /go/bin/harbor_adminserver \ + && chmod u+x /go/bin/harbor_adminserver +WORKDIR /go/bin/ +ENTRYPOINT ["/go/bin/harbor_adminserver"] diff --git a/make/dev/docker-compose.yml b/make/dev/docker-compose.yml index 33c7f2be3..c94a2d875 100644 --- a/make/dev/docker-compose.yml +++ b/make/dev/docker-compose.yml @@ -40,6 +40,22 @@ services: options: syslog-address: "tcp://127.0.0.1:1514" tag: "mysql" + adminserver: + build: + context: ../../ + dockerfile: make/dev/adminserver/Dockerfile + env_file: + - ../common/config/adminserver/env + restart: always + volumes: + - /data/config/:/etc/harbor/ + depends_on: + - log + logging: + driver: "syslog" + options: + syslog-address: "tcp://127.0.0.1:1514" + tag: "adminserver" ui: build: context: ../../ @@ -52,6 +68,8 @@ services: - ../common/config/ui/private_key.pem:/etc/ui/private_key.pem depends_on: - log + - adminserver + - registry logging: driver: "syslog" options: @@ -69,6 +87,7 @@ services: - ../common/config/jobservice/app.conf:/etc/jobservice/app.conf depends_on: - ui + - adminserver logging: driver: "syslog" options: diff --git a/make/docker-compose.tpl b/make/docker-compose.tpl index ac6c33d71..b66eb2698 100644 --- a/make/docker-compose.tpl +++ b/make/docker-compose.tpl @@ -41,6 +41,21 @@ services: options: syslog-address: "tcp://127.0.0.1:1514" tag: "mysql" + adminserver: + image: vmware/harbor-adminserver + container_name: harbor-adminserver + env_file: + - ./common/config/adminserver/env + restart: always + volumes: + - /data/config/:/etc/harbor/ + depends_on: + - log + logging: + driver: "syslog" + options: + syslog-address: "tcp://127.0.0.1:1514" + tag: "adminserver" ui: image: vmware/harbor-ui container_name: harbor-ui @@ -53,6 +68,8 @@ services: - /data:/harbor_storage depends_on: - log + - adminserver + - registry logging: driver: "syslog" options: @@ -69,6 +86,7 @@ services: - ./common/config/jobservice/app.conf:/etc/jobservice/app.conf depends_on: - ui + - adminserver logging: driver: "syslog" options: diff --git a/make/photon/adminserver/Dockerfile b/make/photon/adminserver/Dockerfile new file mode 100644 index 000000000..cb145275b --- /dev/null +++ b/make/photon/adminserver/Dockerfile @@ -0,0 +1,8 @@ +FROM library/photon:1.0 + +RUN mkdir /harbor/ +COPY ./make/dev/adminserver/harbor_adminserver /harbor/ + +RUN chmod u+x /harbor/harbor_adminserver +WORKDIR /harbor/ +ENTRYPOINT ["/harbor/harbor_adminserver"] diff --git a/src/adminserver/api/base.go b/src/adminserver/api/base.go new file mode 100644 index 000000000..47938c8b7 --- /dev/null +++ b/src/adminserver/api/base.go @@ -0,0 +1,29 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "net/http" +) + +func handleInternalServerError(w http.ResponseWriter) { + http.Error(w, http.StatusText(http.StatusInternalServerError), + http.StatusInternalServerError) +} + +func handleBadRequestError(w http.ResponseWriter, error string) { + http.Error(w, error, http.StatusBadRequest) +} diff --git a/src/adminserver/api/cfg.go b/src/adminserver/api/cfg.go new file mode 100644 index 000000000..09499fec6 --- /dev/null +++ b/src/adminserver/api/cfg.go @@ -0,0 +1,161 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "strconv" + + cfg "github.com/vmware/harbor/src/adminserver/systemcfg" + comcfg "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" +) + +// ListCfgs lists configurations +func ListCfgs(w http.ResponseWriter, r *http.Request) { + cfg, err := cfg.GetSystemCfg() + if err != nil { + log.Errorf("failed to get system configurations: %v", err) + handleInternalServerError(w) + return + } + + b, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + log.Errorf("failed to marshal configurations: %v", err) + handleInternalServerError(w) + return + } + if _, err = w.Write(b); err != nil { + log.Errorf("failed to write response: %v", err) + } +} + +// UpdateCfgs updates configurations +func UpdateCfgs(w http.ResponseWriter, r *http.Request) { + b, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Errorf("failed to read request body: %v", err) + handleInternalServerError(w) + return + } + + m := &map[string]string{} + if err = json.Unmarshal(b, m); err != nil { + handleBadRequestError(w, err.Error()) + return + } + + log.Info(m) + + system, err := cfg.GetSystemCfg() + if err != nil { + handleInternalServerError(w) + return + } + + if err := populate(system, *m); err != nil { + log.Errorf("failed to populate system configurations: %v", err) + handleInternalServerError(w) + return + } + + log.Info(system.Authentication.SelfRegistration) + + if err = cfg.UpdateSystemCfg(system); err != nil { + log.Errorf("failed to update system configurations: %v", err) + handleInternalServerError(w) + return + } +} + +// populate attrs of cfg according to m +func populate(cfg *models.SystemCfg, m map[string]string) error { + if mode, ok := m[comcfg.AUTH_MODE]; ok { + cfg.Authentication.Mode = mode + } + if value, ok := m[comcfg.SELF_REGISTRATION]; ok { + cfg.Authentication.SelfRegistration = value == "true" + } + if url, ok := m[comcfg.LDAP_URL]; ok { + cfg.Authentication.LDAP.URL = url + } + if dn, ok := m[comcfg.LDAP_SEARCH_DN]; ok { + cfg.Authentication.LDAP.SearchDN = dn + } + if pwd, ok := m[comcfg.LDAP_SEARCH_PWD]; ok { + cfg.Authentication.LDAP.SearchPwd = pwd + } + if dn, ok := m[comcfg.LDAP_BASE_DN]; ok { + cfg.Authentication.LDAP.BaseDN = dn + } + if uid, ok := m[comcfg.LDAP_UID]; ok { + cfg.Authentication.LDAP.UID = uid + } + if filter, ok := m[comcfg.LDAP_FILTER]; ok { + cfg.Authentication.LDAP.Filter = filter + } + if scope, ok := m[comcfg.LDAP_SCOPE]; ok { + i, err := strconv.Atoi(scope) + if err != nil { + return err + } + cfg.Authentication.LDAP.Scope = i + } + + if value, ok := m[comcfg.EMAIL_SERVER]; ok { + cfg.Email.Host = value + } + if value, ok := m[comcfg.EMAIL_SERVER_PORT]; ok { + cfg.Email.Port = value + } + if value, ok := m[comcfg.EMAIL_USERNAME]; ok { + cfg.Email.Username = value + } + if value, ok := m[comcfg.EMAIL_PWD]; ok { + cfg.Email.Host = value + } + if value, ok := m[comcfg.EMAIL_SSL]; ok { + cfg.Email.Password = value + } + if value, ok := m[comcfg.EMAIL_FROM]; ok { + cfg.Email.From = value + } + if value, ok := m[comcfg.EMAIL_IDENTITY]; ok { + cfg.Email.Identity = value + } + + if value, ok := m[comcfg.PROJECT_CREATION_RESTRICTION]; ok { + cfg.ProjectCreationRestriction = value + } + + if value, ok := m[comcfg.VERIFY_REMOTE_CERT]; ok { + cfg.VerifyRemoteCert = value == "true" + } + + if value, ok := m[comcfg.MAX_JOB_WORKERS]; ok { + if i, err := strconv.Atoi(value); err != nil { + return err + } else { + cfg.MaxJobWorkers = i + } + } + + return nil +} diff --git a/src/adminserver/main.go b/src/adminserver/main.go new file mode 100644 index 000000000..d83d95d1a --- /dev/null +++ b/src/adminserver/main.go @@ -0,0 +1,60 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "net/http" + "os" + + cfg "github.com/vmware/harbor/src/adminserver/systemcfg" + "github.com/vmware/harbor/src/common/utils/log" +) + +// Server for admin component +type Server struct { + Port string + Handler http.Handler +} + +// Serve the API +func (s *Server) Serve() error { + server := &http.Server{ + Addr: ":" + s.Port, + Handler: s.Handler, + } + + return server.ListenAndServe() +} + +func main() { + log.Info("initializing system configurations...") + if err := cfg.Init(); err != nil { + log.Fatalf("failed to initialize the system: %v", err) + } + log.Info("system initialization completed") + + port := os.Getenv("PORT") + if len(port) == 0 { + port = "80" + } + server := &Server{ + Port: port, + Handler: newHandler(), + } + if err := server.Serve(); err != nil { + log.Fatal(err) + } +} diff --git a/src/adminserver/router.go b/src/adminserver/router.go new file mode 100644 index 000000000..49083f253 --- /dev/null +++ b/src/adminserver/router.go @@ -0,0 +1,30 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "net/http" + + "github.com/gorilla/mux" + "github.com/vmware/harbor/src/adminserver/api" +) + +func newHandler() http.Handler { + r := mux.NewRouter() + r.HandleFunc("/api/configurations", api.ListCfgs).Methods("GET") + r.HandleFunc("/api/configurations", api.UpdateCfgs).Methods("PUT") + return r +} diff --git a/src/adminserver/systemcfg/driver.go b/src/adminserver/systemcfg/driver.go new file mode 100644 index 000000000..bfb110f24 --- /dev/null +++ b/src/adminserver/systemcfg/driver.go @@ -0,0 +1,30 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package systemcfg + +import ( + "github.com/vmware/harbor/src/common/models" +) + +// Driver defines methods that a configuration store driver must implement +type Driver interface { + // Name returns a human-readable name of the driver + Name() string + // Read reads the configurations from store + Read() (*models.SystemCfg, error) + // Write writes the configurations to store + Write(*models.SystemCfg) error +} diff --git a/src/adminserver/systemcfg/driver_json.go b/src/adminserver/systemcfg/driver_json.go new file mode 100644 index 000000000..62e2c2ee7 --- /dev/null +++ b/src/adminserver/systemcfg/driver_json.go @@ -0,0 +1,104 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package systemcfg + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "sync" + + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" +) + +const ( + // the default path of configuration file + defaultPath = "/etc/harbor/config.json" +) + +type cfgStore struct { + path string // the path of cfg file + sync.RWMutex +} + +// NewCfgStore returns an instance of cfgStore that stores the configurations +// in a json file. The file will be created if it does not exist. +func NewCfgStore(path ...string) (Driver, error) { + p := defaultPath + if len(path) != 0 { + p = path[0] + } + if _, err := os.Stat(p); os.IsNotExist(err) { + log.Infof("the configuration file %s does not exist, creating it...", p) + if err = os.MkdirAll(filepath.Dir(p), 0600); err != nil { + return nil, err + } + if err = ioutil.WriteFile(p, []byte{}, 0600); err != nil { + return nil, err + } + } + + return &cfgStore{ + path: p, + }, nil +} + +// Name ... +func (c *cfgStore) Name() string { + return "JSON" +} + +// Read ... +func (c *cfgStore) Read() (*models.SystemCfg, error) { + c.RLock() + defer c.RUnlock() + + b, err := ioutil.ReadFile(c.path) + if err != nil { + return nil, err + } + + // empty file + if len(b) == 0 { + return nil, nil + } + + config := &models.SystemCfg{} + if err = json.Unmarshal(b, config); err != nil { + return nil, err + } + + return config, nil +} + +// Write ... +func (c *cfgStore) Write(config *models.SystemCfg) error { + b, err := json.MarshalIndent(config, "", " ") + if err != nil { + return err + } + + c.Lock() + defer c.Unlock() + + if err = ioutil.WriteFile(c.path, b, 0600); err != nil { + return err + } + + return nil +} diff --git a/src/adminserver/systemcfg/driver_json_test.go b/src/adminserver/systemcfg/driver_json_test.go new file mode 100644 index 000000000..83a672cec --- /dev/null +++ b/src/adminserver/systemcfg/driver_json_test.go @@ -0,0 +1,50 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package systemcfg + +import ( + "os" + "testing" +) + +func TestReadWrite(t *testing.T) { + path := "/tmp/config.json" + store, err := NewCfgStore(path) + if err != nil { + t.Fatalf("failed to create json cfg store: %v", err) + } + defer func() { + if err := os.Remove(path); err != nil { + t.Fatalf("failed to remove the json file %s: %v", path, err) + } + }() + + config := &cfg.SystemCfg{ + Authentication: &cfg.Authentication{ + LDAP: &cfg.LDAP{}, + }, + Database: &cfg.Database{ + MySQL: &cfg.MySQL{}, + }, + } + if err := store.Write(config); err != nil { + t.Fatalf("failed to write configurations to json file: %v", err) + } + + if _, err = store.Read(); err != nil { + t.Fatalf("failed to read configurations from json file: %v", err) + } +} diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go new file mode 100644 index 000000000..dda9cf144 --- /dev/null +++ b/src/adminserver/systemcfg/systemcfg.go @@ -0,0 +1,176 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package systemcfg + +import ( + "fmt" + "os" + "strconv" + + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" +) + +var store Driver + +// Init system configurations. Read from config store first, if null read from env +func Init() (err error) { + s := getCfgStore() + switch s { + case "json": + store, err = NewCfgStore() + if err != nil { + return + } + default: + return fmt.Errorf("unsupported configuration store driver %s", s) + } + + log.Infof("configuration store driver: %s", store.Name()) + cfg, err := store.Read() + if err != nil { + return err + } + + if cfg == nil { + log.Info("configurations read from store driver are null, initializing system from environment variables...") + cfg, err = initFromEnv() + if err != nil { + return err + } + } else { + //read the following attrs from env every time boots up, + //and sync them into cfg store + cfg.DomainName = os.Getenv("EXT_ENDPOINT") + cfg.Database.MySQL.Password = os.Getenv("MYSQL_PWD") + cfg.JobLogDir = os.Getenv("LOG_DIR") + cfg.CompressJS = os.Getenv("USE_COMPRESSED_JS") == "on" + exp, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION")) + if err != nil { + return err + } + cfg.TokenExpiration = exp + cfg.SecretKey = os.Getenv("SECRET_KEY") + + cfgExp, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) + if err != nil { + return err + } + cfg.CfgExpiration = cfgExp + } + + if err = store.Write(cfg); err != nil { + return err + } + + return nil +} + +func getCfgStore() string { + return "json" +} + +func initFromEnv() (*models.SystemCfg, error) { + cfg := &models.SystemCfg{} + cfg.DomainName = os.Getenv("EXT_ENDPOINT") + cfg.Authentication = &models.Authentication{ + Mode: os.Getenv("AUTH_MODE"), + SelfRegistration: os.Getenv("SELF_REGISTRATION") == "true", + LDAP: &models.LDAP{ + URL: os.Getenv("LDAP_URL"), + SearchDN: os.Getenv("LDAP_SEARCH_DN"), + SearchPwd: os.Getenv("LDAP_SEARCH_PWD"), + BaseDN: os.Getenv("LDAP_BASE_DN"), + Filter: os.Getenv("LDAP_FILTER"), + UID: os.Getenv("LDAP_UID"), + }, + } + scope, err := strconv.Atoi(os.Getenv("LDAP_SCOPE")) + if err != nil { + return nil, err + } + cfg.Authentication.LDAP.Scope = scope + cfg.Database = &models.Database{ + Type: os.Getenv("DATABASE_TYPE"), + MySQL: &models.MySQL{ + Host: os.Getenv("MYSQL_HOST"), + Username: os.Getenv("MYSQL_USR"), + Password: os.Getenv("MYSQL_PWD"), + Database: os.Getenv("MYSQL_DATABASE"), + }, + SQLite: &models.SQLite{ + File: os.Getenv("SQLITE_FILE"), + }, + } + port, err := strconv.Atoi(os.Getenv("MYSQL_PORT")) + if err != nil { + return nil, err + } + cfg.Database.MySQL.Port = port + + cfg.TokenService = &models.TokenService{ + URL: os.Getenv("TOKEN_SERVICE_URL"), + } + cfg.Registry = &models.Registry{ + URL: os.Getenv("REGISTRY_URL"), + } + cfg.Email = &models.Email{ + Host: os.Getenv("EMAIL_HOST"), + Port: os.Getenv("EMAIL_PORT"), + Username: os.Getenv("EMAIL_USR"), + Password: os.Getenv("EMAIL_PWD"), + TLS: os.Getenv("EMAIL_TLS") == "true", + From: os.Getenv("EMAIL_FROM"), + Identity: os.Getenv("EMAIL_IDENTITY"), + } + cfg.VerifyRemoteCert = os.Getenv("VERIFY_REMOTE_CERT") == "true" + cfg.ProjectCreationRestriction = os.Getenv("PROJECT_CREATION_RESTRICTION") + + workers, err := strconv.Atoi(os.Getenv("MAX_JOB_WORKERS")) + if err != nil { + return nil, err + } + cfg.MaxJobWorkers = workers + cfg.JobLogDir = os.Getenv("LOG_DIR") + cfg.InitialAdminPwd = os.Getenv("HARBOR_ADMIN_PASSWORD") + cfg.CompressJS = os.Getenv("USE_COMPRESSED_JS") == "on" + + tokenExp, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION")) + if err != nil { + return nil, err + } + cfg.TokenExpiration = tokenExp + + cfg.SecretKey = os.Getenv("SECRET_KEY") + + cfgExp, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) + if err != nil { + return nil, err + } + cfg.CfgExpiration = cfgExp + + return cfg, nil +} + +// GetSystemCfg returns the system configurations +func GetSystemCfg() (*models.SystemCfg, error) { + return store.Read() +} + +// UpdateSystemCfg updates the system configurations +func UpdateSystemCfg(cfg *models.SystemCfg) error { + return store.Write(cfg) +} diff --git a/src/common/api/base.go b/src/common/api/base.go index 53d916849..227327339 100644 --- a/src/common/api/base.go +++ b/src/common/api/base.go @@ -22,11 +22,11 @@ import ( "strconv" "github.com/astaxie/beego/validation" - "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/auth" + "github.com/vmware/harbor/src/ui/config" "github.com/astaxie/beego" ) @@ -212,6 +212,10 @@ func (b *BaseAPI) GetPaginationParams() (page, pageSize int64) { } // GetIsInsecure ... -func GetIsInsecure() bool { - return !config.VerifyRemoteCert() +func GetIsInsecure() (bool, error) { + verify, err := config.VerifyRemoteCert() + if err != nil { + return false, err + } + return !verify, nil } diff --git a/src/common/config/config.go b/src/common/config/config.go index e9c54c27c..756bff8d1 100644 --- a/src/common/config/config.go +++ b/src/common/config/config.go @@ -17,162 +17,119 @@ package config import ( + "bytes" "fmt" - "os" + "io/ioutil" + "net/http" "strings" + + "github.com/astaxie/beego/cache" + "github.com/vmware/harbor/src/common/utils" ) -// ConfLoader is the interface to load configurations -type ConfLoader interface { - // Load will load configuration from different source into a string map, the values in the map will be parsed in to configurations. - Load() (map[string]string, error) +const ( + //auth mode + DB_AUTH = "db_auth" + LDAP_AUTH = "ldap_auth" + + //project_creation_restriction + PRO_CRT_RESTR_EVERYONE = "everyone" + PRO_CRT_RESTR_ADM_ONLY = "adminonly" + + LDAP_SCOPE_BASE = "1" + LDAP_SCOPE_ONELEVEL = "2" + LDAP_SCOPE_SUBTREE = "3" + + AUTH_MODE = "auth_mode" + SELF_REGISTRATION = "self_registration" + LDAP_URL = "ldap_url" + LDAP_SEARCH_DN = "ldap_search_dn" + LDAP_SEARCH_PWD = "ldap_search_pwd" + LDAP_BASE_DN = "ldap_base_dn" + LDAP_UID = "ldap_uid" + LDAP_FILTER = "ldap_filter" + LDAP_SCOPE = "ldap_scope" + EMAIL_SERVER = "email_server" + EMAIL_SERVER_PORT = "email_server_port" + EMAIL_USERNAME = "email_server_username" + EMAIL_PWD = "email_server_pwd" + EMAIL_FROM = "email_from" + EMAIL_SSL = "email_ssl" + EMAIL_IDENTITY = "email_identity" + PROJECT_CREATION_RESTRICTION = "project_creation_restriction" + VERIFY_REMOTE_CERT = "verify_remote_cert" + MAX_JOB_WORKERS = "max_job_workers" + CFG_EXPIRATION = "cfg_expiration" +) + +type Manager struct { + Key string + Cache cache.Cache + Loader *Loader } -// EnvConfigLoader loads the config from env vars. -type EnvConfigLoader struct { - Keys []string -} - -// Load ... -func (ec *EnvConfigLoader) Load() (map[string]string, error) { - m := make(map[string]string) - for _, k := range ec.Keys { - m[k] = os.Getenv(k) +func NewManager(key, url string) *Manager { + return &Manager{ + Key: key, + Cache: cache.NewMemoryCache(), + Loader: NewLoader(url), } - return m, nil } -// ConfParser ... -type ConfParser interface { - - //Parse parse the input raw map into a config map - Parse(raw map[string]string, config map[string]interface{}) error -} - -// Config wraps a map for the processed configuration values, -// and loader parser to read configuration from external source and process the values. -type Config struct { - Config map[string]interface{} - Loader ConfLoader - Parser ConfParser -} - -// Load reload the configurations -func (conf *Config) Load() error { - rawMap, err := conf.Loader.Load() - if err != nil { - return err +func (m *Manager) GetFromCache() interface{} { + value := m.Cache.Get(m.Key) + if value != nil { + return value } - err = conf.Parser.Parse(rawMap, conf.Config) - return err -} - -// MySQLSetting wraps the settings of a MySQL DB -type MySQLSetting struct { - Database string - User string - Password string - Host string - Port string -} - -// SQLiteSetting wraps the settings of a SQLite DB -type SQLiteSetting struct { - FilePath string -} - -type commonParser struct{} - -// Parse parses the db settings, veryfy_remote_cert, ext_endpoint, token_endpoint -func (cp *commonParser) Parse(raw map[string]string, config map[string]interface{}) error { - db := strings.ToLower(raw["DATABASE"]) - if db == "mysql" || db == "" { - db = "mysql" - mySQLDB := raw["MYSQL_DATABASE"] - if len(mySQLDB) == 0 { - mySQLDB = "registry" - } - setting := MySQLSetting{ - mySQLDB, - raw["MYSQL_USR"], - raw["MYSQL_PWD"], - raw["MYSQL_HOST"], - raw["MYSQL_PORT"], - } - config["mysql"] = setting - } else if db == "sqlite" { - f := raw["SQLITE_FILE"] - if len(f) == 0 { - f = "registry.db" - } - setting := SQLiteSetting{ - f, - } - config["sqlite"] = setting - } else { - return fmt.Errorf("Invalid DB: %s", db) - } - config["database"] = db - - //By default it's true - config["verify_remote_cert"] = raw["VERIFY_REMOTE_CERT"] != "off" - - config["ext_endpoint"] = raw["EXT_ENDPOINT"] - config["token_endpoint"] = raw["TOKEN_ENDPOINT"] - config["log_level"] = raw["LOG_LEVEL"] return nil } -var commonConfig *Config +type Loader struct { + url string + client *http.Client +} -func init() { - commonKeys := []string{"DATABASE", "MYSQL_DATABASE", "MYSQL_USR", "MYSQL_PWD", "MYSQL_HOST", "MYSQL_PORT", "SQLITE_FILE", "VERIFY_REMOTE_CERT", "EXT_ENDPOINT", "TOKEN_ENDPOINT", "LOG_LEVEL"} - commonConfig = &Config{ - Config: make(map[string]interface{}), - Loader: &EnvConfigLoader{Keys: commonKeys}, - Parser: &commonParser{}, - } - if err := commonConfig.Load(); err != nil { - panic(err) +func NewLoader(url string) *Loader { + return &Loader{ + url: url, + client: &http.Client{}, } } -// Reload will reload the configuration. -func Reload() error { - return commonConfig.Load() +func (l *Loader) Init() error { + addr := l.url + if strings.Contains(addr, "://") { + addr = strings.Split(addr, "://")[1] + } + return utils.TestTCPConn(addr, 60, 2) } -// Database returns the DB type in configuration. -func Database() string { - return commonConfig.Config["database"].(string) +func (l *Loader) Load() ([]byte, error) { + resp, err := l.client.Get(l.url + "/api/configurations") + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return b, nil } -// MySQL returns the mysql setting in configuration. -func MySQL() MySQLSetting { - return commonConfig.Config["mysql"].(MySQLSetting) -} +func (l *Loader) Upload(b []byte) error { + req, err := http.NewRequest("PUT", l.url+"/api/configurations", bytes.NewReader(b)) + if err != nil { + return err + } + resp, err := l.client.Do(req) + if err != nil { + return err + } -// SQLite returns the SQLite setting -func SQLite() SQLiteSetting { - return commonConfig.Config["sqlite"].(SQLiteSetting) -} + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected http status code: %d", resp.StatusCode) + } -// VerifyRemoteCert returns bool value. -func VerifyRemoteCert() bool { - return commonConfig.Config["verify_remote_cert"].(bool) -} - -// ExtEndpoint ... -func ExtEndpoint() string { - return commonConfig.Config["ext_endpoint"].(string) -} - -// TokenEndpoint returns the endpoint string of token service, which can be accessed by internal service of Harbor. -func TokenEndpoint() string { - return commonConfig.Config["token_endpoint"].(string) -} - -// LogLevel returns the log level in string format. -func LogLevel() string { - return commonConfig.Config["log_level"].(string) + return nil } diff --git a/src/common/dao/base.go b/src/common/dao/base.go index 9028ce587..e05f77561 100644 --- a/src/common/dao/base.go +++ b/src/common/dao/base.go @@ -17,11 +17,12 @@ package dao import ( "fmt" + "strconv" "strings" "sync" "github.com/astaxie/beego/orm" - "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" ) @@ -39,27 +40,32 @@ type Database interface { } // InitDatabase initializes the database -func InitDatabase() { - database, err := getDatabase() +func InitDatabase(database *models.Database) error { + db, err := getDatabase(database) if err != nil { - panic(err) + return err } - log.Infof("initializing database: %s", database.String()) - if err := database.Register(); err != nil { - panic(err) + log.Infof("initializing database: %s", db.String()) + if err := db.Register(); err != nil { + return err } + log.Info("initialize database completed") + return nil } -func getDatabase() (db Database, err error) { - switch config.Database() { +func getDatabase(database *models.Database) (db Database, err error) { + switch database.Type { case "", "mysql": - db = NewMySQL(config.MySQL().Host, config.MySQL().Port, config.MySQL().User, - config.MySQL().Password, config.MySQL().Database) + db = NewMySQL(database.MySQL.Host, + strconv.Itoa(database.MySQL.Port), + database.MySQL.Username, + database.MySQL.Password, + database.MySQL.Database) case "sqlite": - db = NewSQLite(config.SQLite().FilePath) + db = NewSQLite(database.SQLite.File) default: - err = fmt.Errorf("invalid database: %s", config.Database()) + err = fmt.Errorf("invalid database: %s", database.Type) } return } diff --git a/src/common/dao/config.go b/src/common/dao/config.go new file mode 100644 index 000000000..eb700c8af --- /dev/null +++ b/src/common/dao/config.go @@ -0,0 +1,32 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package dao + +import ( + "github.com/vmware/harbor/src/common/models" +) + +// AuthModeCanBeModified determines whether auth mode can be +// modified or not. Auth mode can modified when there is only admin +// user in database. +func AuthModeCanBeModified() (bool, error) { + c, err := GetOrmer().QueryTable(&models.User{}).Count() + if err != nil { + return false, err + } + + return c == 1, nil +} diff --git a/src/common/dao/config_test.go b/src/common/dao/config_test.go new file mode 100644 index 000000000..ce624e313 --- /dev/null +++ b/src/common/dao/config_test.go @@ -0,0 +1,236 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package dao + +/* +import ( + "testing" + + "github.com/vmware/harbor/src/common/models" +) + +func deleteConfigByKey(key string) error { + if _, err := GetOrmer().Raw("delete from properties where k = ?", key). + Exec(); err != nil { + return err + } + return nil +} + +func TestGetConfigByKey(t *testing.T) { + cfg := &models.Config{ + Key: "key", + Value: "value", + } + + if err := InsertConfig(cfg); err != nil { + t.Fatalf("failed to insert configuration into table: %v", err) + } + defer func(key string) { + if err := deleteConfigByKey(key); err != nil { + t.Fatalf("failed to delete configuration %s: %v", key, err) + } + }(cfg.Key) + + config, err := GetConfigByKey(cfg.Key) + if err != nil { + t.Fatalf("failed to get configuration by key %s: %v", cfg.Key, err) + } + + if config == nil { + t.Fatal("configuration is nil") + } + + if config.Value != cfg.Value { + t.Fatalf("unexpected value: %s != %s", config.Value, cfg.Value) + } +} + +func TestListConfigs(t *testing.T) { + configs, err := ListConfigs() + if err != nil { + t.Fatalf("failed to list configurations: %v", err) + } + size := len(configs) + + cfg := &models.Config{ + Key: "key", + Value: "value", + } + if err := InsertConfig(cfg); err != nil { + t.Fatalf("failed to insert configuration into table: %v", err) + } + defer func(key string) { + if err := deleteConfigByKey(key); err != nil { + t.Fatalf("failed to delete configuration %s: %v", key, err) + } + }(cfg.Key) + + configs, err = ListConfigs() + if err != nil { + t.Fatalf("failed to list configurations: %v", err) + } + + if size+1 != len(configs) { + t.Fatalf("unexpected length of configurations: %d != %d", len(configs), size+1) + } +} + +func TestInsertConfig(t *testing.T) { + cfg := &models.Config{ + Key: "key1", + Value: "value1", + } + + if err := InsertConfig(cfg); err != nil { + t.Fatalf("failed to insert configuration into table: %v", err) + } + defer func(key string) { + if err := deleteConfigByKey(key); err != nil { + t.Fatalf("failed to delete configuration %s: %v", key, err) + } + }(cfg.Key) + + config, err := GetConfigByKey(cfg.Key) + if err != nil { + t.Fatalf("failed to get configuration by key %s: %v", cfg.Key, err) + } + if config == nil { + t.Fatal("configuration is nil") + } + + if config.Value != cfg.Value { + t.Fatalf("unexpected value: %s != %s", config.Value, cfg.Value) + } +} + +func TestUpdateConfig(t *testing.T) { + cfg := &models.Config{ + Key: "key", + Value: "value", + } + + if err := InsertConfig(cfg); err != nil { + t.Fatalf("failed to insert configuration into table: %v", err) + } + defer func(key string) { + if err := deleteConfigByKey(key); err != nil { + t.Fatalf("failed to delete configuration %s: %v", key, err) + } + }(cfg.Key) + + newCfg := &models.Config{ + Key: "key", + Value: "new_value", + } + if err := UpdateConfig(newCfg); err != nil { + t.Fatalf("failed to update configuration: %v", err) + } + + config, err := GetConfigByKey(cfg.Key) + if err != nil { + t.Fatalf("failed to get configuration by key %s: %v", cfg.Key, err) + } + + if config == nil { + t.Fatal("configuration is nil") + } + + if config.Value != newCfg.Value { + t.Fatalf("unexpected value: %s != %s", config.Value, newCfg.Value) + } +} + +func TestInsertOrUpdateConfigs(t *testing.T) { + cfg1 := &models.Config{ + Key: "key1", + Value: "value1", + } + + if err := InsertConfig(cfg1); err != nil { + t.Fatalf("failed to insert configuration into table: %v", err) + } + defer func(key string) { + if err := deleteConfigByKey(key); err != nil { + t.Fatalf("failed to delete configuration %s: %v", key, err) + } + }(cfg1.Key) + + cfg2 := &models.Config{ + Key: "key2", + Value: "value2", + } + + if err := InsertOrUpdateConfigs([]*models.Config{cfg1, cfg2}); err != nil { + t.Fatalf("failed to insert or update configurations: %v", err) + } + defer func(key string) { + if err := deleteConfigByKey(key); err != nil { + t.Fatalf("failed to delete configuration %s: %v", key, err) + } + }(cfg2.Key) +} + +func TestAuthModeCanBeModified(t *testing.T) { + c, err := GetOrmer().QueryTable(&models.User{}).Count() + if err != nil { + t.Fatalf("failed to count users: %v", err) + } + + if c == 1 { + flag, err := AuthModeCanBeModified() + if err != nil { + t.Fatalf("failed to determine whether auth mode can be modified: %v", err) + } + if !flag { + t.Errorf("unexpected result: %t != %t", flag, true) + } + + user := models.User{ + Username: "user_for_config_test", + Email: "user_for_config_test@vmware.com", + Password: "P@ssword", + Realname: "user_for_config_test", + } + id, err := Register(user) + if err != nil { + t.Fatalf("failed to register user: %v", err) + } + defer func(id int64) { + if err := deleteUser(id); err != nil { + t.Fatalf("failed to delete user %d: %v", id, err) + } + }(id) + + flag, err = AuthModeCanBeModified() + if err != nil { + t.Fatalf("failed to determine whether auth mode can be modified: %v", err) + } + if flag { + t.Errorf("unexpected result: %t != %t", flag, false) + } + + } else { + flag, err := AuthModeCanBeModified() + if err != nil { + t.Fatalf("failed to determine whether auth mode can be modified: %v", err) + } + if flag { + t.Errorf("unexpected result: %t != %t", flag, false) + } + } +} +*/ diff --git a/src/common/dao/dao_test.go b/src/common/dao/dao_test.go index 861bf3ee7..dd3594c79 100644 --- a/src/common/dao/dao_test.go +++ b/src/common/dao/dao_test.go @@ -17,10 +17,12 @@ package dao import ( "os" + "strconv" "testing" "time" "github.com/astaxie/beego/orm" + //"github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" @@ -42,7 +44,7 @@ func execUpdate(o orm.Ormer, sql string, params ...interface{}) error { func clearUp(username string) { var err error - o := orm.NewOrm() + o := GetOrmer() o.Begin() err = execUpdate(o, `delete @@ -156,53 +158,63 @@ func TestMain(m *testing.M) { } func testForMySQL(m *testing.M) int { - db := os.Getenv("DATABASE") - defer os.Setenv("DATABASE", db) - - os.Setenv("DATABASE", "mysql") - - dbHost := os.Getenv("DB_HOST") + dbHost := os.Getenv("MYSQL_HOST") if len(dbHost) == 0 { - log.Fatalf("environment variable DB_HOST is not set") + log.Fatalf("environment variable MYSQL_HOST is not set") } - dbUser := os.Getenv("DB_USR") + dbUser := os.Getenv("MYSQL_USR") if len(dbUser) == 0 { - log.Fatalf("environment variable DB_USR is not set") + log.Fatalf("environment variable MYSQL_USR is not set") } - dbPort := os.Getenv("DB_PORT") - if len(dbPort) == 0 { - log.Fatalf("environment variable DB_PORT is not set") + dbPortStr := os.Getenv("MYSQL_PORT") + if len(dbPortStr) == 0 { + log.Fatalf("environment variable MYSQL_PORT is not set") + } + dbPort, err := strconv.Atoi(dbPortStr) + if err != nil { + log.Fatalf("invalid MYSQL_PORT: %v", err) } - dbPassword := os.Getenv("DB_PWD") - log.Infof("DB_HOST: %s, DB_USR: %s, DB_PORT: %s, DB_PWD: %s\n", dbHost, dbUser, dbPort, dbPassword) + dbPassword := os.Getenv("MYSQL_PWD") + dbDatabase := os.Getenv("MYSQL_DATABASE") + if len(dbDatabase) == 0 { + log.Fatalf("environment variable MYSQL_DATABASE is not set") + } - os.Setenv("MYSQL_HOST", dbHost) - os.Setenv("MYSQL_PORT", dbPort) - os.Setenv("MYSQL_USR", dbUser) - os.Setenv("MYSQL_PWD", dbPassword) + database := &models.Database{ + Type: "mysql", + MySQL: &models.MySQL{ + Host: dbHost, + Port: dbPort, + Username: dbUser, + Password: dbPassword, + Database: dbDatabase, + }, + } - return testForAll(m) + log.Infof("MYSQL_HOST: %s, MYSQL_USR: %s, MYSQL_PORT: %s, MYSQL_PWD: %s\n", dbHost, dbUser, dbPort, dbPassword) + + return testForAll(m, database) } func testForSQLite(m *testing.M) int { - db := os.Getenv("DATABASE") - defer os.Setenv("DATABASE", db) - - os.Setenv("DATABASE", "sqlite") - file := os.Getenv("SQLITE_FILE") if len(file) == 0 { - os.Setenv("SQLITE_FILE", "/registry.db") - defer os.Setenv("SQLITE_FILE", "") + log.Fatalf("environment variable SQLITE_FILE is not set") } - return testForAll(m) + database := &models.Database{ + Type: "sqlite", + SQLite: &models.SQLite{ + File: file, + }, + } + + return testForAll(m, database) } -func testForAll(m *testing.M) int { - os.Setenv("AUTH_MODE", "db_auth") - initDatabaseForTest() +func testForAll(m *testing.M, database *models.Database) int { + initDatabaseForTest(database) clearUp(username) return m.Run() @@ -210,8 +222,8 @@ func testForAll(m *testing.M) int { var defaultRegistered = false -func initDatabaseForTest() { - database, err := getDatabase() +func initDatabaseForTest(db *models.Database) { + database, err := getDatabase(db) if err != nil { panic(err) } @@ -226,6 +238,12 @@ func initDatabaseForTest() { if err := database.Register(alias); err != nil { panic(err) } + + if alias != "default" { + if err = globalOrm.Using(alias); err != nil { + log.Fatalf("failed to create new orm: %v", err) + } + } } func TestRegister(t *testing.T) { diff --git a/src/common/dao/mysql.go b/src/common/dao/mysql.go index a050b054e..8a9c2c304 100644 --- a/src/common/dao/mysql.go +++ b/src/common/dao/mysql.go @@ -16,15 +16,11 @@ package dao import ( - "errors" "fmt" - "net" - - "time" "github.com/astaxie/beego/orm" _ "github.com/go-sql-driver/mysql" //register mysql driver - "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/common/utils" ) type mysql struct { @@ -48,7 +44,8 @@ func NewMySQL(host, port, usr, pwd, database string) Database { // Register registers MySQL as the underlying database used func (m *mysql) Register(alias ...string) error { - if err := m.testConn(m.host, m.port); err != nil { + + if err := utils.TestTCPConn(m.host+":"+m.port, 60, 2); err != nil { return err } @@ -65,30 +62,6 @@ func (m *mysql) Register(alias ...string) error { return orm.RegisterDataBase(an, "mysql", conn) } -func (m *mysql) testConn(host, port string) error { - ch := make(chan int, 1) - go func() { - var err error - var c net.Conn - for { - c, err = net.DialTimeout("tcp", host+":"+port, 20*time.Second) - if err == nil { - c.Close() - ch <- 1 - } else { - log.Errorf("failed to connect to db, retry after 2 seconds :%v", err) - time.Sleep(2 * time.Second) - } - } - }() - select { - case <-ch: - return nil - case <-time.After(60 * time.Second): - return errors.New("failed to connect to database after 60 seconds") - } -} - // Name returns the name of MySQL func (m *mysql) Name() string { return "MySQL" diff --git a/src/common/dao/user_test.go b/src/common/dao/user_test.go index 7d421d848..c227d835e 100644 --- a/src/common/dao/user_test.go +++ b/src/common/dao/user_test.go @@ -38,6 +38,11 @@ func TestDeleteUser(t *testing.T) { if err != nil { t.Fatalf("failed to register user: %v", err) } + defer func(id int64) { + if err := deleteUser(id); err != nil { + t.Fatalf("failed to delete user %d: %v", id, err) + } + }(id) err = DeleteUser(int(id)) if err != nil { @@ -67,3 +72,11 @@ func TestDeleteUser(t *testing.T) { expected) } } + +func deleteUser(id int64) error { + if _, err := GetOrmer().QueryTable(&models.User{}). + Filter("UserID", id).Delete(); err != nil { + return err + } + return nil +} diff --git a/src/common/models/config.go b/src/common/models/config.go new file mode 100644 index 000000000..4039ea7ef --- /dev/null +++ b/src/common/models/config.go @@ -0,0 +1,95 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package models + +// Authentication ... +type Authentication struct { + Mode string `json:"mode"` + SelfRegistration bool `json:"self_registration"` + LDAP *LDAP `json:"ldap,omitempty"` +} + +// LDAP ... +type LDAP struct { + URL string `json:"url"` + SearchDN string `json:"search_dn"` + SearchPwd string `json:"search_pwd"` + BaseDN string `json:"base_dn"` + Filter string `json:"filter"` + UID string `json:"uid"` + Scope int `json:"scope"` +} + +// Database ... +type Database struct { + Type string `json:"type"` + MySQL *MySQL `json:"mysql,omitempty"` + SQLite *SQLite `json:"sqlite,omitempty"` +} + +// MySQL ... +type MySQL struct { + Host string `json:"host"` + Port int `json:"port"` + Username string `json:"username"` + Password string `json:"password"` + Database string `json:"database"` +} + +// SQLite ... +type SQLite struct { + File string `json:"file"` +} + +// Email ... +type Email struct { + Host string `json:"host"` + Port string `json:"port"` + Username string `json:"username"` + Password string `json:"password"` + TLS bool `json:"tls"` + Identity string `json:"identity"` + From string `json:"from"` +} + +// Registry ... +type Registry struct { + URL string `json:"url"` +} + +// TokenService ... +type TokenService struct { + URL string `json:"url"` +} + +// SystemCfg holds all configurations of system +type SystemCfg struct { + DomainName string `json:"domain_name"` // Harbor external URL: protocal://host:port + Authentication *Authentication `json:"authentication"` + Database *Database `json:"database"` + TokenService *TokenService `json:"token_service"` + Registry *Registry `json:"registry"` + Email *Email `json:"email"` + VerifyRemoteCert bool `json:"verify_remote_cert"` + ProjectCreationRestriction string `json:"project_creation_restriction"` + MaxJobWorkers int `json:"max_job_workers"` + JobLogDir string `json:"job_log_dir"` + InitialAdminPwd string `json:"initial_admin_pwd"` + CompressJS bool `json:"compress_js"` //TODO remove + TokenExpiration int `json:"token_expiration"` // in minute + SecretKey string `json:"secret_key"` + CfgExpiration int `json:"cfg_expiration"` +} diff --git a/src/common/utils/mail.go b/src/common/utils/email/mail.go similarity index 89% rename from src/common/utils/mail.go rename to src/common/utils/email/mail.go index c4a364cb8..d82f1d9bf 100644 --- a/src/common/utils/mail.go +++ b/src/common/utils/email/mail.go @@ -13,17 +13,19 @@ limitations under the License. */ -package utils +package email import ( "bytes" "crypto/tls" - "strings" + //"strings" "net/smtp" "text/template" - "github.com/astaxie/beego" + //"github.com/astaxie/beego" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/ui/config" ) // Mail holds information about content of Email @@ -34,24 +36,15 @@ type Mail struct { Message string } -// MailConfig holds information about Email configurations -type MailConfig struct { - Identity string - Host string - Port string - Username string - Password string - TLS bool -} - -var mc MailConfig +var mc models.Email // SendMail sends Email according to the configurations func (m Mail) SendMail() error { - - if mc.Host == "" { - loadConfig() + mc, err := config.Email() + if err != nil { + return err } + mailTemplate, err := template.ParseFiles("views/mail.tpl") if err != nil { return err @@ -123,6 +116,7 @@ func sendMailWithTLS(m Mail, auth smtp.Auth, content []byte) error { return client.Quit() } +/* func loadConfig() { config, err := beego.AppConfig.GetSection("mail") if err != nil { @@ -142,3 +136,4 @@ func loadConfig() { TLS: useTLS, } } +*/ diff --git a/src/common/utils/log/logger.go b/src/common/utils/log/logger.go index dceb70a97..db5d3b4a2 100644 --- a/src/common/utils/log/logger.go +++ b/src/common/utils/log/logger.go @@ -22,8 +22,6 @@ import ( "runtime" "sync" "time" - - "github.com/vmware/harbor/src/common/config" ) var logger = New(os.Stdout, NewTextFormatter(), WarningLevel) @@ -31,7 +29,7 @@ var logger = New(os.Stdout, NewTextFormatter(), WarningLevel) func init() { logger.callDepth = 4 - lvl := config.LogLevel() + lvl := os.Getenv("LOG_LEVEL") if len(lvl) == 0 { logger.SetLevel(InfoLevel) return diff --git a/src/common/utils/registry/auth/tokenauthorizer.go b/src/common/utils/registry/auth/tokenauthorizer.go index 14b230e5a..917e43f38 100644 --- a/src/common/utils/registry/auth/tokenauthorizer.go +++ b/src/common/utils/registry/auth/tokenauthorizer.go @@ -25,7 +25,7 @@ import ( "sync" "time" - "github.com/vmware/harbor/src/common/config" + //"github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/common/utils/registry" registry_error "github.com/vmware/harbor/src/common/utils/registry/error" @@ -234,12 +234,15 @@ func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes [] // 2. the realm field returned by registry is an IP which can not reachable // inside Harbor func tokenURL(realm string) string { - extEndpoint := config.ExtEndpoint() - tokenEndpoint := config.TokenEndpoint() - if len(extEndpoint) != 0 && len(tokenEndpoint) != 0 && - strings.Contains(realm, extEndpoint) { - realm = strings.TrimRight(tokenEndpoint, "/") + "/service/token" - } + //TODO + /* + extEndpoint := config.ExtEndpoint() + tokenEndpoint := config.TokenEndpoint() + if len(extEndpoint) != 0 && len(tokenEndpoint) != 0 && + strings.Contains(realm, extEndpoint) { + realm = strings.TrimRight(tokenEndpoint, "/") + "/service/token" + } + */ return realm } diff --git a/src/common/utils/utils.go b/src/common/utils/utils.go index 4e930514e..a3ee7a5c7 100644 --- a/src/common/utils/utils.go +++ b/src/common/utils/utils.go @@ -16,10 +16,14 @@ package utils import ( + "fmt" "math/rand" + "net" "net/url" "strings" "time" + + "github.com/vmware/harbor/src/common/utils/log" ) // FormatEndpoint formats endpoint @@ -70,3 +74,37 @@ func GenerateRandomString() string { } return string(result) } + +// timeout in second +func TestTCPConn(addr string, timeout, interval int) error { + success := make(chan int) + cancel := make(chan int) + + go func() { + for { + select { + case <-cancel: + break + default: + conn, err := net.DialTimeout("tcp", addr, time.Duration(timeout)*time.Second) + if err != nil { + log.Errorf("failed to connect to tcp://%s, retry after %d seconds :%v", + addr, interval, err) + time.Sleep(time.Duration(interval) * time.Second) + continue + } + conn.Close() + success <- 1 + break + } + } + }() + + select { + case <-success: + return nil + case <-time.After(time.Duration(timeout) * time.Second): + cancel <- 1 + return fmt.Errorf("failed to connect to tcp:%s after %d seconds", addr, timeout) + } +} diff --git a/src/jobservice/api/replication.go b/src/jobservice/api/replication.go index 51807ed39..25aef81e4 100644 --- a/src/jobservice/api/replication.go +++ b/src/jobservice/api/replication.go @@ -25,12 +25,12 @@ import ( "github.com/vmware/harbor/src/common/api" "github.com/vmware/harbor/src/common/dao" - "github.com/vmware/harbor/src/jobservice/job" - "github.com/vmware/harbor/src/jobservice/config" - "github.com/vmware/harbor/src/jobservice/utils" "github.com/vmware/harbor/src/common/models" u "github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" + "github.com/vmware/harbor/src/jobservice/job" + "github.com/vmware/harbor/src/jobservice/utils" ) // ReplicationJob handles /api/replicationJobs /api/replicationJobs/:id/log @@ -171,7 +171,13 @@ func (rj *ReplicationJob) GetLog() { rj.RenderError(http.StatusBadRequest, "Invalid job id") return } - logFile := utils.GetJobLogPath(jid) + logFile, err := utils.GetJobLogPath(jid) + if err != nil { + log.Errorf("failed to get log path of job %s: %v", idStr, err) + rj.RenderError(http.StatusInternalServerError, + http.StatusText(http.StatusInternalServerError)) + return + } rj.Ctx.Output.Download(logFile) } diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index 2e1b2c31d..18999e855 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -16,121 +16,149 @@ package config import ( - "fmt" + "encoding/json" "os" - "strconv" + "time" - "github.com/astaxie/beego" - "github.com/vmware/harbor/src/common/utils/log" + comcfg "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/models" + //"github.com/vmware/harbor/src/common/utils/log" ) -const defaultMaxWorkers int = 10 +var mg *comcfg.Manager -var maxJobWorkers int -var localUIURL string -var localRegURL string -var logDir string -var uiSecret string -var secretKey string -var verifyRemoteCert string +// Configuration holds configurations of Jobservice +type Configuration struct { + Database *models.Database `json:"database"` + Registry *models.Registry `json:"registry"` + VerifyRemoteCert bool `json:"verify_remote_cert"` + MaxJobWorkers int `json:"max_job_workers"` + JobLogDir string `json:"job_log_dir"` + SecretKey string `json:"secret_key"` + CfgExpiration int `json:"cfg_expiration"` +} -func init() { - maxWorkersEnv := os.Getenv("MAX_JOB_WORKERS") - maxWorkers64, err := strconv.ParseInt(maxWorkersEnv, 10, 32) - maxJobWorkers = int(maxWorkers64) +func Init() error { + adminServerURL := os.Getenv("ADMIN_SERVER_URL") + if len(adminServerURL) == 0 { + adminServerURL = "http://admin_server" + } + mg = comcfg.NewManager("cfg", adminServerURL) + + if err := mg.Loader.Init(); err != nil { + return err + } + + if err := load(); err != nil { + return err + } + + path, err := LogDir() if err != nil { - log.Warningf("Failed to parse max works setting, error: %v, the default value: %d will be used", err, defaultMaxWorkers) - maxJobWorkers = defaultMaxWorkers + return err + } + if err := os.MkdirAll(path, 0600); err != nil { + return err } - localRegURL = os.Getenv("REGISTRY_URL") - if len(localRegURL) == 0 { - localRegURL = "http://registry:5000" + return nil +} + +// get returns configurations of jobservice from cache, +// if cache is null, it loads first +func get() (*Configuration, error) { + cfg := mg.GetFromCache() + if cfg != nil { + return cfg.(*Configuration), nil } - localUIURL = os.Getenv("UI_URL") - if len(localUIURL) == 0 { - localUIURL = "http://ui" + if err := load(); err != nil { + return nil, err } - logDir = os.Getenv("LOG_DIR") - if len(logDir) == 0 { - logDir = "/var/log" - } + return mg.GetFromCache().(*Configuration), nil +} - f, err := os.Open(logDir) - defer f.Close() +// load loads configurations of jobservice and puts them into cache +func load() error { + raw, err := mg.Loader.Load() if err != nil { - panic(err) + return err } - finfo, err := f.Stat() + + cfg := &Configuration{} + if err = json.Unmarshal(raw, cfg); err != nil { + return err + } + + if err = mg.Cache.Put(mg.Key, cfg, + time.Duration(cfg.CfgExpiration)*time.Second); err != nil { + return err + } + + return nil +} + +// VerifyRemoteCert returns bool value. +func VerifyRemoteCert() (bool, error) { + cfg, err := get() if err != nil { - panic(err) - } - if !finfo.IsDir() { - panic(fmt.Sprintf("%s is not a direcotry", logDir)) + return true, err } + return cfg.VerifyRemoteCert, nil +} - uiSecret = os.Getenv("UI_SECRET") - if len(uiSecret) == 0 { - panic("UI Secret is not set") +// Database ... +func Database() (*models.Database, error) { + cfg, err := get() + if err != nil { + return nil, err } - - verifyRemoteCert = os.Getenv("VERIFY_REMOTE_CERT") - if len(verifyRemoteCert) == 0 { - verifyRemoteCert = "on" - } - - configPath := os.Getenv("CONFIG_PATH") - if len(configPath) != 0 { - log.Infof("Config path: %s", configPath) - beego.LoadAppConfig("ini", configPath) - } - - secretKey = os.Getenv("SECRET_KEY") - if len(secretKey) != 16 { - panic("The length of secretkey has to be 16 characters!") - } - - log.Debugf("config: maxJobWorkers: %d", maxJobWorkers) - log.Debugf("config: localUIURL: %s", localUIURL) - log.Debugf("config: localRegURL: %s", localRegURL) - log.Debugf("config: verifyRemoteCert: %s", verifyRemoteCert) - log.Debugf("config: logDir: %s", logDir) - log.Debugf("config: uiSecret: ******") + return cfg.Database, nil } // MaxJobWorkers ... -func MaxJobWorkers() int { - return maxJobWorkers +func MaxJobWorkers() (int, error) { + cfg, err := get() + if err != nil { + return 0, err + } + return cfg.MaxJobWorkers, nil } // LocalUIURL returns the local ui url, job service will use this URL to call API hosted on ui process func LocalUIURL() string { - return localUIURL + return "http://ui" } // LocalRegURL returns the local registry url, job service will use this URL to pull image from the registry -func LocalRegURL() string { - return localRegURL +func LocalRegURL() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.Registry.URL, nil } // LogDir returns the absolute path to which the log file will be written -func LogDir() string { - return logDir +func LogDir() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.JobLogDir, nil } // UISecret will return the value of secret cookie for jobsevice to call UI API. func UISecret() string { - return uiSecret + return os.Getenv("UI_SECRET") } // SecretKey will return the secret key for encryption/decryption password in target. -func SecretKey() string { - return secretKey -} - -// VerifyRemoteCert return the flag to tell jobservice whether or not verify the cert of remote registry -func VerifyRemoteCert() bool { - return verifyRemoteCert != "off" +func SecretKey() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.SecretKey, nil } diff --git a/src/jobservice/job/statemachine.go b/src/jobservice/job/statemachine.go index fd12d5443..b6c9cd47a 100644 --- a/src/jobservice/job/statemachine.go +++ b/src/jobservice/job/statemachine.go @@ -20,12 +20,12 @@ import ( "sync" "github.com/vmware/harbor/src/common/dao" - "github.com/vmware/harbor/src/jobservice/config" - "github.com/vmware/harbor/src/jobservice/replication" - "github.com/vmware/harbor/src/jobservice/utils" "github.com/vmware/harbor/src/common/models" uti "github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" + "github.com/vmware/harbor/src/jobservice/replication" + "github.com/vmware/harbor/src/jobservice/utils" ) // RepJobParm wraps the parm of a job @@ -184,14 +184,17 @@ func (sm *SM) Init() { } // Reset resets the state machine so it will start handling another job. -func (sm *SM) Reset(jid int64) error { +func (sm *SM) Reset(jid int64) (err error) { //To ensure the new jobID is visible to the thread to stop the SM sm.lock.Lock() sm.JobID = jid sm.desiredState = "" sm.lock.Unlock() - sm.Logger = utils.NewLogger(sm.JobID) + sm.Logger, err = utils.NewLogger(sm.JobID) + if err != nil { + return + } //init parms job, err := dao.GetRepJob(sm.JobID) if err != nil { @@ -207,13 +210,22 @@ func (sm *SM) Reset(jid int64) error { if policy == nil { return fmt.Errorf("The policy doesn't exist in DB, policy id:%d", job.PolicyID) } + + regURL, err := config.LocalRegURL() + if err != nil { + return err + } + verify, err := config.VerifyRemoteCert() + if err != nil { + return err + } sm.Parms = &RepJobParm{ - LocalRegURL: config.LocalRegURL(), + LocalRegURL: regURL, Repository: job.Repository, Tags: job.TagList, Enabled: policy.Enabled, Operation: job.Operation, - Insecure: !config.VerifyRemoteCert(), + Insecure: !verify, } if policy.Enabled == 0 { //worker will cancel this job @@ -231,7 +243,11 @@ func (sm *SM) Reset(jid int64) error { pwd := target.Password if len(pwd) != 0 { - pwd, err = uti.ReversibleDecrypt(pwd, config.SecretKey()) + key, err := config.SecretKey() + if err != nil { + return err + } + pwd, err = uti.ReversibleDecrypt(pwd, key) if err != nil { return fmt.Errorf("failed to decrypt password: %v", err) } diff --git a/src/jobservice/job/workerpool.go b/src/jobservice/job/workerpool.go index a1034441a..06301cd69 100644 --- a/src/jobservice/job/workerpool.go +++ b/src/jobservice/job/workerpool.go @@ -17,9 +17,9 @@ package job import ( "github.com/vmware/harbor/src/common/dao" - "github.com/vmware/harbor/src/jobservice/config" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" ) type workerPool struct { @@ -111,17 +111,22 @@ func NewWorker(id int) *Worker { } // InitWorkerPool create workers according to configuration. -func InitWorkerPool() { - WorkerPool = &workerPool{ - workerChan: make(chan *Worker, config.MaxJobWorkers()), - workerList: make([]*Worker, 0, config.MaxJobWorkers()), +func InitWorkerPool() error { + n, err := config.MaxJobWorkers() + if err != nil { + return err } - for i := 0; i < config.MaxJobWorkers(); i++ { + WorkerPool = &workerPool{ + workerChan: make(chan *Worker, n), + workerList: make([]*Worker, 0, n), + } + for i := 0; i < n; i++ { worker := NewWorker(i) WorkerPool.workerList = append(WorkerPool.workerList, worker) worker.Start() log.Debugf("worker %d started", worker.ID) } + return nil } // Dispatch will listen to the jobQueue of job service and try to pick a free worker from the worker pool and assign the job to it. diff --git a/src/jobservice/main.go b/src/jobservice/main.go index 5467d96c6..33516761a 100644 --- a/src/jobservice/main.go +++ b/src/jobservice/main.go @@ -18,13 +18,28 @@ package main import ( "github.com/astaxie/beego" "github.com/vmware/harbor/src/common/dao" - "github.com/vmware/harbor/src/jobservice/job" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" + "github.com/vmware/harbor/src/jobservice/job" ) func main() { - dao.InitDatabase() + log.Info("initializing configurations...") + if err := config.Init(); err != nil { + log.Fatalf("failed to initialize configurations: %v", err) + } + log.Info("configurations initialization completed") + + database, err := config.Database() + if err != nil { + log.Fatalf("failed to get database configurations: %v", err) + } + + if err := dao.InitDatabase(database); err != nil { + log.Fatalf("failed to initialize database: %v", err) + } + initRouters() job.InitWorkerPool() go job.Dispatch() @@ -48,3 +63,13 @@ func resumeJobs() { log.Warningf("Failed to jobs to resume, error: %v", err) } } + +/* +func init() { + configPath := os.Getenv("CONFIG_PATH") + if len(configPath) != 0 { + log.Infof("Config path: %s", configPath) + beego.LoadAppConfig("ini", configPath) + } +} +*/ diff --git a/src/jobservice/utils/logger.go b/src/jobservice/utils/logger.go index 6a358285c..7b3b164f4 100644 --- a/src/jobservice/utils/logger.go +++ b/src/jobservice/utils/logger.go @@ -18,16 +18,20 @@ package utils import ( "fmt" - "github.com/vmware/harbor/src/jobservice/config" - "github.com/vmware/harbor/src/common/utils/log" "os" "path/filepath" "strconv" + + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" ) // NewLogger create a logger for a speicified job -func NewLogger(jobID int64) *log.Logger { - logFile := GetJobLogPath(jobID) +func NewLogger(jobID int64) (*log.Logger, error) { + logFile, err := GetJobLogPath(jobID) + if err != nil { + return nil, err + } d := filepath.Dir(logFile) if _, err := os.Stat(d); os.IsNotExist(err) { err := os.MkdirAll(d, 0660) @@ -40,11 +44,11 @@ func NewLogger(jobID int64) *log.Logger { log.Errorf("Failed to open log file %s, the log of job %d will be printed to standard output, the error: %v", logFile, jobID, err) f = os.Stdout } - return log.New(f, log.NewTextFormatter(), log.InfoLevel) + return log.New(f, log.NewTextFormatter(), log.InfoLevel), nil } // GetJobLogPath returns the absolute path in which the job log file is located. -func GetJobLogPath(jobID int64) string { +func GetJobLogPath(jobID int64) (string, error) { f := fmt.Sprintf("job_%d.log", jobID) k := jobID / 1000 p := "" @@ -61,6 +65,10 @@ func GetJobLogPath(jobID int64) string { p = filepath.Join(d, p) } - p = filepath.Join(config.LogDir(), p, f) - return p + base, err := config.LogDir() + if err != nil { + return "", err + } + p = filepath.Join(base, p, f) + return p, nil } diff --git a/src/ui/api/config.go b/src/ui/api/config.go new file mode 100644 index 000000000..70e41459f --- /dev/null +++ b/src/ui/api/config.go @@ -0,0 +1,245 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "fmt" + "net/http" + "strconv" + //"strings" + + "github.com/vmware/harbor/src/common/api" + comcfg "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/dao" + //"github.com/vmware/harbor/src/common/models" + //"github.com/vmware/harbor/src/common/utils" + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/config" +) + +type ConfigAPI struct { + api.BaseAPI +} + +// Prepare validates the user +func (c *ConfigAPI) Prepare() { + userID := c.ValidateUser() + isSysAdmin, err := dao.IsAdminRole(userID) + if err != nil { + log.Errorf("failed to check the role of user: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if !isSysAdmin { + c.CustomAbort(http.StatusForbidden, http.StatusText(http.StatusForbidden)) + } +} + +// Get returns configurations +func (c *ConfigAPI) Get() { + cfg, err := config.GetSystemCfg() + if err != nil { + log.Errorf("failed to get configurations: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + //TODO filter attr in sys config + + c.Data["json"] = cfg + c.ServeJSON() +} + +// Put updates configurations +func (c *ConfigAPI) Put() { + m := map[string]string{} + c.DecodeJSONReq(&m) + if err := validateCfg(m); err != nil { + c.CustomAbort(http.StatusBadRequest, err.Error()) + } + + if value, ok := m[comcfg.AUTH_MODE]; ok { + mode, err := config.AuthMode() + if err != nil { + log.Errorf("failed to get auth mode: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if mode != value { + flag, err := authModeCanBeModified() + if err != nil { + log.Errorf("failed to determine whether auth mode can be modified: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if !flag { + c.CustomAbort(http.StatusBadRequest, + fmt.Sprintf("%s can not be modified as new users have been inserted into database", + comcfg.AUTH_MODE)) + } + } + } + + log.Info(m) + + if err := config.Upload(m); err != nil { + log.Errorf("failed to upload configurations: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if err := config.Load(); err != nil { + log.Errorf("failed to load configurations: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } +} + +// TODO ldap timeout, scope value +func validateCfg(c map[string]string) error { + if value, ok := c[comcfg.AUTH_MODE]; ok { + if value != comcfg.DB_AUTH && value != comcfg.LDAP_AUTH { + return fmt.Errorf("invalid %s, shoud be %s or %s", comcfg.AUTH_MODE, comcfg.DB_AUTH, comcfg.LDAP_AUTH) + } + + if value == comcfg.LDAP_AUTH { + if _, ok := c[comcfg.LDAP_URL]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAP_URL) + } + if _, ok := c[comcfg.LDAP_BASE_DN]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAP_BASE_DN) + } + if _, ok := c[comcfg.LDAP_UID]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAP_UID) + } + if _, ok := c[comcfg.LDAP_SCOPE]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAP_SCOPE) + } + } + } + + if ldapURL, ok := c[comcfg.LDAP_URL]; ok && len(ldapURL) == 0 { + return fmt.Errorf("%s is empty", comcfg.LDAP_URL) + } + if baseDN, ok := c[comcfg.LDAP_BASE_DN]; ok && len(baseDN) == 0 { + return fmt.Errorf("%s is empty", comcfg.LDAP_BASE_DN) + } + if uID, ok := c[comcfg.LDAP_UID]; ok && len(uID) == 0 { + return fmt.Errorf("%s is empty", comcfg.LDAP_UID) + } + if scope, ok := c[comcfg.LDAP_SCOPE]; ok && + scope != comcfg.LDAP_SCOPE_BASE && + scope != comcfg.LDAP_SCOPE_ONELEVEL && + scope != comcfg.LDAP_SCOPE_SUBTREE { + return fmt.Errorf("invalid %s, should be %s, %s or %s", + comcfg.LDAP_SCOPE, + comcfg.LDAP_SCOPE_BASE, + comcfg.LDAP_SCOPE_ONELEVEL, + comcfg.LDAP_SCOPE_SUBTREE) + } + + if self, ok := c[comcfg.SELF_REGISTRATION]; ok && + self != "true" && self != "false" { + return fmt.Errorf("%s should be %s or %s", + comcfg.SELF_REGISTRATION, "true", "false") + } + + if port, ok := c[comcfg.EMAIL_SERVER_PORT]; ok { + if p, err := strconv.Atoi(port); err != nil || p < 0 || p > 65535 { + return fmt.Errorf("invalid %s", comcfg.EMAIL_SERVER_PORT) + } + } + + if ssl, ok := c[comcfg.EMAIL_SSL]; ok && ssl != "true" && ssl != "false" { + return fmt.Errorf("%s should be true or false", comcfg.EMAIL_SSL) + } + + if crt, ok := c[comcfg.PROJECT_CREATION_RESTRICTION]; ok && + crt != comcfg.PRO_CRT_RESTR_EVERYONE && + crt != comcfg.PRO_CRT_RESTR_ADM_ONLY { + return fmt.Errorf("invalid %s, should be %s or %s", + comcfg.PROJECT_CREATION_RESTRICTION, + comcfg.PRO_CRT_RESTR_ADM_ONLY, + comcfg.PRO_CRT_RESTR_EVERYONE) + } + + if verify, ok := c[comcfg.VERIFY_REMOTE_CERT]; ok && verify != "true" && verify != "false" { + return fmt.Errorf("invalid %s, should be true or false", comcfg.VERIFY_REMOTE_CERT) + } + + if worker, ok := c[comcfg.MAX_JOB_WORKERS]; ok { + if w, err := strconv.Atoi(worker); err != nil || w <= 0 { + return fmt.Errorf("invalid %s", comcfg.MAX_JOB_WORKERS) + } + } + + return nil +} + +/* +func convert() ([]*models.Config, error) { + cfgs := []*models.Config{} + var err error + pwdKeys := []string{config.LDAP_SEARCH_PWD, config.EMAIL_PWD} + for _, pwdKey := range pwdKeys { + if pwd, ok := c[pwdKey]; ok && len(pwd) != 0 { + c[pwdKey], err = utils.ReversibleEncrypt(pwd, ui_cfg.SecretKey()) + if err != nil { + return nil, err + } + } + } + + for _, key := range configKeys { + if value, ok := c[key]; ok { + cfgs = append(cfgs, &models.Config{ + Key: key, + Value: value, + }) + } + } + + return cfgs, nil +} +*/ +/* +//[]*models.Config >> cfgForGet +func convert(cfg *config.Configuration) (map[string]interface{}, error) { + result := map[string]interface{}{} + + for _, config := range configs { + cfg[config.Key] = &value{ + Value: config.Value, + Editable: true, + } + } + + dels := []string{config.LDAP_SEARCH_PWD, config.EMAIL_PWD} + for _, del := range dels { + if _, ok := cfg[del]; ok { + delete(cfg, del) + } + } + + flag, err := authModeCanBeModified() + if err != nil { + return nil, err + } + cfg[config.AUTH_MODE].Editable = flag + + return cfgForGet(cfg), nil +} +*/ +func authModeCanBeModified() (bool, error) { + return dao.AuthModeCanBeModified() +} diff --git a/src/ui/api/project.go b/src/ui/api/project.go index 7ba50fa3b..58ec9b96f 100644 --- a/src/ui/api/project.go +++ b/src/ui/api/project.go @@ -77,7 +77,13 @@ func (p *ProjectAPI) Post() { if err != nil { log.Errorf("Failed to check admin role: %v", err) } - if !isSysAdmin && config.OnlyAdminCreateProject() { + + onlyAdmin, err := config.OnlyAdminCreateProject() + if err != nil { + log.Errorf("failed to determine whether only admin can create projects: %v", err) + p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + if !isSysAdmin && onlyAdmin { log.Errorf("Only sys admin can create project") p.RenderError(http.StatusForbidden, "Only system admin can create project") return diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index 4db0d1eb1..9bc7036c0 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -361,11 +361,19 @@ func (ra *RepositoryAPI) GetManifests() { } func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repository, err error) { - endpoint := config.InternalRegistryURL() + endpoint, err := config.RegistryURL() + if err != nil { + return nil, err + } + + insecure, err := api.GetIsInsecure() + if err != nil { + return nil, err + } username, password, ok := ra.Ctx.Request.BasicAuth() if ok { - return newRepositoryClient(endpoint, api.GetIsInsecure(), username, password, + return newRepositoryClient(endpoint, insecure, username, password, repoName, "repository", repoName, "pull", "push", "*") } @@ -374,7 +382,7 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo return nil, err } - return cache.NewRepositoryClient(endpoint, api.GetIsInsecure(), username, repoName, + return cache.NewRepositoryClient(endpoint, insecure, username, repoName, "repository", repoName, "pull", "push", "*") } diff --git a/src/ui/api/target.go b/src/ui/api/target.go index abe59a09c..c1f88d2ba 100644 --- a/src/ui/api/target.go +++ b/src/ui/api/target.go @@ -41,7 +41,12 @@ type TargetAPI struct { // Prepare validates the user func (t *TargetAPI) Prepare() { - t.secretKey = config.SecretKey() + var err error + t.secretKey, err = config.SecretKey() + if err != nil { + log.Errorf("failed to get secret key: %v", err) + t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } userID := t.ValidateUser() isSysAdmin, err := dao.IsAdminRole(userID) @@ -97,7 +102,12 @@ func (t *TargetAPI) Ping() { password = t.GetString("password") } - registry, err := newRegistryClient(endpoint, api.GetIsInsecure(), username, password, + insecure, err := api.GetIsInsecure() + if err != nil { + log.Errorf("failed to check whether insecure or not: %v", err) + t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + registry, err := newRegistryClient(endpoint, insecure, username, password, "", "", "") if err != nil { // timeout, dns resolve error, connection refused, etc. diff --git a/src/ui/api/user.go b/src/ui/api/user.go index 7c1212b45..d5ca84efb 100644 --- a/src/ui/api/user.go +++ b/src/ui/api/user.go @@ -46,10 +46,21 @@ type passwordReq struct { // Prepare validates the URL and parms func (ua *UserAPI) Prepare() { + mode, err := config.AuthMode() + if err != nil { + log.Errorf("failed to get auth mode: %v", err) + ua.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } - ua.AuthMode = config.AuthMode() + ua.AuthMode = mode - ua.SelfRegistration = config.SelfRegistration() + self, err := config.SelfRegistration() + if err != nil { + log.Errorf("failed to get self registration: %v", err) + ua.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + ua.SelfRegistration = self if ua.Ctx.Input.IsPost() { sessionUserID := ua.GetSession("userId") @@ -82,7 +93,6 @@ func (ua *UserAPI) Prepare() { } } - var err error ua.IsAdmin, err = dao.IsAdminRole(ua.currentUserID) if err != nil { log.Errorf("Error occurred in IsAdminRole:%v", err) @@ -234,7 +244,7 @@ func (ua *UserAPI) Delete() { return } - if config.AuthMode() == "ldap_auth" { + if ua.AuthMode == "ldap_auth" { ua.CustomAbort(http.StatusForbidden, "user can not be deleted in LDAP authentication mode") } diff --git a/src/ui/api/utils.go b/src/ui/api/utils.go index 185db1f18..73a11c869 100644 --- a/src/ui/api/utils.go +++ b/src/ui/api/utils.go @@ -20,11 +20,9 @@ import ( "encoding/json" "fmt" "io/ioutil" - "net" "net/http" "sort" "strings" - "time" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" @@ -242,7 +240,7 @@ func addAuthentication(req *http.Request) { // SyncRegistry syncs the repositories of registry with database. func SyncRegistry() error { - log.Debugf("Start syncing repositories from registry to DB... ") + log.Infof("Start syncing repositories from registry to DB... ") reposInRegistry, err := catalog() if err != nil { @@ -304,7 +302,7 @@ func SyncRegistry() error { } } - log.Debugf("Sync repositories from registry to DB is done.") + log.Infof("Sync repositories from registry to DB is done.") return nil } @@ -350,7 +348,10 @@ func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string } // TODO remove the workaround when the bug of registry is fixed - endpoint := config.InternalRegistryURL() + endpoint, err := config.RegistryURL() + if err != nil { + return needsAdd, needsDel, err + } client, err := cache.NewRepositoryClient(endpoint, true, "admin", repoInR, "repository", repoInR) if err != nil { @@ -372,7 +373,10 @@ func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string j++ } else { // TODO remove the workaround when the bug of registry is fixed - endpoint := config.InternalRegistryURL() + endpoint, err := config.RegistryURL() + if err != nil { + return needsAdd, needsDel, err + } client, err := cache.NewRepositoryClient(endpoint, true, "admin", repoInR, "repository", repoInR) if err != nil { @@ -422,32 +426,18 @@ func projectExists(repository string) (bool, error) { } func initRegistryClient() (r *registry.Registry, err error) { - endpoint := config.InternalRegistryURL() - - addr := endpoint - if strings.Contains(endpoint, "/") { - addr = endpoint[strings.LastIndex(endpoint, "/")+1:] + endpoint, err := config.RegistryURL() + if err != nil { + return nil, err } - ch := make(chan int, 1) - go func() { - var err error - var c net.Conn - for { - c, err = net.DialTimeout("tcp", addr, 20*time.Second) - if err == nil { - c.Close() - ch <- 1 - } else { - log.Errorf("failed to connect to registry client, retry after 2 seconds :%v", err) - time.Sleep(2 * time.Second) - } - } - }() - select { - case <-ch: - case <-time.After(60 * time.Second): - panic("Failed to connect to registry client after 60 seconds") + addr := endpoint + if strings.Contains(endpoint, "://") { + addr = strings.Split(endpoint, "://")[1] + } + + if err := utils.TestTCPConn(addr, 60, 2); err != nil { + return nil, err } registryClient, err := cache.NewRegistryClient(endpoint, true, "admin", diff --git a/src/ui/auth/authenticator.go b/src/ui/auth/authenticator.go index 23abdc8f2..d255b60a9 100644 --- a/src/ui/auth/authenticator.go +++ b/src/ui/auth/authenticator.go @@ -50,7 +50,10 @@ func Register(name string, authenticator Authenticator) { // Login authenticates user credentials based on setting. func Login(m models.AuthModel) (*models.User, error) { - var authMode = config.AuthMode() + authMode, err := config.AuthMode() + if err != nil { + return nil, err + } if authMode == "" || m.Principal == "admin" { authMode = "db_auth" } diff --git a/src/ui/auth/ldap/ldap.go b/src/ui/auth/ldap/ldap.go index 46c563cbe..df27ba837 100644 --- a/src/ui/auth/ldap/ldap.go +++ b/src/ui/auth/ldap/ldap.go @@ -46,7 +46,13 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { return nil, fmt.Errorf("the principal contains meta char: %q", c) } } - ldapURL := config.LDAP().URL + + settings, err := config.LDAP() + if err != nil { + return nil, err + } + + ldapURL := settings.URL if ldapURL == "" { return nil, errors.New("can not get any available LDAP_URL") } @@ -57,16 +63,16 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { } ldap.SetOption(openldap.LDAP_OPT_PROTOCOL_VERSION, openldap.LDAP_VERSION3) - ldapBaseDn := config.LDAP().BaseDn + ldapBaseDn := settings.BaseDN if ldapBaseDn == "" { return nil, errors.New("can not get any available LDAP_BASE_DN") } log.Debug("baseDn:", ldapBaseDn) - ldapSearchDn := config.LDAP().SearchDn + ldapSearchDn := settings.SearchDN if ldapSearchDn != "" { log.Debug("Search DN: ", ldapSearchDn) - ldapSearchPwd := config.LDAP().SearchPwd + ldapSearchPwd := settings.SearchPwd err = ldap.Bind(ldapSearchDn, ldapSearchPwd) if err != nil { log.Debug("Bind search dn error", err) @@ -74,8 +80,8 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { } } - attrName := config.LDAP().UID - filter := config.LDAP().Filter + attrName := settings.UID + filter := settings.Filter if filter != "" { filter = "(&" + filter + "(" + attrName + "=" + m.Principal + "))" } else { @@ -83,11 +89,11 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { } log.Debug("one or more filter", filter) - ldapScope := config.LDAP().Scope + ldapScope := settings.Scope var scope int - if ldapScope == "1" { + if ldapScope == 1 { scope = openldap.LDAP_SCOPE_BASE - } else if ldapScope == "2" { + } else if ldapScope == 2 { scope = openldap.LDAP_SCOPE_ONELEVEL } else { scope = openldap.LDAP_SCOPE_SUBTREE diff --git a/src/ui/config/config.go b/src/ui/config/config.go index 783779c4d..aa726e810 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -13,143 +13,225 @@ limitations under the License. */ -// Package config provides methods to get configurations required by code in src/ui package config import ( - "strconv" - "strings" + "encoding/json" + "os" + "time" - commonConfig "github.com/vmware/harbor/src/common/config" - "github.com/vmware/harbor/src/common/utils/log" + comcfg "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/models" ) -// LDAPSetting wraps the setting of an LDAP server -type LDAPSetting struct { - URL string - BaseDn string - SearchDn string - SearchPwd string - UID string - Filter string - Scope string +var mg *comcfg.Manager + +type Configuration struct { + DomainName string `json:"domain_name"` // Harbor external URL: protocal://host:port + Authentication *models.Authentication `json:"authentication"` + Database *models.Database `json:"database"` + TokenService *models.TokenService `json:"token_service"` + Registry *models.Registry `json:"registry"` + Email *models.Email `json:"email"` + VerifyRemoteCert bool `json:"verify_remote_cert"` + ProjectCreationRestriction string `json:"project_creation_restriction"` + InitialAdminPwd string `json:"initial_admin_pwd"` + //TODO remove + CompressJS bool `json:"compress_js"` + TokenExpiration int `json:"token_expiration"` + SecretKey string `json:"secret_key"` + CfgExpiration int `json:"cfg_expiration` } -type uiParser struct{} +func Init() error { + adminServerURL := os.Getenv("ADMIN_SERVER_URL") + if len(adminServerURL) == 0 { + adminServerURL = "http://admin_server" + } + mg = comcfg.NewManager("cfg", adminServerURL) -// Parse parses the auth settings url settings and other configuration consumed by code under src/ui -func (up *uiParser) Parse(raw map[string]string, config map[string]interface{}) error { - mode := raw["AUTH_MODE"] - if mode == "ldap_auth" { - setting := LDAPSetting{ - URL: raw["LDAP_URL"], - BaseDn: raw["LDAP_BASE_DN"], - SearchDn: raw["LDAP_SEARCH_DN"], - SearchPwd: raw["LDAP_SEARCH_PWD"], - UID: raw["LDAP_UID"], - Filter: raw["LDAP_FILTER"], - Scope: raw["LDAP_SCOPE"], - } - config["ldap"] = setting + if err := mg.Loader.Init(); err != nil { + return err } - config["auth_mode"] = mode - var tokenExpiration = 30 //minutes - if len(raw["TOKEN_EXPIRATION"]) > 0 { - i, err := strconv.Atoi(raw["TOKEN_EXPIRATION"]) - if err != nil { - log.Warningf("failed to parse token expiration: %v, using default value %d", err, tokenExpiration) - } else if i <= 0 { - log.Warningf("invalid token expiration, using default value: %d minutes", tokenExpiration) - } else { - tokenExpiration = i - } + + if err := Load(); err != nil { + return err } - config["token_exp"] = tokenExpiration - config["admin_password"] = raw["HARBOR_ADMIN_PASSWORD"] - config["ext_reg_url"] = raw["EXT_REG_URL"] - config["ui_secret"] = raw["UI_SECRET"] - config["secret_key"] = raw["SECRET_KEY"] - config["self_registration"] = raw["SELF_REGISTRATION"] != "off" - config["admin_create_project"] = strings.ToLower(raw["PROJECT_CREATION_RESTRICTION"]) == "adminonly" - registryURL := raw["REGISTRY_URL"] - registryURL = strings.TrimRight(registryURL, "/") - config["internal_registry_url"] = registryURL - jobserviceURL := raw["JOB_SERVICE_URL"] - jobserviceURL = strings.TrimRight(jobserviceURL, "/") - config["internal_jobservice_url"] = jobserviceURL + return nil } -var uiConfig *commonConfig.Config +// Get returns configurations of UI, if cache is null, it loads first +func get() (*Configuration, error) { + cfg := mg.GetFromCache() + if cfg != nil { + return cfg.(*Configuration), nil + } -func init() { - uiKeys := []string{"AUTH_MODE", "LDAP_URL", "LDAP_BASE_DN", "LDAP_SEARCH_DN", "LDAP_SEARCH_PWD", "LDAP_UID", "LDAP_FILTER", "LDAP_SCOPE", "TOKEN_EXPIRATION", "HARBOR_ADMIN_PASSWORD", "EXT_REG_URL", "UI_SECRET", "SECRET_KEY", "SELF_REGISTRATION", "PROJECT_CREATION_RESTRICTION", "REGISTRY_URL", "JOB_SERVICE_URL"} - uiConfig = &commonConfig.Config{ - Config: make(map[string]interface{}), - Loader: &commonConfig.EnvConfigLoader{Keys: uiKeys}, - Parser: &uiParser{}, - } - if err := uiConfig.Load(); err != nil { - panic(err) + if err := Load(); err != nil { + return nil, err } + + return mg.GetFromCache().(*Configuration), nil } -// Reload ... -func Reload() error { - return uiConfig.Load() +// Load loads configurations of UI and puts them into cache +func Load() error { + raw, err := mg.Loader.Load() + if err != nil { + return err + } + + cfg := &Configuration{} + if err = json.Unmarshal(raw, cfg); err != nil { + return err + } + + if err = mg.Cache.Put(mg.Key, cfg, + time.Duration(cfg.CfgExpiration)*time.Second); err != nil { + return err + } + + return nil +} + +// Upload uploads all system configutations to admin server +func Upload(cfg map[string]string) error { + b, err := json.Marshal(cfg) + if err != nil { + return err + } + return mg.Loader.Upload(b) +} + +// GetSystemCfg returns the system configurations +func GetSystemCfg() (*models.SystemCfg, error) { + raw, err := mg.Loader.Load() + if err != nil { + return nil, err + } + + cfg := &models.SystemCfg{} + if err = json.Unmarshal(raw, cfg); err != nil { + return nil, err + } + return cfg, nil } // AuthMode ... -func AuthMode() string { - return uiConfig.Config["auth_mode"].(string) +func AuthMode() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.Authentication.Mode, nil } // LDAP returns the setting of ldap server -func LDAP() LDAPSetting { - return uiConfig.Config["ldap"].(LDAPSetting) +func LDAP() (*models.LDAP, error) { + cfg, err := get() + if err != nil { + return nil, err + } + return cfg.Authentication.LDAP, nil } // TokenExpiration returns the token expiration time (in minute) -func TokenExpiration() int { - return uiConfig.Config["token_exp"].(int) +func TokenExpiration() (int, error) { + cfg, err := get() + if err != nil { + return 0, err + } + return cfg.TokenExpiration, nil } -// ExtRegistryURL returns the registry URL to exposed to external client -func ExtRegistryURL() string { - return uiConfig.Config["ext_reg_url"].(string) -} - -// UISecret returns the value of UI secret cookie, used for communication between UI and JobService -func UISecret() string { - return uiConfig.Config["ui_secret"].(string) +// DomainName returns the external URL of Harbor: protocal://host:port +func DomainName() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.DomainName, nil } // SecretKey returns the secret key to encrypt the password of target -func SecretKey() string { - return uiConfig.Config["secret_key"].(string) +func SecretKey() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.SecretKey, nil } // SelfRegistration returns the enablement of self registration -func SelfRegistration() bool { - return uiConfig.Config["self_registration"].(bool) +func SelfRegistration() (bool, error) { + cfg, err := get() + if err != nil { + return false, err + } + return cfg.Authentication.SelfRegistration, nil } -// InternalRegistryURL returns registry URL for internal communication between Harbor containers -func InternalRegistryURL() string { - return uiConfig.Config["internal_registry_url"].(string) +// RegistryURL ... +func RegistryURL() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.Registry.URL, nil } // InternalJobServiceURL returns jobservice URL for internal communication between Harbor containers func InternalJobServiceURL() string { - return uiConfig.Config["internal_jobservice_url"].(string) + return "http://jobservice" } // InitialAdminPassword returns the initial password for administrator -func InitialAdminPassword() string { - return uiConfig.Config["admin_password"].(string) +func InitialAdminPassword() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.InitialAdminPwd, nil } +// TODO // OnlyAdminCreateProject returns the flag to restrict that only sys admin can create project -func OnlyAdminCreateProject() bool { - return uiConfig.Config["admin_create_project"].(bool) +func OnlyAdminCreateProject() (bool, error) { + cfg, err := get() + if err != nil { + return true, err + } + return cfg.ProjectCreationRestriction == comcfg.PRO_CRT_RESTR_ADM_ONLY, nil +} + +// VerifyRemoteCert returns bool value. +func VerifyRemoteCert() (bool, error) { + cfg, err := get() + if err != nil { + return true, err + } + return cfg.VerifyRemoteCert, nil +} + +func Email() (*models.Email, error) { + cfg, err := get() + if err != nil { + return nil, err + } + return cfg.Email, nil +} + +func Database() (*models.Database, error) { + cfg, err := get() + if err != nil { + return nil, err + } + return cfg.Database, nil +} + +// TODO +// UISecret returns the value of UI secret cookie, used for communication between UI and JobService +func UISecret() string { + return os.Getenv("UI_SECRET") } diff --git a/src/ui/controllers/base.go b/src/ui/controllers/base.go index 165136a1f..691880af2 100644 --- a/src/ui/controllers/base.go +++ b/src/ui/controllers/base.go @@ -103,7 +103,12 @@ func (b *BaseController) Prepare() { b.Data["CurLang"] = curLang.Name b.Data["RestLangs"] = restLangs - authMode := config.AuthMode() + authMode, err := config.AuthMode() + if err != nil { + log.Errorf("failed to get auth mode: %v", err) + b.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + if authMode == "" { authMode = "db_auth" } @@ -120,9 +125,13 @@ func (b *BaseController) Prepare() { b.UseCompressedJS = false } - b.SelfRegistration = config.SelfRegistration() + b.SelfRegistration, err = config.SelfRegistration() + if err != nil { + log.Errorf("failed to get self registration: %v", err) + b.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } - b.Data["SelfRegistration"] = config.SelfRegistration() + b.Data["SelfRegistration"] = b.SelfRegistration sessionUserID := b.GetSession("userId") if sessionUserID != nil { diff --git a/src/ui/controllers/password.go b/src/ui/controllers/password.go index c3dce0801..786b010a2 100644 --- a/src/ui/controllers/password.go +++ b/src/ui/controllers/password.go @@ -6,12 +6,12 @@ import ( "regexp" "text/template" - "github.com/astaxie/beego" - "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils" + email_util "github.com/vmware/harbor/src/common/utils/email" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/config" ) type messageDetail struct { @@ -49,7 +49,11 @@ func (cc *CommonController) SendEmail() { message := new(bytes.Buffer) - harborURL := config.ExtEndpoint() + harborURL, err := config.DomainName() + if err != nil { + log.Errorf("failed to get domain name: %v", err) + cc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } if harborURL == "" { harborURL = "localhost" } @@ -65,14 +69,14 @@ func (cc *CommonController) SendEmail() { cc.CustomAbort(http.StatusInternalServerError, "internal_error") } - config, err := beego.AppConfig.GetSection("mail") + emailSettings, err := config.Email() if err != nil { - log.Errorf("Can not load app.conf: %v", err) + log.Errorf("failed to get email configurations: %v", err) cc.CustomAbort(http.StatusInternalServerError, "internal_error") } - mail := utils.Mail{ - From: config["from"], + mail := email_util.Mail{ + From: emailSettings.From, To: []string{email}, Subject: cc.Tr("reset_email_subject"), Message: message.String()} diff --git a/src/ui/controllers/project.go b/src/ui/controllers/project.go index 71c7242b0..d2e1ece59 100644 --- a/src/ui/controllers/project.go +++ b/src/ui/controllers/project.go @@ -1,6 +1,8 @@ package controllers import ( + "net/http" + "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/config" @@ -23,6 +25,11 @@ func (pc *ProjectController) Get() { isSysAdmin = false } } - pc.Data["CanCreate"] = !config.OnlyAdminCreateProject() || isSysAdmin + onlyAdmin, err := config.OnlyAdminCreateProject() + if err != nil { + log.Errorf("failed to determine whether only admin can create projects: %v", err) + pc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + pc.Data["CanCreate"] = !onlyAdmin || isSysAdmin pc.Forward("page_title_project", "project.htm") } diff --git a/src/ui/controllers/repository.go b/src/ui/controllers/repository.go index b85ad8782..0fddff097 100644 --- a/src/ui/controllers/repository.go +++ b/src/ui/controllers/repository.go @@ -1,6 +1,10 @@ package controllers import ( + "net/http" + "strings" + + "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/config" ) @@ -11,6 +15,11 @@ type RepositoryController struct { // Get renders repository page func (rc *RepositoryController) Get() { - rc.Data["HarborRegUrl"] = config.ExtRegistryURL() + url, err := config.DomainName() + if err != nil { + log.Errorf("failed to get domain name: %v", err) + rc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + rc.Data["HarborRegUrl"] = strings.Split(url, "://")[1] rc.Forward("page_title_repository", "repository.htm") } diff --git a/src/ui/main.go b/src/ui/main.go index 6c4427279..14770c41b 100644 --- a/src/ui/main.go +++ b/src/ui/main.go @@ -64,7 +64,6 @@ func updateInitPassword(userID int, password string) error { } func main() { - beego.BConfig.WebConfig.Session.SessionOn = true //TODO redisURL := os.Getenv("_REDIS_URL") @@ -72,12 +71,28 @@ func main() { beego.BConfig.WebConfig.Session.SessionProvider = "redis" beego.BConfig.WebConfig.Session.SessionProviderConfig = redisURL } - // beego.AddTemplateExt("htm") - dao.InitDatabase() + log.Info("initializing configurations...") + if err := config.Init(); err != nil { + log.Fatalf("failed to initialize configurations: %v", err) + } + log.Info("configurations initialization completed") - if err := updateInitPassword(adminUserID, config.InitialAdminPassword()); err != nil { + database, err := config.Database() + if err != nil { + log.Fatalf("failed to get database configuration: %v", err) + } + + if err := dao.InitDatabase(database); err != nil { + log.Fatalf("failed to initialize database: %v", err) + } + + password, err := config.InitialAdminPassword() + if err != nil { + log.Fatalf("failed to get admin's initia password: %v", err) + } + if err := updateInitPassword(adminUserID, password); err != nil { log.Error(err) } initRouters() diff --git a/src/ui/router.go b/src/ui/router.go index 537748b03..0bf121c2e 100644 --- a/src/ui/router.go +++ b/src/ui/router.go @@ -84,6 +84,7 @@ func initRouters() { beego.Router("/api/users/:id/sysadmin", &api.UserAPI{}, "put:ToggleUserAdminRole") beego.Router("/api/repositories/top", &api.RepositoryAPI{}, "get:GetTopRepos") beego.Router("/api/logs", &api.LogAPI{}) + beego.Router("/api/configurations", &api.ConfigAPI{}) beego.Router("/api/systeminfo/volumes", &api.SystemInfoAPI{}, "get:GetVolumeInfo") beego.Router("/api/systeminfo/getcert", &api.SystemInfoAPI{}, "get:GetCert") diff --git a/src/ui/service/token/authutils.go b/src/ui/service/token/authutils.go index 1e687365b..3bda33ca4 100644 --- a/src/ui/service/token/authutils.go +++ b/src/ui/service/token/authutils.go @@ -37,13 +37,6 @@ const ( privateKey = "/etc/ui/private_key.pem" ) -var expiration int //minutes - -func init() { - expiration = config.TokenExpiration() - log.Infof("token expiration: %d minutes", expiration) -} - // GetResourceActions ... func GetResourceActions(scopes []string) []*token.ResourceActions { log.Debugf("scopes: %+v", scopes) @@ -91,7 +84,12 @@ func FilterAccess(username string, a *token.ResourceActions) { repoLength := len(repoSplit) if repoLength > 1 { //Only check the permission when the requested image has a namespace, i.e. project var projectName string - registryURL := config.ExtRegistryURL() + registryURL, err := config.DomainName() + if err != nil { + log.Errorf("failed to get domain name: %v", err) + return + } + registryURL = strings.Split(registryURL, "://")[1] if repoSplit[0] == registryURL { projectName = repoSplit[1] log.Infof("Detected Registry URL in Project Name. Assuming this is a notary request and setting Project Name as %s\n", projectName) @@ -153,6 +151,11 @@ func MakeToken(username, service string, access []*token.ResourceActions) (token if err != nil { return "", 0, nil, err } + expiration, err := config.TokenExpiration() + if err != nil { + return "", 0, nil, err + } + tk, expiresIn, issuedAt, err := makeTokenCore(issuer, username, service, expiration, access, pk) if err != nil { return "", 0, nil, err From 7f949b1a95e6f68d927277fda977addf39e1604b Mon Sep 17 00:00:00 2001 From: wemeya <1013939285@qq.com> Date: Thu, 19 Jan 2017 10:09:56 +0800 Subject: [PATCH 04/22] add test for ldap.go (#1307) LGTM --- src/ui/api/harborapi_test.go | 17 +++++++ src/ui/api/ldap_test.go | 95 +++++++++++++++++++++++++++++++++++ tests/apitests/apilib/ldap.go | 34 +++++++++++++ tests/docker-compose.test.yml | 31 ++++++++++++ 4 files changed, 177 insertions(+) create mode 100644 src/ui/api/ldap_test.go create mode 100644 tests/apitests/apilib/ldap.go diff --git a/src/ui/api/harborapi_test.go b/src/ui/api/harborapi_test.go index 4fd57a1c3..979f661d1 100644 --- a/src/ui/api/harborapi_test.go +++ b/src/ui/api/harborapi_test.go @@ -90,6 +90,7 @@ func init() { beego.Router("/api/policies/replication/:id([0-9]+)/enablement", &RepPolicyAPI{}, "put:UpdateEnablement") beego.Router("/api/systeminfo/volumes", &SystemInfoAPI{}, "get:GetVolumeInfo") beego.Router("/api/systeminfo/getcert", &SystemInfoAPI{}, "get:GetCert") + beego.Router("/api/ldap/ping", &LdapAPI{}, "post:Ping") _ = updateInitPassword(1, "Harbor12345") @@ -897,3 +898,19 @@ func (a testapi) CertGet(authInfo usrInfo) (int, []byte, error) { httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo) return httpStatusCode, body, err } + +//Post ldap test +func (a testapi) LdapPost(authInfo usrInfo, ldapConf apilib.LdapConf) (int, error) { + + _sling := sling.New().Post(a.basePath) + + // create path and map variables + path := "/api/ldap/ping" + + _sling = _sling.Path(path) + + // body params + _sling = _sling.BodyJSON(ldapConf) + httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo) + return httpStatusCode, err +} diff --git a/src/ui/api/ldap_test.go b/src/ui/api/ldap_test.go new file mode 100644 index 000000000..bd0ed861f --- /dev/null +++ b/src/ui/api/ldap_test.go @@ -0,0 +1,95 @@ +package api + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/tests/apitests/apilib" + "testing" +) + +var ldapConf apilib.LdapConf + +func TestLdapPost(t *testing.T) { + fmt.Println("Testing ldap post") + assert := assert.New(t) + apiTest := newHarborAPI() + + //case 1: ping ldap server without admin role + CommonAddUser() + code, err := apiTest.LdapPost(*testUser, ldapConf) + if err != nil { + t.Error("Error occured while ping ldap server") + t.Log(err) + } else { + assert.Equal(403, code, "Ping ldap server status should be 403") + } + //case 2: ping ldap server with admin role, but empty ldapConf + code, err = apiTest.LdapPost(*admin, ldapConf) + if err != nil { + t.Error("Error occured while ping ldap server") + t.Log(err) + } else { + assert.Equal(400, code, "Ping ldap server status should be 400") + } + + //case 3: ping ldap server with admin role, but bad format of ldapConf + ldapConf.LdapURL = "http://127.0.0.1" + code, err = apiTest.LdapPost(*admin, ldapConf) + if err != nil { + t.Error("Error occured while ping ldap server") + t.Log(err) + } else { + assert.Equal(400, code, "Ping ldap server status should be 400") + } + //case 4: ping ldap server with admin role, but bad format of ldapConf + ldapConf.LdapURL = "127.0.0.1:sss" + code, err = apiTest.LdapPost(*admin, ldapConf) + if err != nil { + t.Error("Error occured while ping ldap server") + t.Log(err) + } else { + assert.Equal(400, code, "Ping ldap server status should be 400") + } + //case 5: ping ldap server with admin role, ldap protocol, without port + ldapConf.LdapURL = "127.0.0.1" + code, err = apiTest.LdapPost(*admin, ldapConf) + if err != nil { + t.Error("Error occured while ping ldap server") + t.Log(err) + } else { + assert.Equal(200, code, "Ping ldap server status should be 200") + } + //not success, will try later + /* + //case 6: ping ldap server with admin role, ldaps protocol without port + ldapConf.LdapURL = "ldaps://127.0.0.1" + code, err = apiTest.LdapPost(*admin, ldapConf) + if err != nil { + t.Error("Error occured while ping ldap server") + t.Log(err) + } else { + assert.Equal(200, code, "Ping ldap server status should be 200") + }*/ + //case 7: ping ldap server with admin role, ldap protocol, port, ldapSearchDn, but wrong password + ldapConf.LdapURL = "ldap://127.0.0.1:389" + ldapConf.LdapSearchDn = "cn=admin,dc=example,dc=org" + code, err = apiTest.LdapPost(*admin, ldapConf) + if err != nil { + t.Error("Error occured while ping ldap server") + t.Log(err) + } else { + assert.Equal(400, code, "Ping ldap server status should be 400") + } + //case 8: ping ldap server with admin role, ldap protocol, port, ldapSearchDn, right password + ldapConf.LdapURL = "ldap://127.0.0.1:389" + ldapConf.LdapSearchDn = "cn=admin,dc=example,dc=org" + ldapConf.LdapSearchPassword = "admin" + code, err = apiTest.LdapPost(*admin, ldapConf) + if err != nil { + t.Error("Error occured while ping ldap server") + t.Log(err) + } else { + assert.Equal(200, code, "Ping ldap server status should be 200") + } + CommonDelUser() +} diff --git a/tests/apitests/apilib/ldap.go b/tests/apitests/apilib/ldap.go new file mode 100644 index 000000000..704d232ce --- /dev/null +++ b/tests/apitests/apilib/ldap.go @@ -0,0 +1,34 @@ +/* + * Harbor API + * + * These APIs provide services for manipulating Harbor project. + * + * OpenAPI spec version: 0.3.0 + * + * Generated by: https://github.com/swagger-api/swagger-codegen.git + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package apilib + +type LdapConf struct { + LdapURL string `json:"ldap_url"` + LdapSearchDn string `json:"ldap_search_dn"` + LdapSearchPassword string `json:"ldap_search_password"` + LdapBaseDn string `json:"ldap_base_dn"` + LdapFilter string `json:"ldap_filter"` + LdapUID string `json:"ldap_uid"` + LdapScope int `json:"ldap_scope"` + LdapConnectionTimeout int `json:"ldap_connection_timeout"` +} diff --git a/tests/docker-compose.test.yml b/tests/docker-compose.test.yml index 5a92685f3..63424ad62 100644 --- a/tests/docker-compose.test.yml +++ b/tests/docker-compose.test.yml @@ -21,3 +21,34 @@ services: - ./common/config/db/env ports: - 3306:3306 + ldap: + image: osixia/openldap:1.1.7 + restart: always + environment: + LDAP_LOG_LEVEL: "256" + LDAP_ORGANISATION: "Example Inc." + LDAP_DOMAIN: "example.org" + LDAP_BASE_DN: "" + LDAP_ADMIN_PASSWORD: "admin" + LDAP_CONFIG_PASSWORD: "config" + LDAP_READONLY_USER: "false" + LDAP_BACKEND: "hdb" + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_CIPHER_SUITE: "SECURE256:-VERS-SSL3.0" + LDAP_TLS_PROTOCOL_MIN: "3.1" + LDAP_TLS_VERIFY_CLIENT: "demand" + LDAP_REPLICATION: "false" + LDAP_REMOVE_CONFIG_AFTER_SETUP: "true" + LDAP_SSL_HELPER_PREFIX: "ldap" + volumes: + - /var/lib/ldap + - /etc/ldap/slapd.d + - /container/service/slapd/assets/certs/ + hostname: "example.org" + ports: + - 389:389 + - 636:636 From f1f78a56499feec5cedc0117abce39dbb45d9d8a Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 12 Jan 2017 18:29:02 +0800 Subject: [PATCH 05/22] update --- .travis.yml | 11 +- make/common/templates/adminserver/env | 4 +- make/common/templates/jobservice/env | 3 +- make/harbor.cfg | 2 +- make/prepare | 80 +++---- src/adminserver/api/base.go | 5 + src/adminserver/api/cfg.go | 100 ++++++--- .../systemcfg/{ => store}/driver.go | 2 +- .../systemcfg/{ => store/json}/driver_json.go | 10 +- .../{ => store/json}/driver_json_test.go | 14 +- src/adminserver/systemcfg/systemcfg.go | 153 ++++++------- src/adminserver/systemcfg/systemcfg_test.go | 71 ++++++ src/common/api/base_test.go | 2 + src/common/config/config.go | 191 ++++++++++++---- src/common/config/config_test.go | 96 --------- src/common/dao/dao_test.go | 2 +- src/common/models/config.go | 8 +- src/common/utils/email/mail.go | 2 +- src/common/utils/test/adminserver.go | 59 +++++ src/common/utils/test/test.go | 8 +- src/common/utils/utils.go | 5 +- src/common/utils/utils_test.go | 10 + src/jobservice/config/config.go | 76 +++---- src/jobservice/config/config_test.go | 66 +++++- src/jobservice/main.go | 4 +- src/ui/api/config.go | 122 ++++++----- src/ui/api/harborapi_test.go | 16 +- src/ui/api/target_test.go | 9 +- src/ui/config/config.go | 67 +++--- src/ui/config/config_test.go | 203 +++++++----------- src/ui/controllers/base.go | 6 +- src/ui/controllers/controllers_test.go | 6 +- src/ui/ui_test.go | 2 + tests/docker-compose.test.yml | 11 + tests/startuptest.go | 57 ++--- tests/testprepare.sh | 6 +- 36 files changed, 867 insertions(+), 622 deletions(-) rename src/adminserver/systemcfg/{ => store}/driver.go (98%) rename src/adminserver/systemcfg/{ => store/json}/driver_json.go (90%) rename src/adminserver/systemcfg/{ => store/json}/driver_json_test.go (84%) create mode 100644 src/adminserver/systemcfg/systemcfg_test.go create mode 100644 src/common/utils/test/adminserver.go diff --git a/.travis.yml b/.travis.yml index a80c95942..7337b0bdd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,14 +13,13 @@ services: dist: trusty env: - DB_HOST: 127.0.0.1 - DB_PORT: 3306 - DB_USR: root - DB_PWD: root123 MYSQL_HOST: localhost MYSQL_PORT: 3306 MYSQL_USR: root MYSQL_PWD: root123 + MYSQL_DATABASE: registry + SQLITE_FILE: /tmp/registry.db + ADMIN_SERVER_URL: http://127.0.0.1:8888 DOCKER_COMPOSE_VERSION: 1.7.1 HARBOR_ADMIN: admin HARBOR_ADMIN_PASSWD: Harbor12345 @@ -70,7 +69,7 @@ install: before_script: # create tables and load data # - mysql < ./make/db/registry.sql -uroot --verbose - - sudo sqlite3 /registry.db < make/common/db/registry_sqlite.sql + - sudo sqlite3 /tmp/registry.db < make/common/db/registry_sqlite.sql script: - sudo mkdir -p /harbor_storage/ca_download @@ -88,7 +87,7 @@ script: - goveralls -coverprofile=profile.cov -service=travis-ci - docker-compose -f make/docker-compose.test.yml down - + - sudo make/prepare - docker-compose -f make/dev/docker-compose.yml up -d - docker ps diff --git a/make/common/templates/adminserver/env b/make/common/templates/adminserver/env index 959082ce3..ff0153215 100644 --- a/make/common/templates/adminserver/env +++ b/make/common/templates/adminserver/env @@ -22,7 +22,7 @@ EMAIL_HOST=$email_host EMAIL_PORT=$email_port EMAIL_USR=$email_usr EMAIL_PWD=$email_pwd -EMAIL_TLS=$email_tls +EMAIL_SSL=$email_ssl EMAIL_FROM=$email_from EMAIL_IDENTITY=$email_identity HARBOR_ADMIN_PASSWORD=$harbor_admin_password @@ -33,6 +33,6 @@ LOG_DIR=/var/log/jobs UI_SECRET=$ui_secret SECRET_KEY=$secret_key TOKEN_EXPIRATION=$token_expiration -CFG_EXPIRATION=$cfg_expiration +CFG_EXPIRATION=5 USE_COMPRESSED_JS=$use_compressed_js GODEBUG=netdns=cgo \ No newline at end of file diff --git a/make/common/templates/jobservice/env b/make/common/templates/jobservice/env index d75972ac3..06c8b0f22 100644 --- a/make/common/templates/jobservice/env +++ b/make/common/templates/jobservice/env @@ -1,5 +1,4 @@ LOG_LEVEL=debug -UI_SECRET=$ui_secret CONFIG_PATH=/etc/jobservice/app.conf -MAX_JOB_WORKERS=$max_job_workers +UI_SECRET=$ui_secret GODEBUG=netdns=cgo diff --git a/make/harbor.cfg b/make/harbor.cfg index 69118dbf3..507533edb 100644 --- a/make/harbor.cfg +++ b/make/harbor.cfg @@ -53,7 +53,7 @@ ldap_uid = uid ldap_scope = 3 #Timeout (in seconds) when connecting to an LDAP Server. The default value (and most reasonable) is 5 seconds. -ldap_connect_timeout = 5 +ldap_timeout = 5 #The password for the root user of mysql db, change this before any production use. db_password = root123 diff --git a/make/prepare b/make/prepare index 2610d5fa2..28d46a376 100755 --- a/make/prepare +++ b/make/prepare @@ -77,10 +77,10 @@ hostname = rcp.get("configuration", "hostname") protocol = rcp.get("configuration", "ui_url_protocol") ui_url = protocol + "://" + hostname email_identity = rcp.get("configuration", "email_identity") -email_server = rcp.get("configuration", "email_server") -email_server_port = rcp.get("configuration", "email_server_port") -email_username = rcp.get("configuration", "email_username") -email_password = rcp.get("configuration", "email_password") +email_host = rcp.get("configuration", "email_server") +email_port = rcp.get("configuration", "email_server_port") +email_usr = rcp.get("configuration", "email_username") +email_pwd = rcp.get("configuration", "email_password") email_from = rcp.get("configuration", "email_from") email_ssl = rcp.get("configuration", "email_ssl") harbor_admin_password = rcp.get("configuration", "harbor_admin_password") @@ -101,7 +101,7 @@ else: ldap_filter = "" ldap_uid = rcp.get("configuration", "ldap_uid") ldap_scope = rcp.get("configuration", "ldap_scope") -ldap_connect_timeout = rcp.get("configuration", "ldap_connect_timeout") +ldap_timeout = rcp.get("configuration", "ldap_timeout") db_password = rcp.get("configuration", "db_password") self_registration = rcp.get("configuration", "self_registration") use_compressed_js = rcp.get("configuration", "use_compressed_js") @@ -126,6 +126,10 @@ secret_key = get_secret_key(secretkey_path) ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16)) +adminserver_config_dir = os.path.join(config_dir,"adminserver") +if not os.path.exists(adminserver_config_dir): + os.makedirs(os.path.join(config_dir, "adminserver")) + ui_config_dir = os.path.join(config_dir,"ui") if not os.path.exists(ui_config_dir): os.makedirs(os.path.join(config_dir, "ui")) @@ -152,6 +156,7 @@ def render(src, dest, **kw): f.write(t.substitute(**kw)) print("Generated configuration file: %s" % dest) +adminserver_conf_env = os.path.join(config_dir, "adminserver", "env") ui_conf_env = os.path.join(config_dir, "ui", "env") ui_conf = os.path.join(config_dir, "ui", "app.conf") jobservice_conf = os.path.join(config_dir, "jobservice", "app.conf") @@ -187,14 +192,12 @@ if protocol == "https": else: render(os.path.join(templates_dir, "nginx", "nginx.http.conf"), nginx_conf) - -render(os.path.join(templates_dir, "ui", "env"), - ui_conf_env, - hostname=hostname, - db_password=db_password, - ui_url=ui_url, - auth_mode=auth_mode, - harbor_admin_password=harbor_admin_password, + +render(os.path.join(templates_dir, "adminserver", "env"), + adminserver_conf_env, + ui_url=ui_url, + auth_mode=auth_mode, + self_registration=self_registration, ldap_url=ldap_url, ldap_searchdn =ldap_searchdn, ldap_search_pwd =ldap_search_pwd, @@ -202,27 +205,31 @@ render(os.path.join(templates_dir, "ui", "env"), ldap_filter=ldap_filter, ldap_uid=ldap_uid, ldap_scope=ldap_scope, - ldap_connect_timeout=ldap_connect_timeout, - self_registration=self_registration, - use_compressed_js=use_compressed_js, - ui_secret=ui_secret, + ldap_timeout=ldap_timeout, + db_password=db_password, + email_host=email_host, + email_port=email_port, + email_usr=email_usr, + email_pwd=email_pwd, + email_ssl=email_ssl, + email_from=email_from, + email_identity=email_identity, + harbor_admin_password=harbor_admin_password, + project_creation_restriction=proj_cre_restriction, + verify_remote_cert=verify_remote_cert, + max_job_workers=max_job_workers, + ui_secret=ui_secret, secret_key=secret_key, - verify_remote_cert=verify_remote_cert, - project_creation_restriction=proj_cre_restriction, - token_expiration=token_expiration) + token_expiration=token_expiration, + use_compressed_js=use_compressed_js + ) -render(os.path.join(templates_dir, "ui", "app.conf"), - ui_conf, - email_identity=email_identity, - email_server=email_server, - email_server_port=email_server_port, - email_username=email_username, - email_password=email_password, - email_from=email_from, - email_ssl=email_ssl, - ui_url=ui_url) +render(os.path.join(templates_dir, "ui", "env"), + ui_conf_env, + ui_secret=ui_secret) -render(os.path.join(templates_dir, "registry", "config.yml"), +render(os.path.join(templates_dir, "registry", + "config.yml"), registry_conf, ui_url=ui_url) @@ -232,16 +239,15 @@ render(os.path.join(templates_dir, "db", "env"), render(os.path.join(templates_dir, "jobservice", "env"), job_conf_env, - db_password=db_password, - ui_secret=ui_secret, - max_job_workers=max_job_workers, - secret_key=secret_key, - ui_url=ui_url, - verify_remote_cert=verify_remote_cert) + ui_secret=ui_secret) print("Generated configuration file: %s" % jobservice_conf) shutil.copyfile(os.path.join(templates_dir, "jobservice", "app.conf"), jobservice_conf) +print("Generated configuration file: %s" % ui_conf) +shutil.copyfile(os.path.join(templates_dir, "ui", "app.conf"), ui_conf) + + def validate_crt_subj(dirty_subj): subj_list = [item for item in dirty_subj.strip().split("/") \ if len(item.split("=")) == 2 and len(item.split("=")[1]) > 0] diff --git a/src/adminserver/api/base.go b/src/adminserver/api/base.go index 47938c8b7..e4221bb64 100644 --- a/src/adminserver/api/base.go +++ b/src/adminserver/api/base.go @@ -27,3 +27,8 @@ func handleInternalServerError(w http.ResponseWriter) { func handleBadRequestError(w http.ResponseWriter, error string) { http.Error(w, error, http.StatusBadRequest) } + +func handleUnauthorized(w http.ResponseWriter) { + http.Error(w, http.StatusText(http.StatusUnauthorized), + http.StatusUnauthorized) +} diff --git a/src/adminserver/api/cfg.go b/src/adminserver/api/cfg.go index 09499fec6..e2a41412b 100644 --- a/src/adminserver/api/cfg.go +++ b/src/adminserver/api/cfg.go @@ -19,6 +19,7 @@ import ( "encoding/json" "io/ioutil" "net/http" + "os" "strconv" cfg "github.com/vmware/harbor/src/adminserver/systemcfg" @@ -27,8 +28,32 @@ import ( "github.com/vmware/harbor/src/common/utils/log" ) +func isAuthenticated(r *http.Request) (bool, error) { + secret := os.Getenv("UI_SECRET") + c, err := r.Cookie("secret") + if err != nil { + if err == http.ErrNoCookie { + return false, nil + } + return false, err + } + return c != nil && c.Value == secret, nil +} + // ListCfgs lists configurations func ListCfgs(w http.ResponseWriter, r *http.Request) { + authenticated, err := isAuthenticated(r) + if err != nil { + log.Errorf("failed to check whether the request is authenticated or not: %v", err) + handleInternalServerError(w) + return + } + + if !authenticated { + handleUnauthorized(w) + return + } + cfg, err := cfg.GetSystemCfg() if err != nil { log.Errorf("failed to get system configurations: %v", err) @@ -49,6 +74,18 @@ func ListCfgs(w http.ResponseWriter, r *http.Request) { // UpdateCfgs updates configurations func UpdateCfgs(w http.ResponseWriter, r *http.Request) { + authenticated, err := isAuthenticated(r) + if err != nil { + log.Errorf("failed to check whether the request is authenticated or not: %v", err) + handleInternalServerError(w) + return + } + + if !authenticated { + handleUnauthorized(w) + return + } + b, err := ioutil.ReadAll(r.Body) if err != nil { log.Errorf("failed to read request body: %v", err) @@ -62,8 +99,6 @@ func UpdateCfgs(w http.ResponseWriter, r *http.Request) { return } - log.Info(m) - system, err := cfg.GetSystemCfg() if err != nil { handleInternalServerError(w) @@ -76,8 +111,6 @@ func UpdateCfgs(w http.ResponseWriter, r *http.Request) { return } - log.Info(system.Authentication.SelfRegistration) - if err = cfg.UpdateSystemCfg(system); err != nil { log.Errorf("failed to update system configurations: %v", err) handleInternalServerError(w) @@ -87,74 +120,73 @@ func UpdateCfgs(w http.ResponseWriter, r *http.Request) { // populate attrs of cfg according to m func populate(cfg *models.SystemCfg, m map[string]string) error { - if mode, ok := m[comcfg.AUTH_MODE]; ok { + if mode, ok := m[comcfg.AUTHMode]; ok { cfg.Authentication.Mode = mode } - if value, ok := m[comcfg.SELF_REGISTRATION]; ok { - cfg.Authentication.SelfRegistration = value == "true" + if value, ok := m[comcfg.SelfRegistration]; ok { + cfg.Authentication.SelfRegistration = value == "1" } - if url, ok := m[comcfg.LDAP_URL]; ok { + if url, ok := m[comcfg.LDAPURL]; ok { cfg.Authentication.LDAP.URL = url } - if dn, ok := m[comcfg.LDAP_SEARCH_DN]; ok { + if dn, ok := m[comcfg.LDAPSearchDN]; ok { cfg.Authentication.LDAP.SearchDN = dn } - if pwd, ok := m[comcfg.LDAP_SEARCH_PWD]; ok { + if pwd, ok := m[comcfg.LDAPSearchPwd]; ok { cfg.Authentication.LDAP.SearchPwd = pwd } - if dn, ok := m[comcfg.LDAP_BASE_DN]; ok { + if dn, ok := m[comcfg.LDAPBaseDN]; ok { cfg.Authentication.LDAP.BaseDN = dn } - if uid, ok := m[comcfg.LDAP_UID]; ok { + if uid, ok := m[comcfg.LDAPUID]; ok { cfg.Authentication.LDAP.UID = uid } - if filter, ok := m[comcfg.LDAP_FILTER]; ok { + if filter, ok := m[comcfg.LDAPFilter]; ok { cfg.Authentication.LDAP.Filter = filter } - if scope, ok := m[comcfg.LDAP_SCOPE]; ok { + if scope, ok := m[comcfg.LDAPScope]; ok { i, err := strconv.Atoi(scope) if err != nil { return err } cfg.Authentication.LDAP.Scope = i } + if timeout, ok := m[comcfg.LDAPTimeout]; ok { + i, err := strconv.Atoi(timeout) + if err != nil { + return err + } + cfg.Authentication.LDAP.Timeout = i + } - if value, ok := m[comcfg.EMAIL_SERVER]; ok { + if value, ok := m[comcfg.EmailHost]; ok { cfg.Email.Host = value } - if value, ok := m[comcfg.EMAIL_SERVER_PORT]; ok { + if value, ok := m[comcfg.EmailPort]; ok { cfg.Email.Port = value } - if value, ok := m[comcfg.EMAIL_USERNAME]; ok { + if value, ok := m[comcfg.EmailUsername]; ok { cfg.Email.Username = value } - if value, ok := m[comcfg.EMAIL_PWD]; ok { - cfg.Email.Host = value - } - if value, ok := m[comcfg.EMAIL_SSL]; ok { + if value, ok := m[comcfg.EmailPassword]; ok { cfg.Email.Password = value } - if value, ok := m[comcfg.EMAIL_FROM]; ok { + if value, ok := m[comcfg.EmailSSL]; ok { + cfg.Email.SSL = value == "1" + } + if value, ok := m[comcfg.EmailFrom]; ok { cfg.Email.From = value } - if value, ok := m[comcfg.EMAIL_IDENTITY]; ok { + if value, ok := m[comcfg.EmailIdentity]; ok { cfg.Email.Identity = value } - if value, ok := m[comcfg.PROJECT_CREATION_RESTRICTION]; ok { + if value, ok := m[comcfg.ProjectCreationRestriction]; ok { cfg.ProjectCreationRestriction = value } - if value, ok := m[comcfg.VERIFY_REMOTE_CERT]; ok { - cfg.VerifyRemoteCert = value == "true" - } - - if value, ok := m[comcfg.MAX_JOB_WORKERS]; ok { - if i, err := strconv.Atoi(value); err != nil { - return err - } else { - cfg.MaxJobWorkers = i - } + if value, ok := m[comcfg.VerifyRemoteCert]; ok { + cfg.VerifyRemoteCert = value == "1" } return nil diff --git a/src/adminserver/systemcfg/driver.go b/src/adminserver/systemcfg/store/driver.go similarity index 98% rename from src/adminserver/systemcfg/driver.go rename to src/adminserver/systemcfg/store/driver.go index bfb110f24..c0a9a2832 100644 --- a/src/adminserver/systemcfg/driver.go +++ b/src/adminserver/systemcfg/store/driver.go @@ -13,7 +13,7 @@ limitations under the License. */ -package systemcfg +package store import ( "github.com/vmware/harbor/src/common/models" diff --git a/src/adminserver/systemcfg/driver_json.go b/src/adminserver/systemcfg/store/json/driver_json.go similarity index 90% rename from src/adminserver/systemcfg/driver_json.go rename to src/adminserver/systemcfg/store/json/driver_json.go index 62e2c2ee7..4785e6006 100644 --- a/src/adminserver/systemcfg/driver_json.go +++ b/src/adminserver/systemcfg/store/json/driver_json.go @@ -13,7 +13,7 @@ limitations under the License. */ -package systemcfg +package json import ( "encoding/json" @@ -22,6 +22,7 @@ import ( "path/filepath" "sync" + "github.com/vmware/harbor/src/adminserver/systemcfg/store" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" ) @@ -38,11 +39,14 @@ type cfgStore struct { // NewCfgStore returns an instance of cfgStore that stores the configurations // in a json file. The file will be created if it does not exist. -func NewCfgStore(path ...string) (Driver, error) { +func NewCfgStore(path ...string) (store.Driver, error) { p := defaultPath - if len(path) != 0 { + if len(path) > 0 && len(path[0]) > 0 { p = path[0] } + + log.Debugf("path of configuration file: %s", p) + if _, err := os.Stat(p); os.IsNotExist(err) { log.Infof("the configuration file %s does not exist, creating it...", p) if err = os.MkdirAll(filepath.Dir(p), 0600); err != nil { diff --git a/src/adminserver/systemcfg/driver_json_test.go b/src/adminserver/systemcfg/store/json/driver_json_test.go similarity index 84% rename from src/adminserver/systemcfg/driver_json_test.go rename to src/adminserver/systemcfg/store/json/driver_json_test.go index 83a672cec..c551049f6 100644 --- a/src/adminserver/systemcfg/driver_json_test.go +++ b/src/adminserver/systemcfg/store/json/driver_json_test.go @@ -13,11 +13,13 @@ limitations under the License. */ -package systemcfg +package json import ( "os" "testing" + + "github.com/vmware/harbor/src/common/models" ) func TestReadWrite(t *testing.T) { @@ -32,12 +34,12 @@ func TestReadWrite(t *testing.T) { } }() - config := &cfg.SystemCfg{ - Authentication: &cfg.Authentication{ - LDAP: &cfg.LDAP{}, + config := &models.SystemCfg{ + Authentication: &models.Authentication{ + LDAP: &models.LDAP{}, }, - Database: &cfg.Database{ - MySQL: &cfg.MySQL{}, + Database: &models.Database{ + MySQL: &models.MySQL{}, }, } if err := store.Write(config); err != nil { diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index 4bb50cf4b..57f0323e8 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -20,18 +20,21 @@ import ( "os" "strconv" + "github.com/vmware/harbor/src/adminserver/systemcfg/store" + "github.com/vmware/harbor/src/adminserver/systemcfg/store/json" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" ) -var store Driver +var cfgStore store.Driver // Init system configurations. Read from config store first, if null read from env func Init() (err error) { s := getCfgStore() switch s { case "json": - store, err = NewCfgStore() + path := os.Getenv("JSON_STORE_PATH") + cfgStore, err = json.NewCfgStore(path) if err != nil { return } @@ -39,8 +42,8 @@ func Init() (err error) { return fmt.Errorf("unsupported configuration store driver %s", s) } - log.Infof("configuration store driver: %s", store.Name()) - cfg, err := store.Read() + log.Infof("configuration store driver: %s", cfgStore.Name()) + cfg, err := cfgStore.Read() if err != nil { return err } @@ -52,27 +55,13 @@ func Init() (err error) { return err } } else { - //read the following attrs from env every time boots up, - //and sync them into cfg store - cfg.DomainName = os.Getenv("EXT_ENDPOINT") - cfg.Database.MySQL.Password = os.Getenv("MYSQL_PWD") - cfg.JobLogDir = os.Getenv("LOG_DIR") - cfg.CompressJS = os.Getenv("USE_COMPRESSED_JS") == "on" - exp, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION")) - if err != nil { + if err := readFromEnv(cfg); err != nil { return err } - cfg.TokenExpiration = exp - cfg.SecretKey = os.Getenv("SECRET_KEY") - - cfgExp, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) - if err != nil { - return err - } - cfg.CfgExpiration = cfgExp } - if err = store.Write(cfg); err != nil { + //sync configurations into cfg store + if err = cfgStore.Write(cfg); err != nil { return err } @@ -80,15 +69,78 @@ func Init() (err error) { } func getCfgStore() string { - return "json" + t := os.Getenv("CFG_STORE_TYPE") + if len(t) == 0 { + t = "json" + } + return t +} + +//read the following attrs from env every time boots up +func readFromEnv(cfg *models.SystemCfg) error { + cfg.DomainName = os.Getenv("EXT_ENDPOINT") + + cfg.Database = &models.Database{ + Type: os.Getenv("DATABASE_TYPE"), + MySQL: &models.MySQL{ + Host: os.Getenv("MYSQL_HOST"), + Username: os.Getenv("MYSQL_USR"), + Password: os.Getenv("MYSQL_PWD"), + Database: os.Getenv("MYSQL_DATABASE"), + }, + SQLite: &models.SQLite{ + File: os.Getenv("SQLITE_FILE"), + }, + } + port, err := strconv.Atoi(os.Getenv("MYSQL_PORT")) + if err != nil { + return err + } + cfg.Database.MySQL.Port = port + + cfg.TokenService = &models.TokenService{ + URL: os.Getenv("TOKEN_SERVICE_URL"), + } + cfg.Registry = &models.Registry{ + URL: os.Getenv("REGISTRY_URL"), + } + + //TODO remove + cfg.JobLogDir = os.Getenv("LOG_DIR") + //TODO remove + cfg.CompressJS = os.Getenv("USE_COMPRESSED_JS") == "on" + exp, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION")) + if err != nil { + return err + } + cfg.TokenExpiration = exp + cfg.SecretKey = os.Getenv("SECRET_KEY") + + cfgExp, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) + if err != nil { + return err + } + cfg.CfgExpiration = cfgExp + + workers, err := strconv.Atoi(os.Getenv("MAX_JOB_WORKERS")) + if err != nil { + return err + } + cfg.MaxJobWorkers = workers + + return nil } func initFromEnv() (*models.SystemCfg, error) { cfg := &models.SystemCfg{} - cfg.DomainName = os.Getenv("EXT_ENDPOINT") + + if err := readFromEnv(cfg); err != nil { + return nil, err + } + cfg.Authentication = &models.Authentication{ Mode: os.Getenv("AUTH_MODE"), - SelfRegistration: os.Getenv("SELF_REGISTRATION") == "true", + SelfRegistration: os.Getenv("SELF_REGISTRATION") == "on", LDAP: &models.LDAP{ URL: os.Getenv("LDAP_URL"), SearchDN: os.Getenv("LDAP_SEARCH_DN"), @@ -108,74 +160,29 @@ func initFromEnv() (*models.SystemCfg, error) { return nil, err } cfg.Authentication.LDAP.Timeout = timeout - cfg.Database = &models.Database{ - Type: os.Getenv("DATABASE_TYPE"), - MySQL: &models.MySQL{ - Host: os.Getenv("MYSQL_HOST"), - Username: os.Getenv("MYSQL_USR"), - Password: os.Getenv("MYSQL_PWD"), - Database: os.Getenv("MYSQL_DATABASE"), - }, - SQLite: &models.SQLite{ - File: os.Getenv("SQLITE_FILE"), - }, - } - port, err := strconv.Atoi(os.Getenv("MYSQL_PORT")) - if err != nil { - return nil, err - } - cfg.Database.MySQL.Port = port - cfg.TokenService = &models.TokenService{ - URL: os.Getenv("TOKEN_SERVICE_URL"), - } - cfg.Registry = &models.Registry{ - URL: os.Getenv("REGISTRY_URL"), - } cfg.Email = &models.Email{ Host: os.Getenv("EMAIL_HOST"), Port: os.Getenv("EMAIL_PORT"), Username: os.Getenv("EMAIL_USR"), Password: os.Getenv("EMAIL_PWD"), - TLS: os.Getenv("EMAIL_TLS") == "true", + SSL: os.Getenv("EMAIL_SSL") == "true", From: os.Getenv("EMAIL_FROM"), Identity: os.Getenv("EMAIL_IDENTITY"), } - cfg.VerifyRemoteCert = os.Getenv("VERIFY_REMOTE_CERT") == "true" + cfg.VerifyRemoteCert = os.Getenv("VERIFY_REMOTE_CERT") == "on" cfg.ProjectCreationRestriction = os.Getenv("PROJECT_CREATION_RESTRICTION") - workers, err := strconv.Atoi(os.Getenv("MAX_JOB_WORKERS")) - if err != nil { - return nil, err - } - cfg.MaxJobWorkers = workers - cfg.JobLogDir = os.Getenv("LOG_DIR") cfg.InitialAdminPwd = os.Getenv("HARBOR_ADMIN_PASSWORD") - cfg.CompressJS = os.Getenv("USE_COMPRESSED_JS") == "on" - - tokenExp, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION")) - if err != nil { - return nil, err - } - cfg.TokenExpiration = tokenExp - - cfg.SecretKey = os.Getenv("SECRET_KEY") - - cfgExp, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) - if err != nil { - return nil, err - } - cfg.CfgExpiration = cfgExp - return cfg, nil } // GetSystemCfg returns the system configurations func GetSystemCfg() (*models.SystemCfg, error) { - return store.Read() + return cfgStore.Read() } // UpdateSystemCfg updates the system configurations func UpdateSystemCfg(cfg *models.SystemCfg) error { - return store.Write(cfg) + return cfgStore.Write(cfg) } diff --git a/src/adminserver/systemcfg/systemcfg_test.go b/src/adminserver/systemcfg/systemcfg_test.go new file mode 100644 index 000000000..f3f4a7f1b --- /dev/null +++ b/src/adminserver/systemcfg/systemcfg_test.go @@ -0,0 +1,71 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package systemcfg + +/* +import ( + "os" + "testing" +) + + +func TestSystemcfg(t *testing.T) { + key := "JSON_STORE_PATH" + tmpPath := "/tmp/config.json" + originalPath := os.Getenv(key) + defer func() { + if err := os.Remove(tmpPath); err != nil { + t.Errorf("failed to remove %s: %v", tmpPath, err) + } + + if len(originalPath) == 0 { + if err := os.Unsetenv(key); err != nil { + t.Fatalf("failed to unset env %s: %v", key, err) + } + return + } + + if err := os.Setenv(key, originalPath); err != nil { + t.Fatalf("failed to set env %s: %v", key, err) + } + }() + + if err := os.Setenv(key, tmpPath); err != nil { + t.Fatalf("failed to set env %s: %v", key, err) + } + + m := map[string]string{ + "LDAP_SCOPE": "1", + "LDAP_TIMEOUT": "30", + "MYSQL_PORT": "3306", + "MAX_JOB_WORKERS": "3", + "TOKEN_EXPIRATION": "30", + "CFG_EXPIRATION": "5", + } + + for k, v := range m { + if err := os.Setenv(k, v); err != nil { + t.Errorf("failed to set env %s: %v", k, err) + return + } + } + + if err := Init(); err != nil { + t.Errorf("failed to initialize system configurations: %v", err) + return + } +} +*/ diff --git a/src/common/api/base_test.go b/src/common/api/base_test.go index df8486ebf..0352b6e76 100644 --- a/src/common/api/base_test.go +++ b/src/common/api/base_test.go @@ -14,6 +14,7 @@ */ package api +/* import ( "github.com/vmware/harbor/src/common/config" "os" @@ -31,3 +32,4 @@ func TestGetIsInsecure(t *testing.T) { } os.Unsetenv("VERIFY_REMOTE_CERT") } +*/ diff --git a/src/common/config/config.go b/src/common/config/config.go index 756bff8d1..4c9aeaa61 100644 --- a/src/common/config/config.go +++ b/src/common/config/config.go @@ -18,94 +18,185 @@ package config import ( "bytes" + "encoding/json" "fmt" "io/ioutil" "net/http" "strings" + "time" "github.com/astaxie/beego/cache" "github.com/vmware/harbor/src/common/utils" + "github.com/vmware/harbor/src/common/utils/log" ) +// const variables const ( - //auth mode - DB_AUTH = "db_auth" - LDAP_AUTH = "ldap_auth" + DBAuth = "db_auth" + LDAPAuth = "ldap_auth" + ProCrtRestrEveryone = "everyone" + ProCrtRestrAdmOnly = "adminonly" + LDAPScopeBase = "1" + LDAPScopeOnelevel = "2" + LDAPScopeSubtree = "3" - //project_creation_restriction - PRO_CRT_RESTR_EVERYONE = "everyone" - PRO_CRT_RESTR_ADM_ONLY = "adminonly" - - LDAP_SCOPE_BASE = "1" - LDAP_SCOPE_ONELEVEL = "2" - LDAP_SCOPE_SUBTREE = "3" - - AUTH_MODE = "auth_mode" - SELF_REGISTRATION = "self_registration" - LDAP_URL = "ldap_url" - LDAP_SEARCH_DN = "ldap_search_dn" - LDAP_SEARCH_PWD = "ldap_search_pwd" - LDAP_BASE_DN = "ldap_base_dn" - LDAP_UID = "ldap_uid" - LDAP_FILTER = "ldap_filter" - LDAP_SCOPE = "ldap_scope" - EMAIL_SERVER = "email_server" - EMAIL_SERVER_PORT = "email_server_port" - EMAIL_USERNAME = "email_server_username" - EMAIL_PWD = "email_server_pwd" - EMAIL_FROM = "email_from" - EMAIL_SSL = "email_ssl" - EMAIL_IDENTITY = "email_identity" - PROJECT_CREATION_RESTRICTION = "project_creation_restriction" - VERIFY_REMOTE_CERT = "verify_remote_cert" - MAX_JOB_WORKERS = "max_job_workers" - CFG_EXPIRATION = "cfg_expiration" + AUTHMode = "auth_mode" + SelfRegistration = "self_registration" + LDAPURL = "ldap_url" + LDAPSearchDN = "ldap_search_dn" + LDAPSearchPwd = "ldap_search_password" + LDAPBaseDN = "ldap_base_dn" + LDAPUID = "ldap_uid" + LDAPFilter = "ldap_filter" + LDAPScope = "ldap_scope" + LDAPTimeout = "ldap_timeout" + EmailHost = "email_host" + EmailPort = "email_port" + EmailUsername = "email_username" + EmailPassword = "email_password" + EmailFrom = "email_from" + EmailSSL = "email_ssl" + EmailIdentity = "email_identity" + ProjectCreationRestriction = "project_creation_restriction" + VerifyRemoteCert = "verify_remote_cert" + MaxJobWorkers = "max_job_workers" + CfgExpiration = "cfg_expiration" ) +// Manager manages configurations type Manager struct { - Key string - Cache cache.Cache Loader *Loader + Parser Parser + Cache bool + cache cache.Cache + key string } -func NewManager(key, url string) *Manager { - return &Manager{ - Key: key, - Cache: cache.NewMemoryCache(), - Loader: NewLoader(url), +// Parser parses []byte to a specific configuration +type Parser interface { + // Parse ... + Parse([]byte) (interface{}, error) +} + +// NewManager returns an instance of Manager +// url: the url from which loader loads configurations +func NewManager(url, secret string, parser Parser, enableCache bool) *Manager { + m := &Manager{ + Loader: NewLoader(url, secret), + Parser: parser, } -} -func (m *Manager) GetFromCache() interface{} { - value := m.Cache.Get(m.Key) - if value != nil { - return value + if enableCache { + m.Cache = true + m.cache = cache.NewMemoryCache() + m.key = "cfg" } - return nil + + return m } +// Init loader +func (m *Manager) Init() error { + return m.Loader.Init() +} + +// Load configurations, if cache is enabled, cache the configurations +func (m *Manager) Load() (interface{}, error) { + b, err := m.Loader.Load() + if err != nil { + return nil, err + } + + c, err := m.Parser.Parse(b) + if err != nil { + return nil, err + } + + if m.Cache { + expi, err := parseExpiration(b) + if err != nil { + return nil, err + } + if err = m.cache.Put(m.key, c, + time.Duration(expi)*time.Second); err != nil { + return nil, err + } + } + + return c, nil +} + +func parseExpiration(b []byte) (int, error) { + expi := &struct { + Expi int `json:"cfg_expiration"` + }{} + if err := json.Unmarshal(b, expi); err != nil { + return 0, err + } + return expi.Expi, nil +} + +// Get : if cache is enabled, read configurations from cache, +// if cache is null or cache is disabled it loads configurations directly +func (m *Manager) Get() (interface{}, error) { + if m.Cache { + c := m.cache.Get(m.key) + if c != nil { + return c, nil + } + } + return m.Load() +} + +// Upload configurations +func (m *Manager) Upload(b []byte) error { + return m.Loader.Upload(b) +} + +// Loader loads and uploads configurations type Loader struct { url string + secret string client *http.Client } -func NewLoader(url string) *Loader { +// NewLoader ... +func NewLoader(url, secret string) *Loader { return &Loader{ url: url, + secret: secret, client: &http.Client{}, } } +// Init waits remote server to be ready by testing connections with it func (l *Loader) Init() error { addr := l.url if strings.Contains(addr, "://") { addr = strings.Split(addr, "://")[1] } + + if !strings.Contains(addr, ":") { + addr = addr + ":80" + } + return utils.TestTCPConn(addr, 60, 2) } +// Load configurations from remote server func (l *Loader) Load() ([]byte, error) { - resp, err := l.client.Get(l.url + "/api/configurations") + log.Debug("loading configurations...") + req, err := http.NewRequest("GET", l.url+"/api/configurations", nil) + if err != nil { + return nil, err + } + + req.AddCookie(&http.Cookie{ + Name: "secret", + Value: l.secret, + }) + + resp, err := l.client.Do(req) if err != nil { return nil, err } @@ -114,14 +205,22 @@ func (l *Loader) Load() ([]byte, error) { if err != nil { return nil, err } + log.Debug("configurations load completed") return b, nil } +// Upload configuratons to remote server func (l *Loader) Upload(b []byte) error { req, err := http.NewRequest("PUT", l.url+"/api/configurations", bytes.NewReader(b)) if err != nil { return err } + + req.AddCookie(&http.Cookie{ + Name: "secret", + Value: l.secret, + }) + resp, err := l.client.Do(req) if err != nil { return err diff --git a/src/common/config/config_test.go b/src/common/config/config_test.go index 95c954975..81448edf3 100644 --- a/src/common/config/config_test.go +++ b/src/common/config/config_test.go @@ -13,99 +13,3 @@ limitations under the License. */ package config - -import ( - "os" - "testing" -) - -func TestEnvConfLoader(t *testing.T) { - os.Unsetenv("KEY2") - os.Setenv("KEY1", "V1") - os.Setenv("KEY3", "V3") - keys := []string{"KEY1", "KEY2"} - ecl := EnvConfigLoader{ - keys, - } - m, err := ecl.Load() - if err != nil { - t.Errorf("Error loading the configuration via env: %v", err) - } - if m["KEY1"] != "V1" { - t.Errorf("The value for key KEY1 should be V1, but infact: %s", m["KEY1"]) - } - if len(m["KEY2"]) > 0 { - t.Errorf("The value for key KEY2 should be emptye, but infact: %s", m["KEY2"]) - } - if _, ok := m["KEY3"]; ok { - t.Errorf("The KEY3 should not be in result as it's not in the initial key list") - } - os.Unsetenv("KEY1") - os.Unsetenv("KEY3") -} - -func TestCommonConfig(t *testing.T) { - - mysql := MySQLSetting{"registry", "root", "password", "127.0.0.1", "3306"} - sqlite := SQLiteSetting{"file.db"} - verify := "off" - ext := "http://harbor" - token := "http://token" - loglevel := "info" - - os.Setenv("DATABASE", "") - os.Setenv("MYSQL_DATABASE", mysql.Database) - os.Setenv("MYSQL_USR", mysql.User) - os.Setenv("MYSQL_PWD", mysql.Password) - os.Setenv("MYSQL_HOST", mysql.Host) - os.Setenv("MYSQL_PORT", mysql.Port) - os.Setenv("SQLITE_FILE", sqlite.FilePath) - os.Setenv("VERIFY_REMOTE_CERT", verify) - os.Setenv("EXT_ENDPOINT", ext) - os.Setenv("TOKEN_ENDPOINT", token) - os.Setenv("LOG_LEVEL", loglevel) - - err := Reload() - if err != nil { - t.Errorf("Unexpected error when loading the configurations, error: %v", err) - } - if Database() != "mysql" { - t.Errorf("Expected Database value: mysql, fact: %s", mysql) - } - if MySQL() != mysql { - t.Errorf("Expected MySQL setting: %+v, fact: %+v", mysql, MySQL()) - } - if VerifyRemoteCert() { - t.Errorf("Expected VerifyRemoteCert: false, env var: %s, fact: %v", verify, VerifyRemoteCert()) - } - if ExtEndpoint() != ext { - t.Errorf("Expected ExtEndpoint: %s, fact: %s", ext, ExtEndpoint()) - } - if TokenEndpoint() != token { - t.Errorf("Expected TokenEndpoint: %s, fact: %s", token, TokenEndpoint()) - } - if LogLevel() != loglevel { - t.Errorf("Expected LogLevel: %s, fact: %s", loglevel, LogLevel()) - } - os.Setenv("DATABASE", "sqlite") - err = Reload() - if err != nil { - t.Errorf("Unexpected error when loading the configurations, error: %v", err) - } - if SQLite() != sqlite { - t.Errorf("Expected SQLite setting: %+v, fact %+v", sqlite, SQLite()) - } - - os.Unsetenv("DATABASE") - os.Unsetenv("MYSQL_DATABASE") - os.Unsetenv("MYSQL_USR") - os.Unsetenv("MYSQL_PWD") - os.Unsetenv("MYSQL_HOST") - os.Unsetenv("MYSQL_PORT") - os.Unsetenv("SQLITE_FILE") - os.Unsetenv("VERIFY_REMOTE_CERT") - os.Unsetenv("EXT_ENDPOINT") - os.Unsetenv("TOKEN_ENDPOINT") - os.Unsetenv("LOG_LEVEL") - -} diff --git a/src/common/dao/dao_test.go b/src/common/dao/dao_test.go index 26056116f..2a1144d9b 100644 --- a/src/common/dao/dao_test.go +++ b/src/common/dao/dao_test.go @@ -137,7 +137,7 @@ const publicityOn = 1 const publicityOff = 0 func TestMain(m *testing.M) { - databases := []string{"mysql", "sqlite"} + databases := []string{"mysql"} for _, database := range databases { log.Infof("run test cases for database: %s", database) diff --git a/src/common/models/config.go b/src/common/models/config.go index 9697a32ba..953f08e6c 100644 --- a/src/common/models/config.go +++ b/src/common/models/config.go @@ -46,7 +46,7 @@ type MySQL struct { Host string `json:"host"` Port int `json:"port"` Username string `json:"username"` - Password string `json:"password"` + Password string `json:"password,omitempty"` Database string `json:"database"` } @@ -61,7 +61,7 @@ type Email struct { Port string `json:"port"` Username string `json:"username"` Password string `json:"password"` - TLS bool `json:"tls"` + SSL bool `json:"ssl"` Identity string `json:"identity"` From string `json:"from"` } @@ -88,9 +88,9 @@ type SystemCfg struct { ProjectCreationRestriction string `json:"project_creation_restriction"` MaxJobWorkers int `json:"max_job_workers"` JobLogDir string `json:"job_log_dir"` - InitialAdminPwd string `json:"initial_admin_pwd"` + InitialAdminPwd string `json:"initial_admin_pwd,omitempty"` CompressJS bool `json:"compress_js"` //TODO remove TokenExpiration int `json:"token_expiration"` // in minute - SecretKey string `json:"secret_key"` + SecretKey string `json:"secret_key,omitempty"` CfgExpiration int `json:"cfg_expiration"` } diff --git a/src/common/utils/email/mail.go b/src/common/utils/email/mail.go index d82f1d9bf..61fc05531 100644 --- a/src/common/utils/email/mail.go +++ b/src/common/utils/email/mail.go @@ -57,7 +57,7 @@ func (m Mail) SendMail() error { content := mailContent.Bytes() auth := smtp.PlainAuth(mc.Identity, mc.Username, mc.Password, mc.Host) - if mc.TLS { + if mc.SSL { err = sendMailWithTLS(m, auth, content) } else { err = sendMail(m, auth, content) diff --git a/src/common/utils/test/adminserver.go b/src/common/utils/test/adminserver.go new file mode 100644 index 000000000..538b861bd --- /dev/null +++ b/src/common/utils/test/adminserver.go @@ -0,0 +1,59 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + + "github.com/vmware/harbor/src/common/models" +) + +// NewAdminserver returns a mock admin server +func NewAdminserver() (*httptest.Server, error) { + m := []*RequestHandlerMapping{} + b, err := json.Marshal(&models.SystemCfg{ + Authentication: &models.Authentication{ + Mode: "db_auth", + }, + Registry: &models.Registry{}, + }) + if err != nil { + return nil, err + } + + resp := &Response{ + StatusCode: http.StatusOK, + Body: b, + } + + m = append(m, &RequestHandlerMapping{ + Method: "GET", + Pattern: "/api/configurations", + Handler: Handler(resp), + }) + + m = append(m, &RequestHandlerMapping{ + Method: "PUT", + Pattern: "/api/configurations", + Handler: Handler(&Response{ + StatusCode: http.StatusOK, + }), + }) + + return NewServer(m...), nil +} diff --git a/src/common/utils/test/test.go b/src/common/utils/test/test.go index 37a435c24..78ea01bcd 100644 --- a/src/common/utils/test/test.go +++ b/src/common/utils/test/test.go @@ -21,6 +21,8 @@ import ( "net/http" "net/http/httptest" "strings" + + "github.com/gorilla/mux" ) // RequestHandlerMapping is a mapping between request and its handler @@ -78,11 +80,11 @@ func Handler(resp *Response) func(http.ResponseWriter, *http.Request) { // NewServer creates a HTTP server for unit test func NewServer(mappings ...*RequestHandlerMapping) *httptest.Server { - mux := http.NewServeMux() + r := mux.NewRouter() for _, mapping := range mappings { - mux.Handle(mapping.Pattern, mapping) + r.PathPrefix(mapping.Pattern).Handler(mapping).Methods(mapping.Method) } - return httptest.NewServer(mux) + return httptest.NewServer(r) } diff --git a/src/common/utils/utils.go b/src/common/utils/utils.go index a3ee7a5c7..a113ac033 100644 --- a/src/common/utils/utils.go +++ b/src/common/utils/utils.go @@ -75,7 +75,10 @@ func GenerateRandomString() string { return string(result) } -// timeout in second +// TestTCPConn tests TCP connection +// timeout: the total time before returning if something is wrong +// with the connection, in second +// interval: the interval time for retring after failure, in second func TestTCPConn(addr string, timeout, interval int) error { success := make(chan int) cancel := make(chan int) diff --git a/src/common/utils/utils_test.go b/src/common/utils/utils_test.go index cdef47de1..7421ad9c8 100644 --- a/src/common/utils/utils_test.go +++ b/src/common/utils/utils_test.go @@ -17,6 +17,7 @@ package utils import ( "encoding/base64" + "net/http/httptest" "strings" "testing" ) @@ -178,3 +179,12 @@ func TestParseLink(t *testing.T) { t.Errorf("unexpected prev: %s != %s", links.Next(), next) } } + +func TestTestTCPConn(t *testing.T) { + server := httptest.NewServer(nil) + defer server.Close() + addr := strings.TrimLeft(server.URL, "http://") + if err := TestTCPConn(addr, 60, 2); err != nil { + t.Fatalf("failed to test tcp connection of %s: %v", addr, err) + } +} diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index 18999e855..51bf1823f 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -18,16 +18,14 @@ package config import ( "encoding/json" "os" - "time" comcfg "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/models" - //"github.com/vmware/harbor/src/common/utils/log" ) var mg *comcfg.Manager -// Configuration holds configurations of Jobservice +// Configuration of Jobservice type Configuration struct { Database *models.Database `json:"database"` Registry *models.Registry `json:"registry"` @@ -38,65 +36,42 @@ type Configuration struct { CfgExpiration int `json:"cfg_expiration"` } +type parser struct { +} + +func (p *parser) Parse(b []byte) (interface{}, error) { + c := &Configuration{} + if err := json.Unmarshal(b, c); err != nil { + return nil, err + } + return c, nil +} + +// Init configurations func Init() error { adminServerURL := os.Getenv("ADMIN_SERVER_URL") if len(adminServerURL) == 0 { - adminServerURL = "http://admin_server" + adminServerURL = "http://adminserver" } - mg = comcfg.NewManager("cfg", adminServerURL) + mg = comcfg.NewManager(adminServerURL, UISecret(), &parser{}, true) - if err := mg.Loader.Init(); err != nil { + if err := mg.Init(); err != nil { return err } - if err := load(); err != nil { - return err - } - - path, err := LogDir() - if err != nil { - return err - } - if err := os.MkdirAll(path, 0600); err != nil { + if _, err := mg.Load(); err != nil { return err } return nil } -// get returns configurations of jobservice from cache, -// if cache is null, it loads first func get() (*Configuration, error) { - cfg := mg.GetFromCache() - if cfg != nil { - return cfg.(*Configuration), nil - } - - if err := load(); err != nil { + c, err := mg.Get() + if err != nil { return nil, err } - - return mg.GetFromCache().(*Configuration), nil -} - -// load loads configurations of jobservice and puts them into cache -func load() error { - raw, err := mg.Loader.Load() - if err != nil { - return err - } - - cfg := &Configuration{} - if err = json.Unmarshal(raw, cfg); err != nil { - return err - } - - if err = mg.Cache.Put(mg.Key, cfg, - time.Duration(cfg.CfgExpiration)*time.Second); err != nil { - return err - } - - return nil + return c.(*Configuration), nil } // VerifyRemoteCert returns bool value. @@ -149,11 +124,6 @@ func LogDir() (string, error) { return cfg.JobLogDir, nil } -// UISecret will return the value of secret cookie for jobsevice to call UI API. -func UISecret() string { - return os.Getenv("UI_SECRET") -} - // SecretKey will return the secret key for encryption/decryption password in target. func SecretKey() (string, error) { cfg, err := get() @@ -162,3 +132,9 @@ func SecretKey() (string, error) { } return cfg.SecretKey, nil } + +// UISecret returns the value of UI secret cookie, used for communication between UI and JobService +// TODO +func UISecret() string { + return os.Getenv("UI_SECRET") +} diff --git a/src/jobservice/config/config_test.go b/src/jobservice/config/config_test.go index f33c38a2e..0abcb1655 100644 --- a/src/jobservice/config/config_test.go +++ b/src/jobservice/config/config_test.go @@ -1,9 +1,71 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package config import ( + "os" "testing" + + "github.com/vmware/harbor/src/common/utils/test" ) -func TestMain(t *testing.T) { -} +// test functions under package jobservice/config +func TestConfig(t *testing.T) { + server, err := test.NewAdminserver() + if err != nil { + t.Fatalf("failed to create a mock admin server: %v", err) + } + defer server.Close() + url := os.Getenv("ADMIN_SERVER_URL") + defer os.Setenv("ADMIN_SERVER_URL", url) + + if err := os.Setenv("ADMIN_SERVER_URL", server.URL); err != nil { + t.Fatalf("failed to set env %s: %v", "ADMIN_SERVER_URL", err) + } + + if err := Init(); err != nil { + t.Fatalf("failed to initialize configurations: %v", err) + } + + if _, err := VerifyRemoteCert(); err != nil { + t.Fatalf("failed to get verify remote cert: %v", err) + } + + if _, err := Database(); err != nil { + t.Fatalf("failed to get database settings: %v", err) + } + + if _, err := MaxJobWorkers(); err != nil { + t.Fatalf("failed to get max job workers: %v", err) + } + + LocalUIURL() + + if _, err := LocalRegURL(); err != nil { + t.Fatalf("failed to get registry URL: %v", err) + } + + if _, err := LogDir(); err != nil { + t.Fatalf("failed to get log directory: %v", err) + } + + if _, err := SecretKey(); err != nil { + t.Fatalf("failed to get secret key: %v", err) + } + + UISecret() +} diff --git a/src/jobservice/main.go b/src/jobservice/main.go index 33516761a..00a91921c 100644 --- a/src/jobservice/main.go +++ b/src/jobservice/main.go @@ -16,6 +16,8 @@ package main import ( + "os" + "github.com/astaxie/beego" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" @@ -64,7 +66,6 @@ func resumeJobs() { } } -/* func init() { configPath := os.Getenv("CONFIG_PATH") if len(configPath) != 0 { @@ -72,4 +73,3 @@ func init() { beego.LoadAppConfig("ini", configPath) } } -*/ diff --git a/src/ui/api/config.go b/src/ui/api/config.go index 70e41459f..1c17ad8e0 100644 --- a/src/ui/api/config.go +++ b/src/ui/api/config.go @@ -30,6 +30,7 @@ import ( "github.com/vmware/harbor/src/ui/config" ) +// ConfigAPI ... type ConfigAPI struct { api.BaseAPI } @@ -56,9 +57,25 @@ func (c *ConfigAPI) Get() { c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } - //TODO filter attr in sys config + if cfg.Database.MySQL != nil { + cfg.Database.MySQL.Password = "" + } - c.Data["json"] = cfg + cfg.InitialAdminPwd = "" + cfg.SecretKey = "" + + m := map[string]interface{}{} + m["config"] = cfg + + editable, err := dao.AuthModeCanBeModified() + if err != nil { + log.Errorf("failed to determinie whether auth mode can be modified: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + m["auth_mode_editable"] = editable + + c.Data["json"] = m c.ServeJSON() } @@ -70,7 +87,7 @@ func (c *ConfigAPI) Put() { c.CustomAbort(http.StatusBadRequest, err.Error()) } - if value, ok := m[comcfg.AUTH_MODE]; ok { + if value, ok := m[comcfg.AUTHMode]; ok { mode, err := config.AuthMode() if err != nil { log.Errorf("failed to get auth mode: %v", err) @@ -87,13 +104,11 @@ func (c *ConfigAPI) Put() { if !flag { c.CustomAbort(http.StatusBadRequest, fmt.Sprintf("%s can not be modified as new users have been inserted into database", - comcfg.AUTH_MODE)) + comcfg.AUTHMode)) } } } - log.Info(m) - if err := config.Upload(m); err != nil { log.Errorf("failed to upload configurations: %v", err) c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) @@ -105,82 +120,81 @@ func (c *ConfigAPI) Put() { } } -// TODO ldap timeout, scope value func validateCfg(c map[string]string) error { - if value, ok := c[comcfg.AUTH_MODE]; ok { - if value != comcfg.DB_AUTH && value != comcfg.LDAP_AUTH { - return fmt.Errorf("invalid %s, shoud be %s or %s", comcfg.AUTH_MODE, comcfg.DB_AUTH, comcfg.LDAP_AUTH) + if value, ok := c[comcfg.AUTHMode]; ok { + if value != comcfg.DBAuth && value != comcfg.LDAPAuth { + return fmt.Errorf("invalid %s, shoud be %s or %s", comcfg.AUTHMode, comcfg.DBAuth, comcfg.LDAPAuth) } - if value == comcfg.LDAP_AUTH { - if _, ok := c[comcfg.LDAP_URL]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAP_URL) + if value == comcfg.LDAPAuth { + if _, ok := c[comcfg.LDAPURL]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAPURL) } - if _, ok := c[comcfg.LDAP_BASE_DN]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAP_BASE_DN) + if _, ok := c[comcfg.LDAPBaseDN]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAPBaseDN) } - if _, ok := c[comcfg.LDAP_UID]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAP_UID) + if _, ok := c[comcfg.LDAPUID]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAPUID) } - if _, ok := c[comcfg.LDAP_SCOPE]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAP_SCOPE) + if _, ok := c[comcfg.LDAPScope]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAPScope) } } } - if ldapURL, ok := c[comcfg.LDAP_URL]; ok && len(ldapURL) == 0 { - return fmt.Errorf("%s is empty", comcfg.LDAP_URL) + if ldapURL, ok := c[comcfg.LDAPURL]; ok && len(ldapURL) == 0 { + return fmt.Errorf("%s is empty", comcfg.LDAPURL) } - if baseDN, ok := c[comcfg.LDAP_BASE_DN]; ok && len(baseDN) == 0 { - return fmt.Errorf("%s is empty", comcfg.LDAP_BASE_DN) + if baseDN, ok := c[comcfg.LDAPBaseDN]; ok && len(baseDN) == 0 { + return fmt.Errorf("%s is empty", comcfg.LDAPBaseDN) } - if uID, ok := c[comcfg.LDAP_UID]; ok && len(uID) == 0 { - return fmt.Errorf("%s is empty", comcfg.LDAP_UID) + if uID, ok := c[comcfg.LDAPUID]; ok && len(uID) == 0 { + return fmt.Errorf("%s is empty", comcfg.LDAPUID) } - if scope, ok := c[comcfg.LDAP_SCOPE]; ok && - scope != comcfg.LDAP_SCOPE_BASE && - scope != comcfg.LDAP_SCOPE_ONELEVEL && - scope != comcfg.LDAP_SCOPE_SUBTREE { + if scope, ok := c[comcfg.LDAPScope]; ok && + scope != comcfg.LDAPScopeBase && + scope != comcfg.LDAPScopeOnelevel && + scope != comcfg.LDAPScopeSubtree { return fmt.Errorf("invalid %s, should be %s, %s or %s", - comcfg.LDAP_SCOPE, - comcfg.LDAP_SCOPE_BASE, - comcfg.LDAP_SCOPE_ONELEVEL, - comcfg.LDAP_SCOPE_SUBTREE) + comcfg.LDAPScope, + comcfg.LDAPScopeBase, + comcfg.LDAPScopeOnelevel, + comcfg.LDAPScopeSubtree) + } + if timeout, ok := c[comcfg.LDAPTimeout]; ok { + if t, err := strconv.Atoi(timeout); err != nil || t < 0 { + return fmt.Errorf("invalid %s", comcfg.LDAPTimeout) + } } - if self, ok := c[comcfg.SELF_REGISTRATION]; ok && - self != "true" && self != "false" { + if self, ok := c[comcfg.SelfRegistration]; ok && + self != "0" && self != "1" { return fmt.Errorf("%s should be %s or %s", - comcfg.SELF_REGISTRATION, "true", "false") + comcfg.SelfRegistration, "0", "1") } - if port, ok := c[comcfg.EMAIL_SERVER_PORT]; ok { + if port, ok := c[comcfg.EmailPort]; ok { if p, err := strconv.Atoi(port); err != nil || p < 0 || p > 65535 { - return fmt.Errorf("invalid %s", comcfg.EMAIL_SERVER_PORT) + return fmt.Errorf("invalid %s", comcfg.EmailPort) } } - if ssl, ok := c[comcfg.EMAIL_SSL]; ok && ssl != "true" && ssl != "false" { - return fmt.Errorf("%s should be true or false", comcfg.EMAIL_SSL) + if ssl, ok := c[comcfg.EmailSSL]; ok && ssl != "0" && ssl != "1" { + return fmt.Errorf("%s should be %s or %s", comcfg.EmailSSL, "0", "1") } - if crt, ok := c[comcfg.PROJECT_CREATION_RESTRICTION]; ok && - crt != comcfg.PRO_CRT_RESTR_EVERYONE && - crt != comcfg.PRO_CRT_RESTR_ADM_ONLY { + if crt, ok := c[comcfg.ProjectCreationRestriction]; ok && + crt != comcfg.ProCrtRestrEveryone && + crt != comcfg.ProCrtRestrAdmOnly { return fmt.Errorf("invalid %s, should be %s or %s", - comcfg.PROJECT_CREATION_RESTRICTION, - comcfg.PRO_CRT_RESTR_ADM_ONLY, - comcfg.PRO_CRT_RESTR_EVERYONE) + comcfg.ProjectCreationRestriction, + comcfg.ProCrtRestrAdmOnly, + comcfg.ProCrtRestrEveryone) } - if verify, ok := c[comcfg.VERIFY_REMOTE_CERT]; ok && verify != "true" && verify != "false" { - return fmt.Errorf("invalid %s, should be true or false", comcfg.VERIFY_REMOTE_CERT) - } - - if worker, ok := c[comcfg.MAX_JOB_WORKERS]; ok { - if w, err := strconv.Atoi(worker); err != nil || w <= 0 { - return fmt.Errorf("invalid %s", comcfg.MAX_JOB_WORKERS) - } + if verify, ok := c[comcfg.VerifyRemoteCert]; ok && verify != "0" && verify != "1" { + return fmt.Errorf("invalid %s, should be %s or %s", + comcfg.VerifyRemoteCert, "0", "1") } return nil diff --git a/src/ui/api/harborapi_test.go b/src/ui/api/harborapi_test.go index 4fd57a1c3..7d85420a6 100644 --- a/src/ui/api/harborapi_test.go +++ b/src/ui/api/harborapi_test.go @@ -14,6 +14,7 @@ import ( "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils" + "github.com/vmware/harbor/src/ui/config" "github.com/vmware/harbor/tests/apitests/apilib" // "strconv" // "strings" @@ -57,7 +58,14 @@ type usrInfo struct { } func init() { - dao.InitDatabase() + if err := config.Init(); err != nil { + log.Fatalf("failed to initialize configurations: %v", err) + } + database, err := config.Database() + if err != nil { + log.Fatalf("failed to get database configurations: %v", err) + } + dao.InitDatabase(database) _, file, _, _ := runtime.Caller(1) apppath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, ".."+string(filepath.Separator)))) beego.BConfig.WebConfig.Session.SessionOn = true @@ -512,7 +520,7 @@ func (a testapi) GetReposTop(authInfo usrInfo, count string) (int, error) { //-------------------------Targets Test---------------------------------------// //Create a new replication target -func (a testapi) AddTargets(authInfo usrInfo, repTarget apilib.RepTargetPost) (int, error) { +func (a testapi) AddTargets(authInfo usrInfo, repTarget apilib.RepTargetPost) (int, string, error) { _sling := sling.New().Post(a.basePath) path := "/api/targets" @@ -520,8 +528,8 @@ func (a testapi) AddTargets(authInfo usrInfo, repTarget apilib.RepTargetPost) (i _sling = _sling.Path(path) _sling = _sling.BodyJSON(repTarget) - httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo) - return httpStatusCode, err + httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo) + return httpStatusCode, string(body), err } //List filters targets by name diff --git a/src/ui/api/target_test.go b/src/ui/api/target_test.go index 94b6b25e0..62709a39e 100644 --- a/src/ui/api/target_test.go +++ b/src/ui/api/target_test.go @@ -30,17 +30,18 @@ func TestTargetsPost(t *testing.T) { //-------------------case 1 : response code = 201------------------------// fmt.Println("case 1 : response code = 201") - httpStatusCode, err = apiTest.AddTargets(*admin, *repTargets) + httpStatusCode, body, err := apiTest.AddTargets(*admin, *repTargets) if err != nil { t.Error("Error whihle add targets", err.Error()) t.Log(err) } else { assert.Equal(int(201), httpStatusCode, "httpStatusCode should be 201") + t.Log(body) } //-----------case 2 : response code = 409,name is already used-----------// fmt.Println("case 2 : response code = 409,name is already used") - httpStatusCode, err = apiTest.AddTargets(*admin, *repTargets) + httpStatusCode, _, err = apiTest.AddTargets(*admin, *repTargets) if err != nil { t.Error("Error whihle add targets", err.Error()) t.Log(err) @@ -51,7 +52,7 @@ func TestTargetsPost(t *testing.T) { //-----------case 3 : response code = 409,name is already used-----------// fmt.Println("case 3 : response code = 409,endPoint is already used") repTargets.Username = "errName" - httpStatusCode, err = apiTest.AddTargets(*admin, *repTargets) + httpStatusCode, _, err = apiTest.AddTargets(*admin, *repTargets) if err != nil { t.Error("Error whihle add targets", err.Error()) t.Log(err) @@ -61,7 +62,7 @@ func TestTargetsPost(t *testing.T) { //--------case 4 : response code = 401,User need to log in first.--------// fmt.Println("case 4 : response code = 401,User need to log in first.") - httpStatusCode, err = apiTest.AddTargets(*unknownUsr, *repTargets) + httpStatusCode, _, err = apiTest.AddTargets(*unknownUsr, *repTargets) if err != nil { t.Error("Error whihle add targets", err.Error()) t.Log(err) diff --git a/src/ui/config/config.go b/src/ui/config/config.go index aa726e810..ccb760f80 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -18,14 +18,15 @@ package config import ( "encoding/json" "os" - "time" comcfg "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" ) var mg *comcfg.Manager +// Configuration of UI type Configuration struct { DomainName string `json:"domain_name"` // Harbor external URL: protocal://host:port Authentication *models.Authentication `json:"authentication"` @@ -40,59 +41,52 @@ type Configuration struct { CompressJS bool `json:"compress_js"` TokenExpiration int `json:"token_expiration"` SecretKey string `json:"secret_key"` - CfgExpiration int `json:"cfg_expiration` + CfgExpiration int `json:"cfg_expiration"` } +type parser struct { +} + +func (p *parser) Parse(b []byte) (interface{}, error) { + c := &Configuration{} + if err := json.Unmarshal(b, c); err != nil { + return nil, err + } + return c, nil +} + +// Init configurations func Init() error { adminServerURL := os.Getenv("ADMIN_SERVER_URL") if len(adminServerURL) == 0 { - adminServerURL = "http://admin_server" + adminServerURL = "http://adminserver" } - mg = comcfg.NewManager("cfg", adminServerURL) + log.Debugf("admin server URL: %s", adminServerURL) + mg = comcfg.NewManager(adminServerURL, UISecret(), &parser{}, true) - if err := mg.Loader.Init(); err != nil { + if err := mg.Init(); err != nil { return err } - if err := Load(); err != nil { + if _, err := mg.Load(); err != nil { return err } return nil } -// Get returns configurations of UI, if cache is null, it loads first func get() (*Configuration, error) { - cfg := mg.GetFromCache() - if cfg != nil { - return cfg.(*Configuration), nil - } - - if err := Load(); err != nil { + c, err := mg.Get() + if err != nil { return nil, err } - - return mg.GetFromCache().(*Configuration), nil + return c.(*Configuration), nil } -// Load loads configurations of UI and puts them into cache +// Load configurations func Load() error { - raw, err := mg.Loader.Load() - if err != nil { - return err - } - - cfg := &Configuration{} - if err = json.Unmarshal(raw, cfg); err != nil { - return err - } - - if err = mg.Cache.Put(mg.Key, cfg, - time.Duration(cfg.CfgExpiration)*time.Second); err != nil { - return err - } - - return nil + _, err := mg.Load() + return err } // Upload uploads all system configutations to admin server @@ -101,7 +95,7 @@ func Upload(cfg map[string]string) error { if err != nil { return err } - return mg.Loader.Upload(b) + return mg.Upload(b) } // GetSystemCfg returns the system configurations @@ -195,14 +189,13 @@ func InitialAdminPassword() (string, error) { return cfg.InitialAdminPwd, nil } -// TODO // OnlyAdminCreateProject returns the flag to restrict that only sys admin can create project func OnlyAdminCreateProject() (bool, error) { cfg, err := get() if err != nil { return true, err } - return cfg.ProjectCreationRestriction == comcfg.PRO_CRT_RESTR_ADM_ONLY, nil + return cfg.ProjectCreationRestriction == comcfg.ProCrtRestrAdmOnly, nil } // VerifyRemoteCert returns bool value. @@ -214,6 +207,7 @@ func VerifyRemoteCert() (bool, error) { return cfg.VerifyRemoteCert, nil } +// Email returns email server settings func Email() (*models.Email, error) { cfg, err := get() if err != nil { @@ -222,6 +216,7 @@ func Email() (*models.Email, error) { return cfg.Email, nil } +// Database returns database settings func Database() (*models.Database, error) { cfg, err := get() if err != nil { @@ -230,8 +225,8 @@ func Database() (*models.Database, error) { return cfg.Database, nil } -// TODO // UISecret returns the value of UI secret cookie, used for communication between UI and JobService +// TODO func UISecret() string { return os.Getenv("UI_SECRET") } diff --git a/src/ui/config/config_test.go b/src/ui/config/config_test.go index 3f18b40b1..e71160d2e 100644 --- a/src/ui/config/config_test.go +++ b/src/ui/config/config_test.go @@ -17,131 +17,92 @@ package config import ( "os" "testing" + + "github.com/vmware/harbor/src/common/utils/test" ) -var ( - auth = "ldap_auth" - ldap = LDAPSetting{ - "ldap://test.ldap.com", - "ou=people", - "dc=whatever,dc=org", - "1234567", - "cn", - "uid", - "2", - "5", - } - tokenExp = "3" - tokenExpRes = 3 - adminPassword = "password" - externalRegURL = "127.0.0.1" - uiSecret = "ffadsdfsdf" - secretKey = "keykey" - selfRegistration = "off" - projectCreationRestriction = "adminonly" - internalRegistryURL = "http://registry:5000" - jobServiceURL = "http://jobservice" -) - -func TestMain(m *testing.M) { - - os.Setenv("AUTH_MODE", auth) - os.Setenv("LDAP_URL", ldap.URL) - os.Setenv("LDAP_BASE_DN", ldap.BaseDn) - os.Setenv("LDAP_SEARCH_DN", ldap.SearchDn) - os.Setenv("LDAP_SEARCH_PWD", ldap.SearchPwd) - os.Setenv("LDAP_UID", ldap.UID) - os.Setenv("LDAP_SCOPE", ldap.Scope) - os.Setenv("LDAP_FILTER", ldap.Filter) - os.Setenv("LDAP_CONNECT_TIMEOUT", ldap.ConnectTimeout) - os.Setenv("TOKEN_EXPIRATION", tokenExp) - os.Setenv("HARBOR_ADMIN_PASSWORD", adminPassword) - os.Setenv("EXT_REG_URL", externalRegURL) - os.Setenv("UI_SECRET", uiSecret) - os.Setenv("SECRET_KEY", secretKey) - os.Setenv("SELF_REGISTRATION", selfRegistration) - os.Setenv("PROJECT_CREATION_RESTRICTION", projectCreationRestriction) - os.Setenv("REGISTRY_URL", internalRegistryURL) - os.Setenv("JOB_SERVICE_URL", jobServiceURL) - - err := Reload() +// test functions under package ui/config +func TestConfig(t *testing.T) { + server, err := test.NewAdminserver() if err != nil { - panic(err) + t.Fatalf("failed to create a mock admin server: %v", err) } - rc := m.Run() + defer server.Close() - os.Unsetenv("AUTH_MODE") - os.Unsetenv("LDAP_URL") - os.Unsetenv("LDAP_BASE_DN") - os.Unsetenv("LDAP_SEARCH_DN") - os.Unsetenv("LDAP_SEARCH_PWD") - os.Unsetenv("LDAP_UID") - os.Unsetenv("LDAP_SCOPE") - os.Unsetenv("LDAP_FILTER") - os.Unsetenv("LDAP_CONNECT_TIMEOUT") - os.Unsetenv("TOKEN_EXPIRATION") - os.Unsetenv("HARBOR_ADMIN_PASSWORD") - os.Unsetenv("EXT_REG_URL") - os.Unsetenv("UI_SECRET") - os.Unsetenv("SECRET_KEY") - os.Unsetenv("SELF_REGISTRATION") - os.Unsetenv("CREATE_PROJECT_RESTRICTION") - os.Unsetenv("REGISTRY_URL") - os.Unsetenv("JOB_SERVICE_URL") + url := os.Getenv("ADMIN_SERVER_URL") + defer os.Setenv("ADMIN_SERVER_URL", url) - os.Exit(rc) -} - -func TestAuth(t *testing.T) { - if AuthMode() != auth { - t.Errorf("Expected auth mode:%s, in fact: %s", auth, AuthMode()) - } - if LDAP() != ldap { - t.Errorf("Expected ldap setting: %+v, in fact: %+v", ldap, LDAP()) - } -} - -func TestTokenExpiration(t *testing.T) { - if TokenExpiration() != tokenExpRes { - t.Errorf("Expected token expiration: %d, in fact: %d", tokenExpRes, TokenExpiration()) - } -} - -func TestURLs(t *testing.T) { - if InternalRegistryURL() != internalRegistryURL { - t.Errorf("Expected internal Registry URL: %s, in fact: %s", internalRegistryURL, InternalRegistryURL()) - } - if InternalJobServiceURL() != jobServiceURL { - t.Errorf("Expected internal jobservice URL: %s, in fact: %s", jobServiceURL, InternalJobServiceURL()) - } - if ExtRegistryURL() != externalRegURL { - t.Errorf("Expected External Registry URL: %s, in fact: %s", externalRegURL, ExtRegistryURL()) - } -} - -func TestSelfRegistration(t *testing.T) { - if SelfRegistration() { - t.Errorf("Expected Self Registration to be false") - } -} - -func TestSecrets(t *testing.T) { - if SecretKey() != secretKey { - t.Errorf("Expected Secrect Key :%s, in fact: %s", secretKey, SecretKey()) - } - if UISecret() != uiSecret { - t.Errorf("Expected UI Secret: %s, in fact: %s", uiSecret, UISecret()) - } -} - -func TestProjectCreationRestrict(t *testing.T) { - if !OnlyAdminCreateProject() { - t.Errorf("Expected OnlyAdminCreateProject to be true") - } -} - -func TestInitAdminPassword(t *testing.T) { - if InitialAdminPassword() != adminPassword { - t.Errorf("Expected adminPassword: %s, in fact: %s", adminPassword, InitialAdminPassword()) - } + if err := os.Setenv("ADMIN_SERVER_URL", server.URL); err != nil { + t.Fatalf("failed to set env %s: %v", "ADMIN_SERVER_URL", err) + } + + if err := Init(); err != nil { + t.Fatalf("failed to initialize configurations: %v", err) + } + + if err := Load(); err != nil { + t.Fatalf("failed to load configurations: %v", err) + } + + if err := Upload(map[string]string{}); err != nil { + t.Fatalf("failed to upload configurations: %v", err) + } + + if _, err := GetSystemCfg(); err != nil { + t.Fatalf("failed to get system configurations: %v", err) + } + + mode, err := AuthMode() + if err != nil { + t.Fatalf("failed to get auth mode: %v", err) + } + if mode != "db_auth" { + t.Errorf("unexpected mode: %s != %s", mode, "db_auth") + } + + if _, err := LDAP(); err != nil { + t.Fatalf("failed to get ldap settings: %v", err) + } + + if _, err := TokenExpiration(); err != nil { + t.Fatalf("failed to get token expiration: %v", err) + } + + if _, err := DomainName(); err != nil { + t.Fatalf("failed to get domain name: %v", err) + } + + if _, err := SecretKey(); err != nil { + t.Fatalf("failed to get secret key: %v", err) + } + + if _, err := SelfRegistration(); err != nil { + t.Fatalf("failed to get self registration: %v", err) + } + + if _, err := RegistryURL(); err != nil { + t.Fatalf("failed to get registry URL: %v", err) + } + + InternalJobServiceURL() + + InitialAdminPassword() + + if _, err := OnlyAdminCreateProject(); err != nil { + t.Fatalf("failed to get onldy admin create project: %v", err) + } + + if _, err := VerifyRemoteCert(); err != nil { + t.Fatalf("failed to get verify remote cert: %v", err) + } + + if _, err := Email(); err != nil { + t.Fatalf("failed to get email settings: %v", err) + } + + if _, err := Database(); err != nil { + t.Fatalf("failed to get database: %v", err) + } + + UISecret() } diff --git a/src/ui/controllers/base.go b/src/ui/controllers/base.go index 691880af2..044784f82 100644 --- a/src/ui/controllers/base.go +++ b/src/ui/controllers/base.go @@ -244,12 +244,13 @@ func (cc *CommonController) UserExists() { } func init() { - //conf/app.conf -> os.Getenv("config_path") configPath := os.Getenv("CONFIG_PATH") if len(configPath) != 0 { log.Infof("Config path: %s", configPath) - beego.LoadAppConfig("ini", configPath) + if err := beego.LoadAppConfig("ini", configPath); err != nil { + log.Errorf("failed to load app config: %v", err) + } } beego.AddFuncMap("i18n", i18n.Tr) @@ -272,5 +273,4 @@ func init() { log.Errorf("Fail to set message file: %s", err.Error()) } } - } diff --git a/src/ui/controllers/controllers_test.go b/src/ui/controllers/controllers_test.go index c80377417..7fd92023c 100644 --- a/src/ui/controllers/controllers_test.go +++ b/src/ui/controllers/controllers_test.go @@ -14,6 +14,8 @@ import ( "github.com/astaxie/beego" //"github.com/dghubble/sling" "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/config" ) //const ( @@ -29,6 +31,9 @@ import ( //var admin *usrInfo func init() { + if err := config.Init(); err != nil { + log.Fatalf("failed to initialize configurations: %v", err) + } _, file, _, _ := runtime.Caller(1) apppath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, ".."+string(filepath.Separator)))) @@ -63,7 +68,6 @@ func init() { //Init user Info //admin = &usrInfo{adminName, adminPwd} - } // TestMain is a sample to run an endpoint test diff --git a/src/ui/ui_test.go b/src/ui/ui_test.go index 59b3724bc..d9bd9c664 100644 --- a/src/ui/ui_test.go +++ b/src/ui/ui_test.go @@ -1,5 +1,6 @@ package main +/* import ( "testing" ) @@ -7,3 +8,4 @@ import ( func TestMain(t *testing.T) { } +*/ diff --git a/tests/docker-compose.test.yml b/tests/docker-compose.test.yml index 5a92685f3..5d9d3906c 100644 --- a/tests/docker-compose.test.yml +++ b/tests/docker-compose.test.yml @@ -21,3 +21,14 @@ services: - ./common/config/db/env ports: - 3306:3306 + adminserver: + build: + context: ../ + dockerfile: make/dev/adminserver/Dockerfile + env_file: + - ./common/config/adminserver/env + restart: always + volumes: + - /data/config/:/etc/harbor/ + ports: + - 8888:80 \ No newline at end of file diff --git a/tests/startuptest.go b/tests/startuptest.go index f611ef58c..e60823f8d 100644 --- a/tests/startuptest.go +++ b/tests/startuptest.go @@ -2,35 +2,38 @@ package main import ( - "fmt" - "io/ioutil" - "net/http" - "os" - "strings" - "time" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + "time" ) func main() { - time.Sleep(60*time.Second) - for _, url := range os.Args[1:] { - resp, err := http.Get(url) - if err != nil { - fmt.Fprintf(os.Stderr, "fetch: %v\n", err) - os.Exit(1) - } - b, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - fmt.Fprintf(os.Stderr, "fetch: reading %s: %v\n", url, err) - os.Exit(1) - } -// fmt.Printf("%s", b) - if strings.Contains(string(b), "Harbor") { - fmt.Printf("sucess!\n") - } else { - os.Exit(1) - } + time.Sleep(60 * time.Second) + for _, url := range os.Args[1:] { + resp, err := http.Get(url) + if err != nil { + fmt.Fprintf(os.Stderr, "fetch: %v\n", err) + os.Exit(1) + } + b, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "fetch: reading %s: %v\n", url, err) + os.Exit(1) + } + // fmt.Printf("%s", b) - } + if strings.Contains(string(b), "Harbor") { + fmt.Printf("sucess!\n") + } else { + fmt.Println("the response does not contain \"Harbor\"!") + + fmt.Println(string(b)) + os.Exit(1) + } + + } } - diff --git a/tests/testprepare.sh b/tests/testprepare.sh index debcad72d..89200e1d0 100755 --- a/tests/testprepare.sh +++ b/tests/testprepare.sh @@ -1,5 +1,5 @@ #!/bin/bash - +set -e cp tests/docker-compose.test.yml make/. mkdir /etc/ui @@ -7,3 +7,7 @@ cp make/common/config/ui/private_key.pem /etc/ui/. mkdir conf cp make/common/config/ui/app.conf conf/. + +sed -i -r "s/MYSQL_HOST=mysql/MYSQL_HOST=127.0.0.1/" make/common/config/adminserver/env +sed -i -r "s|REGISTRY_URL=http://registry:5000|REGISTRY_URL=http://127.0.0.1:5000|" make/common/config/adminserver/env +sed -i -r "s/UI_SECRET=.*/UI_SECRET=$UI_SECRET/" make/common/config/adminserver/env From e21787e68adcf45161a6c257ea714b279485528a Mon Sep 17 00:00:00 2001 From: yhua Date: Fri, 20 Jan 2017 16:15:56 +0800 Subject: [PATCH 06/22] update swagger.yaml --- docs/swagger.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 3b125b55b..9b66b5f47 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1827,13 +1827,13 @@ definitions: ldap_url: type: string description: The url of ldap service. - ldap_searchdn: + ldap_search_dn: type: string description: The search dn of ldap service. - ldap_search_pwd: + ldap_search_password: type: string description: The search password of ldap service. - ldap_basedn: + ldap_base_dn: type: string description: The base dn of ldap service. ldap_filter: @@ -1846,7 +1846,7 @@ definitions: type: integer format: int64 description: The serach scope of ldap service. - ldap_connect_timeout: + ldap_connection_timeout: type: integer format: int64 description: The connect timeout of ldap service(second). From c7a05e0571711370e4f180ef399f3dd2562c2dbe Mon Sep 17 00:00:00 2001 From: yhua Date: Fri, 20 Jan 2017 16:25:03 +0800 Subject: [PATCH 07/22] fix typo --- docs/swagger.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 9b66b5f47..6bbd2ed21 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -6,7 +6,7 @@ info: description: These APIs provide services for manipulating Harbor project. version: "0.3.0" # the domain of the service -host: localshot +host: localhost # array of all schemes that your API supports schemes: - http From bcc6a4bbf2ce996a133062ab6e21a9d12128ce31 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 20 Jan 2017 13:30:49 +0800 Subject: [PATCH 08/22] update --- .../systemcfg/store/json/driver_json_test.go | 11 +- src/adminserver/systemcfg/systemcfg_test.go | 78 ++++++--- src/common/api/base.go | 10 -- src/common/api/base_test.go | 20 --- src/common/dao/config_test.go | 164 ------------------ src/ui/api/repository.go | 6 +- src/ui/api/target.go | 4 +- 7 files changed, 69 insertions(+), 224 deletions(-) diff --git a/src/adminserver/systemcfg/store/json/driver_json_test.go b/src/adminserver/systemcfg/store/json/driver_json_test.go index c551049f6..eef48ee36 100644 --- a/src/adminserver/systemcfg/store/json/driver_json_test.go +++ b/src/adminserver/systemcfg/store/json/driver_json_test.go @@ -34,6 +34,11 @@ func TestReadWrite(t *testing.T) { } }() + if store.Name() != "JSON" { + t.Errorf("unexpected name: %s != %s", store.Name(), "JSON") + return + } + config := &models.SystemCfg{ Authentication: &models.Authentication{ LDAP: &models.LDAP{}, @@ -43,10 +48,12 @@ func TestReadWrite(t *testing.T) { }, } if err := store.Write(config); err != nil { - t.Fatalf("failed to write configurations to json file: %v", err) + t.Errorf("failed to write configurations to json file: %v", err) + return } if _, err = store.Read(); err != nil { - t.Fatalf("failed to read configurations from json file: %v", err) + t.Errorf("failed to read configurations from json file: %v", err) + return } } diff --git a/src/adminserver/systemcfg/systemcfg_test.go b/src/adminserver/systemcfg/systemcfg_test.go index f3f4a7f1b..cc28c2b38 100644 --- a/src/adminserver/systemcfg/systemcfg_test.go +++ b/src/adminserver/systemcfg/systemcfg_test.go @@ -15,39 +15,31 @@ package systemcfg -/* import ( "os" "testing" + + comcfg "github.com/vmware/harbor/src/common/config" ) - +// test functions under adminserver/systemcfg func TestSystemcfg(t *testing.T) { key := "JSON_STORE_PATH" - tmpPath := "/tmp/config.json" - originalPath := os.Getenv(key) - defer func() { - if err := os.Remove(tmpPath); err != nil { - t.Errorf("failed to remove %s: %v", tmpPath, err) + path := "/tmp/config.json" + if _, err := os.Stat(path); err == nil { + if err := os.Remove(path); err != nil { + t.Fatalf("failed to remove %s: %v", path, err) } + } else if !os.IsNotExist(err) { + t.Fatalf("failed to check the existence of %s: %v", path, err) + } - if len(originalPath) == 0 { - if err := os.Unsetenv(key); err != nil { - t.Fatalf("failed to unset env %s: %v", key, err) - } - return - } - - if err := os.Setenv(key, originalPath); err != nil { - t.Fatalf("failed to set env %s: %v", key, err) - } - }() - - if err := os.Setenv(key, tmpPath); err != nil { + if err := os.Setenv(key, path); err != nil { t.Fatalf("failed to set env %s: %v", key, err) } m := map[string]string{ + "AUTH_MODE": comcfg.DBAuth, "LDAP_SCOPE": "1", "LDAP_TIMEOUT": "30", "MYSQL_PORT": "3306", @@ -58,8 +50,7 @@ func TestSystemcfg(t *testing.T) { for k, v := range m { if err := os.Setenv(k, v); err != nil { - t.Errorf("failed to set env %s: %v", k, err) - return + t.Fatalf("failed to set env %s: %v", k, err) } } @@ -67,5 +58,46 @@ func TestSystemcfg(t *testing.T) { t.Errorf("failed to initialize system configurations: %v", err) return } + defer func() { + if err := os.Remove(path); err != nil { + t.Fatalf("failed to remove %s: %v", path, err) + } + }() + + // run Init again to make sure it works well when the configuration file + // already exists + if err := Init(); err != nil { + t.Errorf("failed to initialize system configurations: %v", err) + return + } + + cfg, err := GetSystemCfg() + if err != nil { + t.Errorf("failed to get system configurations: %v", err) + return + } + + if cfg.Authentication.Mode != comcfg.DBAuth { + t.Errorf("unexpected auth mode: %s != %s", + cfg.Authentication.Mode, comcfg.DBAuth) + return + } + + cfg.Authentication.Mode = comcfg.LDAPAuth + if err = UpdateSystemCfg(cfg); err != nil { + t.Errorf("failed to update system configurations: %v", err) + return + } + + cfg, err = GetSystemCfg() + if err != nil { + t.Errorf("failed to get system configurations: %v", err) + return + } + + if cfg.Authentication.Mode != comcfg.LDAPAuth { + t.Errorf("unexpected auth mode: %s != %s", + cfg.Authentication.Mode, comcfg.DBAuth) + return + } } -*/ diff --git a/src/common/api/base.go b/src/common/api/base.go index 227327339..38635f76d 100644 --- a/src/common/api/base.go +++ b/src/common/api/base.go @@ -26,7 +26,6 @@ import ( "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/auth" - "github.com/vmware/harbor/src/ui/config" "github.com/astaxie/beego" ) @@ -210,12 +209,3 @@ func (b *BaseAPI) GetPaginationParams() (page, pageSize int64) { return page, pageSize } - -// GetIsInsecure ... -func GetIsInsecure() (bool, error) { - verify, err := config.VerifyRemoteCert() - if err != nil { - return false, err - } - return !verify, nil -} diff --git a/src/common/api/base_test.go b/src/common/api/base_test.go index 0352b6e76..386e30e52 100644 --- a/src/common/api/base_test.go +++ b/src/common/api/base_test.go @@ -13,23 +13,3 @@ limitations under the License. */ package api - -/* -import ( - "github.com/vmware/harbor/src/common/config" - "os" - "testing" -) - -func TestGetIsInsecure(t *testing.T) { - os.Setenv("VERIFY_REMOTE_CERT", "off") - err := config.Reload() - if err != nil { - t.Errorf("Failed to load config, error: %v", err) - } - if !GetIsInsecure() { - t.Errorf("GetIsInsecure() should be true when VERIFY_REMOTE_CERT is off, in fact: false") - } - os.Unsetenv("VERIFY_REMOTE_CERT") -} -*/ diff --git a/src/common/dao/config_test.go b/src/common/dao/config_test.go index ce624e313..a874a6c6e 100644 --- a/src/common/dao/config_test.go +++ b/src/common/dao/config_test.go @@ -15,175 +15,12 @@ package dao -/* import ( "testing" "github.com/vmware/harbor/src/common/models" ) -func deleteConfigByKey(key string) error { - if _, err := GetOrmer().Raw("delete from properties where k = ?", key). - Exec(); err != nil { - return err - } - return nil -} - -func TestGetConfigByKey(t *testing.T) { - cfg := &models.Config{ - Key: "key", - Value: "value", - } - - if err := InsertConfig(cfg); err != nil { - t.Fatalf("failed to insert configuration into table: %v", err) - } - defer func(key string) { - if err := deleteConfigByKey(key); err != nil { - t.Fatalf("failed to delete configuration %s: %v", key, err) - } - }(cfg.Key) - - config, err := GetConfigByKey(cfg.Key) - if err != nil { - t.Fatalf("failed to get configuration by key %s: %v", cfg.Key, err) - } - - if config == nil { - t.Fatal("configuration is nil") - } - - if config.Value != cfg.Value { - t.Fatalf("unexpected value: %s != %s", config.Value, cfg.Value) - } -} - -func TestListConfigs(t *testing.T) { - configs, err := ListConfigs() - if err != nil { - t.Fatalf("failed to list configurations: %v", err) - } - size := len(configs) - - cfg := &models.Config{ - Key: "key", - Value: "value", - } - if err := InsertConfig(cfg); err != nil { - t.Fatalf("failed to insert configuration into table: %v", err) - } - defer func(key string) { - if err := deleteConfigByKey(key); err != nil { - t.Fatalf("failed to delete configuration %s: %v", key, err) - } - }(cfg.Key) - - configs, err = ListConfigs() - if err != nil { - t.Fatalf("failed to list configurations: %v", err) - } - - if size+1 != len(configs) { - t.Fatalf("unexpected length of configurations: %d != %d", len(configs), size+1) - } -} - -func TestInsertConfig(t *testing.T) { - cfg := &models.Config{ - Key: "key1", - Value: "value1", - } - - if err := InsertConfig(cfg); err != nil { - t.Fatalf("failed to insert configuration into table: %v", err) - } - defer func(key string) { - if err := deleteConfigByKey(key); err != nil { - t.Fatalf("failed to delete configuration %s: %v", key, err) - } - }(cfg.Key) - - config, err := GetConfigByKey(cfg.Key) - if err != nil { - t.Fatalf("failed to get configuration by key %s: %v", cfg.Key, err) - } - if config == nil { - t.Fatal("configuration is nil") - } - - if config.Value != cfg.Value { - t.Fatalf("unexpected value: %s != %s", config.Value, cfg.Value) - } -} - -func TestUpdateConfig(t *testing.T) { - cfg := &models.Config{ - Key: "key", - Value: "value", - } - - if err := InsertConfig(cfg); err != nil { - t.Fatalf("failed to insert configuration into table: %v", err) - } - defer func(key string) { - if err := deleteConfigByKey(key); err != nil { - t.Fatalf("failed to delete configuration %s: %v", key, err) - } - }(cfg.Key) - - newCfg := &models.Config{ - Key: "key", - Value: "new_value", - } - if err := UpdateConfig(newCfg); err != nil { - t.Fatalf("failed to update configuration: %v", err) - } - - config, err := GetConfigByKey(cfg.Key) - if err != nil { - t.Fatalf("failed to get configuration by key %s: %v", cfg.Key, err) - } - - if config == nil { - t.Fatal("configuration is nil") - } - - if config.Value != newCfg.Value { - t.Fatalf("unexpected value: %s != %s", config.Value, newCfg.Value) - } -} - -func TestInsertOrUpdateConfigs(t *testing.T) { - cfg1 := &models.Config{ - Key: "key1", - Value: "value1", - } - - if err := InsertConfig(cfg1); err != nil { - t.Fatalf("failed to insert configuration into table: %v", err) - } - defer func(key string) { - if err := deleteConfigByKey(key); err != nil { - t.Fatalf("failed to delete configuration %s: %v", key, err) - } - }(cfg1.Key) - - cfg2 := &models.Config{ - Key: "key2", - Value: "value2", - } - - if err := InsertOrUpdateConfigs([]*models.Config{cfg1, cfg2}); err != nil { - t.Fatalf("failed to insert or update configurations: %v", err) - } - defer func(key string) { - if err := deleteConfigByKey(key); err != nil { - t.Fatalf("failed to delete configuration %s: %v", key, err) - } - }(cfg2.Key) -} - func TestAuthModeCanBeModified(t *testing.T) { c, err := GetOrmer().QueryTable(&models.User{}).Count() if err != nil { @@ -233,4 +70,3 @@ func TestAuthModeCanBeModified(t *testing.T) { } } } -*/ diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index 5eca32805..b229c0e96 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -366,14 +366,14 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo return nil, err } - insecure, err := api.GetIsInsecure() + verify, err := config.VerifyRemoteCert() if err != nil { return nil, err } username, password, ok := ra.Ctx.Request.BasicAuth() if ok { - return newRepositoryClient(endpoint, insecure, username, password, + return newRepositoryClient(endpoint, !verify, username, password, repoName, "repository", repoName, "pull", "push", "*") } @@ -382,7 +382,7 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo return nil, err } - return cache.NewRepositoryClient(endpoint, insecure, username, repoName, + return cache.NewRepositoryClient(endpoint, !verify, username, repoName, "repository", repoName, "pull", "push", "*") } diff --git a/src/ui/api/target.go b/src/ui/api/target.go index c1f88d2ba..becda414e 100644 --- a/src/ui/api/target.go +++ b/src/ui/api/target.go @@ -102,12 +102,12 @@ func (t *TargetAPI) Ping() { password = t.GetString("password") } - insecure, err := api.GetIsInsecure() + verify, err := config.VerifyRemoteCert() if err != nil { log.Errorf("failed to check whether insecure or not: %v", err) t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } - registry, err := newRegistryClient(endpoint, insecure, username, password, + registry, err := newRegistryClient(endpoint, !verify, username, password, "", "", "") if err != nil { // timeout, dns resolve error, connection refused, etc. From f113f4a54f2ab841df64e23a43dd713c5ee5e772 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 20 Jan 2017 13:30:49 +0800 Subject: [PATCH 09/22] update --- src/adminserver/api/base_test.go | 32 ++++ src/adminserver/api/cfg.go | 25 +-- src/adminserver/api/cfg_test.go | 25 +++ src/adminserver/systemcfg/store/driver.go | 13 +- .../systemcfg/store/json/driver_json.go | 34 +++- .../systemcfg/store/json/driver_json_test.go | 22 +-- src/adminserver/systemcfg/systemcfg.go | 115 ++++++------ src/adminserver/systemcfg/systemcfg_test.go | 79 ++++++--- src/common/api/base.go | 10 -- src/common/api/base_test.go | 20 --- src/common/config/config.go | 72 +++++--- src/common/dao/config_test.go | 164 ------------------ src/common/models/config.go | 18 +- src/common/utils/email/mail.go | 5 +- src/common/utils/test/adminserver.go | 44 ++++- src/jobservice/config/config.go | 70 +++----- src/ui/api/config.go | 152 ++++++++++------ src/ui/api/repository.go | 6 +- src/ui/api/target.go | 4 +- src/ui/auth/ldap/ldap.go | 2 +- src/ui/config/config.go | 132 +++++++------- src/ui/config/config_test.go | 2 +- 22 files changed, 512 insertions(+), 534 deletions(-) create mode 100644 src/adminserver/api/base_test.go create mode 100644 src/adminserver/api/cfg_test.go diff --git a/src/adminserver/api/base_test.go b/src/adminserver/api/base_test.go new file mode 100644 index 000000000..3ba79b912 --- /dev/null +++ b/src/adminserver/api/base_test.go @@ -0,0 +1,32 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestHandleInternalServerError(t *testing.T) { + w := httptest.NewRecorder() + handleInternalServerError(w) + + if w.Code != http.StatusInternalServerError { + t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusInternalServerError) + } + +} diff --git a/src/adminserver/api/cfg.go b/src/adminserver/api/cfg.go index e2a41412b..79b0b4dcf 100644 --- a/src/adminserver/api/cfg.go +++ b/src/adminserver/api/cfg.go @@ -20,11 +20,8 @@ import ( "io/ioutil" "net/http" "os" - "strconv" cfg "github.com/vmware/harbor/src/adminserver/systemcfg" - comcfg "github.com/vmware/harbor/src/common/config" - "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" ) @@ -93,31 +90,20 @@ func UpdateCfgs(w http.ResponseWriter, r *http.Request) { return } - m := &map[string]string{} - if err = json.Unmarshal(b, m); err != nil { + m := map[string]interface{}{} + if err = json.Unmarshal(b, &m); err != nil { handleBadRequestError(w, err.Error()) return } - system, err := cfg.GetSystemCfg() - if err != nil { - handleInternalServerError(w) - return - } - - if err := populate(system, *m); err != nil { - log.Errorf("failed to populate system configurations: %v", err) - handleInternalServerError(w) - return - } - - if err = cfg.UpdateSystemCfg(system); err != nil { + if err = cfg.UpdateSystemCfg(m); err != nil { log.Errorf("failed to update system configurations: %v", err) handleInternalServerError(w) return } } +/* // populate attrs of cfg according to m func populate(cfg *models.SystemCfg, m map[string]string) error { if mode, ok := m[comcfg.AUTHMode]; ok { @@ -133,7 +119,7 @@ func populate(cfg *models.SystemCfg, m map[string]string) error { cfg.Authentication.LDAP.SearchDN = dn } if pwd, ok := m[comcfg.LDAPSearchPwd]; ok { - cfg.Authentication.LDAP.SearchPwd = pwd + cfg.Authentication.LDAP.SearchPassword = pwd } if dn, ok := m[comcfg.LDAPBaseDN]; ok { cfg.Authentication.LDAP.BaseDN = dn @@ -191,3 +177,4 @@ func populate(cfg *models.SystemCfg, m map[string]string) error { return nil } +*/ diff --git a/src/adminserver/api/cfg_test.go b/src/adminserver/api/cfg_test.go new file mode 100644 index 000000000..320aa5e50 --- /dev/null +++ b/src/adminserver/api/cfg_test.go @@ -0,0 +1,25 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "net/http/httptest" +) + +func test() { + httptest.NewRecorder() + +} diff --git a/src/adminserver/systemcfg/store/driver.go b/src/adminserver/systemcfg/store/driver.go index c0a9a2832..3a36d572c 100644 --- a/src/adminserver/systemcfg/store/driver.go +++ b/src/adminserver/systemcfg/store/driver.go @@ -15,16 +15,13 @@ package store -import ( - "github.com/vmware/harbor/src/common/models" -) - // Driver defines methods that a configuration store driver must implement type Driver interface { // Name returns a human-readable name of the driver Name() string - // Read reads the configurations from store - Read() (*models.SystemCfg, error) - // Write writes the configurations to store - Write(*models.SystemCfg) error + // Read reads all the configurations from store + Read() (map[string]interface{}, error) + // Write writes the configurations to store, the configurations can be + // part of all + Write(map[string]interface{}) error } diff --git a/src/adminserver/systemcfg/store/json/driver_json.go b/src/adminserver/systemcfg/store/json/driver_json.go index 4785e6006..90180ba5e 100644 --- a/src/adminserver/systemcfg/store/json/driver_json.go +++ b/src/adminserver/systemcfg/store/json/driver_json.go @@ -23,7 +23,6 @@ import ( "sync" "github.com/vmware/harbor/src/adminserver/systemcfg/store" - "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" ) @@ -68,11 +67,15 @@ func (c *cfgStore) Name() string { } // Read ... -func (c *cfgStore) Read() (*models.SystemCfg, error) { +func (c *cfgStore) Read() (map[string]interface{}, error) { c.RLock() defer c.RUnlock() - b, err := ioutil.ReadFile(c.path) + return read(c.path) +} + +func read(path string) (map[string]interface{}, error) { + b, err := ioutil.ReadFile(path) if err != nil { return nil, err } @@ -82,8 +85,8 @@ func (c *cfgStore) Read() (*models.SystemCfg, error) { return nil, nil } - config := &models.SystemCfg{} - if err = json.Unmarshal(b, config); err != nil { + config := map[string]interface{}{} + if err = json.Unmarshal(b, &config); err != nil { return nil, err } @@ -91,14 +94,27 @@ func (c *cfgStore) Read() (*models.SystemCfg, error) { } // Write ... -func (c *cfgStore) Write(config *models.SystemCfg) error { - b, err := json.MarshalIndent(config, "", " ") +func (c *cfgStore) Write(config map[string]interface{}) error { + c.Lock() + defer c.Unlock() + + cfg, err := read(c.path) if err != nil { return err } - c.Lock() - defer c.Unlock() + if cfg == nil { + cfg = config + } else { + for k, v := range config { + cfg[k] = v + } + } + + b, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return err + } if err = ioutil.WriteFile(c.path, b, 0600); err != nil { return err diff --git a/src/adminserver/systemcfg/store/json/driver_json_test.go b/src/adminserver/systemcfg/store/json/driver_json_test.go index c551049f6..9a5ce4968 100644 --- a/src/adminserver/systemcfg/store/json/driver_json_test.go +++ b/src/adminserver/systemcfg/store/json/driver_json_test.go @@ -18,8 +18,6 @@ package json import ( "os" "testing" - - "github.com/vmware/harbor/src/common/models" ) func TestReadWrite(t *testing.T) { @@ -34,19 +32,21 @@ func TestReadWrite(t *testing.T) { } }() - config := &models.SystemCfg{ - Authentication: &models.Authentication{ - LDAP: &models.LDAP{}, - }, - Database: &models.Database{ - MySQL: &models.MySQL{}, - }, + if store.Name() != "JSON" { + t.Errorf("unexpected name: %s != %s", store.Name(), "JSON") + return + } + + config := map[string]interface{}{ + "key": "value", } if err := store.Write(config); err != nil { - t.Fatalf("failed to write configurations to json file: %v", err) + t.Errorf("failed to write configurations to json file: %v", err) + return } if _, err = store.Read(); err != nil { - t.Fatalf("failed to read configurations from json file: %v", err) + t.Errorf("failed to read configurations from json file: %v", err) + return } } diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index 57f0323e8..d1d45c203 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -22,7 +22,7 @@ import ( "github.com/vmware/harbor/src/adminserver/systemcfg/store" "github.com/vmware/harbor/src/adminserver/systemcfg/store/json" - "github.com/vmware/harbor/src/common/models" + comcfg "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/utils/log" ) @@ -77,112 +77,95 @@ func getCfgStore() string { } //read the following attrs from env every time boots up -func readFromEnv(cfg *models.SystemCfg) error { - cfg.DomainName = os.Getenv("EXT_ENDPOINT") +func readFromEnv(cfg map[string]interface{}) error { + cfg[comcfg.DomainName] = os.Getenv("EXT_ENDPOINT") - cfg.Database = &models.Database{ - Type: os.Getenv("DATABASE_TYPE"), - MySQL: &models.MySQL{ - Host: os.Getenv("MYSQL_HOST"), - Username: os.Getenv("MYSQL_USR"), - Password: os.Getenv("MYSQL_PWD"), - Database: os.Getenv("MYSQL_DATABASE"), - }, - SQLite: &models.SQLite{ - File: os.Getenv("SQLITE_FILE"), - }, - } + cfg[comcfg.DatabaseType] = os.Getenv("DATABASE_TYPE") + cfg[comcfg.MySQLHost] = os.Getenv("MYSQL_HOST") port, err := strconv.Atoi(os.Getenv("MYSQL_PORT")) if err != nil { return err } - cfg.Database.MySQL.Port = port - - cfg.TokenService = &models.TokenService{ - URL: os.Getenv("TOKEN_SERVICE_URL"), - } - cfg.Registry = &models.Registry{ - URL: os.Getenv("REGISTRY_URL"), - } - - //TODO remove - cfg.JobLogDir = os.Getenv("LOG_DIR") - //TODO remove - cfg.CompressJS = os.Getenv("USE_COMPRESSED_JS") == "on" - exp, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION")) + cfg[comcfg.MySQLPort] = port + cfg[comcfg.MySQLUsername] = os.Getenv("MYSQL_USR") + cfg[comcfg.MySQLPassword] = os.Getenv("MYSQL_PWD") + cfg[comcfg.MySQLDatabase] = os.Getenv("MYSQL_DATABASE") + cfg[comcfg.SQLiteFile] = os.Getenv("SQLITE_FILE") + cfg[comcfg.TokenServiceURL] = os.Getenv("TOKEN_SERVICE_URL") + tokenExpi, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION")) if err != nil { return err } - cfg.TokenExpiration = exp - cfg.SecretKey = os.Getenv("SECRET_KEY") - - cfgExp, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) + cfg[comcfg.TokenExpiration] = tokenExpi + cfg[comcfg.RegistryURL] = os.Getenv("REGISTRY_URL") + //TODO remove + cfg[comcfg.JobLogDir] = os.Getenv("LOG_DIR") + //TODO remove + cfg[comcfg.UseCompressedJS] = os.Getenv("USE_COMPRESSED_JS") == "on" + cfg[comcfg.SecretKey] = os.Getenv("SECRET_KEY") + cfgExpi, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) if err != nil { return err } - cfg.CfgExpiration = cfgExp - + cfg[comcfg.CfgExpiration] = cfgExpi workers, err := strconv.Atoi(os.Getenv("MAX_JOB_WORKERS")) if err != nil { return err } - cfg.MaxJobWorkers = workers + cfg[comcfg.MaxJobWorkers] = workers return nil } -func initFromEnv() (*models.SystemCfg, error) { - cfg := &models.SystemCfg{} +func initFromEnv() (map[string]interface{}, error) { + cfg := map[string]interface{}{} if err := readFromEnv(cfg); err != nil { return nil, err } - cfg.Authentication = &models.Authentication{ - Mode: os.Getenv("AUTH_MODE"), - SelfRegistration: os.Getenv("SELF_REGISTRATION") == "on", - LDAP: &models.LDAP{ - URL: os.Getenv("LDAP_URL"), - SearchDN: os.Getenv("LDAP_SEARCH_DN"), - SearchPwd: os.Getenv("LDAP_SEARCH_PWD"), - BaseDN: os.Getenv("LDAP_BASE_DN"), - Filter: os.Getenv("LDAP_FILTER"), - UID: os.Getenv("LDAP_UID"), - }, - } + cfg[comcfg.AUTHMode] = os.Getenv("AUTH_MODE") + cfg[comcfg.SelfRegistration] = os.Getenv("SELF_REGISTRATION") == "on" + cfg[comcfg.LDAPURL] = os.Getenv("LDAP_URL") + cfg[comcfg.LDAPSearchDN] = os.Getenv("LDAP_SEARCH_DN") + cfg[comcfg.LDAPSearchPwd] = os.Getenv("LDAP_SEARCH_PWD") + cfg[comcfg.LDAPBaseDN] = os.Getenv("LDAP_BASE_DN") + cfg[comcfg.LDAPFilter] = os.Getenv("LDAP_FILTER") + cfg[comcfg.LDAPUID] = os.Getenv("LDAP_UID") scope, err := strconv.Atoi(os.Getenv("LDAP_SCOPE")) if err != nil { return nil, err } - cfg.Authentication.LDAP.Scope = scope + cfg[comcfg.LDAPScope] = scope timeout, err := strconv.Atoi(os.Getenv("LDAP_TIMEOUT")) if err != nil { return nil, err } - cfg.Authentication.LDAP.Timeout = timeout - - cfg.Email = &models.Email{ - Host: os.Getenv("EMAIL_HOST"), - Port: os.Getenv("EMAIL_PORT"), - Username: os.Getenv("EMAIL_USR"), - Password: os.Getenv("EMAIL_PWD"), - SSL: os.Getenv("EMAIL_SSL") == "true", - From: os.Getenv("EMAIL_FROM"), - Identity: os.Getenv("EMAIL_IDENTITY"), + cfg[comcfg.LDAPTimeout] = timeout + cfg[comcfg.EmailHost] = os.Getenv("EMAIL_HOST") + port, err := strconv.Atoi(os.Getenv("EMAIL_PORT")) + if err != nil { + return nil, err } - cfg.VerifyRemoteCert = os.Getenv("VERIFY_REMOTE_CERT") == "on" - cfg.ProjectCreationRestriction = os.Getenv("PROJECT_CREATION_RESTRICTION") + cfg[comcfg.EmailPort] = port + cfg[comcfg.EmailUsername] = os.Getenv("EMAIL_USR") + cfg[comcfg.EmailPassword] = os.Getenv("EMAIL_PWD") + cfg[comcfg.EmailSSL] = os.Getenv("EMAIL_SSL") == "true" + cfg[comcfg.EmailFrom] = os.Getenv("EMAIL_FROM") + cfg[comcfg.EmailIdentity] = os.Getenv("EMAIL_IDENTITY") + cfg[comcfg.VerifyRemoteCert] = os.Getenv("VERIFY_REMOTE_CERT") == "on" + cfg[comcfg.ProjectCreationRestriction] = os.Getenv("PROJECT_CREATION_RESTRICTION") + cfg[comcfg.AdminInitialPassword] = os.Getenv("HARBOR_ADMIN_PASSWORD") - cfg.InitialAdminPwd = os.Getenv("HARBOR_ADMIN_PASSWORD") return cfg, nil } // GetSystemCfg returns the system configurations -func GetSystemCfg() (*models.SystemCfg, error) { +func GetSystemCfg() (map[string]interface{}, error) { return cfgStore.Read() } // UpdateSystemCfg updates the system configurations -func UpdateSystemCfg(cfg *models.SystemCfg) error { +func UpdateSystemCfg(cfg map[string]interface{}) error { return cfgStore.Write(cfg) } diff --git a/src/adminserver/systemcfg/systemcfg_test.go b/src/adminserver/systemcfg/systemcfg_test.go index f3f4a7f1b..14b38ed00 100644 --- a/src/adminserver/systemcfg/systemcfg_test.go +++ b/src/adminserver/systemcfg/systemcfg_test.go @@ -15,51 +15,43 @@ package systemcfg -/* import ( "os" "testing" + + comcfg "github.com/vmware/harbor/src/common/config" ) - +// test functions under adminserver/systemcfg func TestSystemcfg(t *testing.T) { key := "JSON_STORE_PATH" - tmpPath := "/tmp/config.json" - originalPath := os.Getenv(key) - defer func() { - if err := os.Remove(tmpPath); err != nil { - t.Errorf("failed to remove %s: %v", tmpPath, err) + path := "/tmp/config.json" + if _, err := os.Stat(path); err == nil { + if err := os.Remove(path); err != nil { + t.Fatalf("failed to remove %s: %v", path, err) } + } else if !os.IsNotExist(err) { + t.Fatalf("failed to check the existence of %s: %v", path, err) + } - if len(originalPath) == 0 { - if err := os.Unsetenv(key); err != nil { - t.Fatalf("failed to unset env %s: %v", key, err) - } - return - } - - if err := os.Setenv(key, originalPath); err != nil { - t.Fatalf("failed to set env %s: %v", key, err) - } - }() - - if err := os.Setenv(key, tmpPath); err != nil { + if err := os.Setenv(key, path); err != nil { t.Fatalf("failed to set env %s: %v", key, err) } m := map[string]string{ + "AUTH_MODE": comcfg.DBAuth, "LDAP_SCOPE": "1", "LDAP_TIMEOUT": "30", "MYSQL_PORT": "3306", "MAX_JOB_WORKERS": "3", "TOKEN_EXPIRATION": "30", "CFG_EXPIRATION": "5", + "EMAIL_PORT": "25", } for k, v := range m { if err := os.Setenv(k, v); err != nil { - t.Errorf("failed to set env %s: %v", k, err) - return + t.Fatalf("failed to set env %s: %v", k, err) } } @@ -67,5 +59,46 @@ func TestSystemcfg(t *testing.T) { t.Errorf("failed to initialize system configurations: %v", err) return } + defer func() { + if err := os.Remove(path); err != nil { + t.Fatalf("failed to remove %s: %v", path, err) + } + }() + + // run Init again to make sure it works well when the configuration file + // already exists + if err := Init(); err != nil { + t.Errorf("failed to initialize system configurations: %v", err) + return + } + + cfg, err := GetSystemCfg() + if err != nil { + t.Errorf("failed to get system configurations: %v", err) + return + } + + if cfg[comcfg.AUTHMode] != comcfg.DBAuth { + t.Errorf("unexpected auth mode: %s != %s", + cfg[comcfg.AUTHMode], comcfg.DBAuth) + return + } + + cfg[comcfg.AUTHMode] = comcfg.LDAPAuth + if err = UpdateSystemCfg(cfg); err != nil { + t.Errorf("failed to update system configurations: %v", err) + return + } + + cfg, err = GetSystemCfg() + if err != nil { + t.Errorf("failed to get system configurations: %v", err) + return + } + + if cfg[comcfg.AUTHMode] != comcfg.LDAPAuth { + t.Errorf("unexpected auth mode: %s != %s", + cfg[comcfg.AUTHMode], comcfg.DBAuth) + return + } } -*/ diff --git a/src/common/api/base.go b/src/common/api/base.go index 227327339..38635f76d 100644 --- a/src/common/api/base.go +++ b/src/common/api/base.go @@ -26,7 +26,6 @@ import ( "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/auth" - "github.com/vmware/harbor/src/ui/config" "github.com/astaxie/beego" ) @@ -210,12 +209,3 @@ func (b *BaseAPI) GetPaginationParams() (page, pageSize int64) { return page, pageSize } - -// GetIsInsecure ... -func GetIsInsecure() (bool, error) { - verify, err := config.VerifyRemoteCert() - if err != nil { - return false, err - } - return !verify, nil -} diff --git a/src/common/api/base_test.go b/src/common/api/base_test.go index 0352b6e76..386e30e52 100644 --- a/src/common/api/base_test.go +++ b/src/common/api/base_test.go @@ -13,23 +13,3 @@ limitations under the License. */ package api - -/* -import ( - "github.com/vmware/harbor/src/common/config" - "os" - "testing" -) - -func TestGetIsInsecure(t *testing.T) { - os.Setenv("VERIFY_REMOTE_CERT", "off") - err := config.Reload() - if err != nil { - t.Errorf("Failed to load config, error: %v", err) - } - if !GetIsInsecure() { - t.Errorf("GetIsInsecure() should be true when VERIFY_REMOTE_CERT is off, in fact: false") - } - os.Unsetenv("VERIFY_REMOTE_CERT") -} -*/ diff --git a/src/common/config/config.go b/src/common/config/config.go index 4c9aeaa61..295564d66 100644 --- a/src/common/config/config.go +++ b/src/common/config/config.go @@ -40,7 +40,15 @@ const ( LDAPScopeOnelevel = "2" LDAPScopeSubtree = "3" + DomainName = "domain_name" AUTHMode = "auth_mode" + DatabaseType = "database_type" + MySQLHost = "mysql_host" + MySQLPort = "mysql_port" + MySQLUsername = "mysql_username" + MySQLPassword = "mysql_password" + MySQLDatabase = "mysql_database" + SQLiteFile = "sqlite_file" SelfRegistration = "self_registration" LDAPURL = "ldap_url" LDAPSearchDN = "ldap_search_dn" @@ -50,6 +58,8 @@ const ( LDAPFilter = "ldap_filter" LDAPScope = "ldap_scope" LDAPTimeout = "ldap_timeout" + TokenServiceURL = "token_service_url" + RegistryURL = "registry_url" EmailHost = "email_host" EmailPort = "email_port" EmailUsername = "email_username" @@ -60,30 +70,29 @@ const ( ProjectCreationRestriction = "project_creation_restriction" VerifyRemoteCert = "verify_remote_cert" MaxJobWorkers = "max_job_workers" + TokenExpiration = "token_expiration" CfgExpiration = "cfg_expiration" + JobLogDir = "job_log_dir" + UseCompressedJS = "use_compressed_js" + SecretKey = "secret_key" + AdminInitialPassword = "admin_initial_password" ) // Manager manages configurations type Manager struct { Loader *Loader - Parser Parser + Parser *Parser Cache bool cache cache.Cache key string } -// Parser parses []byte to a specific configuration -type Parser interface { - // Parse ... - Parse([]byte) (interface{}, error) -} - // NewManager returns an instance of Manager // url: the url from which loader loads configurations -func NewManager(url, secret string, parser Parser, enableCache bool) *Manager { +func NewManager(url, secret string, enableCache bool) *Manager { m := &Manager{ Loader: NewLoader(url, secret), - Parser: parser, + Parser: &Parser{}, } if enableCache { @@ -101,7 +110,7 @@ func (m *Manager) Init() error { } // Load configurations, if cache is enabled, cache the configurations -func (m *Manager) Load() (interface{}, error) { +func (m *Manager) Load() (map[string]interface{}, error) { b, err := m.Loader.Load() if err != nil { return nil, err @@ -113,7 +122,7 @@ func (m *Manager) Load() (interface{}, error) { } if m.Cache { - expi, err := parseExpiration(b) + expi, err := getCfgExpiration(c) if err != nil { return nil, err } @@ -126,23 +135,26 @@ func (m *Manager) Load() (interface{}, error) { return c, nil } -func parseExpiration(b []byte) (int, error) { - expi := &struct { - Expi int `json:"cfg_expiration"` - }{} - if err := json.Unmarshal(b, expi); err != nil { - return 0, err +func getCfgExpiration(m map[string]interface{}) (int, error) { + if m == nil { + return 0, fmt.Errorf("can not get cfg expiration as configurations are null") } - return expi.Expi, nil + + expi, ok := m[CfgExpiration] + if !ok { + return 0, fmt.Errorf("cfg expiration is not set") + } + + return int(expi.(float64)), nil } // Get : if cache is enabled, read configurations from cache, // if cache is null or cache is disabled it loads configurations directly -func (m *Manager) Get() (interface{}, error) { +func (m *Manager) Get() (map[string]interface{}, error) { if m.Cache { c := m.cache.Get(m.key) if c != nil { - return c, nil + return c.(map[string]interface{}), nil } } return m.Load() @@ -201,6 +213,11 @@ func (l *Loader) Load() ([]byte, error) { return nil, err } defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%d", resp.StatusCode) + } + b, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, err @@ -230,5 +247,20 @@ func (l *Loader) Upload(b []byte) error { return fmt.Errorf("unexpected http status code: %d", resp.StatusCode) } + log.Debug("configurations uploaded") + return nil } + +// Parser parses configurations +type Parser struct { +} + +// Parse parses []byte to a map configuration +func (p *Parser) Parse(b []byte) (map[string]interface{}, error) { + c := map[string]interface{}{} + if err := json.Unmarshal(b, &c); err != nil { + return nil, err + } + return c, nil +} diff --git a/src/common/dao/config_test.go b/src/common/dao/config_test.go index ce624e313..a874a6c6e 100644 --- a/src/common/dao/config_test.go +++ b/src/common/dao/config_test.go @@ -15,175 +15,12 @@ package dao -/* import ( "testing" "github.com/vmware/harbor/src/common/models" ) -func deleteConfigByKey(key string) error { - if _, err := GetOrmer().Raw("delete from properties where k = ?", key). - Exec(); err != nil { - return err - } - return nil -} - -func TestGetConfigByKey(t *testing.T) { - cfg := &models.Config{ - Key: "key", - Value: "value", - } - - if err := InsertConfig(cfg); err != nil { - t.Fatalf("failed to insert configuration into table: %v", err) - } - defer func(key string) { - if err := deleteConfigByKey(key); err != nil { - t.Fatalf("failed to delete configuration %s: %v", key, err) - } - }(cfg.Key) - - config, err := GetConfigByKey(cfg.Key) - if err != nil { - t.Fatalf("failed to get configuration by key %s: %v", cfg.Key, err) - } - - if config == nil { - t.Fatal("configuration is nil") - } - - if config.Value != cfg.Value { - t.Fatalf("unexpected value: %s != %s", config.Value, cfg.Value) - } -} - -func TestListConfigs(t *testing.T) { - configs, err := ListConfigs() - if err != nil { - t.Fatalf("failed to list configurations: %v", err) - } - size := len(configs) - - cfg := &models.Config{ - Key: "key", - Value: "value", - } - if err := InsertConfig(cfg); err != nil { - t.Fatalf("failed to insert configuration into table: %v", err) - } - defer func(key string) { - if err := deleteConfigByKey(key); err != nil { - t.Fatalf("failed to delete configuration %s: %v", key, err) - } - }(cfg.Key) - - configs, err = ListConfigs() - if err != nil { - t.Fatalf("failed to list configurations: %v", err) - } - - if size+1 != len(configs) { - t.Fatalf("unexpected length of configurations: %d != %d", len(configs), size+1) - } -} - -func TestInsertConfig(t *testing.T) { - cfg := &models.Config{ - Key: "key1", - Value: "value1", - } - - if err := InsertConfig(cfg); err != nil { - t.Fatalf("failed to insert configuration into table: %v", err) - } - defer func(key string) { - if err := deleteConfigByKey(key); err != nil { - t.Fatalf("failed to delete configuration %s: %v", key, err) - } - }(cfg.Key) - - config, err := GetConfigByKey(cfg.Key) - if err != nil { - t.Fatalf("failed to get configuration by key %s: %v", cfg.Key, err) - } - if config == nil { - t.Fatal("configuration is nil") - } - - if config.Value != cfg.Value { - t.Fatalf("unexpected value: %s != %s", config.Value, cfg.Value) - } -} - -func TestUpdateConfig(t *testing.T) { - cfg := &models.Config{ - Key: "key", - Value: "value", - } - - if err := InsertConfig(cfg); err != nil { - t.Fatalf("failed to insert configuration into table: %v", err) - } - defer func(key string) { - if err := deleteConfigByKey(key); err != nil { - t.Fatalf("failed to delete configuration %s: %v", key, err) - } - }(cfg.Key) - - newCfg := &models.Config{ - Key: "key", - Value: "new_value", - } - if err := UpdateConfig(newCfg); err != nil { - t.Fatalf("failed to update configuration: %v", err) - } - - config, err := GetConfigByKey(cfg.Key) - if err != nil { - t.Fatalf("failed to get configuration by key %s: %v", cfg.Key, err) - } - - if config == nil { - t.Fatal("configuration is nil") - } - - if config.Value != newCfg.Value { - t.Fatalf("unexpected value: %s != %s", config.Value, newCfg.Value) - } -} - -func TestInsertOrUpdateConfigs(t *testing.T) { - cfg1 := &models.Config{ - Key: "key1", - Value: "value1", - } - - if err := InsertConfig(cfg1); err != nil { - t.Fatalf("failed to insert configuration into table: %v", err) - } - defer func(key string) { - if err := deleteConfigByKey(key); err != nil { - t.Fatalf("failed to delete configuration %s: %v", key, err) - } - }(cfg1.Key) - - cfg2 := &models.Config{ - Key: "key2", - Value: "value2", - } - - if err := InsertOrUpdateConfigs([]*models.Config{cfg1, cfg2}); err != nil { - t.Fatalf("failed to insert or update configurations: %v", err) - } - defer func(key string) { - if err := deleteConfigByKey(key); err != nil { - t.Fatalf("failed to delete configuration %s: %v", key, err) - } - }(cfg2.Key) -} - func TestAuthModeCanBeModified(t *testing.T) { c, err := GetOrmer().QueryTable(&models.User{}).Count() if err != nil { @@ -233,4 +70,3 @@ func TestAuthModeCanBeModified(t *testing.T) { } } } -*/ diff --git a/src/common/models/config.go b/src/common/models/config.go index 953f08e6c..50a73edef 100644 --- a/src/common/models/config.go +++ b/src/common/models/config.go @@ -24,14 +24,14 @@ type Authentication struct { // LDAP ... type LDAP struct { - URL string `json:"url"` - SearchDN string `json:"search_dn"` - SearchPwd string `json:"search_pwd"` - BaseDN string `json:"base_dn"` - Filter string `json:"filter"` - UID string `json:"uid"` - Scope int `json:"scope"` - Timeout int `json:"timeout"` // in second + URL string `json:"url"` + SearchDN string `json:"search_dn"` + SearchPassword string `json:"search_password"` + BaseDN string `json:"base_dn"` + Filter string `json:"filter"` + UID string `json:"uid"` + Scope int `json:"scope"` + Timeout int `json:"timeout"` // in second } // Database ... @@ -58,7 +58,7 @@ type SQLite struct { // Email ... type Email struct { Host string `json:"host"` - Port string `json:"port"` + Port int `json:"port"` Username string `json:"username"` Password string `json:"password"` SSL bool `json:"ssl"` diff --git a/src/common/utils/email/mail.go b/src/common/utils/email/mail.go index 61fc05531..f3c5108b5 100644 --- a/src/common/utils/email/mail.go +++ b/src/common/utils/email/mail.go @@ -18,6 +18,7 @@ package email import ( "bytes" "crypto/tls" + "strconv" //"strings" "net/smtp" @@ -67,11 +68,11 @@ func (m Mail) SendMail() error { } func sendMail(m Mail, auth smtp.Auth, content []byte) error { - return smtp.SendMail(mc.Host+":"+mc.Port, auth, m.From, m.To, content) + return smtp.SendMail(mc.Host+":"+strconv.Itoa(mc.Port), auth, m.From, m.To, content) } func sendMailWithTLS(m Mail, auth smtp.Auth, content []byte) error { - conn, err := tls.Dial("tcp", mc.Host+":"+mc.Port, nil) + conn, err := tls.Dial("tcp", mc.Host+":"+strconv.Itoa(mc.Port), nil) if err != nil { return err } diff --git a/src/common/utils/test/adminserver.go b/src/common/utils/test/adminserver.go index 538b861bd..18e1b91fd 100644 --- a/src/common/utils/test/adminserver.go +++ b/src/common/utils/test/adminserver.go @@ -20,17 +20,49 @@ import ( "net/http" "net/http/httptest" - "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/config" ) // NewAdminserver returns a mock admin server func NewAdminserver() (*httptest.Server, error) { m := []*RequestHandlerMapping{} - b, err := json.Marshal(&models.SystemCfg{ - Authentication: &models.Authentication{ - Mode: "db_auth", - }, - Registry: &models.Registry{}, + b, err := json.Marshal(map[string]interface{}{ + config.DomainName: "host01.com", + config.AUTHMode: config.DBAuth, + config.DatabaseType: "mysql", + config.MySQLHost: "127.0.0.1", + config.MySQLPort: 3306, + config.MySQLUsername: "user01", + config.MySQLPassword: "password", + config.MySQLDatabase: "registry", + config.SQLiteFile: "/tmp/registry.db", + config.SelfRegistration: true, + config.LDAPURL: "ldap://127.0.0.1", + config.LDAPSearchDN: "uid=searchuser,ou=people,dc=mydomain,dc=com", + config.LDAPSearchPwd: "password", + config.LDAPBaseDN: "ou=people,dc=mydomain,dc=com", + config.LDAPUID: "uid", + config.LDAPFilter: "", + config.LDAPScope: 3, + config.LDAPTimeout: 30, + config.TokenServiceURL: "http://token_service", + config.RegistryURL: "http://registry", + config.EmailHost: "127.0.0.1", + config.EmailPort: 25, + config.EmailUsername: "user01", + config.EmailPassword: "password", + config.EmailFrom: "from", + config.EmailSSL: true, + config.EmailIdentity: "", + config.ProjectCreationRestriction: config.ProCrtRestrAdmOnly, + config.VerifyRemoteCert: false, + config.MaxJobWorkers: 3, + config.TokenExpiration: 30, + config.CfgExpiration: 5, + config.JobLogDir: "/var/log/jobs", + config.UseCompressedJS: true, + config.SecretKey: "secret", + config.AdminInitialPassword: "password", }) if err != nil { return nil, err diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index 51bf1823f..e4ba0f9b1 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -16,7 +16,6 @@ package config import ( - "encoding/json" "os" comcfg "github.com/vmware/harbor/src/common/config" @@ -25,35 +24,13 @@ import ( var mg *comcfg.Manager -// Configuration of Jobservice -type Configuration struct { - Database *models.Database `json:"database"` - Registry *models.Registry `json:"registry"` - VerifyRemoteCert bool `json:"verify_remote_cert"` - MaxJobWorkers int `json:"max_job_workers"` - JobLogDir string `json:"job_log_dir"` - SecretKey string `json:"secret_key"` - CfgExpiration int `json:"cfg_expiration"` -} - -type parser struct { -} - -func (p *parser) Parse(b []byte) (interface{}, error) { - c := &Configuration{} - if err := json.Unmarshal(b, c); err != nil { - return nil, err - } - return c, nil -} - // Init configurations func Init() error { adminServerURL := os.Getenv("ADMIN_SERVER_URL") if len(adminServerURL) == 0 { adminServerURL = "http://adminserver" } - mg = comcfg.NewManager(adminServerURL, UISecret(), &parser{}, true) + mg = comcfg.NewManager(adminServerURL, UISecret(), true) if err := mg.Init(); err != nil { return err @@ -66,39 +43,44 @@ func Init() error { return nil } -func get() (*Configuration, error) { - c, err := mg.Get() - if err != nil { - return nil, err - } - return c.(*Configuration), nil -} - // VerifyRemoteCert returns bool value. func VerifyRemoteCert() (bool, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return true, err } - return cfg.VerifyRemoteCert, nil + return cfg[comcfg.VerifyRemoteCert].(bool), nil } // Database ... func Database() (*models.Database, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return nil, err } - return cfg.Database, nil + database := &models.Database{} + database.Type = cfg[comcfg.DatabaseType].(string) + mysql := &models.MySQL{} + mysql.Host = cfg[comcfg.MySQLHost].(string) + mysql.Port = int(cfg[comcfg.MySQLPort].(float64)) + mysql.Username = cfg[comcfg.MySQLUsername].(string) + mysql.Password = cfg[comcfg.MySQLPassword].(string) + mysql.Database = cfg[comcfg.MySQLDatabase].(string) + database.MySQL = mysql + sqlite := &models.SQLite{} + sqlite.File = cfg[comcfg.SQLiteFile].(string) + database.SQLite = sqlite + + return database, nil } // MaxJobWorkers ... func MaxJobWorkers() (int, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return 0, err } - return cfg.MaxJobWorkers, nil + return int(cfg[comcfg.MaxJobWorkers].(float64)), nil } // LocalUIURL returns the local ui url, job service will use this URL to call API hosted on ui process @@ -108,29 +90,29 @@ func LocalUIURL() string { // LocalRegURL returns the local registry url, job service will use this URL to pull image from the registry func LocalRegURL() (string, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return "", err } - return cfg.Registry.URL, nil + return cfg[comcfg.RegistryURL].(string), nil } // LogDir returns the absolute path to which the log file will be written func LogDir() (string, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return "", err } - return cfg.JobLogDir, nil + return cfg[comcfg.JobLogDir].(string), nil } // SecretKey will return the secret key for encryption/decryption password in target. func SecretKey() (string, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return "", err } - return cfg.SecretKey, nil + return cfg[comcfg.SecretKey].(string), nil } // UISecret returns the value of UI secret cookie, used for communication between UI and JobService diff --git a/src/ui/api/config.go b/src/ui/api/config.go index 1c17ad8e0..7c1f2a2f1 100644 --- a/src/ui/api/config.go +++ b/src/ui/api/config.go @@ -30,6 +30,41 @@ import ( "github.com/vmware/harbor/src/ui/config" ) +// keys of attrs which user can modify +var validKeys = []string{ + comcfg.AUTHMode, + comcfg.EmailFrom, + comcfg.EmailHost, + comcfg.EmailIdentity, + comcfg.EmailPassword, + comcfg.EmailPort, + comcfg.EmailSSL, + comcfg.EmailUsername, + comcfg.LDAPBaseDN, + comcfg.LDAPFilter, + comcfg.LDAPScope, + comcfg.LDAPSearchDN, + comcfg.LDAPSearchPwd, + comcfg.LDAPTimeout, + comcfg.LDAPUID, + comcfg.LDAPURL, + comcfg.ProjectCreationRestriction, + comcfg.SelfRegistration, + comcfg.VerifyRemoteCert, +} + +var numKeys = []string{ + comcfg.EmailPort, + comcfg.LDAPScope, + comcfg.LDAPTimeout, +} + +var boolKeys = []string{ + comcfg.EmailSSL, + comcfg.SelfRegistration, + comcfg.VerifyRemoteCert, +} + // ConfigAPI ... type ConfigAPI struct { api.BaseAPI @@ -49,6 +84,11 @@ func (c *ConfigAPI) Prepare() { } } +type value struct { + Value interface{} `json:"value"` + Editable bool `json:"editable"` +} + // Get returns configurations func (c *ConfigAPI) Get() { cfg, err := config.GetSystemCfg() @@ -57,24 +97,12 @@ func (c *ConfigAPI) Get() { c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } - if cfg.Database.MySQL != nil { - cfg.Database.MySQL.Password = "" - } - - cfg.InitialAdminPwd = "" - cfg.SecretKey = "" - - m := map[string]interface{}{} - m["config"] = cfg - - editable, err := dao.AuthModeCanBeModified() + m, err := convertForGet(cfg) if err != nil { - log.Errorf("failed to determinie whether auth mode can be modified: %v", err) + log.Errorf("failed to convert configurations: %v", err) c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } - m["auth_mode_editable"] = editable - c.Data["json"] = m c.ServeJSON() } @@ -83,11 +111,19 @@ func (c *ConfigAPI) Get() { func (c *ConfigAPI) Put() { m := map[string]string{} c.DecodeJSONReq(&m) - if err := validateCfg(m); err != nil { + + cfg := map[string]string{} + for _, k := range validKeys { + if v, ok := m[k]; ok { + cfg[k] = v + } + } + + if err := validateCfg(cfg); err != nil { c.CustomAbort(http.StatusBadRequest, err.Error()) } - if value, ok := m[comcfg.AUTHMode]; ok { + if value, ok := cfg[comcfg.AUTHMode]; ok { mode, err := config.AuthMode() if err != nil { log.Errorf("failed to get auth mode: %v", err) @@ -109,7 +145,13 @@ func (c *ConfigAPI) Put() { } } - if err := config.Upload(m); err != nil { + result, err := convertForPut(cfg) + if err != nil { + log.Errorf("failed to convert configurations: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if err := config.Upload(result); err != nil { log.Errorf("failed to upload configurations: %v", err) c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } @@ -200,60 +242,72 @@ func validateCfg(c map[string]string) error { return nil } -/* -func convert() ([]*models.Config, error) { - cfgs := []*models.Config{} - var err error - pwdKeys := []string{config.LDAP_SEARCH_PWD, config.EMAIL_PWD} - for _, pwdKey := range pwdKeys { - if pwd, ok := c[pwdKey]; ok && len(pwd) != 0 { - c[pwdKey], err = utils.ReversibleEncrypt(pwd, ui_cfg.SecretKey()) - if err != nil { - return nil, err +//encode passwords and convert map[string]string to map[string]interface{} +func convertForPut(m map[string]string) (map[string]interface{}, error) { + cfg := map[string]interface{}{} + + /* + pwdKeys := []string{config.LDAP_SEARCH_PWD, config.EMAIL_PWD} + for _, pwdKey := range pwdKeys { + if pwd, ok := c[pwdKey]; ok && len(pwd) != 0 { + c[pwdKey], err = utils.ReversibleEncrypt(pwd, ui_cfg.SecretKey()) + if err != nil { + return nil, err + } } } + */ + for k, v := range m { + cfg[k] = v } - for _, key := range configKeys { - if value, ok := c[key]; ok { - cfgs = append(cfgs, &models.Config{ - Key: key, - Value: value, - }) + for _, k := range numKeys { + v, err := strconv.Atoi(cfg[k].(string)) + if err != nil { + return nil, err } + cfg[k] = v } - return cfgs, nil + for _, k := range boolKeys { + cfg[k] = cfg[k] == "1" + } + + return cfg, nil } -*/ -/* -//[]*models.Config >> cfgForGet -func convert(cfg *config.Configuration) (map[string]interface{}, error) { - result := map[string]interface{}{} - for _, config := range configs { - cfg[config.Key] = &value{ - Value: config.Value, - Editable: true, - } - } +// delete sensitive attrs and add editable field to every attr +func convertForGet(cfg map[string]interface{}) (map[string]*value, error) { + result := map[string]*value{} - dels := []string{config.LDAP_SEARCH_PWD, config.EMAIL_PWD} + dels := []string{ + comcfg.AdminInitialPassword, + comcfg.EmailPassword, + comcfg.LDAPSearchPwd, + comcfg.MySQLPassword, + comcfg.SecretKey} for _, del := range dels { if _, ok := cfg[del]; ok { delete(cfg, del) } } + for k, v := range cfg { + result[k] = &value{ + Value: v, + Editable: true, + } + } + flag, err := authModeCanBeModified() if err != nil { return nil, err } - cfg[config.AUTH_MODE].Editable = flag + result[comcfg.AUTHMode].Editable = flag - return cfgForGet(cfg), nil + return result, nil } -*/ + func authModeCanBeModified() (bool, error) { return dao.AuthModeCanBeModified() } diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index 5eca32805..b229c0e96 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -366,14 +366,14 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo return nil, err } - insecure, err := api.GetIsInsecure() + verify, err := config.VerifyRemoteCert() if err != nil { return nil, err } username, password, ok := ra.Ctx.Request.BasicAuth() if ok { - return newRepositoryClient(endpoint, insecure, username, password, + return newRepositoryClient(endpoint, !verify, username, password, repoName, "repository", repoName, "pull", "push", "*") } @@ -382,7 +382,7 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo return nil, err } - return cache.NewRepositoryClient(endpoint, insecure, username, repoName, + return cache.NewRepositoryClient(endpoint, !verify, username, repoName, "repository", repoName, "pull", "push", "*") } diff --git a/src/ui/api/target.go b/src/ui/api/target.go index c1f88d2ba..becda414e 100644 --- a/src/ui/api/target.go +++ b/src/ui/api/target.go @@ -102,12 +102,12 @@ func (t *TargetAPI) Ping() { password = t.GetString("password") } - insecure, err := api.GetIsInsecure() + verify, err := config.VerifyRemoteCert() if err != nil { log.Errorf("failed to check whether insecure or not: %v", err) t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } - registry, err := newRegistryClient(endpoint, insecure, username, password, + registry, err := newRegistryClient(endpoint, !verify, username, password, "", "", "") if err != nil { // timeout, dns resolve error, connection refused, etc. diff --git a/src/ui/auth/ldap/ldap.go b/src/ui/auth/ldap/ldap.go index e3928a201..7b531e118 100644 --- a/src/ui/auth/ldap/ldap.go +++ b/src/ui/auth/ldap/ldap.go @@ -116,7 +116,7 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { ldapSearchDn := settings.SearchDN if ldapSearchDn != "" { log.Debug("Search DN: ", ldapSearchDn) - ldapSearchPwd := settings.SearchPwd + ldapSearchPwd := settings.SearchPassword err = ldap.Bind(ldapSearchDn, ldapSearchPwd) if err != nil { log.Debug("Bind search dn error", err) diff --git a/src/ui/config/config.go b/src/ui/config/config.go index ccb760f80..5833b16e7 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -26,35 +26,6 @@ import ( var mg *comcfg.Manager -// Configuration of UI -type Configuration struct { - DomainName string `json:"domain_name"` // Harbor external URL: protocal://host:port - Authentication *models.Authentication `json:"authentication"` - Database *models.Database `json:"database"` - TokenService *models.TokenService `json:"token_service"` - Registry *models.Registry `json:"registry"` - Email *models.Email `json:"email"` - VerifyRemoteCert bool `json:"verify_remote_cert"` - ProjectCreationRestriction string `json:"project_creation_restriction"` - InitialAdminPwd string `json:"initial_admin_pwd"` - //TODO remove - CompressJS bool `json:"compress_js"` - TokenExpiration int `json:"token_expiration"` - SecretKey string `json:"secret_key"` - CfgExpiration int `json:"cfg_expiration"` -} - -type parser struct { -} - -func (p *parser) Parse(b []byte) (interface{}, error) { - c := &Configuration{} - if err := json.Unmarshal(b, c); err != nil { - return nil, err - } - return c, nil -} - // Init configurations func Init() error { adminServerURL := os.Getenv("ADMIN_SERVER_URL") @@ -62,7 +33,7 @@ func Init() error { adminServerURL = "http://adminserver" } log.Debugf("admin server URL: %s", adminServerURL) - mg = comcfg.NewManager(adminServerURL, UISecret(), &parser{}, true) + mg = comcfg.NewManager(adminServerURL, UISecret(), true) if err := mg.Init(); err != nil { return err @@ -75,14 +46,6 @@ func Init() error { return nil } -func get() (*Configuration, error) { - c, err := mg.Get() - if err != nil { - return nil, err - } - return c.(*Configuration), nil -} - // Load configurations func Load() error { _, err := mg.Load() @@ -90,7 +53,7 @@ func Load() error { } // Upload uploads all system configutations to admin server -func Upload(cfg map[string]string) error { +func Upload(cfg map[string]interface{}) error { b, err := json.Marshal(cfg) if err != nil { return err @@ -99,80 +62,92 @@ func Upload(cfg map[string]string) error { } // GetSystemCfg returns the system configurations -func GetSystemCfg() (*models.SystemCfg, error) { +func GetSystemCfg() (map[string]interface{}, error) { raw, err := mg.Loader.Load() if err != nil { return nil, err } - cfg := &models.SystemCfg{} - if err = json.Unmarshal(raw, cfg); err != nil { + c, err := mg.Parser.Parse(raw) + if err != nil { return nil, err } - return cfg, nil + + return c, nil } // AuthMode ... func AuthMode() (string, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return "", err } - return cfg.Authentication.Mode, nil + return cfg[comcfg.AUTHMode].(string), nil } // LDAP returns the setting of ldap server func LDAP() (*models.LDAP, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return nil, err } - return cfg.Authentication.LDAP, nil + + ldap := &models.LDAP{} + ldap.URL = cfg[comcfg.LDAPURL].(string) + ldap.SearchDN = cfg[comcfg.LDAPSearchDN].(string) + ldap.SearchPassword = cfg[comcfg.LDAPSearchPwd].(string) + ldap.BaseDN = cfg[comcfg.LDAPBaseDN].(string) + ldap.UID = cfg[comcfg.LDAPUID].(string) + ldap.Filter = cfg[comcfg.LDAPFilter].(string) + ldap.Scope = int(cfg[comcfg.LDAPScope].(float64)) + ldap.Timeout = int(cfg[comcfg.LDAPTimeout].(float64)) + + return ldap, nil } // TokenExpiration returns the token expiration time (in minute) func TokenExpiration() (int, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return 0, err } - return cfg.TokenExpiration, nil + return int(cfg[comcfg.TokenExpiration].(float64)), nil } // DomainName returns the external URL of Harbor: protocal://host:port func DomainName() (string, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return "", err } - return cfg.DomainName, nil + return cfg[comcfg.DomainName].(string), nil } // SecretKey returns the secret key to encrypt the password of target func SecretKey() (string, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return "", err } - return cfg.SecretKey, nil + return cfg[comcfg.SecretKey].(string), nil } // SelfRegistration returns the enablement of self registration func SelfRegistration() (bool, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return false, err } - return cfg.Authentication.SelfRegistration, nil + return cfg[comcfg.SelfRegistration].(bool), nil } // RegistryURL ... func RegistryURL() (string, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return "", err } - return cfg.Registry.URL, nil + return cfg[comcfg.RegistryURL].(string), nil } // InternalJobServiceURL returns jobservice URL for internal communication between Harbor containers @@ -182,47 +157,70 @@ func InternalJobServiceURL() string { // InitialAdminPassword returns the initial password for administrator func InitialAdminPassword() (string, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return "", err } - return cfg.InitialAdminPwd, nil + return cfg[comcfg.AdminInitialPassword].(string), nil } // OnlyAdminCreateProject returns the flag to restrict that only sys admin can create project func OnlyAdminCreateProject() (bool, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return true, err } - return cfg.ProjectCreationRestriction == comcfg.ProCrtRestrAdmOnly, nil + return cfg[comcfg.ProjectCreationRestriction].(string) == comcfg.ProCrtRestrAdmOnly, nil } // VerifyRemoteCert returns bool value. func VerifyRemoteCert() (bool, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return true, err } - return cfg.VerifyRemoteCert, nil + return cfg[comcfg.VerifyRemoteCert].(bool), nil } // Email returns email server settings func Email() (*models.Email, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return nil, err } - return cfg.Email, nil + + email := &models.Email{} + email.Host = cfg[comcfg.EmailHost].(string) + email.Port = int(cfg[comcfg.EmailPort].(float64)) + email.Username = cfg[comcfg.EmailUsername].(string) + email.Password = cfg[comcfg.EmailPassword].(string) + email.SSL = cfg[comcfg.EmailSSL].(bool) + email.From = cfg[comcfg.EmailFrom].(string) + email.Identity = cfg[comcfg.EmailIdentity].(string) + + return email, nil } // Database returns database settings func Database() (*models.Database, error) { - cfg, err := get() + cfg, err := mg.Get() if err != nil { return nil, err } - return cfg.Database, nil + database := &models.Database{} + database.Type = cfg[comcfg.DatabaseType].(string) + mysql := &models.MySQL{} + mysql.Host = cfg[comcfg.MySQLHost].(string) + mysql.Port = int(cfg[comcfg.MySQLPort].(float64)) + mysql.Username = cfg[comcfg.MySQLUsername].(string) + mysql.Password = cfg[comcfg.MySQLPassword].(string) + mysql.Database = cfg[comcfg.MySQLDatabase].(string) + database.MySQL = mysql + sqlite := &models.SQLite{} + sqlite.File = cfg[comcfg.SQLiteFile].(string) + database.SQLite = sqlite + + return database, nil } // UISecret returns the value of UI secret cookie, used for communication between UI and JobService diff --git a/src/ui/config/config_test.go b/src/ui/config/config_test.go index e71160d2e..4846dd952 100644 --- a/src/ui/config/config_test.go +++ b/src/ui/config/config_test.go @@ -44,7 +44,7 @@ func TestConfig(t *testing.T) { t.Fatalf("failed to load configurations: %v", err) } - if err := Upload(map[string]string{}); err != nil { + if err := Upload(map[string]interface{}{}); err != nil { t.Fatalf("failed to upload configurations: %v", err) } From 9ec11ac67210c4ba8abe90393b024b4be35536a8 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 8 Feb 2017 17:17:33 +0800 Subject: [PATCH 10/22] add test cases --- .travis.yml | 2 +- src/adminserver/api/cfg.go | 76 ---------------- src/adminserver/api/cfg_test.go | 146 ++++++++++++++++++++++++++++++- src/common/config/config_test.go | 3 + src/common/models/config.go | 4 + src/ui/api/config.go | 8 ++ src/ui/api/config_test.go | 70 +++++++++++++++ src/ui/api/harborapi_test.go | 21 +++++ 8 files changed, 251 insertions(+), 79 deletions(-) create mode 100644 src/ui/api/config_test.go diff --git a/.travis.yml b/.travis.yml index 7337b0bdd..8c734d966 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ env: MAX_JOB_WORKERS: 3 SECRET_KEY: 1234567890123456 AUTH_MODE: db_auth - SELF_REGISTRATION: "on" + SELF_REGISTRATION: on before_install: - sudo ./tests/hostcfg.sh diff --git a/src/adminserver/api/cfg.go b/src/adminserver/api/cfg.go index 79b0b4dcf..6a2198c6e 100644 --- a/src/adminserver/api/cfg.go +++ b/src/adminserver/api/cfg.go @@ -102,79 +102,3 @@ func UpdateCfgs(w http.ResponseWriter, r *http.Request) { return } } - -/* -// populate attrs of cfg according to m -func populate(cfg *models.SystemCfg, m map[string]string) error { - if mode, ok := m[comcfg.AUTHMode]; ok { - cfg.Authentication.Mode = mode - } - if value, ok := m[comcfg.SelfRegistration]; ok { - cfg.Authentication.SelfRegistration = value == "1" - } - if url, ok := m[comcfg.LDAPURL]; ok { - cfg.Authentication.LDAP.URL = url - } - if dn, ok := m[comcfg.LDAPSearchDN]; ok { - cfg.Authentication.LDAP.SearchDN = dn - } - if pwd, ok := m[comcfg.LDAPSearchPwd]; ok { - cfg.Authentication.LDAP.SearchPassword = pwd - } - if dn, ok := m[comcfg.LDAPBaseDN]; ok { - cfg.Authentication.LDAP.BaseDN = dn - } - if uid, ok := m[comcfg.LDAPUID]; ok { - cfg.Authentication.LDAP.UID = uid - } - if filter, ok := m[comcfg.LDAPFilter]; ok { - cfg.Authentication.LDAP.Filter = filter - } - if scope, ok := m[comcfg.LDAPScope]; ok { - i, err := strconv.Atoi(scope) - if err != nil { - return err - } - cfg.Authentication.LDAP.Scope = i - } - if timeout, ok := m[comcfg.LDAPTimeout]; ok { - i, err := strconv.Atoi(timeout) - if err != nil { - return err - } - cfg.Authentication.LDAP.Timeout = i - } - - if value, ok := m[comcfg.EmailHost]; ok { - cfg.Email.Host = value - } - if value, ok := m[comcfg.EmailPort]; ok { - cfg.Email.Port = value - } - if value, ok := m[comcfg.EmailUsername]; ok { - cfg.Email.Username = value - } - if value, ok := m[comcfg.EmailPassword]; ok { - cfg.Email.Password = value - } - if value, ok := m[comcfg.EmailSSL]; ok { - cfg.Email.SSL = value == "1" - } - if value, ok := m[comcfg.EmailFrom]; ok { - cfg.Email.From = value - } - if value, ok := m[comcfg.EmailIdentity]; ok { - cfg.Email.Identity = value - } - - if value, ok := m[comcfg.ProjectCreationRestriction]; ok { - cfg.ProjectCreationRestriction = value - } - - if value, ok := m[comcfg.VerifyRemoteCert]; ok { - cfg.VerifyRemoteCert = value == "1" - } - - return nil -} -*/ diff --git a/src/adminserver/api/cfg_test.go b/src/adminserver/api/cfg_test.go index 320aa5e50..8ae8553c2 100644 --- a/src/adminserver/api/cfg_test.go +++ b/src/adminserver/api/cfg_test.go @@ -16,10 +16,152 @@ package api import ( + "bytes" + "encoding/json" + "io" + "io/ioutil" + "net/http" "net/http/httptest" + "os" + "testing" + + "github.com/vmware/harbor/src/adminserver/systemcfg" + "github.com/vmware/harbor/src/common/config" ) -func test() { - httptest.NewRecorder() +func TestConfigAPI(t *testing.T) { + path := "/tmp/config.json" + secret := "secret" + envs := map[string]string{ + "JSON_STORE_PATH": path, + "UI_SECRET": secret, + "MYSQL_PORT": "3306", + "TOKEN_EXPIRATION": "30", + "CFG_EXPIRATION": "5", + "MAX_JOB_WORKERS": "3", + "LDAP_SCOPE": "3", + "LDAP_TIMEOUT": "30", + "EMAIL_PORT": "25", + } + + for k, v := range envs { + if err := os.Setenv(k, v); err != nil { + t.Fatalf("failed to set env %s: %v", k, err) + } + } + defer os.Remove(path) + + if err := systemcfg.Init(); err != nil { + t.Errorf("failed to initialize systemconfigurations: %v", err) + return + } + + r, err := http.NewRequest("GET", "", nil) + if err != nil { + t.Errorf("failed to create request: %v", err) + return + } + + w := httptest.NewRecorder() + ListCfgs(w, r) + if w.Code != http.StatusUnauthorized { + t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusUnauthorized) + return + } + + r.AddCookie(&http.Cookie{ + Name: "secret", + Value: secret, + }) + + w = httptest.NewRecorder() + ListCfgs(w, r) + if w.Code != http.StatusOK { + t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusOK) + return + } + + m, err := parse(w.Body) + if err != nil { + t.Errorf("failed to parse response body: %v", err) + return + } + + scope := int(m[config.LDAPScope].(float64)) + if scope != 3 { + t.Errorf("unexpected ldap scope: %d != %d", scope, 3) + return + } + + // modify configurations + c := map[string]interface{}{ + config.AUTHMode: config.LDAPAuth, + } + + b, err := json.Marshal(c) + if err != nil { + t.Errorf("failed to marshal configuartions: %v", err) + return + } + + w = httptest.NewRecorder() + r, err = http.NewRequest("GET", "", bytes.NewReader(b)) + if err != nil { + t.Errorf("failed to create request: %v", err) + return + } + r.AddCookie(&http.Cookie{ + Name: "secret", + Value: secret, + }) + + UpdateCfgs(w, r) + + if w.Code != http.StatusOK { + t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusOK) + return + } + + // confirm the modification is done + r, err = http.NewRequest("GET", "", nil) + if err != nil { + t.Errorf("failed to create request: %v", err) + return + } + r.AddCookie(&http.Cookie{ + Name: "secret", + Value: secret, + }) + w = httptest.NewRecorder() + ListCfgs(w, r) + if w.Code != http.StatusOK { + t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusOK) + return + } + + m, err = parse(w.Body) + if err != nil { + t.Errorf("failed to parse response body: %v", err) + return + } + + mode := m[config.AUTHMode].(string) + if mode != config.LDAPAuth { + t.Errorf("unexpected ldap scope: %s != %s", mode, config.LDAPAuth) + return + } +} + +func parse(reader io.Reader) (map[string]interface{}, error) { + b, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + + m := map[string]interface{}{} + if err := json.Unmarshal(b, &m); err != nil { + return nil, err + } + return m, nil } diff --git a/src/common/config/config_test.go b/src/common/config/config_test.go index 81448edf3..77b8b7829 100644 --- a/src/common/config/config_test.go +++ b/src/common/config/config_test.go @@ -13,3 +13,6 @@ limitations under the License. */ package config + +// the functions in common/config/config.go have been tested +// by cases in UI and Jobservice diff --git a/src/common/models/config.go b/src/common/models/config.go index 50a73edef..16f518d2b 100644 --- a/src/common/models/config.go +++ b/src/common/models/config.go @@ -15,12 +15,14 @@ package models +/* // Authentication ... type Authentication struct { Mode string `json:"mode"` SelfRegistration bool `json:"self_registration"` LDAP *LDAP `json:"ldap,omitempty"` } +*/ // LDAP ... type LDAP struct { @@ -66,6 +68,7 @@ type Email struct { From string `json:"from"` } +/* // Registry ... type Registry struct { URL string `json:"url"` @@ -94,3 +97,4 @@ type SystemCfg struct { SecretKey string `json:"secret_key,omitempty"` CfgExpiration int `json:"cfg_expiration"` } +*/ diff --git a/src/ui/api/config.go b/src/ui/api/config.go index 7c1f2a2f1..89f5371a0 100644 --- a/src/ui/api/config.go +++ b/src/ui/api/config.go @@ -262,6 +262,10 @@ func convertForPut(m map[string]string) (map[string]interface{}, error) { } for _, k := range numKeys { + if _, ok := cfg[k]; !ok { + continue + } + v, err := strconv.Atoi(cfg[k].(string)) if err != nil { return nil, err @@ -270,6 +274,10 @@ func convertForPut(m map[string]string) (map[string]interface{}, error) { } for _, k := range boolKeys { + if _, ok := cfg[k]; !ok { + continue + } + cfg[k] = cfg[k] == "1" } diff --git a/src/ui/api/config_test.go b/src/ui/api/config_test.go new file mode 100644 index 000000000..ca4b4e2c2 --- /dev/null +++ b/src/ui/api/config_test.go @@ -0,0 +1,70 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/common/config" +) + +func TestGetConfig(t *testing.T) { + fmt.Println("Testing getting configurations") + assert := assert.New(t) + apiTest := newHarborAPI() + + //case 1: get configurations without admin role + code, _, err := apiTest.GetConfig(*testUser) + if err != nil { + t.Fatalf("failed to get configurations: %v", err) + } + + assert.Equal(401, code, "the status code of getting configurations with non-admin user should be 401") + + //case 2: get configurations with admin role + code, cfg, err := apiTest.GetConfig(*admin) + if err != nil { + t.Fatalf("failed to get configurations: %v", err) + } + + if !assert.Equal(200, code, "the status code of getting configurations with admin user should be 200") { + return + } + + mode := cfg[config.AUTHMode].Value.(string) + assert.Equal(config.DBAuth, mode, fmt.Sprintf("the auth mode should be %s", config.DBAuth)) +} + +func TestPutConfig(t *testing.T) { + fmt.Println("Testing modifying configurations") + assert := assert.New(t) + apiTest := newHarborAPI() + + cfg := map[string]string{ + config.VerifyRemoteCert: "0", + } + + code, err := apiTest.PutConfig(*admin, cfg) + if err != nil { + t.Fatalf("failed to get configurations: %v", err) + } + + if !assert.Equal(200, code, "the status code of modifying configurations with admin user should be 200") { + return + } +} diff --git a/src/ui/api/harborapi_test.go b/src/ui/api/harborapi_test.go index ac6dd6819..c55fb243c 100644 --- a/src/ui/api/harborapi_test.go +++ b/src/ui/api/harborapi_test.go @@ -99,6 +99,7 @@ func init() { beego.Router("/api/systeminfo/volumes", &SystemInfoAPI{}, "get:GetVolumeInfo") beego.Router("/api/systeminfo/getcert", &SystemInfoAPI{}, "get:GetCert") beego.Router("/api/ldap/ping", &LdapAPI{}, "post:Ping") + beego.Router("/api/configurations", &ConfigAPI{}) _ = updateInitPassword(1, "Harbor12345") @@ -922,3 +923,23 @@ func (a testapi) LdapPost(authInfo usrInfo, ldapConf apilib.LdapConf) (int, erro httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo) return httpStatusCode, err } + +func (a testapi) GetConfig(authInfo usrInfo) (int, map[string]*value, error) { + _sling := sling.New().Base(a.basePath).Get("/api/configurations") + + cfg := map[string]*value{} + + code, body, err := request(_sling, jsonAcceptHeader, authInfo) + if err == nil && code == 200 { + err = json.Unmarshal(body, &cfg) + } + return code, cfg, err +} + +func (a testapi) PutConfig(authInfo usrInfo, cfg map[string]string) (int, error) { + _sling := sling.New().Base(a.basePath).Put("/api/configurations").BodyJSON(cfg) + + code, _, err := request(_sling, jsonAcceptHeader, authInfo) + + return code, err +} From 1fbb28ad8c37c7f945ca7151c57d5d46c643f97f Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 9 Feb 2017 13:58:48 +0800 Subject: [PATCH 11/22] update --- .travis.yml | 1 + docs/swagger.yaml | 40 +++++++++++++++++++ src/common/dao/dao_test.go | 2 +- .../utils/registry/auth/tokenauthorizer.go | 16 ++++---- src/jobservice/config/config.go | 9 +++++ src/jobservice/replication/transfer.go | 10 +++++ src/ui/api/repository.go | 10 +++++ src/ui/api/target.go | 10 +++++ 8 files changed, 88 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8c734d966..05dc4a095 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,6 +70,7 @@ before_script: # create tables and load data # - mysql < ./make/db/registry.sql -uroot --verbose - sudo sqlite3 /tmp/registry.db < make/common/db/registry_sqlite.sql + - sudo chmod 777 /tmp/registry.db script: - sudo mkdir -p /harbor_storage/ca_download diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 6bbd2ed21..460bd6a85 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1371,6 +1371,46 @@ paths: description: Inviald ldap configuration parameters. 500: description: Unexpected internal errors. + /configurations: + get: + summary: Get system configurations. + description: | + This endpoint is for retrieving system configurations that only provides for admin user. + tags: + - Products + responses: + 200: + description: Get system configurations successfully. The response body is a map. + schema: + type: object + 401: + description: User need to log in first. + 403: + description: User does not have permission of admin role. + 500: + description: Unexpected internal errors. + put: + summary: Modify system configurations. + description: | + This endpoint is for modifying system configurations that only provides for admin user. + tags: + - Products + parameters: + - name: configurations + in: body + required: true + schema: + type: object + description: The configurations map need to be modified, the following are keys "auth_mode", "email_from", "email_host", "email_identity", "email_password", "email_port", "email_ssl", "email_username", "ldap_base_dn", "ldap_filter", "ldap_scope", "ldap_search_dn", "ldap_search_password", "ldap_timeout", "ldap_uid", "ldap_url", "project_creation_restriction", "self_registration", "verify_remote_cert". + responses: + 200: + description: Modify system configurations successfully. + 401: + description: User need to log in first. + 403: + description: User does not have permission of admin role. + 500: + description: Unexpected internal errors. definitions: Search: type: object diff --git a/src/common/dao/dao_test.go b/src/common/dao/dao_test.go index 2a1144d9b..26056116f 100644 --- a/src/common/dao/dao_test.go +++ b/src/common/dao/dao_test.go @@ -137,7 +137,7 @@ const publicityOn = 1 const publicityOff = 0 func TestMain(m *testing.M) { - databases := []string{"mysql"} + databases := []string{"mysql", "sqlite"} for _, database := range databases { log.Infof("run test cases for database: %s", database) diff --git a/src/common/utils/registry/auth/tokenauthorizer.go b/src/common/utils/registry/auth/tokenauthorizer.go index 917e43f38..0543ef524 100644 --- a/src/common/utils/registry/auth/tokenauthorizer.go +++ b/src/common/utils/registry/auth/tokenauthorizer.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "net/http" "net/url" + "os" "strings" "sync" "time" @@ -234,15 +235,12 @@ func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes [] // 2. the realm field returned by registry is an IP which can not reachable // inside Harbor func tokenURL(realm string) string { - //TODO - /* - extEndpoint := config.ExtEndpoint() - tokenEndpoint := config.TokenEndpoint() - if len(extEndpoint) != 0 && len(tokenEndpoint) != 0 && - strings.Contains(realm, extEndpoint) { - realm = strings.TrimRight(tokenEndpoint, "/") + "/service/token" - } - */ + + domainName := os.Getenv("DOMAIN_NAME") + if len(domainName) != 0 && strings.Contains(realm, domainName) { + realm = "http://ui/service/token" + } + return realm } diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index e4ba0f9b1..d4916dc22 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -120,3 +120,12 @@ func SecretKey() (string, error) { func UISecret() string { return os.Getenv("UI_SECRET") } + +// DomainName ... +func DomainName() (string, error) { + cfg, err := mg.Get() + if err != nil { + return "", err + } + return cfg[comcfg.DomainName].(string), nil +} diff --git a/src/jobservice/replication/transfer.go b/src/jobservice/replication/transfer.go index db70003cc..186718ee2 100644 --- a/src/jobservice/replication/transfer.go +++ b/src/jobservice/replication/transfer.go @@ -23,6 +23,7 @@ import ( "fmt" "io/ioutil" "net/http" + "os" "strings" "github.com/docker/distribution" @@ -33,6 +34,7 @@ import ( "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/common/utils/registry" "github.com/vmware/harbor/src/common/utils/registry/auth" + "github.com/vmware/harbor/src/jobservice/config" ) const ( @@ -460,6 +462,14 @@ func (m *ManifestPusher) enter() (string, error) { func newRepositoryClient(endpoint string, insecure bool, credential auth.Credential, repository, scopeType, scopeName string, scopeActions ...string) (*registry.Repository, error) { + domain, err := config.DomainName() + if err != nil { + return nil, err + } + if err := os.Setenv("DOMAIN_NAME", domain); err != nil { + return nil, err + } + authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, scopeType, scopeName, scopeActions...) store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer) diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index b229c0e96..19de01090 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -19,6 +19,7 @@ import ( "fmt" "io/ioutil" "net/http" + "os" "sort" "github.com/docker/distribution/manifest/schema1" @@ -442,6 +443,15 @@ func newRepositoryClient(endpoint string, insecure bool, username, password, rep scopeActions ...string) (*registry.Repository, error) { credential := auth.NewBasicAuthCredential(username, password) + + domain, err := config.DomainName() + if err != nil { + return nil, err + } + if err := os.Setenv("DOMAIN_NAME", domain); err != nil { + return nil, err + } + authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, scopeType, scopeName, scopeActions...) store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer) diff --git a/src/ui/api/target.go b/src/ui/api/target.go index becda414e..af3888f37 100644 --- a/src/ui/api/target.go +++ b/src/ui/api/target.go @@ -20,6 +20,7 @@ import ( "net" "net/http" "net/url" + "os" "strconv" "github.com/vmware/harbor/src/common/api" @@ -340,6 +341,15 @@ func (t *TargetAPI) Delete() { func newRegistryClient(endpoint string, insecure bool, username, password, scopeType, scopeName string, scopeActions ...string) (*registry.Registry, error) { credential := auth.NewBasicAuthCredential(username, password) + + domain, err := config.DomainName() + if err != nil { + return nil, err + } + if err := os.Setenv("DOMAIN_NAME", domain); err != nil { + return nil, err + } + authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, scopeType, scopeName, scopeActions...) store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer) From 6dc6b4fa79de79d60745ca001076ded2d9a39281 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 10 Feb 2017 16:36:16 +0800 Subject: [PATCH 12/22] update Makefile --- Makefile | 42 +++++++++++----- docs/compile_guide.md | 8 ---- make/ubuntu/Makefile | 79 ------------------------------- make/ubuntu/jobservice/Dockerfile | 11 ----- make/ubuntu/log/Dockerfile | 18 ------- make/ubuntu/ui/Dockerfile | 26 ---------- src/common/dao/config.go | 4 +- src/common/dao/config_test.go | 2 +- 8 files changed, 33 insertions(+), 157 deletions(-) delete mode 100644 make/ubuntu/Makefile delete mode 100644 make/ubuntu/jobservice/Dockerfile delete mode 100644 make/ubuntu/log/Dockerfile delete mode 100644 make/ubuntu/ui/Dockerfile diff --git a/Makefile b/Makefile index 974118b9c..cd4fd9b98 100644 --- a/Makefile +++ b/Makefile @@ -4,18 +4,17 @@ # # all: prepare env, compile binarys, build images and install images # prepare: prepare env -# compile: compile ui and jobservice code +# compile: compile adminserver, ui and jobservice code # # compile_golangimage: # compile from golang image # for example: make compile_golangimage -e GOBUILDIMAGE= \ # golang:1.7.3 -# compile_ui, compile_jobservice: compile specific binary +# compile_adminserver, compile_ui, compile_jobservice: compile specific binary # # build: build Harbor docker images (defuault: build_photon) # for example: make build -e BASEIMAGE=photon -# build_photon: build Harbor docker images from photon bsaeimage -# build_ubuntu: build Harbor docker images from ubuntu baseimage +# build_photon: build Harbor docker images from photon baseimage # # install: include compile binarys, build images, prepare specific \ # version composefile and startup Harbor instance @@ -46,7 +45,7 @@ # # clean: remove binary, Harbor images, specific version docker-compose \ # file, specific version tag and online/offline install package -# cleanbinary: remove ui and jobservice binary +# cleanbinary: remove adminserver, ui and jobservice binary # cleanimage: remove Harbor images # cleandockercomposefile: # remove specific version docker-compose @@ -100,14 +99,19 @@ GOBUILDIMAGE=reg.mydomain.com/library/harborgo[:tag] GOBUILDPATH=$(GOBASEPATH)/harbor GOIMAGEBUILDCMD=/usr/local/go/bin/go GOIMAGEBUILD=$(GOIMAGEBUILDCMD) build +GOBUILDPATH_ADMINSERVER=$(GOBUILDPATH)/src/adminserver GOBUILDPATH_UI=$(GOBUILDPATH)/src/ui GOBUILDPATH_JOBSERVICE=$(GOBUILDPATH)/src/jobservice GOBUILDMAKEPATH=$(GOBUILDPATH)/make +GOBUILDMAKEPATH_ADMINSERVER=$(GOBUILDMAKEPATH)/dev/adminserver GOBUILDMAKEPATH_UI=$(GOBUILDMAKEPATH)/dev/ui GOBUILDMAKEPATH_JOBSERVICE=$(GOBUILDMAKEPATH)/dev/jobservice GOLANGDOCKERFILENAME=Dockerfile.golang # binary +ADMINSERVERSOURCECODE=$(SRCPATH)/adminserver +ADMINSERVERBINARYPATH=$(MAKEDEVPATH)/adminserver +ADMINSERVERBINARYNAME=harbor_adminserver UISOURCECODE=$(SRCPATH)/ui UIBINARYPATH=$(MAKEDEVPATH)/ui UIBINARYNAME=harbor_ui @@ -125,7 +129,6 @@ CONFIGFILE=harbor.cfg # makefile MAKEFILEPATH_PHOTON=$(MAKEPATH)/photon -MAKEFILEPATH_UBUNTU=$(MAKEPATH)/ubuntu # common dockerfile DOCKERFILEPATH_COMMON=$(MAKEPATH)/common @@ -133,6 +136,7 @@ DOCKERFILEPATH_DB=$(DOCKERFILEPATH_COMMON)/db DOCKERFILENAME_DB=Dockerfile # docker image name +DOCKERIMAGENAME_ADMINSERVER=vmware/harbor-adminserver DOCKERIMAGENAME_UI=vmware/harbor-ui DOCKERIMAGENAME_JOBSERVICE=vmware/harbor-jobservice DOCKERIMAGENAME_LOG=vmware/harbor-log @@ -177,6 +181,11 @@ version: check_environment: @$(MAKEPATH)/$(CHECKENVCMD) +compile_adminserver: + @echo "compiling binary for adminserver..." + @$(GOBUILD) -o $(ADMINSERVERBINARYPATH)/$(ADMINSERVERBINARYNAME) $(ADMINSERVERSOURCECODE) + @echo "Done." + compile_ui: @echo "compiling binary for ui..." @$(GOBUILD) -o $(UIBINARYPATH)/$(UIBINARYNAME) $(UISOURCECODE) @@ -187,9 +196,15 @@ compile_jobservice: @$(GOBUILD) -o $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) $(JOBSERVICESOURCECODE) @echo "Done." -compile_normal: compile_ui compile_jobservice +compile_normal: compile_adminserver compile_ui compile_jobservice compile_golangimage: + @echo "compiling binary for adminserver (golang image)..." + @echo $(GOBASEPATH) + @echo $(GOBUILDPATH) + @$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_ADMINSERVER) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -v -o $(GOBUILDMAKEPATH_ADMINSERVER)/$(ADMINSERVERBINARYNAME) + @echo "Done." + @echo "compiling binary for ui (golang image)..." @echo $(GOBASEPATH) @echo $(GOBUILDPATH) @@ -214,9 +229,6 @@ build_common: version build_photon: build_common make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) -build_ubuntu: build_common - make -f $(MAKEFILEPATH_UBUNTU)/Makefile build -e DEVFLAG=$(DEVFLAG) - build: build_$(BASEIMAGE) modify_composefile: @@ -240,7 +252,6 @@ package_online: modify_composefile @cp NOTICE $(HARBORPKG)/NOTICE @$(TARCMD) -zcvf harbor-online-installer-$(VERSIONTAG).tgz \ --exclude=$(HARBORPKG)/common/db --exclude=$(HARBORPKG)/common/config\ - --exclude=$(HARBORPKG)/common/log --exclude=$(HARBORPKG)/ubuntu \ --exclude=$(HARBORPKG)/photon --exclude=$(HARBORPKG)/kubernetes \ --exclude=$(HARBORPKG)/dev --exclude=$(DOCKERCOMPOSETPLFILENAME) \ --exclude=$(HARBORPKG)/checkenv.sh \ @@ -264,6 +275,7 @@ package_offline: compile build modify_composefile @echo "saving harbor docker image" @$(DOCKERSAVE) -o $(HARBORPKG)/$(DOCKERIMGFILE).$(VERSIONTAG).tgz \ + $(DOCKERIMAGENAME_ADMINSERVER):$(VERSIONTAG) \ $(DOCKERIMAGENAME_UI):$(VERSIONTAG) \ $(DOCKERIMAGENAME_LOG):$(VERSIONTAG) \ $(DOCKERIMAGENAME_DB):$(VERSIONTAG) \ @@ -272,7 +284,6 @@ package_offline: compile build modify_composefile @$(TARCMD) -zcvf harbor-offline-installer-$(VERSIONTAG).tgz \ --exclude=$(HARBORPKG)/common/db --exclude=$(HARBORPKG)/common/config\ - --exclude=$(HARBORPKG)/common/log --exclude=$(HARBORPKG)/ubuntu \ --exclude=$(HARBORPKG)/photon --exclude=$(HARBORPKG)/kubernetes \ --exclude=$(HARBORPKG)/dev --exclude=$(DOCKERCOMPOSETPLFILENAME) \ --exclude=$(HARBORPKG)/checkenv.sh \ @@ -285,6 +296,11 @@ package_offline: compile build modify_composefile pushimage: @echo "pushing harbor images ..." + @$(DOCKERTAG) $(DOCKERIMAGENAME_ADMINSERVER):$(VERSIONTAG) $(REGISTRYSERVER)$(DOCKERIMAGENAME_ADMINSERVER):$(VERSIONTAG) + @$(PUSHSCRIPTPATH)/$(PUSHSCRIPTNAME) $(REGISTRYSERVER)$(DOCKERIMAGENAME_ADMINSERVER):$(VERSIONTAG) \ + $(REGISTRYUSER) $(REGISTRYPASSWORD) $(REGISTRYSERVER) + @$(DOCKERRMIMAGE) $(REGISTRYSERVER)$(DOCKERIMAGENAME_ADMINSERVER):$(VERSIONTAG) + @$(DOCKERTAG) $(DOCKERIMAGENAME_UI):$(VERSIONTAG) $(REGISTRYSERVER)$(DOCKERIMAGENAME_UI):$(VERSIONTAG) @$(PUSHSCRIPTPATH)/$(PUSHSCRIPTNAME) $(REGISTRYSERVER)$(DOCKERIMAGENAME_UI):$(VERSIONTAG) \ $(REGISTRYUSER) $(REGISTRYPASSWORD) $(REGISTRYSERVER) @@ -317,11 +333,13 @@ down: cleanbinary: @echo "cleaning binary..." + @if [ -f $(ADMINSERVERBINARYPATH)/$(ADMINSERVERBINARYNAME) ] ; then rm $(ADMINSERVERBINARYPATH)/$(ADMINSERVERBINARYNAME) ; fi @if [ -f $(UIBINARYPATH)/$(UIBINARYNAME) ] ; then rm $(UIBINARYPATH)/$(UIBINARYNAME) ; fi @if [ -f $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) ] ; then rm $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) ; fi cleanimage: @echo "cleaning image for photon..." + - $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_ADMINSERVER):$(VERSIONTAG) - $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_UI):$(VERSIONTAG) - $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_DB):$(VERSIONTAG) - $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_JOBSERVICE):$(VERSIONTAG) diff --git a/docs/compile_guide.md b/docs/compile_guide.md index 76a973647..7459a97d0 100644 --- a/docs/compile_guide.md +++ b/docs/compile_guide.md @@ -127,7 +127,6 @@ compile_ui | compile ui binary compile_jobservice | compile jobservice binary build | build Harbor docker images (default: using build_photon) build_photon | build Harbor docker images from Photon OS base image -build_ubuntu | build Harbor docker images from Ubuntu base image install | compile binaries, build images, prepare specific version of compose file and startup Harbor instance start | startup Harbor instance down | shutdown Harbor instance @@ -143,13 +142,6 @@ cleanpackage | remove online/offline install package #### EXAMPLE: -#### Build Harbor images based on Ubuntu - - ```sh - $ make build -e BASEIMAGE=ubuntu - - ``` - #### Push Harbor images to specific registry server ```sh diff --git a/make/ubuntu/Makefile b/make/ubuntu/Makefile deleted file mode 100644 index 81fac2acd..000000000 --- a/make/ubuntu/Makefile +++ /dev/null @@ -1,79 +0,0 @@ -# Makefile for a harbor project -# -# Targets: -# -# build: build harbor ubuntu images -# clean: clean ui and jobservice harbor images - -# common -SHELL := /bin/bash -BUILDPATH=$(CURDIR) -MAKEPATH=$(BUILDPATH)/make -MAKEDEVPATH=$(MAKEPATH)/dev -SRCPATH=./src -TOOLSPATH=$(BUILDPATH)/tools -CHECKENVCMD=checkenv.sh -DEVFLAG=true - -# docker parameters -DOCKERCMD=$(shell which docker) -DOCKERBUILD=$(DOCKERCMD) build -DOCKERRMIMAGE=$(DOCKERCMD) rmi -DOCKERIMASES=$(DOCKERCMD) images - -# binary -UISOURCECODE=$(SRCPATH)/ui -UIBINARYPATH=$(MAKEDEVPATH)/ui -UIBINARYNAME=harbor_ui -JOBSERVICESOURCECODE=$(SRCPATH)/jobservice -JOBSERVICEBINARYPATH=$(MAKEDEVPATH)/jobservice -JOBSERVICEBINARYNAME=harbor_jobservice - -# ubuntu dockerfile -DOCKERFILEPATH=$(MAKEPATH)/ubuntu -DOCKERFILEPATH_UI=$(DOCKERFILEPATH)/ui -DOCKERFILENAME_UI=Dockerfile -DOCKERIMAGENAME_UI=vmware/harbor-ui -DOCKERFILEPATH_JOBSERVICE=$(DOCKERFILEPATH)/jobservice -DOCKERFILENAME_JOBSERVICE=Dockerfile -DOCKERIMAGENAME_JOBSERVICE=vmware/harbor-jobservice -DOCKERFILEPATH_LOG=$(DOCKERFILEPATH)/log -DOCKERFILENAME_LOG=Dockerfile -DOCKERIMAGENAME_LOG=vmware/harbor-log - -# version prepare -VERSIONFILEPATH=$(SRCPATH)/views/sections -VERSIONFILENAME=header-content.htm -GITCMD=$(shell which git) -GITTAG=$(GITCMD) describe --tags -ifeq ($(DEVFLAG), true) - VERSIONTAG=dev -else - VERSIONTAG=$(shell $(GITTAG)) -endif - -check_environment: - @$(MAKEPATH)/$(CHECKENVCMD) - -build: - @echo "building ui container for ubuntu..." - $(DOCKERBUILD) -f $(DOCKERFILEPATH_UI)/$(DOCKERFILENAME_UI) -t $(DOCKERIMAGENAME_UI):$(VERSIONTAG) . - @echo "Done." - - @echo "building jobservice container for ubuntu..." - $(DOCKERBUILD) -f $(DOCKERFILEPATH_JOBSERVICE)/$(DOCKERFILENAME_JOBSERVICE) -t $(DOCKERIMAGENAME_JOBSERVICE):$(VERSIONTAG) . - @echo "Done." - - @echo "building log container for ubuntu..." - $(DOCKERBUILD) -f $(DOCKERFILEPATH_LOG)/$(DOCKERFILENAME_LOG) -t $(DOCKERIMAGENAME_LOG):$(VERSIONTAG) . - @echo "Done." - -cleanimage: - @echo "cleaning image for ubuntu..." - - $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_UI):$(VERSIONTAG) - - $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_JOBSERVICE):$(VERSIONTAG) - - $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_LOG):$(VERSIONTAG) - -.PHONY: clean -clean: cleanimage - diff --git a/make/ubuntu/jobservice/Dockerfile b/make/ubuntu/jobservice/Dockerfile deleted file mode 100644 index 29814703d..000000000 --- a/make/ubuntu/jobservice/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM library/ubuntu:14.04 - -MAINTAINER jiangd@vmware.com - -RUN mkdir /harbor/ -COPY ./make/dev/jobservice/harbor_jobservice /harbor/ - -RUN chmod u+x /harbor/harbor_jobservice - -WORKDIR /harbor/ -ENTRYPOINT ["/harbor/harbor_jobservice"] diff --git a/make/ubuntu/log/Dockerfile b/make/ubuntu/log/Dockerfile deleted file mode 100644 index 3e1c7520e..000000000 --- a/make/ubuntu/log/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM library/ubuntu:14.04 - -RUN rm /etc/rsyslog.d/* && rm /etc/rsyslog.conf - -ADD make/common/log/rsyslog.conf /etc/rsyslog.conf - -# rotate logs weekly -# notes: file name cannot contain dot, or the script will not run -ADD make/common/log/rotate.sh /etc/cron.weekly/rotate - -# rsyslog configuration file for docker -ADD make/common/log/rsyslog_docker.conf /etc/rsyslog.d/ - -VOLUME /var/log/docker/ - -EXPOSE 514 - -CMD cron && rsyslogd -n diff --git a/make/ubuntu/ui/Dockerfile b/make/ubuntu/ui/Dockerfile deleted file mode 100644 index f44d4fdf9..000000000 --- a/make/ubuntu/ui/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM library/ubuntu:14.04 - -MAINTAINER jiangd@vmware.com - -ENV MYSQL_USR root \ - MYSQL_PWD root \ - REGISTRY_URL localhost:5000 - -RUN mkdir /harbor/ -COPY ./make/dev/ui/harbor_ui /harbor/ - -COPY ./src/ui/views /harbor/views -COPY ./src/ui/static /harbor/static -COPY ./src/favicon.ico /harbor/favicon.ico -COPY ./make/jsminify.sh /tmp/jsminify.sh - -RUN chmod u+x /harbor/harbor_ui \ - && timestamp=`date '+%s'` \ - && /tmp/jsminify.sh /harbor/views/sections/script-include.htm /harbor/static/resources/js/harbor.app.min.$timestamp.js /harbor/ \ - && sed -i "s/harbor\.app\.min\.js/harbor\.app\.min\.$timestamp\.js/g" /harbor/views/sections/script-min-include.htm - -WORKDIR /harbor/ -ENTRYPOINT ["/harbor/harbor_ui"] - -EXPOSE 80 - diff --git a/src/common/dao/config.go b/src/common/dao/config.go index eb700c8af..a0a2ab1fd 100644 --- a/src/common/dao/config.go +++ b/src/common/dao/config.go @@ -27,6 +27,6 @@ func AuthModeCanBeModified() (bool, error) { if err != nil { return false, err } - - return c == 1, nil + // admin and anonymous + return c == 2, nil } diff --git a/src/common/dao/config_test.go b/src/common/dao/config_test.go index a874a6c6e..b8eb43721 100644 --- a/src/common/dao/config_test.go +++ b/src/common/dao/config_test.go @@ -27,7 +27,7 @@ func TestAuthModeCanBeModified(t *testing.T) { t.Fatalf("failed to count users: %v", err) } - if c == 1 { + if c == 2 { flag, err := AuthModeCanBeModified() if err != nil { t.Fatalf("failed to determine whether auth mode can be modified: %v", err) From 421288046ed36c7fb0b45c2de6858b0fa3c462cd Mon Sep 17 00:00:00 2001 From: kunw Date: Mon, 13 Feb 2017 13:13:49 +0800 Subject: [PATCH 13/22] Add configuration features to UI. --- src/ui/static/resources/css/admin-options.css | 6 +- .../configuration.directive.html | 267 +++++++++++--- .../configuration.directive.js | 348 +++++++++++++++++- .../system-management.directive.js | 1 + .../navigation-admin-options.directive.html | 8 +- .../navigation-admin-options.directive.js | 13 +- .../navigation-details.directive.js | 11 +- .../js/services/i18n/locale_messages_en-US.js | 54 ++- .../js/services/i18n/locale_messages_zh-CN.js | 54 ++- .../services.system-configuration.js | 43 +++ src/ui/views/navigation-detail.htm | 2 +- src/ui/views/sections/script-include.htm | 1 + 12 files changed, 725 insertions(+), 83 deletions(-) create mode 100644 src/ui/static/resources/js/services/system-info/services.system-configuration.js diff --git a/src/ui/static/resources/css/admin-options.css b/src/ui/static/resources/css/admin-options.css index 2cd2966f7..a7830e15c 100644 --- a/src/ui/static/resources/css/admin-options.css +++ b/src/ui/static/resources/css/admin-options.css @@ -14,7 +14,7 @@ */ .switch-pane-admin-options { display: inline; - width: 245px; + width: 340px; float: right; list-style-type: none; } @@ -28,4 +28,8 @@ .switch-pane-admin-options li .active { border-bottom: 2px solid rgb(0, 84, 190); font-weight: bold; +} + +.inline-help-config { + padding: 6px; } \ No newline at end of file diff --git a/src/ui/static/resources/js/components/system-management/configuration.directive.html b/src/ui/static/resources/js/components/system-management/configuration.directive.html index 3bde51eb8..4e330755c 100644 --- a/src/ui/static/resources/js/components/system-management/configuration.directive.html +++ b/src/ui/static/resources/js/components/system-management/configuration.directive.html @@ -12,65 +12,224 @@ See the License for the specific language governing permissions and limitations under the License. --> -
-
-
System Settings
-
-
-
-
- -
- -
- Host name is required. +
+ + +
+
+ +
+ +
+ +
+
+ +
-
+
+ +
+ +
+
+ +
+
+
+ +
+ +
+
+ +
+
+
+ +
+ +
+
+ +
+
+
+ +
+ +
+
+ +
+
+
+ +
+ +
+
+ +
+
+
+ +
+ +
+
+ +
+
+
+ +
+ +
+
+ +
+
+
+ +
+ +
+ // 'timeout_is_required' | tr // + // 'invalid_timeout' | tr // + // 'invalid_timeout' | tr // +
+
+
+
+ +
+ +
+
+ +
+
+
+
+ // vm.pingMessage // + // vm.pingMessage // +
+
+
+
+
+ + + +
+
+
+
-
- -
- -
- Url protocol is required. +
+
+
+ +
+ +
+
+ +
-
-
-
- -
- -
- Email server is required. +
+ +
+ +
+ // 'invalid_port_number' | tr // + // 'invalid_port_number' | tr // +
+
+
+ +
-
+
+ +
+ +
+
+ +
+
+
+ +
+ +
+
+ +
+
+
+ +
+ +
+
+ +
+
+
+ +
+ +
+
+ +
+
+
+
+
+ + +
+
+
+
-
- -
- -
- LDAP URL is required. +
+
+
+ +
+ +
+
+ +
-
+
+ +
+ +
+
+ +
+
+
+
+
+ + +
+
+
+
-
-
Registration
-
-
-
-
- -
- -
-
-
-
- - -
-
-
- \ No newline at end of file +
\ No newline at end of file diff --git a/src/ui/static/resources/js/components/system-management/configuration.directive.js b/src/ui/static/resources/js/components/system-management/configuration.directive.js index bc21345ad..45db3711d 100644 --- a/src/ui/static/resources/js/components/system-management/configuration.directive.js +++ b/src/ui/static/resources/js/components/system-management/configuration.directive.js @@ -18,51 +18,365 @@ angular .module('harbor.system.management') + .constant('defaultPassword', '12345678') .directive('configuration', configuration); - ConfigurationController.$inject = []; + ConfigurationController.$inject = ['$scope', 'ConfigurationService', 'defaultPassword', '$filter', 'trFilter']; - function ConfigurationController() { + function ConfigurationController($scope, ConfigurationService, defaultPassword, $filter, trFilter) { var vm = this; - - vm.registrationOptions = [ - { - 'name': 'on', - 'value': true + + vm.toggleBooleans = [ + { + 'name': 'True', + 'value': true }, { - 'name': 'off', + 'name': 'False', 'value': false } ]; - vm.currentRegistration = { - 'name': 'on', - 'value': true + + vm.toggleCustoms = [ + { + 'name': 'Admin Only', + 'value': 'adminonly', + }, + { + 'name': 'Everyone', + 'value': 'everyone' + } + ]; + + vm.supportedAuths = [ + { + 'name': 'DB auth', + 'value': 'db_auth' + }, + { + 'name': 'LDAP auth', + 'value': 'ldap_auth' + } + ]; + + var confKeyDefinitions = { + 'auth_mode': { type: 'auth', attr: 'authMode' }, + 'self_registration': { type: 'auth', attr: 'selfRegistration' }, + 'ldap_url': { type: 'auth', attr: 'ldapURL' }, + 'ldap_search_dn': { type: 'auth', attr: 'ldapSearchDN' }, + 'ldap_search_password': { type: 'auth', attr: 'ldapSearchPassword' }, + 'ldap_base_dn': { type: 'auth', attr: 'ldapBaseDN' }, + 'ldap_uid': { type: 'auth', attr: 'ldapUID' }, + 'ldap_filter': { type: 'auth', attr: 'ldapFilter' }, + 'ldap_connection_timeout': { type: 'auth', attr: 'ldapConnectionTimeout' }, + 'ldap_scope': { type: 'auth', attr: 'ldapScope' }, + 'email_host': { type: 'email', attr: 'server' }, + 'email_port': { type: 'email', attr: 'serverPort' }, + 'email_username': { type: 'email', attr: 'username' }, + 'email_password': { type: 'email', attr: 'password' }, + 'email_from': { type: 'email', attr: 'from' }, + 'email_ssl': { type: 'email', attr: 'SSL' }, + 'project_creation_restriction': { type: 'system', attr: 'projectCreationRestriction' }, + 'verify_remote_cert': { type: 'system', attr: 'verifyRemoteCert' } }; - vm.changeSettings = changeSettings; + $scope.auth = {}; + $scope.email = {}; + $scope.system = {}; - vm.selectRegistration = selectRegistration; + vm.retrieve = retrieve; - function selectRegistration() { + vm.saveAuthConf = saveAuthConf; + vm.saveEmailConf = saveEmailConf; + vm.saveSystemConf = saveSystemConf; + + vm.gatherUpdateItems = gatherUpdateItems; + vm.clearUp = clearUp; + vm.hasChanged = hasChanged; + vm.setMaskPassword = setMaskPassword; + vm.undo = undo; + + vm.pingLDAP = pingLDAP; + vm.pingTIP = false; + vm.isError = false; + vm.pingMessage = ''; + + vm.retrieve(); + + function retrieve() { + + vm.ldapSearchPasswordChanged = false; + vm.emailPasswordChanged = false; + vm.changedItems = {}; + vm.updatedItems = {}; + vm.warning = {}; + vm.editable = {}; + ConfigurationService + .get() + .then(getConfigurationSuccess, getConfigurationFailed); + } + + function getConfigurationSuccess(response) { + var data = response.data || []; + for(var key in data) { + var mappedDef = keyMapping(key); + if(mappedDef) { + $scope[mappedDef['type']][mappedDef['attr']] = { 'target': mappedDef['type'] + '.' + mappedDef['attr'], 'data': valueMapping(data[key]['value']) }; + $scope.$watch(mappedDef['type'] + '.' + mappedDef['attr'], onChangedCallback, true); + $scope[mappedDef['type']][mappedDef['attr']]['origin'] = { 'target': mappedDef['type'] + '.' + mappedDef['attr'], 'data': valueMapping(data[key]['value']) }; + vm.editable[mappedDef['type'] + '.' + mappedDef['attr']] = data[key]['editable']; + } + } + + $scope.auth.ldapSearchPassword = { 'target': 'auth.ldapSearchPassword', 'data': defaultPassword}; + $scope.email.password = { 'target': 'email.password', 'data': defaultPassword}; + + $scope.$watch('auth.ldapSearchPassword', onChangedCallback, true); + $scope.$watch('email.password', onChangedCallback, true); + + $scope.auth.ldapSearchPassword.actual = { 'target': 'auth.ldapSearchPassword', 'data': ''}; + $scope.email.password.actual = { 'target': 'email.password', 'data': ''}; + } + + function keyMapping(confKey) { + for (var key in confKeyDefinitions) { + if (confKey === key) { + return confKeyDefinitions[key]; + } + } + return null; } - function changeSettings(system) { - console.log(system); + function valueMapping(value) { + switch(value) { + case true: + return vm.toggleBooleans[0]; + case false: + return vm.toggleBooleans[1]; + case 'db_auth': + return vm.supportedAuths[0]; + case 'ldap_auth': + return vm.supportedAuths[1]; + case 'adminonly': + return vm.toggleCustoms[0]; + case 'everyone': + return vm.toggleCustoms[1]; + default: + return value; + } } + + function onChangedCallback(current, previous) { + if(!angular.equals(current, previous)) { + var compositeKey = current.target.split("."); + vm.changed = false; + var changedData = {}; + switch(current.target) { + case 'auth.ldapSearchPassword': + if(vm.ldapSearchPasswordChanged) { + vm.changed = true; + changedData = $scope.auth.ldapSearchPassword.actual.data; + } + break; + case 'email.password': + if(vm.emailPasswordChanged) { + vm.changed = true; + changedData = $scope.email.password.actual.data; + } + break; + default: + if(!angular.equals(current.data, $scope[compositeKey[0]][compositeKey[1]]['origin']['data'])) { + vm.changed = true; + changedData = current.data; + } + } + if(vm.changed) { + vm.changedItems[current.target] = changedData; + vm.warning[current.target] = true; + } else { + delete vm.changedItems[current.target]; + vm.warning[current.target] = false; + } + } + } + + function getConfigurationFailed(response) { + console.log('Failed to get configurations.'); + } + + function updateConfigurationSuccess(response) { + $scope.$emit('modalTitle', $filter('tr')('update_configuration_title', [])); + $scope.$emit('modalMessage', $filter('tr')('successful_update_configuration', [])); + var emitInfo = { + 'confirmOnly': true, + 'contentType': 'text/plain', + 'action' : function() { + vm.retrieve(); + } + }; + $scope.$emit('raiseInfo', emitInfo); + console.log('Updated system configuration successfully.'); + } + + function updateConfigurationFailed() { + $scope.$emit('modalTitle', $filter('tr')('update_configuration_title', [])); + $scope.$emit('modalMessage', $filter('tr')('failed_to_update_configuration', [])); + $scope.$emit('raiseError', true); + console.log('Failed to update system configurations.'); + } + + function gatherUpdateItems() { + vm.updatedItems = {}; + for(var key in confKeyDefinitions) { + var value = confKeyDefinitions[key]; + var compositeKey = value.type + '.' + value.attr; + for(var itemKey in vm.changedItems) { + var item = vm.changedItems[itemKey]; + if (compositeKey === itemKey) { + (typeof item === 'object' && item) ? vm.updatedItems[key] = ((typeof item.value === 'boolean') ? Number(item.value) + '' : item.value) : vm.updatedItems[key] = String(item) || ''; + } + } + } + } + + function saveAuthConf(auth) { + vm.gatherUpdateItems(); + console.log('auth changed:' + angular.toJson(vm.updatedItems)); + ConfigurationService + .update(vm.updatedItems) + .then(updateConfigurationSuccess, updateConfigurationFailed); + } + + function saveEmailConf(email) { + vm.gatherUpdateItems(); + console.log('email changed:' + angular.toJson(vm.updatedItems)); + ConfigurationService + .update(vm.updatedItems) + .then(updateConfigurationSuccess, updateConfigurationFailed); + } + + function saveSystemConf(system) { + vm.gatherUpdateItems(); + console.log('system changed:' + angular.toJson(vm.updatedItems)); + ConfigurationService + .update(vm.updatedItems) + .then(updateConfigurationSuccess, updateConfigurationFailed); + } + + function clearUp(input) { + switch(input.target) { + case 'auth.ldapSearchPassword': + $scope.auth.ldapSearchPassword.data = ''; + break; + case 'email.password': + $scope.email.password.data = ''; + break; + } + } + + function hasChanged(input) { + switch(input.target) { + case 'auth.ldapSearchPassword': + vm.ldapSearchPasswordChanged = true; + $scope.auth.ldapSearchPassword.actual.data = input.data; + break; + case 'email.password': + vm.emailPasswordChanged = true; + $scope.email.password.actual.data = input.data; + break; + } + } + + function setMaskPassword(input) { + input.data = defaultPassword; + } + + function undo() { + vm.retrieve(); + } + + function pingLDAP(auth) { + var keyset = [ + {'name': 'ldapURL' , 'attr': 'ldap_url'}, + {'name': 'ldapSearchDN', 'attr': 'ldap_search_dn'}, + {'name': 'ldapSearchPassword' , 'attr': 'ldap_search_password'}, + {'name': 'ldapConnectionTimeout', 'attr': 'ldap_connection_timeout'} + ]; + var ldapConf = {}; + + for(var i = 0; i < keyset.length; i++) { + var key = keyset[i]; + var value; + if(key.name === 'ldapSearchPassword') { + value = auth[key.name]['actual']['data']; + }else { + value = auth[key.name]['data']; + } + ldapConf[key.attr] = value; + } + + vm.pingMessage = $filter('tr')('pinging_target'); + vm.pingTIP = true; + vm.isError = false; + + ConfigurationService + .pingLDAP(ldapConf) + .then(pingLDAPSuccess, pingLDAPFailed); + } + + function pingLDAPSuccess(response) { + vm.pingTIP = false; + vm.pingMessage = $filter('tr')('successful_ping_target'); + } + + function pingLDAPFailed(response) { + vm.isError = true; + vm.pingTIP = false; + vm.pingMessage = $filter('tr')('failed_to_ping_target'); + console.log('Failed to ping LDAP target:' + response.data); + } + } - function configuration() { + configuration.$inject = ['$filter', 'trFilter']; + + function configuration($filter, trFilter) { var directive = { 'restrict': 'E', 'templateUrl': '/static/resources/js/components/system-management/configuration.directive.html', 'scope': true, + 'link': link, 'controller': ConfigurationController, 'controllerAs': 'vm', 'bindToController': true }; return directive; + + function link(scope, element, attrs, ctrl) { + element.find('#ulTabHeader a').on('click', function(e) { + e.preventDefault(); + ctrl.gatherUpdateItems(); + if(!angular.equals(ctrl.updatedItems, {})) { + var emitInfo = { + 'confirmOnly': true, + 'contentType': 'text/html', + 'action' : function() { + return; + } + }; + scope.$emit('modalTitle', $filter('tr')('caution')); + scope.$emit('modalMessage', $filter('tr')('please_save_changes')); + scope.$emit('raiseInfo', emitInfo); + scope.$apply(); + e.stopPropagation(); + }else{ + $(this).tab('show'); + } + + }); + element.find('#ulTabHeader a:first').trigger('click'); + } } })(); \ No newline at end of file diff --git a/src/ui/static/resources/js/components/system-management/system-management.directive.js b/src/ui/static/resources/js/components/system-management/system-management.directive.js index 68d17938d..f04f54549 100644 --- a/src/ui/static/resources/js/components/system-management/system-management.directive.js +++ b/src/ui/static/resources/js/components/system-management/system-management.directive.js @@ -29,6 +29,7 @@ switch(currentTarget) { case 'destinations': case 'replication': + case 'configuration': $location.path('/' + currentTarget); vm.target = currentTarget; break; diff --git a/src/ui/static/resources/js/layout/navigation/navigation-admin-options.directive.html b/src/ui/static/resources/js/layout/navigation/navigation-admin-options.directive.html index ecbd0fafe..ab1641250 100644 --- a/src/ui/static/resources/js/layout/navigation/navigation-admin-options.directive.html +++ b/src/ui/static/resources/js/layout/navigation/navigation-admin-options.directive.html @@ -12,10 +12,8 @@ See the License for the specific language governing permissions and limitations under the License. --> -
    + \ No newline at end of file diff --git a/src/ui/static/resources/js/layout/navigation/navigation-admin-options.directive.js b/src/ui/static/resources/js/layout/navigation/navigation-admin-options.directive.js index 89de7e4a4..ffc293f46 100644 --- a/src/ui/static/resources/js/layout/navigation/navigation-admin-options.directive.js +++ b/src/ui/static/resources/js/layout/navigation/navigation-admin-options.directive.js @@ -27,7 +27,9 @@ vm.path = $location.path(); } - function navigationAdminOptions() { + navigationAdminOptions.$inject = ['I18nService']; + + function navigationAdminOptions(I18nService) { var directive = { 'restrict': 'E', 'templateUrl': '/static/resources/js/layout/navigation/navigation-admin-options.directive.html', @@ -44,7 +46,14 @@ function link(scope, element, attrs, ctrl) { var visited = ctrl.path.substring(1); console.log('visited:' + visited); - + + var lang = I18nService().getCurrentLanguage(); + ctrl.customPos = {}; + + if(lang === 'zh-CN') { + ctrl.customPos = {'position': 'relative', 'left': '14%'}; + } + if(visited) { element.find('a[tag="' + visited + '"]').addClass('active'); }else{ diff --git a/src/ui/static/resources/js/layout/navigation/navigation-details.directive.js b/src/ui/static/resources/js/layout/navigation/navigation-details.directive.js index 5d6b06d45..b11cc2623 100644 --- a/src/ui/static/resources/js/layout/navigation/navigation-details.directive.js +++ b/src/ui/static/resources/js/layout/navigation/navigation-details.directive.js @@ -35,7 +35,9 @@ vm.path = $location.path(); } - function navigationDetails() { + navigationDetails.$inject = ['I18nService']; + + function navigationDetails(I18nService) { var directive = { restrict: 'E', templateUrl: '/navigation_detail?timestamp=' + new Date().getTime(), @@ -53,6 +55,13 @@ function link(scope, element, attrs, ctrl) { + var lang = I18nService().getCurrentLanguage(); + ctrl.customPos = {}; + + if(lang === 'zh-CN') { + ctrl.customPos = {'position': 'relative', 'left': '8%'}; + } + var visited = ctrl.path.substring(1); if(visited) { diff --git a/src/ui/static/resources/js/services/i18n/locale_messages_en-US.js b/src/ui/static/resources/js/services/i18n/locale_messages_en-US.js index de5a02338..650f75aa5 100644 --- a/src/ui/static/resources/js/services/i18n/locale_messages_en-US.js +++ b/src/ui/static/resources/js/services/i18n/locale_messages_en-US.js @@ -287,5 +287,57 @@ var locale_messages = { 'confirm_to_toggle_enabled_policy': 'After enabling the replication policy, all repositories under the project will be replicated to the destination registry. Please confirm to continue.', 'confirm_to_toggle_disabled_policy_title': 'Disable Policy', 'confirm_to_toggle_disabled_policy': 'After disabling the policy, all unfinished replication jobs of this policy will be stopped and canceled. Please confirm to continue.', - 'begin_date_is_later_than_end_date': 'Begin date should not be later than end date.' + 'begin_date_is_later_than_end_date': 'Begin date should not be later than end date.', + 'configuration': 'Configuration', + 'authentication': 'Authentication', + 'email_settings': 'Email Settings', + 'system_settings': 'System Settings', + 'authentication_mode': 'Authentication Mode', + 'authentication_mode_desc': 'The default authentication mode is db_auth. Set it to ldap_auth when users\' credentials are stored in an LDAP or AD server. Note: this option can only be set once.', + 'self_registration': 'Self Registration', + 'self_registration_desc': 'Determine whether the self-registration is allowed or not. Set this to off to disable a user\'s self-registration in Harbor. This flag has no effect when users are stored in LDAP or AD.', + 'ldap_url': 'LDAP URL', + 'ldap_url_desc': 'The URL of an LDAP/AD server.', + 'ldap_search_dn': 'LDAP Search DN', + 'ldap_search_dn_desc': 'A user\'s DN who has the permission to search the LDAP/AD server. Leave blank if your LDAP/AD server supports anonymous search, otherwise you should configure this DN and LDAP Search Password.', + 'ldap_search_password': 'LDAP Search Password', + 'ldap_search_password_desc': 'The password of the user for LDAP search. Leave blank if your LDAP/AD server supports anonymous search.', + 'ldap_base_dn': 'LDAP Base DN', + 'ldap_base_dn_desc': 'The base DN of a node from which to look up a user for authentication. The search scope includes subtree of the node.', + 'ldap_uid': 'LDAP UID', + 'ldap_uid_desc': 'The attribute used in a search to match a user, it could be uid, cn, email, sAMAccountName or other attributes depending on your LDAP/AD server.', + 'ldap_filter': 'LDAP Filter', + 'ldap_filter_desc': 'Search filter for LDAP/AD, make sure the syntax of the filter is correct.', + 'ldap_connection_timeout': 'LDAP Connection Timeout(sec.)', + 'ldap_scope': 'LDAP Scope', + 'ldap_scope_desc': 'The scope to search for users.(1-LDAP_SCOPE_BASE, 2-LDAP_SCOPE_ONELEVEL, 3-LDAP_SCOPE_SUBTREE)', + 'email_server': 'Email Server', + 'email_server_desc': 'The mail server to send out emails to reset password.', + 'email_server_port': 'Email Server Port', + 'email_server_port_desc': 'The port of mail server.', + 'email_username': 'Email Username', + 'email_username_desc': 'The user from whom the password reset email is sent. Usually this is a system email address.', + 'email_password': 'Email Password', + 'email_password_desc': 'The password of the user from whom the password reset email is sent.', + 'email_from': 'Email From', + 'email_from_desc': 'The name of the email sender.', + 'email_ssl': 'Email SSL', + 'email_ssl_desc': 'Whether to enable secure mail transmission.', + 'project_creation_restriction': 'Project Creation Restriction', + 'project_creation_restriction_desc': 'The flag to control what users have permission to create projects. Be default everyone can create a project, set to "adminonly" such that only admin can create project.', + 'verify_remote_cert': 'Verify Remote Cert.', + 'verify_remote_cert_desc': 'Determine whether the image replication should verify the certificate of a remote Harbor registry. Set this flag to off when the remote registry uses a self-signed or untrusted certificate.', + 'max_job_workers': 'Max Job Workers', + 'max_job_workers_desc': 'Maximum number of job workers in job service.', + 'please_save_changes': 'Please save changes before leaving this page.', + 'undo': 'Undo', + 'invalid_port_number': 'Invalid port number.', + 'max_job_workers_is_required': 'Maxystem job workers number is required.', + 'timeout_is_required': 'Timeout value is required.', + 'invalid_timeout': 'Invalid timeout value.', + 'ldap_scope_is_required': 'Scope is required.', + 'invalid_ldap_scope': 'Invalid Scope value.', + 'update_configuration_title': 'Update Configuration', + 'successful_update_configuration': 'Update configurations successfully.', + 'failed_to_update_configuration': 'Failed to update configuration.' }; \ No newline at end of file diff --git a/src/ui/static/resources/js/services/i18n/locale_messages_zh-CN.js b/src/ui/static/resources/js/services/i18n/locale_messages_zh-CN.js index 451351edf..fbd1bcfc5 100644 --- a/src/ui/static/resources/js/services/i18n/locale_messages_zh-CN.js +++ b/src/ui/static/resources/js/services/i18n/locale_messages_zh-CN.js @@ -287,5 +287,57 @@ var locale_messages = { 'confirm_to_toggle_enabled_policy': '启用策略后，该项目下的所有镜像仓库将复制到目标实例。请确认继续。', 'confirm_to_toggle_disabled_policy_title': '停用策略', 'confirm_to_toggle_disabled_policy': '停用策略后，所有未完成的复制任务将被终止和取消。请确认继续。', - 'begin_date_is_later_than_end_date': '起始日期不能晚于结束日期。' + 'begin_date_is_later_than_end_date': '起始日期不能晚于结束日期。', + 'configuration': '设置', + 'authentication': '认证设置', + 'email_settings': '邮箱设置', + 'system_settings': '系统设置', + 'authentication_mode': '认证模式', + 'authentication_mode_desc': '默认的认证模式是： 数据库认证。 当用户信息存储在LDAP或AD服务器时，应设置为LDAP认证模式。 注意：该选项只能被设置一次。', + 'self_registration': '自注册', + 'self_registration_desc': '确定是否允许或禁止用户自注册。 关闭该项会禁用Harbor用户注册。 当用户数据存储在LDAP或AD时，该选项无效。 ', + 'ldap_url': 'LDAP URL', + 'ldap_url_desc': 'LDAP/AD服务URL。', + 'ldap_search_dn': 'LDAP Search DN', + 'ldap_search_dn_desc': '提供一个拥有检索LDAP/AD权限的用户DN。 如果LDAP/AD允许匿名访问该项可以置空， 否则需提供用户DN和密码。', + 'ldap_search_password': 'LDAP Search 密码', + 'ldap_search_password_desc': '检索LDAP的用户密码。 如果LDAP/AD允许匿名访问该项可以置空。', + 'ldap_base_dn': 'LDAP Base DN', + 'ldap_base_dn_desc': '用于查询认证用户的基本DN节点。 检索范围包含子树节点。', + 'ldap_uid': 'LDAP UID', + 'ldap_uid_desc': '该属性用于检索匹配用户， 可以是uid， cn, Email, sAMAccountName或是其他属性， 取决于LDAP/AD服务。', + 'ldap_filter': 'LDAP 过滤器', + 'ldap_filter_desc': '用于过滤LDAP/AP检索， 请确保正确语法形式。', + 'ldap_connection_timeout': 'LDAP连接超时(秒)', + 'ldap_scope': 'LDAP检索范围', + 'ldap_scope_desc': '指定用户检索范围。(1-LDAP_SCOPE_BASE, 2-LDAP_SCOPE_ONELEVEL, 3-LDAP_SCOPE_SUBTREE)', + 'email_server': '邮箱服务地址', + 'email_server_desc': '用以发送重置密码的邮箱服务地址。', + 'email_server_port': '邮箱服务端口号', + 'email_server_port_desc': '邮箱服务端口号', + 'email_username': '邮箱用户名', + 'email_username_desc': '发送重置密码邮箱的用户。 通常是系统邮箱地址。', + 'email_password': '邮箱密码', + 'email_password_desc': '发送重置密码邮箱的密码。', + 'email_from': '寄信人', + 'email_from_desc': '邮箱寄信人名。', + 'email_ssl': '邮箱SSL', + 'email_ssl_desc': '是否启用安全邮箱传输。', + 'project_creation_restriction': '项目创建约束', + 'project_creation_restriction_desc': '此标志用于控制用户是否有权限创建项目。 默认允许任何人创建项目， 当设置为"adminonly"只允许管理员创建项目。', + 'verify_remote_cert': '验证远程服务证书', + 'verify_remote_cert_desc': '确定镜像复制是否检查远程Harbor服务证书。 当使用非受信或自签名证书时应该关闭该检查。', + 'max_job_workers': '最大任务调度数', + 'max_job_workers_desc': '任务调度服务最大调度数。', + 'please_save_changes': '请在离开此页之前保存。', + 'undo': '撤销', + 'invalid_port_number': '无效的端口号。', + 'max_job_workers_is_required': '最大任务数为必填项。', + 'timeout_is_required': '超时时间为必填项。', + 'invalid_timeout': '无效的超时时间。', + 'ldap_scope_is_required': '范围值为必填项。', + 'invalid_ldap_scope': '无效的范围值。', + 'update_configuration_title': '修改设置', + 'successful_update_configuration': '修改设置成功。', + 'failed_to_update_configuration': '修改设置失败。' }; \ No newline at end of file diff --git a/src/ui/static/resources/js/services/system-info/services.system-configuration.js b/src/ui/static/resources/js/services/system-info/services.system-configuration.js new file mode 100644 index 000000000..8c168cb90 --- /dev/null +++ b/src/ui/static/resources/js/services/system-info/services.system-configuration.js @@ -0,0 +1,43 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +(function() { + + 'use strict'; + angular.module('harbor.services.system.info') + .service('ConfigurationService', ConfigurationService); + + ConfigurationService.$inject = ['$http', '$q', '$timeout']; + + function ConfigurationService($http, $q, $timeout) { + this.get = get; + this.update = update; + this.pingLDAP = pingLDAP; + + function get() { + return $http.get('/api/configurations'); + } + + function update(updates) { + return $http.put('/api/configurations', updates); + } + + function pingLDAP(ldapConf) { + return $http + .post('/api/ldap/ping', ldapConf); + } + + } + +})(); \ No newline at end of file diff --git a/src/ui/views/navigation-detail.htm b/src/ui/views/navigation-detail.htm index 9fc68cfad..ad6bdd107 100644 --- a/src/ui/views/navigation-detail.htm +++ b/src/ui/views/navigation-detail.htm @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. --> -
      +
      • // 'repositories' | tr //|
        • {{ if eq .IsAdmin 1 }}
      • // 'replication' | tr //|
        • diff --git a/src/ui/views/sections/script-include.htm b/src/ui/views/sections/script-include.htm index 187387a07..e8d3d9d1a 100644 --- a/src/ui/views/sections/script-include.htm +++ b/src/ui/views/sections/script-include.htm @@ -112,6 +112,7 @@ + From 06519bb3f245f6b74b6fbbe7679aef40517352c8 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Mon, 13 Feb 2017 17:05:25 +0800 Subject: [PATCH 14/22] update --- make/dev/docker-compose.yml | 2 +- src/adminserver/systemcfg/systemcfg.go | 2 +- src/common/config/config.go | 2 +- .../utils/registry/auth/tokenauthorizer.go | 28 ++++--- .../registry/auth/tokenauthorizer_test.go | 2 +- src/common/utils/test/adminserver.go | 84 ++++++++++--------- src/jobservice/config/config.go | 11 ++- src/jobservice/config/config_test.go | 12 ++- src/jobservice/replication/transfer.go | 12 +-- src/ui/api/config.go | 71 +++++++++++----- src/ui/api/repository.go | 12 +-- src/ui/api/target.go | 12 +-- src/ui/config/config.go | 11 ++- src/ui/config/config_test.go | 18 ++-- src/ui/controllers/password.go | 2 +- src/ui/controllers/repository.go | 2 +- src/ui/service/token/authutils.go | 2 +- 17 files changed, 166 insertions(+), 119 deletions(-) diff --git a/make/dev/docker-compose.yml b/make/dev/docker-compose.yml index c94a2d875..716f642ba 100644 --- a/make/dev/docker-compose.yml +++ b/make/dev/docker-compose.yml @@ -3,7 +3,7 @@ services: log: build: context: ../../ - dockerfile: make/ubuntu/log/Dockerfile + dockerfile: make/photon/log/Dockerfile restart: always volumes: - /var/log/harbor/:/var/log/docker/ diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index d1d45c203..4d22adc22 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -78,7 +78,7 @@ func getCfgStore() string { //read the following attrs from env every time boots up func readFromEnv(cfg map[string]interface{}) error { - cfg[comcfg.DomainName] = os.Getenv("EXT_ENDPOINT") + cfg[comcfg.ExtEndpoint] = os.Getenv("EXT_ENDPOINT") cfg[comcfg.DatabaseType] = os.Getenv("DATABASE_TYPE") cfg[comcfg.MySQLHost] = os.Getenv("MYSQL_HOST") diff --git a/src/common/config/config.go b/src/common/config/config.go index 295564d66..29c59f948 100644 --- a/src/common/config/config.go +++ b/src/common/config/config.go @@ -40,7 +40,7 @@ const ( LDAPScopeOnelevel = "2" LDAPScopeSubtree = "3" - DomainName = "domain_name" + ExtEndpoint = "ext_endpoint" AUTHMode = "auth_mode" DatabaseType = "database_type" MySQLHost = "mysql_host" diff --git a/src/common/utils/registry/auth/tokenauthorizer.go b/src/common/utils/registry/auth/tokenauthorizer.go index 0543ef524..b309689c3 100644 --- a/src/common/utils/registry/auth/tokenauthorizer.go +++ b/src/common/utils/registry/auth/tokenauthorizer.go @@ -21,7 +21,6 @@ import ( "io/ioutil" "net/http" "net/url" - "os" "strings" "sync" "time" @@ -134,19 +133,24 @@ func (t *tokenAuthorizer) updateCachedToken(token string, expiresIn int) { // Implements interface Authorizer type standardTokenAuthorizer struct { tokenAuthorizer - client *http.Client - credential Credential + client *http.Client + credential Credential + tokenServiceEndpoint string } // NewStandardTokenAuthorizer returns a standard token authorizer. The authorizer will request a token // from token server and add it to the origin request -func NewStandardTokenAuthorizer(credential Credential, insecure bool, scopeType, scopeName string, scopeActions ...string) Authorizer { +// If tokenServiceURL is set, the token request will be sent to it instead of the server get from authorizer +// The usage please refer to the function tokenURL +func NewStandardTokenAuthorizer(credential Credential, insecure bool, + tokenServiceEndpoint string, scopeType, scopeName string, scopeActions ...string) Authorizer { authorizer := &standardTokenAuthorizer{ client: &http.Client{ Transport: registry.GetHTTPTransport(insecure), Timeout: 30 * time.Second, }, - credential: credential, + credential: credential, + tokenServiceEndpoint: tokenServiceEndpoint, } if len(scopeType) != 0 || len(scopeName) != 0 { @@ -163,7 +167,7 @@ func NewStandardTokenAuthorizer(credential Credential, insecure bool, scopeType, } func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) { - realm = tokenURL(realm) + realm = s.tokenURL(realm) u, err := url.Parse(realm) if err != nil { @@ -230,17 +234,15 @@ func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes [] // when the registry client is used inside Harbor, the token request // can be posted to token service directly rather than going through nginx. -// this solution can resolve two problems: +// If realm is set as the internal url of token service, this can resolve +// two problems: // 1. performance issue // 2. the realm field returned by registry is an IP which can not reachable // inside Harbor -func tokenURL(realm string) string { - - domainName := os.Getenv("DOMAIN_NAME") - if len(domainName) != 0 && strings.Contains(realm, domainName) { - realm = "http://ui/service/token" +func (s *standardTokenAuthorizer) tokenURL(realm string) string { + if len(s.tokenServiceEndpoint) != 0 { + return s.tokenServiceEndpoint } - return realm } diff --git a/src/common/utils/registry/auth/tokenauthorizer_test.go b/src/common/utils/registry/auth/tokenauthorizer_test.go index 2d843d887..3a582bb64 100644 --- a/src/common/utils/registry/auth/tokenauthorizer_test.go +++ b/src/common/utils/registry/auth/tokenauthorizer_test.go @@ -40,7 +40,7 @@ func TestAuthorizeOfStandardTokenAuthorizer(t *testing.T) { }) defer server.Close() - authorizer := NewStandardTokenAuthorizer(nil, false, "repository", "library/ubuntu", "pull") + authorizer := NewStandardTokenAuthorizer(nil, false, "", "repository", "library/ubuntu", "pull") req, err := http.NewRequest("GET", "http://registry", nil) if err != nil { t.Fatalf("failed to create request: %v", err) diff --git a/src/common/utils/test/adminserver.go b/src/common/utils/test/adminserver.go index 18e1b91fd..93621d0fc 100644 --- a/src/common/utils/test/adminserver.go +++ b/src/common/utils/test/adminserver.go @@ -23,47 +23,53 @@ import ( "github.com/vmware/harbor/src/common/config" ) +var adminServerDefaultConfig = map[string]interface{}{ + config.ExtEndpoint: "host01.com", + config.AUTHMode: config.DBAuth, + config.DatabaseType: "mysql", + config.MySQLHost: "127.0.0.1", + config.MySQLPort: 3306, + config.MySQLUsername: "user01", + config.MySQLPassword: "password", + config.MySQLDatabase: "registry", + config.SQLiteFile: "/tmp/registry.db", + config.SelfRegistration: true, + config.LDAPURL: "ldap://127.0.0.1", + config.LDAPSearchDN: "uid=searchuser,ou=people,dc=mydomain,dc=com", + config.LDAPSearchPwd: "password", + config.LDAPBaseDN: "ou=people,dc=mydomain,dc=com", + config.LDAPUID: "uid", + config.LDAPFilter: "", + config.LDAPScope: 3, + config.LDAPTimeout: 30, + config.TokenServiceURL: "http://token_service", + config.RegistryURL: "http://registry", + config.EmailHost: "127.0.0.1", + config.EmailPort: 25, + config.EmailUsername: "user01", + config.EmailPassword: "password", + config.EmailFrom: "from", + config.EmailSSL: true, + config.EmailIdentity: "", + config.ProjectCreationRestriction: config.ProCrtRestrAdmOnly, + config.VerifyRemoteCert: false, + config.MaxJobWorkers: 3, + config.TokenExpiration: 30, + config.CfgExpiration: 5, + config.JobLogDir: "/var/log/jobs", + config.UseCompressedJS: true, + config.SecretKey: "secret", + config.AdminInitialPassword: "password", +} + // NewAdminserver returns a mock admin server -func NewAdminserver() (*httptest.Server, error) { +func NewAdminserver(config map[string]interface{}) (*httptest.Server, error) { m := []*RequestHandlerMapping{} - b, err := json.Marshal(map[string]interface{}{ - config.DomainName: "host01.com", - config.AUTHMode: config.DBAuth, - config.DatabaseType: "mysql", - config.MySQLHost: "127.0.0.1", - config.MySQLPort: 3306, - config.MySQLUsername: "user01", - config.MySQLPassword: "password", - config.MySQLDatabase: "registry", - config.SQLiteFile: "/tmp/registry.db", - config.SelfRegistration: true, - config.LDAPURL: "ldap://127.0.0.1", - config.LDAPSearchDN: "uid=searchuser,ou=people,dc=mydomain,dc=com", - config.LDAPSearchPwd: "password", - config.LDAPBaseDN: "ou=people,dc=mydomain,dc=com", - config.LDAPUID: "uid", - config.LDAPFilter: "", - config.LDAPScope: 3, - config.LDAPTimeout: 30, - config.TokenServiceURL: "http://token_service", - config.RegistryURL: "http://registry", - config.EmailHost: "127.0.0.1", - config.EmailPort: 25, - config.EmailUsername: "user01", - config.EmailPassword: "password", - config.EmailFrom: "from", - config.EmailSSL: true, - config.EmailIdentity: "", - config.ProjectCreationRestriction: config.ProCrtRestrAdmOnly, - config.VerifyRemoteCert: false, - config.MaxJobWorkers: 3, - config.TokenExpiration: 30, - config.CfgExpiration: 5, - config.JobLogDir: "/var/log/jobs", - config.UseCompressedJS: true, - config.SecretKey: "secret", - config.AdminInitialPassword: "password", - }) + if config == nil { + config = adminServerDefaultConfig + } + + b, err := json.Marshal(config) if err != nil { return nil, err } diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index d4916dc22..164f07f91 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -121,11 +121,16 @@ func UISecret() string { return os.Getenv("UI_SECRET") } -// DomainName ... -func DomainName() (string, error) { +// ExtEndpoint ... +func ExtEndpoint() (string, error) { cfg, err := mg.Get() if err != nil { return "", err } - return cfg[comcfg.DomainName].(string), nil + return cfg[comcfg.ExtEndpoint].(string), nil +} + +// InternalTokenServiceEndpoint ... +func InternalTokenServiceEndpoint() string { + return "http://ui/service/token" } diff --git a/src/jobservice/config/config_test.go b/src/jobservice/config/config_test.go index 0abcb1655..c70af060c 100644 --- a/src/jobservice/config/config_test.go +++ b/src/jobservice/config/config_test.go @@ -24,7 +24,7 @@ import ( // test functions under package jobservice/config func TestConfig(t *testing.T) { - server, err := test.NewAdminserver() + server, err := test.NewAdminserver(nil) if err != nil { t.Fatalf("failed to create a mock admin server: %v", err) } @@ -53,8 +53,6 @@ func TestConfig(t *testing.T) { t.Fatalf("failed to get max job workers: %v", err) } - LocalUIURL() - if _, err := LocalRegURL(); err != nil { t.Fatalf("failed to get registry URL: %v", err) } @@ -67,5 +65,11 @@ func TestConfig(t *testing.T) { t.Fatalf("failed to get secret key: %v", err) } - UISecret() + if len(InternalTokenServiceEndpoint()) == 0 { + t.Error("the internal token service endpoint is null") + } + + if _, err := ExtEndpoint(); err != nil { + t.Fatalf("failed to get ext endpoint: %v", err) + } } diff --git a/src/jobservice/replication/transfer.go b/src/jobservice/replication/transfer.go index 186718ee2..20f2751a1 100644 --- a/src/jobservice/replication/transfer.go +++ b/src/jobservice/replication/transfer.go @@ -139,7 +139,7 @@ func (i *Initializer) enter() (string, error) { c := &http.Cookie{Name: models.UISecretCookie, Value: i.srcSecret} srcCred := auth.NewCookieCredential(c) srcClient, err := newRepositoryClient(i.srcURL, i.insecure, srcCred, - i.repository, "repository", i.repository, "pull", "push", "*") + config.InternalTokenServiceEndpoint(), i.repository, "repository", i.repository, "pull", "push", "*") if err != nil { i.logger.Errorf("an error occurred while creating source repository client: %v", err) return "", err @@ -148,7 +148,7 @@ func (i *Initializer) enter() (string, error) { dstCred := auth.NewBasicAuthCredential(i.dstUsr, i.dstPwd) dstClient, err := newRepositoryClient(i.dstURL, i.insecure, dstCred, - i.repository, "repository", i.repository, "pull", "push", "*") + "", i.repository, "repository", i.repository, "pull", "push", "*") if err != nil { i.logger.Errorf("an error occurred while creating destination repository client: %v", err) return "", err @@ -459,10 +459,11 @@ func (m *ManifestPusher) enter() (string, error) { return StatePullManifest, nil } -func newRepositoryClient(endpoint string, insecure bool, credential auth.Credential, repository, scopeType, scopeName string, +func newRepositoryClient(endpoint string, insecure bool, credential auth.Credential, + tokenServiceEndpoint, repository, scopeType, scopeName string, scopeActions ...string) (*registry.Repository, error) { - domain, err := config.DomainName() + domain, err := config.ExtEndpoint() if err != nil { return nil, err } @@ -470,7 +471,8 @@ func newRepositoryClient(endpoint string, insecure bool, credential auth.Credent return nil, err } - authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, scopeType, scopeName, scopeActions...) + authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, + tokenServiceEndpoint, scopeType, scopeName, scopeActions...) store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer) if err != nil { diff --git a/src/ui/api/config.go b/src/ui/api/config.go index 89f5371a0..98657f18a 100644 --- a/src/ui/api/config.go +++ b/src/ui/api/config.go @@ -119,7 +119,14 @@ func (c *ConfigAPI) Put() { } } - if err := validateCfg(cfg); err != nil { + isSysErr, err := validateCfg(cfg) + + if err != nil { + if isSysErr { + log.Errorf("failed to validate configurations: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + c.CustomAbort(http.StatusBadRequest, err.Error()) } @@ -162,42 +169,68 @@ func (c *ConfigAPI) Put() { } } -func validateCfg(c map[string]string) error { +func validateCfg(c map[string]string) (bool, error) { + isSysErr := false + + mode, err := config.AuthMode() + if err != nil { + isSysErr = true + return isSysErr, err + } + if value, ok := c[comcfg.AUTHMode]; ok { if value != comcfg.DBAuth && value != comcfg.LDAPAuth { - return fmt.Errorf("invalid %s, shoud be %s or %s", comcfg.AUTHMode, comcfg.DBAuth, comcfg.LDAPAuth) + return isSysErr, fmt.Errorf("invalid %s, shoud be %s or %s", comcfg.AUTHMode, comcfg.DBAuth, comcfg.LDAPAuth) + } + mode = value + } + + if mode == comcfg.LDAPAuth { + ldap, err := config.LDAP() + if err != nil { + isSysErr = true + return isSysErr, err } - if value == comcfg.LDAPAuth { + if len(ldap.URL) == 0 { if _, ok := c[comcfg.LDAPURL]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAPURL) + return isSysErr, fmt.Errorf("%s is missing", comcfg.LDAPURL) } + } + + if len(ldap.BaseDN) == 0 { if _, ok := c[comcfg.LDAPBaseDN]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAPBaseDN) + return isSysErr, fmt.Errorf("%s is missing", comcfg.LDAPBaseDN) } + } + if len(ldap.UID) == 0 { if _, ok := c[comcfg.LDAPUID]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAPUID) + return isSysErr, fmt.Errorf("%s is missing", comcfg.LDAPUID) } + } + if ldap.Scope == 0 { if _, ok := c[comcfg.LDAPScope]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAPScope) + return isSysErr, fmt.Errorf("%s is missing", comcfg.LDAPScope) } } } + log.Infof("===========%v", c) + if ldapURL, ok := c[comcfg.LDAPURL]; ok && len(ldapURL) == 0 { - return fmt.Errorf("%s is empty", comcfg.LDAPURL) + return isSysErr, fmt.Errorf("%s is empty", comcfg.LDAPURL) } if baseDN, ok := c[comcfg.LDAPBaseDN]; ok && len(baseDN) == 0 { - return fmt.Errorf("%s is empty", comcfg.LDAPBaseDN) + return isSysErr, fmt.Errorf("%s is empty", comcfg.LDAPBaseDN) } if uID, ok := c[comcfg.LDAPUID]; ok && len(uID) == 0 { - return fmt.Errorf("%s is empty", comcfg.LDAPUID) + return isSysErr, fmt.Errorf("%s is empty", comcfg.LDAPUID) } if scope, ok := c[comcfg.LDAPScope]; ok && scope != comcfg.LDAPScopeBase && scope != comcfg.LDAPScopeOnelevel && scope != comcfg.LDAPScopeSubtree { - return fmt.Errorf("invalid %s, should be %s, %s or %s", + return isSysErr, fmt.Errorf("invalid %s, should be %s, %s or %s", comcfg.LDAPScope, comcfg.LDAPScopeBase, comcfg.LDAPScopeOnelevel, @@ -205,41 +238,41 @@ func validateCfg(c map[string]string) error { } if timeout, ok := c[comcfg.LDAPTimeout]; ok { if t, err := strconv.Atoi(timeout); err != nil || t < 0 { - return fmt.Errorf("invalid %s", comcfg.LDAPTimeout) + return isSysErr, fmt.Errorf("invalid %s", comcfg.LDAPTimeout) } } if self, ok := c[comcfg.SelfRegistration]; ok && self != "0" && self != "1" { - return fmt.Errorf("%s should be %s or %s", + return isSysErr, fmt.Errorf("%s should be %s or %s", comcfg.SelfRegistration, "0", "1") } if port, ok := c[comcfg.EmailPort]; ok { if p, err := strconv.Atoi(port); err != nil || p < 0 || p > 65535 { - return fmt.Errorf("invalid %s", comcfg.EmailPort) + return isSysErr, fmt.Errorf("invalid %s", comcfg.EmailPort) } } if ssl, ok := c[comcfg.EmailSSL]; ok && ssl != "0" && ssl != "1" { - return fmt.Errorf("%s should be %s or %s", comcfg.EmailSSL, "0", "1") + return isSysErr, fmt.Errorf("%s should be %s or %s", comcfg.EmailSSL, "0", "1") } if crt, ok := c[comcfg.ProjectCreationRestriction]; ok && crt != comcfg.ProCrtRestrEveryone && crt != comcfg.ProCrtRestrAdmOnly { - return fmt.Errorf("invalid %s, should be %s or %s", + return isSysErr, fmt.Errorf("invalid %s, should be %s or %s", comcfg.ProjectCreationRestriction, comcfg.ProCrtRestrAdmOnly, comcfg.ProCrtRestrEveryone) } if verify, ok := c[comcfg.VerifyRemoteCert]; ok && verify != "0" && verify != "1" { - return fmt.Errorf("invalid %s, should be %s or %s", + return isSysErr, fmt.Errorf("invalid %s, should be %s or %s", comcfg.VerifyRemoteCert, "0", "1") } - return nil + return isSysErr, nil } //encode passwords and convert map[string]string to map[string]interface{} diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index 19de01090..012ced2d3 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -19,7 +19,6 @@ import ( "fmt" "io/ioutil" "net/http" - "os" "sort" "github.com/docker/distribution/manifest/schema1" @@ -444,15 +443,8 @@ func newRepositoryClient(endpoint string, insecure bool, username, password, rep credential := auth.NewBasicAuthCredential(username, password) - domain, err := config.DomainName() - if err != nil { - return nil, err - } - if err := os.Setenv("DOMAIN_NAME", domain); err != nil { - return nil, err - } - - authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, scopeType, scopeName, scopeActions...) + authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, + config.InternalTokenServiceEndpoint(), scopeType, scopeName, scopeActions...) store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer) if err != nil { diff --git a/src/ui/api/target.go b/src/ui/api/target.go index af3888f37..0fc039f57 100644 --- a/src/ui/api/target.go +++ b/src/ui/api/target.go @@ -20,7 +20,6 @@ import ( "net" "net/http" "net/url" - "os" "strconv" "github.com/vmware/harbor/src/common/api" @@ -342,15 +341,8 @@ func newRegistryClient(endpoint string, insecure bool, username, password, scope scopeActions ...string) (*registry.Registry, error) { credential := auth.NewBasicAuthCredential(username, password) - domain, err := config.DomainName() - if err != nil { - return nil, err - } - if err := os.Setenv("DOMAIN_NAME", domain); err != nil { - return nil, err - } - - authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, scopeType, scopeName, scopeActions...) + authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, + "", scopeType, scopeName, scopeActions...) store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer) if err != nil { diff --git a/src/ui/config/config.go b/src/ui/config/config.go index 5833b16e7..21bbe288b 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -114,13 +114,13 @@ func TokenExpiration() (int, error) { return int(cfg[comcfg.TokenExpiration].(float64)), nil } -// DomainName returns the external URL of Harbor: protocal://host:port -func DomainName() (string, error) { +// ExtEndpoint returns the external URL of Harbor: protocal://host:port +func ExtEndpoint() (string, error) { cfg, err := mg.Get() if err != nil { return "", err } - return cfg[comcfg.DomainName].(string), nil + return cfg[comcfg.ExtEndpoint].(string), nil } // SecretKey returns the secret key to encrypt the password of target @@ -155,6 +155,11 @@ func InternalJobServiceURL() string { return "http://jobservice" } +// InternalTokenServiceEndpoint returns token service endpoint for internal communication between Harbor containers +func InternalTokenServiceEndpoint() string { + return "http://ui/service/token" +} + // InitialAdminPassword returns the initial password for administrator func InitialAdminPassword() (string, error) { cfg, err := mg.Get() diff --git a/src/ui/config/config_test.go b/src/ui/config/config_test.go index 4846dd952..e06798c52 100644 --- a/src/ui/config/config_test.go +++ b/src/ui/config/config_test.go @@ -23,7 +23,7 @@ import ( // test functions under package ui/config func TestConfig(t *testing.T) { - server, err := test.NewAdminserver() + server, err := test.NewAdminserver(nil) if err != nil { t.Fatalf("failed to create a mock admin server: %v", err) } @@ -68,7 +68,7 @@ func TestConfig(t *testing.T) { t.Fatalf("failed to get token expiration: %v", err) } - if _, err := DomainName(); err != nil { + if _, err := ExtEndpoint(); err != nil { t.Fatalf("failed to get domain name: %v", err) } @@ -84,9 +84,17 @@ func TestConfig(t *testing.T) { t.Fatalf("failed to get registry URL: %v", err) } - InternalJobServiceURL() + if len(InternalJobServiceURL()) == 0 { + t.Error("the internal job service url is null") + } - InitialAdminPassword() + if len(InternalTokenServiceEndpoint()) == 0 { + t.Error("the internal token service endpoint is null") + } + + if _, err := InitialAdminPassword(); err != nil { + t.Fatalf("failed to get initial admin password: %v", err) + } if _, err := OnlyAdminCreateProject(); err != nil { t.Fatalf("failed to get onldy admin create project: %v", err) @@ -103,6 +111,4 @@ func TestConfig(t *testing.T) { if _, err := Database(); err != nil { t.Fatalf("failed to get database: %v", err) } - - UISecret() } diff --git a/src/ui/controllers/password.go b/src/ui/controllers/password.go index 786b010a2..60a3140ee 100644 --- a/src/ui/controllers/password.go +++ b/src/ui/controllers/password.go @@ -49,7 +49,7 @@ func (cc *CommonController) SendEmail() { message := new(bytes.Buffer) - harborURL, err := config.DomainName() + harborURL, err := config.ExtEndpoint() if err != nil { log.Errorf("failed to get domain name: %v", err) cc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) diff --git a/src/ui/controllers/repository.go b/src/ui/controllers/repository.go index 0fddff097..a3ada104d 100644 --- a/src/ui/controllers/repository.go +++ b/src/ui/controllers/repository.go @@ -15,7 +15,7 @@ type RepositoryController struct { // Get renders repository page func (rc *RepositoryController) Get() { - url, err := config.DomainName() + url, err := config.ExtEndpoint() if err != nil { log.Errorf("failed to get domain name: %v", err) rc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) diff --git a/src/ui/service/token/authutils.go b/src/ui/service/token/authutils.go index 3bda33ca4..8c14df5d6 100644 --- a/src/ui/service/token/authutils.go +++ b/src/ui/service/token/authutils.go @@ -84,7 +84,7 @@ func FilterAccess(username string, a *token.ResourceActions) { repoLength := len(repoSplit) if repoLength > 1 { //Only check the permission when the requested image has a namespace, i.e. project var projectName string - registryURL, err := config.DomainName() + registryURL, err := config.ExtEndpoint() if err != nil { log.Errorf("failed to get domain name: %v", err) return From 09dc6909ae12f6bea076a6c6edc07e0942742313 Mon Sep 17 00:00:00 2001 From: kunw Date: Tue, 14 Feb 2017 12:34:47 +0800 Subject: [PATCH 15/22] Update for some changes about configuration settings of UI. --- .../components/system-management/configuration.directive.js | 2 +- .../resources/js/services/i18n/locale_messages_en-US.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ui/static/resources/js/components/system-management/configuration.directive.js b/src/ui/static/resources/js/components/system-management/configuration.directive.js index 45db3711d..7a270c6f2 100644 --- a/src/ui/static/resources/js/components/system-management/configuration.directive.js +++ b/src/ui/static/resources/js/components/system-management/configuration.directive.js @@ -68,7 +68,7 @@ 'ldap_base_dn': { type: 'auth', attr: 'ldapBaseDN' }, 'ldap_uid': { type: 'auth', attr: 'ldapUID' }, 'ldap_filter': { type: 'auth', attr: 'ldapFilter' }, - 'ldap_connection_timeout': { type: 'auth', attr: 'ldapConnectionTimeout' }, + 'ldap_timeout': { type: 'auth', attr: 'ldapConnectionTimeout' }, 'ldap_scope': { type: 'auth', attr: 'ldapScope' }, 'email_host': { type: 'email', attr: 'server' }, 'email_port': { type: 'email', attr: 'serverPort' }, diff --git a/src/ui/static/resources/js/services/i18n/locale_messages_en-US.js b/src/ui/static/resources/js/services/i18n/locale_messages_en-US.js index 650f75aa5..87af074eb 100644 --- a/src/ui/static/resources/js/services/i18n/locale_messages_en-US.js +++ b/src/ui/static/resources/js/services/i18n/locale_messages_en-US.js @@ -332,12 +332,12 @@ var locale_messages = { 'please_save_changes': 'Please save changes before leaving this page.', 'undo': 'Undo', 'invalid_port_number': 'Invalid port number.', - 'max_job_workers_is_required': 'Maxystem job workers number is required.', + 'max_job_workers_is_required': 'Max job workers number is required.', 'timeout_is_required': 'Timeout value is required.', 'invalid_timeout': 'Invalid timeout value.', 'ldap_scope_is_required': 'Scope is required.', 'invalid_ldap_scope': 'Invalid Scope value.', - 'update_configuration_title': 'Update Configuration', - 'successful_update_configuration': 'Update configurations successfully.', + 'update_configuration_title': 'Update Configuration(s)', + 'successful_update_configuration': 'Configuration(s) updated successfully.', 'failed_to_update_configuration': 'Failed to update configuration.' }; \ No newline at end of file From 2e3174f404ce06bc38e70ca279e867b754d446a9 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Mon, 13 Feb 2017 17:05:25 +0800 Subject: [PATCH 16/22] update --- make/dev/docker-compose.yml | 2 +- src/adminserver/systemcfg/systemcfg.go | 2 +- src/common/config/config.go | 2 +- .../utils/registry/auth/tokenauthorizer.go | 28 ++++--- .../registry/auth/tokenauthorizer_test.go | 2 +- src/common/utils/test/adminserver.go | 84 ++++++++++--------- src/jobservice/config/config.go | 11 ++- src/jobservice/config/config_test.go | 12 ++- src/jobservice/replication/transfer.go | 19 ++--- src/ui/api/config.go | 71 +++++++++++----- src/ui/api/repository.go | 12 +-- src/ui/api/target.go | 12 +-- src/ui/config/config.go | 11 ++- src/ui/config/config_test.go | 18 ++-- src/ui/controllers/password.go | 2 +- src/ui/controllers/repository.go | 2 +- src/ui/service/token/authutils.go | 2 +- 17 files changed, 165 insertions(+), 127 deletions(-) diff --git a/make/dev/docker-compose.yml b/make/dev/docker-compose.yml index c94a2d875..716f642ba 100644 --- a/make/dev/docker-compose.yml +++ b/make/dev/docker-compose.yml @@ -3,7 +3,7 @@ services: log: build: context: ../../ - dockerfile: make/ubuntu/log/Dockerfile + dockerfile: make/photon/log/Dockerfile restart: always volumes: - /var/log/harbor/:/var/log/docker/ diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index d1d45c203..4d22adc22 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -78,7 +78,7 @@ func getCfgStore() string { //read the following attrs from env every time boots up func readFromEnv(cfg map[string]interface{}) error { - cfg[comcfg.DomainName] = os.Getenv("EXT_ENDPOINT") + cfg[comcfg.ExtEndpoint] = os.Getenv("EXT_ENDPOINT") cfg[comcfg.DatabaseType] = os.Getenv("DATABASE_TYPE") cfg[comcfg.MySQLHost] = os.Getenv("MYSQL_HOST") diff --git a/src/common/config/config.go b/src/common/config/config.go index 295564d66..29c59f948 100644 --- a/src/common/config/config.go +++ b/src/common/config/config.go @@ -40,7 +40,7 @@ const ( LDAPScopeOnelevel = "2" LDAPScopeSubtree = "3" - DomainName = "domain_name" + ExtEndpoint = "ext_endpoint" AUTHMode = "auth_mode" DatabaseType = "database_type" MySQLHost = "mysql_host" diff --git a/src/common/utils/registry/auth/tokenauthorizer.go b/src/common/utils/registry/auth/tokenauthorizer.go index 0543ef524..b65474c9b 100644 --- a/src/common/utils/registry/auth/tokenauthorizer.go +++ b/src/common/utils/registry/auth/tokenauthorizer.go @@ -21,7 +21,6 @@ import ( "io/ioutil" "net/http" "net/url" - "os" "strings" "sync" "time" @@ -134,19 +133,24 @@ func (t *tokenAuthorizer) updateCachedToken(token string, expiresIn int) { // Implements interface Authorizer type standardTokenAuthorizer struct { tokenAuthorizer - client *http.Client - credential Credential + client *http.Client + credential Credential + tokenServiceEndpoint string } // NewStandardTokenAuthorizer returns a standard token authorizer. The authorizer will request a token // from token server and add it to the origin request -func NewStandardTokenAuthorizer(credential Credential, insecure bool, scopeType, scopeName string, scopeActions ...string) Authorizer { +// If tokenServiceEndpoint is set, the token request will be sent to it instead of the server get from authorizer +// The usage please refer to the function tokenURL +func NewStandardTokenAuthorizer(credential Credential, insecure bool, + tokenServiceEndpoint string, scopeType, scopeName string, scopeActions ...string) Authorizer { authorizer := &standardTokenAuthorizer{ client: &http.Client{ Transport: registry.GetHTTPTransport(insecure), Timeout: 30 * time.Second, }, - credential: credential, + credential: credential, + tokenServiceEndpoint: tokenServiceEndpoint, } if len(scopeType) != 0 || len(scopeName) != 0 { @@ -163,7 +167,7 @@ func NewStandardTokenAuthorizer(credential Credential, insecure bool, scopeType, } func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) { - realm = tokenURL(realm) + realm = s.tokenURL(realm) u, err := url.Parse(realm) if err != nil { @@ -230,17 +234,15 @@ func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes [] // when the registry client is used inside Harbor, the token request // can be posted to token service directly rather than going through nginx. -// this solution can resolve two problems: +// If realm is set as the internal url of token service, this can resolve +// two problems: // 1. performance issue // 2. the realm field returned by registry is an IP which can not reachable // inside Harbor -func tokenURL(realm string) string { - - domainName := os.Getenv("DOMAIN_NAME") - if len(domainName) != 0 && strings.Contains(realm, domainName) { - realm = "http://ui/service/token" +func (s *standardTokenAuthorizer) tokenURL(realm string) string { + if len(s.tokenServiceEndpoint) != 0 { + return s.tokenServiceEndpoint } - return realm } diff --git a/src/common/utils/registry/auth/tokenauthorizer_test.go b/src/common/utils/registry/auth/tokenauthorizer_test.go index 2d843d887..3a582bb64 100644 --- a/src/common/utils/registry/auth/tokenauthorizer_test.go +++ b/src/common/utils/registry/auth/tokenauthorizer_test.go @@ -40,7 +40,7 @@ func TestAuthorizeOfStandardTokenAuthorizer(t *testing.T) { }) defer server.Close() - authorizer := NewStandardTokenAuthorizer(nil, false, "repository", "library/ubuntu", "pull") + authorizer := NewStandardTokenAuthorizer(nil, false, "", "repository", "library/ubuntu", "pull") req, err := http.NewRequest("GET", "http://registry", nil) if err != nil { t.Fatalf("failed to create request: %v", err) diff --git a/src/common/utils/test/adminserver.go b/src/common/utils/test/adminserver.go index 18e1b91fd..93621d0fc 100644 --- a/src/common/utils/test/adminserver.go +++ b/src/common/utils/test/adminserver.go @@ -23,47 +23,53 @@ import ( "github.com/vmware/harbor/src/common/config" ) +var adminServerDefaultConfig = map[string]interface{}{ + config.ExtEndpoint: "host01.com", + config.AUTHMode: config.DBAuth, + config.DatabaseType: "mysql", + config.MySQLHost: "127.0.0.1", + config.MySQLPort: 3306, + config.MySQLUsername: "user01", + config.MySQLPassword: "password", + config.MySQLDatabase: "registry", + config.SQLiteFile: "/tmp/registry.db", + config.SelfRegistration: true, + config.LDAPURL: "ldap://127.0.0.1", + config.LDAPSearchDN: "uid=searchuser,ou=people,dc=mydomain,dc=com", + config.LDAPSearchPwd: "password", + config.LDAPBaseDN: "ou=people,dc=mydomain,dc=com", + config.LDAPUID: "uid", + config.LDAPFilter: "", + config.LDAPScope: 3, + config.LDAPTimeout: 30, + config.TokenServiceURL: "http://token_service", + config.RegistryURL: "http://registry", + config.EmailHost: "127.0.0.1", + config.EmailPort: 25, + config.EmailUsername: "user01", + config.EmailPassword: "password", + config.EmailFrom: "from", + config.EmailSSL: true, + config.EmailIdentity: "", + config.ProjectCreationRestriction: config.ProCrtRestrAdmOnly, + config.VerifyRemoteCert: false, + config.MaxJobWorkers: 3, + config.TokenExpiration: 30, + config.CfgExpiration: 5, + config.JobLogDir: "/var/log/jobs", + config.UseCompressedJS: true, + config.SecretKey: "secret", + config.AdminInitialPassword: "password", +} + // NewAdminserver returns a mock admin server -func NewAdminserver() (*httptest.Server, error) { +func NewAdminserver(config map[string]interface{}) (*httptest.Server, error) { m := []*RequestHandlerMapping{} - b, err := json.Marshal(map[string]interface{}{ - config.DomainName: "host01.com", - config.AUTHMode: config.DBAuth, - config.DatabaseType: "mysql", - config.MySQLHost: "127.0.0.1", - config.MySQLPort: 3306, - config.MySQLUsername: "user01", - config.MySQLPassword: "password", - config.MySQLDatabase: "registry", - config.SQLiteFile: "/tmp/registry.db", - config.SelfRegistration: true, - config.LDAPURL: "ldap://127.0.0.1", - config.LDAPSearchDN: "uid=searchuser,ou=people,dc=mydomain,dc=com", - config.LDAPSearchPwd: "password", - config.LDAPBaseDN: "ou=people,dc=mydomain,dc=com", - config.LDAPUID: "uid", - config.LDAPFilter: "", - config.LDAPScope: 3, - config.LDAPTimeout: 30, - config.TokenServiceURL: "http://token_service", - config.RegistryURL: "http://registry", - config.EmailHost: "127.0.0.1", - config.EmailPort: 25, - config.EmailUsername: "user01", - config.EmailPassword: "password", - config.EmailFrom: "from", - config.EmailSSL: true, - config.EmailIdentity: "", - config.ProjectCreationRestriction: config.ProCrtRestrAdmOnly, - config.VerifyRemoteCert: false, - config.MaxJobWorkers: 3, - config.TokenExpiration: 30, - config.CfgExpiration: 5, - config.JobLogDir: "/var/log/jobs", - config.UseCompressedJS: true, - config.SecretKey: "secret", - config.AdminInitialPassword: "password", - }) + if config == nil { + config = adminServerDefaultConfig + } + + b, err := json.Marshal(config) if err != nil { return nil, err } diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index d4916dc22..164f07f91 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -121,11 +121,16 @@ func UISecret() string { return os.Getenv("UI_SECRET") } -// DomainName ... -func DomainName() (string, error) { +// ExtEndpoint ... +func ExtEndpoint() (string, error) { cfg, err := mg.Get() if err != nil { return "", err } - return cfg[comcfg.DomainName].(string), nil + return cfg[comcfg.ExtEndpoint].(string), nil +} + +// InternalTokenServiceEndpoint ... +func InternalTokenServiceEndpoint() string { + return "http://ui/service/token" } diff --git a/src/jobservice/config/config_test.go b/src/jobservice/config/config_test.go index 0abcb1655..c70af060c 100644 --- a/src/jobservice/config/config_test.go +++ b/src/jobservice/config/config_test.go @@ -24,7 +24,7 @@ import ( // test functions under package jobservice/config func TestConfig(t *testing.T) { - server, err := test.NewAdminserver() + server, err := test.NewAdminserver(nil) if err != nil { t.Fatalf("failed to create a mock admin server: %v", err) } @@ -53,8 +53,6 @@ func TestConfig(t *testing.T) { t.Fatalf("failed to get max job workers: %v", err) } - LocalUIURL() - if _, err := LocalRegURL(); err != nil { t.Fatalf("failed to get registry URL: %v", err) } @@ -67,5 +65,11 @@ func TestConfig(t *testing.T) { t.Fatalf("failed to get secret key: %v", err) } - UISecret() + if len(InternalTokenServiceEndpoint()) == 0 { + t.Error("the internal token service endpoint is null") + } + + if _, err := ExtEndpoint(); err != nil { + t.Fatalf("failed to get ext endpoint: %v", err) + } } diff --git a/src/jobservice/replication/transfer.go b/src/jobservice/replication/transfer.go index 186718ee2..970de022f 100644 --- a/src/jobservice/replication/transfer.go +++ b/src/jobservice/replication/transfer.go @@ -23,7 +23,6 @@ import ( "fmt" "io/ioutil" "net/http" - "os" "strings" "github.com/docker/distribution" @@ -139,7 +138,7 @@ func (i *Initializer) enter() (string, error) { c := &http.Cookie{Name: models.UISecretCookie, Value: i.srcSecret} srcCred := auth.NewCookieCredential(c) srcClient, err := newRepositoryClient(i.srcURL, i.insecure, srcCred, - i.repository, "repository", i.repository, "pull", "push", "*") + config.InternalTokenServiceEndpoint(), i.repository, "repository", i.repository, "pull", "push", "*") if err != nil { i.logger.Errorf("an error occurred while creating source repository client: %v", err) return "", err @@ -148,7 +147,7 @@ func (i *Initializer) enter() (string, error) { dstCred := auth.NewBasicAuthCredential(i.dstUsr, i.dstPwd) dstClient, err := newRepositoryClient(i.dstURL, i.insecure, dstCred, - i.repository, "repository", i.repository, "pull", "push", "*") + "", i.repository, "repository", i.repository, "pull", "push", "*") if err != nil { i.logger.Errorf("an error occurred while creating destination repository client: %v", err) return "", err @@ -459,18 +458,12 @@ func (m *ManifestPusher) enter() (string, error) { return StatePullManifest, nil } -func newRepositoryClient(endpoint string, insecure bool, credential auth.Credential, repository, scopeType, scopeName string, +func newRepositoryClient(endpoint string, insecure bool, credential auth.Credential, + tokenServiceEndpoint, repository, scopeType, scopeName string, scopeActions ...string) (*registry.Repository, error) { - domain, err := config.DomainName() - if err != nil { - return nil, err - } - if err := os.Setenv("DOMAIN_NAME", domain); err != nil { - return nil, err - } - - authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, scopeType, scopeName, scopeActions...) + authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, + tokenServiceEndpoint, scopeType, scopeName, scopeActions...) store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer) if err != nil { diff --git a/src/ui/api/config.go b/src/ui/api/config.go index 89f5371a0..98657f18a 100644 --- a/src/ui/api/config.go +++ b/src/ui/api/config.go @@ -119,7 +119,14 @@ func (c *ConfigAPI) Put() { } } - if err := validateCfg(cfg); err != nil { + isSysErr, err := validateCfg(cfg) + + if err != nil { + if isSysErr { + log.Errorf("failed to validate configurations: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + c.CustomAbort(http.StatusBadRequest, err.Error()) } @@ -162,42 +169,68 @@ func (c *ConfigAPI) Put() { } } -func validateCfg(c map[string]string) error { +func validateCfg(c map[string]string) (bool, error) { + isSysErr := false + + mode, err := config.AuthMode() + if err != nil { + isSysErr = true + return isSysErr, err + } + if value, ok := c[comcfg.AUTHMode]; ok { if value != comcfg.DBAuth && value != comcfg.LDAPAuth { - return fmt.Errorf("invalid %s, shoud be %s or %s", comcfg.AUTHMode, comcfg.DBAuth, comcfg.LDAPAuth) + return isSysErr, fmt.Errorf("invalid %s, shoud be %s or %s", comcfg.AUTHMode, comcfg.DBAuth, comcfg.LDAPAuth) + } + mode = value + } + + if mode == comcfg.LDAPAuth { + ldap, err := config.LDAP() + if err != nil { + isSysErr = true + return isSysErr, err } - if value == comcfg.LDAPAuth { + if len(ldap.URL) == 0 { if _, ok := c[comcfg.LDAPURL]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAPURL) + return isSysErr, fmt.Errorf("%s is missing", comcfg.LDAPURL) } + } + + if len(ldap.BaseDN) == 0 { if _, ok := c[comcfg.LDAPBaseDN]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAPBaseDN) + return isSysErr, fmt.Errorf("%s is missing", comcfg.LDAPBaseDN) } + } + if len(ldap.UID) == 0 { if _, ok := c[comcfg.LDAPUID]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAPUID) + return isSysErr, fmt.Errorf("%s is missing", comcfg.LDAPUID) } + } + if ldap.Scope == 0 { if _, ok := c[comcfg.LDAPScope]; !ok { - return fmt.Errorf("%s is missing", comcfg.LDAPScope) + return isSysErr, fmt.Errorf("%s is missing", comcfg.LDAPScope) } } } + log.Infof("===========%v", c) + if ldapURL, ok := c[comcfg.LDAPURL]; ok && len(ldapURL) == 0 { - return fmt.Errorf("%s is empty", comcfg.LDAPURL) + return isSysErr, fmt.Errorf("%s is empty", comcfg.LDAPURL) } if baseDN, ok := c[comcfg.LDAPBaseDN]; ok && len(baseDN) == 0 { - return fmt.Errorf("%s is empty", comcfg.LDAPBaseDN) + return isSysErr, fmt.Errorf("%s is empty", comcfg.LDAPBaseDN) } if uID, ok := c[comcfg.LDAPUID]; ok && len(uID) == 0 { - return fmt.Errorf("%s is empty", comcfg.LDAPUID) + return isSysErr, fmt.Errorf("%s is empty", comcfg.LDAPUID) } if scope, ok := c[comcfg.LDAPScope]; ok && scope != comcfg.LDAPScopeBase && scope != comcfg.LDAPScopeOnelevel && scope != comcfg.LDAPScopeSubtree { - return fmt.Errorf("invalid %s, should be %s, %s or %s", + return isSysErr, fmt.Errorf("invalid %s, should be %s, %s or %s", comcfg.LDAPScope, comcfg.LDAPScopeBase, comcfg.LDAPScopeOnelevel, @@ -205,41 +238,41 @@ func validateCfg(c map[string]string) error { } if timeout, ok := c[comcfg.LDAPTimeout]; ok { if t, err := strconv.Atoi(timeout); err != nil || t < 0 { - return fmt.Errorf("invalid %s", comcfg.LDAPTimeout) + return isSysErr, fmt.Errorf("invalid %s", comcfg.LDAPTimeout) } } if self, ok := c[comcfg.SelfRegistration]; ok && self != "0" && self != "1" { - return fmt.Errorf("%s should be %s or %s", + return isSysErr, fmt.Errorf("%s should be %s or %s", comcfg.SelfRegistration, "0", "1") } if port, ok := c[comcfg.EmailPort]; ok { if p, err := strconv.Atoi(port); err != nil || p < 0 || p > 65535 { - return fmt.Errorf("invalid %s", comcfg.EmailPort) + return isSysErr, fmt.Errorf("invalid %s", comcfg.EmailPort) } } if ssl, ok := c[comcfg.EmailSSL]; ok && ssl != "0" && ssl != "1" { - return fmt.Errorf("%s should be %s or %s", comcfg.EmailSSL, "0", "1") + return isSysErr, fmt.Errorf("%s should be %s or %s", comcfg.EmailSSL, "0", "1") } if crt, ok := c[comcfg.ProjectCreationRestriction]; ok && crt != comcfg.ProCrtRestrEveryone && crt != comcfg.ProCrtRestrAdmOnly { - return fmt.Errorf("invalid %s, should be %s or %s", + return isSysErr, fmt.Errorf("invalid %s, should be %s or %s", comcfg.ProjectCreationRestriction, comcfg.ProCrtRestrAdmOnly, comcfg.ProCrtRestrEveryone) } if verify, ok := c[comcfg.VerifyRemoteCert]; ok && verify != "0" && verify != "1" { - return fmt.Errorf("invalid %s, should be %s or %s", + return isSysErr, fmt.Errorf("invalid %s, should be %s or %s", comcfg.VerifyRemoteCert, "0", "1") } - return nil + return isSysErr, nil } //encode passwords and convert map[string]string to map[string]interface{} diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index 19de01090..012ced2d3 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -19,7 +19,6 @@ import ( "fmt" "io/ioutil" "net/http" - "os" "sort" "github.com/docker/distribution/manifest/schema1" @@ -444,15 +443,8 @@ func newRepositoryClient(endpoint string, insecure bool, username, password, rep credential := auth.NewBasicAuthCredential(username, password) - domain, err := config.DomainName() - if err != nil { - return nil, err - } - if err := os.Setenv("DOMAIN_NAME", domain); err != nil { - return nil, err - } - - authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, scopeType, scopeName, scopeActions...) + authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, + config.InternalTokenServiceEndpoint(), scopeType, scopeName, scopeActions...) store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer) if err != nil { diff --git a/src/ui/api/target.go b/src/ui/api/target.go index af3888f37..0fc039f57 100644 --- a/src/ui/api/target.go +++ b/src/ui/api/target.go @@ -20,7 +20,6 @@ import ( "net" "net/http" "net/url" - "os" "strconv" "github.com/vmware/harbor/src/common/api" @@ -342,15 +341,8 @@ func newRegistryClient(endpoint string, insecure bool, username, password, scope scopeActions ...string) (*registry.Registry, error) { credential := auth.NewBasicAuthCredential(username, password) - domain, err := config.DomainName() - if err != nil { - return nil, err - } - if err := os.Setenv("DOMAIN_NAME", domain); err != nil { - return nil, err - } - - authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, scopeType, scopeName, scopeActions...) + authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, + "", scopeType, scopeName, scopeActions...) store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer) if err != nil { diff --git a/src/ui/config/config.go b/src/ui/config/config.go index 5833b16e7..21bbe288b 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -114,13 +114,13 @@ func TokenExpiration() (int, error) { return int(cfg[comcfg.TokenExpiration].(float64)), nil } -// DomainName returns the external URL of Harbor: protocal://host:port -func DomainName() (string, error) { +// ExtEndpoint returns the external URL of Harbor: protocal://host:port +func ExtEndpoint() (string, error) { cfg, err := mg.Get() if err != nil { return "", err } - return cfg[comcfg.DomainName].(string), nil + return cfg[comcfg.ExtEndpoint].(string), nil } // SecretKey returns the secret key to encrypt the password of target @@ -155,6 +155,11 @@ func InternalJobServiceURL() string { return "http://jobservice" } +// InternalTokenServiceEndpoint returns token service endpoint for internal communication between Harbor containers +func InternalTokenServiceEndpoint() string { + return "http://ui/service/token" +} + // InitialAdminPassword returns the initial password for administrator func InitialAdminPassword() (string, error) { cfg, err := mg.Get() diff --git a/src/ui/config/config_test.go b/src/ui/config/config_test.go index 4846dd952..e06798c52 100644 --- a/src/ui/config/config_test.go +++ b/src/ui/config/config_test.go @@ -23,7 +23,7 @@ import ( // test functions under package ui/config func TestConfig(t *testing.T) { - server, err := test.NewAdminserver() + server, err := test.NewAdminserver(nil) if err != nil { t.Fatalf("failed to create a mock admin server: %v", err) } @@ -68,7 +68,7 @@ func TestConfig(t *testing.T) { t.Fatalf("failed to get token expiration: %v", err) } - if _, err := DomainName(); err != nil { + if _, err := ExtEndpoint(); err != nil { t.Fatalf("failed to get domain name: %v", err) } @@ -84,9 +84,17 @@ func TestConfig(t *testing.T) { t.Fatalf("failed to get registry URL: %v", err) } - InternalJobServiceURL() + if len(InternalJobServiceURL()) == 0 { + t.Error("the internal job service url is null") + } - InitialAdminPassword() + if len(InternalTokenServiceEndpoint()) == 0 { + t.Error("the internal token service endpoint is null") + } + + if _, err := InitialAdminPassword(); err != nil { + t.Fatalf("failed to get initial admin password: %v", err) + } if _, err := OnlyAdminCreateProject(); err != nil { t.Fatalf("failed to get onldy admin create project: %v", err) @@ -103,6 +111,4 @@ func TestConfig(t *testing.T) { if _, err := Database(); err != nil { t.Fatalf("failed to get database: %v", err) } - - UISecret() } diff --git a/src/ui/controllers/password.go b/src/ui/controllers/password.go index 786b010a2..60a3140ee 100644 --- a/src/ui/controllers/password.go +++ b/src/ui/controllers/password.go @@ -49,7 +49,7 @@ func (cc *CommonController) SendEmail() { message := new(bytes.Buffer) - harborURL, err := config.DomainName() + harborURL, err := config.ExtEndpoint() if err != nil { log.Errorf("failed to get domain name: %v", err) cc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) diff --git a/src/ui/controllers/repository.go b/src/ui/controllers/repository.go index 0fddff097..a3ada104d 100644 --- a/src/ui/controllers/repository.go +++ b/src/ui/controllers/repository.go @@ -15,7 +15,7 @@ type RepositoryController struct { // Get renders repository page func (rc *RepositoryController) Get() { - url, err := config.DomainName() + url, err := config.ExtEndpoint() if err != nil { log.Errorf("failed to get domain name: %v", err) rc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) diff --git a/src/ui/service/token/authutils.go b/src/ui/service/token/authutils.go index 3bda33ca4..8c14df5d6 100644 --- a/src/ui/service/token/authutils.go +++ b/src/ui/service/token/authutils.go @@ -84,7 +84,7 @@ func FilterAccess(username string, a *token.ResourceActions) { repoLength := len(repoSplit) if repoLength > 1 { //Only check the permission when the requested image has a namespace, i.e. project var projectName string - registryURL, err := config.DomainName() + registryURL, err := config.ExtEndpoint() if err != nil { log.Errorf("failed to get domain name: %v", err) return From 390f89ee0a46cf7aa946cfbf96e6715864bf68ab Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 15 Feb 2017 15:25:59 +0800 Subject: [PATCH 17/22] encrypt passwords and secret --- .travis.yml | 2 +- make/dev/docker-compose.yml | 3 + make/docker-compose.tpl | 3 + make/prepare | 9 +++ src/adminserver/api/cfg.go | 5 +- src/adminserver/api/cfg_test.go | 67 +++++++++++++------ src/adminserver/config/config.go | 66 +++++++++++++++++++ src/adminserver/config/config_test.go | 67 +++++++++++++++++++ src/adminserver/main.go | 12 +++- src/adminserver/systemcfg/systemcfg.go | 71 +++++++++++++++++++-- src/adminserver/systemcfg/systemcfg_test.go | 20 +++--- src/common/config/config.go | 1 - src/common/utils/test/adminserver.go | 3 +- src/common/utils/test/key.go | 41 ++++++++++++ src/jobservice/config/config.go | 51 ++++++++++++--- src/jobservice/config/config_test.go | 15 ++++- src/ui/api/config.go | 5 +- src/ui/config/config.go | 48 +++++++++++--- src/ui/config/config_test.go | 15 ++++- tests/docker-compose.test.yml | 1 + tests/testprepare.sh | 2 + 21 files changed, 441 insertions(+), 66 deletions(-) create mode 100644 src/adminserver/config/config.go create mode 100644 src/adminserver/config/config_test.go create mode 100644 src/common/utils/test/key.go diff --git a/.travis.yml b/.travis.yml index 05dc4a095..c7b40e1bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,9 +25,9 @@ env: HARBOR_ADMIN_PASSWD: Harbor12345 UI_SECRET: tempString MAX_JOB_WORKERS: 3 - SECRET_KEY: 1234567890123456 AUTH_MODE: db_auth SELF_REGISTRATION: on + KEY_PATH: /data/secretkey before_install: - sudo ./tests/hostcfg.sh diff --git a/make/dev/docker-compose.yml b/make/dev/docker-compose.yml index 716f642ba..5f8574c15 100644 --- a/make/dev/docker-compose.yml +++ b/make/dev/docker-compose.yml @@ -49,6 +49,7 @@ services: restart: always volumes: - /data/config/:/etc/harbor/ + - /data/secretkey:/harbor/secretkey depends_on: - log logging: @@ -66,6 +67,7 @@ services: volumes: - ../common/config/ui/app.conf:/etc/ui/app.conf - ../common/config/ui/private_key.pem:/etc/ui/private_key.pem + - /data/secretkey:/harbor/secretkey depends_on: - log - adminserver @@ -85,6 +87,7 @@ services: volumes: - /data/job_logs:/var/log/jobs - ../common/config/jobservice/app.conf:/etc/jobservice/app.conf + - /data/secretkey:/harbor/secretkey depends_on: - ui - adminserver diff --git a/make/docker-compose.tpl b/make/docker-compose.tpl index b66eb2698..bdcb5a50a 100644 --- a/make/docker-compose.tpl +++ b/make/docker-compose.tpl @@ -49,6 +49,7 @@ services: restart: always volumes: - /data/config/:/etc/harbor/ + - /data/secretkey:/harbor/secretkey depends_on: - log logging: @@ -66,6 +67,7 @@ services: - ./common/config/ui/app.conf:/etc/ui/app.conf - ./common/config/ui/private_key.pem:/etc/ui/private_key.pem - /data:/harbor_storage + - /data/secretkey:/harbor/secretkey depends_on: - log - adminserver @@ -84,6 +86,7 @@ services: volumes: - /data/job_logs:/var/log/jobs - ./common/config/jobservice/app.conf:/etc/jobservice/app.conf + - /data/secretkey:/harbor/secretkey depends_on: - ui - adminserver diff --git a/make/prepare b/make/prepare index 28d46a376..de132c2a0 100755 --- a/make/prepare +++ b/make/prepare @@ -10,6 +10,9 @@ import argparse import subprocess import shutil from io import open +import base64 +from Crypto import Random +from Crypto.Cipher import AES if sys.version_info[:3][0] == 2: import ConfigParser as ConfigParser @@ -54,6 +57,11 @@ def get_secret_key(path): print("generated and saved secret key") return key +def aesEncrypt(key, raw): + iv = Random.new().read(AES.block_size) + cipher = AES.new(key, AES.MODE_CFB, iv) + return base64.b64encode(iv + cipher.encrypt(raw)) + base_dir = os.path.dirname(__file__) config_dir = os.path.join(base_dir, "common/config") templates_dir = os.path.join(base_dir, "common/templates") @@ -125,6 +133,7 @@ secret_key = get_secret_key(secretkey_path) ######## ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16)) +#ui_secret = aesEncrypt(secret_key, ui_secret_raw) adminserver_config_dir = os.path.join(config_dir,"adminserver") if not os.path.exists(adminserver_config_dir): diff --git a/src/adminserver/api/cfg.go b/src/adminserver/api/cfg.go index 6a2198c6e..3e89da5de 100644 --- a/src/adminserver/api/cfg.go +++ b/src/adminserver/api/cfg.go @@ -19,14 +19,13 @@ import ( "encoding/json" "io/ioutil" "net/http" - "os" + "github.com/vmware/harbor/src/adminserver/config" cfg "github.com/vmware/harbor/src/adminserver/systemcfg" "github.com/vmware/harbor/src/common/utils/log" ) func isAuthenticated(r *http.Request) (bool, error) { - secret := os.Getenv("UI_SECRET") c, err := r.Cookie("secret") if err != nil { if err == http.ErrNoCookie { @@ -34,7 +33,7 @@ func isAuthenticated(r *http.Request) (bool, error) { } return false, err } - return c != nil && c.Value == secret, nil + return c != nil && c.Value == config.Secret(), nil } // ListCfgs lists configurations diff --git a/src/adminserver/api/cfg_test.go b/src/adminserver/api/cfg_test.go index 8ae8553c2..3dcf9191f 100644 --- a/src/adminserver/api/cfg_test.go +++ b/src/adminserver/api/cfg_test.go @@ -25,24 +25,48 @@ import ( "os" "testing" + "github.com/vmware/harbor/src/adminserver/config" "github.com/vmware/harbor/src/adminserver/systemcfg" - "github.com/vmware/harbor/src/common/config" + comcfg "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/utils/test" ) func TestConfigAPI(t *testing.T) { - path := "/tmp/config.json" - secret := "secret" + configPath := "/tmp/config.json" + secretKeyPath := "/tmp/secretkey" + _, err := test.GenerateKey(secretKeyPath) + if err != nil { + t.Errorf("failed to generate secret key: %v", err) + return + } + defer os.Remove(secretKeyPath) + + secret := "secret" + /* + secretPlaintext := "secret" + secretCiphertext, err := utils.ReversibleEncrypt(secretPlaintext, string(data)) + if err != nil { + t.Errorf("failed to encrypt secret: %v", err) + return + } + */ envs := map[string]string{ - "JSON_STORE_PATH": path, - "UI_SECRET": secret, - "MYSQL_PORT": "3306", - "TOKEN_EXPIRATION": "30", - "CFG_EXPIRATION": "5", - "MAX_JOB_WORKERS": "3", - "LDAP_SCOPE": "3", - "LDAP_TIMEOUT": "30", - "EMAIL_PORT": "25", + + "JSON_STORE_PATH": configPath, + "KEY_PATH": secretKeyPath, + "UI_SECRET": secret, + "MYSQL_PORT": "3306", + "TOKEN_EXPIRATION": "30", + "CFG_EXPIRATION": "5", + "MAX_JOB_WORKERS": "3", + "LDAP_SCOPE": "3", + "LDAP_TIMEOUT": "30", + "EMAIL_PORT": "25", + "MYSQL_PWD": "", + "LDAP_SEARCH_PWD": "", + "EMAIL_PWD": "", + "HARBOR_ADMIN_PASSWORD": "", } for k, v := range envs { @@ -50,10 +74,15 @@ func TestConfigAPI(t *testing.T) { t.Fatalf("failed to set env %s: %v", k, err) } } - defer os.Remove(path) + defer os.Remove(configPath) + + if err := config.Init(); err != nil { + t.Errorf("failed to load configurations of adminserver: %v", err) + return + } if err := systemcfg.Init(); err != nil { - t.Errorf("failed to initialize systemconfigurations: %v", err) + t.Errorf("failed to initialize system configurations: %v", err) return } @@ -88,7 +117,7 @@ func TestConfigAPI(t *testing.T) { return } - scope := int(m[config.LDAPScope].(float64)) + scope := int(m[comcfg.LDAPScope].(float64)) if scope != 3 { t.Errorf("unexpected ldap scope: %d != %d", scope, 3) return @@ -96,7 +125,7 @@ func TestConfigAPI(t *testing.T) { // modify configurations c := map[string]interface{}{ - config.AUTHMode: config.LDAPAuth, + comcfg.AUTHMode: comcfg.LDAPAuth, } b, err := json.Marshal(c) @@ -146,9 +175,9 @@ func TestConfigAPI(t *testing.T) { return } - mode := m[config.AUTHMode].(string) - if mode != config.LDAPAuth { - t.Errorf("unexpected ldap scope: %s != %s", mode, config.LDAPAuth) + mode := m[comcfg.AUTHMode].(string) + if mode != comcfg.LDAPAuth { + t.Errorf("unexpected ldap scope: %s != %s", mode, comcfg.LDAPAuth) return } } diff --git a/src/adminserver/config/config.go b/src/adminserver/config/config.go new file mode 100644 index 000000000..e908517c9 --- /dev/null +++ b/src/adminserver/config/config.go @@ -0,0 +1,66 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "io/ioutil" + "os" + + //"github.com/vmware/harbor/src/common/utils" +) + +const defaultKeyPath string = "/harbor/secretkey" + +var ( + secret string + secretKey string +) + +// Init configurations used by adminserver +func Init() error { + path := os.Getenv("KEY_PATH") + if len(path) == 0 { + path = defaultKeyPath + } + + b, err := ioutil.ReadFile(path) + if err != nil { + return err + } + secretKey = string(b) + + secret = os.Getenv("UI_SECRET") + + /* + secretCipherText := os.Getenv("UI_SECRET") + + secret, err = utils.ReversibleDecrypt(secretCipherText, secretKey) + if err != nil { + return err + } + */ + return nil +} + +// Secret is used by API to authenticate requests +func Secret() string { + return secret +} + +// SecretKey is used to encrypt or decrypt +func SecretKey() string { + return secretKey +} diff --git a/src/adminserver/config/config_test.go b/src/adminserver/config/config_test.go new file mode 100644 index 000000000..19c43d4d9 --- /dev/null +++ b/src/adminserver/config/config_test.go @@ -0,0 +1,67 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "os" + "testing" + + "github.com/vmware/harbor/src/common/utils/test" +) + +func TestConfig(t *testing.T) { + secretKeyPath := "/tmp/secretkey" + + secretKey, err := test.GenerateKey(secretKeyPath) + if err != nil { + t.Errorf("failed to generate secret key: %v", err) + return + } + defer os.Remove(secretKeyPath) + + secret := "secret" + /* + secretPlaintext := "secret" + secretCiphertext, err := utils.ReversibleEncrypt(secretPlaintext, string(data)) + if err != nil { + t.Errorf("failed to encrypt secret: %v", err) + return + } + */ + envs := map[string]string{ + "KEY_PATH": secretKeyPath, + "UI_SECRET": secret, + } + + for k, v := range envs { + if err := os.Setenv(k, v); err != nil { + t.Fatalf("failed to set env %s: %v", k, err) + } + } + + if err := Init(); err != nil { + t.Errorf("failed to load configurations of adminserver: %v", err) + return + } + + if SecretKey() != secretKey { + t.Errorf("unexpected secret key: %s != %s", SecretKey(), secretKey) + } + + if Secret() != secret { + t.Errorf("unexpected secret: %s != %s", Secret(), secret) + } +} diff --git a/src/adminserver/main.go b/src/adminserver/main.go index d83d95d1a..d225e7bab 100644 --- a/src/adminserver/main.go +++ b/src/adminserver/main.go @@ -19,7 +19,8 @@ import ( "net/http" "os" - cfg "github.com/vmware/harbor/src/adminserver/systemcfg" + "github.com/vmware/harbor/src/adminserver/config" + syscfg "github.com/vmware/harbor/src/adminserver/systemcfg" "github.com/vmware/harbor/src/common/utils/log" ) @@ -40,8 +41,15 @@ func (s *Server) Serve() error { } func main() { + + log.Info("loading configurations of adminserver...") + if err := config.Init(); err != nil { + log.Fatalf("failed to load configurations of adminserver: %v", err) + } + log.Info("load completed") + log.Info("initializing system configurations...") - if err := cfg.Init(); err != nil { + if err := syscfg.Init(); err != nil { log.Fatalf("failed to initialize the system: %v", err) } log.Info("system initialization completed") diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index 4d22adc22..f12127888 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -20,12 +20,22 @@ import ( "os" "strconv" + "github.com/vmware/harbor/src/adminserver/config" "github.com/vmware/harbor/src/adminserver/systemcfg/store" "github.com/vmware/harbor/src/adminserver/systemcfg/store/json" comcfg "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" ) +// Keys need to be encrypted or decrypted +var Keys = []string{ + comcfg.EmailPassword, + comcfg.LDAPSearchPwd, + comcfg.MySQLPassword, + comcfg.AdminInitialPassword, +} + var cfgStore store.Driver // Init system configurations. Read from config store first, if null read from env @@ -43,7 +53,7 @@ func Init() (err error) { } log.Infof("configuration store driver: %s", cfgStore.Name()) - cfg, err := cfgStore.Read() + cfg, err := GetSystemCfg() if err != nil { return err } @@ -61,7 +71,7 @@ func Init() (err error) { } //sync configurations into cfg store - if err = cfgStore.Write(cfg); err != nil { + if err = UpdateSystemCfg(cfg); err != nil { return err } @@ -102,7 +112,6 @@ func readFromEnv(cfg map[string]interface{}) error { cfg[comcfg.JobLogDir] = os.Getenv("LOG_DIR") //TODO remove cfg[comcfg.UseCompressedJS] = os.Getenv("USE_COMPRESSED_JS") == "on" - cfg[comcfg.SecretKey] = os.Getenv("SECRET_KEY") cfgExpi, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) if err != nil { return err @@ -162,10 +171,64 @@ func initFromEnv() (map[string]interface{}, error) { // GetSystemCfg returns the system configurations func GetSystemCfg() (map[string]interface{}, error) { - return cfgStore.Read() + m, err := cfgStore.Read() + if err != nil { + return nil, err + } + + if err = decrypt(m, Keys, config.SecretKey()); err != nil { + return nil, err + } + + return m, nil } // UpdateSystemCfg updates the system configurations func UpdateSystemCfg(cfg map[string]interface{}) error { + + if err := encrypt(cfg, Keys, config.SecretKey()); err != nil { + return err + } + return cfgStore.Write(cfg) } + +func encrypt(m map[string]interface{}, keys []string, secretKey string) error { + for _, key := range keys { + v, ok := m[key] + if !ok { + continue + } + + if len(v.(string)) == 0 { + continue + } + + cipherText, err := utils.ReversibleEncrypt(v.(string), secretKey) + if err != nil { + return err + } + m[key] = cipherText + } + return nil +} + +func decrypt(m map[string]interface{}, keys []string, secretKey string) error { + for _, key := range keys { + v, ok := m[key] + if !ok { + continue + } + + if len(v.(string)) == 0 { + continue + } + + text, err := utils.ReversibleDecrypt(v.(string), secretKey) + if err != nil { + return err + } + m[key] = text + } + return nil +} diff --git a/src/adminserver/systemcfg/systemcfg_test.go b/src/adminserver/systemcfg/systemcfg_test.go index d04f31414..0f6db5ec2 100644 --- a/src/adminserver/systemcfg/systemcfg_test.go +++ b/src/adminserver/systemcfg/systemcfg_test.go @@ -39,14 +39,18 @@ func TestSystemcfg(t *testing.T) { } m := map[string]string{ - "AUTH_MODE": comcfg.DBAuth, - "LDAP_SCOPE": "1", - "LDAP_TIMEOUT": "30", - "MYSQL_PORT": "3306", - "MAX_JOB_WORKERS": "3", - "TOKEN_EXPIRATION": "30", - "CFG_EXPIRATION": "5", - "EMAIL_PORT": "25", + "AUTH_MODE": comcfg.DBAuth, + "LDAP_SCOPE": "1", + "LDAP_TIMEOUT": "30", + "MYSQL_PORT": "3306", + "MAX_JOB_WORKERS": "3", + "TOKEN_EXPIRATION": "30", + "CFG_EXPIRATION": "5", + "EMAIL_PORT": "25", + "MYSQL_PWD": "", + "LDAP_SEARCH_PWD": "", + "EMAIL_PWD": "", + "HARBOR_ADMIN_PASSWORD": "", } for k, v := range m { diff --git a/src/common/config/config.go b/src/common/config/config.go index 29c59f948..ff3b86c22 100644 --- a/src/common/config/config.go +++ b/src/common/config/config.go @@ -74,7 +74,6 @@ const ( CfgExpiration = "cfg_expiration" JobLogDir = "job_log_dir" UseCompressedJS = "use_compressed_js" - SecretKey = "secret_key" AdminInitialPassword = "admin_initial_password" ) diff --git a/src/common/utils/test/adminserver.go b/src/common/utils/test/adminserver.go index 93621d0fc..8f53eacee 100644 --- a/src/common/utils/test/adminserver.go +++ b/src/common/utils/test/adminserver.go @@ -24,7 +24,7 @@ import ( ) var adminServerDefaultConfig = map[string]interface{}{ - config.ExtEndpoint: "host01.com", + config.ExtEndpoint: "host01.com", config.AUTHMode: config.DBAuth, config.DatabaseType: "mysql", config.MySQLHost: "127.0.0.1", @@ -58,7 +58,6 @@ var adminServerDefaultConfig = map[string]interface{}{ config.CfgExpiration: 5, config.JobLogDir: "/var/log/jobs", config.UseCompressedJS: true, - config.SecretKey: "secret", config.AdminInitialPassword: "password", } diff --git a/src/common/utils/test/key.go b/src/common/utils/test/key.go new file mode 100644 index 000000000..bf7f3c80e --- /dev/null +++ b/src/common/utils/test/key.go @@ -0,0 +1,41 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "crypto/aes" + "crypto/rand" + "fmt" + "io/ioutil" +) + +// GenerateKey generates aes key +func GenerateKey(path string) (string, error) { + data := make([]byte, aes.BlockSize) + n, err := rand.Read(data) + if err != nil { + return "", fmt.Errorf("failed to generate random bytes: %v", err) + } + if n != aes.BlockSize { + return "", fmt.Errorf("the length of random bytes %d != %d", n, aes.BlockSize) + } + + if err = ioutil.WriteFile(path, data, 0777); err != nil { + return "", fmt.Errorf("failed write secret key to file %s: %v", path, err) + } + + return string(data), nil +} diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index 164f07f91..b57930a19 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -16,16 +16,55 @@ package config import ( + "io/ioutil" "os" comcfg "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/models" + //"github.com/vmware/harbor/src/common/utils" + //"github.com/vmware/harbor/src/common/utils/log" ) +const defaultKeyPath string = "/harbor/secretkey" + var mg *comcfg.Manager +var ( + secret string + secretKey string +) + +func initSecretAndKey() error { + path := os.Getenv("KEY_PATH") + if len(path) == 0 { + path = defaultKeyPath + } + + b, err := ioutil.ReadFile(path) + if err != nil { + return err + } + secretKey = string(b) + + secret = os.Getenv("UI_SECRET") + + /* + secretCipherText := os.Getenv("UI_SECRET") + + secret, err = utils.ReversibleDecrypt(secretCipherText, secretKey) + if err != nil { + log.Errorf("failed to decrypt secret: %v", err) + } + */ + return nil +} + // Init configurations func Init() error { + if err := initSecretAndKey(); err != nil { + return err + } + adminServerURL := os.Getenv("ADMIN_SERVER_URL") if len(adminServerURL) == 0 { adminServerURL = "http://adminserver" @@ -108,17 +147,13 @@ func LogDir() (string, error) { // SecretKey will return the secret key for encryption/decryption password in target. func SecretKey() (string, error) { - cfg, err := mg.Get() - if err != nil { - return "", err - } - return cfg[comcfg.SecretKey].(string), nil + return secretKey, nil } -// UISecret returns the value of UI secret cookie, used for communication between UI and JobService -// TODO +// UISecret returns a secret used for communication of UI, JobService +// and Adminserver func UISecret() string { - return os.Getenv("UI_SECRET") + return secret } // ExtEndpoint ... diff --git a/src/jobservice/config/config_test.go b/src/jobservice/config/config_test.go index c70af060c..556386020 100644 --- a/src/jobservice/config/config_test.go +++ b/src/jobservice/config/config_test.go @@ -30,13 +30,22 @@ func TestConfig(t *testing.T) { } defer server.Close() - url := os.Getenv("ADMIN_SERVER_URL") - defer os.Setenv("ADMIN_SERVER_URL", url) - if err := os.Setenv("ADMIN_SERVER_URL", server.URL); err != nil { t.Fatalf("failed to set env %s: %v", "ADMIN_SERVER_URL", err) } + secretKeyPath := "/tmp/secretkey" + _, err = test.GenerateKey(secretKeyPath) + if err != nil { + t.Errorf("failed to generate secret key: %v", err) + return + } + defer os.Remove(secretKeyPath) + + if err := os.Setenv("KEY_PATH", secretKeyPath); err != nil { + t.Fatalf("failed to set env %s: %v", "KEY_PATH", err) + } + if err := Init(); err != nil { t.Fatalf("failed to initialize configurations: %v", err) } diff --git a/src/ui/api/config.go b/src/ui/api/config.go index 98657f18a..05c4c5c44 100644 --- a/src/ui/api/config.go +++ b/src/ui/api/config.go @@ -215,8 +215,6 @@ func validateCfg(c map[string]string) (bool, error) { } } - log.Infof("===========%v", c) - if ldapURL, ok := c[comcfg.LDAPURL]; ok && len(ldapURL) == 0 { return isSysErr, fmt.Errorf("%s is empty", comcfg.LDAPURL) } @@ -325,8 +323,7 @@ func convertForGet(cfg map[string]interface{}) (map[string]*value, error) { comcfg.AdminInitialPassword, comcfg.EmailPassword, comcfg.LDAPSearchPwd, - comcfg.MySQLPassword, - comcfg.SecretKey} + comcfg.MySQLPassword} for _, del := range dels { if _, ok := cfg[del]; ok { delete(cfg, del) diff --git a/src/ui/config/config.go b/src/ui/config/config.go index 21bbe288b..d810da1df 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -17,17 +17,53 @@ package config import ( "encoding/json" + "io/ioutil" "os" comcfg "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/models" + //"github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" ) +const defaultKeyPath string = "/harbor/secretkey" + var mg *comcfg.Manager +var ( + secret string + secretKey string +) + +func initSecretAndKey() error { + path := os.Getenv("KEY_PATH") + if len(path) == 0 { + path = defaultKeyPath + } + + b, err := ioutil.ReadFile(path) + if err != nil { + return err + } + secretKey = string(b) + + secret = os.Getenv("UI_SECRET") + /* + secretCipherText := os.Getenv("UI_SECRET") + secret, err = utils.ReversibleDecrypt(secretCipherText, secretKey) + if err != nil { + return err + } + */ + return nil +} + // Init configurations func Init() error { + if err := initSecretAndKey(); err != nil { + return err + } + adminServerURL := os.Getenv("ADMIN_SERVER_URL") if len(adminServerURL) == 0 { adminServerURL = "http://adminserver" @@ -125,11 +161,7 @@ func ExtEndpoint() (string, error) { // SecretKey returns the secret key to encrypt the password of target func SecretKey() (string, error) { - cfg, err := mg.Get() - if err != nil { - return "", err - } - return cfg[comcfg.SecretKey].(string), nil + return secretKey, nil } // SelfRegistration returns the enablement of self registration @@ -228,8 +260,8 @@ func Database() (*models.Database, error) { return database, nil } -// UISecret returns the value of UI secret cookie, used for communication between UI and JobService -// TODO +// UISecret returns a secret used for communication of UI, JobService +// and Adminserver func UISecret() string { - return os.Getenv("UI_SECRET") + return secret } diff --git a/src/ui/config/config_test.go b/src/ui/config/config_test.go index e06798c52..789cf53d6 100644 --- a/src/ui/config/config_test.go +++ b/src/ui/config/config_test.go @@ -29,13 +29,22 @@ func TestConfig(t *testing.T) { } defer server.Close() - url := os.Getenv("ADMIN_SERVER_URL") - defer os.Setenv("ADMIN_SERVER_URL", url) - if err := os.Setenv("ADMIN_SERVER_URL", server.URL); err != nil { t.Fatalf("failed to set env %s: %v", "ADMIN_SERVER_URL", err) } + secretKeyPath := "/tmp/secretkey" + _, err = test.GenerateKey(secretKeyPath) + if err != nil { + t.Errorf("failed to generate secret key: %v", err) + return + } + defer os.Remove(secretKeyPath) + + if err := os.Setenv("KEY_PATH", secretKeyPath); err != nil { + t.Fatalf("failed to set env %s: %v", "KEY_PATH", err) + } + if err := Init(); err != nil { t.Fatalf("failed to initialize configurations: %v", err) } diff --git a/tests/docker-compose.test.yml b/tests/docker-compose.test.yml index 0b8499a22..92b5d68d4 100644 --- a/tests/docker-compose.test.yml +++ b/tests/docker-compose.test.yml @@ -30,6 +30,7 @@ services: restart: always volumes: - /data/config/:/etc/harbor/ + - /data/secretkey:/harbor/secretkey ports: - 8888:80 ldap: diff --git a/tests/testprepare.sh b/tests/testprepare.sh index 89200e1d0..cb337e2f9 100755 --- a/tests/testprepare.sh +++ b/tests/testprepare.sh @@ -11,3 +11,5 @@ cp make/common/config/ui/app.conf conf/. sed -i -r "s/MYSQL_HOST=mysql/MYSQL_HOST=127.0.0.1/" make/common/config/adminserver/env sed -i -r "s|REGISTRY_URL=http://registry:5000|REGISTRY_URL=http://127.0.0.1:5000|" make/common/config/adminserver/env sed -i -r "s/UI_SECRET=.*/UI_SECRET=$UI_SECRET/" make/common/config/adminserver/env + +chmod 777 /data/ \ No newline at end of file From bf39b3a956db8e0d8447cf13bcca376b38a372a7 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Tue, 21 Feb 2017 13:45:12 +0800 Subject: [PATCH 18/22] modify Makefile of Harbor based on photon (#1404) LGTM --- make/photon/Makefile | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/make/photon/Makefile b/make/photon/Makefile index 0a1aeaaad..1b7673ed5 100644 --- a/make/photon/Makefile +++ b/make/photon/Makefile @@ -3,7 +3,7 @@ # Targets: # # build: build harbor photon images -# clean: clean ui and jobservice harbor images +# clean: clean adminserver, ui and jobservice harbor images # common SHELL := /bin/bash @@ -22,6 +22,9 @@ DOCKERRMIMAGE=$(DOCKERCMD) rmi DOCKERIMASES=$(DOCKERCMD) images # binary +ADMINSERVERSOURCECODE=$(SRCPATH)/adminserver +ADMINSERVERBINARYPATH=$(MAKEDEVPATH)/adminserver +ADMINSERVERBINARYNAME=harbor_adminserver UISOURCECODE=$(SRCPATH)/ui UIBINARYPATH=$(MAKEDEVPATH)/ui UIBINARYNAME=harbor_ui @@ -31,6 +34,9 @@ JOBSERVICEBINARYNAME=harbor_jobservice # photon dockerfile DOCKERFILEPATH=$(MAKEPATH)/photon +DOCKERFILEPATH_ADMINSERVER=$(DOCKERFILEPATH)/adminserver +DOCKERFILENAME_ADMINSERVER=Dockerfile +DOCKERIMAGENAME_ADMINSERVER=vmware/harbor-adminserver DOCKERFILEPATH_UI=$(DOCKERFILEPATH)/ui DOCKERFILENAME_UI=Dockerfile DOCKERIMAGENAME_UI=vmware/harbor-ui @@ -56,6 +62,10 @@ check_environment: @$(MAKEPATH)/$(CHECKENVCMD) build: + @echo "building adminserver container for photon..." + $(DOCKERBUILD) -f $(DOCKERFILEPATH_ADMINSERVER)/$(DOCKERFILENAME_ADMINSERVER) -t $(DOCKERIMAGENAME_ADMINSERVER):$(VERSIONTAG) . + @echo "Done." + @echo "building ui container for photon..." $(DOCKERBUILD) -f $(DOCKERFILEPATH_UI)/$(DOCKERFILENAME_UI) -t $(DOCKERIMAGENAME_UI):$(VERSIONTAG) . @echo "Done." @@ -70,6 +80,7 @@ build: cleanimage: @echo "cleaning image for photon..." + - $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_ADMINSERVER):$(VERSIONTAG) - $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_UI):$(VERSIONTAG) - $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_JOBSERVICE):$(VERSIONTAG) - $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_LOG):$(VERSIONTAG) From 40eb6bb7d33af4528e66b03a29c32904fc02d1b5 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 22 Feb 2017 13:22:52 +0800 Subject: [PATCH 19/22] encrypt passwords enhancement --- make/dev/docker-compose.yml | 8 +- make/docker-compose.tpl | 8 +- make/prepare | 9 --- src/adminserver/api/cfg.go | 5 +- src/adminserver/api/cfg_test.go | 19 +---- src/adminserver/config/config.go | 66 --------------- src/adminserver/config/config_test.go | 67 ---------------- src/adminserver/main.go | 8 -- src/adminserver/systemcfg/systemcfg.go | 89 +++++++++++++++------ src/adminserver/systemcfg/systemcfg_test.go | 39 +++++---- src/common/config/keyprovider.go | 47 +++++++++++ src/common/config/keyprovider_test.go | 43 ++++++++++ src/jobservice/config/config.go | 58 +++++--------- src/ui/config/config.go | 52 ++++-------- tests/docker-compose.test.yml | 4 +- 15 files changed, 231 insertions(+), 291 deletions(-) delete mode 100644 src/adminserver/config/config.go delete mode 100644 src/adminserver/config/config_test.go create mode 100644 src/common/config/keyprovider.go create mode 100644 src/common/config/keyprovider_test.go diff --git a/make/dev/docker-compose.yml b/make/dev/docker-compose.yml index 5f8574c15..d49f0db26 100644 --- a/make/dev/docker-compose.yml +++ b/make/dev/docker-compose.yml @@ -48,8 +48,8 @@ services: - ../common/config/adminserver/env restart: always volumes: - - /data/config/:/etc/harbor/ - - /data/secretkey:/harbor/secretkey + - /data/config/:/etc/adminserver/ + - /data/secretkey:/etc/adminserver/key depends_on: - log logging: @@ -67,7 +67,7 @@ services: volumes: - ../common/config/ui/app.conf:/etc/ui/app.conf - ../common/config/ui/private_key.pem:/etc/ui/private_key.pem - - /data/secretkey:/harbor/secretkey + - /data/secretkey:/etc/ui/key depends_on: - log - adminserver @@ -87,7 +87,7 @@ services: volumes: - /data/job_logs:/var/log/jobs - ../common/config/jobservice/app.conf:/etc/jobservice/app.conf - - /data/secretkey:/harbor/secretkey + - /data/secretkey:/etc/jobservice/key depends_on: - ui - adminserver diff --git a/make/docker-compose.tpl b/make/docker-compose.tpl index bdcb5a50a..7138e7c6b 100644 --- a/make/docker-compose.tpl +++ b/make/docker-compose.tpl @@ -48,8 +48,8 @@ services: - ./common/config/adminserver/env restart: always volumes: - - /data/config/:/etc/harbor/ - - /data/secretkey:/harbor/secretkey + - /data/config/:/etc/adminserver/ + - /data/secretkey:/etc/adminserver/key depends_on: - log logging: @@ -67,7 +67,7 @@ services: - ./common/config/ui/app.conf:/etc/ui/app.conf - ./common/config/ui/private_key.pem:/etc/ui/private_key.pem - /data:/harbor_storage - - /data/secretkey:/harbor/secretkey + - /data/secretkey:/etc/ui/key depends_on: - log - adminserver @@ -86,7 +86,7 @@ services: volumes: - /data/job_logs:/var/log/jobs - ./common/config/jobservice/app.conf:/etc/jobservice/app.conf - - /data/secretkey:/harbor/secretkey + - /data/secretkey:/etc/jobservice/key depends_on: - ui - adminserver diff --git a/make/prepare b/make/prepare index de132c2a0..28d46a376 100755 --- a/make/prepare +++ b/make/prepare @@ -10,9 +10,6 @@ import argparse import subprocess import shutil from io import open -import base64 -from Crypto import Random -from Crypto.Cipher import AES if sys.version_info[:3][0] == 2: import ConfigParser as ConfigParser @@ -57,11 +54,6 @@ def get_secret_key(path): print("generated and saved secret key") return key -def aesEncrypt(key, raw): - iv = Random.new().read(AES.block_size) - cipher = AES.new(key, AES.MODE_CFB, iv) - return base64.b64encode(iv + cipher.encrypt(raw)) - base_dir = os.path.dirname(__file__) config_dir = os.path.join(base_dir, "common/config") templates_dir = os.path.join(base_dir, "common/templates") @@ -133,7 +125,6 @@ secret_key = get_secret_key(secretkey_path) ######## ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16)) -#ui_secret = aesEncrypt(secret_key, ui_secret_raw) adminserver_config_dir = os.path.join(config_dir,"adminserver") if not os.path.exists(adminserver_config_dir): diff --git a/src/adminserver/api/cfg.go b/src/adminserver/api/cfg.go index 3e89da5de..6a2198c6e 100644 --- a/src/adminserver/api/cfg.go +++ b/src/adminserver/api/cfg.go @@ -19,13 +19,14 @@ import ( "encoding/json" "io/ioutil" "net/http" + "os" - "github.com/vmware/harbor/src/adminserver/config" cfg "github.com/vmware/harbor/src/adminserver/systemcfg" "github.com/vmware/harbor/src/common/utils/log" ) func isAuthenticated(r *http.Request) (bool, error) { + secret := os.Getenv("UI_SECRET") c, err := r.Cookie("secret") if err != nil { if err == http.ErrNoCookie { @@ -33,7 +34,7 @@ func isAuthenticated(r *http.Request) (bool, error) { } return false, err } - return c != nil && c.Value == config.Secret(), nil + return c != nil && c.Value == secret, nil } // ListCfgs lists configurations diff --git a/src/adminserver/api/cfg_test.go b/src/adminserver/api/cfg_test.go index 3dcf9191f..79c267a99 100644 --- a/src/adminserver/api/cfg_test.go +++ b/src/adminserver/api/cfg_test.go @@ -25,7 +25,6 @@ import ( "os" "testing" - "github.com/vmware/harbor/src/adminserver/config" "github.com/vmware/harbor/src/adminserver/systemcfg" comcfg "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/utils/test" @@ -43,17 +42,9 @@ func TestConfigAPI(t *testing.T) { defer os.Remove(secretKeyPath) secret := "secret" - /* - secretPlaintext := "secret" - secretCiphertext, err := utils.ReversibleEncrypt(secretPlaintext, string(data)) - if err != nil { - t.Errorf("failed to encrypt secret: %v", err) - return - } - */ envs := map[string]string{ - "JSON_STORE_PATH": configPath, + "JSON_CFG_STORE_PATH": configPath, "KEY_PATH": secretKeyPath, "UI_SECRET": secret, "MYSQL_PORT": "3306", @@ -71,16 +62,12 @@ func TestConfigAPI(t *testing.T) { for k, v := range envs { if err := os.Setenv(k, v); err != nil { - t.Fatalf("failed to set env %s: %v", k, err) + t.Errorf("failed to set env %s: %v", k, err) + return } } defer os.Remove(configPath) - if err := config.Init(); err != nil { - t.Errorf("failed to load configurations of adminserver: %v", err) - return - } - if err := systemcfg.Init(); err != nil { t.Errorf("failed to initialize system configurations: %v", err) return diff --git a/src/adminserver/config/config.go b/src/adminserver/config/config.go deleted file mode 100644 index e908517c9..000000000 --- a/src/adminserver/config/config.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright (c) 2016 VMware, Inc. All Rights Reserved. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package config - -import ( - "io/ioutil" - "os" - - //"github.com/vmware/harbor/src/common/utils" -) - -const defaultKeyPath string = "/harbor/secretkey" - -var ( - secret string - secretKey string -) - -// Init configurations used by adminserver -func Init() error { - path := os.Getenv("KEY_PATH") - if len(path) == 0 { - path = defaultKeyPath - } - - b, err := ioutil.ReadFile(path) - if err != nil { - return err - } - secretKey = string(b) - - secret = os.Getenv("UI_SECRET") - - /* - secretCipherText := os.Getenv("UI_SECRET") - - secret, err = utils.ReversibleDecrypt(secretCipherText, secretKey) - if err != nil { - return err - } - */ - return nil -} - -// Secret is used by API to authenticate requests -func Secret() string { - return secret -} - -// SecretKey is used to encrypt or decrypt -func SecretKey() string { - return secretKey -} diff --git a/src/adminserver/config/config_test.go b/src/adminserver/config/config_test.go deleted file mode 100644 index 19c43d4d9..000000000 --- a/src/adminserver/config/config_test.go +++ /dev/null @@ -1,67 +0,0 @@ -/* - Copyright (c) 2016 VMware, Inc. All Rights Reserved. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package config - -import ( - "os" - "testing" - - "github.com/vmware/harbor/src/common/utils/test" -) - -func TestConfig(t *testing.T) { - secretKeyPath := "/tmp/secretkey" - - secretKey, err := test.GenerateKey(secretKeyPath) - if err != nil { - t.Errorf("failed to generate secret key: %v", err) - return - } - defer os.Remove(secretKeyPath) - - secret := "secret" - /* - secretPlaintext := "secret" - secretCiphertext, err := utils.ReversibleEncrypt(secretPlaintext, string(data)) - if err != nil { - t.Errorf("failed to encrypt secret: %v", err) - return - } - */ - envs := map[string]string{ - "KEY_PATH": secretKeyPath, - "UI_SECRET": secret, - } - - for k, v := range envs { - if err := os.Setenv(k, v); err != nil { - t.Fatalf("failed to set env %s: %v", k, err) - } - } - - if err := Init(); err != nil { - t.Errorf("failed to load configurations of adminserver: %v", err) - return - } - - if SecretKey() != secretKey { - t.Errorf("unexpected secret key: %s != %s", SecretKey(), secretKey) - } - - if Secret() != secret { - t.Errorf("unexpected secret: %s != %s", Secret(), secret) - } -} diff --git a/src/adminserver/main.go b/src/adminserver/main.go index d225e7bab..90b889372 100644 --- a/src/adminserver/main.go +++ b/src/adminserver/main.go @@ -19,7 +19,6 @@ import ( "net/http" "os" - "github.com/vmware/harbor/src/adminserver/config" syscfg "github.com/vmware/harbor/src/adminserver/systemcfg" "github.com/vmware/harbor/src/common/utils/log" ) @@ -41,13 +40,6 @@ func (s *Server) Serve() error { } func main() { - - log.Info("loading configurations of adminserver...") - if err := config.Init(); err != nil { - log.Fatalf("failed to load configurations of adminserver: %v", err) - } - log.Info("load completed") - log.Info("initializing system configurations...") if err := syscfg.Init(); err != nil { log.Fatalf("failed to initialize the system: %v", err) diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index f12127888..efef094e7 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -20,7 +20,6 @@ import ( "os" "strconv" - "github.com/vmware/harbor/src/adminserver/config" "github.com/vmware/harbor/src/adminserver/systemcfg/store" "github.com/vmware/harbor/src/adminserver/systemcfg/store/json" comcfg "github.com/vmware/harbor/src/common/config" @@ -28,31 +27,34 @@ import ( "github.com/vmware/harbor/src/common/utils/log" ) -// Keys need to be encrypted or decrypted -var Keys = []string{ - comcfg.EmailPassword, - comcfg.LDAPSearchPwd, - comcfg.MySQLPassword, - comcfg.AdminInitialPassword, -} +const ( + defaultCfgStoreDriver string = "json" + defaultJSONCfgStorePath string = "/etc/adminserver/config.json" + defaultKeyPath string = "/etc/adminserver/key" +) -var cfgStore store.Driver +var ( + // attrs need to be encrypted or decrypted + attrs = []string{ + comcfg.EmailPassword, + comcfg.LDAPSearchPwd, + comcfg.MySQLPassword, + comcfg.AdminInitialPassword, + } + cfgStore store.Driver + keyProvider comcfg.KeyProvider +) // Init system configurations. Read from config store first, if null read from env func Init() (err error) { - s := getCfgStore() - switch s { - case "json": - path := os.Getenv("JSON_STORE_PATH") - cfgStore, err = json.NewCfgStore(path) - if err != nil { - return - } - default: - return fmt.Errorf("unsupported configuration store driver %s", s) + //init configuation store + if err = initCfgStore(); err != nil { + return err } - log.Infof("configuration store driver: %s", cfgStore.Name()) + //init key provider + initKeyProvider() + cfg, err := GetSystemCfg() if err != nil { return err @@ -78,12 +80,37 @@ func Init() (err error) { return nil } -func getCfgStore() string { - t := os.Getenv("CFG_STORE_TYPE") +func initCfgStore() (err error) { + t := os.Getenv("CFG_STORE_DRIVER") if len(t) == 0 { - t = "json" + t = defaultCfgStoreDriver } - return t + log.Infof("configuration store driver: %s", t) + + switch t { + case "json": + path := os.Getenv("JSON_CFG_STORE_PATH") + if len(path) == 0 { + path = defaultJSONCfgStorePath + } + log.Infof("json configuration store path: %s", path) + + cfgStore, err = json.NewCfgStore(path) + default: + err = fmt.Errorf("unsupported configuration store driver %s", t) + } + + return err +} + +func initKeyProvider() { + path := os.Getenv("KEY_PATH") + if len(path) == 0 { + path = defaultKeyPath + } + log.Infof("key path: %s", path) + + keyProvider = comcfg.NewFileKeyProvider(path) } //read the following attrs from env every time boots up @@ -176,7 +203,12 @@ func GetSystemCfg() (map[string]interface{}, error) { return nil, err } - if err = decrypt(m, Keys, config.SecretKey()); err != nil { + key, err := keyProvider.Get(nil) + if err != nil { + return nil, fmt.Errorf("failed to get key: %v", err) + } + + if err = decrypt(m, attrs, key); err != nil { return nil, err } @@ -186,7 +218,12 @@ func GetSystemCfg() (map[string]interface{}, error) { // UpdateSystemCfg updates the system configurations func UpdateSystemCfg(cfg map[string]interface{}) error { - if err := encrypt(cfg, Keys, config.SecretKey()); err != nil { + key, err := keyProvider.Get(nil) + if err != nil { + return fmt.Errorf("failed to get key: %v", err) + } + + if err := encrypt(cfg, attrs, key); err != nil { return err } diff --git a/src/adminserver/systemcfg/systemcfg_test.go b/src/adminserver/systemcfg/systemcfg_test.go index 0f6db5ec2..4b0d3f557 100644 --- a/src/adminserver/systemcfg/systemcfg_test.go +++ b/src/adminserver/systemcfg/systemcfg_test.go @@ -20,22 +20,37 @@ import ( "testing" comcfg "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/utils/test" ) -// test functions under adminserver/systemcfg +// test functions in adminserver/systemcfg/systemcfg.go func TestSystemcfg(t *testing.T) { - key := "JSON_STORE_PATH" - path := "/tmp/config.json" - if _, err := os.Stat(path); err == nil { - if err := os.Remove(path); err != nil { - t.Fatalf("failed to remove %s: %v", path, err) + configPath := "/tmp/config.json" + if _, err := os.Stat(configPath); err == nil { + if err := os.Remove(configPath); err != nil { + t.Errorf("failed to remove %s: %v", configPath, err) + return } } else if !os.IsNotExist(err) { - t.Fatalf("failed to check the existence of %s: %v", path, err) + t.Errorf("failed to check the existence of %s: %v", configPath, err) + return } - if err := os.Setenv(key, path); err != nil { - t.Fatalf("failed to set env %s: %v", key, err) + if err := os.Setenv("JSON_CFG_STORE_PATH", configPath); err != nil { + t.Errorf("failed to set env: %v", err) + return + } + + keyPath := "/tmp/secretkey" + if _, err := test.GenerateKey(keyPath); err != nil { + t.Errorf("failed to generate key: %v", err) + return + } + defer os.Remove(keyPath) + + if err := os.Setenv("KEY_PATH", keyPath); err != nil { + t.Errorf("failed to set env: %v", err) + return } m := map[string]string{ @@ -63,11 +78,7 @@ func TestSystemcfg(t *testing.T) { t.Errorf("failed to initialize system configurations: %v", err) return } - defer func() { - if err := os.Remove(path); err != nil { - t.Fatalf("failed to remove %s: %v", path, err) - } - }() + defer os.Remove(configPath) // run Init again to make sure it works well when the configuration file // already exists diff --git a/src/common/config/keyprovider.go b/src/common/config/keyprovider.go new file mode 100644 index 000000000..a33319026 --- /dev/null +++ b/src/common/config/keyprovider.go @@ -0,0 +1,47 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "io/ioutil" +) + +// KeyProvider provides the key used to encrypt and decrypt attrs +type KeyProvider interface { + // Get returns the key + // params can be used to pass parameters in different implements + Get(params map[string]interface{}) (string, error) +} + +// FileKeyProvider reads key from file +type FileKeyProvider struct { + path string +} + +// NewFileKeyProvider returns an instance of FileKeyProvider +// path: where the key should be read from +func NewFileKeyProvider(path string) KeyProvider { + return &FileKeyProvider{ + path: path, + } +} + +// Get returns the key read from file +func (f *FileKeyProvider) Get(params map[string]interface{}) (string, error) { + b, err := ioutil.ReadFile(f.path) + if err != nil { + return "", err + } + return string(b), nil +} diff --git a/src/common/config/keyprovider_test.go b/src/common/config/keyprovider_test.go new file mode 100644 index 000000000..9fe05da0f --- /dev/null +++ b/src/common/config/keyprovider_test.go @@ -0,0 +1,43 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestGetOfFileKeyProvider(t *testing.T) { + path := "/tmp/key" + key := "key_content" + + if err := ioutil.WriteFile(path, []byte(key), 0777); err != nil { + t.Errorf("failed to write to file %s: %v", path, err) + return + } + defer os.Remove(path) + + provider := NewFileKeyProvider(path) + k, err := provider.Get(nil) + if err != nil { + t.Errorf("failed to get key from the file provider: %v", err) + return + } + + if k != key { + t.Errorf("unexpected key: %s != %s", k, key) + return + } +} diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index b57930a19..0a13ab5d4 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -16,54 +16,26 @@ package config import ( - "io/ioutil" "os" comcfg "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/models" - //"github.com/vmware/harbor/src/common/utils" - //"github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/common/utils/log" ) -const defaultKeyPath string = "/harbor/secretkey" - -var mg *comcfg.Manager +const ( + defaultKeyPath string = "/etc/jobservice/key" +) var ( - secret string - secretKey string + mg *comcfg.Manager + keyProvider comcfg.KeyProvider ) -func initSecretAndKey() error { - path := os.Getenv("KEY_PATH") - if len(path) == 0 { - path = defaultKeyPath - } - - b, err := ioutil.ReadFile(path) - if err != nil { - return err - } - secretKey = string(b) - - secret = os.Getenv("UI_SECRET") - - /* - secretCipherText := os.Getenv("UI_SECRET") - - secret, err = utils.ReversibleDecrypt(secretCipherText, secretKey) - if err != nil { - log.Errorf("failed to decrypt secret: %v", err) - } - */ - return nil -} - // Init configurations func Init() error { - if err := initSecretAndKey(); err != nil { - return err - } + //init key provider + initKeyProvider() adminServerURL := os.Getenv("ADMIN_SERVER_URL") if len(adminServerURL) == 0 { @@ -82,6 +54,16 @@ func Init() error { return nil } +func initKeyProvider() { + path := os.Getenv("KEY_PATH") + if len(path) == 0 { + path = defaultKeyPath + } + log.Infof("key path: %s", path) + + keyProvider = comcfg.NewFileKeyProvider(path) +} + // VerifyRemoteCert returns bool value. func VerifyRemoteCert() (bool, error) { cfg, err := mg.Get() @@ -147,13 +129,13 @@ func LogDir() (string, error) { // SecretKey will return the secret key for encryption/decryption password in target. func SecretKey() (string, error) { - return secretKey, nil + return keyProvider.Get(nil) } // UISecret returns a secret used for communication of UI, JobService // and Adminserver func UISecret() string { - return secret + return os.Getenv("UI_SECRET") } // ExtEndpoint ... diff --git a/src/ui/config/config.go b/src/ui/config/config.go index d810da1df..fb100fe7f 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -17,52 +17,24 @@ package config import ( "encoding/json" - "io/ioutil" "os" comcfg "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/models" - //"github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" ) -const defaultKeyPath string = "/harbor/secretkey" - -var mg *comcfg.Manager +const defaultKeyPath string = "/etc/ui/key" var ( - secret string - secretKey string + mg *comcfg.Manager + keyProvider comcfg.KeyProvider ) -func initSecretAndKey() error { - path := os.Getenv("KEY_PATH") - if len(path) == 0 { - path = defaultKeyPath - } - - b, err := ioutil.ReadFile(path) - if err != nil { - return err - } - secretKey = string(b) - - secret = os.Getenv("UI_SECRET") - /* - secretCipherText := os.Getenv("UI_SECRET") - secret, err = utils.ReversibleDecrypt(secretCipherText, secretKey) - if err != nil { - return err - } - */ - return nil -} - // Init configurations func Init() error { - if err := initSecretAndKey(); err != nil { - return err - } + //init key provider + initKeyProvider() adminServerURL := os.Getenv("ADMIN_SERVER_URL") if len(adminServerURL) == 0 { @@ -82,6 +54,16 @@ func Init() error { return nil } +func initKeyProvider() { + path := os.Getenv("KEY_PATH") + if len(path) == 0 { + path = defaultKeyPath + } + log.Infof("key path: %s", path) + + keyProvider = comcfg.NewFileKeyProvider(path) +} + // Load configurations func Load() error { _, err := mg.Load() @@ -161,7 +143,7 @@ func ExtEndpoint() (string, error) { // SecretKey returns the secret key to encrypt the password of target func SecretKey() (string, error) { - return secretKey, nil + return keyProvider.Get(nil) } // SelfRegistration returns the enablement of self registration @@ -263,5 +245,5 @@ func Database() (*models.Database, error) { // UISecret returns a secret used for communication of UI, JobService // and Adminserver func UISecret() string { - return secret + return os.Getenv("UI_SECRET") } diff --git a/tests/docker-compose.test.yml b/tests/docker-compose.test.yml index 92b5d68d4..8d209b770 100644 --- a/tests/docker-compose.test.yml +++ b/tests/docker-compose.test.yml @@ -29,8 +29,8 @@ services: - ./common/config/adminserver/env restart: always volumes: - - /data/config/:/etc/harbor/ - - /data/secretkey:/harbor/secretkey + - /data/config/:/etc/adminserver/ + - /data/secretkey:/etc/adminserver/key ports: - 8888:80 ldap: From e2c7cfc0adee842be8f161255ac3b59c381cb556 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 23 Feb 2017 14:24:49 +0800 Subject: [PATCH 20/22] support changing all configurations through API --- .travis.yml | 1 + src/adminserver/systemcfg/systemcfg.go | 209 ++++++++++++++----------- src/ui/api/config.go | 164 ++++++++++--------- 3 files changed, 208 insertions(+), 166 deletions(-) diff --git a/.travis.yml b/.travis.yml index c7b40e1bc..170581ecf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,6 +89,7 @@ script: - docker-compose -f make/docker-compose.test.yml down - sudo make/prepare + - sudo rm -rf /data/config/* - docker-compose -f make/dev/docker-compose.yml up -d - docker ps diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index efef094e7..24d13ba84 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -19,6 +19,7 @@ import ( "fmt" "os" "strconv" + "strings" "github.com/vmware/harbor/src/adminserver/systemcfg/store" "github.com/vmware/harbor/src/adminserver/systemcfg/store/json" @@ -34,6 +35,9 @@ const ( ) var ( + cfgStore store.Driver + keyProvider comcfg.KeyProvider + // attrs need to be encrypted or decrypted attrs = []string{ comcfg.EmailPassword, @@ -41,11 +45,98 @@ var ( comcfg.MySQLPassword, comcfg.AdminInitialPassword, } - cfgStore store.Driver - keyProvider comcfg.KeyProvider + + // envs are configurations need read from environment variables + envs = map[string]interface{}{ + comcfg.ExtEndpoint: "EXT_ENDPOINT", + comcfg.AUTHMode: "AUTH_MODE", + comcfg.SelfRegistration: &parser{ + env: "SELF_REGISTRATION", + parse: parseStringToBool, + }, + comcfg.DatabaseType: "DATABASE_TYPE", + comcfg.MySQLHost: "MYSQL_HOST", + comcfg.MySQLPort: &parser{ + env: "MYSQL_PORT", + parse: parseStringToInt, + }, + comcfg.MySQLUsername: "MYSQL_USR", + comcfg.MySQLPassword: "MYSQL_PWD", + comcfg.MySQLDatabase: "MYSQL_DATABASE", + comcfg.SQLiteFile: "SQLITE_FILE", + comcfg.LDAPURL: "LDAP_URL", + comcfg.LDAPSearchDN: "LDAP_SEARCH_DN", + comcfg.LDAPSearchPwd: "LDAP_SEARCH_PWD", + comcfg.LDAPBaseDN: "LDAP_BASE_DN", + comcfg.LDAPFilter: "LDAP_FILTER", + comcfg.LDAPUID: "LDAP_UID", + comcfg.LDAPScope: &parser{ + env: "LDAP_SCOPE", + parse: parseStringToInt, + }, + comcfg.LDAPTimeout: &parser{ + env: "LDAP_TIMEOUT", + parse: parseStringToInt, + }, + comcfg.EmailHost: "EMAIL_HOST", + comcfg.EmailPort: &parser{ + env: "EMAIL_PORT", + parse: parseStringToInt, + }, + comcfg.EmailUsername: "EMAIL_USR", + comcfg.EmailPassword: "EMAIL_PWD", + comcfg.EmailSSL: &parser{ + env: "EMAIL_SSL", + parse: parseStringToBool, + }, + comcfg.EmailFrom: "EMAIL_FROM", + comcfg.EmailIdentity: "EMAIL_IDENTITY", + comcfg.RegistryURL: "REGISTRY_URL", + comcfg.TokenExpiration: &parser{ + env: "TOKEN_EXPIRATION", + parse: parseStringToInt, + }, + comcfg.JobLogDir: "LOG_DIR", + comcfg.UseCompressedJS: &parser{ + env: "USE_COMPRESSED_JS", + parse: parseStringToBool, + }, + comcfg.CfgExpiration: &parser{ + env: "CFG_EXPIRATION", + parse: parseStringToInt, + }, + comcfg.MaxJobWorkers: &parser{ + env: "MAX_JOB_WORKERS", + parse: parseStringToInt, + }, + comcfg.VerifyRemoteCert: &parser{ + env: "VERIFY_REMOTE_CERT", + parse: parseStringToBool, + }, + comcfg.ProjectCreationRestriction: "PROJECT_CREATION_RESTRICTION", + comcfg.AdminInitialPassword: "HARBOR_ADMIN_PASSWORD", + } ) -// Init system configurations. Read from config store first, if null read from env +type parser struct { + // the name of env + env string + // parse the value of env, e.g. parse string to int or + // parse string to bool + parse func(string) (interface{}, error) +} + +func parseStringToInt(str string) (interface{}, error) { + return strconv.Atoi(str) +} + +func parseStringToBool(str string) (interface{}, error) { + return strings.ToLower(str) == "true" || + strings.ToLower(str) == "on", nil +} + +// Init system configurations. Read from config store first, +// if null read from env func Init() (err error) { //init configuation store if err = initCfgStore(); err != nil { @@ -60,24 +151,18 @@ func Init() (err error) { return err } - if cfg == nil { - log.Info("configurations read from store driver are null, initializing system from environment variables...") - cfg, err = initFromEnv() - if err != nil { - return err - } - } else { - if err := readFromEnv(cfg); err != nil { - return err - } + if cfg != nil { + return nil } - //sync configurations into cfg store - if err = UpdateSystemCfg(cfg); err != nil { + log.Info("configurations read from store driver are null, initializing system from environment variables...") + cfg, err = loadFromEnv() + if err != nil { return err } - return nil + //sync configurations into cfg store + return UpdateSystemCfg(cfg) } func initCfgStore() (err error) { @@ -113,85 +198,27 @@ func initKeyProvider() { keyProvider = comcfg.NewFileKeyProvider(path) } -//read the following attrs from env every time boots up -func readFromEnv(cfg map[string]interface{}) error { - cfg[comcfg.ExtEndpoint] = os.Getenv("EXT_ENDPOINT") - - cfg[comcfg.DatabaseType] = os.Getenv("DATABASE_TYPE") - cfg[comcfg.MySQLHost] = os.Getenv("MYSQL_HOST") - port, err := strconv.Atoi(os.Getenv("MYSQL_PORT")) - if err != nil { - return err - } - cfg[comcfg.MySQLPort] = port - cfg[comcfg.MySQLUsername] = os.Getenv("MYSQL_USR") - cfg[comcfg.MySQLPassword] = os.Getenv("MYSQL_PWD") - cfg[comcfg.MySQLDatabase] = os.Getenv("MYSQL_DATABASE") - cfg[comcfg.SQLiteFile] = os.Getenv("SQLITE_FILE") - cfg[comcfg.TokenServiceURL] = os.Getenv("TOKEN_SERVICE_URL") - tokenExpi, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION")) - if err != nil { - return err - } - cfg[comcfg.TokenExpiration] = tokenExpi - cfg[comcfg.RegistryURL] = os.Getenv("REGISTRY_URL") - //TODO remove - cfg[comcfg.JobLogDir] = os.Getenv("LOG_DIR") - //TODO remove - cfg[comcfg.UseCompressedJS] = os.Getenv("USE_COMPRESSED_JS") == "on" - cfgExpi, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) - if err != nil { - return err - } - cfg[comcfg.CfgExpiration] = cfgExpi - workers, err := strconv.Atoi(os.Getenv("MAX_JOB_WORKERS")) - if err != nil { - return err - } - cfg[comcfg.MaxJobWorkers] = workers - - return nil -} - -func initFromEnv() (map[string]interface{}, error) { +//load the configurations from env +func loadFromEnv() (map[string]interface{}, error) { cfg := map[string]interface{}{} - if err := readFromEnv(cfg); err != nil { - return nil, err - } + for k, v := range envs { + if str, ok := v.(string); ok { + cfg[k] = os.Getenv(str) + continue + } - cfg[comcfg.AUTHMode] = os.Getenv("AUTH_MODE") - cfg[comcfg.SelfRegistration] = os.Getenv("SELF_REGISTRATION") == "on" - cfg[comcfg.LDAPURL] = os.Getenv("LDAP_URL") - cfg[comcfg.LDAPSearchDN] = os.Getenv("LDAP_SEARCH_DN") - cfg[comcfg.LDAPSearchPwd] = os.Getenv("LDAP_SEARCH_PWD") - cfg[comcfg.LDAPBaseDN] = os.Getenv("LDAP_BASE_DN") - cfg[comcfg.LDAPFilter] = os.Getenv("LDAP_FILTER") - cfg[comcfg.LDAPUID] = os.Getenv("LDAP_UID") - scope, err := strconv.Atoi(os.Getenv("LDAP_SCOPE")) - if err != nil { - return nil, err + if parser, ok := v.(*parser); ok { + i, err := parser.parse(os.Getenv(parser.env)) + if err != nil { + return nil, err + } + cfg[k] = i + continue + } + + return nil, fmt.Errorf("%v is not string or parse type", v) } - cfg[comcfg.LDAPScope] = scope - timeout, err := strconv.Atoi(os.Getenv("LDAP_TIMEOUT")) - if err != nil { - return nil, err - } - cfg[comcfg.LDAPTimeout] = timeout - cfg[comcfg.EmailHost] = os.Getenv("EMAIL_HOST") - port, err := strconv.Atoi(os.Getenv("EMAIL_PORT")) - if err != nil { - return nil, err - } - cfg[comcfg.EmailPort] = port - cfg[comcfg.EmailUsername] = os.Getenv("EMAIL_USR") - cfg[comcfg.EmailPassword] = os.Getenv("EMAIL_PWD") - cfg[comcfg.EmailSSL] = os.Getenv("EMAIL_SSL") == "true" - cfg[comcfg.EmailFrom] = os.Getenv("EMAIL_FROM") - cfg[comcfg.EmailIdentity] = os.Getenv("EMAIL_IDENTITY") - cfg[comcfg.VerifyRemoteCert] = os.Getenv("VERIFY_REMOTE_CERT") == "on" - cfg[comcfg.ProjectCreationRestriction] = os.Getenv("PROJECT_CREATION_RESTRICTION") - cfg[comcfg.AdminInitialPassword] = os.Getenv("HARBOR_ADMIN_PASSWORD") return cfg, nil } diff --git a/src/ui/api/config.go b/src/ui/api/config.go index 05c4c5c44..ab833c961 100644 --- a/src/ui/api/config.go +++ b/src/ui/api/config.go @@ -19,51 +19,78 @@ import ( "fmt" "net/http" "strconv" - //"strings" "github.com/vmware/harbor/src/common/api" comcfg "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/dao" - //"github.com/vmware/harbor/src/common/models" - //"github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/config" ) -// keys of attrs which user can modify -var validKeys = []string{ - comcfg.AUTHMode, - comcfg.EmailFrom, - comcfg.EmailHost, - comcfg.EmailIdentity, - comcfg.EmailPassword, - comcfg.EmailPort, - comcfg.EmailSSL, - comcfg.EmailUsername, - comcfg.LDAPBaseDN, - comcfg.LDAPFilter, - comcfg.LDAPScope, - comcfg.LDAPSearchDN, - comcfg.LDAPSearchPwd, - comcfg.LDAPTimeout, - comcfg.LDAPUID, - comcfg.LDAPURL, - comcfg.ProjectCreationRestriction, - comcfg.SelfRegistration, - comcfg.VerifyRemoteCert, -} +var ( + // valid keys of configurations which user can modify + validKeys = []string{ + comcfg.ExtEndpoint, + comcfg.AUTHMode, + comcfg.DatabaseType, + comcfg.MySQLHost, + comcfg.MySQLPort, + comcfg.MySQLUsername, + comcfg.MySQLPassword, + comcfg.MySQLDatabase, + comcfg.SQLiteFile, + comcfg.SelfRegistration, + comcfg.LDAPURL, + comcfg.LDAPSearchDN, + comcfg.LDAPSearchPwd, + comcfg.LDAPBaseDN, + comcfg.LDAPUID, + comcfg.LDAPFilter, + comcfg.LDAPScope, + comcfg.LDAPTimeout, + comcfg.TokenServiceURL, + comcfg.RegistryURL, + comcfg.EmailHost, + comcfg.EmailPort, + comcfg.EmailUsername, + comcfg.EmailPassword, + comcfg.EmailFrom, + comcfg.EmailSSL, + comcfg.EmailIdentity, + comcfg.ProjectCreationRestriction, + comcfg.VerifyRemoteCert, + comcfg.MaxJobWorkers, + comcfg.TokenExpiration, + comcfg.CfgExpiration, + comcfg.JobLogDir, + comcfg.UseCompressedJS, + comcfg.AdminInitialPassword, + } -var numKeys = []string{ - comcfg.EmailPort, - comcfg.LDAPScope, - comcfg.LDAPTimeout, -} + numKeys = []string{ + comcfg.EmailPort, + comcfg.LDAPScope, + comcfg.LDAPTimeout, + comcfg.MySQLPort, + comcfg.MaxJobWorkers, + comcfg.TokenExpiration, + comcfg.CfgExpiration, + } -var boolKeys = []string{ - comcfg.EmailSSL, - comcfg.SelfRegistration, - comcfg.VerifyRemoteCert, -} + boolKeys = []string{ + comcfg.EmailSSL, + comcfg.SelfRegistration, + comcfg.VerifyRemoteCert, + comcfg.UseCompressedJS, + } + + passwordKeys = []string{ + comcfg.AdminInitialPassword, + comcfg.EmailPassword, + comcfg.LDAPSearchPwd, + comcfg.MySQLPassword, + } +) // ConfigAPI ... type ConfigAPI struct { @@ -234,26 +261,34 @@ func validateCfg(c map[string]string) (bool, error) { comcfg.LDAPScopeOnelevel, comcfg.LDAPScopeSubtree) } - if timeout, ok := c[comcfg.LDAPTimeout]; ok { - if t, err := strconv.Atoi(timeout); err != nil || t < 0 { - return isSysErr, fmt.Errorf("invalid %s", comcfg.LDAPTimeout) + + for _, k := range boolKeys { + v, ok := c[k] + if !ok { + continue + } + + if v != "0" && v != "1" { + return isSysErr, fmt.Errorf("%s should be %s or %s", + k, "0", "1") } } - if self, ok := c[comcfg.SelfRegistration]; ok && - self != "0" && self != "1" { - return isSysErr, fmt.Errorf("%s should be %s or %s", - comcfg.SelfRegistration, "0", "1") - } - - if port, ok := c[comcfg.EmailPort]; ok { - if p, err := strconv.Atoi(port); err != nil || p < 0 || p > 65535 { - return isSysErr, fmt.Errorf("invalid %s", comcfg.EmailPort) + for _, k := range numKeys { + v, ok := c[k] + if !ok { + continue } - } - if ssl, ok := c[comcfg.EmailSSL]; ok && ssl != "0" && ssl != "1" { - return isSysErr, fmt.Errorf("%s should be %s or %s", comcfg.EmailSSL, "0", "1") + n, err := strconv.Atoi(v) + if err != nil || n < 0 { + return isSysErr, fmt.Errorf("invalid %s: %s", k, v) + } + + if (k == comcfg.EmailPort || + k == comcfg.MySQLPort) && n > 65535 { + return isSysErr, fmt.Errorf("invalid %s: %s", k, v) + } } if crt, ok := c[comcfg.ProjectCreationRestriction]; ok && @@ -265,29 +300,13 @@ func validateCfg(c map[string]string) (bool, error) { comcfg.ProCrtRestrEveryone) } - if verify, ok := c[comcfg.VerifyRemoteCert]; ok && verify != "0" && verify != "1" { - return isSysErr, fmt.Errorf("invalid %s, should be %s or %s", - comcfg.VerifyRemoteCert, "0", "1") - } - return isSysErr, nil } -//encode passwords and convert map[string]string to map[string]interface{} +//convert map[string]string to map[string]interface{} func convertForPut(m map[string]string) (map[string]interface{}, error) { cfg := map[string]interface{}{} - /* - pwdKeys := []string{config.LDAP_SEARCH_PWD, config.EMAIL_PWD} - for _, pwdKey := range pwdKeys { - if pwd, ok := c[pwdKey]; ok && len(pwd) != 0 { - c[pwdKey], err = utils.ReversibleEncrypt(pwd, ui_cfg.SecretKey()) - if err != nil { - return nil, err - } - } - } - */ for k, v := range m { cfg[k] = v } @@ -319,14 +338,9 @@ func convertForPut(m map[string]string) (map[string]interface{}, error) { func convertForGet(cfg map[string]interface{}) (map[string]*value, error) { result := map[string]*value{} - dels := []string{ - comcfg.AdminInitialPassword, - comcfg.EmailPassword, - comcfg.LDAPSearchPwd, - comcfg.MySQLPassword} - for _, del := range dels { - if _, ok := cfg[del]; ok { - delete(cfg, del) + for _, k := range passwordKeys { + if _, ok := cfg[k]; ok { + delete(cfg, k) } } From a1858098c5224127de933caa347eb9fa8877925d Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 23 Feb 2017 18:02:19 +0800 Subject: [PATCH 21/22] using different secret to mark himself when communicates with other components --- make/common/templates/adminserver/env | 2 +- make/common/templates/jobservice/env | 1 + make/common/templates/ui/env | 1 + make/prepare | 9 ++++++--- src/adminserver/api/cfg.go | 6 ++++-- src/common/models/replication_job.go | 2 +- src/jobservice/api/replication.go | 2 +- src/jobservice/config/config.go | 12 +++++++++--- src/jobservice/job/statemachine.go | 2 +- src/ui/api/repository.go | 2 +- src/ui/config/config.go | 10 ++++++++-- src/ui/service/token/token.go | 7 ++++--- src/ui/service/utils/utils.go | 8 +++----- 13 files changed, 41 insertions(+), 23 deletions(-) diff --git a/make/common/templates/adminserver/env b/make/common/templates/adminserver/env index ff0153215..d537720f8 100644 --- a/make/common/templates/adminserver/env +++ b/make/common/templates/adminserver/env @@ -31,7 +31,7 @@ VERIFY_REMOTE_CERT=$verify_remote_cert MAX_JOB_WORKERS=$max_job_workers LOG_DIR=/var/log/jobs UI_SECRET=$ui_secret -SECRET_KEY=$secret_key +JOBSERVICE_SECRET=$jobservice_secret TOKEN_EXPIRATION=$token_expiration CFG_EXPIRATION=5 USE_COMPRESSED_JS=$use_compressed_js diff --git a/make/common/templates/jobservice/env b/make/common/templates/jobservice/env index 06c8b0f22..c5e37fc0f 100644 --- a/make/common/templates/jobservice/env +++ b/make/common/templates/jobservice/env @@ -1,4 +1,5 @@ LOG_LEVEL=debug CONFIG_PATH=/etc/jobservice/app.conf UI_SECRET=$ui_secret +JOBSERVICE_SECRET=$jobservice_secret GODEBUG=netdns=cgo diff --git a/make/common/templates/ui/env b/make/common/templates/ui/env index fc0d133ab..1f457e257 100644 --- a/make/common/templates/ui/env +++ b/make/common/templates/ui/env @@ -1,4 +1,5 @@ LOG_LEVEL=debug CONFIG_PATH=/etc/ui/app.conf UI_SECRET=$ui_secret +JOBSERVICE_SECRET=$jobservice_secret GODEBUG=netdns=cgo diff --git a/make/prepare b/make/prepare index 28d46a376..7fb47671e 100755 --- a/make/prepare +++ b/make/prepare @@ -125,6 +125,7 @@ secret_key = get_secret_key(secretkey_path) ######## ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16)) +jobservice_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16)) adminserver_config_dir = os.path.join(config_dir,"adminserver") if not os.path.exists(adminserver_config_dir): @@ -219,14 +220,15 @@ render(os.path.join(templates_dir, "adminserver", "env"), verify_remote_cert=verify_remote_cert, max_job_workers=max_job_workers, ui_secret=ui_secret, - secret_key=secret_key, + jobservice_secret=jobservice_secret, token_expiration=token_expiration, use_compressed_js=use_compressed_js ) render(os.path.join(templates_dir, "ui", "env"), ui_conf_env, - ui_secret=ui_secret) + ui_secret=ui_secret, + jobservice_secret=jobservice_secret,) render(os.path.join(templates_dir, "registry", "config.yml"), @@ -239,7 +241,8 @@ render(os.path.join(templates_dir, "db", "env"), render(os.path.join(templates_dir, "jobservice", "env"), job_conf_env, - ui_secret=ui_secret) + ui_secret=ui_secret, + jobservice_secret=jobservice_secret) print("Generated configuration file: %s" % jobservice_conf) shutil.copyfile(os.path.join(templates_dir, "jobservice", "app.conf"), jobservice_conf) diff --git a/src/adminserver/api/cfg.go b/src/adminserver/api/cfg.go index 6a2198c6e..23469eef8 100644 --- a/src/adminserver/api/cfg.go +++ b/src/adminserver/api/cfg.go @@ -26,7 +26,8 @@ import ( ) func isAuthenticated(r *http.Request) (bool, error) { - secret := os.Getenv("UI_SECRET") + uiSecret := os.Getenv("UI_SECRET") + jobserviceSecret := os.Getenv("JOBSERVICE_SECRET") c, err := r.Cookie("secret") if err != nil { if err == http.ErrNoCookie { @@ -34,7 +35,8 @@ func isAuthenticated(r *http.Request) (bool, error) { } return false, err } - return c != nil && c.Value == secret, nil + return c != nil && (c.Value == uiSecret || + c.Value == jobserviceSecret), nil } // ListCfgs lists configurations diff --git a/src/common/models/replication_job.go b/src/common/models/replication_job.go index 2908e4569..ae47371fd 100644 --- a/src/common/models/replication_job.go +++ b/src/common/models/replication_job.go @@ -44,7 +44,7 @@ const ( //RepOpDelete represents the operation of a job to remove repository from a remote registry/harbor instance. RepOpDelete string = "delete" //UISecretCookie is the cookie name to contain the UI secret - UISecretCookie string = "uisecret" + UISecretCookie string = "secret" ) // RepPolicy is the model for a replication policy, which associate to a project and a target (destination) diff --git a/src/jobservice/api/replication.go b/src/jobservice/api/replication.go index 25aef81e4..bfe7dd5c5 100644 --- a/src/jobservice/api/replication.go +++ b/src/jobservice/api/replication.go @@ -194,7 +194,7 @@ func getRepoList(projectID int64) ([]string, error) { return repositories, err } - req.AddCookie(&http.Cookie{Name: models.UISecretCookie, Value: config.UISecret()}) + req.AddCookie(&http.Cookie{Name: models.UISecretCookie, Value: config.JobserviceSecret()}) resp, err := client.Do(req) if err != nil { diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index 0a13ab5d4..c58f20b78 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -41,7 +41,7 @@ func Init() error { if len(adminServerURL) == 0 { adminServerURL = "http://adminserver" } - mg = comcfg.NewManager(adminServerURL, UISecret(), true) + mg = comcfg.NewManager(adminServerURL, JobserviceSecret(), true) if err := mg.Init(); err != nil { return err @@ -132,12 +132,18 @@ func SecretKey() (string, error) { return keyProvider.Get(nil) } -// UISecret returns a secret used for communication of UI, JobService -// and Adminserver +// UISecret returns a secret to mark UI when communicate with other +// component func UISecret() string { return os.Getenv("UI_SECRET") } +// JobserviceSecret returns a secret to mark Jobservice when communicate with +// other component +func JobserviceSecret() string { + return os.Getenv("JOBSERVICE_SECRET") +} + // ExtEndpoint ... func ExtEndpoint() (string, error) { cfg, err := mg.Get() diff --git a/src/jobservice/job/statemachine.go b/src/jobservice/job/statemachine.go index b6c9cd47a..db0d79a79 100644 --- a/src/jobservice/job/statemachine.go +++ b/src/jobservice/job/statemachine.go @@ -285,7 +285,7 @@ func addTestTransition(sm *SM) error { } func addImgTransferTransition(sm *SM) { - base := replication.InitBaseHandler(sm.Parms.Repository, sm.Parms.LocalRegURL, config.UISecret(), + base := replication.InitBaseHandler(sm.Parms.Repository, sm.Parms.LocalRegURL, config.JobserviceSecret(), sm.Parms.TargetURL, sm.Parms.TargetUsername, sm.Parms.TargetPassword, sm.Parms.Insecure, sm.Parms.Tags, sm.Logger) diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index 012ced2d3..94aa302ee 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -66,7 +66,7 @@ func (ra *RepositoryAPI) Get() { if project.Public == 0 { var userID int - if svc_utils.VerifySecret(ra.Ctx.Request) { + if svc_utils.VerifySecret(ra.Ctx.Request, config.JobserviceSecret()) { userID = 1 } else { userID = ra.ValidateUser() diff --git a/src/ui/config/config.go b/src/ui/config/config.go index fb100fe7f..d6d34b906 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -242,8 +242,14 @@ func Database() (*models.Database, error) { return database, nil } -// UISecret returns a secret used for communication of UI, JobService -// and Adminserver +// UISecret returns a secret to mark UI when communicate with +// other component func UISecret() string { return os.Getenv("UI_SECRET") } + +// JobserviceSecret returns a secret to mark Jobservice when communicate with +// other component +func JobserviceSecret() string { + return os.Getenv("JOBSERVICE_SECRET") +} diff --git a/src/ui/service/token/token.go b/src/ui/service/token/token.go index 2d408cbc5..b144e322a 100644 --- a/src/ui/service/token/token.go +++ b/src/ui/service/token/token.go @@ -19,10 +19,11 @@ import ( "net/http" "time" - "github.com/vmware/harbor/src/ui/auth" "github.com/vmware/harbor/src/common/models" - svc_utils "github.com/vmware/harbor/src/ui/service/utils" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/auth" + "github.com/vmware/harbor/src/ui/config" + svc_utils "github.com/vmware/harbor/src/ui/service/utils" "github.com/astaxie/beego" "github.com/docker/distribution/registry/auth/token" @@ -45,7 +46,7 @@ func (h *Handler) Get() { access := GetResourceActions(scopes) log.Infof("request url: %v", request.URL.String()) - if svc_utils.VerifySecret(request) { + if svc_utils.VerifySecret(request, config.JobserviceSecret()) { log.Debugf("Will grant all access as this request is from job service with legal secret.") username = "job-service-user" } else { diff --git a/src/ui/service/utils/utils.go b/src/ui/service/utils/utils.go index 007b9b720..44674d127 100644 --- a/src/ui/service/utils/utils.go +++ b/src/ui/service/utils/utils.go @@ -20,15 +20,13 @@ import ( "net/http" "github.com/vmware/harbor/src/common/utils/log" - "github.com/vmware/harbor/src/ui/config" ) // VerifySecret verifies the UI_SECRET cookie in a http request. -func VerifySecret(r *http.Request) bool { - secret := config.UISecret() - c, err := r.Cookie("uisecret") +func VerifySecret(r *http.Request, expectedSecret string) bool { + c, err := r.Cookie("secret") if err != nil { log.Warningf("Failed to get secret cookie, error: %v", err) } - return c != nil && c.Value == secret + return c != nil && c.Value == expectedSecret } From 9f3f48be596294842e25e7cb7b5c8a8354232de7 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 24 Feb 2017 14:14:36 +0800 Subject: [PATCH 22/22] add harbor network to adminserver --- make/docker-compose.tpl | 2 ++ src/ui/service/token/authutils.go | 12 +----------- src/ui/service/token/token_test.go | 8 +++++++- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/make/docker-compose.tpl b/make/docker-compose.tpl index 812164785..389ba2133 100644 --- a/make/docker-compose.tpl +++ b/make/docker-compose.tpl @@ -56,6 +56,8 @@ services: volumes: - /data/config/:/etc/adminserver/ - /data/secretkey:/etc/adminserver/key + networks: + - harbor depends_on: - log logging: diff --git a/src/ui/service/token/authutils.go b/src/ui/service/token/authutils.go index 740c9b3e6..8f577dfa3 100644 --- a/src/ui/service/token/authutils.go +++ b/src/ui/service/token/authutils.go @@ -33,23 +33,13 @@ import ( ) const ( - issuer = "registry-token-issuer" - defaultTokenExpiration = 30 + issuer = "registry-token-issuer" ) -var expiration int //minutes var privateKey string func init() { - var err error - expiration, err = config.TokenExpiration() - if err != nil { - log.Errorf("failed to get token expiration: %v, will use the default value %d", - err, defaultTokenExpiration) - expiration = defaultTokenExpiration - } privateKey = "/etc/ui/private_key.pem" - log.Infof("token expiration: %d minutes", expiration) } // GetResourceActions ... diff --git a/src/ui/service/token/token_test.go b/src/ui/service/token/token_test.go index 5cd8517db..499fb56f2 100644 --- a/src/ui/service/token/token_test.go +++ b/src/ui/service/token/token_test.go @@ -14,9 +14,16 @@ import ( "path" "runtime" "testing" + + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/config" ) func TestMain(m *testing.M) { + if err := config.Init(); err != nil { + log.Fatalf("failed to initialize configurations: %v", err) + } + result := m.Run() if result != 0 { os.Exit(result) @@ -73,7 +80,6 @@ func TestMakeToken(t *testing.T) { pk, crt := getKeyAndCertPath() //overwrite the config values for testing. privateKey = pk - expiration = 10 ra := []*token.ResourceActions{&token.ResourceActions{ Type: "repository", Name: "10.117.4.142/notary-test/hello-world-2",