diff --git a/make/common/templates/adminserver/env b/make/common/templates/adminserver/env index cc1040fb0..d1228bebb 100644 --- a/make/common/templates/adminserver/env +++ b/make/common/templates/adminserver/env @@ -1,6 +1,6 @@ PORT=8080 LOG_LEVEL=debug -EXT_ENDPOINT=$ui_url +EXT_ENDPOINT=$public_url AUTH_MODE=$auth_mode SELF_REGISTRATION=$self_registration LDAP_URL=$ldap_url @@ -22,8 +22,8 @@ MYSQL_PORT=$db_port MYSQL_USR=$db_user MYSQL_PWD=$db_password MYSQL_DATABASE=registry -REGISTRY_URL=http://registry:5000 -TOKEN_SERVICE_URL=http://ui/service/token +REGISTRY_URL=$registry_url +TOKEN_SERVICE_URL=$token_service_url EMAIL_HOST=$email_host EMAIL_PORT=$email_port EMAIL_USR=$email_usr @@ -53,7 +53,9 @@ UAA_ENDPOINT=$uaa_endpoint UAA_CLIENTID=$uaa_clientid UAA_CLIENTSECRET=$uaa_clientsecret UAA_VERIFY_CERT=$uaa_verify_cert -UI_URL=http://ui:8080 -JOBSERVICE_URL=http://jobservice:8080 +UI_URL=$ui_url +JOBSERVICE_URL=$jobservice_url +CLAIR_URL=$clair_url +NOTARY_URL=$notary_url REGISTRY_STORAGE_PROVIDER_NAME=$storage_provider_name READ_ONLY=false diff --git a/make/common/templates/jobservice/env b/make/common/templates/jobservice/env index e5ccac91e..7684d5afd 100644 --- a/make/common/templates/jobservice/env +++ b/make/common/templates/jobservice/env @@ -2,5 +2,5 @@ LOG_LEVEL=debug CONFIG_PATH=/etc/jobservice/app.conf UI_SECRET=$ui_secret JOBSERVICE_SECRET=$jobservice_secret -ADMINSERVER_URL=http://adminserver:8080 +ADMINSERVER_URL=$adminserver_url GODEBUG=netdns=cgo diff --git a/make/common/templates/registry/config.yml b/make/common/templates/registry/config.yml index e30dfb47c..c39c42aa1 100644 --- a/make/common/templates/registry/config.yml +++ b/make/common/templates/registry/config.yml @@ -20,14 +20,14 @@ http: auth: token: issuer: harbor-token-issuer - realm: $ui_url/service/token + realm: $public_url/service/token rootcertbundle: /etc/registry/root.crt service: harbor-registry notifications: endpoints: - name: harbor disabled: false - url: http://ui:8080/service/notifications + url: $ui_url/service/notifications timeout: 3000ms threshold: 5 backoff: 1s diff --git a/make/common/templates/registry/config_ha.yml b/make/common/templates/registry/config_ha.yml index e4570db2c..46776c8ff 100644 --- a/make/common/templates/registry/config_ha.yml +++ b/make/common/templates/registry/config_ha.yml @@ -31,7 +31,7 @@ http: auth: token: issuer: harbor-token-issuer - realm: $ui_url/service/token + realm: $public_url/service/token rootcertbundle: /etc/registry/root.crt service: harbor-registry @@ -39,7 +39,7 @@ notifications: endpoints: - name: harbor disabled: false - url: http://ui:8080/service/notifications + url: $ui_url/service/notifications timeout: 3000ms threshold: 5 backoff: 1s diff --git a/make/common/templates/ui/env b/make/common/templates/ui/env index 3a7000135..f9129643f 100644 --- a/make/common/templates/ui/env +++ b/make/common/templates/ui/env @@ -3,6 +3,6 @@ CONFIG_PATH=/etc/ui/app.conf UI_SECRET=$ui_secret JOBSERVICE_SECRET=$jobservice_secret GODEBUG=netdns=cgo -ADMINSERVER_URL=http://adminserver:8080 +ADMINSERVER_URL=$adminserver_url UAA_CA_ROOT=/etc/ui/certificates/uaa_ca.pem _REDIS_URL=$redis_url diff --git a/make/harbor.cfg b/make/harbor.cfg index 738fccdca..a7c851b07 100644 --- a/make/harbor.cfg +++ b/make/harbor.cfg @@ -171,3 +171,4 @@ registry_storage_provider_name = filesystem #registry_storage_provider_config is a comma separated "key: value" pairs, e.g. "key1: value, key2: value2". #Refer to https://docs.docker.com/registry/configuration/#storage for all available configuration. registry_storage_provider_config = + diff --git a/make/prepare b/make/prepare index 73addcc16..20988764c 100755 --- a/make/prepare +++ b/make/prepare @@ -195,7 +195,7 @@ reload_config = rcp.get("configuration", "reload_config") if rcp.has_option( "configuration", "reload_config") else "false" hostname = rcp.get("configuration", "hostname") protocol = rcp.get("configuration", "ui_url_protocol") -ui_url = protocol + "://" + hostname +public_url = protocol + "://" + hostname email_identity = rcp.get("configuration", "email_identity") email_host = rcp.get("configuration", "email_server") email_port = rcp.get("configuration", "email_server_port") @@ -270,7 +270,6 @@ storage_provider_name = rcp.get("configuration", "registry_storage_provider_name storage_provider_config = rcp.get("configuration", "registry_storage_provider_config").strip() # yaml requires 1 or more spaces between the key and value storage_provider_config = storage_provider_config.replace(":", ": ", 1) - ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16)) jobservice_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16)) @@ -298,6 +297,14 @@ job_conf_env = os.path.join(config_dir, "jobservice", "env") nginx_conf = os.path.join(config_dir, "nginx", "nginx.conf") cert_dir = os.path.join(config_dir, "nginx", "cert") log_rotate_config = os.path.join(config_dir, "log", "logrotate.conf") +adminserver_url = "http://adminserver:8080" +registry_url = "http://registry:5000" +ui_url = "http://ui:8080" +token_service_url = "http://ui:8080/service/token" + +jobservice_url = "http://jobservice:8080" +clair_url = "http://clair:6060" +notary_url = "http://notary-server:4443" if protocol == "https": target_cert_path = os.path.join(cert_dir, os.path.basename(cert_path)) @@ -317,6 +324,7 @@ else: render(os.path.join(templates_dir, "adminserver", "env"), adminserver_conf_env, reload_config=reload_config, + public_url=public_url, ui_url=ui_url, auth_mode=auth_mode, self_registration=self_registration, @@ -363,14 +371,20 @@ render(os.path.join(templates_dir, "adminserver", "env"), uaa_clientid=uaa_clientid, uaa_clientsecret=uaa_clientsecret, uaa_verify_cert=uaa_verify_cert, - storage_provider_name=storage_provider_name + storage_provider_name=storage_provider_name, + registry_url=registry_url, + token_service_url=token_service_url, + jobservice_url=jobservice_url, + clair_url=clair_url, + notary_url=notary_url ) render(os.path.join(templates_dir, "ui", "env"), ui_conf_env, ui_secret=ui_secret, jobservice_secret=jobservice_secret, - redis_url = redis_url + redis_url = redis_url, + adminserver_url = adminserver_url ) registry_config_file = "config_ha.yml" if args.ha_mode else "config.yml" @@ -385,6 +399,7 @@ storage_provider_info = ('\n' + ' ' * 4).join( render(os.path.join(templates_dir, "registry", registry_config_file), registry_conf, storage_provider_info=storage_provider_info, + public_url=public_url, ui_url=ui_url, redis_url=redis_url) @@ -395,7 +410,8 @@ render(os.path.join(templates_dir, "db", "env"), render(os.path.join(templates_dir, "jobservice", "env"), job_conf_env, ui_secret=ui_secret, - jobservice_secret=jobservice_secret) + jobservice_secret=jobservice_secret, + adminserver_url=adminserver_url) render(os.path.join(templates_dir, "log", "logrotate.conf"), log_rotate_config, @@ -522,7 +538,7 @@ if args.notary_mode: shutil.copy2(os.path.join(notary_temp_dir, "signer-config.json"), notary_config_dir) render(os.path.join(notary_temp_dir, "server-config.json"), os.path.join(notary_config_dir, "server-config.json"), - token_endpoint=ui_url) + token_endpoint=public_url) print("Copying nginx configuration file for notary") shutil.copy2(os.path.join(templates_dir, "nginx", "notary.upstream.conf"), nginx_conf_d) diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index ed5bce093..c80b049ed 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -151,6 +151,9 @@ var ( }, common.UIURL: "UI_URL", common.JobServiceURL: "JOBSERVICE_URL", + common.TokenServiceURL: "TOKEN_SERVICE_URL", + common.ClairURL: "CLAIR_URL", + common.NotaryURL: "NOTARY_URL", common.RegistryStorageProviderName: "REGISTRY_STORAGE_PROVIDER_NAME", common.ReadOnly: &parser{ env: "READ_ONLY", @@ -202,6 +205,12 @@ var ( parse: parseStringToBool, }, common.RegistryStorageProviderName: "REGISTRY_STORAGE_PROVIDER_NAME", + common.UIURL: "UI_URL", + common.JobServiceURL: "JOBSERVICE_URL", + common.RegistryURL: "REGISTRY_URL", + common.TokenServiceURL: "TOKEN_SERVICE_URL", + common.ClairURL: "CLAIR_URL", + common.NotaryURL: "NOTARY_URL", } ) diff --git a/src/common/const.go b/src/common/const.go index 59806e554..1eba447db 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -100,4 +100,10 @@ const ( UserMember = "u" GroupMember = "g" ReadOnly = "read_only" + ClairURL = "clair_url" + NotaryURL = "notary_url" + DefaultAdminserverEndpoint = "http://adminserver:8080" + DefaultJobserviceEndpoint = "http://jobservice:8080" + DefaultUIEndpoint = "http://ui:8080" + DefaultNotaryEndpoint = "http://notary-server:4443" ) diff --git a/src/common/utils/clair/utils.go b/src/common/utils/clair/utils.go index 4e1690e2f..8a2dfdc31 100644 --- a/src/common/utils/clair/utils.go +++ b/src/common/utils/clair/utils.go @@ -15,11 +15,9 @@ package clair import ( - "github.com/vmware/harbor/src/common" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" - "fmt" "strings" ) @@ -44,7 +42,7 @@ func ParseClairSev(clairSev string) models.Severity { } // UpdateScanOverview qeuries the vulnerability based on the layerName and update the record in img_scan_overview table based on digest. -func UpdateScanOverview(digest, layerName string, l ...*log.Logger) error { +func UpdateScanOverview(digest, layerName string, clairEndpoint string, l ...*log.Logger) error { var logger *log.Logger if len(l) > 1 { return fmt.Errorf("More than one logger specified") @@ -53,7 +51,7 @@ func UpdateScanOverview(digest, layerName string, l ...*log.Logger) error { } else { logger = log.DefaultLogger() } - client := NewClient(common.DefaultClairEndpoint, logger) + client := NewClient(clairEndpoint, logger) res, err := client.GetResult(layerName) if err != nil { logger.Errorf("Failed to get result from Clair, error: %v", err) diff --git a/src/common/utils/test/adminserver.go b/src/common/utils/test/adminserver.go index 203ae05f2..c3b6ebe13 100644 --- a/src/common/utils/test/adminserver.go +++ b/src/common/utils/test/adminserver.go @@ -76,6 +76,7 @@ var adminServerDefaultConfig = map[string]interface{}{ common.UIURL: "http://myui:8888/", common.JobServiceURL: "http://myjob:8888/", common.ReadOnly: false, + common.NotaryURL: "http://notary-server:4443", } // NewAdminserver returns a mock admin server diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index 4992c2363..c0f615d30 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -46,7 +46,7 @@ func Init() error { adminServerURL := os.Getenv("ADMINSERVER_URL") if len(adminServerURL) == 0 { - adminServerURL = "http://adminserver" + adminServerURL = common.DefaultAdminserverEndpoint } log.Infof("initializing client for adminserver %s ...", adminServerURL) cfg := &client.Config{ @@ -112,7 +112,7 @@ func LocalUIURL() string { cfg, err := mg.Get() if err != nil { log.Warningf("Failed to Get job service UI URL from backend, error: %v, will return default value.") - return "http://ui" + return common.DefaultUIEndpoint } return strings.TrimSuffix(cfg[common.UIURL].(string), "/") @@ -169,5 +169,12 @@ func InternalTokenServiceEndpoint() string { // ClairEndpoint returns the end point of clair instance, by default it's the one deployed within Harbor. func ClairEndpoint() string { - return common.DefaultClairEndpoint -} + cfg, err :=mg.Get() + if err != nil { + return common.DefaultClairEndpoint + } + if cfg[common.ClairURL] == nil { + return common.DefaultClairEndpoint + } + return cfg[common.ClairURL].(string) +} \ No newline at end of file diff --git a/src/jobservice/scan/handlers.go b/src/jobservice/scan/handlers.go index 0850ce6c8..c1d9e3bc8 100644 --- a/src/jobservice/scan/handlers.go +++ b/src/jobservice/scan/handlers.go @@ -134,7 +134,8 @@ func (sh *SummarizeHandler) Enter() (string, error) { logger.Infof("Entered summarize handler") layerName := sh.Context.layers[len(sh.Context.layers)-1].Name logger.Infof("Top layer's name: %s, will use it to get the vulnerability result of image", layerName) - if err := clair.UpdateScanOverview(sh.Context.Digest, layerName); err != nil { + clairURL := config.ClairEndpoint() + if err := clair.UpdateScanOverview(sh.Context.Digest, layerName, clairURL); err != nil { return "", err } return models.JobFinished, nil diff --git a/src/ui/config/config.go b/src/ui/config/config.go index 8bdfcf402..7a0ceb625 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -66,7 +66,7 @@ func Init() error { initKeyProvider() adminServerURL := os.Getenv("ADMINSERVER_URL") if len(adminServerURL) == 0 { - adminServerURL = "http://adminserver" + adminServerURL = common.DefaultAdminserverEndpoint } return InitByURL(adminServerURL) @@ -295,19 +295,18 @@ func InternalJobServiceURL() string { cfg, err := mg.Get() if err != nil { log.Warningf("Failed to Get job service URL from backend, error: %v, will return default value.") - - return "http://jobservice" + return common.DefaultJobserviceEndpoint } if cfg[common.JobServiceURL] == nil { - return "http://jobservice" + return common.DefaultJobserviceEndpoint } return strings.TrimSuffix(cfg[common.JobServiceURL].(string), "/") } // InternalTokenServiceEndpoint returns token service endpoint for internal communication between Harbor containers func InternalTokenServiceEndpoint() string { - uiURL := "http://ui" + uiURL := common.DefaultUIEndpoint cfg, err := mg.Get() if err != nil { log.Warningf("Failed to Get job service UI URL from backend, error: %v, will use default value.") @@ -321,7 +320,15 @@ func InternalTokenServiceEndpoint() string { // InternalNotaryEndpoint returns notary server endpoint for internal communication between Harbor containers // This is currently a conventional value and can be unaccessible when Harbor is not deployed with Notary. func InternalNotaryEndpoint() string { - return "http://notary-server:4443" + cfg, err := mg.Get() + if err != nil { + log.Warningf("Failed to get Notary endpoint from backend, error: %v, will use default value.") + return common.DefaultNotaryEndpoint + } + if cfg[common.NotaryURL] == nil { + return common.DefaultNotaryEndpoint + } + return cfg[common.NotaryURL].(string) } // InitialAdminPassword returns the initial password for administrator @@ -401,7 +408,7 @@ func JobserviceSecret() string { func WithNotary() bool { cfg, err := mg.Get() if err != nil { - log.Errorf("Failed to get configuration, will return WithNotary == false") + log.Warningf("Failed to get configuration, will return WithNotary == false") return false } return cfg[common.WithNotary].(bool) @@ -419,7 +426,12 @@ func WithClair() bool { // ClairEndpoint returns the end point of clair instance, by default it's the one deployed within Harbor. func ClairEndpoint() string { - return common.DefaultClairEndpoint + cfg, err := mg.Get() + if err != nil { + log.Errorf("Failed to get configuration, use default clair endpoint") + return common.DefaultClairEndpoint + } + return cfg[common.ClairURL].(string) } // ClairDB return Clair db info diff --git a/src/ui/proxy/interceptors.go b/src/ui/proxy/interceptors.go index b18d1c155..5c9924d0a 100644 --- a/src/ui/proxy/interceptors.go +++ b/src/ui/proxy/interceptors.go @@ -35,7 +35,7 @@ const ( var rec *httptest.ResponseRecorder // NotaryEndpoint , exported for testing. -var NotaryEndpoint = config.InternalNotaryEndpoint() +var NotaryEndpoint ="" // MatchPullManifest checks if the request looks like a request to pull manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values func MatchPullManifest(req *http.Request) (bool, string, string) { @@ -294,6 +294,9 @@ func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) } func matchNotaryDigest(img imageInfo) (bool, error) { + if NotaryEndpoint == "" { + NotaryEndpoint = config.InternalNotaryEndpoint() + } targets, err := notary.GetInternalTargets(NotaryEndpoint, tokenUsername, img.repository) if err != nil { return false, err diff --git a/src/ui/service/notifications/clair/handler.go b/src/ui/service/notifications/clair/handler.go index 82e8867a2..0dcd852b0 100644 --- a/src/ui/service/notifications/clair/handler.go +++ b/src/ui/service/notifications/clair/handler.go @@ -32,7 +32,7 @@ const ( ) var ( - clairClient = clair.NewClient(config.ClairEndpoint(), nil) + clairClient *clair.Client ) // Handler handles reqeust on /service/notifications/clair/, which listens to clair's notifications. @@ -43,7 +43,10 @@ type Handler struct { // Handle ... func (h *Handler) Handle() { - var ne models.ClairNotificationEnvelope + if clairClient == nil { + clairClient = clair.NewClient(config.ClairEndpoint(), nil) + } + var ne models.ClairNotificationEnvelope if err := json.Unmarshal(h.Ctx.Input.CopyBody(1<<32), &ne); err != nil { log.Errorf("Failed to decode the request: %v", err) return @@ -85,7 +88,7 @@ func (h *Handler) Handle() { return } for _, e := range l { - if err := clair.UpdateScanOverview(e.Digest, e.DetailsKey); err != nil { + if err := clair.UpdateScanOverview(e.Digest, e.DetailsKey,config.ClairEndpoint()); err != nil { log.Errorf("Failed to refresh scan overview for image: %s", e.Digest) } else { log.Debugf("Refreshed scan overview for record with digest: %s", e.Digest) diff --git a/src/ui/ui_test.go b/src/ui/ui_test.go deleted file mode 100644 index 453775329..000000000 --- a/src/ui/ui_test.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2017 VMware, Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package main