Merge master into job_service and fix conflicts

src/common/utils/test/adminserver.go

@@ -61,7 +61,7 @@ var adminServerDefaultConfig = map[string]interface{}{
 	common.TokenExpiration:            30,
 	common.CfgExpiration:              5,
 	common.AdminInitialPassword:       "password",
-	common.AdmiralEndpoint:            "http://www.vmware.com",
+	common.AdmiralEndpoint:            "",
 	common.WithNotary:                 false,
 	common.WithClair:                  false,
 	common.ClairDBUsername:            "postgres",
@@ -84,8 +84,13 @@ func NewAdminserver(config map[string]interface{}) (*httptest.Server, error) {
 	m := []*RequestHandlerMapping{}
 	if config == nil {
 		config = adminServerDefaultConfig
+	} else {
+		for k, v := range adminServerDefaultConfig {
+			if _, ok := config[k]; !ok {
+				config[k] = v
+			}
+		}
 	}
-
 	b, err := json.Marshal(config)
 	if err != nil {
 		return nil, err

src/ui/config/config.go

@@ -16,8 +16,10 @@ package config
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"strconv"
@@ -29,7 +31,6 @@ import (
 	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/secret"
 	"github.com/vmware/harbor/src/common/utils/log"
-	jobservice_client "github.com/vmware/harbor/src/jobservice/client"
 	"github.com/vmware/harbor/src/ui/promgr"
 	"github.com/vmware/harbor/src/ui/promgr/pmsdriver"
 	"github.com/vmware/harbor/src/ui/promgr/pmsdriver/admiral"
@@ -56,8 +57,8 @@ var (
 	AdmiralClient *http.Client
 	// TokenReader is used in integration mode to read token
 	TokenReader admiral.TokenReader
-	// GlobalJobserviceClient is a global client for jobservice
+
-	GlobalJobserviceClient jobservice_client.Client
+	defaultCACertPath = "/etc/ui/ca/ca.crt"
 )
 
 // Init configurations
@@ -94,7 +95,10 @@ func InitByURL(adminServerURL string) error {
 	initSecretStore()
 
 	// init project manager based on deploy mode
-	initProjectManager()
+	if err := initProjectManager(); err != nil {
+		log.Errorf("Failed to initialise project manager, error: %v", err)
+		return err
+	}
 
 	return nil
 }
@@ -115,20 +119,28 @@ func initSecretStore() {
 	SecretStore = secret.NewStore(m)
 }
 
-func initProjectManager() {
+func initProjectManager() error {
 	var driver pmsdriver.PMSDriver
 	if WithAdmiral() {
-		// integration with admiral
+		log.Debugf("Initialising Admiral client with certificate: %s", defaultCACertPath)
-		log.Info("initializing the project manager based on PMS...")
+		content, err := ioutil.ReadFile(defaultCACertPath)
-		// TODO read ca/cert file and pass it to the TLS config
+		if err != nil {
+			return err
+		}
+		pool := x509.NewCertPool()
+		if ok := pool.AppendCertsFromPEM(content); !ok {
+			return fmt.Errorf("failed to append cert content into cert pool")
+		}
 		AdmiralClient = &http.Client{
 			Transport: &http.Transport{
 				TLSClientConfig: &tls.Config{
-					InsecureSkipVerify: true,
+					RootCAs: pool,
 				},
 			},
 		}
 
+		// integration with admiral
+		log.Info("initializing the project manager based on PMS...")
 		path := os.Getenv("SERVICE_TOKEN_FILE_PATH")
 		if len(path) == 0 {
 			path = defaultTokenFilePath
@@ -144,6 +156,7 @@ func initProjectManager() {
 		driver = local.NewDriver()
 	}
 	GlobalProjectMgr = promgr.NewDefaultProjectManager(driver, true)
+	return nil
 
 }
 

src/ui/config/config_test.go

@@ -15,6 +15,8 @@ package config
 
 import (
 	"os"
+	"path"
+	"runtime"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -24,7 +26,12 @@ import (
 
 // test functions under package ui/config
 func TestConfig(t *testing.T) {
-	server, err := test.NewAdminserver(nil)
+
+	defaultCACertPath = path.Join(currPath(), "test", "ca.crt")
+	c := map[string]interface{}{
+		common.AdmiralEndpoint: "http://www.vmware.com",
+	}
+	server, err := test.NewAdminserver(c)
 	if err != nil {
 		t.Fatalf("failed to create a mock admin server: %v", err)
 	}
@@ -190,3 +197,11 @@ func TestConfig(t *testing.T) {
 	assert.Equal("http://myui:8888/service/token", InternalTokenServiceEndpoint())
 
 }
+
+func currPath() string {
+	_, f, _, ok := runtime.Caller(0)
+	if !ok {
+		panic("Failed to get current directory")
+	}
+	return path.Dir(f)
+}

src/ui/config/test/ca.crt

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIIC7TCCAdWgAwIBAgIJAKmFRnILlp3XMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV
+BAMMAmNhMB4XDTE3MDkyNDA3MDA1M1oXDTI3MDkyMjA3MDA1M1owDTELMAkGA1UE
+AwwCY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCr4+HxXkY81j1p
+5OD3htFkbJI+XulBgc7ja5YorU323VB7JfNBnau3rDZS8NdyvkLLEQT4rKw5Dd4p
+phlmdKsmIq9ej1OlDjWnCOGr+HG2jG5POgPYRCf5WgCGoQ4eUIA+IXcVroG8f1YM
+LDzZEBKlEP80W0zyh0ma/BYN8HG4Ica4q/iIjffJc7ob/tWFGt2HobI9wbTSyBgR
+s7JSs6MBIISXGAuOE3cs7vJNzKtWhQSBw4j8FFUZSYCyONFYfOg2OtZG6z1XhpTC
+rfVMm6cEsYla/mf9bJB2AqtRiUdUZwAOWQbalWPFKEO73Bj4/5sVNHKFCd/S6J1z
+LHaWM0W7AgMBAAGjUDBOMB0GA1UdDgQWBBR0jFgTuL9K2iWE0wzU7r4RZT0k+zAf
+BgNVHSMEGDAWgBR0jFgTuL9K2iWE0wzU7r4RZT0k+zAMBgNVHRMEBTADAQH/MA0G
+CSqGSIb3DQEBCwUAA4IBAQCemrfEKHPe5ahb2III89+iuIDmbPgVESXqnf88UUdS
+Iv+htE8hu9CkSemsErXcC0kUbPSM0vWN9IbHINq78cXucVyi+YTzaKJ8zsK01/zf
+x0xYeK5bffYTQzs+BopTCwVqd9zHSs9a2zPnsBVHXCn25j30anQgQH9ODsspXZ3i
+WUAkEOmZDnNuX7tGDesA+7h8BPcZ8zrz94kxsrdneMXuHdT1iHxS/hTxTEUUhOMF
+FntwT6zx3fGL4cNG06d+pdjjp+CuUR+8GRxeASbYBWhXeiY1ykipiptxkp1zhZ3x
+SNandCCdeMRntnNs/+xvRhsEGbhyrvzg2WFL2NrqiKtg
+-----END CERTIFICATE-----

src/ui/filter/readonly.go

@@ -42,6 +42,7 @@ func filter(req *http.Request, resp http.ResponseWriter) {
 	}
 	if matchRepoTagDelete(req) {
 		resp.WriteHeader(http.StatusServiceUnavailable)
+		resp.Write([]byte("The system is in read only mode. Any modification is not prohibited."))
 	}
 }
 

src/ui/proxy/interceptor_test.go

@@ -195,8 +195,10 @@ func TestCopyResp(t *testing.T) {
 
 func TestMarshalError(t *testing.T) {
 	assert := assert.New(t)
-	js := marshalError("Not Found")
+	js1 := marshalError("PROJECT_POLICY_VIOLATION", "Not Found")
-	assert.Equal("{\"errors\":[{\"code\":\"PROJECT_POLICY_VIOLATION\",\"message\":\"Not Found\",\"detail\":\"Not Found\"}]}", js)
+	assert.Equal("{\"errors\":[{\"code\":\"PROJECT_POLICY_VIOLATION\",\"message\":\"Not Found\",\"detail\":\"Not Found\"}]}", js1)
+	js2 := marshalError("DENIED", "The action is denied")
+	assert.Equal("{\"errors\":[{\"code\":\"DENIED\",\"message\":\"The action is denied\",\"detail\":\"The action is denied\"}]}", js2)
 }
 
 func TestIsDigest(t *testing.T) {

src/ui/proxy/interceptors.go

@@ -35,7 +35,7 @@ const (
 var rec *httptest.ResponseRecorder
 
 // NotaryEndpoint , exported for testing.
-var NotaryEndpoint =""
+var NotaryEndpoint = ""
 
 // MatchPullManifest checks if the request looks like a request to pull manifest.  If it is returns the image and tag/sha256 digest as 2nd and 3rd return values
 func MatchPullManifest(req *http.Request) (bool, string, string) {
@@ -123,20 +123,20 @@ func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	if flag {
 		components := strings.SplitN(repository, "/", 2)
 		if len(components) < 2 {
-			http.Error(rw, marshalError(fmt.Sprintf("Bad repository name: %s", repository)), http.StatusBadRequest)
+			http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Bad repository name: %s", repository)), http.StatusBadRequest)
 			return
 		}
 
 		client, err := uiutils.NewRepositoryClientForUI(tokenUsername, repository)
 		if err != nil {
 			log.Errorf("Error creating repository Client: %v", err)
-			http.Error(rw, marshalError(fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError)
+			http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError)
 			return
 		}
 		digest, _, err := client.ManifestExist(reference)
 		if err != nil {
 			log.Errorf("Failed to get digest for reference: %s, error: %v", reference, err)
-			http.Error(rw, marshalError(fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError)
+			http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError)
 			return
 		}
 
@@ -161,7 +161,7 @@ type readonlyHandler struct {
 func (rh readonlyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	if config.ReadOnly() {
 		if req.Method == http.MethodDelete || req.Method == http.MethodPost || req.Method == http.MethodPatch {
-			http.Error(rw, "Upload/Delete is prohibited in read only mode.", http.StatusServiceUnavailable)
+			http.Error(rw, marshalError("DENIED", "The system is in read only mode. Any modification is not prohibited."), http.StatusForbidden)
 			return
 		}
 	}
@@ -241,12 +241,12 @@ func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Reque
 	}
 	match, err := matchNotaryDigest(img)
 	if err != nil {
-		http.Error(rw, marshalError("Failed in communication with Notary please check the log"), http.StatusInternalServerError)
+		http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed in communication with Notary please check the log"), http.StatusInternalServerError)
 		return
 	}
 	if !match {
 		log.Debugf("digest mismatch, failing the response.")
-		http.Error(rw, marshalError("The image is not signed in Notary."), http.StatusPreconditionFailed)
+		http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "The image is not signed in Notary."), http.StatusPreconditionFailed)
 		return
 	}
 	cth.next.ServeHTTP(rw, req)
@@ -275,19 +275,19 @@ func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request)
 	overview, err := dao.GetImgScanOverview(img.digest)
 	if err != nil {
 		log.Errorf("failed to get ImgScanOverview with repo: %s, reference: %s, digest: %s. Error: %v", img.repository, img.reference, img.digest, err)
-		http.Error(rw, marshalError("Failed to get ImgScanOverview."), http.StatusPreconditionFailed)
+		http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get ImgScanOverview."), http.StatusPreconditionFailed)
 		return
 	}
 	// severity is 0 means that the image fails to scan or not scanned successfully.
 	if overview == nil || overview.Sev == 0 {
 		log.Debugf("cannot get the image scan overview info, failing the response.")
-		http.Error(rw, marshalError("Cannot get the image severity."), http.StatusPreconditionFailed)
+		http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Cannot get the image severity."), http.StatusPreconditionFailed)
 		return
 	}
 	imageSev := overview.Sev
 	if imageSev >= int(projectVulnerableSeverity) {
 		log.Debugf("the image severity: %q is higher then project setting: %q, failing the response.", models.Severity(imageSev), projectVulnerableSeverity)
-		http.Error(rw, marshalError(fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", models.Severity(imageSev), projectVulnerableSeverity)), http.StatusPreconditionFailed)
+		http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", models.Severity(imageSev), projectVulnerableSeverity)), http.StatusPreconditionFailed)
 		return
 	}
 	vh.next.ServeHTTP(rw, req)
@@ -340,12 +340,12 @@ func copyResp(rec *httptest.ResponseRecorder, rw http.ResponseWriter) {
 	rw.Write(rec.Body.Bytes())
 }
 
-func marshalError(msg string) string {
+func marshalError(code, msg string) string {
 	var tmpErrs struct {
 		Errors []JSONError `json:"errors,omitempty"`
 	}
 	tmpErrs.Errors = append(tmpErrs.Errors, JSONError{
-		Code:    "PROJECT_POLICY_VIOLATION",
+		Code:    code,
 		Message: msg,
 		Detail:  msg,
 	})

tools/migration/cfg/migrate.py

@@ -5,6 +5,7 @@
 from __future__ import print_function
 import argparse
 import os
+import sys
 import utils
 import importlib
 import glob
@@ -24,9 +25,12 @@ def main():
     input_version =  utils.get_conf_version(args.input_path)
     curr_dir = os.path.dirname(__file__)
     chain = []
+    if input_version == target_version:
+        print ("Version of input harbor.cfg is identical to target %s, no need to upgrade" % input_version)
+        sys.exit(0)
     if not search(curr_dir, input_version, target_version, chain):
         print ("No migrator for version: %s" % input_version)
-        os.exit(1)
+        sys.exit(1)
     else:
         print ("input version: %s, migrator chain: %s" % (input_version, chain))
     curr_input_path = args.input_path

tools/migration/cfg/migrator_1_5_0/__init__.py

@@ -26,12 +26,12 @@ default = {
 
 def migrate(input_cfg, output_cfg):
     d = utils.read_conf(input_cfg)
-    keys = default.keys()
+    keys = list(default.keys())
     keys.extend(['hostname', 'ui_url_protocol', 'max_job_workers', 'customize_crt', 
             'ssl_cert', 'ssl_cert_key', 'secretkey_path', 'admiral_url', 'db_password', 'clair_db_password'])
     val = {}
     for k in keys:
-        if d.has_key(k):
+        if k in d:
             val[k] = d[k]
         else:
             val[k] = default[k]

tools/migration/cfg/utils.py

@@ -29,13 +29,13 @@ def read_conf(path):
 def get_conf_version(path):
     d = read_conf(path)
 #    print json.dumps(d,indent=4)
-    if d.has_key("_version"): # >=1.5.0
+    if "_version" in d: # >=1.5.0
         return d["_version"]
-    if not d.has_key("clair_db_password"):
+    if not "clair_db_password" in d:
         return "unsupported"
-    if d.has_key("registry_storage_provider_name"):
+    if "registry_storage_provider_name" in d:
         return "1.4.0"
-    if d.has_key("uaa_endpoint"):
+    if "uaa_endpoint" in d:
         return "1.3.0"
     return "1.2.0"