Refactor chart API endpoints

add /chartrepo after /api
add /chartrepo before /index.yaml,/:repo/index.yaml and /:repo/charts/:filename
add debug messages to the potential debug points
add more supporting mime types "multipart/form-data", "application/octet-stream" to the API endpoints

update travis file to inject ENV for UT cases of chart repo

.travis.yml

@@ -28,6 +28,7 @@ env:
   SELF_REGISTRATION: on
   KEY_PATH: /data/secretkey
   REDIS_HOST: localhost
+  BACKEND_CHART_SERVER: http://localhost:9090
 
 before_install:
    - sudo ./tests/hostcfg.sh

src/chartserver/client.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"strings"
 	"time"
 )
@@ -42,12 +43,17 @@ func NewChartClient(credentail *Credential) *ChartClient { //Create http client
 }
 
 //GetContent get the bytes from the specified url
-func (cc *ChartClient) GetContent(url string) ([]byte, error) {
-	if len(strings.TrimSpace(url)) == 0 {
+func (cc *ChartClient) GetContent(addr string) ([]byte, error) {
+	if len(strings.TrimSpace(addr)) == 0 {
 		return nil, errors.New("empty url is not allowed")
 	}
 
-	request, err := http.NewRequest(http.MethodGet, url, nil)
+	fullURI, err := url.Parse(addr)
+	if err != nil {
+		return nil, fmt.Errorf("invalid url: %s", err.Error())
+	}
+
+	request, err := http.NewRequest(http.MethodGet, addr, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -69,7 +75,7 @@ func (cc *ChartClient) GetContent(url string) ([]byte, error) {
 	defer response.Body.Close()
 
 	if response.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("failed to retrieve content from url '%s' with error: %s", url, content)
+		return nil, fmt.Errorf("failed to retrieve content from '%s' with error: %s", fullURI.Path, content)
 	}
 
 	return content, nil

src/chartserver/manipulation_handler.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"path"
 	"strings"
 
 	"github.com/ghodss/yaml"
@@ -43,10 +44,10 @@ type ManipulationHandler struct {
 
 //ListCharts lists all the charts under the specified namespace
 func (mh *ManipulationHandler) ListCharts(w http.ResponseWriter, req *http.Request) {
-	rootURL := strings.TrimSuffix(mh.backendServerAddress.String(), "/")
-	fullURL := fmt.Sprintf("%s%s", rootURL, req.RequestURI)
+	url := strings.TrimPrefix(req.URL.String(), "/")
+	url = fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), url)
 
-	content, err := mh.apiClient.GetContent(fullURL)
+	content, err := mh.apiClient.GetContent(url)
 	if err != nil {
 		writeInternalError(w, err)
 		return
@@ -76,7 +77,7 @@ func (mh *ManipulationHandler) GetChart(w http.ResponseWriter, req *http.Request
 //This handler should return the details of the chart version,
 //maybe including metadata,dependencies and values etc.
 func (mh *ManipulationHandler) GetChartVersion(w http.ResponseWriter, req *http.Request) {
-	chartV, err := mh.getChartVersion(req.RequestURI)
+	chartV, err := mh.getChartVersion(req.URL.String())
 	if err != nil {
 		writeInternalError(w, err)
 		return
@@ -149,11 +150,10 @@ func (mh *ManipulationHandler) DeleteChartVersion(w http.ResponseWriter, req *ht
 }
 
 //Get the basic metadata of chart version
-func (mh *ManipulationHandler) getChartVersion(path string) (*helm_repo.ChartVersion, error) {
-	rootURL := strings.TrimSuffix(mh.backendServerAddress.String(), "/")
-	fullURL := fmt.Sprintf("%s%s", rootURL, path)
+func (mh *ManipulationHandler) getChartVersion(subPath string) (*helm_repo.ChartVersion, error) {
+	url := fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), strings.TrimPrefix(subPath, "/"))
 
-	content, err := mh.apiClient.GetContent(fullURL)
+	content, err := mh.apiClient.GetContent(url)
 	if err != nil {
 		return nil, err
 	}
@@ -167,9 +167,9 @@ func (mh *ManipulationHandler) getChartVersion(path string) (*helm_repo.ChartVer
 }
 
 //Get the content bytes of the chart version
-func (mh *ManipulationHandler) getChartVersionContent(namespace string, path string) ([]byte, error) {
-	rootURL := strings.TrimSuffix(mh.backendServerAddress.String(), "/")
-	fullPath := fmt.Sprintf("%s/%s/%s", rootURL, namespace, path)
+func (mh *ManipulationHandler) getChartVersionContent(namespace string, subPath string) ([]byte, error) {
+	url := path.Join(namespace, subPath)
+	url = fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), url)
 
-	return mh.apiClient.GetContent(fullPath)
+	return mh.apiClient.GetContent(url)
 }

src/chartserver/repo_handler.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
-	"strings"
+	"path"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -14,6 +14,8 @@ import (
 	"github.com/ghodss/yaml"
 	"github.com/vmware/harbor/src/ui/filter"
 	helm_repo "k8s.io/helm/pkg/repo"
+
+	hlog "github.com/vmware/harbor/src/common/utils/log"
 )
 
 const (
@@ -209,8 +211,9 @@ func (rh *RepositoryHandler) DownloadChartObject(w http.ResponseWriter, req *htt
 //Get the index yaml file under the specified namespace from the backend server
 func (rh *RepositoryHandler) getIndexYamlWithNS(namespace string) (*helm_repo.IndexFile, error) {
 	//Join url path
-	rootURL := strings.TrimSuffix(rh.backendServerAddress.String(), "/")
-	url := fmt.Sprintf("%s/%s/index.yaml", rootURL, namespace)
+	url := path.Join(namespace, "index.yaml")
+	url = fmt.Sprintf("%s/%s", rh.backendServerAddress.String(), url)
+	hlog.Debugf("Getting index.yaml from '%s'", url)
 
 	content, err := rh.apiClient.GetContent(url)
 	if err != nil {
@@ -238,7 +241,7 @@ func (rh *RepositoryHandler) mergeIndexFile(namespace string,
 			version.Name = nameWithNS
 			//Currently there is only one url
 			for index, url := range version.URLs {
-				version.URLs[index] = fmt.Sprintf("%s/%s", namespace, url)
+				version.URLs[index] = path.Join(namespace, url)
 			}
 		}
 

src/ui/api/chart_repository.go

@@ -13,8 +13,12 @@ import (
 )
 
 const (
-	backendChartServerAddr = "BACKEND_CHART_SERVER"
-	namespaceParam         = ":repo"
+	backendChartServerAddr  = "BACKEND_CHART_SERVER"
+	namespaceParam          = ":repo"
+	defaultRepo             = "library"
+	rootUploadingEndpoint   = "/api/chartrepo/charts"
+	rootIndexEndpoint       = "/chartrepo/index.yaml"
+	chartRepoHealthEndpoint = "/api/chartrepo/health"
 
 	accessLevelPublic = iota
 	accessLevelRead
@@ -49,11 +53,28 @@ func (cra *ChartRepositoryAPI) Prepare() {
 	// -/index.yaml
 	// -/api/chartserver/health
 	incomingURI := cra.Ctx.Request.RequestURI
-	if incomingURI != "/index.yaml" && incomingURI != "/api/chartserver/health" {
+	if incomingURI == rootUploadingEndpoint {
+		//Forward to the default repository
+		cra.namespace = defaultRepo
+	}
+
+	if incomingURI != rootIndexEndpoint &&
+		incomingURI != chartRepoHealthEndpoint {
 		if !cra.requireNamespace(cra.namespace) {
 			return
 		}
 	}
+
+	//Rewrite URL path
+	cra.rewriteURLPath(cra.Ctx.Request)
+}
+
+//UploadChartVersionToDefaultNS ...
+func (cra *ChartRepositoryAPI) UploadChartVersionToDefaultNS() {
+	res := make(map[string]interface{})
+	res["result"] = "done"
+	cra.Data["json"] = res
+	cra.ServeJSON()
 }
 
 //GetHealthStatus handles GET /api/chartserver/health
@@ -63,11 +84,7 @@ func (cra *ChartRepositoryAPI) GetHealthStatus() {
 		return
 	}
 
-	//Override the request path to '/health'
-	req := cra.Ctx.Request
-	req.URL.Path = "/health"
-
-	chartController.GetBaseHandler().GetHealthStatus(cra.Ctx.ResponseWriter, req)
+	chartController.GetBaseHandler().GetHealthStatus(cra.Ctx.ResponseWriter, cra.Ctx.Request)
 }
 
 //GetIndexByRepo handles GET /:repo/index.yaml
@@ -146,6 +163,8 @@ func (cra *ChartRepositoryAPI) DeleteChartVersion() {
 
 //UploadChartVersion handles POST /api/:repo/charts
 func (cra *ChartRepositoryAPI) UploadChartVersion() {
+	hlog.Debugf("Header of request of uploading chart: %#v, content-len=%d", cra.Ctx.Request.Header, cra.Ctx.Request.ContentLength)
+
 	//Check access
 	if !cra.requireAccess(cra.namespace, accessLevelWrite) {
 		return
@@ -164,6 +183,41 @@ func (cra *ChartRepositoryAPI) UploadChartProvFile() {
 	chartController.GetManipulationHandler().UploadProvenanceFile(cra.Ctx.ResponseWriter, cra.Ctx.Request)
 }
 
+//Rewrite the incoming URL with the right backend URL pattern
+//Remove 'chartrepo' from the endpoints of manipulation API
+//Remove 'chartrepo' from the endpoints of repository services
+func (cra *ChartRepositoryAPI) rewriteURLPath(req *http.Request) {
+	incomingURLPath := req.RequestURI
+
+	defer func() {
+		hlog.Debugf("Incoming URL '%s' is rewritten to '%s'", incomingURLPath, req.URL.String())
+	}()
+
+	//Health check endpoint
+	if incomingURLPath == chartRepoHealthEndpoint {
+		req.URL.Path = "/health"
+		return
+	}
+
+	//Root uploading endpoint
+	if incomingURLPath == rootUploadingEndpoint {
+		req.URL.Path = strings.Replace(incomingURLPath, "chartrepo", defaultRepo, 1)
+		return
+	}
+
+	//Repository endpoints
+	if strings.HasPrefix(incomingURLPath, "/chartrepo") {
+		req.URL.Path = strings.TrimPrefix(incomingURLPath, "/chartrepo")
+		return
+	}
+
+	//API endpoints
+	if strings.HasPrefix(incomingURLPath, "/api/chartrepo") {
+		req.URL.Path = strings.Replace(incomingURLPath, "/chartrepo", "", 1)
+		return
+	}
+}
+
 //Check if there exists a valid namespace
 //Return true if it does
 //Return false if it does not
@@ -243,17 +297,23 @@ func (cra *ChartRepositoryAPI) requireAccess(namespace string, accessLevel uint)
 
 //Initialize the chart service controller
 func initializeChartController() *chartserver.Controller {
-	addr := os.Getenv(backendChartServerAddr)
+	addr := strings.TrimSpace(os.Getenv(backendChartServerAddr))
+	if len(addr) == 0 {
+		hlog.Fatal("The address of chart storage server is not set")
+	}
+
+	addr = strings.TrimSuffix(addr, "/")
 	url, err := url.Parse(addr)
 	if err != nil {
-		hlog.Fatal("chart storage server is not correctly configured")
+		hlog.Fatal("Chart storage server is not correctly configured")
 	}
 
 	controller, err := chartserver.NewController(url)
 	if err != nil {
-		hlog.Fatal("failed to initialize chart API controller")
+		hlog.Fatal("Failed to initialize chart API controller")
 	}
 
+	hlog.Debugf("Chart storage server is set to %s", url.String())
 	hlog.Info("API controller for chart repository server is successfully initialized")
 
 	return controller

src/ui/filter/mediatype.go

@@ -15,9 +15,12 @@
 package filter
 
 import (
-	beegoctx "github.com/astaxie/beego/context"
 	"net/http"
 	"strings"
+
+	beegoctx "github.com/astaxie/beego/context"
+
+	hlog "github.com/vmware/harbor/src/common/utils/log"
 )
 
 //MediaTypeFilter filters the POST request, it returns 415 if the content type of the request
@@ -34,6 +37,8 @@ func filterContentType(req *http.Request, resp http.ResponseWriter, mediaType ..
 	}
 	v := req.Header.Get("Content-Type")
 	mimeType := strings.Split(v, ";")[0]
+	hlog.Debugf("Mimetype of incoming request %s: %s", req.RequestURI, mimeType)
+
 	for _, t := range mediaType {
 		if t == mimeType {
 			return

src/ui/main.go

@@ -144,7 +144,7 @@ func main() {
 	filter.Init()
 	beego.InsertFilter("/*", beego.BeforeRouter, filter.SecurityFilter)
 	beego.InsertFilter("/*", beego.BeforeRouter, filter.ReadonlyFilter)
-	beego.InsertFilter("/api/*", beego.BeforeRouter, filter.MediaTypeFilter("application/json"))
+	beego.InsertFilter("/api/*", beego.BeforeRouter, filter.MediaTypeFilter("application/json", "multipart/form-data", "application/octet-stream"))
 
 	initRouters()
 	if err := api.SyncRegistry(config.GlobalProjectMgr); err != nil {

src/ui/router.go

@@ -123,19 +123,21 @@ func initRouters() {
 	beego.Router("/registryproxy/*", &controllers.RegistryProxy{}, "*:Handle")
 
 	//APIs for chart repository
+	//Charts are controlled under projects
 	chartRepositoryAPIType := &api.ChartRepositoryAPI{}
-	beego.Router("/api/chartserver/health", chartRepositoryAPIType, "get:GetHealthStatus")
-	beego.Router("/api/:repo/charts", chartRepositoryAPIType, "get:ListCharts")
-	beego.Router("/api/:repo/charts/:name", chartRepositoryAPIType, "get:ListChartVersions")
-	beego.Router("/api/:repo/charts/:name/:version", chartRepositoryAPIType, "get:GetChartVersion")
-	beego.Router("/api/:repo/charts/:name/:version", chartRepositoryAPIType, "delete:DeleteChartVersion")
-	beego.Router("/api/:repo/charts", chartRepositoryAPIType, "post:UploadChartVersion")
-	beego.Router("/api/:repo/prov", chartRepositoryAPIType, "post:UploadChartProvFile")
+	beego.Router("/api/chartrepo/health", chartRepositoryAPIType, "get:GetHealthStatus")
+	beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "get:ListCharts")
+	beego.Router("/api/chartrepo/:repo/charts/:name", chartRepositoryAPIType, "get:ListChartVersions")
+	beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "get:GetChartVersion")
+	beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "delete:DeleteChartVersion")
+	beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "post:UploadChartVersion")
+	beego.Router("/api/chartrepo/:repo/prov", chartRepositoryAPIType, "post:UploadChartProvFile")
+	beego.Router("/api/chartrepo/charts", chartRepositoryAPIType, "post:UploadChartVersion")
 
 	//Repository services
-	beego.Router("/:repo/index.yaml", chartRepositoryAPIType, "get:GetIndexByRepo")
-	beego.Router("/index.yaml", chartRepositoryAPIType, "get:GetIndex")
-	beego.Router("/:repo/charts/:filename", chartRepositoryAPIType, "get:DownloadChart")
+	beego.Router("/chartrepo/:repo/index.yaml", chartRepositoryAPIType, "get:GetIndexByRepo")
+	beego.Router("/chartrepo/index.yaml", chartRepositoryAPIType, "get:GetIndex")
+	beego.Router("/chartrepo/:repo/charts/:filename", chartRepositoryAPIType, "get:DownloadChart")
 
 	//Error pages
 	beego.ErrorController(&controllers.ErrorController{})