diff --git a/.travis.yml b/.travis.yml index 9d0b8d016..0f30b1e14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ env: SELF_REGISTRATION: on KEY_PATH: /data/secretkey REDIS_HOST: localhost + BACKEND_CHART_SERVER: http://localhost:9090 before_install: - sudo ./tests/hostcfg.sh diff --git a/src/chartserver/client.go b/src/chartserver/client.go index 0deda03e9..41c0ffb88 100644 --- a/src/chartserver/client.go +++ b/src/chartserver/client.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "strings" "time" ) @@ -42,12 +43,17 @@ func NewChartClient(credentail *Credential) *ChartClient { //Create http client } //GetContent get the bytes from the specified url -func (cc *ChartClient) GetContent(url string) ([]byte, error) { - if len(strings.TrimSpace(url)) == 0 { +func (cc *ChartClient) GetContent(addr string) ([]byte, error) { + if len(strings.TrimSpace(addr)) == 0 { return nil, errors.New("empty url is not allowed") } - request, err := http.NewRequest(http.MethodGet, url, nil) + fullURI, err := url.Parse(addr) + if err != nil { + return nil, fmt.Errorf("invalid url: %s", err.Error()) + } + + request, err := http.NewRequest(http.MethodGet, addr, nil) if err != nil { return nil, err } @@ -69,7 +75,7 @@ func (cc *ChartClient) GetContent(url string) ([]byte, error) { defer response.Body.Close() if response.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to retrieve content from url '%s' with error: %s", url, content) + return nil, fmt.Errorf("failed to retrieve content from '%s' with error: %s", fullURI.Path, content) } return content, nil diff --git a/src/chartserver/manipulation_handler.go b/src/chartserver/manipulation_handler.go index e246d4f6c..ae79dafbd 100644 --- a/src/chartserver/manipulation_handler.go +++ b/src/chartserver/manipulation_handler.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/url" + "path" "strings" "github.com/ghodss/yaml" @@ -43,10 +44,10 @@ type ManipulationHandler struct { //ListCharts lists all the charts under the specified namespace func (mh *ManipulationHandler) ListCharts(w http.ResponseWriter, req *http.Request) { - rootURL := strings.TrimSuffix(mh.backendServerAddress.String(), "/") - fullURL := fmt.Sprintf("%s%s", rootURL, req.RequestURI) + url := strings.TrimPrefix(req.URL.String(), "/") + url = fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), url) - content, err := mh.apiClient.GetContent(fullURL) + content, err := mh.apiClient.GetContent(url) if err != nil { writeInternalError(w, err) return @@ -76,7 +77,7 @@ func (mh *ManipulationHandler) GetChart(w http.ResponseWriter, req *http.Request //This handler should return the details of the chart version, //maybe including metadata,dependencies and values etc. func (mh *ManipulationHandler) GetChartVersion(w http.ResponseWriter, req *http.Request) { - chartV, err := mh.getChartVersion(req.RequestURI) + chartV, err := mh.getChartVersion(req.URL.String()) if err != nil { writeInternalError(w, err) return @@ -149,11 +150,10 @@ func (mh *ManipulationHandler) DeleteChartVersion(w http.ResponseWriter, req *ht } //Get the basic metadata of chart version -func (mh *ManipulationHandler) getChartVersion(path string) (*helm_repo.ChartVersion, error) { - rootURL := strings.TrimSuffix(mh.backendServerAddress.String(), "/") - fullURL := fmt.Sprintf("%s%s", rootURL, path) +func (mh *ManipulationHandler) getChartVersion(subPath string) (*helm_repo.ChartVersion, error) { + url := fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), strings.TrimPrefix(subPath, "/")) - content, err := mh.apiClient.GetContent(fullURL) + content, err := mh.apiClient.GetContent(url) if err != nil { return nil, err } @@ -167,9 +167,9 @@ func (mh *ManipulationHandler) getChartVersion(path string) (*helm_repo.ChartVer } //Get the content bytes of the chart version -func (mh *ManipulationHandler) getChartVersionContent(namespace string, path string) ([]byte, error) { - rootURL := strings.TrimSuffix(mh.backendServerAddress.String(), "/") - fullPath := fmt.Sprintf("%s/%s/%s", rootURL, namespace, path) +func (mh *ManipulationHandler) getChartVersionContent(namespace string, subPath string) ([]byte, error) { + url := path.Join(namespace, subPath) + url = fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), url) - return mh.apiClient.GetContent(fullPath) + return mh.apiClient.GetContent(url) } diff --git a/src/chartserver/repo_handler.go b/src/chartserver/repo_handler.go index 401cdaad1..6f510dd12 100644 --- a/src/chartserver/repo_handler.go +++ b/src/chartserver/repo_handler.go @@ -6,7 +6,7 @@ import ( "fmt" "net/http" "net/url" - "strings" + "path" "sync" "sync/atomic" "time" @@ -14,6 +14,8 @@ import ( "github.com/ghodss/yaml" "github.com/vmware/harbor/src/ui/filter" helm_repo "k8s.io/helm/pkg/repo" + + hlog "github.com/vmware/harbor/src/common/utils/log" ) const ( @@ -209,8 +211,9 @@ func (rh *RepositoryHandler) DownloadChartObject(w http.ResponseWriter, req *htt //Get the index yaml file under the specified namespace from the backend server func (rh *RepositoryHandler) getIndexYamlWithNS(namespace string) (*helm_repo.IndexFile, error) { //Join url path - rootURL := strings.TrimSuffix(rh.backendServerAddress.String(), "/") - url := fmt.Sprintf("%s/%s/index.yaml", rootURL, namespace) + url := path.Join(namespace, "index.yaml") + url = fmt.Sprintf("%s/%s", rh.backendServerAddress.String(), url) + hlog.Debugf("Getting index.yaml from '%s'", url) content, err := rh.apiClient.GetContent(url) if err != nil { @@ -238,7 +241,7 @@ func (rh *RepositoryHandler) mergeIndexFile(namespace string, version.Name = nameWithNS //Currently there is only one url for index, url := range version.URLs { - version.URLs[index] = fmt.Sprintf("%s/%s", namespace, url) + version.URLs[index] = path.Join(namespace, url) } } diff --git a/src/ui/api/chart_repository.go b/src/ui/api/chart_repository.go index 1d8d89634..30b0c7e2d 100644 --- a/src/ui/api/chart_repository.go +++ b/src/ui/api/chart_repository.go @@ -13,8 +13,12 @@ import ( ) const ( - backendChartServerAddr = "BACKEND_CHART_SERVER" - namespaceParam = ":repo" + backendChartServerAddr = "BACKEND_CHART_SERVER" + namespaceParam = ":repo" + defaultRepo = "library" + rootUploadingEndpoint = "/api/chartrepo/charts" + rootIndexEndpoint = "/chartrepo/index.yaml" + chartRepoHealthEndpoint = "/api/chartrepo/health" accessLevelPublic = iota accessLevelRead @@ -49,11 +53,28 @@ func (cra *ChartRepositoryAPI) Prepare() { // -/index.yaml // -/api/chartserver/health incomingURI := cra.Ctx.Request.RequestURI - if incomingURI != "/index.yaml" && incomingURI != "/api/chartserver/health" { + if incomingURI == rootUploadingEndpoint { + //Forward to the default repository + cra.namespace = defaultRepo + } + + if incomingURI != rootIndexEndpoint && + incomingURI != chartRepoHealthEndpoint { if !cra.requireNamespace(cra.namespace) { return } } + + //Rewrite URL path + cra.rewriteURLPath(cra.Ctx.Request) +} + +//UploadChartVersionToDefaultNS ... +func (cra *ChartRepositoryAPI) UploadChartVersionToDefaultNS() { + res := make(map[string]interface{}) + res["result"] = "done" + cra.Data["json"] = res + cra.ServeJSON() } //GetHealthStatus handles GET /api/chartserver/health @@ -63,11 +84,7 @@ func (cra *ChartRepositoryAPI) GetHealthStatus() { return } - //Override the request path to '/health' - req := cra.Ctx.Request - req.URL.Path = "/health" - - chartController.GetBaseHandler().GetHealthStatus(cra.Ctx.ResponseWriter, req) + chartController.GetBaseHandler().GetHealthStatus(cra.Ctx.ResponseWriter, cra.Ctx.Request) } //GetIndexByRepo handles GET /:repo/index.yaml @@ -146,6 +163,8 @@ func (cra *ChartRepositoryAPI) DeleteChartVersion() { //UploadChartVersion handles POST /api/:repo/charts func (cra *ChartRepositoryAPI) UploadChartVersion() { + hlog.Debugf("Header of request of uploading chart: %#v, content-len=%d", cra.Ctx.Request.Header, cra.Ctx.Request.ContentLength) + //Check access if !cra.requireAccess(cra.namespace, accessLevelWrite) { return @@ -164,6 +183,41 @@ func (cra *ChartRepositoryAPI) UploadChartProvFile() { chartController.GetManipulationHandler().UploadProvenanceFile(cra.Ctx.ResponseWriter, cra.Ctx.Request) } +//Rewrite the incoming URL with the right backend URL pattern +//Remove 'chartrepo' from the endpoints of manipulation API +//Remove 'chartrepo' from the endpoints of repository services +func (cra *ChartRepositoryAPI) rewriteURLPath(req *http.Request) { + incomingURLPath := req.RequestURI + + defer func() { + hlog.Debugf("Incoming URL '%s' is rewritten to '%s'", incomingURLPath, req.URL.String()) + }() + + //Health check endpoint + if incomingURLPath == chartRepoHealthEndpoint { + req.URL.Path = "/health" + return + } + + //Root uploading endpoint + if incomingURLPath == rootUploadingEndpoint { + req.URL.Path = strings.Replace(incomingURLPath, "chartrepo", defaultRepo, 1) + return + } + + //Repository endpoints + if strings.HasPrefix(incomingURLPath, "/chartrepo") { + req.URL.Path = strings.TrimPrefix(incomingURLPath, "/chartrepo") + return + } + + //API endpoints + if strings.HasPrefix(incomingURLPath, "/api/chartrepo") { + req.URL.Path = strings.Replace(incomingURLPath, "/chartrepo", "", 1) + return + } +} + //Check if there exists a valid namespace //Return true if it does //Return false if it does not @@ -243,17 +297,23 @@ func (cra *ChartRepositoryAPI) requireAccess(namespace string, accessLevel uint) //Initialize the chart service controller func initializeChartController() *chartserver.Controller { - addr := os.Getenv(backendChartServerAddr) + addr := strings.TrimSpace(os.Getenv(backendChartServerAddr)) + if len(addr) == 0 { + hlog.Fatal("The address of chart storage server is not set") + } + + addr = strings.TrimSuffix(addr, "/") url, err := url.Parse(addr) if err != nil { - hlog.Fatal("chart storage server is not correctly configured") + hlog.Fatal("Chart storage server is not correctly configured") } controller, err := chartserver.NewController(url) if err != nil { - hlog.Fatal("failed to initialize chart API controller") + hlog.Fatal("Failed to initialize chart API controller") } + hlog.Debugf("Chart storage server is set to %s", url.String()) hlog.Info("API controller for chart repository server is successfully initialized") return controller diff --git a/src/ui/filter/mediatype.go b/src/ui/filter/mediatype.go index 8d2765faf..6874856f8 100644 --- a/src/ui/filter/mediatype.go +++ b/src/ui/filter/mediatype.go @@ -15,9 +15,12 @@ package filter import ( - beegoctx "github.com/astaxie/beego/context" "net/http" "strings" + + beegoctx "github.com/astaxie/beego/context" + + hlog "github.com/vmware/harbor/src/common/utils/log" ) //MediaTypeFilter filters the POST request, it returns 415 if the content type of the request @@ -34,6 +37,8 @@ func filterContentType(req *http.Request, resp http.ResponseWriter, mediaType .. } v := req.Header.Get("Content-Type") mimeType := strings.Split(v, ";")[0] + hlog.Debugf("Mimetype of incoming request %s: %s", req.RequestURI, mimeType) + for _, t := range mediaType { if t == mimeType { return diff --git a/src/ui/main.go b/src/ui/main.go index 99720f937..b266e6fe0 100644 --- a/src/ui/main.go +++ b/src/ui/main.go @@ -144,7 +144,7 @@ func main() { filter.Init() beego.InsertFilter("/*", beego.BeforeRouter, filter.SecurityFilter) beego.InsertFilter("/*", beego.BeforeRouter, filter.ReadonlyFilter) - beego.InsertFilter("/api/*", beego.BeforeRouter, filter.MediaTypeFilter("application/json")) + beego.InsertFilter("/api/*", beego.BeforeRouter, filter.MediaTypeFilter("application/json", "multipart/form-data", "application/octet-stream")) initRouters() if err := api.SyncRegistry(config.GlobalProjectMgr); err != nil { diff --git a/src/ui/router.go b/src/ui/router.go index 4445273b8..1d8af4d0f 100644 --- a/src/ui/router.go +++ b/src/ui/router.go @@ -123,19 +123,21 @@ func initRouters() { beego.Router("/registryproxy/*", &controllers.RegistryProxy{}, "*:Handle") //APIs for chart repository + //Charts are controlled under projects chartRepositoryAPIType := &api.ChartRepositoryAPI{} - beego.Router("/api/chartserver/health", chartRepositoryAPIType, "get:GetHealthStatus") - beego.Router("/api/:repo/charts", chartRepositoryAPIType, "get:ListCharts") - beego.Router("/api/:repo/charts/:name", chartRepositoryAPIType, "get:ListChartVersions") - beego.Router("/api/:repo/charts/:name/:version", chartRepositoryAPIType, "get:GetChartVersion") - beego.Router("/api/:repo/charts/:name/:version", chartRepositoryAPIType, "delete:DeleteChartVersion") - beego.Router("/api/:repo/charts", chartRepositoryAPIType, "post:UploadChartVersion") - beego.Router("/api/:repo/prov", chartRepositoryAPIType, "post:UploadChartProvFile") + beego.Router("/api/chartrepo/health", chartRepositoryAPIType, "get:GetHealthStatus") + beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "get:ListCharts") + beego.Router("/api/chartrepo/:repo/charts/:name", chartRepositoryAPIType, "get:ListChartVersions") + beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "get:GetChartVersion") + beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "delete:DeleteChartVersion") + beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "post:UploadChartVersion") + beego.Router("/api/chartrepo/:repo/prov", chartRepositoryAPIType, "post:UploadChartProvFile") + beego.Router("/api/chartrepo/charts", chartRepositoryAPIType, "post:UploadChartVersion") //Repository services - beego.Router("/:repo/index.yaml", chartRepositoryAPIType, "get:GetIndexByRepo") - beego.Router("/index.yaml", chartRepositoryAPIType, "get:GetIndex") - beego.Router("/:repo/charts/:filename", chartRepositoryAPIType, "get:DownloadChart") + beego.Router("/chartrepo/:repo/index.yaml", chartRepositoryAPIType, "get:GetIndexByRepo") + beego.Router("/chartrepo/index.yaml", chartRepositoryAPIType, "get:GetIndex") + beego.Router("/chartrepo/:repo/charts/:filename", chartRepositoryAPIType, "get:DownloadChart") //Error pages beego.ErrorController(&controllers.ErrorController{})