From ef906c96d09886363e17a759b2b117b8efaf74df Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Thu, 9 Mar 2017 20:12:33 +0800 Subject: [PATCH] provide systeminfo API for UI --- docs/swagger.yaml | 37 +++++++++++++++++++ make/common/templates/adminserver/env | 4 +- make/harbor.cfg | 3 ++ make/prepare | 50 +++++++++++++------------ src/adminserver/systemcfg/systemcfg.go | 10 +++++ src/common/config/config.go | 2 + src/common/utils/notary/helper.go | 3 +- src/common/utils/notary/helper_test.go | 9 +++-- src/common/utils/test/adminserver.go | 2 + src/ui/api/harborapi_test.go | 6 +++ src/ui/api/repository.go | 2 +- src/ui/api/systeminfo.go | 46 ++++++++++++++++++++++- src/ui/api/systeminfo_test.go | 13 +++++++ src/ui/config/config.go | 51 +++++++++++++++++++++++++- src/ui/config/config_test.go | 20 ++++++++++ src/ui/router.go | 1 + src/ui/service/token/creator.go | 6 +-- 17 files changed, 228 insertions(+), 37 deletions(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 1b271e85b..211452582 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1343,6 +1343,22 @@ paths: description: User does not have permission of admin role. 500: description: Unexpected internal errors. + /systeminfo: + get: + summary: Get general system info + description: | + This API is for retrieving general system info, this can be called by anonymous request. + tags: + - Products + responses: + 200: + description: Get general info successfully. + schema: + type: object + items: + $ref: "#/definitions/GeneralInfo" + 500: + description: Unexpected internal error. /systeminfo/volumes: get: summary: Get system volume info (total/free size). @@ -1967,6 +1983,27 @@ definitions: type: integer format: int64 description: Free volume size. + GeneralInfo: + type: object + properties: + with_notary: + type: boolean + description: If the Harbor instance is deployed with nested notary. + with_admiral: + type: boolean + description: If the Harbor instance is deployed with Admiral. + admiral_endpoint: + type: string + description: The url of the endpoint of admiral instance. + auth_mode: + type: string + description: The auth mode of current Harbor instance. + project_creation_restriction: + type: string + description: Indicate who can create projects, it could be 'adminonly' or 'everyone'. + self_registration: + type: boolean + description: Indicate whether the Harbor instance enable user to register himself. SystemInfo: type: object properties: diff --git a/make/common/templates/adminserver/env b/make/common/templates/adminserver/env index 02039aa1d..e110a6608 100644 --- a/make/common/templates/adminserver/env +++ b/make/common/templates/adminserver/env @@ -34,4 +34,6 @@ JOBSERVICE_SECRET=$jobservice_secret TOKEN_EXPIRATION=$token_expiration CFG_EXPIRATION=5 USE_COMPRESSED_JS=$use_compressed_js -GODEBUG=netdns=cgo \ No newline at end of file +GODEBUG=netdns=cgo +ADMIRAL_URL=$admiral_url +WITH_NOTARY=$with_notary diff --git a/make/harbor.cfg b/make/harbor.cfg index 03e6f5f5d..844daa49e 100644 --- a/make/harbor.cfg +++ b/make/harbor.cfg @@ -40,6 +40,9 @@ ssl_cert_key = /data/cert/server.key #The path of secretkey storage secretkey_path = /data +#Admiral's url, comment this attribute, or set its value to to NA when Harbor is standalone +admiral_url = NA + #NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES #only take effect in the first boot, the subsequent changes of these properties #should be performed on web ui diff --git a/make/prepare b/make/prepare index 4d54c8833..8f47bd96f 100755 --- a/make/prepare +++ b/make/prepare @@ -147,6 +147,7 @@ token_expiration = rcp.get("configuration", "token_expiration") verify_remote_cert = rcp.get("configuration", "verify_remote_cert") proj_cre_restriction = rcp.get("configuration", "project_creation_restriction") secretkey_path = rcp.get("configuration", "secretkey_path") +admiral_url = rcp.get("configuration", "admiral_url") secret_key = get_secret_key(secretkey_path) ######## @@ -190,10 +191,10 @@ else: nginx_conf) render(os.path.join(templates_dir, "adminserver", "env"), - adminserver_conf_env, - ui_url=ui_url, - auth_mode=auth_mode, - self_registration=self_registration, + adminserver_conf_env, + ui_url=ui_url, + auth_mode=auth_mode, + self_registration=self_registration, ldap_url=ldap_url, ldap_searchdn =ldap_searchdn, ldap_search_pwd =ldap_search_pwd, @@ -203,27 +204,29 @@ render(os.path.join(templates_dir, "adminserver", "env"), ldap_scope=ldap_scope, ldap_timeout=ldap_timeout, db_password=db_password, - email_host=email_host, - email_port=email_port, - email_usr=email_usr, - email_pwd=email_pwd, - email_ssl=email_ssl, - email_from=email_from, - email_identity=email_identity, + email_host=email_host, + email_port=email_port, + email_usr=email_usr, + email_pwd=email_pwd, + email_ssl=email_ssl, + email_from=email_from, + email_identity=email_identity, harbor_admin_password=harbor_admin_password, - project_creation_restriction=proj_cre_restriction, - verify_remote_cert=verify_remote_cert, - max_job_workers=max_job_workers, - ui_secret=ui_secret, - jobservice_secret=jobservice_secret, - token_expiration=token_expiration, - use_compressed_js=use_compressed_js + project_creation_restriction=proj_cre_restriction, + verify_remote_cert=verify_remote_cert, + max_job_workers=max_job_workers, + ui_secret=ui_secret, + jobservice_secret=jobservice_secret, + token_expiration=token_expiration, + admiral_url=admiral_url, + with_notary=args.notary_mode, + use_compressed_js=use_compressed_js ) render(os.path.join(templates_dir, "ui", "env"), - ui_conf_env, - ui_secret=ui_secret, - jobservice_secret=jobservice_secret,) + ui_conf_env, + ui_secret=ui_secret, + jobservice_secret=jobservice_secret,) render(os.path.join(templates_dir, "registry", "config.yml"), @@ -237,8 +240,8 @@ render(os.path.join(templates_dir, "db", "env"), render(os.path.join(templates_dir, "jobservice", "env"), job_conf_env, ui_secret=ui_secret, - jobservice_secret=jobservice_secret) - + jobservice_secret=jobservice_secret) + print("Generated configuration file: %s" % jobservice_conf) shutil.copyfile(os.path.join(templates_dir, "jobservice", "app.conf"), jobservice_conf) @@ -328,5 +331,6 @@ if args.notary_mode: default_alias = ''.join(random.choice(string.ascii_letters) for i in range(8)) render(os.path.join(notary_temp_dir, "signer_env"), os.path.join(notary_config_dir, "signer_env"), alias = default_alias) + print("The configuration files are ready, please use docker-compose to start the service.") diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index 27ebcf753..7420ddf04 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -114,6 +114,11 @@ var ( }, comcfg.ProjectCreationRestriction: "PROJECT_CREATION_RESTRICTION", comcfg.AdminInitialPassword: "HARBOR_ADMIN_PASSWORD", + comcfg.AdmiralEndpoint: "ADMIRAL_URL", + comcfg.WithNotary: &parser{ + env: "WITH_NOTARY", + parse: parseStringToBool, + }, } // configurations need read from environment variables @@ -134,6 +139,11 @@ var ( env: "CFG_EXPIRATION", parse: parseStringToInt, }, + comcfg.AdmiralEndpoint: "ADMIRAL_URL", + comcfg.WithNotary: &parser{ + env: "WITH_NOTARY", + parse: parseStringToBool, + }, } ) diff --git a/src/common/config/config.go b/src/common/config/config.go index ff3b86c22..755a2d3a4 100644 --- a/src/common/config/config.go +++ b/src/common/config/config.go @@ -75,6 +75,8 @@ const ( JobLogDir = "job_log_dir" UseCompressedJS = "use_compressed_js" AdminInitialPassword = "admin_initial_password" + AdmiralEndpoint = "admiral_url" + WithNotary = "with_notary" ) // Manager manages configurations diff --git a/src/common/utils/notary/helper.go b/src/common/utils/notary/helper.go index 019f42889..0c6be9fc6 100644 --- a/src/common/utils/notary/helper.go +++ b/src/common/utils/notary/helper.go @@ -28,7 +28,6 @@ import ( ) var ( - notaryEndpoint = "http://notary-server:4443" notaryCachePath = "/root/notary" trustPin trustpinning.TrustPinConfig mockRetriever notary.PassRetriever @@ -55,7 +54,7 @@ func init() { // GetTargets is a help function called by API to fetch signature information of a given repository. // Per docker's convention the repository should contain the information of endpoint, i.e. it should look // like "10.117.4.117/library/ubuntu", instead of "library/ubuntu" (fqRepo for fully-qualified repo) -func GetTargets(username string, fqRepo string) ([]Target, error) { +func GetTargets(notaryEndpoint string, username string, fqRepo string) ([]Target, error) { res := []Target{} authorizer := auth.NewNotaryUsernameTokenAuthorizer(username, "repository", fqRepo, "pull") store, err := auth.NewAuthorizerStore(strings.Split(notaryEndpoint, "//")[1], true, authorizer) diff --git a/src/common/utils/notary/helper_test.go b/src/common/utils/notary/helper_test.go index 61b802e23..0ef4d225c 100644 --- a/src/common/utils/notary/helper_test.go +++ b/src/common/utils/notary/helper_test.go @@ -5,17 +5,18 @@ import ( "github.com/stretchr/testify/assert" notarytest "github.com/vmware/harbor/src/common/utils/notary/test" + "net/http/httptest" "os" "path" "testing" ) var endpoint = "10.117.4.142" +var notaryServer *httptest.Server func TestMain(m *testing.M) { - notaryServer := notarytest.NewNotaryServer(endpoint) + notaryServer = notarytest.NewNotaryServer(endpoint) defer notaryServer.Close() - notaryEndpoint = notaryServer.URL notaryCachePath = "/tmp/notary" result := m.Run() if result != 0 { @@ -24,12 +25,12 @@ func TestMain(m *testing.M) { } func TestGetTargets(t *testing.T) { - targets, err := GetTargets("admin", path.Join(endpoint, "notary-demo/busybox")) + targets, err := GetTargets(notaryServer.URL, "admin", path.Join(endpoint, "notary-demo/busybox")) assert.Nil(t, err, fmt.Sprintf("Unexpected error: %v", err)) assert.Equal(t, 1, len(targets), "") assert.Equal(t, "1.0", targets[0].Tag, "") - targets, err = GetTargets("admin", path.Join(endpoint, "notary-demo/notexist")) + targets, err = GetTargets(notaryServer.URL, "admin", path.Join(endpoint, "notary-demo/notexist")) assert.Nil(t, err, fmt.Sprintf("Unexpected error: %v", err)) assert.Equal(t, 0, len(targets), "Targets list should be empty for non exist repo.") } diff --git a/src/common/utils/test/adminserver.go b/src/common/utils/test/adminserver.go index 9d772ee38..794394775 100644 --- a/src/common/utils/test/adminserver.go +++ b/src/common/utils/test/adminserver.go @@ -58,6 +58,8 @@ var adminServerDefaultConfig = map[string]interface{}{ config.CfgExpiration: 5, config.UseCompressedJS: true, config.AdminInitialPassword: "password", + config.AdmiralEndpoint: "http://www.vmware.com", + config.WithNotary: false, } // NewAdminserver returns a mock admin server diff --git a/src/ui/api/harborapi_test.go b/src/ui/api/harborapi_test.go index c2c51e2d5..d3bb80f9d 100644 --- a/src/ui/api/harborapi_test.go +++ b/src/ui/api/harborapi_test.go @@ -97,6 +97,7 @@ func init() { beego.Router("/api/policies/replication", &RepPolicyAPI{}, "get:List") beego.Router("/api/policies/replication", &RepPolicyAPI{}, "post:Post;delete:Delete") beego.Router("/api/policies/replication/:id([0-9]+)/enablement", &RepPolicyAPI{}, "put:UpdateEnablement") + beego.Router("/api/systeminfo", &SystemInfoAPI{}, "get:GetGeneralInfo") beego.Router("/api/systeminfo/volumes", &SystemInfoAPI{}, "get:GetVolumeInfo") beego.Router("/api/systeminfo/getcert", &SystemInfoAPI{}, "get:GetCert") beego.Router("/api/ldap/ping", &LdapAPI{}, "post:Ping") @@ -953,6 +954,11 @@ func (a testapi) VolumeInfoGet(authInfo usrInfo) (int, apilib.SystemInfo, error) return httpStatusCode, successPayLoad, err } +func (a testapi) GetGeneralInfo() (int, []byte, error) { + _sling := sling.New().Get(a.basePath).Path("/api/systeminfo") + return request(_sling, jsonAcceptHeader) +} + //Get system cert func (a testapi) CertGet(authInfo usrInfo) (int, []byte, error) { _sling := sling.New().Get(a.basePath) diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index eb443546c..a6363858c 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -554,7 +554,7 @@ func (ra *RepositoryAPI) GetSignatures() { if err != nil { log.Warningf("Error when getting username: %v", err) } - targets, err := notary.GetTargets(username, fqRepo) + targets, err := notary.GetTargets(config.InternalNotaryEndpoint(), username, fqRepo) if err != nil { log.Errorf("Error while fetching signature from notary: %v", err) ra.CustomAbort(http.StatusInternalServerError, "internal error") diff --git a/src/ui/api/systeminfo.go b/src/ui/api/systeminfo.go index 8afedc00f..767d96d7f 100644 --- a/src/ui/api/systeminfo.go +++ b/src/ui/api/systeminfo.go @@ -4,11 +4,14 @@ import ( "net/http" "os" "path/filepath" + "strings" "syscall" "github.com/vmware/harbor/src/common/api" + comcfg "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/config" ) //SystemInfoAPI handle requests for getting system info /api/systeminfo @@ -32,8 +35,19 @@ type Storage struct { Free uint64 `json:"free"` } -// Prepare for validating user if an admin. -func (sia *SystemInfoAPI) Prepare() { +//GeneralInfo wraps common systeminfo for anonymous request +type GeneralInfo struct { + WithNotary bool `json:"with_notary"` + WithAdmiral bool `json:"with_admiral"` + AdmiralEndpoint string `json:"admiral_endpoint"` + AuthMode string `json:"auth_mode"` + RegistryURL string `json:"registry_url"` + ProjectCreationRestrict string `json:"project_creation_restriction"` + SelfRegistration bool `json:"self_registration"` +} + +// validate for validating user if an admin. +func (sia *SystemInfoAPI) validate() { sia.currentUserID = sia.ValidateUser() var err error @@ -46,6 +60,7 @@ func (sia *SystemInfoAPI) Prepare() { // GetVolumeInfo gets specific volume storage info. func (sia *SystemInfoAPI) GetVolumeInfo() { + sia.validate() if !sia.isAdmin { sia.RenderError(http.StatusForbidden, "User does not have admin role.") return @@ -71,6 +86,7 @@ func (sia *SystemInfoAPI) GetVolumeInfo() { //GetCert gets default self-signed certificate. func (sia *SystemInfoAPI) GetCert() { + sia.validate() if sia.isAdmin { if _, err := os.Stat(defaultRootCert); !os.IsNotExist(err) { sia.Ctx.Output.Header("Content-Type", "application/octet-stream") @@ -83,3 +99,29 @@ func (sia *SystemInfoAPI) GetCert() { } sia.CustomAbort(http.StatusForbidden, "") } + +// GetGeneralInfo returns the general system info, which is to be called by anonymous user +func (sia *SystemInfoAPI) GetGeneralInfo() { + cfg, err := config.GetSystemCfg() + if err != nil { + log.Errorf("Error occured getting config: %v", err) + sia.CustomAbort(http.StatusInternalServerError, "Unexpected error") + } + var registryURL string + if l := strings.Split(cfg[comcfg.ExtEndpoint].(string), "://"); len(l) > 1 { + registryURL = l[1] + } else { + registryURL = l[0] + } + info := GeneralInfo{ + AdmiralEndpoint: cfg[comcfg.AdmiralEndpoint].(string), + WithAdmiral: config.WithAdmiral(), + WithNotary: config.WithNotary(), + AuthMode: cfg[comcfg.AUTHMode].(string), + ProjectCreationRestrict: cfg[comcfg.ProjectCreationRestriction].(string), + SelfRegistration: cfg[comcfg.SelfRegistration].(bool), + RegistryURL: registryURL, + } + sia.Data["json"] = info + sia.ServeJSON() +} diff --git a/src/ui/api/systeminfo_test.go b/src/ui/api/systeminfo_test.go index 933e47640..f52a94707 100644 --- a/src/ui/api/systeminfo_test.go +++ b/src/ui/api/systeminfo_test.go @@ -1,6 +1,7 @@ package api import ( + "encoding/json" "fmt" "github.com/stretchr/testify/assert" "testing" @@ -37,6 +38,18 @@ func TestGetVolumeInfo(t *testing.T) { } +func TestGetGeneralInfo(t *testing.T) { + apiTest := newHarborAPI() + code, body, err := apiTest.GetGeneralInfo() + assert := assert.New(t) + assert.Nil(err, fmt.Sprintf("Unexpected Error: %v", err)) + assert.Equal(200, code, fmt.Sprintf("Unexpected status code: %d", code)) + g := &GeneralInfo{} + err = json.Unmarshal(body, g) + assert.Nil(err, fmt.Sprintf("Unexpected Error: %v", err)) + assert.Equal(false, g.WithNotary, "with notary should be false") +} + func TestGetCert(t *testing.T) { fmt.Println("Testing Get Cert") assert := assert.New(t) diff --git a/src/ui/config/config.go b/src/ui/config/config.go index d6d34b906..7868651f4 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -18,6 +18,7 @@ package config import ( "encoding/json" "os" + "strings" comcfg "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/models" @@ -132,7 +133,7 @@ func TokenExpiration() (int, error) { return int(cfg[comcfg.TokenExpiration].(float64)), nil } -// ExtEndpoint returns the external URL of Harbor: protocal://host:port +// ExtEndpoint returns the external URL of Harbor: protocol://host:port func ExtEndpoint() (string, error) { cfg, err := mg.Get() if err != nil { @@ -141,6 +142,19 @@ func ExtEndpoint() (string, error) { return cfg[comcfg.ExtEndpoint].(string), nil } +// ExtURL returns the external URL: host:port +func ExtURL() (string, error) { + endpoint, err := ExtEndpoint() + if err != nil { + return "", err + } + l := strings.Split(endpoint, "://") + if len(l) > 0 { + return l[1], nil + } + return endpoint, nil +} + // SecretKey returns the secret key to encrypt the password of target func SecretKey() (string, error) { return keyProvider.Get(nil) @@ -174,6 +188,12 @@ func InternalTokenServiceEndpoint() string { return "http://ui/service/token" } +// InternalNotaryEndpoint returns notary server endpoint for internal communication between Harbor containers +// This is currently a conventional value and can be unaccessible when Harbor is not deployed with Notary. +func InternalNotaryEndpoint() string { + return "http://notary-server:4443" +} + // InitialAdminPassword returns the initial password for administrator func InitialAdminPassword() (string, error) { cfg, err := mg.Get() @@ -253,3 +273,32 @@ func UISecret() string { func JobserviceSecret() string { return os.Getenv("JOBSERVICE_SECRET") } + +// WithNotary returns a bool value to indicate if Harbor's deployed with Notary +func WithNotary() bool { + cfg, err := mg.Get() + if err != nil { + log.Errorf("Failed to get configuration, will return WithNotary == false") + return false + } + return cfg[comcfg.WithNotary].(bool) +} + +// AdmiralEndpoint returns the URL of admiral, if Harbor is not deployed with admiral it should return an empty string. +func AdmiralEndpoint() string { + cfg, err := mg.Get() + if err != nil { + log.Errorf("Failed to get configuration, will return empty string as admiral's endpoint") + + return "" + } + if e, ok := cfg[comcfg.AdmiralEndpoint].(string); !ok || e == "NA" { + cfg[comcfg.AdmiralEndpoint] = "" + } + return cfg[comcfg.AdmiralEndpoint].(string) +} + +// WithAdmiral returns a bool to indicate if Harbor's deployed with admiral. +func WithAdmiral() bool { + return len(AdmiralEndpoint()) > 0 +} diff --git a/src/ui/config/config_test.go b/src/ui/config/config_test.go index 789cf53d6..981daa384 100644 --- a/src/ui/config/config_test.go +++ b/src/ui/config/config_test.go @@ -120,4 +120,24 @@ func TestConfig(t *testing.T) { if _, err := Database(); err != nil { t.Fatalf("failed to get database: %v", err) } + if InternalNotaryEndpoint() != "http://notary-server:4443" { + t.Errorf("Unexpected notary endpoint: %s", InternalNotaryEndpoint()) + } + if WithNotary() { + t.Errorf("Withnotary should be false") + } + if !WithAdmiral() { + t.Errorf("WithAdmiral should be true") + } + if AdmiralEndpoint() != "http://www.vmware.com" { + t.Errorf("Unexpected admiral endpoint: %s", AdmiralEndpoint()) + } + + extURL, err := ExtURL() + if err != nil { + t.Errorf("Unexpected error getting external URL: %v", err) + } + if extURL != "host01.com" { + t.Errorf(`extURL should be "host01.com".`) + } } diff --git a/src/ui/router.go b/src/ui/router.go index a26c3806d..349aafac8 100644 --- a/src/ui/router.go +++ b/src/ui/router.go @@ -92,6 +92,7 @@ func initRouters() { beego.Router("/api/logs", &api.LogAPI{}) beego.Router("/api/configurations", &api.ConfigAPI{}) + beego.Router("/api/systeminfo", &api.SystemInfoAPI{}, "get:GetGeneralInfo") beego.Router("/api/systeminfo/volumes", &api.SystemInfoAPI{}, "get:GetVolumeInfo") beego.Router("/api/systeminfo/getcert", &api.SystemInfoAPI{}, "get:GetCert") beego.Router("/api/ldap/ping", &api.LdapAPI{}, "post:Ping") diff --git a/src/ui/service/token/creator.go b/src/ui/service/token/creator.go index a572ab932..d1602fb7b 100644 --- a/src/ui/service/token/creator.go +++ b/src/ui/service/token/creator.go @@ -43,14 +43,14 @@ func InitCreators() { }, "registry": ®istryFilter{}, } - ext, err := config.ExtEndpoint() + ext, err := config.ExtURL() if err != nil { - log.Warningf("Failed to get ext enpoint, err: %v, the token service will not be functional with notary requests", err) + log.Warningf("Failed to get ext url, err: %v, the token service will not be functional with notary requests", err) } else { notaryFilterMap = map[string]accessFilter{ "repository": &repositoryFilter{ parser: &endpointParser{ - endpoint: strings.Split(ext, "//")[1], + endpoint: ext, }, }, }