From c34b2872bc6f5b22402c46fcc36e838fb92116ac Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Tue, 15 Nov 2016 18:37:36 +0800 Subject: [PATCH] config refactory for common pkg --- make/common/templates/ui/env | 2 +- src/common/api/base.go | 13 +- src/common/config/config.go | 170 ++++++++++++++++++ src/common/config/config_test.go | 98 ++++++++++ src/common/dao/base.go | 37 +--- src/common/utils/log/logger.go | 5 +- .../utils/registry/auth/tokenauthorizer.go | 12 +- 7 files changed, 287 insertions(+), 50 deletions(-) create mode 100644 src/common/config/config.go create mode 100644 src/common/config/config_test.go diff --git a/make/common/templates/ui/env b/make/common/templates/ui/env index 1b3cea9e3..1f6b2052b 100644 --- a/make/common/templates/ui/env +++ b/make/common/templates/ui/env @@ -23,6 +23,6 @@ USE_COMPRESSED_JS=$use_compressed_js LOG_LEVEL=debug GODEBUG=netdns=cgo EXT_ENDPOINT=$ui_url -TOKEN_URL=http://ui +TOKEN_ENDPOINT=http://ui VERIFY_REMOTE_CERT=$verify_remote_cert TOKEN_EXPIRATION=$token_expiration diff --git a/src/common/api/base.go b/src/common/api/base.go index fd1802979..94375db43 100644 --- a/src/common/api/base.go +++ b/src/common/api/base.go @@ -19,14 +19,14 @@ import ( "encoding/json" "fmt" "net/http" - "os" "strconv" "github.com/astaxie/beego/validation" - "github.com/vmware/harbor/src/ui/auth" + "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/auth" "github.com/astaxie/beego" ) @@ -213,12 +213,5 @@ func (b *BaseAPI) GetPaginationParams() (page, pageSize int64) { // GetIsInsecure ... func GetIsInsecure() bool { - insecure := false - - verifyRemoteCert := os.Getenv("VERIFY_REMOTE_CERT") - if verifyRemoteCert == "off" { - insecure = true - } - - return insecure + return config.VerifyRemoteCert() } diff --git a/src/common/config/config.go b/src/common/config/config.go new file mode 100644 index 000000000..121cc9cb2 --- /dev/null +++ b/src/common/config/config.go @@ -0,0 +1,170 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package config + +import ( + "fmt" + "os" + "strings" +) + +// ConfigLoader is the interface to load configurations +type ConfigLoader interface { + + // Load will load configuration from different source into a string map, the values in the map will be parsed in to configurations. + Load() (map[string]string, error) +} + +// EnvConfigLoader loads the config from env vars. +type EnvConfigLoader struct { + keys []string +} + +// Load ... +func (ec *EnvConfigLoader) Load() (map[string]string, error) { + m := make(map[string]string) + for _, k := range ec.keys { + m[k] = os.Getenv(k) + } + return m, nil +} + +// ConfigParser +type ConfigParser interface { + //Parse parse the input raw map into a config map + Parse(raw map[string]string, config map[string]interface{}) error +} + +type Config struct { + config map[string]interface{} + loader ConfigLoader + parser ConfigParser +} + +func (conf *Config) load() error { + rawMap, err := conf.loader.Load() + if err != nil { + return err + } + err = conf.parser.Parse(rawMap, conf.config) + return err +} + +type MySQLSetting struct { + Database string + User string + Password string + Host string + Port string +} + +type SQLiteSetting struct { + FilePath string +} + +type commonParser struct{} + +// Parse parses the db settings, veryfy_remote_cert, ext_endpoint, token_endpoint +func (cp *commonParser) Parse(raw map[string]string, config map[string]interface{}) error { + db := strings.ToLower(raw["DATABASE"]) + if db == "mysql" || db == "" { + db = "mysql" + mySQLDB := raw["MYSQL_DATABASE"] + if len(mySQLDB) == 0 { + mySQLDB = "registry" + } + setting := MySQLSetting{ + mySQLDB, + raw["MYSQL_USR"], + raw["MYSQL_PWD"], + raw["MYSQL_HOST"], + raw["MYSQL_PORT"], + } + config["mysql"] = setting + } else if db == "sqlite" { + f := raw["SQLITE_FILE"] + if len(f) == 0 { + f = "registry.db" + } + setting := SQLiteSetting{ + f, + } + config["sqlite"] = setting + } else { + return fmt.Errorf("Invalid DB: %s", db) + } + config["database"] = db + + //By default it's true + config["verify_remote_cert"] = raw["VERIFY_REMOTE_CERT"] != "off" + + config["ext_endpoint"] = raw["EXT_ENDPOINT"] + config["token_endpoint"] = raw["TOKEN_ENDPOINT"] + config["log_level"] = raw["LOG_LEVEL"] + return nil +} + +var commonConfig *Config + +func init() { + commonKeys := []string{"DATABASE", "MYSQL_DATABASE", "MYSQL_USR", "MYSQL_PWD", "MYSQL_HOST", "MYSQL_PORT", "SQLITE_FILE", "VERIFY_REMOTE_CERT", "EXT_ENDPOINT", "TOKEN_ENDPOINT", "LOG_LEVEL"} + commonConfig = &Config{ + config: make(map[string]interface{}), + loader: &EnvConfigLoader{keys: commonKeys}, + parser: &commonParser{}, + } + if err := commonConfig.load(); err != nil { + panic(err) + } +} + +// Reload will reload the configuration. +func Reload() error { + return commonConfig.load() +} + +// Database returns the DB type in configuration. +func Database() string { + return commonConfig.config["database"].(string) +} + +// MySQL returns the mysql setting in configuration. +func MySQL() MySQLSetting { + return commonConfig.config["mysql"].(MySQLSetting) +} + +// SQLite returns the SQLite setting +func SQLite() SQLiteSetting { + return commonConfig.config["sqlite"].(SQLiteSetting) +} + +// VerifyRemoteCert returns bool value. +func VerifyRemoteCert() bool { + return commonConfig.config["verify_remote_cert"].(bool) +} + +// ExtEndpoint ... +func ExtEndpoint() string { + return commonConfig.config["ext_endpoint"].(string) +} + +// TokenEndpoint returns the endpoint string of token service, which can be accessed by internal service of Harbor. +func TokenEndpoint() string { + return commonConfig.config["token_endpoint"].(string) +} + +func LogLevel() string { + return commonConfig.config["log_level"].(string) +} diff --git a/src/common/config/config_test.go b/src/common/config/config_test.go new file mode 100644 index 000000000..34d6fb7b5 --- /dev/null +++ b/src/common/config/config_test.go @@ -0,0 +1,98 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package config + +import ( + "os" + "testing" +) + +func TestEnvConfLoader(t *testing.T) { + os.Unsetenv("KEY2") + os.Setenv("KEY1", "V1") + os.Setenv("KEY3", "V3") + keys := []string{"KEY1", "KEY2"} + ecl := EnvConfigLoader{ + keys, + } + m, err := ecl.Load() + if err != nil { + t.Errorf("Error loading the configuration via env: %v", err) + } + if m["KEY1"] != "V1" { + t.Errorf("The value for key KEY1 should be V1, but infact: %s", m["KEY1"]) + } + if len(m["KEY2"]) > 0 { + t.Errorf("The value for key KEY2 should be emptye, but infact: %s", m["KEY2"]) + } + if _, ok := m["KEY3"]; ok { + t.Errorf("The KEY3 should not be in result as it's not in the initial key list") + } + os.Unsetenv("KEY1") + os.Unsetenv("KEY3") +} + +func TestCommonConfig(t *testing.T) { + + mysql := MySQLSetting{"registry", "root", "password", "127.0.0.1", "3306"} + sqlite := SQLiteSetting{"file.db"} + verify := "off" + ext := "http://harbor" + token := "http://token" + loglevel := "info" + + os.Setenv("DATABASE", "") + os.Setenv("MYSQL_DATABASE", mysql.Database) + os.Setenv("MYSQL_USR", mysql.User) + os.Setenv("MYSQL_PWD", mysql.Password) + os.Setenv("MYSQL_HOST", mysql.Host) + os.Setenv("MYSQL_PORT", mysql.Port) + os.Setenv("SQLITE_FILE", sqlite.FilePath) + os.Setenv("VERIFY_REMOTE_CERT", verify) + os.Setenv("EXT_ENDPOINT", ext) + os.Setenv("TOKEN_ENDPOINT", token) + os.Setenv("LOG_LEVEL", loglevel) + + err := Reload() + if err != nil { + t.Errorf("Unexpected error when loading the configurations, error: %v", err) + } + if Database() != "mysql" { + t.Errorf("Expected Database value: mysql, fact: %s", mysql) + } + if MySQL() != mysql { + t.Errorf("Expected MySQL setting: %+v, fact: %+v", mysql, MySQL()) + } + if VerifyRemoteCert() { + t.Errorf("Expected VerifyRemoteCert: false, env var: %s, fact: %v", verify, VerifyRemoteCert()) + } + if ExtEndpoint() != ext { + t.Errorf("Expected ExtEndpoint: %s, fact: %s", ext, ExtEndpoint()) + } + if TokenEndpoint() != token { + t.Errorf("Expected TokenEndpoint: %s, fact: %s", token, TokenEndpoint()) + } + if LogLevel() != loglevel { + t.Errorf("Expected LogLevel: %s, fact: %s", loglevel, LogLevel) + } + os.Setenv("DATABASE", "sqlite") + err = Reload() + if err != nil { + t.Errorf("Unexpected error when loading the configurations, error: %v", err) + } + if SQLite() != sqlite { + t.Errorf("Expected SQLite setting: %+v, fact %+v", sqlite, SQLite()) + } +} diff --git a/src/common/dao/base.go b/src/common/dao/base.go index 55e452f7a..c974cca6e 100644 --- a/src/common/dao/base.go +++ b/src/common/dao/base.go @@ -17,11 +17,10 @@ package dao import ( "fmt" - "os" - "strings" "sync" "github.com/astaxie/beego/orm" + "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/utils/log" ) @@ -52,42 +51,18 @@ func InitDatabase() { } func getDatabase() (db Database, err error) { - switch strings.ToLower(os.Getenv("DATABASE")) { + switch config.Database() { case "", "mysql": - host, port, usr, pwd, database := getMySQLConnInfo() - db = NewMySQL(host, port, usr, pwd, database) + db = NewMySQL(config.MySQL().Host, config.MySQL().Port, config.MySQL().User, + config.MySQL().Port, config.MySQL().Database) case "sqlite": - file := getSQLiteConnInfo() - db = NewSQLite(file) + db = NewSQLite(config.SQLite().FilePath) default: - err = fmt.Errorf("invalid database: %s", os.Getenv("DATABASE")) - } - - return -} - -// TODO read from config -func getMySQLConnInfo() (host, port, username, password, database string) { - host = os.Getenv("MYSQL_HOST") - port = os.Getenv("MYSQL_PORT") - username = os.Getenv("MYSQL_USR") - password = os.Getenv("MYSQL_PWD") - database = os.Getenv("MYSQL_DATABASE") - if len(database) == 0 { - database = "registry" + err = fmt.Errorf("invalid database: %s", config.Database()) } return } -// TODO read from config -func getSQLiteConnInfo() string { - file := os.Getenv("SQLITE_FILE") - if len(file) == 0 { - file = "registry.db" - } - return file -} - var globalOrm orm.Ormer var once sync.Once diff --git a/src/common/utils/log/logger.go b/src/common/utils/log/logger.go index c9a6c5072..dceb70a97 100644 --- a/src/common/utils/log/logger.go +++ b/src/common/utils/log/logger.go @@ -22,6 +22,8 @@ import ( "runtime" "sync" "time" + + "github.com/vmware/harbor/src/common/config" ) var logger = New(os.Stdout, NewTextFormatter(), WarningLevel) @@ -29,8 +31,7 @@ var logger = New(os.Stdout, NewTextFormatter(), WarningLevel) func init() { logger.callDepth = 4 - // TODO add item in configuaration file - lvl := os.Getenv("LOG_LEVEL") + lvl := config.LogLevel() if len(lvl) == 0 { logger.SetLevel(InfoLevel) return diff --git a/src/common/utils/registry/auth/tokenauthorizer.go b/src/common/utils/registry/auth/tokenauthorizer.go index c7147ea57..14b230e5a 100644 --- a/src/common/utils/registry/auth/tokenauthorizer.go +++ b/src/common/utils/registry/auth/tokenauthorizer.go @@ -21,15 +21,15 @@ import ( "io/ioutil" "net/http" "net/url" - "os" "strings" "sync" "time" - token_util "github.com/vmware/harbor/src/ui/service/token" + "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/common/utils/registry" registry_error "github.com/vmware/harbor/src/common/utils/registry/error" + token_util "github.com/vmware/harbor/src/ui/service/token" ) const ( @@ -234,11 +234,11 @@ func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes [] // 2. the realm field returned by registry is an IP which can not reachable // inside Harbor func tokenURL(realm string) string { - extEndpoint := os.Getenv("EXT_ENDPOINT") - tokenURL := os.Getenv("TOKEN_URL") - if len(extEndpoint) != 0 && len(tokenURL) != 0 && + extEndpoint := config.ExtEndpoint() + tokenEndpoint := config.TokenEndpoint() + if len(extEndpoint) != 0 && len(tokenEndpoint) != 0 && strings.Contains(realm, extEndpoint) { - realm = strings.TrimRight(tokenURL, "/") + "/service/token" + realm = strings.TrimRight(tokenEndpoint, "/") + "/service/token" } return realm }