feat(configuration,db): connection pool configs for db

Signed-off-by: He Weiwei <hweiwei@vmware.com>
This commit is contained in:
He Weiwei 2019-08-12 17:14:57 +08:00
parent a545b460ed
commit 98e1f68468
10 changed files with 68 additions and 38 deletions

View File

@ -30,6 +30,11 @@ harbor_admin_password: Harbor12345
database: database:
# The password for the root user of Harbor DB. Change this before any production use. # The password for the root user of Harbor DB. Change this before any production use.
password: root123 password: root123
# The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
max_idle_conns: 50
# The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.
# Note: the default number of connections is 100 for postgres.
max_open_conns: 100
# The default data volume # The default data volume
data_volume: /data data_volume: /data
@ -50,12 +55,12 @@ data_volume: /data
# disabled: false # disabled: false
# Clair configuration # Clair configuration
clair: clair:
# The interval of clair updaters, the unit is hour, set to 0 to disable the updaters. # The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
updaters_interval: 12 updaters_interval: 12
jobservice: jobservice:
# Maximum number of job workers in job service # Maximum number of job workers in job service
max_job_workers: 10 max_job_workers: 10
notification: notification:
@ -74,8 +79,8 @@ log:
local: local:
# Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
rotate_count: 50 rotate_count: 50
# Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes.
# If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G
# are all valid. # are all valid.
rotate_size: 200M rotate_size: 200M
# The directory on your host that store log # The directory on your host that store log

View File

@ -15,6 +15,8 @@ POSTGRESQL_USERNAME={{harbor_db_username}}
POSTGRESQL_PASSWORD={{harbor_db_password}} POSTGRESQL_PASSWORD={{harbor_db_password}}
POSTGRESQL_DATABASE={{harbor_db_name}} POSTGRESQL_DATABASE={{harbor_db_name}}
POSTGRESQL_SSLMODE={{harbor_db_sslmode}} POSTGRESQL_SSLMODE={{harbor_db_sslmode}}
POSTGRESQL_MAX_IDLE_CONNS={{harbor_db_max_idle_conns}}
POSTGRESQL_MAX_OPEN_CONNS={{harbor_db_max_open_conns}}
REGISTRY_URL={{registry_url}} REGISTRY_URL={{registry_url}}
TOKEN_SERVICE_URL={{token_service_url}} TOKEN_SERVICE_URL={{token_service_url}}
HARBOR_ADMIN_PASSWORD={{harbor_admin_password}} HARBOR_ADMIN_PASSWORD={{harbor_admin_password}}

View File

@ -112,6 +112,11 @@ def parse_yaml_config(config_file_path):
config_dict['harbor_db_username'] = 'postgres' config_dict['harbor_db_username'] = 'postgres'
config_dict['harbor_db_password'] = db_configs.get("password") or '' config_dict['harbor_db_password'] = db_configs.get("password") or ''
config_dict['harbor_db_sslmode'] = 'disable' config_dict['harbor_db_sslmode'] = 'disable'
default_max_idle_conns = 2 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns
default_max_open_conns = 0 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxOpenConns
config_dict['harbor_db_max_idle_conns'] = db_configs.get("max_idle_conns") or default_max_idle_conns
config_dict['harbor_db_max_open_conns'] = db_configs.get("max_open_conns") or default_max_open_conns
# clari db # clari db
config_dict['clair_db_host'] = 'postgresql' config_dict['clair_db_host'] = 'postgresql'
config_dict['clair_db_port'] = 5432 config_dict['clair_db_port'] = 5432

View File

@ -210,12 +210,14 @@ func (c *CfgManager) GetDatabaseCfg() *models.Database {
return &models.Database{ return &models.Database{
Type: c.Get(common.DatabaseType).GetString(), Type: c.Get(common.DatabaseType).GetString(),
PostGreSQL: &models.PostGreSQL{ PostGreSQL: &models.PostGreSQL{
Host: c.Get(common.PostGreSQLHOST).GetString(), Host: c.Get(common.PostGreSQLHOST).GetString(),
Port: c.Get(common.PostGreSQLPort).GetInt(), Port: c.Get(common.PostGreSQLPort).GetInt(),
Username: c.Get(common.PostGreSQLUsername).GetString(), Username: c.Get(common.PostGreSQLUsername).GetString(),
Password: c.Get(common.PostGreSQLPassword).GetString(), Password: c.Get(common.PostGreSQLPassword).GetString(),
Database: c.Get(common.PostGreSQLDatabase).GetString(), Database: c.Get(common.PostGreSQLDatabase).GetString(),
SSLMode: c.Get(common.PostGreSQLSSLMode).GetString(), SSLMode: c.Get(common.PostGreSQLSSLMode).GetString(),
MaxIdleConns: c.Get(common.PostGreSQLMaxIdleConns).GetInt(),
MaxOpenConns: c.Get(common.PostGreSQLMaxOpenConns).GetInt(),
}, },
} }
} }

View File

@ -116,6 +116,8 @@ var (
{Name: common.PostGreSQLPort, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_PORT", DefaultValue: "5432", ItemType: &PortType{}, Editable: false}, {Name: common.PostGreSQLPort, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_PORT", DefaultValue: "5432", ItemType: &PortType{}, Editable: false},
{Name: common.PostGreSQLSSLMode, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_SSLMODE", DefaultValue: "disable", ItemType: &StringType{}, Editable: false}, {Name: common.PostGreSQLSSLMode, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_SSLMODE", DefaultValue: "disable", ItemType: &StringType{}, Editable: false},
{Name: common.PostGreSQLUsername, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_USERNAME", DefaultValue: "postgres", ItemType: &StringType{}, Editable: false}, {Name: common.PostGreSQLUsername, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_USERNAME", DefaultValue: "postgres", ItemType: &StringType{}, Editable: false},
{Name: common.PostGreSQLMaxIdleConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_IDLE_CONNS", DefaultValue: "2", ItemType: &IntType{}, Editable: false},
{Name: common.PostGreSQLMaxOpenConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_OPEN_CONNS", DefaultValue: "0", ItemType: &IntType{}, Editable: false},
{Name: common.ProjectCreationRestriction, Scope: UserScope, Group: BasicGroup, EnvKey: "PROJECT_CREATION_RESTRICTION", DefaultValue: common.ProCrtRestrEveryone, ItemType: &ProjectCreationRestrictionType{}, Editable: false}, {Name: common.ProjectCreationRestriction, Scope: UserScope, Group: BasicGroup, EnvKey: "PROJECT_CREATION_RESTRICTION", DefaultValue: common.ProCrtRestrEveryone, ItemType: &ProjectCreationRestrictionType{}, Editable: false},
{Name: common.ReadOnly, Scope: UserScope, Group: BasicGroup, EnvKey: "READ_ONLY", DefaultValue: "false", ItemType: &BoolType{}, Editable: false}, {Name: common.ReadOnly, Scope: UserScope, Group: BasicGroup, EnvKey: "READ_ONLY", DefaultValue: "false", ItemType: &BoolType{}, Editable: false},

View File

@ -53,6 +53,8 @@ const (
PostGreSQLPassword = "postgresql_password" PostGreSQLPassword = "postgresql_password"
PostGreSQLDatabase = "postgresql_database" PostGreSQLDatabase = "postgresql_database"
PostGreSQLSSLMode = "postgresql_sslmode" PostGreSQLSSLMode = "postgresql_sslmode"
PostGreSQLMaxIdleConns = "postgresql_max_idle_conns"
PostGreSQLMaxOpenConns = "postgresql_max_open_conns"
SelfRegistration = "self_registration" SelfRegistration = "self_registration"
CoreURL = "core_url" CoreURL = "core_url"
CoreLocalURL = "core_local_url" CoreLocalURL = "core_local_url"

View File

@ -121,12 +121,16 @@ func getDatabase(database *models.Database) (db Database, err error) {
switch database.Type { switch database.Type {
case "", "postgresql": case "", "postgresql":
db = NewPGSQL(database.PostGreSQL.Host, db = NewPGSQL(
database.PostGreSQL.Host,
strconv.Itoa(database.PostGreSQL.Port), strconv.Itoa(database.PostGreSQL.Port),
database.PostGreSQL.Username, database.PostGreSQL.Username,
database.PostGreSQL.Password, database.PostGreSQL.Password,
database.PostGreSQL.Database, database.PostGreSQL.Database,
database.PostGreSQL.SSLMode) database.PostGreSQL.SSLMode,
database.PostGreSQL.MaxIdleConns,
database.PostGreSQL.MaxOpenConns,
)
default: default:
err = fmt.Errorf("invalid database: %s", database.Type) err = fmt.Errorf("invalid database: %s", database.Type)
} }

View File

@ -31,12 +31,14 @@ import (
const defaultMigrationPath = "migrations/postgresql/" const defaultMigrationPath = "migrations/postgresql/"
type pgsql struct { type pgsql struct {
host string host string
port string port string
usr string usr string
pwd string pwd string
database string database string
sslmode string sslmode string
maxIdleConns int
maxOpenConns int
} }
// Name returns the name of PostgreSQL // Name returns the name of PostgreSQL
@ -51,17 +53,19 @@ func (p *pgsql) String() string {
} }
// NewPGSQL returns an instance of postgres // NewPGSQL returns an instance of postgres
func NewPGSQL(host string, port string, usr string, pwd string, database string, sslmode string) Database { func NewPGSQL(host string, port string, usr string, pwd string, database string, sslmode string, maxIdleConns int, maxOpenConns int) Database {
if len(sslmode) == 0 { if len(sslmode) == 0 {
sslmode = "disable" sslmode = "disable"
} }
return &pgsql{ return &pgsql{
host: host, host: host,
port: port, port: port,
usr: usr, usr: usr,
pwd: pwd, pwd: pwd,
database: database, database: database,
sslmode: sslmode, sslmode: sslmode,
maxIdleConns: maxIdleConns,
maxOpenConns: maxOpenConns,
} }
} }
@ -82,7 +86,7 @@ func (p *pgsql) Register(alias ...string) error {
info := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s", info := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",
p.host, p.port, p.usr, p.pwd, p.database, p.sslmode) p.host, p.port, p.usr, p.pwd, p.database, p.sslmode)
return orm.RegisterDataBase(an, "postgres", info) return orm.RegisterDataBase(an, "postgres", info, p.maxIdleConns, p.maxOpenConns)
} }
// UpgradeSchema calls migrate tool to upgrade schema to the latest based on the SQL scripts. // UpgradeSchema calls migrate tool to upgrade schema to the latest based on the SQL scripts.

View File

@ -45,12 +45,14 @@ type SQLite struct {
// PostGreSQL ... // PostGreSQL ...
type PostGreSQL struct { type PostGreSQL struct {
Host string `json:"host"` Host string `json:"host"`
Port int `json:"port"` Port int `json:"port"`
Username string `json:"username"` Username string `json:"username"`
Password string `json:"password,omitempty"` Password string `json:"password,omitempty"`
Database string `json:"database"` Database string `json:"database"`
SSLMode string `json:"sslmode"` SSLMode string `json:"sslmode"`
MaxIdleConns int `json:"max_idle_conns"`
MaxOpenConns int `json:"max_open_conns"`
} }
// Email ... // Email ...

View File

@ -331,12 +331,14 @@ func Database() (*models.Database, error) {
database := &models.Database{} database := &models.Database{}
database.Type = cfgMgr.Get(common.DatabaseType).GetString() database.Type = cfgMgr.Get(common.DatabaseType).GetString()
postgresql := &models.PostGreSQL{ postgresql := &models.PostGreSQL{
Host: cfgMgr.Get(common.PostGreSQLHOST).GetString(), Host: cfgMgr.Get(common.PostGreSQLHOST).GetString(),
Port: cfgMgr.Get(common.PostGreSQLPort).GetInt(), Port: cfgMgr.Get(common.PostGreSQLPort).GetInt(),
Username: cfgMgr.Get(common.PostGreSQLUsername).GetString(), Username: cfgMgr.Get(common.PostGreSQLUsername).GetString(),
Password: cfgMgr.Get(common.PostGreSQLPassword).GetString(), Password: cfgMgr.Get(common.PostGreSQLPassword).GetString(),
Database: cfgMgr.Get(common.PostGreSQLDatabase).GetString(), Database: cfgMgr.Get(common.PostGreSQLDatabase).GetString(),
SSLMode: cfgMgr.Get(common.PostGreSQLSSLMode).GetString(), SSLMode: cfgMgr.Get(common.PostGreSQLSSLMode).GetString(),
MaxIdleConns: cfgMgr.Get(common.PostGreSQLMaxIdleConns).GetInt(),
MaxOpenConns: cfgMgr.Get(common.PostGreSQLMaxOpenConns).GetInt(),
} }
database.PostGreSQL = postgresql database.PostGreSQL = postgresql