upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor synced 2024-09-20 13:15:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

add parameters for PostgreSQL (#16641)

Browse Source 
Signed-off-by: sayaoailun <guojianwei007@126.com>
This commit is contained in:
Jianwei Guo 2022-11-30 19:08:08 +08:00 committed by GitHub
parent 927a055aa9
commit cb11540a14
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 140 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
make/harbor.yml.tmpl
View File

@ -42,6 +42,12 @@ database:
# The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.
# Note: the default number of connections is 1024 for postgres of harbor.
max_open_conns: 900
# The maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If it <= 0, connections are not closed due to a connection's age.
# The value is a duration string. A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
conn_max_lifetime: 5m
# The maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If it <= 0, connections are not closed due to a connection's idle time.
# The value is a duration string. A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
conn_max_idle_time: 0
# The default data volume
data_volume: /data

2
make/photon/prepare/templates/core/env.jinja
View File

@ -16,6 +16,8 @@ POSTGRESQL_DATABASE={{harbor_db_name}}
POSTGRESQL_SSLMODE={{harbor_db_sslmode}}
POSTGRESQL_MAX_IDLE_CONNS={{harbor_db_max_idle_conns}}
POSTGRESQL_MAX_OPEN_CONNS={{harbor_db_max_open_conns}}
POSTGRESQL_CONN_MAX_LIFETIME={{harbor_db_conn_max_lifetime}}
POSTGRESQL_CONN_MAX_IDLE_TIME={{harbor_db_conn_max_idle_time}}
REGISTRY_URL={{registry_url}}
PORTAL_URL={{portal_url}}
TOKEN_SERVICE_URL={{token_service_url}}

4
make/photon/prepare/templates/exporter/env.jinja
View File

@ -25,4 +25,6 @@ HARBOR_DATABASE_PASSWORD={{harbor_db_password}}
HARBOR_DATABASE_DBNAME={{harbor_db_name}}
HARBOR_DATABASE_SSLMODE={{harbor_db_sslmode}}
HARBOR_DATABASE_MAX_IDLE_CONNS={{harbor_db_max_idle_conns}}
HARBOR_DATABASE_MAX_OPEN_CONNS={{harbor_db_max_open_conns}}
HARBOR_DATABASE_MAX_OPEN_CONNS={{harbor_db_max_open_conns}}
HARBOR_DATABASE_CONN_MAX_LIFETIME={{harbor_db_conn_max_lifetime}}
HARBOR_DATABASE_CONN_MAX_IDLE_TIME={{harbor_db_conn_max_idle_time}}

4
make/photon/prepare/utils/configs.py
View File

@ -158,6 +158,8 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
config_dict['harbor_db_sslmode'] = 'disable'
config_dict['harbor_db_max_idle_conns'] = db_configs.get("max_idle_conns") or default_db_max_idle_conns
config_dict['harbor_db_max_open_conns'] = db_configs.get("max_open_conns") or default_db_max_open_conns
config_dict['harbor_db_conn_max_lifetime'] = db_configs.get("conn_max_lifetime") or '5m'
config_dict['harbor_db_conn_max_idle_time'] = db_configs.get("conn_max_idle_time") or '0'
if with_notary:
# notary signer
@ -288,6 +290,8 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
config_dict['harbor_db_sslmode'] = external_db_configs['harbor']['ssl_mode']
config_dict['harbor_db_max_idle_conns'] = external_db_configs['harbor'].get("max_idle_conns") or default_db_max_idle_conns
config_dict['harbor_db_max_open_conns'] = external_db_configs['harbor'].get("max_open_conns") or default_db_max_open_conns
config_dict['harbor_db_conn_max_lifetime'] = external_db_configs['harbor'].get("conn_max_lifetime") or '5m'
config_dict['harbor_db_conn_max_idle_time'] = external_db_configs['harbor'].get("conn_max_idle_time") or '0'
if with_notary:
# notary signer

29
src/cmd/exporter/main.go
View File

@ -4,6 +4,7 @@ import (
"net/http"
"os"
"strings"
"time"
_ "github.com/jackc/pgx/v4/stdlib" // registry pgx driver
"github.com/prometheus/client_golang/prometheus"
@ -21,17 +22,29 @@ func main() {
viper.SetEnvPrefix("harbor")
viper.AutomaticEnv()
viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
connMaxLifetime, err := time.ParseDuration(viper.GetString("database.conn_max_lifetime"))
if err != nil {
log.Errorf("Failed to parse database.conn_max_lifetime: %v", err)
connMaxLifetime = 5 * time.Minute
}
connMaxIdleTime, err := time.ParseDuration(viper.GetString("database.conn_max_idle_time"))
if err != nil {
log.Errorf("Failed to parse database.conn_max_idle_time: %v", err)
connMaxIdleTime = 0
}
dbCfg := &models.Database{
Type: "postgresql",
PostGreSQL: &models.PostGreSQL{
Host: viper.GetString("database.host"),
Port: viper.GetInt("database.port"),
Username: viper.GetString("database.username"),
Password: viper.GetString("database.password"),
Database: viper.GetString("database.dbname"),
SSLMode: viper.GetString("database.sslmode"),
MaxIdleConns: viper.GetInt("database.max_idle_conns"),
MaxOpenConns: viper.GetInt("database.max_open_conns"),
Host: viper.GetString("database.host"),
Port: viper.GetInt("database.port"),
Username: viper.GetString("database.username"),
Password: viper.GetString("database.password"),
Database: viper.GetString("database.dbname"),
SSLMode: viper.GetString("database.sslmode"),
MaxIdleConns: viper.GetInt("database.max_idle_conns"),
MaxOpenConns: viper.GetInt("database.max_open_conns"),
ConnMaxLifetime: connMaxLifetime,
ConnMaxIdleTime: connMaxIdleTime,
},
}
if err := dao.InitDatabase(dbCfg); err != nil {

2
src/common/const.go
View File

@ -59,6 +59,8 @@ const (
PostGreSQLSSLMode = "postgresql_sslmode"
PostGreSQLMaxIdleConns = "postgresql_max_idle_conns"
PostGreSQLMaxOpenConns = "postgresql_max_open_conns"
PostGreSQLConnMaxLifetime = "postgresql_conn_max_lifetime"
PostGreSQLConnMaxIdleTime = "postgresql_conn_max_idle_time"
SelfRegistration = "self_registration"
CoreURL = "core_url"
CoreLocalURL = "core_local_url"

2
src/common/dao/base.go
View File

@ -85,6 +85,8 @@ func getDatabase(database *models.Database) (db Database, err error) {
database.PostGreSQL.SSLMode,
database.PostGreSQL.MaxIdleConns,
database.PostGreSQL.MaxOpenConns,
database.PostGreSQL.ConnMaxLifetime,
database.PostGreSQL.ConnMaxIdleTime,
)
default:
err = fmt.Errorf("invalid database: %s", database.Type)

46
src/common/dao/pgsql.go
View File

@ -36,14 +36,16 @@ import (
const defaultMigrationPath = "migrations/postgresql/"
type pgsql struct {
host string
port string
usr string
pwd string
database string
sslmode string
maxIdleConns int
maxOpenConns int
host string
port string
usr string
pwd string
database string
sslmode string
maxIdleConns int
maxOpenConns int
connMaxLifetime time.Duration
connMaxIdleTime time.Duration
}
// Name returns the name of PostgreSQL
@ -58,19 +60,21 @@ func (p *pgsql) String() string {
}
// NewPGSQL returns an instance of postgres
func NewPGSQL(host string, port string, usr string, pwd string, database string, sslmode string, maxIdleConns int, maxOpenConns int) Database {
func NewPGSQL(host string, port string, usr string, pwd string, database string, sslmode string, maxIdleConns int, maxOpenConns int, connMaxLifetime time.Duration, connMaxIdleTime time.Duration) Database {
if len(sslmode) == 0 {
sslmode = "disable"
}
return &pgsql{
host: host,
port: port,
usr: usr,
pwd: pwd,
database: database,
sslmode: sslmode,
maxIdleConns: maxIdleConns,
maxOpenConns: maxOpenConns,
host: host,
port: port,
usr: usr,
pwd: pwd,
database: database,
sslmode: sslmode,
maxIdleConns: maxIdleConns,
maxOpenConns: maxOpenConns,
connMaxLifetime: connMaxLifetime,
connMaxIdleTime: connMaxIdleTime,
}
}
@ -92,10 +96,16 @@ func (p *pgsql) Register(alias ...string) error {
p.host, p.port, p.usr, p.pwd, p.database, p.sslmode)
if err := orm.RegisterDataBase(an, "pgx", info, orm.MaxIdleConnections(p.maxIdleConns),
orm.MaxOpenConnections(p.maxOpenConns), orm.ConnMaxLifetime(5*time.Minute)); err != nil {
orm.MaxOpenConnections(p.maxOpenConns), orm.ConnMaxLifetime(p.connMaxLifetime)); err != nil {
return err
}
db, err := orm.GetDB(an)
if err != nil {
return err
}
db.SetConnMaxIdleTime(p.connMaxIdleTime)
return nil
}

20
src/common/models/database.go
View File

@ -14,6 +14,8 @@
package models
import "time"
// Database ...
type Database struct {
Type string `json:"type"`
@ -36,12 +38,14 @@ type SQLite struct {
// PostGreSQL ...
type PostGreSQL struct {
Host string `json:"host"`
Port int `json:"port"`
Username string `json:"username"`
Password string `json:"password,omitempty"`
Database string `json:"database"`
SSLMode string `json:"sslmode"`
MaxIdleConns int `json:"max_idle_conns"`
MaxOpenConns int `json:"max_open_conns"`
Host string `json:"host"`
Port int `json:"port"`
Username string `json:"username"`
Password string `json:"password,omitempty"`
Database string `json:"database"`
SSLMode string `json:"sslmode"`
MaxIdleConns int `json:"max_idle_conns"`
MaxOpenConns int `json:"max_open_conns"`
ConnMaxLifetime time.Duration `json:"conn_max_lifetime"`
ConnMaxIdleTime time.Duration `json:"conn_max_idle_time"`
}

2
src/lib/config/metadata/metadatalist.go
View File

@ -104,6 +104,8 @@ var (
{Name: common.PostGreSQLUsername, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_USERNAME", DefaultValue: "postgres", ItemType: &StringType{}, Editable: false},
{Name: common.PostGreSQLMaxIdleConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_IDLE_CONNS", DefaultValue: "2", ItemType: &IntType{}, Editable: false},
{Name: common.PostGreSQLMaxOpenConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_OPEN_CONNS", DefaultValue: "0", ItemType: &IntType{}, Editable: false},
{Name: common.PostGreSQLConnMaxLifetime, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_CONN_MAX_LIFETIME", DefaultValue: "5m", ItemType: &DurationType{}, Editable: false},
{Name: common.PostGreSQLConnMaxIdleTime, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_CONN_MAX_IDLE_TIME", DefaultValue: "0", ItemType: &DurationType{}, Editable: false},
{Name: common.ProjectCreationRestriction, Scope: UserScope, Group: BasicGroup, EnvKey: "PROJECT_CREATION_RESTRICTION", DefaultValue: common.ProCrtRestrEveryone, ItemType: &ProjectCreationRestrictionType{}, Editable: false, Description: `Indicate who can create projects, it could be ''adminonly'' or ''everyone''.`},
{Name: common.ReadOnly, Scope: UserScope, Group: BasicGroup, EnvKey: "READ_ONLY", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `The flag to indicate whether Harbor is in readonly mode.`},

14
src/lib/config/metadata/type.go
View File

@ -21,6 +21,7 @@ import (
"math"
"strconv"
"strings"
"time"
"github.com/goharbor/harbor/src/common"
)
@ -234,6 +235,19 @@ func (t *QuotaType) validate(str string) error {
return nil
}
// DurationType ...
type DurationType struct {
}
func (t *DurationType) validate(str string) error {
_, err := time.ParseDuration(str)
return err
}
func (t *DurationType) get(str string) (interface{}, error) {
return time.ParseDuration(str)
}
// parseInt64 returns int64 from string which support scientific notation
func parseInt64(str string) (int64, error) {
val, err := strconv.ParseInt(str, 10, 64)

17
src/lib/config/metadata/value.go
View File

@ -16,6 +16,7 @@ package metadata
import (
"errors"
"time"
"github.com/goharbor/harbor/src/lib/log"
)
@ -144,6 +145,22 @@ func (c *ConfigureValue) GetStringToStringMap() map[string]string {
return result
}
// GetDuration - return the time.Duration value of current value
func (c *ConfigureValue) GetDuration() time.Duration {
if item, ok := Instance().GetByName(c.Name); ok {
val, err := item.ItemType.get(c.Value)
if err != nil {
log.Errorf("GetDuration failed, error: %+v", err)
return 0
}
if durationValue, suc := val.(time.Duration); suc {
return durationValue
}
}
log.Errorf("GetDuration failed, the current value's metadata is not defined, %+v", c)
return 0
}
// GetAnyType get the interface{} of current value
func (c *ConfigureValue) GetAnyType() (interface{}, error) {
if item, ok := Instance().GetByName(c.Name); ok {

7
src/lib/config/metadata/value_test.go
View File

@ -17,6 +17,7 @@ package metadata
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
@ -60,6 +61,12 @@ func TestConfigureValue_GetStringToStringMap(t *testing.T) {
assert.Equal(t, val, map[string]interface{}{"sample": "abc"})
Instance().init()
}
func TestConfigureValue_GetDuration(t *testing.T) {
assert.Equal(t, createCfgValue("postgresql_conn_max_lifetime", "5m").GetDuration(), 5*time.Minute)
assert.Equal(t, createCfgValue("postgresql_conn_max_lifetime", "").GetDuration(), time.Duration(0))
}
func TestConfigureValue_GetInt(t *testing.T) {
assert.Equal(t, createCfgValue("ldap_timeout", "5").GetInt(), 5)
}

18
src/lib/config/systemconfig.go
View File

@ -274,14 +274,16 @@ func Database() (*models.Database, error) {
database := &models.Database{}
database.Type = DefaultMgr().Get(backgroundCtx, common.DatabaseType).GetString()
postgresql := &models.PostGreSQL{
Host: DefaultMgr().Get(backgroundCtx, common.PostGreSQLHOST).GetString(),
Port: DefaultMgr().Get(backgroundCtx, common.PostGreSQLPort).GetInt(),
Username: DefaultMgr().Get(backgroundCtx, common.PostGreSQLUsername).GetString(),
Password: DefaultMgr().Get(backgroundCtx, common.PostGreSQLPassword).GetPassword(),
Database: DefaultMgr().Get(backgroundCtx, common.PostGreSQLDatabase).GetString(),
SSLMode: DefaultMgr().Get(backgroundCtx, common.PostGreSQLSSLMode).GetString(),
MaxIdleConns: DefaultMgr().Get(backgroundCtx, common.PostGreSQLMaxIdleConns).GetInt(),
MaxOpenConns: DefaultMgr().Get(backgroundCtx, common.PostGreSQLMaxOpenConns).GetInt(),
Host: DefaultMgr().Get(backgroundCtx, common.PostGreSQLHOST).GetString(),
Port: DefaultMgr().Get(backgroundCtx, common.PostGreSQLPort).GetInt(),
Username: DefaultMgr().Get(backgroundCtx, common.PostGreSQLUsername).GetString(),
Password: DefaultMgr().Get(backgroundCtx, common.PostGreSQLPassword).GetPassword(),
Database: DefaultMgr().Get(backgroundCtx, common.PostGreSQLDatabase).GetString(),
SSLMode: DefaultMgr().Get(backgroundCtx, common.PostGreSQLSSLMode).GetString(),
MaxIdleConns: DefaultMgr().Get(backgroundCtx, common.PostGreSQLMaxIdleConns).GetInt(),
MaxOpenConns: DefaultMgr().Get(backgroundCtx, common.PostGreSQLMaxOpenConns).GetInt(),
ConnMaxLifetime: DefaultMgr().Get(backgroundCtx, common.PostGreSQLConnMaxLifetime).GetDuration(),
ConnMaxIdleTime: DefaultMgr().Get(backgroundCtx, common.PostGreSQLConnMaxIdleTime).GetDuration(),
}
database.PostGreSQL = postgresql

18
src/pkg/config/manager.go
View File

@ -159,14 +159,16 @@ func (c *CfgManager) GetDatabaseCfg() *models.Database {
return &models.Database{
Type: c.Get(ctx, common.DatabaseType).GetString(),
PostGreSQL: &models.PostGreSQL{
Host: c.Get(ctx, common.PostGreSQLHOST).GetString(),
Port: c.Get(ctx, common.PostGreSQLPort).GetInt(),
Username: c.Get(ctx, common.PostGreSQLUsername).GetString(),
Password: c.Get(ctx, common.PostGreSQLPassword).GetString(),
Database: c.Get(ctx, common.PostGreSQLDatabase).GetString(),
SSLMode: c.Get(ctx, common.PostGreSQLSSLMode).GetString(),
MaxIdleConns: c.Get(ctx, common.PostGreSQLMaxIdleConns).GetInt(),
MaxOpenConns: c.Get(ctx, common.PostGreSQLMaxOpenConns).GetInt(),
Host: c.Get(ctx, common.PostGreSQLHOST).GetString(),
Port: c.Get(ctx, common.PostGreSQLPort).GetInt(),
Username: c.Get(ctx, common.PostGreSQLUsername).GetString(),
Password: c.Get(ctx, common.PostGreSQLPassword).GetString(),
Database: c.Get(ctx, common.PostGreSQLDatabase).GetString(),
SSLMode: c.Get(ctx, common.PostGreSQLSSLMode).GetString(),
MaxIdleConns: c.Get(ctx, common.PostGreSQLMaxIdleConns).GetInt(),
MaxOpenConns: c.Get(ctx, common.PostGreSQLMaxOpenConns).GetInt(),
ConnMaxLifetime: c.Get(ctx, common.PostGreSQLConnMaxLifetime).GetDuration(),
ConnMaxIdleTime: c.Get(ctx, common.PostGreSQLConnMaxIdleTime).GetDuration(),
},
}
}