add parameters for PostgreSQL (#16641)

Signed-off-by: sayaoailun <guojianwei007@126.com>

make/harbor.yml.tmpl

@@ -42,6 +42,12 @@ database:
   # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.
   # Note: the default number of connections is 1024 for postgres of harbor.
   max_open_conns: 900
+  # The maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If it <= 0, connections are not closed due to a connection's age.
+  # The value is a duration string. A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+  conn_max_lifetime: 5m
+  # The maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If it <= 0, connections are not closed due to a connection's idle time.
+  # The value is a duration string. A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+  conn_max_idle_time: 0
 
 # The default data volume
 data_volume: /data

make/photon/prepare/templates/core/env.jinja

@@ -16,6 +16,8 @@ POSTGRESQL_DATABASE={{harbor_db_name}}
 POSTGRESQL_SSLMODE={{harbor_db_sslmode}}
 POSTGRESQL_MAX_IDLE_CONNS={{harbor_db_max_idle_conns}}
 POSTGRESQL_MAX_OPEN_CONNS={{harbor_db_max_open_conns}}
+POSTGRESQL_CONN_MAX_LIFETIME={{harbor_db_conn_max_lifetime}}
+POSTGRESQL_CONN_MAX_IDLE_TIME={{harbor_db_conn_max_idle_time}}
 REGISTRY_URL={{registry_url}}
 PORTAL_URL={{portal_url}}
 TOKEN_SERVICE_URL={{token_service_url}}

make/photon/prepare/templates/exporter/env.jinja

@@ -26,3 +26,5 @@ HARBOR_DATABASE_DBNAME={{harbor_db_name}}
 HARBOR_DATABASE_SSLMODE={{harbor_db_sslmode}}
 HARBOR_DATABASE_MAX_IDLE_CONNS={{harbor_db_max_idle_conns}}
 HARBOR_DATABASE_MAX_OPEN_CONNS={{harbor_db_max_open_conns}}
+HARBOR_DATABASE_CONN_MAX_LIFETIME={{harbor_db_conn_max_lifetime}}
+HARBOR_DATABASE_CONN_MAX_IDLE_TIME={{harbor_db_conn_max_idle_time}}

make/photon/prepare/utils/configs.py

@@ -158,6 +158,8 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
         config_dict['harbor_db_sslmode'] = 'disable'
         config_dict['harbor_db_max_idle_conns'] = db_configs.get("max_idle_conns") or default_db_max_idle_conns
         config_dict['harbor_db_max_open_conns'] = db_configs.get("max_open_conns") or default_db_max_open_conns
+        config_dict['harbor_db_conn_max_lifetime'] = db_configs.get("conn_max_lifetime") or '5m'
+        config_dict['harbor_db_conn_max_idle_time'] = db_configs.get("conn_max_idle_time") or '0'
 
         if with_notary:
             # notary signer
@@ -288,6 +290,8 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
         config_dict['harbor_db_sslmode'] = external_db_configs['harbor']['ssl_mode']
         config_dict['harbor_db_max_idle_conns'] = external_db_configs['harbor'].get("max_idle_conns") or default_db_max_idle_conns
         config_dict['harbor_db_max_open_conns'] = external_db_configs['harbor'].get("max_open_conns") or default_db_max_open_conns
+        config_dict['harbor_db_conn_max_lifetime'] = external_db_configs['harbor'].get("conn_max_lifetime") or '5m'
+        config_dict['harbor_db_conn_max_idle_time'] = external_db_configs['harbor'].get("conn_max_idle_time") or '0'
 
         if with_notary:
             # notary signer

src/cmd/exporter/main.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
 	_ "github.com/jackc/pgx/v4/stdlib" // registry pgx driver
 	"github.com/prometheus/client_golang/prometheus"
@@ -21,6 +22,16 @@ func main() {
 	viper.SetEnvPrefix("harbor")
 	viper.AutomaticEnv()
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+	connMaxLifetime, err := time.ParseDuration(viper.GetString("database.conn_max_lifetime"))
+	if err != nil {
+		log.Errorf("Failed to parse database.conn_max_lifetime: %v", err)
+		connMaxLifetime = 5 * time.Minute
+	}
+	connMaxIdleTime, err := time.ParseDuration(viper.GetString("database.conn_max_idle_time"))
+	if err != nil {
+		log.Errorf("Failed to parse database.conn_max_idle_time: %v", err)
+		connMaxIdleTime = 0
+	}
 	dbCfg := &models.Database{
 		Type: "postgresql",
 		PostGreSQL: &models.PostGreSQL{
@@ -32,6 +43,8 @@ func main() {
 			SSLMode:         viper.GetString("database.sslmode"),
 			MaxIdleConns:    viper.GetInt("database.max_idle_conns"),
 			MaxOpenConns:    viper.GetInt("database.max_open_conns"),
+			ConnMaxLifetime: connMaxLifetime,
+			ConnMaxIdleTime: connMaxIdleTime,
 		},
 	}
 	if err := dao.InitDatabase(dbCfg); err != nil {

src/common/const.go

@@ -59,6 +59,8 @@ const (
 	PostGreSQLSSLMode                = "postgresql_sslmode"
 	PostGreSQLMaxIdleConns           = "postgresql_max_idle_conns"
 	PostGreSQLMaxOpenConns           = "postgresql_max_open_conns"
+	PostGreSQLConnMaxLifetime        = "postgresql_conn_max_lifetime"
+	PostGreSQLConnMaxIdleTime        = "postgresql_conn_max_idle_time"
 	SelfRegistration                 = "self_registration"
 	CoreURL                          = "core_url"
 	CoreLocalURL                     = "core_local_url"

src/common/dao/base.go

@@ -85,6 +85,8 @@ func getDatabase(database *models.Database) (db Database, err error) {
 			database.PostGreSQL.SSLMode,
 			database.PostGreSQL.MaxIdleConns,
 			database.PostGreSQL.MaxOpenConns,
+			database.PostGreSQL.ConnMaxLifetime,
+			database.PostGreSQL.ConnMaxIdleTime,
 		)
 	default:
 		err = fmt.Errorf("invalid database: %s", database.Type)

src/common/dao/pgsql.go

@@ -44,6 +44,8 @@ type pgsql struct {
 	sslmode         string
 	maxIdleConns    int
 	maxOpenConns    int
+	connMaxLifetime time.Duration
+	connMaxIdleTime time.Duration
 }
 
 // Name returns the name of PostgreSQL
@@ -58,7 +60,7 @@ func (p *pgsql) String() string {
 }
 
 // NewPGSQL returns an instance of postgres
-func NewPGSQL(host string, port string, usr string, pwd string, database string, sslmode string, maxIdleConns int, maxOpenConns int) Database {
+func NewPGSQL(host string, port string, usr string, pwd string, database string, sslmode string, maxIdleConns int, maxOpenConns int, connMaxLifetime time.Duration, connMaxIdleTime time.Duration) Database {
 	if len(sslmode) == 0 {
 		sslmode = "disable"
 	}
@@ -71,6 +73,8 @@ func NewPGSQL(host string, port string, usr string, pwd string, database string,
 		sslmode:         sslmode,
 		maxIdleConns:    maxIdleConns,
 		maxOpenConns:    maxOpenConns,
+		connMaxLifetime: connMaxLifetime,
+		connMaxIdleTime: connMaxIdleTime,
 	}
 }
 
@@ -92,10 +96,16 @@ func (p *pgsql) Register(alias ...string) error {
 		p.host, p.port, p.usr, p.pwd, p.database, p.sslmode)
 
 	if err := orm.RegisterDataBase(an, "pgx", info, orm.MaxIdleConnections(p.maxIdleConns),
-		orm.MaxOpenConnections(p.maxOpenConns), orm.ConnMaxLifetime(5*time.Minute)); err != nil {
+		orm.MaxOpenConnections(p.maxOpenConns), orm.ConnMaxLifetime(p.connMaxLifetime)); err != nil {
 		return err
 	}
 
+	db, err := orm.GetDB(an)
+	if err != nil {
+		return err
+	}
+	db.SetConnMaxIdleTime(p.connMaxIdleTime)
+
 	return nil
 }
 

src/common/models/database.go

@@ -14,6 +14,8 @@
 
 package models
 
+import "time"
+
 // Database ...
 type Database struct {
 	Type       string      `json:"type"`
@@ -44,4 +46,6 @@ type PostGreSQL struct {
 	SSLMode         string        `json:"sslmode"`
 	MaxIdleConns    int           `json:"max_idle_conns"`
 	MaxOpenConns    int           `json:"max_open_conns"`
+	ConnMaxLifetime time.Duration `json:"conn_max_lifetime"`
+	ConnMaxIdleTime time.Duration `json:"conn_max_idle_time"`
 }

src/lib/config/metadata/metadatalist.go

@@ -104,6 +104,8 @@ var (
 		{Name: common.PostGreSQLUsername, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_USERNAME", DefaultValue: "postgres", ItemType: &StringType{}, Editable: false},
 		{Name: common.PostGreSQLMaxIdleConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_IDLE_CONNS", DefaultValue: "2", ItemType: &IntType{}, Editable: false},
 		{Name: common.PostGreSQLMaxOpenConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_OPEN_CONNS", DefaultValue: "0", ItemType: &IntType{}, Editable: false},
+		{Name: common.PostGreSQLConnMaxLifetime, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_CONN_MAX_LIFETIME", DefaultValue: "5m", ItemType: &DurationType{}, Editable: false},
+		{Name: common.PostGreSQLConnMaxIdleTime, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_CONN_MAX_IDLE_TIME", DefaultValue: "0", ItemType: &DurationType{}, Editable: false},
 
 		{Name: common.ProjectCreationRestriction, Scope: UserScope, Group: BasicGroup, EnvKey: "PROJECT_CREATION_RESTRICTION", DefaultValue: common.ProCrtRestrEveryone, ItemType: &ProjectCreationRestrictionType{}, Editable: false, Description: `Indicate who can create projects, it could be ''adminonly'' or ''everyone''.`},
 		{Name: common.ReadOnly, Scope: UserScope, Group: BasicGroup, EnvKey: "READ_ONLY", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `The flag to indicate whether Harbor is in readonly mode.`},

src/lib/config/metadata/type.go

@@ -21,6 +21,7 @@ import (
 	"math"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/goharbor/harbor/src/common"
 )
@@ -234,6 +235,19 @@ func (t *QuotaType) validate(str string) error {
 	return nil
 }
 
+// DurationType ...
+type DurationType struct {
+}
+
+func (t *DurationType) validate(str string) error {
+	_, err := time.ParseDuration(str)
+	return err
+}
+
+func (t *DurationType) get(str string) (interface{}, error) {
+	return time.ParseDuration(str)
+}
+
 // parseInt64 returns int64 from string which support scientific notation
 func parseInt64(str string) (int64, error) {
 	val, err := strconv.ParseInt(str, 10, 64)

src/lib/config/metadata/value.go

@@ -16,6 +16,7 @@ package metadata
 
 import (
 	"errors"
+	"time"
 
 	"github.com/goharbor/harbor/src/lib/log"
 )
@@ -144,6 +145,22 @@ func (c *ConfigureValue) GetStringToStringMap() map[string]string {
 	return result
 }
 
+// GetDuration - return the time.Duration value of current value
+func (c *ConfigureValue) GetDuration() time.Duration {
+	if item, ok := Instance().GetByName(c.Name); ok {
+		val, err := item.ItemType.get(c.Value)
+		if err != nil {
+			log.Errorf("GetDuration failed, error: %+v", err)
+			return 0
+		}
+		if durationValue, suc := val.(time.Duration); suc {
+			return durationValue
+		}
+	}
+	log.Errorf("GetDuration failed, the current value's metadata is not defined, %+v", c)
+	return 0
+}
+
 // GetAnyType get the interface{} of current value
 func (c *ConfigureValue) GetAnyType() (interface{}, error) {
 	if item, ok := Instance().GetByName(c.Name); ok {

src/lib/config/metadata/value_test.go

@@ -17,6 +17,7 @@ package metadata
 import (
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -60,6 +61,12 @@ func TestConfigureValue_GetStringToStringMap(t *testing.T) {
 	assert.Equal(t, val, map[string]interface{}{"sample": "abc"})
 	Instance().init()
 }
+
+func TestConfigureValue_GetDuration(t *testing.T) {
+	assert.Equal(t, createCfgValue("postgresql_conn_max_lifetime", "5m").GetDuration(), 5*time.Minute)
+	assert.Equal(t, createCfgValue("postgresql_conn_max_lifetime", "").GetDuration(), time.Duration(0))
+}
+
 func TestConfigureValue_GetInt(t *testing.T) {
 	assert.Equal(t, createCfgValue("ldap_timeout", "5").GetInt(), 5)
 }

src/lib/config/systemconfig.go

@@ -282,6 +282,8 @@ func Database() (*models.Database, error) {
 		SSLMode:         DefaultMgr().Get(backgroundCtx, common.PostGreSQLSSLMode).GetString(),
 		MaxIdleConns:    DefaultMgr().Get(backgroundCtx, common.PostGreSQLMaxIdleConns).GetInt(),
 		MaxOpenConns:    DefaultMgr().Get(backgroundCtx, common.PostGreSQLMaxOpenConns).GetInt(),
+		ConnMaxLifetime: DefaultMgr().Get(backgroundCtx, common.PostGreSQLConnMaxLifetime).GetDuration(),
+		ConnMaxIdleTime: DefaultMgr().Get(backgroundCtx, common.PostGreSQLConnMaxIdleTime).GetDuration(),
 	}
 	database.PostGreSQL = postgresql
 

src/pkg/config/manager.go

@@ -167,6 +167,8 @@ func (c *CfgManager) GetDatabaseCfg() *models.Database {
 			SSLMode:         c.Get(ctx, common.PostGreSQLSSLMode).GetString(),
 			MaxIdleConns:    c.Get(ctx, common.PostGreSQLMaxIdleConns).GetInt(),
 			MaxOpenConns:    c.Get(ctx, common.PostGreSQLMaxOpenConns).GetInt(),
+			ConnMaxLifetime: c.Get(ctx, common.PostGreSQLConnMaxLifetime).GetDuration(),
+			ConnMaxIdleTime: c.Get(ctx, common.PostGreSQLConnMaxIdleTime).GetDuration(),
 		},
 	}
 }