Move Settings of HTTP auth proxy (#7047)

Previously the settings of HTTP authproxy were set in environment
variable.
This commit move them to the configuration API

Signed-off-by: Daniel Jiang <jiangd@vmware.com>

src/common/config/metadata/metadatalist.go

@@ -30,7 +30,7 @@ type Item struct {
 	Name string `json:"name,omitempty"`
 	// It can be &IntType{}, &StringType{}, &BoolType{}, &PasswordType{}, &MapType{} etc, any type interface implementation
 	ItemType Type
-	// Is this settign can be modified after configure
+	// TODO: Clarify the usage of this attribute
 	Editable bool `json:"editable,omitempty"`
 }
 
@@ -44,6 +44,7 @@ const (
 	LdapGroupGroup = "ldapgroup"
 	EmailGroup     = "email"
 	UAAGroup       = "uaa"
+	HTTPAuthGroup  = "http_auth"
 	DatabaseGroup  = "database"
 	// Put all config items do not belong a existing group into basic
 	BasicGroup = "basic"
@@ -57,6 +58,7 @@ var (
 	// 2. Get/Set config settings by CfgManager
 	// 3. CfgManager.Load()/CfgManager.Save() to load/save from configure storage.
 	ConfigList = []Item{
+		// TODO: All these Name should be reference to const, see #7040
 		{Name: "admin_initial_password", Scope: SystemScope, Group: BasicGroup, EnvKey: "HARBOR_ADMIN_PASSWORD", DefaultValue: "", ItemType: &PasswordType{}, Editable: true},
 		{Name: "admiral_url", Scope: SystemScope, Group: BasicGroup, EnvKey: "ADMIRAL_URL", DefaultValue: "", ItemType: &StringType{}, Editable: false},
 		{Name: "auth_mode", Scope: UserScope, Group: BasicGroup, EnvKey: "AUTH_MODE", DefaultValue: "db_auth", ItemType: &AuthModeType{}, Editable: false},
@@ -127,6 +129,10 @@ var (
 		{Name: "uaa_endpoint", Scope: UserScope, Group: UAAGroup, EnvKey: "UAA_ENDPOINT", DefaultValue: "", ItemType: &StringType{}, Editable: false},
 		{Name: "uaa_verify_cert", Scope: UserScope, Group: UAAGroup, EnvKey: "UAA_VERIFY_CERT", DefaultValue: "false", ItemType: &BoolType{}, Editable: false},
 
+		{Name: common.HTTPAuthProxyEndpoint, Scope: UserScope, Group: HTTPAuthGroup, EnvKey: "HTTP_AUTHPROXY_ENDPOINT", DefaultValue: "", ItemType: &StringType{}, Editable: false},
+		{Name: common.HTTPAuthProxySkipCertVerify, Scope: UserScope, Group: HTTPAuthGroup, EnvKey: "HTTP_AUTHPROXY_SKIP_CERT_VERIFY", DefaultValue: "false", ItemType: &BoolType{}, Editable: false},
+		{Name: common.HTTPAuthProxyAlwaysOnboard, Scope: UserScope, Group: HTTPAuthGroup, EnvKey: "HTTP_AUTHPROXY_ALWAYS_ONBOARD", DefaultValue: "false", ItemType: &BoolType{}, Editable: false},
+
 		{Name: "with_chartmuseum", Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_CHARTMUSEUM", DefaultValue: "false", ItemType: &BoolType{}, Editable: true},
 		{Name: "with_clair", Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_CLAIR", DefaultValue: "false", ItemType: &BoolType{}, Editable: true},
 		{Name: "with_notary", Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_NOTARY", DefaultValue: "false", ItemType: &BoolType{}, Editable: true},

src/common/config/metadata/type.go

@@ -61,11 +61,11 @@ type AuthModeType struct {
 }
 
 func (t *AuthModeType) validate(str string) error {
-	if str == common.LDAPAuth || str == common.DBAuth || str == common.UAAAuth {
+	if str == common.LDAPAuth || str == common.DBAuth || str == common.UAAAuth || str == common.HTTPAuth {
 		return nil
 	}
-	return fmt.Errorf("invalid %s, shoud be one of %s, %s, %s",
-		common.AUTHMode, common.DBAuth, common.LDAPAuth, common.UAAAuth)
+	return fmt.Errorf("invalid %s, shoud be one of %s, %s, %s, %s",
+		common.AUTHMode, common.DBAuth, common.LDAPAuth, common.UAAAuth, common.HTTPAuth)
 }
 
 // ProjectCreationRestrictionType ...

src/common/const.go

@@ -41,60 +41,64 @@ const (
 	ResourceTypeImage      = "i"
 	ResourceTypeChart      = "c"
 
-	ExtEndpoint                       = "ext_endpoint"
-	AUTHMode                          = "auth_mode"
-	DatabaseType                      = "database_type"
-	PostGreSQLHOST                    = "postgresql_host"
-	PostGreSQLPort                    = "postgresql_port"
-	PostGreSQLUsername                = "postgresql_username"
-	PostGreSQLPassword                = "postgresql_password"
-	PostGreSQLDatabase                = "postgresql_database"
-	PostGreSQLSSLMode                 = "postgresql_sslmode"
-	SelfRegistration                  = "self_registration"
-	CoreURL                           = "core_url"
-	JobServiceURL                     = "jobservice_url"
-	LDAPURL                           = "ldap_url"
-	LDAPSearchDN                      = "ldap_search_dn"
-	LDAPSearchPwd                     = "ldap_search_password"
-	LDAPBaseDN                        = "ldap_base_dn"
-	LDAPUID                           = "ldap_uid"
-	LDAPFilter                        = "ldap_filter"
-	LDAPScope                         = "ldap_scope"
-	LDAPTimeout                       = "ldap_timeout"
-	LDAPVerifyCert                    = "ldap_verify_cert"
-	LDAPGroupBaseDN                   = "ldap_group_base_dn"
-	LDAPGroupSearchFilter             = "ldap_group_search_filter"
-	LDAPGroupAttributeName            = "ldap_group_attribute_name"
-	LDAPGroupSearchScope              = "ldap_group_search_scope"
-	TokenServiceURL                   = "token_service_url"
-	RegistryURL                       = "registry_url"
-	EmailHost                         = "email_host"
-	EmailPort                         = "email_port"
-	EmailUsername                     = "email_username"
-	EmailPassword                     = "email_password"
-	EmailFrom                         = "email_from"
-	EmailSSL                          = "email_ssl"
-	EmailIdentity                     = "email_identity"
-	EmailInsecure                     = "email_insecure"
-	ProjectCreationRestriction        = "project_creation_restriction"
-	MaxJobWorkers                     = "max_job_workers"
-	TokenExpiration                   = "token_expiration"
-	CfgExpiration                     = "cfg_expiration"
-	AdminInitialPassword              = "admin_initial_password"
-	AdmiralEndpoint                   = "admiral_url"
-	WithNotary                        = "with_notary"
-	WithClair                         = "with_clair"
-	ScanAllPolicy                     = "scan_all_policy"
-	ClairDBPassword                   = "clair_db_password"
-	ClairDBHost                       = "clair_db_host"
-	ClairDBPort                       = "clair_db_port"
-	ClairDB                           = "clair_db"
-	ClairDBUsername                   = "clair_db_username"
-	ClairDBSSLMode                    = "clair_db_sslmode"
-	UAAEndpoint                       = "uaa_endpoint"
-	UAAClientID                       = "uaa_client_id"
-	UAAClientSecret                   = "uaa_client_secret"
-	UAAVerifyCert                     = "uaa_verify_cert"
+	ExtEndpoint                 = "ext_endpoint"
+	AUTHMode                    = "auth_mode"
+	DatabaseType                = "database_type"
+	PostGreSQLHOST              = "postgresql_host"
+	PostGreSQLPort              = "postgresql_port"
+	PostGreSQLUsername          = "postgresql_username"
+	PostGreSQLPassword          = "postgresql_password"
+	PostGreSQLDatabase          = "postgresql_database"
+	PostGreSQLSSLMode           = "postgresql_sslmode"
+	SelfRegistration            = "self_registration"
+	CoreURL                     = "core_url"
+	JobServiceURL               = "jobservice_url"
+	LDAPURL                     = "ldap_url"
+	LDAPSearchDN                = "ldap_search_dn"
+	LDAPSearchPwd               = "ldap_search_password"
+	LDAPBaseDN                  = "ldap_base_dn"
+	LDAPUID                     = "ldap_uid"
+	LDAPFilter                  = "ldap_filter"
+	LDAPScope                   = "ldap_scope"
+	LDAPTimeout                 = "ldap_timeout"
+	LDAPVerifyCert              = "ldap_verify_cert"
+	LDAPGroupBaseDN             = "ldap_group_base_dn"
+	LDAPGroupSearchFilter       = "ldap_group_search_filter"
+	LDAPGroupAttributeName      = "ldap_group_attribute_name"
+	LDAPGroupSearchScope        = "ldap_group_search_scope"
+	TokenServiceURL             = "token_service_url"
+	RegistryURL                 = "registry_url"
+	EmailHost                   = "email_host"
+	EmailPort                   = "email_port"
+	EmailUsername               = "email_username"
+	EmailPassword               = "email_password"
+	EmailFrom                   = "email_from"
+	EmailSSL                    = "email_ssl"
+	EmailIdentity               = "email_identity"
+	EmailInsecure               = "email_insecure"
+	ProjectCreationRestriction  = "project_creation_restriction"
+	MaxJobWorkers               = "max_job_workers"
+	TokenExpiration             = "token_expiration"
+	CfgExpiration               = "cfg_expiration"
+	AdminInitialPassword        = "admin_initial_password"
+	AdmiralEndpoint             = "admiral_url"
+	WithNotary                  = "with_notary"
+	WithClair                   = "with_clair"
+	ScanAllPolicy               = "scan_all_policy"
+	ClairDBPassword             = "clair_db_password"
+	ClairDBHost                 = "clair_db_host"
+	ClairDBPort                 = "clair_db_port"
+	ClairDB                     = "clair_db"
+	ClairDBUsername             = "clair_db_username"
+	ClairDBSSLMode              = "clair_db_sslmode"
+	UAAEndpoint                 = "uaa_endpoint"
+	UAAClientID                 = "uaa_client_id"
+	UAAClientSecret             = "uaa_client_secret"
+	UAAVerifyCert               = "uaa_verify_cert"
+	HTTPAuthProxyEndpoint       = "http_authproxy_endpoint"
+	HTTPAuthProxySkipCertVerify = "http_authproxy_skip_cert_verify"
+	HTTPAuthProxyAlwaysOnboard  = "http_authproxy_always_onboard"
+
 	DefaultClairEndpoint              = "http://clair:6060"
 	CfgDriverDB                       = "db"
 	NewHarborAdminName                = "admin@harbor.local"
@@ -107,7 +111,6 @@ const (
 	DefaultCoreEndpoint               = "http://core:8080"
 	DefaultNotaryEndpoint             = "http://notary-server:4443"
 	LdapGroupType                     = 1
-	ReloadKey                         = "reload_key"
 	LdapGroupAdminDn                  = "ldap_group_admin_dn"
 	DefaultRegistryControllerEndpoint = "http://registryctl:8080"
 	WithChartMuseum                   = "with_chartmuseum"

src/common/dao/testutils.go

@@ -133,3 +133,12 @@ func ArrayEqual(arrayA, arrayB []int) bool {
 	}
 	return true
 }
+
+// ClearHTTPAuthProxyUsers remove the records from harbor_users to delete all user imported via
+// HTTP Auth Proxy
+func ClearHTTPAuthProxyUsers() error {
+	o := GetOrmer()
+	sql := "DELETE FROM harbor_user WHERE comment='By Authproxy'"
+	_, err := o.Raw(sql).Exec()
+	return err
+}

src/common/models/config.go

@@ -65,6 +65,13 @@ type Email struct {
 	Insecure bool   `json:"insecure"`
 }
 
+// HTTPAuthProxy wraps the settings for HTTP auth proxy
+type HTTPAuthProxy struct {
+	Endpoint       string `json:"endpoint"`
+	SkipCertVerify bool   `json:"skip_cert_verify"`
+	AlwaysOnBoard  bool   `json:"always_onboard"`
+}
+
 /*
 // Registry ...
 type Registry struct {

src/core/auth/authproxy/auth.go

@@ -22,27 +22,38 @@ import (
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/core/auth"
+	"github.com/goharbor/harbor/src/core/config"
 	"io/ioutil"
 	"net/http"
-	"os"
 	"strings"
 	"sync"
+	"time"
 )
 
+const refreshDuration = 5 * time.Second
+const userEntryComment = "By Authproxy"
+
 // Auth implements HTTP authenticator the required attributes.
 // The attribute Endpoint is the HTTP endpoint to which the POST request should be issued for authentication
 type Auth struct {
 	auth.DefaultAuthenticateHelper
 	sync.Mutex
-	Endpoint       string
-	SkipCertVerify bool
-	AlwaysOnboard  bool
-	client         *http.Client
+	Endpoint         string
+	SkipCertVerify   bool
+	AlwaysOnboard    bool
+	settingTimeStamp time.Time
+	client           *http.Client
 }
 
 // Authenticate issues http POST request to Endpoint if it returns 200 the authentication is considered success.
 func (a *Auth) Authenticate(m models.AuthModel) (*models.User, error) {
-	a.ensure()
+	err := a.ensure()
+	if err != nil {
+		if a.Endpoint == "" {
+			return nil, fmt.Errorf("failed to initialize HTTP Auth Proxy Authenticator, error: %v", err)
+		}
+		log.Warningf("Failed to refresh configuration for HTTP Auth Proxy Authenticator, error: %v, old settings will be used", err)
+	}
 	req, err := http.NewRequest(http.MethodPost, a.Endpoint, nil)
 	if err != nil {
 		return nil, fmt.Errorf("failed to send request, error: %v", err)
@@ -109,7 +120,7 @@ func (a *Auth) fillInModel(u *models.User) error {
 	}
 	u.Realname = u.Username
 	u.Password = "1234567ab"
-	u.Comment = "By Authproxy"
+	u.Comment = userEntryComment
 	if strings.Contains(u.Username, "@") {
 		u.Email = u.Username
 	} else {
@@ -118,13 +129,17 @@ func (a *Auth) fillInModel(u *models.User) error {
 	return nil
 }
 
-func (a *Auth) ensure() {
+func (a *Auth) ensure() error {
 	a.Lock()
 	defer a.Unlock()
-	if a.Endpoint == "" {
-		a.Endpoint = os.Getenv("AUTHPROXY_ENDPOINT")
-		a.SkipCertVerify = strings.EqualFold(os.Getenv("AUTHPROXY_SKIP_CERT_VERIFY"), "true")
-		a.AlwaysOnboard = strings.EqualFold(os.Getenv("AUTHPROXY_ALWAYS_ONBOARD"), "true")
+	if time.Now().Sub(a.settingTimeStamp) >= refreshDuration {
+		setting, err := config.HTTPAuthProxySetting()
+		if err != nil {
+			return err
+		}
+		a.Endpoint = setting.Endpoint
+		a.SkipCertVerify = setting.SkipCertVerify
+		a.AlwaysOnboard = setting.AlwaysOnBoard
 	}
 	if a.client == nil {
 		tr := &http.Transport{
@@ -136,6 +151,7 @@ func (a *Auth) ensure() {
 			Transport: tr,
 		}
 	}
+	return nil
 }
 
 func init() {

src/core/auth/authproxy/auth_test.go

@@ -15,13 +15,16 @@
 package authproxy
 
 import (
+	"github.com/goharbor/harbor/src/common/dao"
 	"github.com/goharbor/harbor/src/common/models"
+	cut "github.com/goharbor/harbor/src/common/utils/test"
 	"github.com/goharbor/harbor/src/core/auth"
 	"github.com/goharbor/harbor/src/core/auth/authproxy/test"
 	"github.com/stretchr/testify/assert"
 	"net/http/httptest"
 	"os"
 	"testing"
+	"time"
 )
 
 var mockSvr *httptest.Server
@@ -30,13 +33,22 @@ var pwd = "1234567ab"
 var cmt = "By Authproxy"
 
 func TestMain(m *testing.M) {
+	cut.InitDatabaseFromEnv()
+	if err := dao.ClearHTTPAuthProxyUsers(); err != nil {
+		panic(err)
+	}
 	mockSvr = test.NewMockServer(map[string]string{"jt": "pp", "Admin@vsphere.local": "Admin!23"})
 	defer mockSvr.Close()
 	a = &Auth{
 		Endpoint:       mockSvr.URL + "/test/login",
 		SkipCertVerify: true,
+		// So it won't require mocking the cfgManager
+		settingTimeStamp: time.Now(),
 	}
 	rc := m.Run()
+	if err := dao.ClearHTTPAuthProxyUsers(); err != nil {
+		panic(err)
+	}
 	if rc != 0 {
 		os.Exit(rc)
 	}
@@ -104,7 +116,6 @@ func TestAuth_Authenticate(t *testing.T) {
 	}
 }
 
-/* TODO: Enable this case after adminserver refactor is merged.
 func TestAuth_PostAuthenticate(t *testing.T) {
 	type tc struct {
 		input  *models.User
@@ -120,7 +131,7 @@ func TestAuth_PostAuthenticate(t *testing.T) {
 				Email:    "jt@placeholder.com",
 				Realname: "jt",
 				Password: pwd,
-				Comment:  fmt.Sprintf(cmtTmpl, mockSvr.URL+"/test/login"),
+				Comment:  userEntryComment,
 			},
 		},
 		{
@@ -129,16 +140,19 @@ func TestAuth_PostAuthenticate(t *testing.T) {
 			},
 			expect: models.User{
 				Username: "Admin@vsphere.local",
-				Email:    "jt@placeholder.com",
+				Email:    "Admin@vsphere.local",
 				Realname: "Admin@vsphere.local",
 				Password: pwd,
-				Comment:  fmt.Sprintf(cmtTmpl, mockSvr.URL+"/test/login"),
+				Comment:  userEntryComment,
 			},
 		},
 	}
 	for _, c := range suite {
 		a.PostAuthenticate(c.input)
-		assert.Equal(t, c.expect, *c.input)
+		assert.Equal(t, c.expect.Username, c.input.Username)
+		assert.Equal(t, c.expect.Email, c.input.Email)
+		assert.Equal(t, c.expect.Realname, c.input.Realname)
+		assert.Equal(t, c.expect.Comment, c.input.Comment)
 	}
+
 }
-*/

src/core/config/config.go

@@ -462,3 +462,17 @@ func GetClairHealthCheckServerURL() string {
 	}
 	return url
 }
+
+// HTTPAuthProxySetting returns the setting of HTTP Auth proxy.  the settings are only meaningful when the auth_mode is
+// set to http_auth
+func HTTPAuthProxySetting() (*models.HTTPAuthProxy, error) {
+	if err := cfgMgr.Load(); err != nil {
+		return nil, err
+	}
+	return &models.HTTPAuthProxy{
+		Endpoint:       cfgMgr.Get(common.HTTPAuthProxyEndpoint).GetString(),
+		SkipCertVerify: cfgMgr.Get(common.HTTPAuthProxySkipCertVerify).GetBool(),
+		AlwaysOnBoard:  cfgMgr.Get(common.HTTPAuthProxyAlwaysOnboard).GetBool(),
+	}, nil
+
+}

src/core/config/config_test.go

@@ -225,3 +225,19 @@ func TestConfigureValue_GetMap(t *testing.T) {
 	}
 	fmt.Printf("%+v\n", policy)
 }
+
+func TestHTTPAuthProxySetting(t *testing.T) {
+	m := map[string]interface{}{
+		common.HTTPAuthProxyAlwaysOnboard:  "true",
+		common.HTTPAuthProxySkipCertVerify: "true",
+		common.HTTPAuthProxyEndpoint:       "https://auth.proxy/suffix",
+	}
+	InitWithSettings(m)
+	v, e := HTTPAuthProxySetting()
+	assert.Nil(t, e)
+	assert.Equal(t, *v, models.HTTPAuthProxy{
+		Endpoint:       "https://auth.proxy/suffix",
+		AlwaysOnBoard:  true,
+		SkipCertVerify: true,
+	})
+}