From c1e676ad9921e90aed759f80aa4784427a49de0f Mon Sep 17 00:00:00 2001 From: Qian Deng Date: Tue, 9 Apr 2019 15:51:51 +0800 Subject: [PATCH 01/15] Add migration script from 1.7.0 to 1.8.0 Add jinja2 to migrator Add template to migrator Add config upgrading script Signed-off-by: Qian Deng --- make/harbor.yml | 2 +- tools/migration/Dockerfile | 2 +- .../migration/cfg/migrator_1_8_0/__init__.py | 47 +++++--- .../cfg/migrator_1_8_0/harbor.yml.jinja | 113 +++++++++++++++++ .../cfg/migrator_1_8_0/harbor.yml.tpl | 114 ------------------ tools/migration/cfg/run.py | 3 +- tools/migration/cfg/utils.py | 22 ++++ 7 files changed, 170 insertions(+), 133 deletions(-) create mode 100644 tools/migration/cfg/migrator_1_8_0/harbor.yml.jinja delete mode 100644 tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl diff --git a/make/harbor.yml b/make/harbor.yml index 7abb44217..1a3f78e37 100644 --- a/make/harbor.yml +++ b/make/harbor.yml @@ -71,7 +71,7 @@ log: location: /var/log/harbor #This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! -_version: 1.7.0 +_version: 1.8.0 # Uncomment external_database if using external database. And the password will replace the the password setting in database. # And currently ontly support postgres. diff --git a/tools/migration/Dockerfile b/tools/migration/Dockerfile index 4bebce16c..222ceb057 100644 --- a/tools/migration/Dockerfile +++ b/tools/migration/Dockerfile @@ -7,7 +7,7 @@ RUN tdnf distro-sync -y \ && tdnf remove -y toybox \ && tdnf install -y sed shadow procps-ng gawk gzip sudo net-tools glibc-i18n >> /dev/null\ && groupadd -r -g 10000 mysql && useradd --no-log-init -r -g 10000 -u 10000 mysql \ - && tdnf install -y mariadb-server mariadb mariadb-devel python2 python2-devel python-pip gcc \ + && tdnf install -y mariadb-server mariadb mariadb-devel python2 python2-devel python-pip gcc PyYAML python-jinja2\ linux-api-headers glibc-devel binutils zlib-devel openssl-devel postgresql python-psycopg2 >> /dev/null \ && pip install mysqlclient alembic \ && mkdir /docker-entrypoint-initdb.d /docker-entrypoint-updatedb.d \ diff --git a/tools/migration/cfg/migrator_1_8_0/__init__.py b/tools/migration/cfg/migrator_1_8_0/__init__.py index db26f8de1..0266db09c 100644 --- a/tools/migration/cfg/migrator_1_8_0/__init__.py +++ b/tools/migration/cfg/migrator_1_8_0/__init__.py @@ -1,14 +1,13 @@ from __future__ import print_function import utils import os +from jinja2 import Environment, FileSystemLoader acceptable_versions = ['1.7.0'] keys = [ 'hostname', 'ui_url_protocol', - 'customize_crt', 'ssl_cert', 'ssl_cert_key', - 'secretkey_path', 'admiral_url', 'log_rotate_count', 'log_rotate_size', @@ -19,18 +18,15 @@ keys = [ 'db_password', 'db_port', 'db_user', - 'clair_db_host', - 'clair_db_password', - 'clair_db_port', - 'clair_db_username', - 'clair_db', - 'uaa_endpoint', - 'uaa_clientid', - 'uaa_clientsecret', - 'uaa_verify_cert', - 'uaa_ca_cert', + 'redis_host', + 'redis_port', + 'redis_password', + 'redis_db_index', + 'clair_updaters_interval', + 'max_job_workers', 'registry_storage_provider_name', - 'registry_storage_provider_config' + 'registry_storage_provider_config', + 'registry_custom_ca_bundle' ] def migrate(input_cfg, output_cfg): @@ -38,5 +34,26 @@ def migrate(input_cfg, output_cfg): val = {} for k in keys: val[k] = d.get(k,'') - tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.yml.tpl') - utils.render(tpl_path, output_cfg, **val) + if val['db_host'] == 'postgresql' and val['db_port'] == 5432 and val['db_user'] == 'postgres': + val['external_db'] = False + else: + val['external_db'] = True + # If using default filesystem, didn't need registry_storage_provider_config config + if val['registry_storage_provider_name'] == 'filesystem' and not val.get('registry_storage_provider_config'): + val['storage_provider_info'] = '' + else: + val['storage_provider_info'] = utils.get_storage_provider_info( + val['registry_storage_provider_name'], + val['registry_storage_provider_config'] + ) + if val['redis_host'] == 'redis' and val['redis_port'] == 6379 and not val['redis_password'] and val['redis_db_index'] == '1,2,3': + val['external_redis'] = False + else: + val['registry_db_index'], val['jobservice_db_index'], val['chartmuseum_db_index'] = map(int, val['redis_db_index'].split(',')) + val['external_redis'] = True + + this_dir = os.path.dirname(__file__) + tpl = Environment(loader=FileSystemLoader(this_dir)).get_template('harbor.yml.jinja') + + with open(output_cfg, 'w') as f: + f.write(tpl.render(**val)) \ No newline at end of file diff --git a/tools/migration/cfg/migrator_1_8_0/harbor.yml.jinja b/tools/migration/cfg/migrator_1_8_0/harbor.yml.jinja new file mode 100644 index 000000000..e70c3ba06 --- /dev/null +++ b/tools/migration/cfg/migrator_1_8_0/harbor.yml.jinja @@ -0,0 +1,113 @@ +## Configuration file of Harbor + +#The IP address or hostname to access admin UI and registry service. +#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. +hostname: {{hostname}} + +# http related comfig +http: + # port for http, default is 80. If https enabled, this port will redirect to https port + port: 80 + +{% if ui_url_protocol == 'https'%} +https: + port: 443 + #The path of cert and key files for nginx + certificate: {{ ssl_cert }} + private_key: {{ ssl_cert_key }} +{% else %} +# https: +# port: 443 +# #The path of cert and key files for nginx +# certificate: /your/certificate/path +# private_key: /your/private/key/path +{% endif %} + +# Uncomment extearnal_url if you want to enable external proxy +# And when it enabled the hostname will no longger used +# external_url: https://reg.mydomain.com:8433 + +# The initial password of Harbor admin +# It only works in first time to install harbor +# Remember Change the admin password from UI after launching Harbor. +harbor_admin_password: Harbor12345 + +## Harbor DB configuration +database: + #The password for the root user of Harbor DB. Change this before any production use. + password: {{ db_password }} + +# The default data volume +data_volume: /data + +# Harbor Storage settings by default is using data_volume dir on local filesystem +# Uncomment storage_service setting If you want to using external storage +storage_service: + # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore + # of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signed certificate. + ca_bundle: {{ registry_custom_ca_bundle }} + + {{storage_provider_info}} + +# Clair configuration +clair: + # The interval of clair updaters, the unit is hour, set to 0 to disable the updaters. + updaters_interval: {{ clair_updaters_interval }} + + # Config http proxy for Clair, e.g. http://my.proxy.com:3128 + # Clair doesn't need to connect to harbor internal components via http proxy. + http_proxy: {{ http_proxy }} + https_proxy: {{ https_proxy }} + no_proxy: {{ no_proxy }} + +jobservice: + # Maximum number of job workers in job service + max_job_workers: {{ max_job_workers }} + +# Log configurations +log: + # options are debug, info, warn, error + level: info + # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. + rotate_count: {{ log_rotate_count }} + # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. + # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G + # are all valid. + rotate_size: {{ log_rotate_size }} + # The directory on your host that store log + location: /var/log/harbor + +#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! +_version: 1.8.0 + +{% if external_db %} +# Uncomment external_database if using external database. And the password will replace the the password setting in database. +# And currently ontly support postgres. +external_database: + host: {{ db_host }} + port: {{ db_port }} + username: {{ db_user }} + password: {{ db_password }} + ssl_mode: disable +{% endif %} + +{% if external_redis %} +external_redis: + host: {{ redis_host }} + port: {{ redis_port }} + password: {{ redis_password }} + # db_index 0 is for core, it's unchangeable + registry_db_index: {{ registry_db_index }} + jobservice_db_index: {{ jobservice_db_index }} + chartmuseum_db_index: {{ chartmuseum_db_index }} +{% else %} +# Umcomments external_redis if using external Redis server +# external_redis: +# host: redis +# port: 6379 +# password: +# # db_index 0 is for core, it's unchangeable +# registry_db_index: 1 +# jobservice_db_index: 2 +# chartmuseum_db_index: 3 +{% endif %} \ No newline at end of file diff --git a/tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl b/tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl deleted file mode 100644 index 7716c5fe1..000000000 --- a/tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl +++ /dev/null @@ -1,114 +0,0 @@ -## Configuration file of Harbor - -#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! -_version: 1.7.0 -#The IP address or hostname to access admin UI and registry service. -#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. -#DO NOT comment out this line, modify the value of "hostname" directly, or the installation will fail. -hostname: $hostname - -#The protocol for accessing the UI and token/notification service, by default it is http. -#It can be set to https if ssl is enabled on nginx. -ui_url_protocol: $ui_url_protocol - -#Maximum number of job workers in job service -max_job_workers: 10 - -#Determine whether or not to generate certificate for the registry's token. -#If the value is on, the prepare script creates new root cert and private key -#for generating token to access the registry. If the value is off the default key/cert will be used. -#This flag also controls the creation of the notary signer's cert. -customize_crt: $customize_crt - -# The default data volume -data_volume: /data - -#The path of cert and key files for nginx, they are applied only the protocol is set to https -ssl_cert: $ssl_cert -ssl_cert_key: $ssl_cert_key - -#The path of secretkey storage -secretkey_path: $secretkey_path - -#Admiral's url, comment this attribute, or set its value to NA when Harbor is standalone -admiral_url: $admiral_url - -log: - #Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. - rotate_count: $log_rotate_count - #Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. - #If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G - #are all valid. - rotate_size: $log_rotate_size - - # The directory that store log files - location: /var/log/harbor - -#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES -#only take effect in the first boot, the subsequent changes of these properties -#should be performed on web ui - -##The initial password of Harbor admin, only works for the first time when Harbor starts. -#It has no effect after the first launch of Harbor. -#Change the admin password from UI after launching Harbor. -harbor_admin_password: Harbor12345 - -database: - #The address of the Harbor database. Only need to change when using external db. - host: $db_host - #The port of Harbor database host - port: $db_port - #The user name of Harbor database - username: $db_user - #The password for the root user of Harbor DB. Change this before any production use. - password: $db_password - -redis: - # Redis connection address - redis_host: redis - # Redis connection port - redis_port: 6379 - # Redis connection password - redis_password: - # Redis connection db index - # db_index 1,2,3 is for registry, jobservice and chartmuseum. - # db_index 0 is for UI, it's unchangeable - redis_db_index: 1,2,3 - -clair: - #Clair DB host address. Only change it when using an exteral DB. - db_host: $clair_db_host - #The password of the Clair's postgres database. Only effective when Harbor is deployed with Clair. - #Please update it before deployment. Subsequent update will cause Clair's API server and Harbor unable to access Clair's database. - db_password: $clair_db_password - #Clair DB connect port - db_port: $clair_db_port - #Clair DB username - db_username: $clair_db_username - #Clair default database - db: $clair_db - #The interval of clair updaters, the unit is hour, set to 0 to disable the updaters. - updaters_interval: 12 - #Config http proxy for Clair, e.g. http://my.proxy.com:3128 - #Clair doesn't need to connect to harbor internal components via http proxy. - http_proxy: $http_proxy - https_proxy: $https_proxy - no_proxy: $no_proxy - -storage: - #Please be aware that the following storage settings will be applied to both docker registry and helm chart repository. - #registry_storage_provider can be: filesystem, s3, gcs, azure, etc. - registry_storage_provider_name: $registry_storage_provider_name - #registry_storage_provider_config is a comma separated "key: value" pairs, e.g. "key1: value, key2: value2". - #To avoid duplicated configurations, both docker registry and chart repository follow the same storage configuration specifications of docker registry. - #Refer to https://docs.docker.com/registry/configuration/#storage for all available configuration. - registry_storage_provider_config: $registry_storage_provider_config - #registry_custom_ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore - #of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signed certificate. - registry_custom_ca_bundle: - -#If reload_config=true, all settings which present in harbor.yml take effect after prepare and restart harbor, it overwrites exsiting settings. -#reload_config=true -#Regular expression to match skipped environment variables -#skip_reload_env_pattern: (^EMAIL.*)|(^LDAP.*) - diff --git a/tools/migration/cfg/run.py b/tools/migration/cfg/run.py index b49c4e9d9..7a03030fd 100644 --- a/tools/migration/cfg/run.py +++ b/tools/migration/cfg/run.py @@ -13,7 +13,7 @@ import shutil import sys def main(): - target_version = '1.7.0' + target_version = '1.8.0' parser = argparse.ArgumentParser(description='migrator of harbor.cfg') parser.add_argument('--input', '-i', action="store", dest='input_path', required=True, help='The path to the old harbor.cfg that provides input value, this required value') parser.add_argument('--output','-o', action="store", dest='output_path', required=False, help='The path of the migrated harbor.cfg, if not set the input file will be overwritten') @@ -62,7 +62,6 @@ def search(basedir, input_ver, target_ver, l): l.append(target_ver) return True return False - if __name__ == "__main__": main() diff --git a/tools/migration/cfg/utils.py b/tools/migration/cfg/utils.py index 7791b7422..078cb1b40 100644 --- a/tools/migration/cfg/utils.py +++ b/tools/migration/cfg/utils.py @@ -43,3 +43,25 @@ def render(src, dest, **kw): t = Template(open(src, 'r').read()) with open(dest, 'w') as f: f.write(t.substitute(**kw)) + +def get_storage_provider_info(provider_name, provider_config): + provider_config = provider_config.strip('" ') + if not provider_config.strip(" "): + return '' + + storage_provider_cfg_map = {} + for k_v in provider_config.split(","): + if k_v > 0: + kvs = k_v.split(": ") # add space suffix to avoid existing ":" in the value + if len(kvs) == 2: + #key must not be empty + if kvs[0].strip() != "": + storage_provider_cfg_map[kvs[0].strip()] = kvs[1].strip() + + # generate storage configuration section in yaml format + + storage_provider_conf_list = [provider_name + ':'] + for config in storage_provider_cfg_map.items(): + storage_provider_conf_list.append('{}: {}'.format(*config)) + storage_provider_info = ('\n' + ' ' * 4).join(storage_provider_conf_list) + return storage_provider_info From 0bb2829d27d645cdef86f6b5c88174b56266eaf1 Mon Sep 17 00:00:00 2001 From: Daniel Jiang Date: Mon, 6 May 2019 16:32:00 +0800 Subject: [PATCH 02/15] Alow user to set CA cert for UAA in harbor.yml Signed-off-by: Daniel Jiang --- make/harbor.yml | 4 ++++ .../templates/docker_compose/docker-compose.yml.jinja | 8 ++++++-- make/photon/prepare/utils/configs.py | 3 +++ make/photon/prepare/utils/docker_compose.py | 4 ++++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/make/harbor.yml b/make/harbor.yml index b3851b282..842fb0673 100644 --- a/make/harbor.yml +++ b/make/harbor.yml @@ -95,3 +95,7 @@ _version: 1.7.0 # registry_db_index: 1 # jobservice_db_index: 2 # chartmuseum_db_index: 3 + +# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert. +# uaa: +# ca_file: /path/to/ca \ No newline at end of file diff --git a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja index db441e229..f0ada5f40 100644 --- a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja +++ b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja @@ -1,4 +1,4 @@ -version: '2' +version: '2.3' services: log: image: goharbor/harbor-log:{{version}} @@ -130,12 +130,16 @@ services: - SETUID volumes: - ./common/config/core/app.conf:/etc/core/app.conf:z - - ./common/config/core/certificates/:/etc/core/certificates/:z - {{data_volume}}/secret/core/private_key.pem:/etc/core/private_key.pem:z - {{data_volume}}/secret/keys/secretkey:/etc/core/key:z - {{data_volume}}/ca_download/:/etc/core/ca/:z - {{data_volume}}/psc/:/etc/core/token/:z - {{data_volume}}/:/data/:z +{% if uaa_ca_file %} + - type: bind + source: {{uaa_ca_file}} + target: /etc/core/certificates/uaa_ca.pem +{% endif %} networks: harbor: {% if with_notary %} diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index 5af7026a3..581269ac7 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -190,4 +190,7 @@ def parse_yaml_config(config_file_path): # Admiral configs config_dict['admiral_url'] = configs.get("admiral_url") or "" + # UAA configs + config_dict['uaa'] = configs.get('uaa') or {} + return config_dict \ No newline at end of file diff --git a/make/photon/prepare/utils/docker_compose.py b/make/photon/prepare/utils/docker_compose.py index ba7829f96..d0c39acd0 100644 --- a/make/photon/prepare/utils/docker_compose.py +++ b/make/photon/prepare/utils/docker_compose.py @@ -43,4 +43,8 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum): rendering_variables['cert_key_path'] = configs['cert_key_path'] rendering_variables['cert_path'] = configs['cert_path'] + uaa_config = configs.get('uaa') or {} + if uaa_config.get('ca_file'): + rendering_variables['uaa_ca_file'] = uaa_config['ca_file'] + render_jinja(docker_compose_template_path, docker_compose_yml_path, **rendering_variables) \ No newline at end of file From 3d43f3fefe259b5670c843ce2b4801d78a642b4d Mon Sep 17 00:00:00 2001 From: FangyuanCheng Date: Tue, 7 May 2019 11:59:05 +0800 Subject: [PATCH 03/15] Fix wrong data displayed when editing replication rule in a certain sequence. Signed-off-by: FangyuanCheng --- .../create-edit-rule/create-edit-rule.component.html | 4 ++-- .../src/create-edit-rule/create-edit-rule.component.ts | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/portal/lib/src/create-edit-rule/create-edit-rule.component.html b/src/portal/lib/src/create-edit-rule/create-edit-rule.component.html index 34a80fa79..34e346769 100644 --- a/src/portal/lib/src/create-edit-rule/create-edit-rule.component.html +++ b/src/portal/lib/src/create-edit-rule/create-edit-rule.component.html @@ -22,7 +22,7 @@
- + @@ -32,7 +32,7 @@
- + diff --git a/src/portal/lib/src/create-edit-rule/create-edit-rule.component.ts b/src/portal/lib/src/create-edit-rule/create-edit-rule.component.ts index 17db76bd5..fdb6ccf82 100644 --- a/src/portal/lib/src/create-edit-rule/create-edit-rule.component.ts +++ b/src/portal/lib/src/create-edit-rule/create-edit-rule.component.ts @@ -133,11 +133,19 @@ export class CreateEditRuleComponent implements OnInit, OnDestroy { equals(c1: any, c2: any): boolean { return c1 && c2 ? c1.id === c2.id : c1 === c2; } - modeChange(): void { + pushModeChange(): void { this.setFilter([]); this.initRegistryInfo(0); } + pullModeChange(): void { + let selectId = this.ruleForm.get('src_registry').value; + if (selectId) { + this.setFilter([]); + this.initRegistryInfo(selectId.id); + } + } + sourceChange($event): void { this.noSelectedEndpoint = false; let selectId = this.ruleForm.get('src_registry').value; From 30b5852b4991b8c82e297f8248e7b1673aaa154b Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Tue, 7 May 2019 12:52:34 +0800 Subject: [PATCH 04/15] Rename the adapter name of docker registry Rename the adapter name of docker registry to "docker-registry" Signed-off-by: Wenkai Yin --- src/replication/adapter/native/adapter.go | 10 ++++------ src/replication/adapter/native/adapter_test.go | 2 +- src/replication/adapter/native/image_registry_test.go | 2 +- src/replication/model/registry.go | 8 ++++---- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/replication/adapter/native/adapter.go b/src/replication/adapter/native/adapter.go index aecac52f3..79a3212bd 100644 --- a/src/replication/adapter/native/adapter.go +++ b/src/replication/adapter/native/adapter.go @@ -20,16 +20,14 @@ import ( "github.com/goharbor/harbor/src/replication/model" ) -const registryTypeNative model.RegistryType = "native" - func init() { - if err := adp.RegisterFactory(registryTypeNative, func(registry *model.Registry) (adp.Adapter, error) { + if err := adp.RegisterFactory(model.RegistryTypeDockerRegistry, func(registry *model.Registry) (adp.Adapter, error) { return newAdapter(registry) }); err != nil { - log.Errorf("failed to register factory for %s: %v", registryTypeNative, err) + log.Errorf("failed to register factory for %s: %v", model.RegistryTypeDockerRegistry, err) return } - log.Infof("the factory for adapter %s registered", registryTypeNative) + log.Infof("the factory for adapter %s registered", model.RegistryTypeDockerRegistry) } func newAdapter(registry *model.Registry) (*native, error) { @@ -52,7 +50,7 @@ var _ adp.Adapter = native{} func (native) Info() (info *model.RegistryInfo, err error) { return &model.RegistryInfo{ - Type: registryTypeNative, + Type: model.RegistryTypeDockerRegistry, SupportedResourceTypes: []model.ResourceType{ model.ResourceTypeImage, }, diff --git a/src/replication/adapter/native/adapter_test.go b/src/replication/adapter/native/adapter_test.go index a6b374267..0c6ff74ff 100644 --- a/src/replication/adapter/native/adapter_test.go +++ b/src/replication/adapter/native/adapter_test.go @@ -57,7 +57,7 @@ func Test_native_Info(t *testing.T) { var info, err = adapter.Info() assert.Nil(t, err) assert.NotNil(t, info) - assert.Equal(t, registryTypeNative, info.Type) + assert.Equal(t, model.RegistryTypeDockerRegistry, info.Type) assert.Equal(t, 1, len(info.SupportedResourceTypes)) assert.Equal(t, 2, len(info.SupportedResourceFilters)) assert.Equal(t, 2, len(info.SupportedTriggers)) diff --git a/src/replication/adapter/native/image_registry_test.go b/src/replication/adapter/native/image_registry_test.go index 767c3b02b..841004a20 100644 --- a/src/replication/adapter/native/image_registry_test.go +++ b/src/replication/adapter/native/image_registry_test.go @@ -65,7 +65,7 @@ func Test_native_FetchImages(t *testing.T) { fmt.Println("mockNativeRegistry URL: ", mock.URL) var registry = &model.Registry{ - Type: registryTypeNative, + Type: model.RegistryTypeDockerRegistry, URL: mock.URL, Insecure: true, } diff --git a/src/replication/model/registry.go b/src/replication/model/registry.go index 9dabda09a..956ba3c1b 100644 --- a/src/replication/model/registry.go +++ b/src/replication/model/registry.go @@ -22,10 +22,10 @@ import ( // const definition const ( - // RegistryTypeHarbor indicates registry type harbor - RegistryTypeHarbor RegistryType = "harbor" - RegistryTypeDockerHub RegistryType = "dockerHub" - RegistryTypeHuawei RegistryType = "Huawei" + RegistryTypeHarbor RegistryType = "harbor" + RegistryTypeDockerHub RegistryType = "docker-hub" + RegistryTypeDockerRegistry RegistryType = "docker-registry" + RegistryTypeHuawei RegistryType = "huawei" FilterStyleTypeText = "input" FilterStyleTypeRadio = "radio" From d74624d306d400dd197bd673f408d9c5615c8eb3 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Tue, 7 May 2019 13:34:48 +0800 Subject: [PATCH 05/15] Iterate all paginations when listing projects and repositories (#7660) Iterate all paginations when listing projects and repositories Signed-off-by: Wenkai Yin --- src/common/http/client.go | 64 +++++++++++++++++++ src/replication/adapter/harbor/adapter.go | 13 +++- .../adapter/harbor/image_registry.go | 5 +- 3 files changed, 77 insertions(+), 5 deletions(-) diff --git a/src/common/http/client.go b/src/common/http/client.go index 03155fc9c..533212dc0 100644 --- a/src/common/http/client.go +++ b/src/common/http/client.go @@ -17,9 +17,13 @@ package http import ( "bytes" "encoding/json" + "errors" "io" "io/ioutil" "net/http" + "net/url" + "reflect" + "strings" "github.com/goharbor/harbor/src/common/http/modifier" ) @@ -168,3 +172,63 @@ func (c *Client) do(req *http.Request) ([]byte, error) { return data, nil } + +// GetAndIteratePagination iterates the pagination header and returns all resources +// The parameter "v" must be a pointer to a slice +func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error { + url, err := url.Parse(endpoint) + if err != nil { + return err + } + + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr { + return errors.New("v should be a pointer to a slice") + } + elemType := rv.Elem().Type() + if elemType.Kind() != reflect.Slice { + return errors.New("v should be a pointer to a slice") + } + + resources := reflect.Indirect(reflect.New(elemType)) + for len(endpoint) > 0 { + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return err + } + resp, err := c.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return &Error{ + Code: resp.StatusCode, + Message: string(data), + } + } + + res := reflect.New(elemType) + if err = json.Unmarshal(data, res.Interface()); err != nil { + return err + } + resources = reflect.AppendSlice(resources, reflect.Indirect(res)) + + endpoint = "" + link := resp.Header.Get("Link") + for _, str := range strings.Split(link, ",") { + if strings.HasSuffix(str, `rel="next"`) && + strings.Index(str, "<") >= 0 && + strings.Index(str, ">") >= 0 { + endpoint = url.Scheme + "://" + url.Host + str[strings.Index(str, "<")+1:strings.Index(str, ">")] + break + } + } + } + rv.Elem().Set(resources) + return nil +} diff --git a/src/replication/adapter/harbor/adapter.go b/src/replication/adapter/harbor/adapter.go index 74614ac6c..7cc83ef37 100644 --- a/src/replication/adapter/harbor/adapter.go +++ b/src/replication/adapter/harbor/adapter.go @@ -211,8 +211,8 @@ type project struct { func (a *adapter) getProjects(name string) ([]*project, error) { projects := []*project{} - url := fmt.Sprintf("%s/api/projects?name=%s&page=1&page_size=1000", a.coreServiceURL, name) - if err := a.client.Get(url, &projects); err != nil { + url := fmt.Sprintf("%s/api/projects?name=%s&page=1&page_size=500", a.coreServiceURL, name) + if err := a.client.GetAndIteratePagination(url, &projects); err != nil { return nil, err } return projects, nil @@ -243,3 +243,12 @@ func (a *adapter) getProject(name string) (*project, error) { } return nil, nil } + +func (a *adapter) getRepositories(projectID int64) ([]*repository, error) { + repositories := []*repository{} + url := fmt.Sprintf("%s/api/repositories?project_id=%d&page=1&page_size=500", a.coreServiceURL, projectID) + if err := a.client.GetAndIteratePagination(url, &repositories); err != nil { + return nil, err + } + return repositories, nil +} diff --git a/src/replication/adapter/harbor/image_registry.go b/src/replication/adapter/harbor/image_registry.go index 4d7ce68d5..87420c4d2 100644 --- a/src/replication/adapter/harbor/image_registry.go +++ b/src/replication/adapter/harbor/image_registry.go @@ -64,9 +64,8 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error } resources := []*model.Resource{} for _, project := range projects { - repositories := []*repository{} - url := fmt.Sprintf("%s/api/repositories?project_id=%d", a.coreServiceURL, project.ID) - if err = a.client.Get(url, &repositories); err != nil { + repositories, err := a.getRepositories(project.ID) + if err != nil { return nil, err } repositories, err = filterRepositories(repositories, filters) From ab08a576e4c60256fefb87132e20d7024c1e154d Mon Sep 17 00:00:00 2001 From: wang yan Date: Mon, 6 May 2019 15:02:23 +0800 Subject: [PATCH 06/15] add multiple manifest intercepetor handler 1, Add a interceptor to block request to upload manifest list 2, Discard notiification without tag. Signed-off-by: wang yan --- src/core/proxy/interceptor_test.go | 47 +++++++++++++++++++ src/core/proxy/interceptors.go | 31 ++++++++++++ src/core/proxy/proxy.go | 2 +- .../service/notifications/registry/handler.go | 32 +++++++------ 4 files changed, 97 insertions(+), 15 deletions(-) diff --git a/src/core/proxy/interceptor_test.go b/src/core/proxy/interceptor_test.go index 40a30c0cb..be445316d 100644 --- a/src/core/proxy/interceptor_test.go +++ b/src/core/proxy/interceptor_test.go @@ -79,6 +79,53 @@ func TestMatchPullManifest(t *testing.T) { assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag7) } +func TestMatchPushManifest(t *testing.T) { + assert := assert.New(t) + req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil) + res1, _, _ := MatchPushManifest(req1) + assert.False(res1, "%s %v is not a request to push manifest", req1.Method, req1.URL) + + req2, _ := http.NewRequest("PUT", "http://192.168.0.3:80/v2/library/ubuntu/manifests/14.04", nil) + res2, repo2, tag2 := MatchPushManifest(req2) + assert.True(res2, "%s %v is a request to push manifest", req2.Method, req2.URL) + assert.Equal("library/ubuntu", repo2) + assert.Equal("14.04", tag2) + + req3, _ := http.NewRequest("GET", "https://192.168.0.5:443/v1/library/ubuntu/manifests/14.04", nil) + res3, _, _ := MatchPushManifest(req3) + assert.False(res3, "%s %v is not a request to push manifest", req3.Method, req3.URL) + + req4, _ := http.NewRequest("PUT", "https://192.168.0.5/v2/library/ubuntu/manifests/14.04", nil) + res4, repo4, tag4 := MatchPushManifest(req4) + assert.True(res4, "%s %v is a request to push manifest", req4.Method, req4.URL) + assert.Equal("library/ubuntu", repo4) + assert.Equal("14.04", tag4) + + req5, _ := http.NewRequest("PUT", "https://myregistry.com/v2/path1/path2/golang/manifests/1.6.2", nil) + res5, repo5, tag5 := MatchPushManifest(req5) + assert.True(res5, "%s %v is a request to push manifest", req5.Method, req5.URL) + assert.Equal("path1/path2/golang", repo5) + assert.Equal("1.6.2", tag5) + + req6, _ := http.NewRequest("PUT", "https://myregistry.com/v2/myproject/registry/manifests/sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", nil) + res6, repo6, tag6 := MatchPushManifest(req6) + assert.True(res6, "%s %v is a request to push manifest", req6.Method, req6.URL) + assert.Equal("myproject/registry", repo6) + assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag6) + + req7, _ := http.NewRequest("PUT", "https://myregistry.com/v2/myproject/manifests/sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", nil) + res7, repo7, tag7 := MatchPushManifest(req7) + assert.True(res7, "%s %v is a request to push manifest", req7.Method, req7.URL) + assert.Equal("myproject", repo7) + assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag7) + + req8, _ := http.NewRequest("PUT", "http://192.168.0.3:80/v2/library/ubuntu/manifests/14.04", nil) + res8, repo8, tag8 := MatchPushManifest(req8) + assert.True(res8, "%s %v is a request to push manifest", req8.Method, req8.URL) + assert.Equal("library/ubuntu", repo8) + assert.Equal("14.04", tag8) +} + func TestMatchListRepos(t *testing.T) { assert := assert.New(t) req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/_catalog", nil) diff --git a/src/core/proxy/interceptors.go b/src/core/proxy/interceptors.go index fb7c29e0e..b8a3fe3b8 100644 --- a/src/core/proxy/interceptors.go +++ b/src/core/proxy/interceptors.go @@ -43,6 +43,18 @@ func MatchPullManifest(req *http.Request) (bool, string, string) { if req.Method != http.MethodGet { return false, "", "" } + return matchManifestURL(req) +} + +// MatchPushManifest checks if the request looks like a request to push manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values +func MatchPushManifest(req *http.Request) (bool, string, string) { + if req.Method != http.MethodPut { + return false, "", "" + } + return matchManifestURL(req) +} + +func matchManifestURL(req *http.Request) (bool, string, string) { re := regexp.MustCompile(manifestURLPattern) s := re.FindStringSubmatch(req.URL.Path) if len(s) == 3 { @@ -168,6 +180,25 @@ func (rh readonlyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { rh.next.ServeHTTP(rw, req) } +type multipleManifestHandler struct { + next http.Handler +} + +// The handler is responsible for blocking request to upload manifest list by docker client, which is not supported so far by Harbor. +func (mh multipleManifestHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + match, _, _ := MatchPushManifest(req) + if match { + contentType := req.Header.Get("Content-type") + // application/vnd.docker.distribution.manifest.list.v2+json + if strings.Contains(contentType, "manifest.list.v2") { + log.Debugf("Content-type: %s is not supported, failing the response.", contentType) + http.Error(rw, marshalError("UNSUPPORTED_MEDIA_TYPE", "Manifest.list is not supported."), http.StatusUnsupportedMediaType) + return + } + } + mh.next.ServeHTTP(rw, req) +} + type listReposHandler struct { next http.Handler } diff --git a/src/core/proxy/proxy.go b/src/core/proxy/proxy.go index 930f9290d..eadbfed38 100644 --- a/src/core/proxy/proxy.go +++ b/src/core/proxy/proxy.go @@ -38,7 +38,7 @@ func Init(urls ...string) error { return err } Proxy = httputil.NewSingleHostReverseProxy(targetURL) - handlers = handlerChain{head: readonlyHandler{next: urlHandler{next: listReposHandler{next: contentTrustHandler{next: vulnerableHandler{next: Proxy}}}}}} + handlers = handlerChain{head: readonlyHandler{next: urlHandler{next: multipleManifestHandler{next: listReposHandler{next: contentTrustHandler{next: vulnerableHandler{next: Proxy}}}}}}} return nil } diff --git a/src/core/service/notifications/registry/handler.go b/src/core/service/notifications/registry/handler.go index 16f8f2585..d3530f979 100644 --- a/src/core/service/notifications/registry/handler.go +++ b/src/core/service/notifications/registry/handler.go @@ -93,20 +93,24 @@ func (n *NotificationHandler) Post() { }() if action == "push" { - go func() { - exist := dao.RepositoryExists(repository) - if exist { - return - } - log.Debugf("Add repository %s into DB.", repository) - repoRecord := models.RepoRecord{ - Name: repository, - ProjectID: pro.ProjectID, - } - if err := dao.AddRepository(repoRecord); err != nil { - log.Errorf("Error happens when adding repository: %v", err) - } - }() + // discard the notification without tag. + if tag != "" { + go func() { + exist := dao.RepositoryExists(repository) + if exist { + return + } + log.Debugf("Add repository %s into DB.", repository) + repoRecord := models.RepoRecord{ + Name: repository, + ProjectID: pro.ProjectID, + } + if err := dao.AddRepository(repoRecord); err != nil { + log.Errorf("Error happens when adding repository: %v", err) + } + }() + } + if !coreutils.WaitForManifestReady(repository, tag, 5) { log.Errorf("Manifest for image %s:%s is not ready, skip the follow up actions.", repository, tag) return From 8d30d24edddc26c94e003d9b1c309ea34a178c20 Mon Sep 17 00:00:00 2001 From: FangyuanCheng Date: Tue, 7 May 2019 17:58:59 +0800 Subject: [PATCH 07/15] Modify the tooltip of the override Signed-off-by: FangyuanCheng --- .../create-edit-endpoint/create-edit-endpoint.component.ts | 4 ++-- src/portal/src/i18n/lang/en-us-lang.json | 4 ++-- src/portal/src/i18n/lang/es-es-lang.json | 4 ++-- src/portal/src/i18n/lang/fr-fr-lang.json | 4 ++-- src/portal/src/i18n/lang/pt-br-lang.json | 4 ++-- src/portal/src/i18n/lang/zh-cn-lang.json | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/portal/lib/src/create-edit-endpoint/create-edit-endpoint.component.ts b/src/portal/lib/src/create-edit-endpoint/create-edit-endpoint.component.ts index 598fde2a1..19cdb38e8 100644 --- a/src/portal/lib/src/create-edit-endpoint/create-edit-endpoint.component.ts +++ b/src/portal/lib/src/create-edit-endpoint/create-edit-endpoint.component.ts @@ -185,7 +185,7 @@ export class CreateEditEndpointComponent this.endpointService.getEndpoint(targetId).subscribe( target => { this.target = target; - this.urlDisabled = this.target.type === 'dockerHub' ? true : false; + this.urlDisabled = this.target.type === 'docker-hub' ? true : false; // Keep data cache this.initVal = clone(target); this.initVal.credential.access_secret = FAKE_PASSWORD; @@ -212,7 +212,7 @@ export class CreateEditEndpointComponent adapterChange($event): void { let selectValue = this.targetForm.controls.adapter.value; - if (selectValue === 'dockerHub') { + if (selectValue === 'docker-hub') { this.urlDisabled = true; this.targetForm.controls.endpointUrl.setValue(DOCKERHUB_URL); } else { diff --git a/src/portal/src/i18n/lang/en-us-lang.json b/src/portal/src/i18n/lang/en-us-lang.json index 4d624672f..647b320bf 100644 --- a/src/portal/src/i18n/lang/en-us-lang.json +++ b/src/portal/src/i18n/lang/en-us-lang.json @@ -62,7 +62,7 @@ "PUSH_BASED": "Push the resources from the local Harbor to the remote registry.", "PULL_BASED": "Pull the resources from the remote registry to the local Harbor.", "DESTINATION_NAMESPACE": "Specify the destination namespace. If empty, the resources will be put under the same namespace as the source.", - "OVERRIDE": "Specify whether to replace the resources at the destination if a resource with the same name exists.", + "OVERRIDE": "Specify whether to override the resources at the destination if a resource with the same name exists.", "EMAIL": "Email should be a valid email address like name@example.com.", "USER_NAME": "Cannot contain special characters and maximum length should be 255 characters.", "FULL_NAME": "Maximum length should be 20 characters.", @@ -374,7 +374,7 @@ "TOTAL": "Total", "OVERRIDE": "Override", "ENABLED_RULE": "Enable rule", - "OVERRIDE_INFO": "Replace the destination resources if name exists", + "OVERRIDE_INFO": "Override", "OPERATION": "Operation", "CURRENT": "current", "FILTER_PLACEHOLDER": "Filter Tasks", diff --git a/src/portal/src/i18n/lang/es-es-lang.json b/src/portal/src/i18n/lang/es-es-lang.json index 2dc683b0c..c14f74648 100644 --- a/src/portal/src/i18n/lang/es-es-lang.json +++ b/src/portal/src/i18n/lang/es-es-lang.json @@ -62,7 +62,7 @@ "PUSH_BASED": "Push the resources from the local Harbor to the remote registry.", "PULL_BASED": "Pull the resources from the remote registry to the local Harbor.", "DESTINATION_NAMESPACE": "Specify the destination namespace. If empty, the resources will be put under the same namespace as the source.", - "OVERRIDE": "Specify whether to replace the resources at the destination if a resource with the same name exists.", + "OVERRIDE": "Specify whether to override the resources at the destination if a resource with the same name exists.", "EMAIL": "El email debe ser una dirección válida como nombre@ejemplo.com.", "USER_NAME": "Debe tener una longitud máxima de 255 caracteres y no puede contener caracteres especiales.", "FULL_NAME": "La longitud máxima debería ser de 20 caracteres.", @@ -374,7 +374,7 @@ "TOTAL": "Total", "OVERRIDE": "Override", "ENABLED_RULE": "Enable rule", - "OVERRIDE_INFO": "Replace the destination resources if name exists", + "OVERRIDE_INFO": "Override", "CURRENT": "current", "FILTER_PLACEHOLDER": "Filter Tasks", "STOP_TITLE": "Confirme Stop Executions", diff --git a/src/portal/src/i18n/lang/fr-fr-lang.json b/src/portal/src/i18n/lang/fr-fr-lang.json index daebf3df3..688604143 100644 --- a/src/portal/src/i18n/lang/fr-fr-lang.json +++ b/src/portal/src/i18n/lang/fr-fr-lang.json @@ -59,7 +59,7 @@ "PUSH_BASED": "Push the resources from the local Harbor to the remote registry.", "PULL_BASED": "Pull the resources from the remote registry to the local Harbor.", "DESTINATION_NAMESPACE": "Specify the destination namespace. If empty, the resources will be put under the same namespace as the source.", - "OVERRIDE": "Specify whether to replace the resources at the destination if a resource with the same name exists.", + "OVERRIDE": "Specify whether to override the resources at the destination if a resource with the same name exists.", "EMAIL": "L'email doit être une adresse email valide comme name@example.com.", "USER_NAME": "Ne peut pas contenir de caractères spéciaux et la longueur maximale doit être de 255 caractères.", "FULL_NAME": "La longueur maximale doit être de 20 caractères.", @@ -366,7 +366,7 @@ "TOTAL": "Total", "OVERRIDE": "Override", "ENABLED_RULE": "Enable rule", - "OVERRIDE_INFO": "Replace the destination resources if name exists", + "OVERRIDE_INFO": "Override", "CURRENT": "current", "FILTER_PLACEHOLDER": "Filter Tasks", "STOP_TITLE": "Confirmer arrêter les exécutions", diff --git a/src/portal/src/i18n/lang/pt-br-lang.json b/src/portal/src/i18n/lang/pt-br-lang.json index 4cef6484d..72976b5a2 100644 --- a/src/portal/src/i18n/lang/pt-br-lang.json +++ b/src/portal/src/i18n/lang/pt-br-lang.json @@ -62,7 +62,7 @@ "PUSH_BASED": "Push the resources from the local Harbor to the remote registry.", "PULL_BASED": "Pull the resources from the remote registry to the local Harbor.", "DESTINATION_NAMESPACE": "Specify the destination namespace. If empty, the resources will be put under the same namespace as the source.", - "OVERRIDE": "Specify whether to replace the resources at the destination if a resource with the same name exists.", + "OVERRIDE": "Specify whether to override the resources at the destination if a resource with the same name exists.", "EMAIL": "Email deve ser um endereço de email válido como nome@exemplo.com.", "USER_NAME": "Não pode conter caracteres especiais e o tamanho máximo deve ser de 255 caracteres.", "FULL_NAME": "Tamanho máximo deve ser de 20 caracteres.", @@ -372,7 +372,7 @@ "TOTAL": "Total", "OVERRIDE": "Override", "ENABLED_RULE": "Enable rule", - "OVERRIDE_INFO": "Replace the destination resources if name exists", + "OVERRIDE_INFO": "Override", "CURRENT": "current", "FILTER_PLACEHOLDER": "Filter Tasks", "STOP_TITLE": "Confirme as execuções de parada", diff --git a/src/portal/src/i18n/lang/zh-cn-lang.json b/src/portal/src/i18n/lang/zh-cn-lang.json index 0efd3e2ba..7638fdd6b 100644 --- a/src/portal/src/i18n/lang/zh-cn-lang.json +++ b/src/portal/src/i18n/lang/zh-cn-lang.json @@ -62,7 +62,7 @@ "PUSH_BASED": "把资源由本地Harbor推送到远端仓库。", "PULL_BASED": "把资源由远端仓库拉取到本地Harbor。", "DESTINATION_NAMESPACE": "指定目的端名称空间。如果不填，资源会被放到和源相同的名称空间下。", - "OVERRIDE": "如果存在具有相同名称的资源，请指定是否替换目标上的资源。", + "OVERRIDE": "如果存在具有相同名称的资源，请指定是否覆盖目标上的资源。", "EMAIL": "请使用正确的邮箱地址，比如name@example.com。", "USER_NAME": "不能包含特殊字符且长度不能超过255。", "FULL_NAME": "长度不能超过20。", @@ -373,7 +373,7 @@ "TOTAL": "总数", "OVERRIDE": "覆盖", "ENABLED_RULE": "启用规则", - "OVERRIDE_INFO": "如果名称存在，则替换目标资源", + "OVERRIDE_INFO": "覆盖", "CURRENT": "当前仓库", "FILTER_PLACEHOLDER": "过滤任务", "STOP_TITLE": "确认停止任务", From 8311ff729ad1652d3bb7cb102f677f41e131b905 Mon Sep 17 00:00:00 2001 From: Steven Ren Date: Tue, 7 May 2019 18:31:50 +0800 Subject: [PATCH 08/15] Don't display password when calling api/users API. This change fixes github issue 6838 Signed-off-by: Steven Ren --- src/core/api/user.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/api/user.go b/src/core/api/user.go index 353001bf7..f0e852cbe 100644 --- a/src/core/api/user.go +++ b/src/core/api/user.go @@ -203,7 +203,10 @@ func (ua *UserAPI) List() { ua.SendInternalServerError(fmt.Errorf("failed to get users: %v", err)) return } - + for i := range users { + user := &users[i] + user.Password = "" + } ua.SetPaginationHeader(total, page, size) ua.Data["json"] = users ua.ServeJSON() From c16b44d30bf73162334baa7f3ed481475a27517d Mon Sep 17 00:00:00 2001 From: Daniel Jiang Date: Tue, 7 May 2019 19:51:09 +0800 Subject: [PATCH 09/15] Make sure panic is not thrown when refresh token Fixes #7695 Signed-off-by: Daniel Jiang --- src/common/utils/oidc/helper.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/common/utils/oidc/helper.go b/src/common/utils/oidc/helper.go index cafb9852b..32fee3c29 100644 --- a/src/common/utils/oidc/helper.go +++ b/src/common/utils/oidc/helper.go @@ -208,5 +208,9 @@ func RefreshToken(ctx context.Context, token *Token) (*Token, error) { if err != nil { return nil, err } - return &Token{Token: *t, IDToken: t.Extra("id_token").(string)}, nil + it, ok := t.Extra("id_token").(string) + if !ok { + return nil, fmt.Errorf("failed to get id_token from refresh response") + } + return &Token{Token: *t, IDToken: it}, nil } From 6d0c2f51f34b3fb4331d05ba856f40d6b78fecd0 Mon Sep 17 00:00:00 2001 From: Yogi_Wang Date: Wed, 8 May 2019 00:40:47 +0800 Subject: [PATCH 10/15] Move redirectUrl tip,fix pattern with endpoint,fix incorrect tip message for invalid input of OIDC scope Signed-off-by: Yogi_Wang --- src/portal/src/app/config/auth/config-auth.component.html | 6 +++--- src/portal/src/app/config/auth/config-auth.component.scss | 4 +--- src/portal/src/i18n/lang/en-us-lang.json | 1 + src/portal/src/i18n/lang/es-es-lang.json | 1 + src/portal/src/i18n/lang/fr-fr-lang.json | 1 + src/portal/src/i18n/lang/pt-br-lang.json | 1 + src/portal/src/i18n/lang/zh-cn-lang.json | 1 + 7 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/portal/src/app/config/auth/config-auth.component.html b/src/portal/src/app/config/auth/config-auth.component.html index 354570e5f..1c637df9f 100644 --- a/src/portal/src/app/config/auth/config-auth.component.html +++ b/src/portal/src/app/config/auth/config-auth.component.html @@ -322,7 +322,6 @@
-
{{ 'CONFIG.OIDC.OIDC_REDIREC_URL' | translate}} {{redirectUrl}}/c/oidc/callback
{{'TOOLTIP.OIDC_VERIFYCERT' | translate}}
+
{{ 'CONFIG.OIDC.OIDC_REDIREC_URL' | translate}} {{redirectUrl}}/c/oidc/callback
diff --git a/src/portal/src/app/config/auth/config-auth.component.scss b/src/portal/src/app/config/auth/config-auth.component.scss index fcfec2514..3a0ff7f5b 100644 --- a/src/portal/src/app/config/auth/config-auth.component.scss +++ b/src/portal/src/app/config/auth/config-auth.component.scss @@ -16,7 +16,5 @@ clr-tooltip { top: -5px; } .oidc-tip { - position: absolute; - top: 170px; - color: rgb(210, 74, 112); + color: rgb(10, 74, 112); } diff --git a/src/portal/src/i18n/lang/en-us-lang.json b/src/portal/src/i18n/lang/en-us-lang.json index 4d624672f..d0e1d1a48 100644 --- a/src/portal/src/i18n/lang/en-us-lang.json +++ b/src/portal/src/i18n/lang/en-us-lang.json @@ -75,6 +75,7 @@ "SIGN_UP_MAIL": "Email is only used for resetting your password.", "SIGN_UP_REAL_NAME": "First and last name", "ITEM_REQUIRED": "Field is required.", + "SCOPE_REQUIRED": "Field is required and should be in scope format.", "NUMBER_REQUIRED": "Field is required and should be numbers.", "PORT_REQUIRED": "Field is required and should be valid port number.", "CRON_REQUIRED": "Field is required and should be in cron format.", diff --git a/src/portal/src/i18n/lang/es-es-lang.json b/src/portal/src/i18n/lang/es-es-lang.json index 2dc683b0c..0e765e347 100644 --- a/src/portal/src/i18n/lang/es-es-lang.json +++ b/src/portal/src/i18n/lang/es-es-lang.json @@ -75,6 +75,7 @@ "SIGN_UP_MAIL": "La dirección de email solamente se utilizar para restablecer la contraseña.", "SIGN_UP_REAL_NAME": "Nombre y apellidos", "ITEM_REQUIRED": "Campo obligatorio.", + "SCOPE_REQUIRED": "Field is required and should be in scope format.", "NUMBER_REQUIRED": "El campo es obligatorio y debería ser un número.", "PORT_REQUIRED": "El campo es obligatorio y debería ser un número de puerto válido.", "CRON_REQUIRED": "El campo es obligatorio y debe estar en formato cron.", diff --git a/src/portal/src/i18n/lang/fr-fr-lang.json b/src/portal/src/i18n/lang/fr-fr-lang.json index daebf3df3..c5cc95047 100644 --- a/src/portal/src/i18n/lang/fr-fr-lang.json +++ b/src/portal/src/i18n/lang/fr-fr-lang.json @@ -72,6 +72,7 @@ "SIGN_UP_MAIL": "L'email n'est utilisé que pour réinitialiser votre mot de passe.", "SIGN_UP_REAL_NAME": "Prénom et nom", "ITEM_REQUIRED": "Le champ est obligatoire.", + "SCOPE_REQUIRED": "Field is required and should be in scope format.", "NUMBER_REQUIRED": "Le champ est obligatoire et doit être numérique.", "PORT_REQUIRED": "Le champ est obligatoire et doit être un numéro de port valide.", "CRON_REQUIRED": "Le champ est obligatoire et doit être au format cron.", diff --git a/src/portal/src/i18n/lang/pt-br-lang.json b/src/portal/src/i18n/lang/pt-br-lang.json index 4cef6484d..c514150f9 100644 --- a/src/portal/src/i18n/lang/pt-br-lang.json +++ b/src/portal/src/i18n/lang/pt-br-lang.json @@ -75,6 +75,7 @@ "SIGN_UP_MAIL": "Email é apenas utilizado para redefinir a senha.", "SIGN_UP_REAL_NAME": "Primeiro e último nome", "ITEM_REQUIRED": "Campo é obrigatório.", + "SCOPE_REQUIRED": "Field is required and should be in scope format.", "NUMBER_REQUIRED": "Campo é obrigatório e deve ser numerico.", "PORT_REQUIRED": "Campo é obrigatório e deve ser um número de porta válido.", "CRON_REQUIRED": "O campo é obrigatório e deve estar no formato cron.", diff --git a/src/portal/src/i18n/lang/zh-cn-lang.json b/src/portal/src/i18n/lang/zh-cn-lang.json index 0efd3e2ba..fe730eb8d 100644 --- a/src/portal/src/i18n/lang/zh-cn-lang.json +++ b/src/portal/src/i18n/lang/zh-cn-lang.json @@ -75,6 +75,7 @@ "SIGN_UP_MAIL": "邮件地址仅用来重置您的密码。", "SIGN_UP_REAL_NAME": "全名", "ITEM_REQUIRED": "此项为必填项。", + "SCOPE_REQUIRED": "此项为必填项且为scope格式。", "NUMBER_REQUIRED": "此项为必填项且为数字。", "PORT_REQUIRED": "此项为必填项且为合法端口号。", "CRON_REQUIRED": "此项为必填项且为cron格式。", From af0a8355eb87f8eff7ba390062de2007f68ef3d7 Mon Sep 17 00:00:00 2001 From: Yogi_Wang Date: Mon, 6 May 2019 20:22:32 +0800 Subject: [PATCH 11/15] Remove required from filed "First and last name" on the OIDC user profile page,fix email username cannot save in profile remove required from filed "First and last name" on the OIDC user profile page. fix email username cannot save in profile Signed-off-by: Yogi_Wang --- .../account-settings/account-settings-modal.component.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/portal/src/app/account/account-settings/account-settings-modal.component.html b/src/portal/src/app/account/account-settings/account-settings-modal.component.html index bdf1c2da2..e709e8317 100644 --- a/src/portal/src/app/account/account-settings/account-settings-modal.component.html +++ b/src/portal/src/app/account/account-settings/account-settings-modal.component.html @@ -34,9 +34,9 @@
- +