From 4d501030b6827da570f2b3e2f40c4a22be95a71e Mon Sep 17 00:00:00 2001 From: danfengliu Date: Mon, 23 Nov 2020 16:26:43 +0800 Subject: [PATCH] Add docker access credential to avoid access limitaton by docker-hub Signed-off-by: danfengliu --- .github/workflows/CI.yml | 41 --- api/v2.0/legacy_swagger.yaml | 25 +- tests/apitests/python/library/artifact.py | 62 +++- tests/apitests/python/library/base.py | 5 +- tests/apitests/python/library/cnab.py | 5 + .../apitests/python/library/configurations.py | 6 +- tests/apitests/python/library/containerd.py | 2 +- tests/apitests/python/library/docker_api.py | 60 ++-- tests/apitests/python/library/preheat.py | 74 +++++ tests/apitests/python/library/project.py | 19 +- tests/apitests/python/library/registry.py | 2 +- tests/apitests/python/library/repository.py | 24 +- tests/apitests/python/library/singularity.py | 2 +- .../python/library/tag_immutability.py | 87 +++++ tests/apitests/python/library/user.py | 30 +- .../test_add_member_to_private_project.py | 7 +- .../python/test_add_replication_rule.py | 10 +- .../python/test_add_sys_label_to_tag.py | 10 +- .../python/test_assign_role_to_ldap_group.py | 33 +- .../apitests/python/test_assign_sys_admin.py | 10 +- .../test_copy_artifact_outside_project.py | 18 +- tests/apitests/python/test_del_repo.py | 12 +- .../python/test_edit_project_creation.py | 10 +- .../python/test_garbage_collection.py | 10 +- tests/apitests/python/test_ldap_admin_role.py | 26 +- .../apitests/python/test_list_helm_charts.py | 12 +- .../python/test_manage_project_member.py | 10 +- tests/apitests/python/test_p2p.py | 94 ++++++ .../test_project_level_cve_allowlist.py | 9 +- ...test_project_level_policy_content_trust.py | 12 +- tests/apitests/python/test_project_quota.py | 10 +- tests/apitests/python/test_proxy_cache.py | 134 ++++++++ ...chart_by_helm2_helm3_with_robot_Account.py | 12 +- .../test_push_chart_by_helm3_chart_cli.py | 16 +- .../apitests/python/test_push_cnab_bundle.py | 45 +-- .../python/test_push_files_by_oras.py | 12 +- .../test_push_image_with_special_name.py | 12 +- .../test_push_index_by_docker_manifest.py | 17 +- .../python/test_push_sif_by_singularity.py | 12 +- tests/apitests/python/test_registry_api.py | 10 +- .../python/test_replication_from_dockerhub.py | 15 +- tests/apitests/python/test_retention.py | 34 +- tests/apitests/python/test_robot_account.py | 6 +- .../python/test_scan_image_artifact.py | 102 +++--- ...t_scan_image_artifact_in_public_project.py | 6 +- tests/apitests/python/test_sign_image.py | 16 +- .../python/test_sys_cve_allowlists.py | 17 +- .../python/test_system_level_scan_all.py | 56 ++-- tests/apitests/python/test_tag_crud.py | 91 ++++++ .../apitests/python/test_tag_immutability.py | 300 ++++++++++++++++++ tests/apitests/python/test_user_view_logs.py | 39 +-- tests/apitests/python/testutils.py | 26 +- tests/apitests/python/update_docker_cfg.sh | 8 +- tests/ci/api_common_install.sh | 15 +- tests/ci/api_run.sh | 5 +- tests/resources/APITest-Util.robot | 3 +- tests/resources/CNAB_Util.robot | 5 +- tests/resources/Docker-Util.robot | 27 +- .../Harbor-Pages/Configuration.robot | 25 +- .../Harbor-Pages/Configuration_Elements.robot | 3 + tests/resources/Harbor-Pages/GC.robot | 2 - tests/resources/Harbor-Pages/LDAP-Mode.robot | 10 + .../Harbor-Pages/Project-Helmcharts.robot | 6 - .../Harbor-Pages/Project-P2P-Preheat.robot | 6 +- .../Harbor-Pages/Project-Repository.robot | 2 - .../Harbor-Pages/Project-Webhooks.robot | 2 - tests/resources/Harbor-Pages/Project.robot | 10 - .../resources/Harbor-Pages/Replication.robot | 86 ++--- .../Harbor-Pages/Replication_Elements.robot | 11 +- tests/resources/Harbor-Pages/ToolKit.robot | 8 - .../resources/Harbor-Pages/UserProfile.robot | 1 - tests/resources/Harbor-Pages/Verify.robot | 10 +- tests/resources/Nightly-Util.robot | 10 +- tests/resources/TestCaseBody.robot | 2 - tests/resources/Util.robot | 11 +- tests/robot-cases/Group0-BAT/API_DB.robot | 20 +- tests/robot-cases/Group0-BAT/API_LDAP.robot | 3 + .../Group0-Util/cnab_push_bundle.sh | 14 + .../Group0-Util/docker_push_manifest_list.sh | 10 +- tests/robot-cases/Group1-Nightly/Common.robot | 56 ++-- .../Group1-Nightly/Common_GC.robot | 3 - tests/robot-cases/Group1-Nightly/LDAP.robot | 11 + .../robot-cases/Group1-Nightly/Nightly.robot | 3 - .../Group1-Nightly/Replication.robot | 41 +-- .../robot-cases/Group1-Nightly/Schedule.robot | 36 ++- tests/robot-cases/Group3-Upgrade/data.json | 33 +- tests/robot-cases/Group3-Upgrade/prepare.py | 9 +- tests/robot-cases/Group3-Upgrade/run.sh | 4 +- tests/robot-cases/Group3-Upgrade/verify.robot | 2 +- 89 files changed, 1468 insertions(+), 760 deletions(-) create mode 100644 tests/apitests/python/library/preheat.py create mode 100644 tests/apitests/python/test_p2p.py create mode 100644 tests/apitests/python/test_proxy_cache.py create mode 100644 tests/apitests/python/test_tag_crud.py diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8f83e9925..c14da59f8 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -113,7 +113,6 @@ jobs: cd src/github.com/goharbor/harbor pwd go env - echo "CNAB_PATH=$(go env GOPATH)/src/github.com/docker" >> $GITHUB_ENV echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV echo "GOPATH=$(go env GOPATH):$GITHUB_WORKSPACE" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH @@ -133,40 +132,6 @@ jobs: curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose chmod +x docker-compose sudo mv docker-compose /usr/local/bin - echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json - sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/ - sudo update-ca-certificates - sudo service docker restart - wget https://get.helm.sh/helm-v2.14.1-linux-386.tar.gz && tar zxvf helm-v2.14.1-linux-386.tar.gz - sudo mv linux-386/helm /usr/local/bin/helm2 - helm2 init --client-only - helm2 plugin list | grep push || helm2 plugin install https://github.com/chartmuseum/helm-push - wget https://get.helm.sh/helm-v3.1.1-linux-386.tar.gz && tar zxvf helm-v3.1.1-linux-386.tar.gz - sudo mv linux-386/helm /usr/local/bin/helm3 - helm3 plugin list | grep push || helm3 plugin install https://github.com/chartmuseum/helm-push - rm -rf $CNAB_PATH;mkdir -p $CNAB_PATH && cd $CNAB_PATH && git clone https://github.com/cnabio/cnab-to-oci.git - cd cnab-to-oci && git checkout v0.3.0-beta4 - go list - make build - sudo mv bin/cnab-to-oci /usr/local/bin - curl -LO https://github.com/deislabs/oras/releases/download/v0.8.1/oras_0.8.1_linux_amd64.tar.gz - mkdir -p oras-install/ - tar -zxf oras_0.8.1_*.tar.gz -C oras-install/ - sudo mv oras-install/oras /usr/local/bin/ - sudo apt-get update && sudo apt-get install -y \ - build-essential \ - uuid-dev \ - libgpgme-dev \ - squashfs-tools \ - libseccomp-dev \ - pkg-config \ - cryptsetup-bin - export VERSION=3.5.3 && \ - wget https://github.com/sylabs/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz && \ - tar -xzf singularity-${VERSION}.tar.gz && cd singularity - ./mconfig && make -C builddir && sudo make -C builddir install - sudo apt-get update -y ; sudo apt-get install -y zbar-tools libzbar-dev python-zbar python3.7 - sudo rm /usr/bin/python ; sudo ln -s /usr/bin/python3.7 /usr/bin/python ; sudo apt-get install -y python3-pip - name: install run: | cd src/github.com/goharbor/harbor @@ -226,12 +191,6 @@ jobs: curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose chmod +x docker-compose sudo mv docker-compose /usr/local/bin - IP=`hostname -I | awk '{print $1}'` - echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json - echo "IP=$IP" >> $GITHUB_ENV - python -V - sudo apt-get update -y ; sudo apt-get install -y zbar-tools libzbar-dev python-zbar python3.7 - sudo rm /usr/bin/python ; sudo ln -s /usr/bin/python3.7 /usr/bin/python ; sudo apt-get install -y python3-pip - name: install run: | cd src/github.com/goharbor/harbor diff --git a/api/v2.0/legacy_swagger.yaml b/api/v2.0/legacy_swagger.yaml index a27619fb0..ecd77ecab 100644 --- a/api/v2.0/legacy_swagger.yaml +++ b/api/v2.0/legacy_swagger.yaml @@ -2781,7 +2781,7 @@ paths: schema: type: array items: - $ref: '#/definitions/ImmutableTagRule' + $ref: '#/definitions/RetentionRule' '400': description: Illegal format of provided ID value. '401': @@ -2801,10 +2801,11 @@ paths: format: int64 required: true description: Relevant project ID. - - name: immutabletagrule + - name: RetentionRule in: body + required: true schema: - $ref: '#/definitions/ImmutableTagRule' + $ref: '#/definitions/RetentionRule' tags: - Products responses: @@ -2834,10 +2835,11 @@ paths: format: int64 required: true description: Immutable tag rule ID. - - name: immutabletagrule + - name: RetentionRule in: body + required: true schema: - $ref: '#/definitions/ImmutableTagRule' + $ref: '#/definitions/RetentionRule' tags: - Products responses: @@ -5207,19 +5209,6 @@ definitions: enabled: type: boolean description: The quota is enable or disable - ImmutableTagRule: - type: object - properties: - id: - type: integer - format: int64 - project_id: - type: integer - format: int64 - tag_filter: - type: string - enabled: - type: boolean ScannerRegistration: type: object diff --git a/tests/apitests/python/library/artifact.py b/tests/apitests/python/library/artifact.py index db6730dfe..b6ec3dcd8 100644 --- a/tests/apitests/python/library/artifact.py +++ b/tests/apitests/python/library/artifact.py @@ -13,7 +13,7 @@ class Artifact(base.Base, object): client = self._get_client(**kwargs) return client.list_artifacts(project_name, repo_name) - def get_reference_info(self, project_name, repo_name, reference, ignore_not_found = False,**kwargs): + def get_reference_info(self, project_name, repo_name, reference, expect_status_code = 200, ignore_not_found = False,**kwargs): client = self._get_client(**kwargs) params = {} if "with_signature" in kwargs: @@ -22,12 +22,21 @@ class Artifact(base.Base, object): params["with_tag"] = kwargs["with_tag"] if "with_scan_overview" in kwargs: params["with_scan_overview"] = kwargs["with_scan_overview"] + if "with_immutable_status" in kwargs: + params["with_immutable_status"] = kwargs["with_immutable_status"] try: - return client.get_artifact_with_http_info(project_name, repo_name, reference, **params) + data, status_code, _ = client.get_artifact_with_http_info(project_name, repo_name, reference, **params) + return data except ApiException as e: if e.status == 404 and ignore_not_found == True: - return [] + return None + else: + raise Exception("Failed to get reference, {} {}".format(e.status, e.body)) + else: + base._assert_status_code(expect_status_code, status_code) + base._assert_status_code(200, status_code) + return None def delete_artifact(self, project_name, repo_name, reference, expect_status_code = 200, expect_response_body = None, **kwargs): client = self._get_client(**kwargs) @@ -39,9 +48,9 @@ class Artifact(base.Base, object): if expect_response_body is not None: base._assert_status_body(expect_response_body, e.body) return - - base._assert_status_code(expect_status_code, status_code) - base._assert_status_code(200, status_code) + else: + base._assert_status_code(expect_status_code, status_code) + base._assert_status_code(200, status_code) def get_addition(self, project_name, repo_name, reference, addition, **kwargs): client = self._get_client(**kwargs) @@ -62,10 +71,10 @@ class Artifact(base.Base, object): if expect_response_body is not None: base._assert_status_body(expect_response_body, e.body) return - - base._assert_status_code(expect_status_code, status_code) - base._assert_status_code(201, status_code) - return data + else: + base._assert_status_code(expect_status_code, status_code) + base._assert_status_code(201, status_code) + return data def create_tag(self, project_name, repo_name, reference, tag_name, expect_status_code = 201, ignore_conflict = False, **kwargs): client = self._get_client(**kwargs) @@ -75,12 +84,19 @@ class Artifact(base.Base, object): except ApiException as e: if e.status == 409 and ignore_conflict == True: return - base._assert_status_code(expect_status_code, status_code) + else: + raise Exception("Create tag error, {}.".format(e.body)) + else: + base._assert_status_code(expect_status_code, status_code) def delete_tag(self, project_name, repo_name, reference, tag_name, expect_status_code = 200, **kwargs): client = self._get_client(**kwargs) - _, status_code, _ = client.delete_tag_with_http_info(project_name, repo_name, reference, tag_name) - base._assert_status_code(expect_status_code, status_code) + try: + _, status_code, _ = client.delete_tag_with_http_info(project_name, repo_name, reference, tag_name) + except ApiException as e: + base._assert_status_code(expect_status_code, e.status) + else: + base._assert_status_code(expect_status_code, status_code) def check_image_scan_result(self, project_name, repo_name, reference, expected_scan_status = "Success", **kwargs): timeout_count = 30 @@ -91,7 +107,7 @@ class Artifact(base.Base, object): if (timeout_count == 0): break artifact = self.get_reference_info(project_name, repo_name, reference, **kwargs) - scan_status = artifact[0].scan_overview['application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0'].scan_status + scan_status = artifact.scan_overview['application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0'].scan_status if scan_status == expected_scan_status: return raise Exception("Scan image result is {}, not as expected {}.".format(scan_status, expected_scan_status)) @@ -99,5 +115,19 @@ class Artifact(base.Base, object): def check_reference_exist(self, project_name, repo_name, reference, ignore_not_found = False, **kwargs): artifact = self.get_reference_info( project_name, repo_name, reference, ignore_not_found=ignore_not_found, **kwargs) return { - 0: False, - }.get(len(artifact), True) + None: False, + }.get(artifact, True) + + def waiting_for_reference_exist(self, project_name, repo_name, reference, ignore_not_found = True, period = 60, loop_count = 20, **kwargs): + _loop_count = loop_count + while True: + print("Waiting for reference {} round...".format(_loop_count)) + _loop_count = _loop_count - 1 + if (_loop_count == 0): + break + artifact = self.get_reference_info(project_name, repo_name, reference, ignore_not_found=ignore_not_found, **kwargs) + print("Returned artifact by get reference info:", artifact) + if artifact and artifact !=[]: + return artifact + time.sleep(period) + raise Exception("Reference is not exist {} {} {}.".format(project_name, repo_name, reference)) diff --git a/tests/apitests/python/library/base.py b/tests/apitests/python/library/base.py index 475187322..89f07cc0e 100644 --- a/tests/apitests/python/library/base.py +++ b/tests/apitests/python/library/base.py @@ -28,7 +28,7 @@ def get_endpoint(): def _create_client(server, credential, debug, api_type="products"): cfg = None - if api_type in ('projectv2', 'artifact', 'repository', 'scan'): + if api_type in ('projectv2', 'artifact', 'repository', 'scan', 'preheat'): cfg = v2_swagger_client.Configuration() else: cfg = swagger_client.Configuration() @@ -55,6 +55,7 @@ def _create_client(server, credential, debug, api_type="products"): "products": swagger_client.ProductsApi(swagger_client.ApiClient(cfg)), "projectv2": v2_swagger_client.ProjectApi(v2_swagger_client.ApiClient(cfg)), "artifact": v2_swagger_client.ArtifactApi(v2_swagger_client.ApiClient(cfg)), + "preheat": v2_swagger_client.PreheatApi(v2_swagger_client.ApiClient(cfg)), "repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)), "scan": v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)), "scanner": swagger_client.ScannersApi(swagger_client.ApiClient(cfg)), @@ -65,7 +66,7 @@ def _assert_status_code(expect_code, return_code): raise Exception(r"HTTPS status code s not as we expected. Expected {}, while actual HTTPS status code is {}.".format(expect_code, return_code)) def _assert_status_body(expect_status_body, returned_status_body): - if expect_status_body.strip() != returned_status_body.strip(): + if str(returned_status_body.strip()).lower().find(expect_status_body.lower()) < 0: raise Exception(r"HTTPS status body s not as we expected. Expected {}, while actual HTTPS status body is {}.".format(expect_status_body, returned_status_body)) def _random_name(prefix): diff --git a/tests/apitests/python/library/cnab.py b/tests/apitests/python/library/cnab.py index f0b7231dc..add51d259 100644 --- a/tests/apitests/python/library/cnab.py +++ b/tests/apitests/python/library/cnab.py @@ -3,6 +3,7 @@ import base import json import docker_api +from testutils import DOCKER_USER, DOCKER_PWD def load_bundle(service_image, invocation_image): bundle_file = "./tests/apitests/python/bundle_data/bundle.json" @@ -41,6 +42,10 @@ def cnab_push_bundle(bundle_file, target): def push_cnab_bundle(harbor_server, user, password, service_image, invocation_image, target, auto_update_bundle = True): docker_api.docker_info_display() + + #Add docker login command to avoid pull request access rate elimitation by docker hub + docker_api.docker_login_cmd("", DOCKER_USER, DOCKER_PWD, enable_manifest = False) + docker_api.docker_login_cmd(harbor_server, user, password, enable_manifest = False) bundle_file = load_bundle(service_image, invocation_image) fixed_bundle_file = cnab_fixup_bundle(bundle_file, target, auto_update_bundle = auto_update_bundle) diff --git a/tests/apitests/python/library/configurations.py b/tests/apitests/python/library/configurations.py index 196d0940a..715b91ae4 100644 --- a/tests/apitests/python/library/configurations.py +++ b/tests/apitests/python/library/configurations.py @@ -21,6 +21,8 @@ def set_configurations(client, expect_status_code = 200, expect_response_body = conf.ldap_group_search_filter = config.get("ldap_group_search_filter") if "ldap_group_search_scope" in config: conf.ldap_group_search_scope = config.get("ldap_group_search_scope") + if "ldap_group_admin_dn" in config: + conf.ldap_group_admin_dn = config.get("ldap_group_admin_dn") try: _, status_code, _ = client.configurations_put_with_http_info(conf) @@ -68,9 +70,9 @@ class Configurations(base.Base): set_configurations(client, expect_status_code = expect_status_code, **config) def set_configurations_of_ldap(self, ldap_filter=None, ldap_group_attribute_name=None, - ldap_group_base_dn=None, ldap_group_search_filter=None, ldap_group_search_scope=None, expect_status_code = 200, **kwargs): + ldap_group_base_dn=None, ldap_group_search_filter=None, ldap_group_search_scope=None, ldap_group_admin_dn=None, expect_status_code = 200, **kwargs): client = self._get_client(**kwargs) config=dict(ldap_filter=ldap_filter, ldap_group_attribute_name=ldap_group_attribute_name, - ldap_group_base_dn=ldap_group_base_dn, ldap_group_search_filter=ldap_group_search_filter, ldap_group_search_scope=ldap_group_search_scope) + ldap_group_base_dn=ldap_group_base_dn, ldap_group_search_filter=ldap_group_search_filter, ldap_group_admin_dn=ldap_group_admin_dn, ldap_group_search_scope=ldap_group_search_scope) set_configurations(client, expect_status_code = expect_status_code, **config) diff --git a/tests/apitests/python/library/containerd.py b/tests/apitests/python/library/containerd.py index 01ecc3cf7..7f318363f 100644 --- a/tests/apitests/python/library/containerd.py +++ b/tests/apitests/python/library/containerd.py @@ -5,7 +5,7 @@ import json import docker_api def ctr_images_pull(username, password, oci): - command = ["sudo", "ctr", "images", "pull", "-u", username+":"+password, oci] + command = ["sudo", "ctr", "images", "pull","--snapshotter", "native", "-u", username+":"+password, oci] print("Command: ", command) ret = base.run_command(command) print("Command return: ", ret) diff --git a/tests/apitests/python/library/docker_api.py b/tests/apitests/python/library/docker_api.py index 581d0c001..2f892ca4c 100644 --- a/tests/apitests/python/library/docker_api.py +++ b/tests/apitests/python/library/docker_api.py @@ -3,6 +3,7 @@ import base import subprocess import json +from testutils import DOCKER_USER, DOCKER_PWD try: import docker @@ -17,13 +18,17 @@ def docker_info_display(): ret = base.run_command(command) print("Command return: ", ret) -def docker_login_cmd(harbor_host, user, password, cfg_file = "./tests/apitests/python/update_docker_cfg.sh", enable_manifest = True): - command = ["sudo", "docker", "login", harbor_host, "-u", user, "-p", password] +def docker_login_cmd(harbor_host, username, password, cfg_file = "./tests/apitests/python/update_docker_cfg.sh", enable_manifest = True): + if username == "" or password == "": + print("[Warnig]: No docker credential was provided.") + return + command = ["sudo", "docker", "login", harbor_host, "-u", username, "-p", password] print( "Docker Login Command: ", command) base.run_command(command) if enable_manifest == True: try: ret = subprocess.check_output([cfg_file], shell=False) + print("docker login cmd ret:", ret) except subprocess.CalledProcessError as exc: raise Exception("Failed to update docker config, error is {} {}.".format(exc.returncode, exc.output)) @@ -46,30 +51,30 @@ def docker_manifest_push(index): manifest_list.append(line[-71:]) return index_sha256, manifest_list -def docker_manifest_push_to_harbor(index, manifests, harbor_server, user, password, cfg_file = "./tests/apitests/python/update_docker_cfg.sh"): - docker_login_cmd(harbor_server, user, password, cfg_file=cfg_file) +def docker_manifest_push_to_harbor(index, manifests, harbor_server, username, password, cfg_file = "./tests/apitests/python/update_docker_cfg.sh"): + docker_login_cmd(harbor_server, username, password, cfg_file=cfg_file) docker_manifest_create(index, manifests) return docker_manifest_push(index) -def list_repositories(harbor_host, user, password, n = None, last = None): +def list_repositories(harbor_host, username, password, n = None, last = None): if n is not None and last is not None: - command = ["curl", "-s", "-u", user+":"+password, "https://"+harbor_host+"/v2/_catalog"+"?n=%d"%n+"&last="+last, "--insecure"] + command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/_catalog"+"?n=%d"%n+"&last="+last, "--insecure"] elif n is not None: - command = ["curl", "-s", "-u", user+":"+password, "https://"+harbor_host+"/v2/_catalog"+"?n=%d"%n, "--insecure"] + command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/_catalog"+"?n=%d"%n, "--insecure"] else: - command = ["curl", "-s", "-u", user+":"+password, "https://"+harbor_host+"/v2/_catalog", "--insecure"] + command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/_catalog", "--insecure"] print( "List Repositories Command: ", command) ret = base.run_command(command) repos = json.loads(ret).get("repositories","") return repos -def list_image_tags(harbor_host, repository, user, password, n = None, last = None): +def list_image_tags(harbor_host, repository, username, password, n = None, last = None): if n is not None and last is not None: - command = ["curl", "-s", "-u", user+":"+password, "https://"+harbor_host+"/v2/"+repository+"/tags/list"+"?n=%d"%n+"&last="+last, "--insecure"] + command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/"+repository+"/tags/list"+"?n=%d"%n+"&last="+last, "--insecure"] elif n is not None: - command = ["curl", "-s", "-u", user+":"+password, "https://"+harbor_host+"/v2/"+repository+"/tags/list"+"?n=%d"%n, "--insecure"] + command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/"+repository+"/tags/list"+"?n=%d"%n, "--insecure"] else: - command = ["curl", "-s", "-u", user+":"+password, "https://"+harbor_host+"/v2/"+repository+"/tags/list", "--insecure"] + command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/"+repository+"/tags/list", "--insecure"] print( "List Image Tags Command: ", command) ret = base.run_command(command) tags = json.loads(ret).get("tags","") @@ -81,10 +86,17 @@ class DockerAPI(object): self.DCLIENT2 = docker.from_env() def docker_login(self, registry, username, password, expected_error_message = None): - if expected_error_message is "": + if username == "" or password == "": + print("[Warnig]: No docker credential was provided.") + return + if expected_error_message == "": expected_error_message = None + if registry == "docker": + registry = None + ret = "" try: - self.DCLIENT.login(registry = registry, username=username, password=password) + ret = self.DCLIENT.login(registry = registry, username=username, password=password) + return ret except docker.errors.APIError as err: if expected_error_message is not None: print( "docker login error:", str(err)) @@ -100,20 +112,18 @@ class DockerAPI(object): _tag = "latest" if expected_error_message is "": expected_error_message = None - caught_err = False ret = "" try: self.DCLIENT.pull(r'{}:{}'.format(image, _tag)) return ret except Exception as err: - caught_err = True if expected_error_message is not None: print( "docker image pull error:", str(err)) if str(err).lower().find(expected_error_message.lower()) < 0: raise Exception(r"Pull image: Return message {} is not as expected {}".format(str(err), expected_error_message)) else: - raise Exception(r" Docker pull image {} failed, error is [{}]".format (image, message)) - if caught_err == False: + raise Exception(r" Docker pull image {} failed, error is [{}]".format (image, str(err))) + else: if expected_error_message is not None: if str(ret).lower().find(expected_error_message.lower()) < 0: raise Exception(r" Failed to catch error [{}] when pull image {}, return message: {}".format (expected_error_message, image, str(ret))) @@ -132,22 +142,19 @@ class DockerAPI(object): raise Exception(r" Docker tag image {} failed, error is [{}]".format (image, str(err))) def docker_image_push(self, harbor_registry, tag, expected_error_message = None): - caught_err = False - ret = "" + ret = None if expected_error_message is "": expected_error_message = None try: - self.DCLIENT.push(harbor_registry, tag) - return ret + ret = self.DCLIENT.push(harbor_registry, tag) except Exception as err: - caught_err = True if expected_error_message is not None: print( "docker image push error:", str(err)) if str(err).lower().find(expected_error_message.lower()) < 0: raise Exception(r"Push image: Return message {} is not as expected {}".format(str(err), expected_error_message)) else: raise Exception(r" Docker push image {} failed, error is [{}]".format (harbor_registry, message)) - if caught_err == False: + else: if expected_error_message is not None: if str(ret).lower().find(expected_error_message.lower()) < 0: raise Exception(r" Failed to catch error [{}] when push image {}, return message: {}". @@ -158,10 +165,10 @@ class DockerAPI(object): format (harbor_registry, ret)) def docker_image_build(self, harbor_registry, tags=None, size=1, expected_error_message = None): - caught_err = False ret = "" try: baseimage='busybox:latest' + self.DCLIENT.login(username=DOCKER_USER, password=DOCKER_PWD) if not self.DCLIENT.images(name=baseimage): self.DCLIENT.pull(baseimage) c=self.DCLIENT.create_container(image='busybox:latest',command='dd if=/dev/urandom of=test bs=1M count=%d' % size ) @@ -185,14 +192,13 @@ class DockerAPI(object): image = self.DCLIENT2.images.get(repo) return repo, image.id except Exception as err: - caught_err = True if expected_error_message is not None: print( "docker image build error:", str(err)) if str(err).lower().find(expected_error_message.lower()) < 0: raise Exception(r"Push image: Return message {} is not as expected {}".format(str(err), expected_error_message)) else: raise Exception(r" Docker build image {} failed, error is [{}]".format (harbor_registry, str(err))) - if caught_err == False: + else: if expected_error_message is not None: if str(ret).lower().find(expected_error_message.lower()) < 0: raise Exception(r" Failed to catch error [{}] when build image {}, return message: {}". diff --git a/tests/apitests/python/library/preheat.py b/tests/apitests/python/library/preheat.py new file mode 100644 index 000000000..e12f2f715 --- /dev/null +++ b/tests/apitests/python/library/preheat.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- + +import base +import v2_swagger_client +from v2_swagger_client.rest import ApiException + +class Preheat(base.Base, object): + def __init__(self): + super(Preheat,self).__init__(api_type = "preheat") + + def create_instance(self, name = None, description="It's a dragonfly instance", vendor="dragonfly", + endpoint_url="http://20.32.244.16", auth_mode="NONE", enabled=True, insecure=True, + expect_status_code = 201, expect_response_body = None, **kwargs): + if name is None: + name = base._random_name("instance") + client = self._get_client(**kwargs) + instance = v2_swagger_client.Instance(name=name, description=description,vendor=vendor, + endpoint=endpoint_url, auth_mode=auth_mode, enabled=enabled) + print("instance:",instance) + try: + _, status_code, header = client.create_instance_with_http_info(instance) + except ApiException as e: + base._assert_status_code(expect_status_code, e.status) + if expect_response_body is not None: + base._assert_status_body(expect_response_body, e.body) + return + base._assert_status_code(expect_status_code, status_code) + base._assert_status_code(201, status_code) + return base._get_id_from_header(header), name + + def create_policy(self, project_name, project_id, provider_id, name = None, description="It's a dragonfly policy", + filters=r'[{"type":"repository","value":"re*"},{"type":"tag","value":"v1.0*"}]', trigger=r'{"type":"manual","trigger_setting":{"cron":""}}', enabled=True, + expect_status_code = 201, expect_response_body = None, **kwargs): + if name is None: + name = base._random_name("policy") + client = self._get_client(**kwargs) + policy = v2_swagger_client.PreheatPolicy(name=name, project_id=project_id, provider_id=provider_id, + description=description,filters=filters, + trigger=trigger, enabled=enabled) + print("policy:",policy) + try: + _, status_code, header = client.create_policy_with_http_info(project_name, policy) + except ApiException as e: + base._assert_status_code(expect_status_code, e.status) + if expect_response_body is not None: + base._assert_status_body(expect_response_body, e.body) + return + base._assert_status_code(expect_status_code, status_code) + base._assert_status_code(201, status_code) + return base._get_id_from_header(header), name + + def get_instance(self, instance_name, **kwargs): + client = self._get_client(**kwargs) + return client.get_instance(instance_name) + + def get_policy(self, project_name, preheat_policy_name, **kwargs): + client = self._get_client(**kwargs) + return client.get_policy(project_name, preheat_policy_name) + + def update_policy(self, project_name, preheat_policy_name, policy, **kwargs): + client = self._get_client(**kwargs) + return client.update_policy(project_name, preheat_policy_name, policy) + + def delete_instance(self, preheat_instance_name, expect_status_code = 200, expect_response_body = None, **kwargs): + client = self._get_client(**kwargs) + try: + _, status_code, _ = _, status_code, _ = client.delete_instance_with_http_info(preheat_instance_name) + except ApiException as e: + base._assert_status_code(expect_status_code, e.status) + if expect_response_body is not None: + base._assert_status_body(expect_response_body, e.body) + else: + base._assert_status_code(expect_status_code, status_code) + base._assert_status_code(200, status_code) diff --git a/tests/apitests/python/library/project.py b/tests/apitests/python/library/project.py index af2aeefc6..ce04eb701 100644 --- a/tests/apitests/python/library/project.py +++ b/tests/apitests/python/library/project.py @@ -30,15 +30,18 @@ class Project(base.Base): kwargs["credential"] = base.Credential('basic_auth', username, password) super(Project, self).__init__(**kwargs) - def create_project(self, name=None, metadata=None, expect_status_code = 201, expect_response_body = None, **kwargs): + def create_project(self, name=None, registry_id=None, metadata=None, expect_status_code = 201, expect_response_body = None, **kwargs): if name is None: name = base._random_name("project") if metadata is None: metadata = {} + if registry_id is None: + registry_id = registry_id + client = self._get_client(**kwargs) try: - _, status_code, header = client.create_project_with_http_info(v2_swagger_client.ProjectReq(project_name=name, metadata=metadata)) + _, status_code, header = client.create_project_with_http_info(v2_swagger_client.ProjectReq(project_name=name, registry_id = registry_id, metadata=metadata)) except ApiException as e: base._assert_status_code(expect_status_code, e.status) if expect_response_body is not None: @@ -189,7 +192,7 @@ class Project(base.Base): base._assert_status_code(expect_status_code, status_code) base._assert_status_code(200, status_code) - def add_project_members(self, project_id, user_id = None, member_role_id = None, _ldap_group_dn=None,expect_status_code = 201, **kwargs): + def add_project_members(self, project_id, user_id = None, member_role_id = None, _ldap_group_dn=None, expect_status_code = 201, **kwargs): kwargs['api_type'] = 'products' projectMember = swagger_client.ProjectMember() if user_id is not None: @@ -203,9 +206,13 @@ class Project(base.Base): client = self._get_client(**kwargs) data = [] - data, status_code, header = client.projects_project_id_members_post_with_http_info(project_id, project_member = projectMember) - base._assert_status_code(expect_status_code, status_code) - return base._get_id_from_header(header) + try: + data, status_code, header = client.projects_project_id_members_post_with_http_info(project_id, project_member = projectMember) + except swagger_client.rest.ApiException as e: + base._assert_status_code(expect_status_code, e.status) + else: + base._assert_status_code(expect_status_code, status_code) + return base._get_id_from_header(header) def add_project_robot_account(self, project_id, project_name, expires_at, robot_name = None, robot_desc = None, has_pull_right = True, has_push_right = True, has_chart_read_right = True, has_chart_create_right = True, expect_status_code = 201, **kwargs): kwargs['api_type'] = 'products' diff --git a/tests/apitests/python/library/registry.py b/tests/apitests/python/library/registry.py index fe25fa7a6..ebfc261fb 100644 --- a/tests/apitests/python/library/registry.py +++ b/tests/apitests/python/library/registry.py @@ -14,7 +14,7 @@ class Registry(base.Base): registry = swagger_client.Registry(name=name, url=url, description= description, type=registry_type, insecure=insecure, credential=registryCredential) - + print("registry:", registry) _, status_code, header = client.registries_post_with_http_info(registry) base._assert_status_code(expect_status_code, status_code) return base._get_id_from_header(header), _ diff --git a/tests/apitests/python/library/repository.py b/tests/apitests/python/library/repository.py index e3190558b..873009370 100644 --- a/tests/apitests/python/library/repository.py +++ b/tests/apitests/python/library/repository.py @@ -5,6 +5,7 @@ import base import swagger_client from docker_api import DockerAPI from swagger_client.rest import ApiException +from testutils import DOCKER_USER, DOCKER_PWD def pull_harbor_image(registry, username, password, image, tag, expected_login_error_message = None, expected_error_message = None): _docker_api = DockerAPI() @@ -13,24 +14,26 @@ def pull_harbor_image(registry, username, password, image, tag, expected_login_e return time.sleep(2) ret = _docker_api.docker_image_pull(r'{}/{}'.format(registry, image), tag = tag, expected_error_message = expected_error_message) + print("Docker pull image return message: {}".format(ret)) def push_image_to_project(project_name, registry, username, password, image, tag, expected_login_error_message = None, expected_error_message = None, profix_for_image = None, new_image=None): + print("Start to push image {}/{}/{}:{}".format(registry, project_name, image, tag) ) _docker_api = DockerAPI() + _docker_api.docker_login("docker", DOCKER_USER, DOCKER_PWD) + _docker_api.docker_image_pull(image, tag = tag) _docker_api.docker_login(registry, username, password, expected_error_message = expected_login_error_message) time.sleep(2) if expected_login_error_message != None: return - _docker_api.docker_image_pull(image, tag = tag) time.sleep(2) - + original_name = image image = new_image or image if profix_for_image == None: - new_harbor_registry, new_tag = _docker_api.docker_image_tag(r'{}:{}'.format(image, tag), r'{}/{}/{}'.format(registry, project_name, image)) + new_harbor_registry, new_tag = _docker_api.docker_image_tag(r'{}:{}'.format(original_name, tag), r'{}/{}/{}'.format(registry, project_name, image), tag = tag) else: - new_harbor_registry, new_tag = _docker_api.docker_image_tag(r'{}:{}'.format(image, tag), r'{}/{}/{}/{}'.format(registry, project_name, profix_for_image, image)) + new_harbor_registry, new_tag = _docker_api.docker_image_tag(r'{}:{}'.format(original_name, tag), r'{}/{}/{}/{}'.format(registry, project_name, profix_for_image, image), tag = tag) time.sleep(2) - _docker_api.docker_image_push(new_harbor_registry, new_tag, expected_error_message = expected_error_message) return r'{}/{}'.format(project_name, image), new_tag @@ -79,9 +82,12 @@ class Repository(base.Base, object): _, status_code, _ = client.delete_repository_with_http_info(project_name, repo_name) except Exception as e: base._assert_status_code(expect_status_code, e.status) - return e.body - base._assert_status_code(expect_status_code, status_code) - base._assert_status_code(200, status_code) + if expect_response_body is not None: + base._assert_status_body(expect_response_body, e.body) + return + else: + base._assert_status_code(expect_status_code, status_code) + base._assert_status_code(200, status_code) def list_repositories(self, project_name, **kwargs): @@ -146,4 +152,4 @@ class Repository(base.Base, object): base._assert_status_code(expect_status_code, status_code) base._assert_status_code(200, status_code) - return data \ No newline at end of file + return data diff --git a/tests/apitests/python/library/singularity.py b/tests/apitests/python/library/singularity.py index b5bc9fd5b..2258b29b7 100644 --- a/tests/apitests/python/library/singularity.py +++ b/tests/apitests/python/library/singularity.py @@ -15,7 +15,7 @@ def singularity_push_to_harbor(harbor_server, sif_file, project, image, tag): ret = base.run_command( [singularity_cmd, "push", sif_file, "oras://"+harbor_server + "/" + project + "/" + image+":"+ tag] ) def singularity_pull(out_file, from_sif_file): - ret = base.run_command( [singularity_cmd, "pull", out_file, from_sif_file] ) + ret = base.run_command( [singularity_cmd, "pull", "--allow-unsigned", out_file, from_sif_file] ) def push_singularity_to_harbor(from_URI, from_namespace, harbor_server, user, password, project, image, tag): tmp_sif_file = image+timestamp+".sif" diff --git a/tests/apitests/python/library/tag_immutability.py b/tests/apitests/python/library/tag_immutability.py index a2f1c4bae..f99d8286a 100644 --- a/tests/apitests/python/library/tag_immutability.py +++ b/tests/apitests/python/library/tag_immutability.py @@ -2,5 +2,92 @@ import base import swagger_client +from swagger_client.rest import ApiException class Tag_Immutability(base.Base): + def create_tag_immutability_policy_rule(self, project_id, selector_repository_decoration = "repoMatches", + selector_repository="**", selector_tag_decoration = "matches", + selector_tag="**", expect_status_code = 201, **kwargs): + #repoExcludes,excludes + client = self._get_client(**kwargs) + retention_rule = swagger_client.RetentionRule( + action="immutable", + template="immutable_template", + priority = 0, + scope_selectors={ + "repository": [ + { + "kind": "doublestar", + "decoration": selector_repository_decoration, + "pattern": selector_repository + } + ] + }, + tag_selectors=[ + { + "kind": "doublestar", + "decoration": selector_tag_decoration, + "pattern": selector_tag + } + ] + ) + try: + _, status_code, header = client.projects_project_id_immutabletagrules_post_with_http_info(project_id, retention_rule) + except ApiException as e: + base._assert_status_code(expect_status_code, e.status) + else: + base._assert_status_code(expect_status_code, status_code) + base._assert_status_code(201, status_code) + return base._get_id_from_header(header) + + def list_tag_immutability_policy_rules(self, project_id, **kwargs): + client = self._get_client(**kwargs) + return client.projects_project_id_immutabletagrules_get(project_id) + + def get_rule(self, project_id, rule_id, **kwargs): + rules = self.list_tag_immutability_policy_rules(project_id, **kwargs) + for r in rules: + if r.id == rule_id: + return r + return None + + def update_tag_immutability_policy_rule(self, project_id, rule_id, selector_repository_decoration = None, + selector_repository=None, selector_tag_decoration = None, + selector_tag=None, disabled = None, expect_status_code = 200, **kwargs): + rule = self.get_rule( project_id, rule_id,**kwargs) + if selector_repository_decoration: + rule.scope_selectors["repository"][0].decoration = selector_repository_decoration + if selector_repository: + rule.scope_selectors["repository"][0].pattern = selector_repository + if selector_tag_decoration: + rule.tag_selectors[0].decoration = selector_tag_decoration + if selector_tag: + rule.tag_selectors[0].pattern = selector_tag + if disabled is not None: + rule.disabled = disabled + client = self._get_client(**kwargs) + try: + _, status_code, header = client.projects_project_id_immutabletagrules_id_put_with_http_info(project_id, rule_id, rule) + except ApiException as e: + base._assert_status_code(expect_status_code, e.status) + if expect_response_body is not None: + base._assert_status_body(expect_response_body, e.body) + else: + base._assert_status_code(expect_status_code, status_code) + base._assert_status_code(200, status_code) + return base._get_id_from_header(header) + + def create_rule(self, project_id, selector_repository_decoration = "repoMatches", selector_repository="**", + selector_tag_decoration = "matches", selector_tag="**", + expect_status_code = 201, disabled = False, **kwargs): + rule_id = self.create_tag_immutability_policy_rule(project_id, selector_repository_decoration = selector_repository_decoration, + selector_repository = selector_repository, + selector_tag_decoration = selector_tag_decoration, + selector_tag = selector_tag, expect_status_code = expect_status_code, **kwargs) + if expect_status_code != 201: + return + self.update_tag_immutability_policy_rule(project_id, rule_id, selector_repository_decoration = selector_repository_decoration, + selector_repository = selector_repository, selector_tag_decoration = selector_tag_decoration, + selector_tag = selector_tag, disabled = disabled, expect_status_code = 200, **kwargs) + return rule_id + diff --git a/tests/apitests/python/library/user.py b/tests/apitests/python/library/user.py index 3be67cbf0..0dcc75d4d 100644 --- a/tests/apitests/python/library/user.py +++ b/tests/apitests/python/library/user.py @@ -2,11 +2,12 @@ import base import swagger_client +from swagger_client.rest import ApiException class User(base.Base): def create_user(self, name=None, - email = None, user_password=None, realname = None, role_id = None, **kwargs): + email = None, user_password=None, realname = None, role_id = None, expect_status_code=201, **kwargs): if name is None: name = base._random_name("user") if realname is None: @@ -20,13 +21,16 @@ class User(base.Base): client = self._get_client(**kwargs) user = swagger_client.User(username = name, email = email, password = user_password, realname = realname, role_id = role_id) - _, status_code, header = client.users_post_with_http_info(user) - base._assert_status_code(201, status_code) + try: + _, status_code, header = client.users_post_with_http_info(user) + except ApiException as e: + base._assert_status_code(expect_status_code, e.status) + else: + base._assert_status_code(expect_status_code, status_code) + return base._get_id_from_header(header), name - return base._get_id_from_header(header), name - - def get_users(self, user_name=None, email=None, page=None, page_size=None, **kwargs): + def get_users(self, user_name=None, email=None, page=None, page_size=None, expect_status_code=200, **kwargs): client = self._get_client(**kwargs) params={} if user_name is not None: @@ -37,9 +41,13 @@ class User(base.Base): params["page"] = page if page_size is not None: params["page_size"] = page_size - data, status_code, _ = client.users_get_with_http_info(**params) - base._assert_status_code(200, status_code) - return data + try: + data, status_code, _ = client.users_get_with_http_info(**params) + except ApiException as e: + base._assert_status_code(expect_status_code, e.status) + else: + base._assert_status_code(expect_status_code, status_code) + return data def get_user_by_id(self, user_id, **kwargs): client = self._get_client(**kwargs) @@ -47,8 +55,8 @@ class User(base.Base): base._assert_status_code(200, status_code) return data - def get_user_by_name(self, name, **kwargs): - users = self.get_users(user_name=name, **kwargs) + def get_user_by_name(self, name, expect_status_code=200, **kwargs): + users = self.get_users(user_name=name, expect_status_code=expect_status_code , **kwargs) for user in users: if user.username == name: return user diff --git a/tests/apitests/python/test_add_member_to_private_project.py b/tests/apitests/python/test_add_member_to_private_project.py index ec1eefbf6..d436a2016 100644 --- a/tests/apitests/python/test_add_member_to_private_project.py +++ b/tests/apitests/python/test_add_member_to_private_project.py @@ -3,18 +3,19 @@ from __future__ import absolute_import import unittest -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning, TEARDOWN from library.project import Project from library.user import User class TestProjects(unittest.TestCase): - """UserGroup unit test stubs""" + @suppress_urllib3_warning def setUp(self): self.project = Project() self.user= User() + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") def tearDown(self): - pass + print("Case completed") def testAddProjectMember(self): """ diff --git a/tests/apitests/python/test_add_replication_rule.py b/tests/apitests/python/test_add_replication_rule.py index eebe740cc..598fd7a9c 100644 --- a/tests/apitests/python/test_add_replication_rule.py +++ b/tests/apitests/python/test_add_replication_rule.py @@ -1,7 +1,7 @@ from __future__ import absolute_import import unittest -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import harbor_server from testutils import TEARDOWN from library.project import Project @@ -11,19 +11,15 @@ from library.registry import Registry import swagger_client class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.project = Project() self.user = User() self.replication = Replication() self.registry = Registry() - @classmethod - def tearDown(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete rule(RA); self.replication.delete_replication_rule(TestProjects.rule_id, **ADMIN_CLIENT) diff --git a/tests/apitests/python/test_add_sys_label_to_tag.py b/tests/apitests/python/test_add_sys_label_to_tag.py index 08d07a9f5..a0373a2d1 100644 --- a/tests/apitests/python/test_add_sys_label_to_tag.py +++ b/tests/apitests/python/test_add_sys_label_to_tag.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import unittest -from testutils import harbor_server +from testutils import harbor_server, suppress_urllib3_warning from testutils import TEARDOWN from testutils import ADMIN_CLIENT from library.artifact import Artifact @@ -13,7 +13,7 @@ from library.repository import push_image_to_project from library.label import Label class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.project = Project() self.user = User() @@ -21,12 +21,8 @@ class TestProjects(unittest.TestCase): self.repo = Repository() self.label = Label() - @classmethod - def tearDown(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete repository(RA) by user(UA); self.repo.delete_repoitory(TestProjects.project_add_g_lbl_name, TestProjects.repo_name.split('/')[1], **TestProjects.USER_add_g_lbl_CLIENT) diff --git a/tests/apitests/python/test_assign_role_to_ldap_group.py b/tests/apitests/python/test_assign_role_to_ldap_group.py index bd360ae0d..f15bf2a6f 100644 --- a/tests/apitests/python/test_assign_role_to_ldap_group.py +++ b/tests/apitests/python/test_assign_role_to_ldap_group.py @@ -1,7 +1,7 @@ from __future__ import absolute_import import unittest -from testutils import harbor_server +from testutils import harbor_server, suppress_urllib3_warning from testutils import TEARDOWN from testutils import ADMIN_CLIENT from testutils import created_user, created_project @@ -10,22 +10,21 @@ from library.user import User from library.repository import Repository from library.repository import push_image_to_project from library.artifact import Artifact -from library.scan import Scan from library.scanner import Scanner from library.configurations import Configurations from library.projectV2 import ProjectV2 class TestAssignRoleToLdapGroup(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.conf= Configurations() self.project = Project() self.artifact = Artifact() self.repo = Repository() - self.scan = Scan() + self.user= User() - @classmethod + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") def tearDown(self): print("Case completed") @@ -38,17 +37,19 @@ class TestAssignRoleToLdapGroup(unittest.TestCase): 2. Create a new public project(PA) by Admin; 3. Add 3 member groups to project(PA); 4. Push image by each member role; - 5. Verfify that admin_user and dev_user can push image, guest_user can not push image; - 6. Verfify that admin_user, dev_user and guest_user can view logs, test user can not view logs. - 7. Delete repository(RA) by user(UA); - 8. Delete project(PA); + 5. Verfify that admin_user can add project member, dev_user and guest_user can not add project member; + 6. Verfify that admin_user and dev_user can push image, guest_user can not push image; + 7. Verfify that admin_user, dev_user and guest_user can view logs, test user can not view logs. + 8. Delete repository(RA) by user(UA); + 9. Delete project(PA); """ url = ADMIN_CLIENT["endpoint"] USER_ADMIN=dict(endpoint = url, username = "admin_user", password = "zhu88jie", repo = "hello-world") USER_DEV=dict(endpoint = url, username = "dev_user", password = "zhu88jie", repo = "alpine") USER_GUEST=dict(endpoint = url, username = "guest_user", password = "zhu88jie", repo = "busybox") USER_TEST=dict(endpoint = url, username = "test", password = "123456") - + USER_MIKE=dict(endpoint = url, username = "mike", password = "zhu88jie") + #USER001 is in group harbor_group3 self.conf.set_configurations_of_ldap(ldap_filter="", ldap_group_attribute_name="cn", ldap_group_base_dn="ou=groups,dc=example,dc=com", ldap_group_search_filter="objectclass=groupOfNames", ldap_group_search_scope=2, **ADMIN_CLIENT) @@ -56,21 +57,29 @@ class TestAssignRoleToLdapGroup(unittest.TestCase): self.project.add_project_members(project_id, member_role_id = 1, _ldap_group_dn = "cn=harbor_admin,ou=groups,dc=example,dc=com", **ADMIN_CLIENT) self.project.add_project_members(project_id, member_role_id = 2, _ldap_group_dn = "cn=harbor_dev,ou=groups,dc=example,dc=com", **ADMIN_CLIENT) self.project.add_project_members(project_id, member_role_id = 3, _ldap_group_dn = "cn=harbor_guest,ou=groups,dc=example,dc=com", **ADMIN_CLIENT) + projects = self.project.get_projects(dict(name=project_name), **USER_ADMIN) self.assertTrue(len(projects) == 1) self.assertEqual(1, projects[0].current_user_role_id) + #Mike has logged in harbor in previous test. + mike = self.user.get_user_by_name(USER_MIKE["username"], **ADMIN_CLIENT) + + #Verify role difference in add project member feature, to distinguish between admin and dev role + self.project.add_project_members(project_id, user_id=mike.user_id, member_role_id = 3, **USER_ADMIN) + self.project.add_project_members(project_id, user_id=mike.user_id, member_role_id = 3, expect_status_code=403, **USER_DEV) + self.project.add_project_members(project_id, user_id=mike.user_id, member_role_id = 3, expect_status_code=403, **USER_GUEST) + repo_name_admin, _ = push_image_to_project(project_name, harbor_server, USER_ADMIN["username"], USER_ADMIN["password"], USER_ADMIN["repo"], "latest") artifacts = self.artifact.list_artifacts(project_name, USER_ADMIN["repo"], **USER_ADMIN) self.assertTrue(len(artifacts) == 1) repo_name_dev, _ = push_image_to_project(project_name, harbor_server, USER_DEV["username"], USER_DEV["password"], USER_DEV["repo"], "latest") artifacts = self.artifact.list_artifacts(project_name, USER_DEV["repo"], **USER_DEV) self.assertTrue(len(artifacts) == 1) - push_image_to_project(project_name, harbor_server, USER_GUEST["username"], USER_GUEST["password"], USER_GUEST["repo"], "latest") + push_image_to_project(project_name, harbor_server, USER_GUEST["username"], USER_GUEST["password"], USER_GUEST["repo"], "latest", expected_error_message = "unauthorized to access repository") artifacts = self.artifact.list_artifacts(project_name, USER_GUEST["repo"], **USER_GUEST) self.assertTrue(len(artifacts) == 0) - self.assertTrue(self.project.query_user_logs(project_name, **USER_ADMIN)>0, "admin user can see logs") self.assertTrue(self.project.query_user_logs(project_name, **USER_DEV)>0, "dev user can see logs") self.assertTrue(self.project.query_user_logs(project_name, **USER_GUEST)>0, "guest user can see logs") diff --git a/tests/apitests/python/test_assign_sys_admin.py b/tests/apitests/python/test_assign_sys_admin.py index 7112ae20e..d01ed55d6 100644 --- a/tests/apitests/python/test_assign_sys_admin.py +++ b/tests/apitests/python/test_assign_sys_admin.py @@ -2,23 +2,19 @@ from __future__ import absolute_import import unittest -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import TEARDOWN from library.user import User from library.configurations import Configurations class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.conf= Configurations() self.user = User() - @classmethod - def tearDown(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete user(UA); self.user.delete_user(TestProjects.user_assign_sys_admin_id, **ADMIN_CLIENT) diff --git a/tests/apitests/python/test_copy_artifact_outside_project.py b/tests/apitests/python/test_copy_artifact_outside_project.py index ac061df6e..8f3d282d0 100644 --- a/tests/apitests/python/test_copy_artifact_outside_project.py +++ b/tests/apitests/python/test_copy_artifact_outside_project.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import unittest -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import harbor_server from testutils import TEARDOWN from library.base import _assert_status_code @@ -15,19 +15,15 @@ from library.repository import push_image_to_project from library.repository import pull_harbor_image class TestProjects(unittest.TestCase): - @classmethod - def setUpClass(self): + @suppress_urllib3_warning + def setUp(self): self.project = Project() self.user = User() self.artifact = Artifact() self.repo = Repository() - @classmethod - def tearDownClass(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete repository(RA); self.repo.delete_repoitory(TestProjects.project_src_repo_name, (TestProjects.src_repo_name).split('/')[1], **TestProjects.USER_RETAG_CLIENT) @@ -97,19 +93,19 @@ class TestProjects(unittest.TestCase): src_tag_data = self.artifact.get_reference_info(TestProjects.project_src_repo_name, TestProjects.src_repo_name.split('/')[1], tag_name, **TestProjects.USER_RETAG_CLIENT) TestProjects.dst_repo_name = TestProjects.project_dst_repo_name+"/"+ dst_repo_sub_name #8. Retag image in project(PA) to project(PB), it should be forbidden; - self.artifact.copy_artifact(TestProjects.project_dst_repo_name, dst_repo_sub_name, TestProjects.src_repo_name+"@"+src_tag_data[0].digest, expect_status_code=403, **TestProjects.USER_RETAG_CLIENT) + self.artifact.copy_artifact(TestProjects.project_dst_repo_name, dst_repo_sub_name, TestProjects.src_repo_name+"@"+src_tag_data.digest, expect_status_code=403, **TestProjects.USER_RETAG_CLIENT) #9. Update role of user-retag as admin member of project(PB); self.project.update_project_member_role(TestProjects.project_dst_repo_id, retag_member_id, 1, **ADMIN_CLIENT) #10. Retag image in project(PA) to project(PB), it should be successful; - self.artifact.copy_artifact(TestProjects.project_dst_repo_name, dst_repo_sub_name, TestProjects.src_repo_name+"@"+src_tag_data[0].digest, **TestProjects.USER_RETAG_CLIENT) + self.artifact.copy_artifact(TestProjects.project_dst_repo_name, dst_repo_sub_name, TestProjects.src_repo_name+"@"+src_tag_data.digest, **TestProjects.USER_RETAG_CLIENT) #11. Get repository(RB)'s image tag detail information; dst_tag_data = self.artifact.get_reference_info(TestProjects.project_dst_repo_name, dst_repo_sub_name, tag_name, **TestProjects.USER_RETAG_CLIENT) #12. Read digest of retaged image, it must be the same with the image in repository(RA); - self.assertEqual(src_tag_data[0].digest, dst_tag_data[0].digest) + self.assertEqual(src_tag_data.digest, dst_tag_data.digest) #13. Pull image from project(PB) by user_retag, it must be successful;" pull_harbor_image(harbor_server, user_retag_name, user_retag_password, TestProjects.dst_repo_name, tag_name) diff --git a/tests/apitests/python/test_del_repo.py b/tests/apitests/python/test_del_repo.py index dff8bf845..a6d6cb5c5 100644 --- a/tests/apitests/python/test_del_repo.py +++ b/tests/apitests/python/test_del_repo.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import unittest -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import harbor_server from testutils import TEARDOWN from library.base import _assert_status_code @@ -13,18 +13,14 @@ from library.repository import Repository from library.repository import push_image_to_project class TestProjects(unittest.TestCase): - @classmethod - def setUpClass(self): + @suppress_urllib3_warning + def setUp(self): self.project= Project() self.user= User() self.repo= Repository() - @classmethod - def tearDownClass(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete project(PA); self.project.delete_project(TestProjects.project_del_repo_id, **TestProjects.USER_del_repo_CLIENT) diff --git a/tests/apitests/python/test_edit_project_creation.py b/tests/apitests/python/test_edit_project_creation.py index e4dc4e7f5..398d941e2 100644 --- a/tests/apitests/python/test_edit_project_creation.py +++ b/tests/apitests/python/test_edit_project_creation.py @@ -1,25 +1,21 @@ from __future__ import absolute_import import unittest -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import TEARDOWN from library.project import Project from library.user import User from library.configurations import Configurations class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.conf= Configurations() self.project= Project() self.user= User() - @classmethod - def tearDown(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete project(PA); self.project.delete_project(TestProjects.project_edit_project_creation_id, **TestProjects.USER_edit_project_creation_CLIENT) diff --git a/tests/apitests/python/test_garbage_collection.py b/tests/apitests/python/test_garbage_collection.py index a453d80de..412c5c3ff 100644 --- a/tests/apitests/python/test_garbage_collection.py +++ b/tests/apitests/python/test_garbage_collection.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import unittest import time -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import TEARDOWN from testutils import harbor_server from library.user import User @@ -16,7 +16,7 @@ from library.repository import push_special_image_to_project from library.artifact import Artifact class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.system = System() self.project = Project() @@ -27,12 +27,8 @@ class TestProjects(unittest.TestCase): self.repo_name_untag = "test_untag" self.tag = "v1.0" - @classmethod + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") def tearDown(self): - print("Case completed") - - @unittest.skipIf(TEARDOWN == True, "Test data won't be erased.") - def test_ClearData(self): #2. Delete project(PA); self.project.delete_project(TestProjects.project_gc_id, **TestProjects.USER_GC_CLIENT) diff --git a/tests/apitests/python/test_ldap_admin_role.py b/tests/apitests/python/test_ldap_admin_role.py index f1ad2a4d9..f1c2716ab 100644 --- a/tests/apitests/python/test_ldap_admin_role.py +++ b/tests/apitests/python/test_ldap_admin_role.py @@ -1,7 +1,7 @@ from __future__ import absolute_import import unittest -from testutils import harbor_server +from testutils import harbor_server, suppress_urllib3_warning from testutils import TEARDOWN from testutils import ADMIN_CLIENT from library.user import User @@ -10,18 +10,15 @@ from library.configurations import Configurations class TestLdapAdminRole(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): url = ADMIN_CLIENT["endpoint"] self.conf= Configurations() - self.uesr = User() + self.user = User() self.project = Project() self.USER_MIKE=dict(endpoint = url, username = "mike", password = "zhu88jie") + self.project_id = None - @classmethod - def tearDown(self): - self.project.delete_project(TestLdapAdminRole.project_id, **self.USER_MIKE) - print("Case completed") def testLdapAdminRole(self): """ @@ -31,19 +28,28 @@ class TestLdapAdminRole(unittest.TestCase): 1. Set LDAP Auth configurations; 2. Create a new public project(PA) by LDAP user mike; 3. Check project is created successfully; - 4. Check mike is not admin; + 4. Check mike's SysAdminFlag is false, but AdminRoleInAuth should be true 5. Delete project(PA); """ self.conf.set_configurations_of_ldap(ldap_group_admin_dn="cn=harbor_users,ou=groups,dc=example,dc=com", **ADMIN_CLIENT) - TestLdapAdminRole.project_id, project_name = self.project.create_project(metadata = {"public": "false"}, **self.USER_MIKE) + self.project_id, project_name = self.project.create_project(metadata = {"public": "false"}, **self.USER_MIKE) + print("self.project_id:", self.project_id) self.project.check_project_name_exist(name=project_name, **self.USER_MIKE) - _user = self.uesr.get_user_by_name(self.USER_MIKE["username"], **ADMIN_CLIENT) + _user = self.user.get_user_current(**self.USER_MIKE) + print( _user) self.assertFalse(_user.sysadmin_flag) + self.assertTrue(_user.admin_role_in_auth) + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def tearDown(self): + print("self.project_id:", self.project_id) + self.project.delete_project(self.project_id, **self.USER_MIKE) + print("Case completed") + if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/apitests/python/test_list_helm_charts.py b/tests/apitests/python/test_list_helm_charts.py index 5b65226d5..6876e3801 100644 --- a/tests/apitests/python/test_list_helm_charts.py +++ b/tests/apitests/python/test_list_helm_charts.py @@ -2,25 +2,22 @@ from __future__ import absolute_import import unittest -from testutils import ADMIN_CLIENT, CHART_API_CLIENT +from testutils import ADMIN_CLIENT, CHART_API_CLIENT, suppress_urllib3_warning from testutils import TEARDOWN +import base from library.user import User from library.project import Project from library.chart import Chart class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.chart= Chart() self.project= Project() self.user= User() - @classmethod - def tearDown(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete chart file; self.chart.delete_chart_with_version(TestProjects.project_chart_name, TestProjects.CHART_NAME, TestProjects.VERSION, **CHART_API_CLIENT) @@ -50,6 +47,7 @@ class TestProjects(unittest.TestCase): TestProjects.CHART_NAME = 'mariadb' TestProjects.VERSION = '4.3.1' + base.run_command( ["curl", r"-o", "./tests/apitests/python/mariadb-4.3.1.tgz", "https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz"]) #1. Create a new user(UA); TestProjects.user_chart_id, user_chart_name = self.user.create_user(user_password = user_chart_password, **ADMIN_CLIENT) diff --git a/tests/apitests/python/test_manage_project_member.py b/tests/apitests/python/test_manage_project_member.py index 0bde95351..19c291fd4 100644 --- a/tests/apitests/python/test_manage_project_member.py +++ b/tests/apitests/python/test_manage_project_member.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import unittest -from testutils import harbor_server +from testutils import harbor_server, suppress_urllib3_warning from testutils import TEARDOWN from testutils import ADMIN_CLIENT from library.project import Project @@ -11,18 +11,14 @@ from library.repository import push_image_to_project from library.repository import Repository class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.project = Project() self.user = User() self.repo = Repository() - @classmethod - def tearDown(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete repository(RA) by admin; self.repo.delete_repoitory(TestProjects.project_alice_name, TestProjects.repo_name.split('/')[1], **ADMIN_CLIENT) diff --git a/tests/apitests/python/test_p2p.py b/tests/apitests/python/test_p2p.py new file mode 100644 index 000000000..f28aac5b6 --- /dev/null +++ b/tests/apitests/python/test_p2p.py @@ -0,0 +1,94 @@ +from __future__ import absolute_import + +import unittest +import sys + +from testutils import ADMIN_CLIENT, TEARDOWN, suppress_urllib3_warning + +from library.project import Project +from library.user import User +from library.repository import Repository +from library.registry import Registry +from library.artifact import Artifact +from library.preheat import Preheat +import v2_swagger_client + +class TestP2P(unittest.TestCase): + @suppress_urllib3_warning + def setUp(self): + self.url = ADMIN_CLIENT["endpoint"] + self.user_password = "Aa123456" + self.project= Project() + self.user= User() + self.repo= Repository() + self.registry = Registry() + self.artifact = Artifact() + self.preheat = Preheat() + + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def tearDown(self): + print("Case completed") + + def do_validate(self, registry_type): + """ + Test case: + Proxy Cache Image From Harbor + Test step and expected result: + 1. Create a new registry; + 2. Create a new project; + 3. Add a new user as a member of project; + 4. Pull image from this project by docker CLI; + 5. Pull image from this project by ctr CLI; + 6. Pull manifest index from this project by docker CLI; + 7. Pull manifest from this project by ctr CLI; + 8. Image pulled by docker CLI should be cached; + 9. Image pulled by ctr CLI should be cached; + 10. Manifest index pulled by docker CLI should be cached; + 11. Manifest index pulled by ctr CLI should be cached; + Tear down: + 1. Delete project(PA); + 2. Delete user(UA). + """ + _, user_name = self.user.create_user(user_password = self.user_password, **ADMIN_CLIENT) + USER_CLIENT=dict(with_signature = True, endpoint = self.url, username = user_name, password = self.user_password) + + #2. Create a new distribution instance; + _, instance_name = self.preheat.create_instance( **ADMIN_CLIENT) + + #This need to be removed once issue #13378 fixed. + instance = self.preheat.get_instance(instance_name) + print("instance:", instance) + + #2. Create a new project; + project_id, project_name = self.project.create_project(metadata = {"public": "false"}, **USER_CLIENT) + print("project_id:",project_id) + print("project_name:",project_name) + + #This need to be removed once issue #13378 fixed. + _, policy_name = self.preheat.create_policy(project_name, project_id, instance.id, **USER_CLIENT) + #policy_id, _ = self.preheat.create_policy(project_name, project_id, instance_id, **USER_CLIENT) + policy = self.preheat.get_policy(project_name, policy_name) + print("policy:", policy) + + policy_new = v2_swagger_client.PreheatPolicy(id = policy.id, name="policy_new_name", project_id=project_id, provider_id=instance.id, + description="edit this policy",filters=r'[{"type":"repository","value":"zgila/alpine*"},{"type":"tag","value":"v1.0*"},{"type":"label","value":"release"}]', + trigger=r'{"type":"scheduled","trigger_setting":{"cron":"0 8 * * * *"}}', enabled=False) + + self.preheat.update_policy(project_name, policy.name, policy_new, **USER_CLIENT) + + self.preheat.delete_instance(instance.name, expect_status_code=403, **USER_CLIENT) + + self.project.delete_project(project_id, **USER_CLIENT) + + self.preheat.delete_instance(instance.name, **ADMIN_CLIENT) + + def test_create_instance(self): + self.do_validate("harbor") + +if __name__ == '__main__': + suite = unittest.TestSuite(unittest.makeSuite(TestP2P)) + result = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=True).run(suite) + print("Test result:",result) + if not result.wasSuccessful(): + raise Exception(r"P2P test failed!") + diff --git a/tests/apitests/python/test_project_level_cve_allowlist.py b/tests/apitests/python/test_project_level_cve_allowlist.py index 2a7bb804a..2e37dfe78 100644 --- a/tests/apitests/python/test_project_level_cve_allowlist.py +++ b/tests/apitests/python/test_project_level_cve_allowlist.py @@ -4,7 +4,7 @@ import unittest import v2_swagger_client import time -from testutils import ADMIN_CLIENT, TEARDOWN +from testutils import ADMIN_CLIENT, TEARDOWN, suppress_urllib3_warning from library.project import Project from library.user import User @@ -32,7 +32,7 @@ class TestProjectCVEAllowlist(unittest.TestCase): 2. Delete project(PA) 3. Delete User(RA) """ - + @suppress_urllib3_warning def setUp(self): self.user = User() self.project = Project() @@ -49,11 +49,8 @@ class TestProjectCVEAllowlist(unittest.TestCase): m_id = self.project.add_project_members(self.project_pa_id, user_id=self.user_ra_id, member_role_id=3, **ADMIN_CLIENT) self.member_id = int(m_id) - def tearDown(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): print("Tearing down...") self.project.delete_project_member(self.project_pa_id, self.member_id, **ADMIN_CLIENT) self.project.delete_project(self.project_pa_id,**ADMIN_CLIENT) diff --git a/tests/apitests/python/test_project_level_policy_content_trust.py b/tests/apitests/python/test_project_level_policy_content_trust.py index 8de52e52a..e598d3c55 100644 --- a/tests/apitests/python/test_project_level_policy_content_trust.py +++ b/tests/apitests/python/test_project_level_policy_content_trust.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import unittest -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import harbor_server from testutils import TEARDOWN from library.artifact import Artifact @@ -13,19 +13,15 @@ from library.repository import push_image_to_project from library.repository import pull_harbor_image class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.project= Project() self.user= User() self.artifact= Artifact() self.repo= Repository() - @classmethod - def tearDown(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete repository(RA) by user(UA); self.repo.delete_repoitory(TestProjects.project_content_trust_name, TestProjects.repo_name.split('/')[1], **TestProjects.USER_CONTENT_TRUST_CLIENT) @@ -71,7 +67,7 @@ class TestProjects(unittest.TestCase): #4. Image(IA) should exist; artifact = self.artifact.get_reference_info(TestProjects.project_content_trust_name, image, tag, **TestProjects.USER_CONTENT_TRUST_CLIENT) - self.assertEqual(artifact[0].tags[0].name, tag) + self.assertEqual(artifact.tags[0].name, tag) #5. Pull image(IA) successfully; pull_harbor_image(harbor_server, admin_name, admin_password, TestProjects.repo_name, tag) diff --git a/tests/apitests/python/test_project_quota.py b/tests/apitests/python/test_project_quota.py index 0cb4efea0..6bab2dc98 100644 --- a/tests/apitests/python/test_project_quota.py +++ b/tests/apitests/python/test_project_quota.py @@ -1,20 +1,20 @@ from __future__ import absolute_import import unittest -from testutils import harbor_server, created_project, created_user -from testutils import ADMIN_CLIENT +from testutils import harbor_server, created_project, created_user, \ + TEARDOWN, ADMIN_CLIENT,suppress_urllib3_warning from library.repository import Repository from library.repository import push_image_to_project from library.system import System class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(cls): cls.repo = Repository() cls.system = System() - @classmethod - def tearDown(cls): + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def tearDown(self): print("Case completed") def testProjectQuota(self): diff --git a/tests/apitests/python/test_proxy_cache.py b/tests/apitests/python/test_proxy_cache.py new file mode 100644 index 000000000..e01a7c0ed --- /dev/null +++ b/tests/apitests/python/test_proxy_cache.py @@ -0,0 +1,134 @@ +from __future__ import absolute_import + + +import unittest +import urllib +import sys + +from testutils import ADMIN_CLIENT, suppress_urllib3_warning, DOCKER_USER, DOCKER_PWD +from testutils import harbor_server +from testutils import TEARDOWN +from library.base import _random_name +from library.project import Project +from library.user import User +from library.repository import Repository +from library.registry import Registry +from library.repository import pull_harbor_image +from library.artifact import Artifact +import library.containerd + +class TestProxyCache(unittest.TestCase): + @suppress_urllib3_warning + def setUp(self): + self.url = ADMIN_CLIENT["endpoint"] + self.user_password = "Aa123456" + self.project= Project() + self.user= User() + self.repo= Repository() + self.registry = Registry() + self.artifact = Artifact() + + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def tearDown(self): + print("Case completed") + + def do_validate(self, registry_type): + """ + Test case: + Proxy Cache Image From Harbor + Test step and expected result: + 1. Create a new registry; + 2. Create a new project; + 3. Add a new user as a member of project; + 4. Pull image from this project by docker CLI; + 5. Pull image from this project by ctr CLI; + 6. Pull manifest index from this project by docker CLI; + 7. Pull manifest from this project by ctr CLI; + 8. Image pulled by docker CLI should be cached; + 9. Image pulled by ctr CLI should be cached; + 10. Manifest index pulled by docker CLI should be cached; + 11. Manifest index pulled by ctr CLI should be cached; + Tear down: + 1. Delete project(PA); + 2. Delete user(UA). + """ + user_id, user_name = self.user.create_user(user_password = self.user_password, **ADMIN_CLIENT) + USER_CLIENT=dict(with_signature = True, endpoint = self.url, username = user_name, password = self.user_password) + + image_for_docker = dict(image = "for_proxy", tag = "1.0") + image_for_ctr = dict(image = "redis", tag = "latest") + index_for_docker = dict(image = "index081597864867", tag = "index_tag081597864867") + access_key = "" + access_secret = "" + + #1. Create a new registry; + if registry_type == "docker-hub": + user_namespace = DOCKER_USER + access_key = user_namespace + access_secret = DOCKER_PWD + registry = "https://hub.docker.com" + # Memo: ctr will not send image pull request if manifest list already exist, so we pull different manifest list for different registry; + index_for_ctr = dict(image = "alpine", tag = "3.12.0") + else: + user_namespace = "nightly" + registry = "https://cicd.harbor.vmwarecna.net" + index_for_ctr = dict(image = "busybox", tag = "1.32.0") + + registry_id, _ = self.registry.create_registry(registry, name=_random_name(registry_type), registry_type=registry_type, access_key = access_key, access_secret = access_secret, insecure=False, **ADMIN_CLIENT) + + print("registry_id:", registry_id) + + #2. Create a new project; + project_id, project_name = self.project.create_project(registry_id = registry_id, metadata = {"public": "false"}, **ADMIN_CLIENT) + print("project_id:",project_id) + print("project_name:",project_name) + + #3. Add a new user as a member of project; + self.project.add_project_members(project_id, user_id=user_id, **ADMIN_CLIENT) + + #4. Pull image from this project by docker CLI; + pull_harbor_image(harbor_server, USER_CLIENT["username"], USER_CLIENT["password"], project_name + "/" + user_namespace + "/" + image_for_docker["image"], image_for_docker["tag"]) + + #5. Pull image from this project by ctr CLI; + oci_ref = harbor_server + "/" + project_name + "/" + user_namespace + "/" + image_for_ctr["image"] + ":" + image_for_ctr["tag"] + library.containerd.ctr_images_pull(user_name, self.user_password, oci_ref) + library.containerd.ctr_images_list(oci_ref = oci_ref) + + #6. Pull manifest index from this project by docker CLI; + index_repo_name = user_namespace + "/" + index_for_docker["image"] + pull_harbor_image(harbor_server, user_name, self.user_password, project_name + "/" + index_repo_name, index_for_docker["tag"]) + + #7. Pull manifest from this project by ctr CLI; + index_repo_name_for_ctr = user_namespace + "/" + index_for_ctr["image"] + oci_ref = harbor_server + "/" + project_name + "/" + index_repo_name_for_ctr + ":" + index_for_ctr["tag"] + library.containerd.ctr_images_pull(user_name, self.user_password, oci_ref) + library.containerd.ctr_images_list(oci_ref = oci_ref) + + #8. Image pulled by docker CLI should be cached; + self.artifact.waiting_for_reference_exist(project_name, urllib.parse.quote(user_namespace + "/" + image_for_docker["image"],'utf-8'), image_for_docker["tag"], **USER_CLIENT) + + #9. Image pulled by ctr CLI should be cached; + self.artifact.waiting_for_reference_exist(project_name, urllib.parse.quote(user_namespace + "/" + image_for_ctr["image"],'utf-8'), image_for_ctr["tag"], **USER_CLIENT) + + #10. Manifest index pulled by docker CLI should be cached; + ret_index_by_d = self.artifact.waiting_for_reference_exist(project_name, urllib.parse.quote(index_repo_name,'utf-8'), index_for_docker["tag"], **USER_CLIENT) + print("Index's reference by docker CLI:", ret_index_by_d.references) + self.assertTrue(len(ret_index_by_d.references) == 1) + + #11. Manifest index pulled by ctr CLI should be cached; + ret_index_by_c = self.artifact.waiting_for_reference_exist(project_name, urllib.parse.quote(index_repo_name_for_ctr,'utf-8'), index_for_ctr["tag"], **USER_CLIENT) + print("Index's reference by ctr CLI:", ret_index_by_c.references) + self.assertTrue(len(ret_index_by_c.references) == 1) + + def test_proxy_cache_from_harbor(self): + self.do_validate("harbor") + + #def test_proxy_cache_from_dockerhub(self): + # self.do_validate("docker-hub") + +if __name__ == '__main__': + suite = unittest.TestSuite(unittest.makeSuite(TestProxyCache)) + result = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=True).run(suite) + if not result.wasSuccessful(): + raise Exception(r"Proxy cache test failed: {}".format(result)) + diff --git a/tests/apitests/python/test_push_chart_by_helm2_helm3_with_robot_Account.py b/tests/apitests/python/test_push_chart_by_helm2_helm3_with_robot_Account.py index a05c451aa..684c9a4dc 100644 --- a/tests/apitests/python/test_push_chart_by_helm2_helm3_with_robot_Account.py +++ b/tests/apitests/python/test_push_chart_by_helm2_helm3_with_robot_Account.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import unittest -from testutils import ADMIN_CLIENT, CHART_API_CLIENT +from testutils import ADMIN_CLIENT, CHART_API_CLIENT, suppress_urllib3_warning from testutils import harbor_server from testutils import TEARDOWN import library.repository @@ -13,8 +13,8 @@ from library.user import User from library.chart import Chart class TestProjects(unittest.TestCase): - @classmethod - def setUpClass(self): + @suppress_urllib3_warning + def setUp(self): self.project= Project() self.user= User() self.chart= Chart() @@ -28,12 +28,8 @@ class TestProjects(unittest.TestCase): self.chart_repo_name = "chart_local" self.repo_name = "harbor_api_test" - @classmethod - def tearDownClass(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete user(UA). self.user.delete_user(TestProjects.user_id, **ADMIN_CLIENT) diff --git a/tests/apitests/python/test_push_chart_by_helm3_chart_cli.py b/tests/apitests/python/test_push_chart_by_helm3_chart_cli.py index 593053810..ca39937d3 100644 --- a/tests/apitests/python/test_push_chart_by_helm3_chart_cli.py +++ b/tests/apitests/python/test_push_chart_by_helm3_chart_cli.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import unittest -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import harbor_server from testutils import TEARDOWN import library.repository @@ -14,8 +14,8 @@ from library.repository import Repository from library.artifact import Artifact class TestProjects(unittest.TestCase): - @classmethod - def setUpClass(self): + @suppress_urllib3_warning + def setUp(self): self.project= Project() self.user= User() self.artifact = Artifact() @@ -27,12 +27,8 @@ class TestProjects(unittest.TestCase): self.verion = "0.2.0" self.repo_name = "harbor_api_test" - @classmethod - def tearDownClass(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete repository chart(CA) by user(UA); self.repo.delete_repoitory(TestProjects.project_push_chart_name, self.repo_name, **TestProjects.USER_CLIENT) @@ -76,8 +72,8 @@ class TestProjects(unittest.TestCase): #5.1 Get chart(CA) by reference successfully; artifact = self.artifact.get_reference_info(TestProjects.project_push_chart_name, self.repo_name, self.verion, **TestProjects.USER_CLIENT) - self.assertEqual(artifact[0].type, 'CHART') - self.assertEqual(artifact[0].tags[0].name, self.verion) + self.assertEqual(artifact.type, 'CHART') + self.assertEqual(artifact.tags[0].name, self.verion) #5.2 Chart bundle can be pulled by ctr successfully; #oci_ref = harbor_server+"/"+TestProjects.project_push_chart_name+"/"+self.repo_name+":"+self.verion diff --git a/tests/apitests/python/test_push_cnab_bundle.py b/tests/apitests/python/test_push_cnab_bundle.py index 38ae6f2e3..86854394b 100644 --- a/tests/apitests/python/test_push_cnab_bundle.py +++ b/tests/apitests/python/test_push_cnab_bundle.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import unittest -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import harbor_server from testutils import TEARDOWN import library.repository @@ -12,11 +12,10 @@ from library.project import Project from library.user import User from library.repository import Repository from library.artifact import Artifact -from library.docker_api import DockerAPI class TestProjects(unittest.TestCase): - @classmethod - def setUpClass(self): + @suppress_urllib3_warning + def setUp(self): self.project= Project() self.user= User() self.artifact = Artifact() @@ -26,12 +25,8 @@ class TestProjects(unittest.TestCase): self.cnab_repo_name = "test_cnab" self.cnab_tag = "test_cnab_tag" - @classmethod - def tearDownClass(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete repository(RA) by user(UA); self.repo.delete_repoitory(TestProjects.project_push_bundle_name, self.cnab_repo_name, **TestProjects.USER_CLIENT) @@ -48,12 +43,11 @@ class TestProjects(unittest.TestCase): Test step and expected result: 1. Create a new user(UA); 2. Create a new project(PA) by user(UA); - 3. Pull images for bundle; - 4. Push bundle to harbor as repository(RA); - 5. Get repository from Harbor successfully; - 6. Verfiy bundle name; - 7. Get artifact by sha256; - 8. Verify artifact information. + 3. Push bundle to harbor as repository(RA); + 4. Get repository from Harbor successfully; + 5. Verfiy bundle name; + 6. Get artifact by sha256; + 7. Verify artifact information. Tear down: 1. Delete repository(RA) by user(UA); 2. Delete project(PA); @@ -67,33 +61,28 @@ class TestProjects(unittest.TestCase): #2. Create a new project(PA) by user(UA); TestProjects.project_push_bundle_id, TestProjects.project_push_bundle_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_CLIENT) - #3. Pull images for bundle; - _docker_api = DockerAPI() - _docker_api.docker_image_pull("alpine", tag = "latest") - _docker_api.docker_image_pull("haproxy", tag = "latest") - - #4. Push bundle to harbor as repository(RA); + #3. Push bundle to harbor as repository(RA); target = harbor_server + "/" + TestProjects.project_push_bundle_name + "/" + self.cnab_repo_name + ":" + self.cnab_tag reference_sha256 = library.cnab.push_cnab_bundle(harbor_server, user_name, self.user_push_cnab_password, "alpine:latest", "haproxy:latest", target) - #5. Get repository from Harbor successfully; + #4. Get repository from Harbor successfully; index_data = self.repo.get_repository(TestProjects.project_push_bundle_name, self.cnab_repo_name, **TestProjects.USER_CLIENT) - #5.2 Cnab bundle can be pulled by ctr successfully; + #4.2 Cnab bundle can be pulled by ctr successfully; # This step might not successful since ctr does't support cnab fully, it might be uncomment sometime in future. # Please keep them in comment! #library.containerd.ctr_images_pull(user_name, self.user_push_cnab_password, target) #library.containerd.ctr_images_list(oci_ref = target) - #6. Verfiy bundle name; + #5. Verfiy bundle name; self.assertEqual(index_data.name, TestProjects.project_push_bundle_name + "/" + self.cnab_repo_name) - #7. Get artifact by sha256; + #6. Get artifact by sha256; artifact = self.artifact.get_reference_info(TestProjects.project_push_bundle_name, self.cnab_repo_name, reference_sha256, **TestProjects.USER_CLIENT) - #8. Verify artifact information; - self.assertEqual(artifact[0].type, 'CNAB') - self.assertEqual(artifact[0].digest, reference_sha256) + #7. Verify artifact information; + self.assertEqual(artifact.type, 'CNAB') + self.assertEqual(artifact.digest, reference_sha256) if __name__ == '__main__': unittest.main() diff --git a/tests/apitests/python/test_push_files_by_oras.py b/tests/apitests/python/test_push_files_by_oras.py index 0f004e6ea..c8d3d04ad 100644 --- a/tests/apitests/python/test_push_files_by_oras.py +++ b/tests/apitests/python/test_push_files_by_oras.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import unittest import urllib -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import harbor_server from testutils import TEARDOWN import library.oras @@ -14,7 +14,7 @@ from library.artifact import Artifact class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.project = Project() self.user = User() @@ -23,12 +23,8 @@ class TestProjects(unittest.TestCase): self.repo_name = "hello-artifact" self.tag = "test_v2" - @classmethod - def tearDown(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete user(UA); self.user.delete_user(TestProjects.user_sign_image_id, **ADMIN_CLIENT) @@ -67,7 +63,7 @@ class TestProjects(unittest.TestCase): #5. Get and verify artifacts by tag; artifact = self.artifact.get_reference_info(TestProjects.project_name, self.repo_name, self.tag, **TestProjects.USER_CLIENT) - self.assertEqual(artifact[0].tags[0].name, self.tag) + self.assertEqual(artifact.tags[0].name, self.tag) #6. ORAS CLI pull artifacts index by tag; md5_list_pull = library.oras.oras_pull(harbor_server, user_name, user_001_password, TestProjects.project_name, self.repo_name, self.tag) diff --git a/tests/apitests/python/test_push_image_with_special_name.py b/tests/apitests/python/test_push_image_with_special_name.py index 93c683e34..f7c6a1ce0 100644 --- a/tests/apitests/python/test_push_image_with_special_name.py +++ b/tests/apitests/python/test_push_image_with_special_name.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import unittest import urllib -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import harbor_server from testutils import TEARDOWN from library.sign import sign_image @@ -13,19 +13,15 @@ from library.repository import Repository from library.repository import push_image_to_project class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.project = Project() self.user = User() self.artifact = Artifact() self.repo = Repository() - @classmethod - def tearDown(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): # remove the deletion as the signed image cannot be deleted. #1. Delete repository(RA) by user(UA); #self.repo.delete_repoitory(TestProjects.project_sign_image_name, TestProjects.repo_name.split('/')[1], **TestProjects.USER_sign_image_CLIENT) @@ -80,7 +76,7 @@ class TestProjects(unittest.TestCase): full_name = urllib.parse.quote(profix+"/"+image,'utf-8') artifact = self.artifact.get_reference_info(TestProjects.project_sign_image_name, full_name, tag, **TestProjects.USER_sign_image_CLIENT) - self.assertEqual(artifact[0].type, 'IMAGE') + self.assertEqual(artifact.type, 'IMAGE') if __name__ == '__main__': unittest.main() diff --git a/tests/apitests/python/test_push_index_by_docker_manifest.py b/tests/apitests/python/test_push_index_by_docker_manifest.py index e1d51a55b..3a66a1ee0 100644 --- a/tests/apitests/python/test_push_index_by_docker_manifest.py +++ b/tests/apitests/python/test_push_index_by_docker_manifest.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import unittest -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import harbor_server from testutils import TEARDOWN import library.repository @@ -18,8 +18,8 @@ from library.repository import push_image_to_project from library.repository import pull_harbor_image class TestProjects(unittest.TestCase): - @classmethod - def setUpClass(self): + @suppress_urllib3_warning + def setUp(self): self.project= Project() self.user= User() self.artifact = Artifact() @@ -31,12 +31,8 @@ class TestProjects(unittest.TestCase): self.image_a = "alpine" self.image_b = "busybox" - @classmethod - def tearDownClass(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete repository(RA,RB,IA) by user(UA); self.repo.delete_repoitory(TestProjects.project_push_index_name, self.index_name, **TestProjects.USER_CLIENT) self.repo.delete_repoitory(TestProjects.project_push_index_name, self.image_a, **TestProjects.USER_CLIENT) @@ -95,10 +91,11 @@ class TestProjects(unittest.TestCase): #6. Get index(IA) by reference successfully; index_data = self.artifact.get_reference_info(TestProjects.project_push_index_name, self.index_name, self.index_tag, **TestProjects.USER_CLIENT) - manifests_sha256_harbor_ret = [index_data[0].references[1].child_digest, index_data[0].references[0].child_digest] + print("===========index_data:",index_data) + manifests_sha256_harbor_ret = [index_data.references[1].child_digest, index_data.references[0].child_digest] #7. Verify harbor index is index(IA) pushed by docker manifest CLI; - self.assertEqual(index_data[0].digest, index_sha256_cli_ret) + self.assertEqual(index_data.digest, index_sha256_cli_ret) self.assertEqual(manifests_sha256_harbor_ret.count(manifests_sha256_cli_ret[0]), 1) self.assertEqual(manifests_sha256_harbor_ret.count(manifests_sha256_cli_ret[1]), 1) diff --git a/tests/apitests/python/test_push_sif_by_singularity.py b/tests/apitests/python/test_push_sif_by_singularity.py index cc2107a77..1be1eda79 100644 --- a/tests/apitests/python/test_push_sif_by_singularity.py +++ b/tests/apitests/python/test_push_sif_by_singularity.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import unittest import urllib -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import harbor_server from testutils import TEARDOWN import library.singularity @@ -14,7 +14,7 @@ from library.artifact import Artifact class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.project = Project() self.user = User() @@ -23,12 +23,8 @@ class TestProjects(unittest.TestCase): self.repo_name = "busybox" self.tag = "1.28" - @classmethod - def tearDown(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete user(UA); self.user.delete_user(TestProjects.user_sign_image_id, **ADMIN_CLIENT) @@ -67,7 +63,7 @@ class TestProjects(unittest.TestCase): #5. Get and verify artifacts by tag; artifact = self.artifact.get_reference_info(TestProjects.project_name, self.repo_name, self.tag, **TestProjects.USER_CLIENT) - self.assertEqual(artifact[0].tags[0].name, self.tag) + self.assertEqual(artifact.tags[0].name, self.tag) #6. Pull sif file from harbor by singularity; library.singularity.singularity_pull(TestProjects.project_name + ".sif", "oras://"+harbor_server + "/" + TestProjects.project_name + "/" + self.repo_name+":"+ self.tag) diff --git a/tests/apitests/python/test_registry_api.py b/tests/apitests/python/test_registry_api.py index b4573c207..95043c85c 100644 --- a/tests/apitests/python/test_registry_api.py +++ b/tests/apitests/python/test_registry_api.py @@ -1,7 +1,7 @@ from __future__ import absolute_import import unittest -from testutils import harbor_server +from testutils import harbor_server, suppress_urllib3_warning from testutils import admin_user from testutils import admin_pwd from testutils import TEARDOWN @@ -20,7 +20,7 @@ import library.base import json class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.system = System() self.project= Project() @@ -29,12 +29,8 @@ class TestProjects(unittest.TestCase): self.repo = Repository() self.repo_name = "hello-world" - @classmethod - def tearDown(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete Alice's repository and Luca's repository; self.repo.delete_repoitory(TestProjects.project_Alice_name, TestProjects.repo_a.split('/')[1], **ADMIN_CLIENT) self.repo.delete_repoitory(TestProjects.project_Alice_name, TestProjects.repo_b.split('/')[1], **ADMIN_CLIENT) diff --git a/tests/apitests/python/test_replication_from_dockerhub.py b/tests/apitests/python/test_replication_from_dockerhub.py index 29fd6c2c5..de088de5d 100644 --- a/tests/apitests/python/test_replication_from_dockerhub.py +++ b/tests/apitests/python/test_replication_from_dockerhub.py @@ -1,7 +1,7 @@ from __future__ import absolute_import import unittest -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import TEARDOWN from library.project import Project from library.user import User @@ -10,9 +10,10 @@ from library.registry import Registry from library.artifact import Artifact from library.repository import Repository import swagger_client +from testutils import DOCKER_USER, DOCKER_PWD class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.project = Project() self.user = User() @@ -23,12 +24,8 @@ class TestProjects(unittest.TestCase): self.image = "alpine" self.tag = "latest" - @classmethod - def tearDown(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete rule(RA); self.replication.delete_replication_rule(TestProjects.rule_id, **ADMIN_CLIENT) @@ -79,7 +76,7 @@ class TestProjects(unittest.TestCase): expected_project_id = TestProjects.project_add_rule_id, **TestProjects.USER_add_rule_CLIENT) #3. Create a new registry; - TestProjects.registry_id, _ = self.registry.create_registry("https://hub.docker.com", registry_type="docker-hub", access_key = "", access_secret = "", insecure=False, **ADMIN_CLIENT) + TestProjects.registry_id, _ = self.registry.create_registry("https://hub.docker.com", registry_type="docker-hub", access_key = DOCKER_USER, access_secret = DOCKER_PWD, insecure=False, **ADMIN_CLIENT) #4. Create a pull-based rule for this registry; TestProjects.rule_id, rule_name = self.replication.create_replication_policy(src_registry=swagger_client.Registry(id=int(TestProjects.registry_id)), @@ -94,7 +91,7 @@ class TestProjects(unittest.TestCase): self.replication.trigger_replication_executions(TestProjects.rule_id, **ADMIN_CLIENT) #7. Wait for completion of this replication job; - self.replication.wait_until_jobs_finish(TestProjects.rule_id, **ADMIN_CLIENT) + self.replication.wait_until_jobs_finish(TestProjects.rule_id,interval=30, **ADMIN_CLIENT) #8. Check image is replicated into target project successfully. artifact = self.artifact.get_reference_info(TestProjects.project_name, self.image, self.tag, **ADMIN_CLIENT) diff --git a/tests/apitests/python/test_retention.py b/tests/apitests/python/test_retention.py index 4e9a491a9..7409d00e3 100644 --- a/tests/apitests/python/test_retention.py +++ b/tests/apitests/python/test_retention.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import unittest import time -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import TEARDOWN from testutils import harbor_server from library.repository import push_special_image_to_project @@ -31,8 +31,8 @@ class TestProjects(unittest.TestCase): Tear Down: 1. Delete project test-retention """ - @classmethod - def setUpClass(self): + @suppress_urllib3_warning + def setUp(self): self.user = User() self.system = System() self.repo = Repository() @@ -42,6 +42,16 @@ class TestProjects(unittest.TestCase): self.repo_name_1 = "test1" self.repo_name_2 = "test2" + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def tearDown(self): + # TODO delete_repoitory will fail when no tags left anymore + # resp=self.repo.list_repositories(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT) + # for repo in resp: + # self.repo.delete_repoitory(repo.name, **TestProjects.USER_RA_CLIENT) + # self.project.delete_project(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT) + # self.user.delete_user(TestProjects.user_ra_id, **ADMIN_CLIENT) + print("Case completed") + def testTagRetention(self): user_ra_password = "Aa123456" user_ra_id, user_ra_name = self.user.create_user(user_password=user_ra_password, **ADMIN_CLIENT) @@ -112,26 +122,12 @@ class TestProjects(unittest.TestCase): #List artifacts successfully, and untagged artifact in test1 should be the only one retained; artifacts_1 = self.artifact.list_artifacts(TestProjects.project_src_repo_name, self.repo_name_1, **TestProjects.USER_RA_CLIENT) self.assertTrue(len(artifacts_1)==1) - self.assertEqual(artifacts_1[0].digest, tag_data_artifact3_image1[0].digest) + self.assertEqual(artifacts_1[0].digest, tag_data_artifact3_image1.digest) #List artifacts successfully, and artifact with latest tag in test2 should be the only one retained; artifacts_2 = self.artifact.list_artifacts(TestProjects.project_src_repo_name, self.repo_name_2, **TestProjects.USER_RA_CLIENT) self.assertTrue(len(artifacts_2)==1) - self.assertEqual(artifacts_2[0].digest, tag_data_artifact2_image2[0].digest) - - @classmethod - def tearDownClass(self): - print("Case completed") - - # TODO delete_repoitory will fail when no tags left anymore - # @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - # def test_ClearData(self): - # resp=self.repo.list_repositories(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT) - # for repo in resp: - # self.repo.delete_repoitory(repo.name, **TestProjects.USER_RA_CLIENT) - # self.project.delete_project(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT) - # self.user.delete_user(TestProjects.user_ra_id, **ADMIN_CLIENT) - + self.assertEqual(artifacts_2[0].digest, tag_data_artifact2_image2.digest) if __name__ == '__main__': unittest.main() diff --git a/tests/apitests/python/test_robot_account.py b/tests/apitests/python/test_robot_account.py index bbb9924f5..81aa2391d 100644 --- a/tests/apitests/python/test_robot_account.py +++ b/tests/apitests/python/test_robot_account.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import unittest -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import TEARDOWN from testutils import harbor_server from library.user import User @@ -13,13 +13,13 @@ from library.repository import push_image_to_project from library.base import _assert_status_code class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.project = Project() self.user = User() self.repo = Repository() - @classmethod + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") def tearDown(self): print("Case completed") diff --git a/tests/apitests/python/test_scan_image_artifact.py b/tests/apitests/python/test_scan_image_artifact.py index 15169c2a5..ce6e5ce47 100644 --- a/tests/apitests/python/test_scan_image_artifact.py +++ b/tests/apitests/python/test_scan_image_artifact.py @@ -1,7 +1,8 @@ from __future__ import absolute_import import unittest +import sys -from testutils import harbor_server +from testutils import harbor_server, suppress_urllib3_warning from testutils import TEARDOWN from testutils import ADMIN_CLIENT from library.project import Project @@ -10,31 +11,41 @@ from library.repository import Repository from library.repository import push_image_to_project from library.artifact import Artifact from library.scan import Scan -from library.scanner import Scanner -class TestProjects(unittest.TestCase): - @classmethod +from library.sign import sign_image + +class TestScan(unittest.TestCase): + @suppress_urllib3_warning def setUp(self): self.project= Project() self.user= User() self.artifact = Artifact() self.repo = Repository() self.scan = Scan() - self.scanner = Scanner() - @classmethod - def tearDown(self): - print("Case completed") + self.url = ADMIN_CLIENT["endpoint"] + self.user_password = "Aa123456" + self.project_id, self.project_name, self.user_id, self.user_name, self.repo_name1, self.repo_name2 = [None] * 6 + self.user_id, self.user_name = self.user.create_user(user_password = self.user_password, **ADMIN_CLIENT) + self.USER_CLIENT = dict(with_signature = True, with_immutable_status = True, endpoint = self.url, username = self.user_name, password = self.user_password, with_scan_overview = True) + + + #2. Create a new private project(PA) by user(UA); + self.project_id, self.project_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT) + + #3. Add user(UA) as a member of project(PA) with project-admin role; + self.project.add_project_members(self.project_id, user_id = self.user_id, **ADMIN_CLIENT) @unittest.skipIf(TEARDOWN == True, "Test data won't be erased.") - def test_ClearData(self): + def do_tearDown(self): #1. Delete repository(RA) by user(UA); - self.repo.delete_repoitory(TestProjects.project_scan_image_name, TestProjects.repo_name.split('/')[1], **TestProjects.USER_SCAN_IMAGE_CLIENT) + self.repo.delete_repoitory(self.project_name, self.repo_name1.split('/')[1], **self.USER_CLIENT) + self.repo.delete_repoitory(self.project_name, self.repo_name2.split('/')[1], **self.USER_CLIENT) #2. Delete project(PA); - self.project.delete_project(TestProjects.project_scan_image_id, **TestProjects.USER_SCAN_IMAGE_CLIENT) + self.project.delete_project(self.project_id, **self.USER_CLIENT) #3. Delete user(UA); - self.user.delete_user(TestProjects.user_scan_image_id, **ADMIN_CLIENT) + self.user.delete_user(self.user_id, **ADMIN_CLIENT) def testScanImageArtifact(self): """ @@ -54,45 +65,58 @@ class TestProjects(unittest.TestCase): 2. Delete project(PA); 3. Delete user(UA); """ - url = ADMIN_CLIENT["endpoint"] - user_001_password = "Aa123456" - - #1. Create user-001 - TestProjects.user_scan_image_id, user_scan_image_name = self.user.create_user(user_password = user_001_password, **ADMIN_CLIENT) - - TestProjects.USER_SCAN_IMAGE_CLIENT=dict(endpoint = url, username = user_scan_image_name, password = user_001_password, with_scan_overview = True) - - #2. Create a new private project(PA) by user(UA); - TestProjects.project_scan_image_id, TestProjects.project_scan_image_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT) - - #3. Add user(UA) as a member of project(PA) with project-admin role; - self.project.add_project_members(TestProjects.project_scan_image_id, user_id=TestProjects.user_scan_image_id, **ADMIN_CLIENT) #4. Get private project of user(UA), user(UA) can see only one private project which is project(PA); self.project.projects_should_exist(dict(public=False), expected_count = 1, - expected_project_id = TestProjects.project_scan_image_id, **TestProjects.USER_SCAN_IMAGE_CLIENT) + expected_project_id = self.project_id, **self.USER_CLIENT) #Note: Please make sure that this Image has never been pulled before by any other cases, # so it is a not-scanned image right after repository creation. image = "docker" src_tag = "1.13" #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); - TestProjects.repo_name, tag = push_image_to_project(TestProjects.project_scan_image_name, harbor_server, user_scan_image_name, user_001_password, image, src_tag) + self.repo_name1, tag = push_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image, src_tag) #6. Send scan image command and get tag(TA) information to check scan result, it should be finished; - self.scan.scan_artifact(TestProjects.project_scan_image_name, TestProjects.repo_name.split('/')[1], tag, **TestProjects.USER_SCAN_IMAGE_CLIENT) - self.artifact.check_image_scan_result(TestProjects.project_scan_image_name, image, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT) + self.scan.scan_artifact(self.project_name, self.repo_name1.split('/')[1], tag, **self.USER_CLIENT) + self.artifact.check_image_scan_result(self.project_name, image, tag, **self.USER_CLIENT) - #7. Swith Scanner; - uuid = self.scanner.scanners_get_uuid(**ADMIN_CLIENT) - self.scanner.scanners_registration_id_patch(uuid, **ADMIN_CLIENT) + self.do_tearDown() - image = "tomcat" - src_tag = "latest" - TestProjects.repo_name, tag = push_image_to_project(TestProjects.project_scan_image_name, harbor_server, user_scan_image_name, user_001_password, image, src_tag) - #8. Send scan another image command and get tag(TA) information to check scan result, it should be finished. - self.scan.scan_artifact(TestProjects.project_scan_image_name, TestProjects.repo_name.split('/')[1], tag, **TestProjects.USER_SCAN_IMAGE_CLIENT) - self.artifact.check_image_scan_result(TestProjects.project_scan_image_name, image, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT) + def testScanSignedImage(self): + """ + Test case: + Scan A Signed Image + Test step and expected result: + 1. Create a new user(UA); + 2. Create a new private project(PA) by user(UA); + 3. Add user(UA) as a member of project(PA) with project-admin role; + 4. Get private project of user(UA), user(UA) can see only one private project which is project(PA); + 5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); + 6. Send scan image command and get tag(TA) information to check scan result, it should be finished; + 7. Swith Scanner; + 8. Send scan another image command and get tag(TA) information to check scan result, it should be finished. + Tear down: + 1. Delete repository(RA) by user(UA); + 2. Delete project(PA); + 3. Delete user(UA); + """ + + #Note: Please make sure that this Image has never been pulled before by any other cases, + # so it is a not-scanned image right after repository creation. + image = "redis" + tag = "latest" + #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); + TestScan.repo_name_1, tag = push_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image, tag) + + sign_image(harbor_server, self.project_name, image, tag) + + #6. Send scan image command and get tag(TA) information to check scan result, it should be finished; + self.scan.scan_artifact(self.project_name, TestScan.repo_name_1.split('/')[1], tag, **self.USER_CLIENT) + self.artifact.check_image_scan_result(self.project_name, image, tag, **self.USER_CLIENT) if __name__ == '__main__': - unittest.main() \ No newline at end of file + suite = unittest.TestSuite(unittest.makeSuite(TestScan)) + result = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=True).run(suite) + if not result.wasSuccessful(): + raise Exception(r"Tag immutability test failed: {}".format(result)) \ No newline at end of file diff --git a/tests/apitests/python/test_scan_image_artifact_in_public_project.py b/tests/apitests/python/test_scan_image_artifact_in_public_project.py index 81a5e4dfd..74f72eb18 100644 --- a/tests/apitests/python/test_scan_image_artifact_in_public_project.py +++ b/tests/apitests/python/test_scan_image_artifact_in_public_project.py @@ -1,7 +1,7 @@ from __future__ import absolute_import import unittest -from testutils import harbor_server +from testutils import harbor_server, TEARDOWN, suppress_urllib3_warning from testutils import created_user, created_project from library.artifact import Artifact from library.repository import Repository, push_image_to_project @@ -9,13 +9,13 @@ from library.scan import Scan class TestScanImageInPublicProject(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.artifact = Artifact() self.repo = Repository() self.scan = Scan() - @classmethod + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") def tearDown(self): print("Case completed") diff --git a/tests/apitests/python/test_sign_image.py b/tests/apitests/python/test_sign_image.py index 95f5b1aa8..8fa410eb4 100644 --- a/tests/apitests/python/test_sign_image.py +++ b/tests/apitests/python/test_sign_image.py @@ -1,7 +1,7 @@ from __future__ import absolute_import import unittest -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import harbor_server from testutils import TEARDOWN from library.sign import sign_image @@ -13,7 +13,7 @@ from library.repository import push_image_to_project from library.repository import push_special_image_to_project class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): self.project = Project() self.user = User() @@ -21,12 +21,8 @@ class TestProjects(unittest.TestCase): self.repo = Repository() self.repo_name_1 = "test1_sign" - @classmethod - def tearDown(self): - print("Case completed") - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): # remove the deletion as the signed image cannot be deleted. #1. Delete repository(RA) by user(UA); #self.repo.delete_repoitory(TestProjects.project_sign_image_name, TestProjects.repo_name.split('/')[1], **TestProjects.USER_sign_image_CLIENT) @@ -80,14 +76,12 @@ class TestProjects(unittest.TestCase): #7. Get signature of image with tag(TA), it should be exist. artifact = self.artifact.get_reference_info(TestProjects.project_sign_image_name, image, tag, **TestProjects.USER_sign_image_CLIENT) - self.assertEqual(artifact[0].tags[0].signed, True) + self.assertEqual(artifact.tags[0].signed, True) push_special_image_to_project(TestProjects.project_sign_image_name, harbor_server, user_sign_image_name, user_001_password, self.repo_name_1, ['1.0']) self.repo.delete_repoitory(TestProjects.project_sign_image_name, self.repo_name_1, **TestProjects.USER_sign_image_CLIENT) - ret = self.repo.delete_repoitory(TestProjects.project_sign_image_name, TestProjects.repo_name.split('/')[1], expect_status_code=412, **TestProjects.USER_sign_image_CLIENT) - self.assertIn("with signature cannot be deleted", ret) - + self.repo.delete_repoitory(TestProjects.project_sign_image_name, TestProjects.repo_name.split('/')[1], expect_status_code=412, expect_response_body = "with signature cannot be deleted", **TestProjects.USER_sign_image_CLIENT) if __name__ == '__main__': unittest.main() diff --git a/tests/apitests/python/test_sys_cve_allowlists.py b/tests/apitests/python/test_sys_cve_allowlists.py index ef0bae891..06093984b 100644 --- a/tests/apitests/python/test_sys_cve_allowlists.py +++ b/tests/apitests/python/test_sys_cve_allowlists.py @@ -4,7 +4,7 @@ import unittest import swagger_client import time -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, TEARDOWN, suppress_urllib3_warning from library.user import User from library.system import System @@ -27,6 +27,7 @@ class TestSysCVEAllowlist(unittest.TestCase): 1. Clear the system level CVE allowlist. 2. Delete User(RA) """ + @suppress_urllib3_warning def setUp(self): self.user = User() self.system = System() @@ -39,6 +40,13 @@ class TestSysCVEAllowlist(unittest.TestCase): password=user_ra_password) self.user_ra_id = int(user_ra_id) + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def tearDown(self): + print("TearDown: Clearing the Allowlist") + self.system.set_cve_allowlist(**ADMIN_CLIENT) + print("TearDown: Deleting user: %d" % self.user_ra_id) + self.user.delete_user(self.user_ra_id, **ADMIN_CLIENT) + def testSysCVEAllowlist(self): # 1. User(RA) reads the system level CVE allowlist and it's empty. wl = self.system.get_cve_allowlist(**self.USER_RA_CLIENT) @@ -62,12 +70,5 @@ class TestSysCVEAllowlist(unittest.TestCase): wl = self.system.get_cve_allowlist(**self.USER_RA_CLIENT) self.assertEqual(exp, wl.expires_at) - def tearDown(self): - print("TearDown: Clearing the Allowlist") - self.system.set_cve_allowlist(**ADMIN_CLIENT) - print("TearDown: Deleting user: %d" % self.user_ra_id) - self.user.delete_user(self.user_ra_id, **ADMIN_CLIENT) - - if __name__ == '__main__': unittest.main() diff --git a/tests/apitests/python/test_system_level_scan_all.py b/tests/apitests/python/test_system_level_scan_all.py index 030b30440..2c0efc6f0 100644 --- a/tests/apitests/python/test_system_level_scan_all.py +++ b/tests/apitests/python/test_system_level_scan_all.py @@ -1,7 +1,7 @@ from __future__ import absolute_import import unittest -from testutils import harbor_server +from testutils import harbor_server, suppress_urllib3_warning from testutils import TEARDOWN from testutils import ADMIN_CLIENT from library.system import System @@ -10,35 +10,29 @@ from library.user import User from library.repository import Repository from library.repository import push_image_to_project from library.artifact import Artifact -from library.scanner import Scanner - -class TestProjects(unittest.TestCase): - @classmethod +class TestScanAll(unittest.TestCase): + @suppress_urllib3_warning def setUp(self): self.system = System() self.project= Project() self.user= User() self.artifact = Artifact() self.repo = Repository() - self.scanner = Scanner() - - @classmethod - def tearDown(self): - print("Case completed") @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): #1. Delete Alice's repository and Luca's repository; - self.repo.delete_repoitory(TestProjects.project_Alice_name, TestProjects.repo_Alice_name.split('/')[1], **ADMIN_CLIENT) - self.repo.delete_repoitory(TestProjects.project_Luca_name, TestProjects.repo_Luca_name.split('/')[1], **ADMIN_CLIENT) + self.repo.delete_repoitory(TestScanAll.project_Alice_name, TestScanAll.repo_Alice_name.split('/')[1], **ADMIN_CLIENT) + self.repo.delete_repoitory(TestScanAll.project_Luca_name, TestScanAll.repo_Luca_name.split('/')[1], **ADMIN_CLIENT) #2. Delete Alice's project and Luca's project; - self.project.delete_project(TestProjects.project_Alice_id, **ADMIN_CLIENT) - self.project.delete_project(TestProjects.project_Luca_id, **ADMIN_CLIENT) + self.project.delete_project(TestScanAll.project_Alice_id, **ADMIN_CLIENT) + self.project.delete_project(TestScanAll.project_Luca_id, **ADMIN_CLIENT) #3. Delete user Alice and Luca. - self.user.delete_user(TestProjects.user_Alice_id, **ADMIN_CLIENT) - self.user.delete_user(TestProjects.user_Luca_id, **ADMIN_CLIENT) + self.user.delete_user(TestScanAll.user_Alice_id, **ADMIN_CLIENT) + self.user.delete_user(TestScanAll.user_Luca_id, **ADMIN_CLIENT) + print("Case completed") def testSystemLevelScanALL(self): """ @@ -59,15 +53,15 @@ class TestProjects(unittest.TestCase): user_common_password = "Aa123456" #1. Create user Alice and Luca; - TestProjects.user_Alice_id, user_Alice_name = self.user.create_user(user_password = user_common_password, **ADMIN_CLIENT) - TestProjects.user_Luca_id, user_Luca_name = self.user.create_user(user_password = user_common_password, **ADMIN_CLIENT) + TestScanAll.user_Alice_id, user_Alice_name = self.user.create_user(user_password = user_common_password, **ADMIN_CLIENT) + TestScanAll.user_Luca_id, user_Luca_name = self.user.create_user(user_password = user_common_password, **ADMIN_CLIENT) USER_ALICE_CLIENT=dict(endpoint = url, username = user_Alice_name, password = user_common_password, with_scan_overview = True) USER_LUCA_CLIENT=dict(endpoint = url, username = user_Luca_name, password = user_common_password, with_scan_overview = True) #2. Create 2 new private projects project_Alice and project_Luca; - TestProjects.project_Alice_id, TestProjects.project_Alice_name = self.project.create_project(metadata = {"public": "false"}, **USER_ALICE_CLIENT) - TestProjects.project_Luca_id, TestProjects.project_Luca_name = self.project.create_project(metadata = {"public": "false"}, **USER_LUCA_CLIENT) + TestScanAll.project_Alice_id, TestScanAll.project_Alice_name = self.project.create_project(metadata = {"public": "false"}, **USER_ALICE_CLIENT) + TestScanAll.project_Luca_id, TestScanAll.project_Luca_name = self.project.create_project(metadata = {"public": "false"}, **USER_LUCA_CLIENT) #3. Push a image to project_Alice and push another image to project_Luca; @@ -77,33 +71,21 @@ class TestProjects(unittest.TestCase): image_a = "mariadb" src_tag = "latest" #3.1 Push a image to project_Alice; - TestProjects.repo_Alice_name, tag_Alice = push_image_to_project(TestProjects.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_a, src_tag) + TestScanAll.repo_Alice_name, tag_Alice = push_image_to_project(TestScanAll.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_a, src_tag) #Note: Please make sure that this Image has never been pulled before by any other cases, # so it is a not-scanned image rigth after repository creation. image_b = "httpd" src_tag = "latest" #3.2 push another image to project_Luca; - TestProjects.repo_Luca_name, tag_Luca = push_image_to_project(TestProjects.project_Luca_name, harbor_server, user_Luca_name, user_common_password, image_b, src_tag) + TestScanAll.repo_Luca_name, tag_Luca = push_image_to_project(TestScanAll.project_Luca_name, harbor_server, user_Luca_name, user_common_password, image_b, src_tag) #4. Trigger scan all event; self.system.scan_now(**ADMIN_CLIENT) #5. Check if image in project_Alice and another image in project_Luca were both scanned. - self.artifact.check_image_scan_result(TestProjects.project_Alice_name, image_a, tag_Alice, **USER_ALICE_CLIENT) - self.artifact.check_image_scan_result(TestProjects.project_Luca_name, image_b, tag_Luca, **USER_LUCA_CLIENT) - - #6. Swith Scanner; - uuid = self.scanner.scanners_get_uuid(**ADMIN_CLIENT) - self.scanner.scanners_registration_id_patch(uuid, **ADMIN_CLIENT) - - #7. Trigger scan all event; - self.system.scan_now(**ADMIN_CLIENT) - - #8. Check if image in project_Alice and another image in project_Luca were both scanned. - self.artifact.check_image_scan_result(TestProjects.project_Alice_name, image_a, tag_Alice, **USER_ALICE_CLIENT) - self.artifact.check_image_scan_result(TestProjects.project_Luca_name, image_b, tag_Luca, **USER_LUCA_CLIENT) - + self.artifact.check_image_scan_result(TestScanAll.project_Alice_name, image_a, tag_Alice, **USER_ALICE_CLIENT) + self.artifact.check_image_scan_result(TestScanAll.project_Luca_name, image_b, tag_Luca, **USER_LUCA_CLIENT) if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/apitests/python/test_tag_crud.py b/tests/apitests/python/test_tag_crud.py new file mode 100644 index 000000000..c68729b2a --- /dev/null +++ b/tests/apitests/python/test_tag_crud.py @@ -0,0 +1,91 @@ +from __future__ import absolute_import + + +import unittest + +from testutils import ADMIN_CLIENT, suppress_urllib3_warning +from testutils import harbor_server +from testutils import TEARDOWN +from library.project import Project +from library.user import User +from library.repository import Repository +from library.artifact import Artifact +from library.repository import push_image_to_project + +class TestProjects(unittest.TestCase): + @suppress_urllib3_warning + def setUp(self): + self.project= Project() + self.user= User() + self.artifact = Artifact() + self.repo= Repository() + self.url = ADMIN_CLIENT["endpoint"] + self.user_password = "Aa123456" + self.repo_name = "hello-world" + + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def tearDown(self): + #1. Delete repository(RA,IA) by user(UA); + self.repo.delete_repoitory(TestProjects.project_name, self.repo_name, **TestProjects.USER_CLIENT) + + #2. Delete project(PA); + self.project.delete_project(TestProjects.project_id, **TestProjects.USER_CLIENT) + + #3. Delete user(UA). + self.user.delete_user(TestProjects.user_id, **ADMIN_CLIENT) + print("Case completed") + + def testCreateDeleteTag(self): + """ + Test case: + Create/Delete tag + Test step and expected result: + 1. Create a new user(UA); + 2. Create a new project(PA) by user(UA); + 3. Push an image(IA) to Harbor by docker successfully; + 4. Create a tag(1.0) for the image(IA); + 5. Get the image(latest) from Harbor successfully; + 6. Verify the image(IA) contains tag named 1.0; + 7. Delete the tag(1.0) from image(IA); + 8. Get the image(IA) from Harbor successfully; + 9. Verify the image(IA) contains no tag named 1.0; + Tear down: + 1. Delete repository(RA,IA) by user(UA); + 2. Delete project(PA); + 3. Delete user(UA). + """ + #1. Create a new user(UA); + TestProjects.user_id, user_name = self.user.create_user(user_password = self.user_password, **ADMIN_CLIENT) + + TestProjects.USER_CLIENT=dict(with_tag = True, endpoint = self.url, username = user_name, password = self.user_password) + + #2. Create a new project(PA) by user(UA); + TestProjects.project_id, TestProjects.project_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_CLIENT) + + #3. Push an image(IA) to Harbor by docker successfully; + _, tag = push_image_to_project(TestProjects.project_name, harbor_server, 'admin', 'Harbor12345', self.repo_name, "latest") + + #4. Create a tag(1.0) for the image(IA) + self.artifact.create_tag(TestProjects.project_name, self.repo_name, tag, "1.0",**TestProjects.USER_CLIENT) + + #5. Get the image(IA) from Harbor successfully; + artifact = self.artifact.get_reference_info(TestProjects.project_name, self.repo_name, tag, **TestProjects.USER_CLIENT) + + #6. Verify the image(IA) contains tag named 1.0; + self.assertEqual(artifact.tags[0].name, "1.0") + self.assertEqual(artifact.tags[1].name, tag) + + #7. Delete the tag(1.0) from image(IA); + self.artifact.delete_tag(TestProjects.project_name, self.repo_name, tag, "1.0",**TestProjects.USER_CLIENT) + + #8. Get the image(latest) from Harbor successfully; + artifact = self.artifact.get_reference_info(TestProjects.project_name, self.repo_name, tag, **TestProjects.USER_CLIENT) + + #9. Verify the image(IA) contains no tag named 1.0; + self.assertEqual(artifact.tags[0].name, tag) + + + +if __name__ == '__main__': + unittest.main() + diff --git a/tests/apitests/python/test_tag_immutability.py b/tests/apitests/python/test_tag_immutability.py index e69de29bb..df3cbf13e 100644 --- a/tests/apitests/python/test_tag_immutability.py +++ b/tests/apitests/python/test_tag_immutability.py @@ -0,0 +1,300 @@ +from __future__ import absolute_import + + +import unittest +import sys + +from testutils import ADMIN_CLIENT, TEARDOWN, suppress_urllib3_warning +from testutils import harbor_server +from library.project import Project +from library.user import User +from library.repository import Repository +from library.repository import push_image_to_project +from library.registry import Registry +from library.artifact import Artifact +from library.tag_immutability import Tag_Immutability +from library.repository import push_special_image_to_project + +class TestTagImmutability(unittest.TestCase): + @suppress_urllib3_warning + def setUp(self): + self.url = ADMIN_CLIENT["endpoint"] + self.user_password = "Aa123456" + self.project= Project() + self.user= User() + self.repo= Repository() + self.registry = Registry() + self.artifact = Artifact() + self.tag_immutability = Tag_Immutability() + self.project_id, self.project_name, self.user_id, self.user_name = [None] * 4 + self.user_id, self.user_name = self.user.create_user(user_password = self.user_password, **ADMIN_CLIENT) + self.USER_CLIENT = dict(with_signature = True, with_immutable_status = True, endpoint = self.url, username = self.user_name, password = self.user_password) + self.exsiting_rule = dict(selector_repository="rel*", selector_tag="v2.*") + self.project_id, self.project_name = self.project.create_project(metadata = {"public": "false"}, **self.USER_CLIENT) + + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def tearDown(self): + print("Case completed") + + def check_tag_immutability(self, artifact, tag_name, status = True): + for tag in artifact.tags: + if tag.name == tag_name: + self.assertTrue(tag.immutable == status) + return + raise Exception("No tag {} found in artifact {}".format(tag, artifact)) + + def test_disability_of_rules(self): + """ + Test case: + Test Disability Of Rules + Test step and expected result: + 1. Create a new project; + 2. Push image A to the project with 2 tags A and B; + 3. Create a disabled rule matched image A with tag A; + 4. Both tags of image A should not be immutable; + 5. Enable this rule; + 6. image A with tag A should be immutable. + """ + image_a = dict(name="image_disability_a", tag1="latest", tag2="6.2.2") + + #1. Create a new project; + project_id, project_name = self.project.create_project(metadata = {"public": "false"}, **self.USER_CLIENT) + + #2. Push image A to the project with 2 tags; + push_special_image_to_project(project_name, harbor_server, self.user_name, self.user_password, image_a["name"], [image_a["tag1"], image_a["tag2"]]) + + #3. Create a disabled rule matched image A; + rule_id = self.tag_immutability.create_rule(project_id, disabled = True, selector_repository=image_a["name"], selector_tag=str(image_a["tag1"])[0:2] + "*", **self.USER_CLIENT) + + #4. Both tags of image A should not be immutable; + artifact_a = self.artifact.get_reference_info(project_name, image_a["name"], image_a["tag2"], **self.USER_CLIENT) + print("[test_disability_of_rules] - artifact:{}".format(artifact_a)) + self.assertTrue(artifact_a) + self.check_tag_immutability(artifact_a, image_a["tag1"], status = False) + self.check_tag_immutability(artifact_a, image_a["tag2"], status = False) + + #5. Enable this rule; + self.tag_immutability.update_tag_immutability_policy_rule(project_id, rule_id, disabled = False, **self.USER_CLIENT) + + #6. image A with tag A should be immutable. + artifact_a = self.artifact.get_reference_info(project_name, image_a["name"], image_a["tag2"], **self.USER_CLIENT) + print("[test_disability_of_rules] - artifact:{}".format(artifact_a)) + self.assertTrue(artifact_a) + self.check_tag_immutability(artifact_a, image_a["tag1"], status = True) + self.check_tag_immutability(artifact_a, image_a["tag2"], status = False) + + def test_artifact_and_repo_is_undeletable(self): + """ + Test case: + Test Artifact And Repo is Undeleteable + Test step and expected result: + 1. Create a new project; + 2. Push image A to the project with 2 tags A and B; + 3. Create a enabled rule matched image A with tag A; + 4. Tag A should be immutable; + 5. Artifact is undeletable; + 6. Repository is undeletable. + """ + image_a = dict(name="image_repo_undeletable_a", tag1="latest", tag2="1.3.2") + + #1. Create a new project; + project_id, project_name = self.project.create_project(metadata = {"public": "false"}, **self.USER_CLIENT) + + #2. Push image A to the project with 2 tags A and B; + push_special_image_to_project(project_name, harbor_server, self.user_name, self.user_password, image_a["name"], [image_a["tag1"], image_a["tag2"]]) + + #3. Create a enabled rule matched image A with tag A; + self.tag_immutability.create_rule(project_id, selector_repository=image_a["name"], selector_tag=str(image_a["tag1"])[0:2] + "*", **self.USER_CLIENT) + + #4. Tag A should be immutable; + artifact_a = self.artifact.get_reference_info(project_name, image_a["name"], image_a["tag2"], **self.USER_CLIENT) + print("[test_artifact_and_repo_is_undeletable] - artifact:{}".format(artifact_a)) + self.assertTrue(artifact_a) + self.check_tag_immutability(artifact_a, image_a["tag1"], status = True) + self.check_tag_immutability(artifact_a, image_a["tag2"], status = False) + + #5. Artifact is undeletable; + self.artifact.delete_artifact(project_name, image_a["name"], image_a["tag1"], expect_status_code = 412,expect_response_body = "configured as immutable, cannot be deleted", **self.USER_CLIENT) + + #6. Repository is undeletable. + self.repo.delete_repoitory(project_name, image_a["name"], expect_status_code = 412, expect_response_body = "configured as immutable, cannot be deleted", **self.USER_CLIENT) + + def test_tag_is_undeletable(self): + """ + Test case: + Test Tag is Undeleteable + Test step and expected result: + 1. Push image A to the project with 2 tags A and B; + 2. Create a enabled rule matched image A with tag A; + 3. Tag A should be immutable; + 4. Tag A is undeletable; + 5. Tag B is deletable. + """ + image_a = dict(name="image_undeletable_a", tag1="latest", tag2="9.3.2") + + #1. Push image A to the project with 2 tags A and B; + push_special_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image_a["name"], [image_a["tag1"], image_a["tag2"]]) + + #2. Create a enabled rule matched image A with tag A; + self.tag_immutability.create_rule(self.project_id, selector_repository=image_a["name"], selector_tag=str(image_a["tag2"])[0:2] + "*", **self.USER_CLIENT) + + #3. Tag A should be immutable; + artifact_a = self.artifact.get_reference_info(self.project_name, image_a["name"], image_a["tag2"], **self.USER_CLIENT) + print("[test_tag_is_undeletable] - artifact:{}".format(artifact_a)) + self.assertTrue(artifact_a) + self.check_tag_immutability(artifact_a, image_a["tag2"], status = True) + + #4. Tag A is undeletable; + self.artifact.delete_tag(self.project_name, image_a["name"], image_a["tag1"], image_a["tag2"], expect_status_code = 412, **self.USER_CLIENT) + + #5. Tag B is deletable. + self.artifact.delete_tag(self.project_name, image_a["name"], image_a["tag1"], image_a["tag1"], **self.USER_CLIENT) + + def test_image_is_unpushable(self): + """ + Test case: + Test Image is Unpushable + Test step and expected result: + 1. Create a new project; + 2. Push image A to the project with 2 tags A and B; + 3. Create a enabled rule matched image A with tag A; + 4. Tag A should be immutable; + 5. Can not push image with the same image name and with the same tag name. + """ + image_a = dict(name="image_unpushable_a", tag1="latest", tag2="1.3.2") + + #1. Create a new project; + project_id, project_name = self.project.create_project(metadata = {"public": "false"}, **self.USER_CLIENT) + + #2. Push image A to the project with 2 tags A and B; + push_special_image_to_project(project_name, harbor_server, self.user_name, self.user_password, image_a["name"], [image_a["tag1"], image_a["tag2"]]) + + #3. Create a enabled rule matched image A with tag A; + self.tag_immutability.create_rule(project_id, selector_repository=image_a["name"], selector_tag=str(image_a["tag1"])[0:2] + "*", **self.USER_CLIENT) + + #4. Tag A should be immutable; + artifact_a = self.artifact.get_reference_info(project_name, image_a["name"], image_a["tag2"], **self.USER_CLIENT) + print("[test_image_is_unpushable] - artifact:{}".format(artifact_a)) + self.assertTrue(artifact_a) + self.check_tag_immutability(artifact_a, image_a["tag1"], status = True) + self.check_tag_immutability(artifact_a, image_a["tag2"], status = False) + + #5. Can not push image with the same image name and with the same tag name. + push_image_to_project(project_name, harbor_server, self.user_name, self.user_password, "tomcat", image_a["tag1"], + new_image = image_a["name"], expected_error_message = "configured as immutable") + + def test_copy_disability(self): + """ + Test case: + Test Copy Disability + Test step and expected result: + 1. Create 2 projects; + 2. Push image A with tag A and B to project A, push image B which has the same image name and tag name to project B; + 3. Create a enabled rule matched image A with tag A; + 4. Tag A should be immutable; + 5. Can not copy artifact from project A to project B with the same repository name. + """ + image_a = dict(name="image_copy_disability_a", tag1="latest", tag2="1.3.2") + + #1. Create 2 projects; + project_id, project_name = self.project.create_project(metadata = {"public": "false"}, **self.USER_CLIENT) + _, project_name_src = self.project.create_project(metadata = {"public": "false"}, **self.USER_CLIENT) + + #2. Push image A with tag A and B to project A, push image B which has the same image name and tag name to project B; + push_special_image_to_project(project_name, harbor_server, self.user_name, self.user_password, image_a["name"], [image_a["tag1"], image_a["tag2"]]) + push_special_image_to_project(project_name_src, harbor_server, self.user_name, self.user_password, image_a["name"], [image_a["tag1"], image_a["tag2"]]) + + #3. Create a enabled rule matched image A with tag A; + self.tag_immutability.create_rule(project_id, selector_repository=image_a["name"], selector_tag=str(image_a["tag1"])[0:2] + "*", **self.USER_CLIENT) + + #4. Tag A should be immutable; + artifact_a = self.artifact.get_reference_info(project_name, image_a["name"], image_a["tag2"], **self.USER_CLIENT) + print("[test_copy_disability] - artifact:{}".format(artifact_a)) + self.assertTrue(artifact_a) + self.check_tag_immutability(artifact_a, image_a["tag1"], status = True) + self.check_tag_immutability(artifact_a, image_a["tag2"], status = False) + + #5. Can not copy artifact from project A to project B with the same repository name. + artifact_a_src = self.artifact.get_reference_info(project_name_src, image_a["name"], image_a["tag2"], **self.USER_CLIENT) + print("[test_copy_disability] - artifact_a_src:{}".format(artifact_a_src)) + self.artifact.copy_artifact(project_name, image_a["name"], project_name_src+"/"+ image_a["name"] + "@" + artifact_a_src.digest, expect_status_code=412, expect_response_body = "configured as immutable, cannot be updated", **self.USER_CLIENT) + + #def test_replication_disability(self): + # pass + + def test_priority_of_rules(self): + """ + Test case: + Test Priority Of Rules(excluding rule will not affect matching rule) + Test step and expected result: + 1. Push image A, B and C, image A has only 1 tag named tag1; + 2. Create a matching rule that matches image A and tag named tag2 which is not exist; + 3. Create a excluding rule to exlude image A and B; + 4. Add a tag named tag2 to image A, tag2 should be immutable; + 5. Tag2 should be immutable; + 6. All tags in image B should be immutable; + 7. All tags in image C should not be immutable; + 8. Disable all rules. + """ + image_a = dict(name="image_priority_a", tag1="latest", tag2="6.3.2") + image_b = dict(name="image_priority_b", tag1="latest", tag2="0.12.0") + image_c = dict(name="image_priority_c", tag1="latest", tag2="3.12.0") + + #1. Push image A, B and C, image A has only 1 tag named tag1; + push_special_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image_a["name"], [image_a["tag1"]]) + push_special_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image_b["name"], [image_b["tag1"],image_b["tag2"]]) + push_special_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image_c["name"], [image_c["tag1"],image_c["tag2"]]) + + #2. Create a matching rule that matches image A and tag named tag2 which is not exist; + rule_id_1 = self.tag_immutability.create_rule(self.project_id, selector_repository=image_a["name"], selector_tag=image_a["tag2"], **self.USER_CLIENT) + + #3. Create a excluding rule to exlude image A and B; + rule_id_2 = self.tag_immutability.create_rule(self.project_id, selector_repository_decoration = "repoExcludes", + selector_repository="{image_priority_a,image_priority_b}", selector_tag="**", **self.USER_CLIENT) + + #4. Add a tag named tag2 to image A, tag2 should be immutable; + self.artifact.create_tag(self.project_name, image_a["name"], image_a["tag1"], image_a["tag2"], **self.USER_CLIENT) + + #5. Tag2 should be immutable; + artifact_a = self.artifact.get_reference_info(self.project_name, image_a["name"], image_a["tag2"], **self.USER_CLIENT) + print("[test_priority_of_rules] - artifact:{}".format(artifact_a)) + self.assertTrue(artifact_a) + self.check_tag_immutability(artifact_a, image_a["tag2"], status = True) + self.check_tag_immutability(artifact_a, image_a["tag1"], status = False) + + #6. All tags in image B should be immutable; + artifact_b = self.artifact.get_reference_info(self.project_name, image_b["name"], image_b["tag2"], **self.USER_CLIENT) + print("[test_priority_of_rules] - artifact:{}".format(artifact_b)) + self.assertTrue(artifact_b) + self.check_tag_immutability(artifact_b, image_b["tag2"], status = False) + self.check_tag_immutability(artifact_b, image_b["tag1"], status = False) + + #7. All tags in image C should not be immutable; + artifact_c = self.artifact.get_reference_info(self.project_name, image_c["name"], image_c["tag2"], **self.USER_CLIENT) + print("[test_priority_of_rules] - artifact:{}".format(artifact_c)) + self.assertTrue(artifact_c) + self.check_tag_immutability(artifact_c, image_c["tag2"], status = True) + self.check_tag_immutability(artifact_c, image_c["tag1"], status = True) + + #8. Disable all rules. + self.tag_immutability.update_tag_immutability_policy_rule(self.project_id, rule_id_1, disabled = True, **self.USER_CLIENT) + self.tag_immutability.update_tag_immutability_policy_rule(self.project_id, rule_id_2, disabled = True, **self.USER_CLIENT) + + def test_add_exsiting_rule(self): + """ + Test case: + Test Priority Of Rules(excluding rule will not affect matching rule) + Test step and expected result: + 1. Push image A and B with no tag; + 2. Create a immutability policy rule A; + 3. Fail to create rule B which has the same config as rule A; + """ + self.tag_immutability.create_tag_immutability_policy_rule(self.project_id, **self.exsiting_rule, **self.USER_CLIENT) + self.tag_immutability.create_tag_immutability_policy_rule(self.project_id, **self.exsiting_rule, expect_status_code = 409, **self.USER_CLIENT) + +if __name__ == '__main__': + suite = unittest.TestSuite(unittest.makeSuite(TestTagImmutability)) + result = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=True).run(suite) + if not result.wasSuccessful(): + raise Exception(r"Tag immutability test failed: {}".format(result)) + diff --git a/tests/apitests/python/test_user_view_logs.py b/tests/apitests/python/test_user_view_logs.py index d25bc5628..64f09807d 100644 --- a/tests/apitests/python/test_user_view_logs.py +++ b/tests/apitests/python/test_user_view_logs.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import unittest import time -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import TEARDOWN from testutils import TestResult from library.user import User @@ -14,30 +14,16 @@ from library.repository import push_image_to_project from testutils import harbor_server class TestProjects(unittest.TestCase): - @classmethod + @suppress_urllib3_warning def setUp(self): - test_result = TestResult() - self.test_result= test_result - - project = Project() - self.project= project - - user = User() - self.user= user - - repo = Repository() - self.repo= repo - - projectv2 = ProjectV2() - self.projectv2= projectv2 - - @classmethod - def tearDown(self): - self.test_result.get_final_result() - print("Case completed") + self.project= Project() + self.user= User() + self.repo= Repository() + self.projectv2= ProjectV2() @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): + def tearDown(self): + print("Case completed") #1. Delete project(PA); self.project.delete_project(TestProjects.project_user_view_logs_id, **TestProjects.USER_USER_VIEW_LOGS_CLIENT) @@ -57,6 +43,7 @@ class TestProjects(unittest.TestCase): 1. Delete project(PA); 2. Delete user(UA). """ + test_result= TestResult() url = ADMIN_CLIENT["endpoint"] admin_name = ADMIN_CLIENT["username"] admin_password = ADMIN_CLIENT["password"] @@ -75,7 +62,7 @@ class TestProjects(unittest.TestCase): operation = "create" log_count = self.projectv2.filter_project_logs(project_user_view_logs_name, user_user_view_logs_name, project_user_view_logs_name, "project", operation, **TestProjects.USER_USER_VIEW_LOGS_CLIENT) if log_count != 1: - self.test_result.add_test_result("1 - Failed to get log with user:{}, resource:{}, resource_type:{} and operation:{}, expect count 1, but actual is {}.". + test_result.add_test_result("1 - Failed to get log with user:{}, resource:{}, resource_type:{} and operation:{}, expect count 1, but actual is {}.". format(user_user_view_logs_name, project_user_view_logs_name, "project", operation, log_count)) #3.1 Push a new image(IA) in project(PA) by admin; @@ -86,7 +73,7 @@ class TestProjects(unittest.TestCase): operation = "create" log_count = self.projectv2.filter_project_logs(project_user_view_logs_name, admin_name, r'{}:{}'.format(repo_name, tag), "artifact", operation, **TestProjects.USER_USER_VIEW_LOGS_CLIENT) if log_count != 1: - self.test_result.add_test_result("2 - Failed to get log with user:{}, resource:{}, resource_type:{} and operation:{}, expect count 1, but actual is {}.". + test_result.add_test_result("2 - Failed to get log with user:{}, resource:{}, resource_type:{} and operation:{}, expect count 1, but actual is {}.". format(user_user_view_logs_name, project_user_view_logs_name, "artifact", operation, log_count)) #4.1 Delete repository(RA) by user(UA); self.repo.delete_repoitory(project_user_view_logs_name, repo_name.split('/')[1], **TestProjects.USER_USER_VIEW_LOGS_CLIENT) @@ -96,8 +83,10 @@ class TestProjects(unittest.TestCase): operation = "delete" log_count = self.projectv2.filter_project_logs(project_user_view_logs_name, user_user_view_logs_name, repo_name, "repository", operation, **TestProjects.USER_USER_VIEW_LOGS_CLIENT) if log_count != 1: - self.test_result.add_test_result("5 - Failed to get log with user:{}, resource:{}, resource_type:{} and operation:{}, expect count 1, but actual is {}.". + test_result.add_test_result("5 - Failed to get log with user:{}, resource:{}, resource_type:{} and operation:{}, expect count 1, but actual is {}.". format(user_user_view_logs_name, project_user_view_logs_name, "repository", operation, log_count)) + test_result.get_final_result() + if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/apitests/python/testutils.py b/tests/apitests/python/testutils.py index b3a56b9f9..10ed9f9d3 100644 --- a/tests/apitests/python/testutils.py +++ b/tests/apitests/python/testutils.py @@ -1,15 +1,19 @@ import time import os import sys +import warnings +from functools import wraps -sys.path.insert(0, os.environ["SWAGGER_CLIENT_PATH"]) +sys.path.insert(0, os.environ.get("SWAGGER_CLIENT_PATH", '')) path=os.getcwd() + "/library" sys.path.insert(0, path) path=os.getcwd() + "/tests/apitests/python/library" sys.path.insert(0, path) - +path=os.getcwd() + "/tests/apitests/python/" +sys.path.insert(0, path) +print(sys.path) import v2_swagger_client from swagger_client.rest import ApiException import swagger_client.models @@ -18,15 +22,17 @@ from pprint import pprint admin_user = "admin" admin_pwd = "Harbor12345" -harbor_server = os.environ["HARBOR_HOST"] +harbor_server = os.environ.get("HARBOR_HOST", '') #CLIENT=dict(endpoint="https://"+harbor_server+"/api") ADMIN_CLIENT=dict(endpoint = os.environ.get("HARBOR_HOST_SCHEMA", "https")+ "://"+harbor_server+"/api/v2.0", username = admin_user, password = admin_pwd) CHART_API_CLIENT=dict(endpoint = os.environ.get("HARBOR_HOST_SCHEMA", "https")+ "://"+harbor_server+"/api", username = admin_user, password = admin_pwd) USER_ROLE=dict(admin=0,normal=1) TEARDOWN = os.environ.get('TEARDOWN', 'true').lower() in ('true', 'yes') notary_url = os.environ.get('NOTARY_URL', 'https://'+harbor_server+':4443') +DOCKER_USER = os.environ.get('DOCKER_USER', '') +DOCKER_PWD = os.environ.get('DOCKER_PWD', '') -def GetProductApi(username, password, harbor_server= os.environ["HARBOR_HOST"]): +def GetProductApi(username, password, harbor_server= os.environ.get("HARBOR_HOST", '')): cfg = swagger_client.Configuration() cfg.host = "https://"+harbor_server+"/api/v2.0" @@ -38,7 +44,7 @@ def GetProductApi(username, password, harbor_server= os.environ["HARBOR_HOST"]): api_instance = swagger_client.ProductsApi(api_client) return api_instance -def GetRepositoryApi(username, password, harbor_server= os.environ["HARBOR_HOST"]): +def GetRepositoryApi(username, password, harbor_server= os.environ.get("HARBOR_HOST", '')): cfg = v2_swagger_client.Configuration() cfg.host = "https://"+harbor_server+"/api/v2.0" @@ -63,6 +69,16 @@ class TestResult(object): print("Error message:", each_err_msg) raise Exception(r"Test case failed with {} errors.".format(self.num_errors)) +def suppress_urllib3_warning(func): + @wraps(func) + def inner_func(*args): + warnings.filterwarnings(action="ignore", + message="unclosed", + category=ResourceWarning) + warnings.filterwarnings(action='ignore', message='Unverified HTTPS request') + func(*args) + return inner_func + from contextlib import contextmanager @contextmanager diff --git a/tests/apitests/python/update_docker_cfg.sh b/tests/apitests/python/update_docker_cfg.sh index 0ab4968ed..2e26e9007 100755 --- a/tests/apitests/python/update_docker_cfg.sh +++ b/tests/apitests/python/update_docker_cfg.sh @@ -1,5 +1,7 @@ #!/bin/sh -sudo sed -i '$d' /$HOME/.docker/config.json -sudo sed -i '$d' /$HOME/.docker/config.json -sudo echo -e "\n },\n \"experimental\": \"enabled\"\n}" >> /$HOME/.docker/config.json +if [ $(cat /$HOME/.docker/config.json |grep experimental |wc -l) -eq 0 ];then + sudo sed -i '$d' /$HOME/.docker/config.json + sudo sed -i '$d' /$HOME/.docker/config.json + sudo echo -e "},\n \"experimental\": \"enabled\"\n}" >> /$HOME/.docker/config.json +fi diff --git a/tests/ci/api_common_install.sh b/tests/ci/api_common_install.sh index a54c8c156..a894949ca 100755 --- a/tests/ci/api_common_install.sh +++ b/tests/ci/api_common_install.sh @@ -10,26 +10,13 @@ set -e if [ -z "$1" ]; then echo no ip specified; exit 1;fi # prepare cert ... sudo ./tests/generateCerts.sh $1 -sudo mkdir -p /etc/docker/certs.d/$1 && sudo cp ./tests/harbor_ca.crt /etc/docker/certs.d/$1/ && rm -rf ~/.docker/ && mkdir -p ~/.docker/tls/$1:4443/ && sudo cp ./tests/harbor_ca.crt ~/.docker/tls/$1:4443/ - +sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests --upgrade sudo ./tests/hostcfg.sh if [ "$2" = 'LDAP' ]; then cd tests && sudo ./ldapprepare.sh && cd .. fi -# prepare a chart file for API_DB test... -sudo curl -o $DIR/../../tests/apitests/python/mariadb-4.3.1.tgz https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz - -sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests && sudo pip install robotframework==3.2.1 robotframework-httplibrary requests --upgrade -sudo make swagger_client -#TODO: Swagger python package used to installed into dist-packages, but it's changed into site-packages all in a sudden, we havn't found the root cause. -# so current workround is to copy swagger packages from site-packages to dist-packages. -package_dir=/usr/lib/python3.7/site-packages -if [ -d $package_dir ] && [ $(find $package_dir -type f -name "*client*.egg" | wc -l) -gt 0 ];then - sudo cp -rf ${package_dir}/* /usr/local/lib/python3.7/dist-packages -fi - if [ $GITHUB_TOKEN ]; then sed "s/# github_token: xxx/github_token: $GITHUB_TOKEN/" -i make/harbor.yml diff --git a/tests/ci/api_run.sh b/tests/ci/api_run.sh index 33ad774bf..67d753715 100755 --- a/tests/ci/api_run.sh +++ b/tests/ci/api_run.sh @@ -18,6 +18,7 @@ harbor_logs_bucket="harbor-ci-logs" #echo "content_language = en" >> $botofile #echo "default_project_id = $GS_PROJECT_ID" >> $botofile DIR="$(cd "$(dirname "$0")" && pwd)" +E2E_IMAGE="goharbor/harbor-e2e-engine:2.6" # GS util function uploader { @@ -30,7 +31,7 @@ set +e docker ps # run db auth api cases if [ "$1" = 'DB' ]; then - robot -X -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_DB.robot + docker run -i --privileged -v $DIR/../../:/drone -v $DIR/../:/ca -w /drone $E2E_IMAGE robot -v DOCKER_USER:${DOCKER_USER} -v DOCKER_PWD:${DOCKER_PWD} -v ip:$2 -v ip1: -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot /drone/tests/robot-cases/Group0-BAT/API_DB.robot elif [ "$1" = 'LDAP' ]; then # run ldap api cases python $DIR/../../tests/configharbor.py -H $IP -u $HARBOR_ADMIN -p $HARBOR_ADMIN_PASSWD -c auth_mode=ldap_auth \ @@ -39,7 +40,7 @@ elif [ "$1" = 'LDAP' ]; then ldap_search_password=admin \ ldap_base_dn=dc=example,dc=com \ ldap_uid=cn - robot -X -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_LDAP.robot + docker run -i --privileged -v $DIR/../../:/drone -v $DIR/../:/ca -w /drone $E2E_IMAGE robot -v DOCKER_USER:${DOCKER_USER} -v DOCKER_PWD:${DOCKER_PWD} -v ip:$2 -v ip1: -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot /drone/tests/robot-cases/Group0-BAT/API_LDAP.robot else rc=999 fi diff --git a/tests/resources/APITest-Util.robot b/tests/resources/APITest-Util.robot index e057de3a3..d608276d3 100644 --- a/tests/resources/APITest-Util.robot +++ b/tests/resources/APITest-Util.robot @@ -12,6 +12,7 @@ Harbor API Test ${current_dir}= Run pwd Log To Console ${current_dir} Log To Console ${ip} - ${rc} ${output}= Run And Return Rc And Output SWAGGER_CLIENT_PATH=${current_dir}/harborclient HARBOR_HOST=${ip} python ${testcase_name} + ${rc} ${output}= Run And Return Rc And Output SWAGGER_CLIENT_PATH=${current_dir}/harborclient HARBOR_HOST=${ip} DOCKER_USER=${DOCKER_USER} DOCKER_PWD=${DOCKER_PWD} python ${testcase_name} Log To Console ${output} + Log ${output} Should Be Equal As Integers ${rc} 0 \ No newline at end of file diff --git a/tests/resources/CNAB_Util.robot b/tests/resources/CNAB_Util.robot index cb2735113..11a777c0a 100644 --- a/tests/resources/CNAB_Util.robot +++ b/tests/resources/CNAB_Util.robot @@ -19,7 +19,8 @@ Library Process *** Keywords *** CNAB Push Bundle - [Arguments] ${ip} ${user} ${pwd} ${target} ${bundle_file} - ${rc} ${output}= Run And Return Rc And Output ./tests/robot-cases/Group0-Util/cnab_push_bundle.sh ${ip} ${user} ${pwd} ${target} ${bundle_file} + [Arguments] ${ip} ${user} ${pwd} ${target} ${bundle_file} ${docker_user} ${docker_pwd} + ${rc} ${output}= Run And Return Rc And Output ./tests/robot-cases/Group0-Util/cnab_push_bundle.sh ${ip} ${user} ${pwd} ${target} ${bundle_file} ${docker_user} ${docker_pwd} + Log To Console ${output} Log ${output} Should Be Equal As Integers ${rc} 0 \ No newline at end of file diff --git a/tests/resources/Docker-Util.robot b/tests/resources/Docker-Util.robot index 78d191de2..dd84f8262 100644 --- a/tests/resources/Docker-Util.robot +++ b/tests/resources/Docker-Util.robot @@ -138,6 +138,17 @@ Start Docker Daemon Locally Sleep 2s [Return] ${handle} +Start Containerd Daemon Locally + ${handle}= Start Process containerd > ./daemon-local.log 2>&1 & shell=True + FOR ${IDX} IN RANGE 5 + ${pid}= Run pidof containerd + Log To Console pid: ${pid} + Exit For Loop If '${pid}' != '${EMPTY}' + Sleep 2s + END + Sleep 2s + [Return] ${handle} + Prepare Docker Cert [Arguments] ${ip} Wait Unitl Command Success mkdir -p /etc/docker/certs.d/${ip} @@ -145,6 +156,13 @@ Prepare Docker Cert Wait Unitl Command Success cp harbor_ca.crt /usr/local/share/ca-certificates/ Wait Unitl Command Success update-ca-certificates +Prepare Docker Cert For Nightly + [Arguments] ${ip} + Wait Unitl Command Success mkdir -p /etc/docker/certs.d/${ip} + Wait Unitl Command Success cp harbor_ca.crt /etc/docker/certs.d/${ip} + Wait Unitl Command Success cp harbor_ca.crt /usr/local/share/ca-certificates/ + Wait Unitl Command Success update-ca-certificates + Kill Local Docker Daemon [Arguments] ${handle} ${dockerd-pid} Terminate Process ${handle} @@ -164,7 +182,7 @@ Docker Login Docker Pull [Arguments] ${image} - ${output}= Retry Keyword N Times When Error 10 Wait Unitl Command Success docker pull ${image} + ${output}= Retry Keyword N Times When Error 2 Wait Unitl Command Success docker pull ${image} Log ${output} Log To Console Docker Pull: ${output} [Return] ${output} @@ -186,17 +204,22 @@ Docker Push Index Docker Image Can Not Be Pulled [Arguments] ${image} FOR ${idx} IN RANGE 0 30 + ${out}= Run Keyword And Ignore Error Docker Login "" ${DOCKER_USER} ${DOCKER_PWD} + Log To Console Return value is ${out} ${out}= Run Keyword And Ignore Error Command Should be Failed docker pull ${image} Exit For Loop If '${out[0]}'=='PASS' + Log To Console Docker pull return value is ${out} Sleep 3 END Log To Console Cannot Pull Image From Docker - Pull Log: ${out[1]} Should Be Equal As Strings '${out[0]}' 'PASS' Docker Image Can Be Pulled - [Arguments] ${image} ${period}=60 ${times}=10 + [Arguments] ${image} ${period}=60 ${times}=2 FOR ${n} IN RANGE 1 ${times} Sleep ${period} + ${out}= Run Keyword And Ignore Error Docker Login "" ${DOCKER_USER} ${DOCKER_PWD} + Log To Console Return value is ${out} ${out}= Run Keyword And Ignore Error Docker Pull ${image} Log To Console Return value is ${out[0]} Exit For Loop If '${out[0]}'=='PASS' diff --git a/tests/resources/Harbor-Pages/Configuration.robot b/tests/resources/Harbor-Pages/Configuration.robot index 87448fc8e..879d940b6 100644 --- a/tests/resources/Harbor-Pages/Configuration.robot +++ b/tests/resources/Harbor-Pages/Configuration.robot @@ -35,13 +35,11 @@ Init LDAP Sleep 1 Input Text xpath=//*[@id='ldapUid'] cn Sleep 1 - Capture Page Screenshot Disable Ldap Verify Cert Checkbox Retry Element Click xpath=${config_auth_save_button_xpath} Sleep 2 Retry Element Click xpath=/html/body/harbor-app/harbor-shell/clr-main-container/div/div/config/div/div/div/button[3] Sleep 1 - Capture Page Screenshot Switch To Configure Retry Element Click xpath=${configuration_xpath} @@ -72,7 +70,6 @@ Test Ldap Connection # ldap checkbox unchecked, click test connection to verify success. Sleep 1 Retry Element Click xpath=${test_ldap_xpath} - Capture Page Screenshot Wait Until Page Contains Connection to LDAP server is verified timeout=15 Test LDAP Server Success @@ -83,7 +80,6 @@ Disable Ldap Verify Cert Checkbox Mouse Down xpath=//*[@id='clr-checkbox-ldapVerifyCert'] Mouse Up xpath=//*[@id='clr-checkbox-ldapVerifyCert'] Sleep 2 - Capture Page Screenshot Ldap Verify Cert Checkbox Should Be Disabled Ldap Verify Cert Checkbox Should Be Disabled @@ -99,7 +95,6 @@ Set Pro Create Admin Only Retry Element Click xpath=//select[@id='proCreation']//option[@value='adminonly'] Sleep 1 Retry Element Click xpath=${config_system_save_button_xpath} - Capture Page Screenshot AdminCreateOnly.png Set Pro Create Every One Retry Element Click xpath=${configuration_xpath} @@ -112,7 +107,6 @@ Set Pro Create Every One Sleep 1 Retry Element Click xpath=${config_system_save_button_xpath} Sleep 2 - Capture Page Screenshot EveryoneCreate.png Disable Self Reg Retry Element Click xpath=${configuration_xpath} @@ -121,7 +115,6 @@ Disable Self Reg Sleep 1 Self Reg Should Be Disabled Retry Element Click xpath=${config_auth_save_button_xpath} - Capture Page Screenshot DisableSelfReg.png Sleep 1 Enable Self Reg @@ -130,7 +123,6 @@ Enable Self Reg Sleep 1 Self Reg Should Be Enabled Retry Element Click xpath=${config_auth_save_button_xpath} - Capture Page Screenshot EnableSelfReg.png Sleep 1 Self Reg Should Be Disabled @@ -183,7 +175,6 @@ Check Verify Remote Cert Mouse Down xpath=//*[@id='clr-checkbox-verifyRemoteCert'] Mouse Up xpath=//*[@id='clr-checkbox-verifyRemoteCert'] Retry Element Click xpath=${config_save_button_xpath} - Capture Page Screenshot RemoteCert.png Sleep 1 Switch To System Replication @@ -276,7 +267,6 @@ Create New Labels Sleep 1 Input Text xpath=//*[@id='description'] global Retry Element Click xpath=//div/form/section/label[4]/button[2] - Capture Page Screenshot Wait Until Page Contains ${labelname} Update A Label @@ -288,7 +278,6 @@ Update A Label Input Text xpath=//*[@id='name'] ${labelname}1 Sleep 1 Retry Element Click xpath=//hbr-create-edit-label//form/section//button[2] - Capture Page Screenshot Wait Until Page Contains ${labelname}1 Delete A Label @@ -297,7 +286,6 @@ Delete A Label Sleep 1 Retry Element Click xpath=//button[contains(.,'Delete')] Sleep 3 - Capture Page Screenshot Retry Element Click xpath=//clr-modal//div//button[contains(.,'DELETE')] Wait Until Page Contains Element //*[@id='contentAll']//div[contains(.,'${labelname}')]/../div/clr-icon[@shape='success-standard'] @@ -334,7 +322,6 @@ Delete Top Item In System CVE Allowlist Retry Element Click ${configuration_system_wl_delete_a_cve_id_icon} END Retry Element Click ${config_system_save_button_xpath} - Capture Page Screenshot Get Project Count Quota Text From Project Quotas List [Arguments] ${project_name} @@ -352,25 +339,23 @@ Get Project Storage Quota Text From Project Quotas List Check Automatic Onboarding And Save Retry Element Click ${cfg_auth_automatic_onboarding_checkbox} Retry Element Click xpath=${config_auth_save_button_xpath} - Capture Page Screenshot Set User Name Claim And Save [Arguments] ${type} Retry Text Input ${cfg_auth_user_name_claim_input} ${type} Retry Element Click xpath=${config_auth_save_button_xpath} - Capture Page Screenshot Select Distribution [Arguments] ${name} - Retry Element Click //div[@class='datagrid-scrolling-cells' and contains(.,'${name}')]//clr-checkbox-wrapper/label + Retry Element Click //clr-dg-row[contains(.,'${name}')]//clr-checkbox-wrapper/label Distribution Exist [Arguments] ${name} ${endpoint} - Retry Wait Until Page Contains Element //div[@class='datagrid-scrolling-cells' and contains(.,'${name}') and contains(.,'${endpoint}')] + Retry Wait Until Page Contains Element //clr-dg-row[contains(.,'${name}') and contains(.,'${endpoint}')] Distribution Not Exist [Arguments] ${name} ${endpoint} - Retry Wait Until Page Not Contains Element //div[@class='datagrid-scrolling-cells' and contains(.,'${name}') and contains(.,'${endpoint}')] + Retry Wait Until Page Not Contains Element //clr-dg-row[contains(.,'${name}') and contains(.,'${endpoint}')] Filter Distribution List [Arguments] ${name} ${endpoint} ${exsit}=${true} @@ -399,7 +384,7 @@ Delete A Distribution ${is_exsit} evaluate not ${deletable} Switch To Distribution Filter Distribution List ${name} ${endpoint} - Retry Double Keywords When Error Select Distribution ${name} Wait Until Element Is Visible //clr-datagrid/clr-dg-footer//clr-checkbox-wrapper/label + Retry Double Keywords When Error Select Distribution ${name} Wait Until Element Is Visible //clr-datagrid//clr-dg-footer//clr-checkbox-wrapper/label Retry Double Keywords When Error Retry Element Click ${distribution_action_btn_id} Wait Until Element Is Visible And Enabled ${distribution_del_btn_id} Retry Double Keywords When Error Retry Element Click ${distribution_del_btn_id} Wait Until Element Is Visible And Enabled ${delete_confirm_btn} Retry Double Keywords When Error Retry Element Click ${delete_confirm_btn} Retry Wait Until Page Not Contains Element ${delete_confirm_btn} @@ -410,7 +395,7 @@ Edit A Distribution [Arguments] ${name} ${endpoint} ${new_endpoint}=${null} Switch To Distribution Filter Distribution List ${name} ${endpoint} - Retry Double Keywords When Error Select Distribution ${name} Wait Until Element Is Visible //clr-datagrid/clr-dg-footer//clr-checkbox-wrapper/label + Retry Double Keywords When Error Select Distribution ${name} Wait Until Element Is Visible //clr-datagrid//clr-dg-footer//clr-checkbox-wrapper/label times=9 Retry Double Keywords When Error Retry Element Click ${distribution_action_btn_id} Wait Until Element Is Visible And Enabled ${distribution_edit_btn_id} Retry Double Keywords When Error Retry Element Click ${distribution_edit_btn_id} Wait Until Element Is Visible And Enabled ${distribution_name_input_id} Retry Text Input ${distribution_endpoint_id} ${new_endpoint} diff --git a/tests/resources/Harbor-Pages/Configuration_Elements.robot b/tests/resources/Harbor-Pages/Configuration_Elements.robot index 428f9ca0b..5928e14d4 100644 --- a/tests/resources/Harbor-Pages/Configuration_Elements.robot +++ b/tests/resources/Harbor-Pages/Configuration_Elements.robot @@ -42,6 +42,9 @@ ${checkbox_delete_untagged_artifacts} //gc-config//clr-toggle-wrapper/label[con ${cfg_auth_automatic_onboarding_checkbox} //clr-checkbox-wrapper//label[contains(@for,'oidcAutoOnboard')] ${cfg_auth_user_name_claim_input} //*[@id='oidcUserClaim'] +${cfg_auth_ldap_group_admin_dn} //*[@id='ldapGroupAdminDN'] + + ${distribution_add_btn_id} //*[@id='new-instance'] ${distribution_provider_select_id} //*[@id='provider'] ${distribution_name_input_id} //*[@id='name'] diff --git a/tests/resources/Harbor-Pages/GC.robot b/tests/resources/Harbor-Pages/GC.robot index 0d111fb95..9c3dc562c 100644 --- a/tests/resources/Harbor-Pages/GC.robot +++ b/tests/resources/Harbor-Pages/GC.robot @@ -22,9 +22,7 @@ Resource ../../resources/Util.robot GC Now [Arguments] ${harbor_url} ${login_user} ${login_pwd} ${untag}=${false} Switch To Garbage Collection - Capture Page Screenshot Run Keyword If '${untag}' == '${true}' Retry Element Click xpath=${checkbox_delete_untagged_artifacts} - Capture Page Screenshot Click GC Now Logout Harbor Sleep 2 diff --git a/tests/resources/Harbor-Pages/LDAP-Mode.robot b/tests/resources/Harbor-Pages/LDAP-Mode.robot index f75dfc24c..8fefe84a1 100644 --- a/tests/resources/Harbor-Pages/LDAP-Mode.robot +++ b/tests/resources/Harbor-Pages/LDAP-Mode.robot @@ -19,6 +19,16 @@ Resource ../../resources/Util.robot *** Variables *** *** Keywords *** +Switch To Configuration Authentication + Sleep 1 + Retry Element Click xpath=${configuration_xpath} + Retry Element Click xpath=${configuration_authentication_tabsheet_id} + +Set LDAP Group Admin DN + [Arguments] ${group_dn} + Switch To Configuration Authentication + Retry Text Input ${cfg_auth_ldap_group_admin_dn} ${group_dn} + Retry Element Click ${config_auth_save_button_xpath} Ldap User Should Not See Change Password Retry Element Click //clr-header//clr-dropdown[2]//button diff --git a/tests/resources/Harbor-Pages/Project-Helmcharts.robot b/tests/resources/Harbor-Pages/Project-Helmcharts.robot index a4bb2ac49..0a5066429 100644 --- a/tests/resources/Harbor-Pages/Project-Helmcharts.robot +++ b/tests/resources/Harbor-Pages/Project-Helmcharts.robot @@ -20,20 +20,17 @@ Upload Chart files Retry Double Keywords When Error Retry Element Click xpath=${upload_action_button} Retry Wait Until Page Not Contains Element xpath=${upload_action_button} Retry Double Keywords When Error Retry Element Click xpath=${upload_chart_button} Retry Wait Until Page Contains Element xpath=${upload_action_button} Retry Wait Until Page Contains ${prometheus_chart_name} - Capture Page Screenshot ${harbor_file_path} Set Variable ${current_dir}/${harbor_chart_filename} ${harbor_prov_file_path} Set Variable ${current_dir}/${harbor_chart_prov_filename} Choose File xpath=${chart_file_browse} ${harbor_file_path} Choose File xpath=${chart_prov_browse} ${harbor_prov_file_path} Retry Double Keywords When Error Retry Element Click xpath=${upload_action_button} Retry Wait Until Page Not Contains Element xpath=${upload_action_button} Retry Wait Until Page Contains ${harbor_chart_name} - Capture Page Screenshot Go Into Chart Version [Arguments] ${chart_name} Retry Element Click xpath=//hbr-helm-chart//a[contains(., '${chart_name}')] Sleep 3 - Capture Page Screenshot viewchartversion.png Go Into Chart Detail [Arguments] ${version_name} @@ -47,10 +44,7 @@ Multi-delete Chart Files Retry Element Click //clr-dg-row[contains(.,'${obj}')]//label END #Retry Element Click xpath=${version_checkbox} - Capture Page Screenshot Retry Double Keywords When Error Retry Element Click xpath=${version_delete} Retry Wait Until Page Contains Element ${version_confirm_delete} - Capture Page Screenshot Retry Double Keywords When Error Retry Element Click ${version_confirm_delete} Retry Wait Until Page Not Contains Element xpath=${version_confirm_delete} Retry Wait Element xpath=//clr-dg-placeholder[contains(.,\"We couldn\'t find any charts!\")] - Capture Page Screenshot diff --git a/tests/resources/Harbor-Pages/Project-P2P-Preheat.robot b/tests/resources/Harbor-Pages/Project-P2P-Preheat.robot index 1c9386c7d..9bcb7007c 100644 --- a/tests/resources/Harbor-Pages/Project-P2P-Preheat.robot +++ b/tests/resources/Harbor-Pages/Project-P2P-Preheat.robot @@ -29,16 +29,16 @@ Select Distribution For P2P Preheat Select P2P Preheat Policy [Arguments] ${name} - Retry Element Click //div[@class='datagrid-scrolling-cells' and contains(.,'${name}')]//clr-radio-wrapper/label + Retry Element Click //clr-dg-row[contains(.,'${name}')]//clr-radio-wrapper/label P2P Preheat Policy Exist [Arguments] ${name} ${repo}=${null} - ${policy_row_xpath}= Set Variable If '${repo}'=='${null}' //div[@class='datagrid-scrolling-cells' and contains(.,'${name}')] //div[@class='datagrid-scrolling-cells' and contains(.,'${name}') and contains(.,'${repo}')] + ${policy_row_xpath}= Set Variable If '${repo}'=='${null}' //clr-dg-row[contains(.,'${name}')] //clr-dg-row[contains(.,'${name}') and contains(.,'${repo}')] Retry Wait Until Page Contains Element ${policy_row_xpath} P2P Preheat Policy Not Exist [Arguments] ${name} - Retry Wait Until Page Not Contains Element //div[@class='datagrid-scrolling-cells' and contains(.,'${name}')] + Retry Wait Until Page Not Contains Element //clr-dg-row[contains(.,'${name}')] Create An New P2P Preheat Policy [Arguments] ${policy_name} ${dist_name} ${repo} ${tag} ${trigger_type}=${null} diff --git a/tests/resources/Harbor-Pages/Project-Repository.robot b/tests/resources/Harbor-Pages/Project-Repository.robot index 0f94df7f7..dddb4e4c7 100644 --- a/tests/resources/Harbor-Pages/Project-Repository.robot +++ b/tests/resources/Harbor-Pages/Project-Repository.robot @@ -20,7 +20,6 @@ Resource ../../resources/Util.robot View Repo Scan Details [Arguments] @{vulnerabilities_level} Retry Element Click xpath=${first_repo_xpath} - Capture Page Screenshot FOR ${item} IN @{vulnerabilities_level} Retry Wait Until Page Contains Element //hbr-artifact-vulnerabilities//clr-dg-row[contains(.,'${item}')] END @@ -30,6 +29,5 @@ View Repo Scan Details View Scan Error Log Retry Wait Until Page Contains View Log Retry Element Click xpath=${view_log_xpath} - Capture Page Screenshot viewlog.png diff --git a/tests/resources/Harbor-Pages/Project-Webhooks.robot b/tests/resources/Harbor-Pages/Project-Webhooks.robot index 5f29ca403..78eab24fe 100644 --- a/tests/resources/Harbor-Pages/Project-Webhooks.robot +++ b/tests/resources/Harbor-Pages/Project-Webhooks.robot @@ -16,7 +16,6 @@ Create A New Webhook Retry Text Input ${webhook_name_xpath} ${webhook_name} Retry Text Input ${webhook_endpoint_id_xpath} ${webhook_endpoint_url} Retry Double Keywords When Error Retry Element Click ${create_webhooks_continue_button_xpath} Retry Wait Until Page Not Contains Element ${create_webhooks_continue_button_xpath} - Capture Page Screenshot Retry Wait Until Page Contains ${webhook_name} Update A Webhook @@ -35,7 +34,6 @@ Update A Webhook Retry Text Input ${webhook_endpoint_id_xpath} ${new_webhook_enpoint} Retry Double Keywords When Error Retry Element Click ${edit_webhooks_save_button_xpath} Retry Wait Until Page Not Contains Element ${edit_webhooks_save_button_xpath} Retry Wait Until Page Contains ${new_webhook_name} - Capture Page Screenshot Enable/Disable State of Same Webhook [Arguments] ${webhook_name} diff --git a/tests/resources/Harbor-Pages/Project.robot b/tests/resources/Harbor-Pages/Project.robot index 45050b16d..6e687fafc 100644 --- a/tests/resources/Harbor-Pages/Project.robot +++ b/tests/resources/Harbor-Pages/Project.robot @@ -24,16 +24,13 @@ Create An New Project And Go Into Project Navigate To Projects Retry Button Click xpath=${create_project_button_xpath} Log To Console Project Name: ${projectname} - Capture Page Screenshot Retry Text Input xpath=${project_name_xpath} ${projectname} ${element_project_public}= Set Variable xpath=${project_public_xpath} Run Keyword If '${public}' == 'true' Run Keywords Wait Until Element Is Visible And Enabled ${element_project_public} AND Retry Element Click ${element_project_public} Run Keyword If '${count_quota}'!='${null}' Input Count Quota ${count_quota} Run Keyword If '${storage_quota}'!='${null}' Input Storage Quota ${storage_quota} ${storage_quota_unit} Run Keyword If '${proxy_cache}' == '${true}' Run Keywords Mouse Down ${project_proxy_cache_switcher_id} AND Mouse Up ${project_proxy_cache_switcher_id} AND Retry Element Click ${project_registry_select_id} AND Retry Element Click xpath=//select[@id='registry']//option[contains(.,'${registry}')] - Capture Page Screenshot Retry Double Keywords When Error Retry Element Click ${create_project_OK_button_xpath} Retry Wait Until Page Not Contains Element ${create_project_OK_button_xpath} - Capture Page Screenshot Sleep 2 Go Into Project ${projectname} has_image=${false} @@ -102,7 +99,6 @@ Search Private Projects Retry Element Click xpath=//select Retry Element Click xpath=//select/option[@value=1] Sleep 1 - Capture Page Screenshot SearchPrivateProjects.png Make Project Private [Arguments] ${projectname} @@ -164,7 +160,6 @@ Advanced Search Should Display # it's not a common keywords, only used into log case. Do Log Advanced Search - Capture Page Screenshot LogAdvancedSearch.png Retry Wait Until Page Contains Element xpath=//clr-dg-row[contains(.,'artifact') and contains(.,'pull')] Retry Wait Until Page Contains Element xpath=//clr-dg-row[contains(.,'artifact') and contains(.,'create')] Retry Wait Until Page Contains Element xpath=//clr-dg-row[contains(.,'artifact') and contains(.,'delete')] @@ -193,7 +188,6 @@ Do Log Advanced Search Retry Element Click xpath=//audit-log//hbr-filter//clr-icon Retry Text Input xpath=//audit-log//hbr-filter//input harbor Sleep 1 - Capture Page Screenshot LogAdvancedSearch2.png ${rc} = Get Element Count //audit-log//clr-dg-row Should Be Equal As Integers ${rc} 0 @@ -239,7 +233,6 @@ Go Into Index And Contain Artifacts FOR ${n} IN RANGE 1 10 ${out} Run Keyword And Ignore Error Page Should Contain Element ${artifact_rows} limit=${limit} Exit For Loop If '${out[0]}'=='PASS' - Capture Page Screenshot gointo_${tag_name}.png Sleep 3 END Run Keyword If '${out[0]}'=='FAIL' Capture Page Screenshot @@ -268,7 +261,6 @@ Edit Repo Info Input Text xpath=//*[@id='info-edit-textarea'] test_description_info Retry Element Click xpath=//*[@id='edit-save'] Retry Wait Until Page Contains test_description_info - Capture Page Screenshot Switch To Project Label Retry Element Click xpath=//project-detail//a[contains(.,'Labels')] @@ -281,7 +273,6 @@ Switch To Project Repo Add Labels To Tag [Arguments] ${tagName} ${labelName} Retry Element Click xpath=//clr-dg-row[contains(.,'${tagName}')]//label - Capture Page Screenshot add_${labelName}.png Retry Element Click xpath=//clr-dg-action-bar//clr-dropdown//span Retry Element Click xpath=//clr-dropdown-menu//clr-dropdown//button[contains(.,'Add Labels')] Retry Element Click xpath=//clr-dropdown//div//label[contains(.,'${labelName}')] @@ -301,7 +292,6 @@ Filter Labels In Tags Retry Element Click xpath=//*[@id='filterArea']//div//button[contains(.,'${labelName2}')] Retry Element Click xpath=//*[@id='filterArea']//hbr-filter/span/clr-icon Sleep 2 - Capture Page Screenshot filter_${labelName2}.png Retry Wait Until Page Contains Element xpath=//clr-dg-row[contains(.,'${labelName2}')] Retry Wait Until Page Not Contains Element xpath=//clr-dg-row[contains(.,'${labelName1}')] diff --git a/tests/resources/Harbor-Pages/Replication.robot b/tests/resources/Harbor-Pages/Replication.robot index 395859430..b3e8bcdfa 100644 --- a/tests/resources/Harbor-Pages/Replication.robot +++ b/tests/resources/Harbor-Pages/Replication.robot @@ -20,12 +20,15 @@ Resource ../../resources/Util.robot *** Keywords *** Filter Replication Rule - [Arguments] ${ruleName} + [Arguments] ${ruleName} ${exist}=${true} ${rule_name_element}= Set Variable xpath=//clr-dg-cell[contains(.,'${ruleName}')] Retry Element Click ${filter_rules_btn} + Retry Clear Element Text ${filter_rules_input} Retry Text Input ${filter_rules_input} ${ruleName} - Retry Wait Until Page Contains Element ${rule_name_element} - Capture Page Screenshot filter_replic_${ruleName}.png + Run Keyword If ${exist}==${true} Retry Wait Until Page Contains Element ${rule_name_element} + ... ELSE Retry Wait Element xpath=//clr-dg-placeholder[contains(.,\"We couldn\'t find any replication rules!\")] + + Filter Registry [Arguments] ${registry_name} @@ -35,7 +38,6 @@ Filter Registry Retry Element Click ${filter_registry_btn} Retry Text Input ${filter_registry_input} ${registry_name} Retry Wait Until Page Contains Element ${registry_name_element} - Capture Page Screenshot filter_repistry_${registry_name}.png Select Dest Registry [Arguments] ${endpoint} @@ -73,8 +75,8 @@ Create A New Endpoint Retry Text Input xpath=${destination_name_xpath} ${name} Run Keyword If '${provider}' == 'harbor' Run keyword Retry Text Input xpath=${destination_url_xpath} ${url} Run Keyword If '${provider}' == 'aws-ecr' or '${provider}' == 'google-gcr' Run keyword Select Destination URL ${url} - Run Keyword If '${provider}' != 'google-gcr' Retry Text Input xpath=${destination_username_xpath} ${username} - Retry Text Input xpath=${destination_password_xpath} ${pwd} + Run Keyword If '${provider}' != 'google-gcr' and '${username}' != '${null}' Retry Text Input xpath=${destination_username_xpath} ${username} + Run Keyword If '${pwd}' != '${null}' Retry Text Input xpath=${destination_password_xpath} ${pwd} #cancel verify cert since we use a selfsigned cert Retry Element Click ${destination_insecure_xpath} Run Keyword If '${save}' == 'Y' Run keyword Retry Double Keywords When Error Retry Element Click ${replication_save_xpath} Retry Wait Until Page Not Contains Element ${replication_save_xpath} @@ -153,21 +155,9 @@ Rename Rule Retry Text Input ${rule_name} ${newname} Retry Element Click ${rule_save_button} -Delete Rule - [Arguments] ${rule} - Retry Element Click ${rule_filter_search} - Retry Text Input ${rule_filter_input} ${rule} - Retry Element Click //clr-dg-row[contains(.,'${rule}')]//label - Retry Element Click ${action_bar_delete} - Retry Wait Until Page Contains Element ${dialog_delete} - #change from click to mouse down and up - Mouse Down ${dialog_delete} - Mouse Up ${dialog_delete} - Sleep 2 - Select Rule [Arguments] ${rule} - Retry Double Keywords When Error Retry Element Click //clr-dg-cell[contains(.,'${rule}')] Retry Wait Element ${replication_exec_id} + Retry Double Keywords When Error Retry Element Click //clr-dg-cell[contains(.,'${rule}')] Retry Wait Element ${replication_rule_exec_id} Stop Jobs Retry Element Click ${stop_jobs_button} @@ -178,42 +168,35 @@ View Job Log Retry Text Input ${job_filter_input} ${job} Retry Link Click //clr-dg-row[contains(.,'${job}')]//a -Find Item And Click Edit Button +Find Registry And Click Edit Button [Arguments] ${name} Filter Object ${name} Retry Select Object ${name} - Retry Element Click ${action_bar_edit} - -Find Item And Click Delete Button - [Arguments] ${name} - Filter Object ${name} - Retry Select Object ${name} - Retry Element Click ${action_bar_delete} + Retry Element Click ${registry_edit_btn} Switch To Replication Manage Page - [Arguments] ${name} Switch To Registries Switch To Replication Manage -Edit Replication Rule By Name +Edit Replication Rule [Arguments] ${name} - Retry Double Keywords When Error Switch To Replication Manage Page "NULL" Find Item And Click Edit Button ${name} + Switch To Replication Manage Page + Filter Replication Rule ${name} + Select Rule ${name} + Retry Double Keywords When Error Retry Element Click ${replication_rule_action_bar_edit} Retry Wait Until Page Contains Edit Replication Rule -Delete Replication Rule By Name - [Arguments] ${name} - Switch To Registries - Switch To Replication Manage - Find Item And Click Delete Button ${name} - -Ensure Delete Replication Rule By Name - [Arguments] ${name} - Delete Replication Rule By Name ${name} - Retry Double Keywords When Error Retry Element Click ${delete_confirm_btn} Retry Wait Until Page Not Contains Element ${delete_confirm_btn} - Retry Wait Element xpath=//clr-dg-placeholder[contains(.,\"We couldn\'t find any replication rules!\")] +Delete Replication Rule + [Arguments] ${name} + Switch To Replication Manage Page + Filter Replication Rule ${name} + Select Rule ${name} + Retry Double Keywords When Error Retry Element Click ${replication_rule_action_bar_delete} Wait Until Page Contains Element ${dialog_delete} + Retry Double Keywords When Error Retry Element Click ${dialog_delete} Retry Wait Until Page Not Contains Element ${dialog_delete} + Filter Replication Rule ${name} exist=${false} Rename Endpoint [arguments] ${name} ${newname} - Find Item And Click Edit Button ${name} + Find Registry And Click Edit Button ${name} Retry Wait Until Page Contains Element ${destination_name_xpath} Retry Text Input ${destination_name_xpath} ${newname} Retry Element Click ${replication_save_xpath} @@ -231,24 +214,9 @@ Delete Endpoint Select Rule And Replicate [Arguments] ${rule_name} Select Rule ${rule_name} - Retry Element Click ${replication_exec_id} + Retry Element Click ${replication_rule_exec_id} Retry Double Keywords When Error Retry Element Click xpath=${dialog_replicate} Retry Wait Until Page Not Contains Element xpath=${dialog_replicate} -Select Rule And Click Edit Button - [Arguments] ${rule_name} - Retry Element Click //clr-dg-row[contains(.,'${rule_name}')]//clr-radio-wrapper/label - Retry Element Click ${edit_replication_rule_id} - -Delete Replication Rule - [Arguments] ${name} - Retry Element Click ${endpoint_filter_search} - Retry Text Input ${endpoint_filter_input} ${name} - #click checkbox before target endpoint - Retry Element Click //clr-dg-row[contains(.,'${name}')]//label - Retry Element Click ${action_bar_delete} - Wait Until Page Contains Element ${dialog_delete} - Retry Element Click ${dialog_delete} - Image Should Be Replicated To Project [Arguments] ${project} ${image} ${period}=60 ${times}=3 FOR ${n} IN RANGE 0 ${times} @@ -268,7 +236,5 @@ Executions Result Count Should Be [Arguments] ${expected_status} ${expected_trigger_type} ${expected_result_count} Sleep 10 ${count}= Get Element Count xpath=//clr-dg-row[contains(.,'${expected_status}') and contains(.,'${expected_trigger_type}')] - Capture Page Screenshot Should Be Equal As Integers ${count} ${expected_result_count} - Capture Page Screenshot diff --git a/tests/resources/Harbor-Pages/Replication_Elements.robot b/tests/resources/Harbor-Pages/Replication_Elements.robot index 97f9d36bb..2ae6e0454 100644 --- a/tests/resources/Harbor-Pages/Replication_Elements.robot +++ b/tests/resources/Harbor-Pages/Replication_Elements.robot @@ -68,9 +68,14 @@ ${rule_resource_selector} //*[@id='select_resource'] ${trigger_mode_selector} //*[@id='ruleTrigger'] ${dest_namespace_xpath} //*[@id='dest_namespace'] ${new_replication_rule_id} //*[@id='new_replication_rule_id'] -${edit_replication_rule_id} //*[@id='edit_replication_rule_id'] -${delete_replication_rule_id} //*[@id='delete_replication_rule_id'] -${replication_exec_id} //*[@id='replication_exe_id'] + +${registry_edit_btn} //button[contains(.,'Edit')] +${registry_del_btn} //button[contains(.,'Delete')] + +${replication_rule_action} //*[@id='rule-action'] +${replication_rule_action_bar_edit} //*[@id='edit_replication_rule_id'] +${replication_rule_action_bar_delete} //*[@id='delete_replication_rule_id'] +${replication_rule_exec_id} //*[@id='replication_exe_id'] ${replication_task_line_1} //clr-datagrid//clr-dg-row/div/div[2]//clr-checkbox-wrapper/label[1] ${is_overide_xpath} //label[contains(.,'Replace the destination resources if name exists')] ${enable_rule_xpath} //label[contains(.,'Enable rule')] diff --git a/tests/resources/Harbor-Pages/ToolKit.robot b/tests/resources/Harbor-Pages/ToolKit.robot index 2cfc0b5c3..46dd526c2 100644 --- a/tests/resources/Harbor-Pages/ToolKit.robot +++ b/tests/resources/Harbor-Pages/ToolKit.robot @@ -25,7 +25,6 @@ Delete Success Retry Wait Until Page Contains Element //*[@id='contentAll']//div[contains(.,'${obj}')]/../div/clr-icon[@shape='success-standard'] END Sleep 1 - Capture Page Screenshot Delete Fail [Arguments] @{obj} @@ -33,7 +32,6 @@ Delete Fail Retry Wait Until Page Contains Element //*[@id='contentAll']//div[contains(.,'${obj}')]/../div/clr-icon[@shape='error-standard'] END Sleep 1 - Capture Page Screenshot Filter Object #Filter project repo user tag. @@ -64,13 +62,10 @@ Multi-delete Object Retry Element Click ${element} END Sleep 1 - Capture Page Screenshot Retry Element Click ${delete_btn} Sleep 1 - Capture Page Screenshot Retry Element Click ${repo_delete_on_card_view_btn} Sleep 1 - Capture Page Screenshot Sleep 1 # This func cannot support as the delete user flow changed. @@ -81,15 +76,12 @@ Multi-delete Artifact Retry Element Click ${element} END Sleep 1 - Capture Page Screenshot Retry Element Click ${artifact_action_xpath} Sleep 1 Retry Element Click ${artifact_action_delete_xpath} Sleep 1 - Capture Page Screenshot Retry Element Click ${repo_delete_on_card_view_btn} Sleep 1 - Capture Page Screenshot Sleep 1 Multi-delete User diff --git a/tests/resources/Harbor-Pages/UserProfile.robot b/tests/resources/Harbor-Pages/UserProfile.robot index 8d58d999e..7ca6cb695 100644 --- a/tests/resources/Harbor-Pages/UserProfile.robot +++ b/tests/resources/Harbor-Pages/UserProfile.robot @@ -41,6 +41,5 @@ Update User Comment Logout Harbor Retry Element Click ${head_admin_xpath} Retry Link Click Log Out - Capture Page Screenshot Logout.png Sleep 2 Wait Until Keyword Succeeds 5x 1 Retry Wait Until Page Contains Element ${sign_in_title_xpath} \ No newline at end of file diff --git a/tests/resources/Harbor-Pages/Verify.robot b/tests/resources/Harbor-Pages/Verify.robot index eea6e9208..963fbe159 100644 --- a/tests/resources/Harbor-Pages/Verify.robot +++ b/tests/resources/Harbor-Pages/Verify.robot @@ -336,8 +336,7 @@ Verify Replicationrule Init Chrome Driver Log To Console -----replicationrule-----"${replicationrule}"------------ Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Switch To Replication Manage - Select Rule And Click Edit Button ${replicationrule} + Edit Replication Rule ${replicationrule} @{is_src_registry}= Get Value From Json ${json} $.replicationrule[?(@.rulename=${replicationrule})].is_src_registry @{trigger_type}= Get Value From Json ${json} $.replicationrule[?(@.rulename=${replicationrule})].trigger_type @{name_filters}= Get Value From Json ${json} $.replicationrule[?(@.rulename=${replicationrule})].name_filters @@ -351,12 +350,13 @@ Verify Replicationrule ${endpoint0}= Set Variable @{endpoint}[0] Log To Console -----endpoint0-----${endpoint0}------------ @{endpoint_type}= Get Value From Json ${json} $.endpoint[?(@.name=${endpoint0})].type + @{endpoint_url}= Get Value From Json ${json} $.endpoint[?(@.name=${endpoint0})].url Retry Textfield Value Should Be ${filter_name_id} @{name_filters}[0] Retry Textfield Value Should Be ${filter_tag_id} @{tag_filters}[0] Retry Textfield Value Should Be ${rule_name_input} ${replicationrule} Retry Textfield Value Should Be ${dest_namespace_xpath} @{dest_namespace}[0] Log To Console -----endpoint_type-----@{endpoint_type}[0]------------ - ${registry}= Set Variable If "@{endpoint_type}[0]"=="harbor" ${endpoint0}-https://${IP} ${endpoint0}-https://hub.docker.com + ${registry}= Set Variable If "@{endpoint_type}[0]"=="harbor" ${endpoint0}-@{endpoint_url}[0] ${endpoint0}-https://hub.docker.com Log To Console -------registry---${registry}------------ Run Keyword If '@{is_src_registry}[0]' == '${true}' Retry List Selection Should Be ${src_registry_dropdown_list} ${registry} ... ELSE Retry List Selection Should Be ${dest_registry_dropdown_list} ${registry} @@ -516,7 +516,7 @@ Verify Distributions ${endpoint}= Get Value From Json ${json} $.distributions[?(@.name=${name})].endpoint ${vendor}= Get Value From Json ${json} $.distributions[?(@.name=${name})].vendor ${auth_mode}= Get Value From Json ${json} $.distributions[?(@.name=${name})].auth_mode - Retry Wait Until Page Contains Element //div[@class='datagrid-scrolling-cells' and contains(.,'${name}') and contains(.,'${endpoint}[0]') and contains(.,'${vendor}[0]') and contains(.,'${auth_mode}[0]')] + Retry Wait Until Page Contains Element //clr-dg-row[contains(.,'${name}') and contains(.,'${endpoint}[0]') and contains(.,'${vendor}[0]') and contains(.,'${auth_mode}[0]')] END Verify P2P Preheat Policy @@ -542,5 +542,5 @@ Loop P2P Preheat Policys [Arguments] ${json} ${project} @{policy_names} FOR ${policy} IN @{policy_names} ${provider_name}= Get Value From Json ${json} $.projects[?(@.name=${project})].p2p_preheat_policy[?(@.name=${policy})].provider_name - Retry Wait Until Page Contains Element //div[@class='datagrid-scrolling-cells' and contains(.,'${policy}') and contains(.,'${provider_name}[0]')] + Retry Wait Until Page Contains Element //clr-dg-row[contains(.,'${policy}') and contains(.,'${provider_name}[0]')] END \ No newline at end of file diff --git a/tests/resources/Nightly-Util.robot b/tests/resources/Nightly-Util.robot index f5f4cba5a..852e5597e 100644 --- a/tests/resources/Nightly-Util.robot +++ b/tests/resources/Nightly-Util.robot @@ -28,6 +28,8 @@ Nightly Test Setup Run Keyword CA setup ${ip} ${HARBOR_PASSWORD} Log To Console Start Docker Daemon Locally ... Run Keyword Start Docker Daemon Locally + Log To Console Start Containerd Daemon Locally ... + Run Keyword Start Containerd Daemon Locally Log To Console wget mariadb ... Run wget ${prometheus_chart_file_url} @@ -45,13 +47,19 @@ Nightly Test Setup For Nightly Run Keyword If '${ip1}' != '${EMPTY}' CA setup For Nightly ${ip1} ${HARBOR_PASSWORD} /ca/ca1.crt Run Keyword If '${ip1}' != '${EMPTY}' Run rm -rf ./harbor_ca.crt Run Keyword CA setup For Nightly ${ip} ${HARBOR_PASSWORD} + Log To Console Start Docker Daemon Locally ... Run Keyword Start Docker Daemon Locally + Log To Console Start Containerd Daemon Locally ... + Run Keyword Start Containerd Daemon Locally + #Prepare docker image for push special image keyword in replication test + Docker Pull ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/busybox:latest + Docker Tag ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/busybox:latest busybox:latest CA Setup For Nightly [Arguments] ${ip} ${HARBOR_PASSWORD} ${cert}=/ca/ca.crt Run cp ${cert} harbor_ca.crt Generate Certificate Authority For Chrome ${HARBOR_PASSWORD} - Prepare Docker Cert ${ip} + Prepare Docker Cert For Nightly ${ip} Prepare Helm Cert Collect Nightly Logs diff --git a/tests/resources/TestCaseBody.robot b/tests/resources/TestCaseBody.robot index e211a6b04..ede514c6f 100644 --- a/tests/resources/TestCaseBody.robot +++ b/tests/resources/TestCaseBody.robot @@ -212,7 +212,6 @@ Helm CLI Push Without Sign In Harbor Switch To Project Charts Go Into Chart Version ${harbor_chart_name} Retry Wait Until Page Contains ${harbor_chart_version} - Capture Page Screenshot Helm3 CLI Push Without Sign In Harbor [Arguments] ${sign_in_user} ${sign_in_pwd} @@ -221,7 +220,6 @@ Helm3 CLI Push Without Sign In Harbor Helm Repo Push ${sign_in_user} ${sign_in_pwd} ${harbor_chart_filename} helm_repo_name=${HARBOR_URL}/chartrepo/project${d} helm_cmd=helm3 Switch To Project Charts Retry Double Keywords When Error Go Into Chart Version ${harbor_chart_name} Retry Wait Until Page Contains ${harbor_chart_version} - Capture Page Screenshot #Important Note: All CVE IDs in CVE Allowlist cases must unique! Body Of Verfiy System Level CVE Allowlist diff --git a/tests/resources/Util.robot b/tests/resources/Util.robot index e1c8e25e3..ce8a45afd 100644 --- a/tests/resources/Util.robot +++ b/tests/resources/Util.robot @@ -150,6 +150,11 @@ Retry Wait Until Page Contains @{param} Create List ${element_xpath} Retry Action Keyword Wait Until Page Contains @{param} +Retry Wait Until Page Does Not Contains + [Arguments] ${element_xpath} + @{param} Create List ${element_xpath} + Retry Action Keyword Wait Until Page Does Not Contain @{param} + Retry Wait Until Page Contains Element [Arguments] ${element_xpath} @{param} Create List ${element_xpath} @@ -213,7 +218,7 @@ Clear Field Of Characters END Wait Unitl Command Success - [Arguments] ${cmd} ${times}=8 + [Arguments] ${cmd} ${times}=2 FOR ${n} IN RANGE 1 ${times} Log Trying ${cmd}: ${n} ... console=True ${rc} ${output}= Run And Return Rc And Output ${cmd} @@ -237,7 +242,6 @@ Retry Keyword N Times When Error Log To Console Trying ${keyword} elements @{elements} ${n} times ... ${out} Run Keyword And Ignore Error ${keyword} @{elements} Log To Console Return value is ${out} and ${out[0]} - Capture Page Screenshot record.png Run Keyword If '${keyword}'=='Make Swagger Client' Exit For Loop If '${out[0]}'=='PASS' and '${out[1]}'=='0' ... ELSE Exit For Loop If '${out[0]}'=='PASS' Sleep 10 @@ -264,14 +268,13 @@ Retry Double Keywords When Error FOR ${n} IN RANGE 1 ${times} Log To Console Trying ${keyword1} and ${keyword2} ${n} times ... ${out1} Run Keyword And Ignore Error ${keyword1} ${element1} - Capture Page Screenshot Sleep 1 ${out2} Run Keyword And Ignore Error ${keyword2} ${element2} - Capture Page Screenshot Log To Console Return value is ${out1[0]} ${out2[0]} Exit For Loop If '${out2[0]}'=='PASS' Sleep 1 END + Capture Page Screenshot Return From Keyword If ${DoAssert} == ${false} '${out2[0]}' Should Be Equal As Strings '${out2[0]}' 'PASS' diff --git a/tests/robot-cases/Group0-BAT/API_DB.robot b/tests/robot-cases/Group0-BAT/API_DB.robot index bb08919e2..1c5a50c7c 100644 --- a/tests/robot-cases/Group0-BAT/API_DB.robot +++ b/tests/robot-cases/Group0-BAT/API_DB.robot @@ -105,16 +105,16 @@ Test Case - Push Cnab Bundle [Tags] push_cnab Harbor API Test ./tests/apitests/python/test_push_cnab_bundle.py -Test Case - Create/Delete tag - [Tags] tag_cuid - Harbor API Test ./tests/apitests/python/test_create_delete_tag.py +Test Case - Tag CRUD + [Tags] tag_crud + Harbor API Test ./tests/apitests/python/test_tag_crud.py Test Case - Scan Image [Tags] scan Harbor API Test ./tests/apitests/python/test_scan_image_artifact.py Test Case - Scan Image In Public Project - [Tags] scan + [Tags] scan_public_project Harbor API Test ./tests/apitests/python/test_scan_image_artifact_in_public_project.py Test Case - Scan All Images @@ -144,3 +144,15 @@ Test Case - Push Chart File To Chart Repository By Helm V2 With Robot Account Test Case - Replication From Dockerhub [Tags] replic_dockerhub Harbor API Test ./tests/apitests/python/test_replication_from_dockerhub.py + +Test Case - Proxy Cache + [Tags] proxy_cache + Harbor API Test ./tests/apitests/python/test_proxy_cache.py + +Test Case - Tag Immutability + [Tags] tag_immutability + Harbor API Test ./tests/apitests/python/test_tag_immutability.py + +Test Case - P2P + [Tags] p2p + Harbor API Test ./tests/apitests/python/test_p2p.py diff --git a/tests/robot-cases/Group0-BAT/API_LDAP.robot b/tests/robot-cases/Group0-BAT/API_LDAP.robot index 2ce9f6948..ce22b5e87 100644 --- a/tests/robot-cases/Group0-BAT/API_LDAP.robot +++ b/tests/robot-cases/Group0-BAT/API_LDAP.robot @@ -10,10 +10,13 @@ Default Tags API *** Test Cases *** Test Case - LDAP Group Admin Role + [Tags] group_admin Harbor API Test ./tests/apitests/python/test_ldap_admin_role.py Test Case - LDAP Group User Group + [Tags] group_user Harbor API Test ./tests/apitests/python/test_user_group.py Test Case - Run LDAP Group Related API Test + [Tags] assign_role Harbor API Test ./tests/apitests/python/test_assign_role_to_ldap_group.py \ No newline at end of file diff --git a/tests/robot-cases/Group0-Util/cnab_push_bundle.sh b/tests/robot-cases/Group0-Util/cnab_push_bundle.sh index c45df9969..c9df10923 100755 --- a/tests/robot-cases/Group0-Util/cnab_push_bundle.sh +++ b/tests/robot-cases/Group0-Util/cnab_push_bundle.sh @@ -1,12 +1,26 @@ #!/bin/bash +set -x IP=$1 USER=$2 PWD=$3 TARGET=$4 BUNDLE_FILE=$5 +DOCKER_USER=$6 +DOCKER_PWD=$7 +echo $DOCKER_USER echo $IP +TOKEN=$(curl --user "$DOCKER_USER:$DOCKER_PWD" "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token) +curl -v -H "Authorization: Bearer $TOKEN" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest +curl -v -H "Authorization: Bearer $TOKEN" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest > limit.log + +docker login -u $DOCKER_USER -p $DOCKER_PWD docker login $IP -u $USER -p $PWD + cnab-to-oci fixup $BUNDLE_FILE --target $TARGET --bundle fixup_bundle.json --auto-update-bundle + +TOKEN=$(curl --user "$DOCKER_USER:$DOCKER_PWD" "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token) +curl -v -H "Authorization: Bearer $TOKEN" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest > limit_after.log + cnab-to-oci push fixup_bundle.json --target $TARGET --auto-update-bundle \ No newline at end of file diff --git a/tests/robot-cases/Group0-Util/docker_push_manifest_list.sh b/tests/robot-cases/Group0-Util/docker_push_manifest_list.sh index db4dba452..55edafe12 100755 --- a/tests/robot-cases/Group0-Util/docker_push_manifest_list.sh +++ b/tests/robot-cases/Group0-Util/docker_push_manifest_list.sh @@ -1,4 +1,6 @@ #!/bin/bash +set -x +set -e IP=$1 USER=$2 @@ -12,9 +14,11 @@ docker login $IP -u $USER -p $PWD cat /$HOME/.docker/config.json -sed -i '$d' /$HOME/.docker/config.json -sed -i '$d' /$HOME/.docker/config.json -echo -e "\n },\n \"experimental\": \"enabled\"\n}" >> /$HOME/.docker/config.json +if [ $(cat /$HOME/.docker/config.json |grep experimental |wc -l) -eq 0 ];then + sudo sed -i '$d' /$HOME/.docker/config.json + sudo sed -i '$d' /$HOME/.docker/config.json + sudo echo -e "},\n \"experimental\": \"enabled\"\n}" >> /$HOME/.docker/config.json +fi cat /$HOME/.docker/config.json diff --git a/tests/robot-cases/Group1-Nightly/Common.robot b/tests/robot-cases/Group1-Nightly/Common.robot index 224d6f11c..9eef9be60 100644 --- a/tests/robot-cases/Group1-Nightly/Common.robot +++ b/tests/robot-cases/Group1-Nightly/Common.robot @@ -29,6 +29,28 @@ Test Case - Sign With Admin Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Close Browser +Test Case - Push CNAB Bundle and Display + [Tags] run-once + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} user010 Test1@34 + Create An New Project And Go Into Project test${d} + + ${target}= Set Variable ${ip}/test${d}/cnab${d}:cnab_tag${d} + Retry Keyword N Times When Error 5 CNAB Push Bundle ${ip} user010 Test1@34 ${target} ./tests/robot-cases/Group0-Util/bundle.json ${DOCKER_USER} ${DOCKER_PWD} + + Go Into Project test${d} + Wait Until Page Contains test${d}/cnab${d} + + Go Into Repo test${d}/cnab${d} + Wait Until Page Contains cnab_tag${d} + Go Into Project test${d} + Wait Until Page Contains test${d}/cnab${d} + Go Into Repo test${d}/cnab${d} + Go Into Index And Contain Artifacts cnab_tag${d} limit=3 + Close Browser + Test Case - Create An New Project Init Chrome Driver ${d}= Get Current Date result_format=%m%s @@ -494,10 +516,8 @@ Test Case - Project Quotas Control Under Copy Sleep 2 Go Into Project project_b_${d} Sleep 2 - Capture Page Screenshot Retry Wait Until Page Contains Element xpath=//clr-dg-cell[contains(.,'${image_a}')]/a Retry Wait Until Page Not Contains Element xpath=//clr-dg-cell[contains(.,'${image_b}')]/a - Capture Page Screenshot Close Browser Test Case - Webhook CRUD @@ -611,27 +631,6 @@ Test Case - Push Docker Manifest Index and Display Go Into Index And Contain Artifacts index_tag${d} limit=2 Close Browser -Test Case - Push CNAB Bundle and Display - Init Chrome Driver - ${d}= Get Current Date result_format=%m%s - - Sign In Harbor ${HARBOR_URL} user010 Test1@34 - Create An New Project And Go Into Project test${d} - - ${target}= Set Variable ${ip}/test${d}/cnab${d}:cnab_tag${d} - Retry Keyword N Times When Error 5 CNAB Push Bundle ${ip} user010 Test1@34 ${target} ./tests/robot-cases/Group0-Util/bundle.json - - Go Into Project test${d} - Wait Until Page Contains test${d}/cnab${d} - - Go Into Repo test${d}/cnab${d} - Wait Until Page Contains cnab_tag${d} - Go Into Project test${d} - Wait Until Page Contains test${d}/cnab${d} - Go Into Repo test${d}/cnab${d} - Go Into Index And Contain Artifacts cnab_tag${d} limit=3 - Close Browser - Test Case - Push Helm Chart and Display Init Chrome Driver ${d}= Get Current Date result_format=%m%s @@ -694,9 +693,10 @@ Test Case - Read Only Mode Close Browser Test Case - Proxy Cache + [Tags] run-once ${d}= Get Current Date result_format=%m%s ${registry}= Set Variable https://hub.docker.com/ - ${user_namespace}= Set Variable danfengliu + ${user_namespace}= Set Variable ${DOCKER_USER} ${image}= Set Variable for_proxy ${tag}= Set Variable 1.0 ${manifest_index}= Set Variable index081597864867 @@ -704,7 +704,7 @@ Test Case - Proxy Cache Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To Registries - Create A New Endpoint docker-hub e1${d} ${registry} ${user_namespace} Aa123456 + Create A New Endpoint docker-hub e1${d} ${registry} ${user_namespace} ${DOCKER_PWD} Create An New Project And Go Into Project project${d} proxy_cache=${true} registry=e1${d} Cannot Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} busybox:latest err_msg=can not push artifact to a proxy project Pull Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${user_namespace}/${image} tag=${tag} @@ -723,7 +723,7 @@ Test Case - Proxy Cache Test Case - Distribution CRUD ${d}= Get Current Date result_format=%m%s ${name}= Set Variable distribution${d} - ${endpoint}= Set Variable https://1.1.1.2 + ${endpoint}= Set Variable https://32.1.1.2 ${endpoint_new}= Set Variable https://10.65.65.42 Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} @@ -732,11 +732,11 @@ Test Case - Distribution CRUD Delete A Distribution ${name} ${endpoint_new} Close Browser -Test Case - P2P Peheat Policy CRUD +Test Case - P2P Preheat Policy CRUD ${d}= Get Current Date result_format=%m%s ${pro_name}= Set Variable project_p2p${d} ${dist_name}= Set Variable distribution${d} - ${endpoint}= Set Variable https://1.1.1.2 + ${endpoint}= Set Variable https://20.76.1.2 ${policy_name}= Set Variable policy${d} ${repo}= Set Variable alpine ${repo_new}= Set Variable redis* diff --git a/tests/robot-cases/Group1-Nightly/Common_GC.robot b/tests/robot-cases/Group1-Nightly/Common_GC.robot index dac172780..0eb1aceb0 100644 --- a/tests/robot-cases/Group1-Nightly/Common_GC.robot +++ b/tests/robot-cases/Group1-Nightly/Common_GC.robot @@ -80,11 +80,8 @@ Test Case - Project Quotas Control Under GC ${image_a_size}= Set Variable 321.03MB ${image_a_ver}= Set Variable 6.8.3 Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Capture Page Screenshot Create An New Project And Go Into Project project${d} storage_quota=${storage_quota} storage_quota_unit=${storage_quota_unit} - Capture Page Screenshot Cannot Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image_a}:${image_a_ver} err_msg=will exceed the configured upper limit of 200.0 MiB - Capture Page Screenshot GC Now ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Retry GC Should Be Successful 4 ${null} @{param} Create List project${d} diff --git a/tests/robot-cases/Group1-Nightly/LDAP.robot b/tests/robot-cases/Group1-Nightly/LDAP.robot index 7b7511692..451c86faf 100644 --- a/tests/robot-cases/Group1-Nightly/LDAP.robot +++ b/tests/robot-cases/Group1-Nightly/LDAP.robot @@ -111,6 +111,17 @@ Test Case - Ldap User Push An Image Test Case - Ldap User Can Not login Docker Login Fail ${ip} testerDeesExist 123456 +Test Case - Ldap Group Admin DN Setting + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Set LDAP Group Admin DN cn=harbor_users,ou=groups,dc=example,dc=com + Logout Harbor + Sign In Harbor ${HARBOR_URL} mike zhu88jie + Switch To Registries + Create A New Endpoint harbor edp1${d} https://cicd.harbor.vmwarecna.net ${null} ${null} Y + + Test Case - Run LDAP Group Related API Test Harbor API Test ./tests/apitests/python/test_ldap_admin_role.py Harbor API Test ./tests/apitests/python/test_user_group.py diff --git a/tests/robot-cases/Group1-Nightly/Nightly.robot b/tests/robot-cases/Group1-Nightly/Nightly.robot index 83b8a4db6..ac6d28473 100644 --- a/tests/robot-cases/Group1-Nightly/Nightly.robot +++ b/tests/robot-cases/Group1-Nightly/Nightly.robot @@ -133,7 +133,6 @@ Test Case - User View Projects Create An New Project test${d}2 Create An New Project test${d}3 Switch To Log - Capture Page Screenshot UserViewProjects.png Wait Until Page Contains test${d}1 Wait Until Page Contains test${d}2 Wait Until Page Contains test${d}3 @@ -368,9 +367,7 @@ TestCase - Project Admin Add Labels To Repo # Add labels Switch To Project Label Create New Labels label111 - Capture Page Screenshot CreateLabel1.png Create New Labels label22 - Capture Page Screenshot CreateLabel2.png Sleep 2 Switch To Project Repo Go Into Repo project${d}/redis diff --git a/tests/robot-cases/Group1-Nightly/Replication.robot b/tests/robot-cases/Group1-Nightly/Replication.robot index 51b297f0d..e6e6a787a 100644 --- a/tests/robot-cases/Group1-Nightly/Replication.robot +++ b/tests/robot-cases/Group1-Nightly/Replication.robot @@ -14,6 +14,7 @@ *** Settings *** Documentation Harbor BATs +Library ../../apitests/python/testutils.py Library ../../apitests/python/library/repository.py Resource ../../resources/Util.robot Default Tags Replication @@ -39,7 +40,6 @@ Test Case - Pro Replication Rules Add Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To Registries - Capture Page Screenshot Switch To Replication Manage Check New Rule UI Without Endpoint Close Browser @@ -56,14 +56,14 @@ Test Case - Harbor Endpoint Verification Endpoint Is Unpingable Close Browser -Test Case - DockerHub Endpoint Add +##Test Case - DockerHub Endpoint Add #This case need vailid info and selfsign cert - Init Chrome Driver - ${d}= Get Current Date result_format=%m%s - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Switch To Registries - Create A New Endpoint docker-hub edp1${d} https://hub.docker.com/ danfengliu Aa123456 Y - Close Browser + ##Init Chrome Driver + ##${d}= Get Current Date result_format=%m%s + ##Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + ##Switch To Registries + ##Create A New Endpoint docker-hub edp1${d} https://hub.docker.com/ ${DOCKER_USER} ${DOCKER_PWD} Y + ##Close Browser Test Case - Harbor Endpoint Add #This case need vailid info and selfsign cert @@ -105,11 +105,12 @@ Test Case - Replication Rule Edit ${cron_str}= Set Variable 10 10 10 * * * Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To Registries - Create A New Endpoint docker-hub ${endpoint1} https://hub.docker.com/ danfengliu Aa123456 Y + #Due to docker-hub access limitation, remove docker-hub endpoint + Create A New Endpoint harbor ${endpoint1} https://cicd.harbor.vmwarecna.net ${null} ${null} Y Create A New Endpoint harbor ${endpoint2} https://${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Y Switch To Replication Manage - Create A Rule With Existing Endpoint ${rule_name_old} pull danfengliu/* image ${endpoint1} project${d} - Edit Replication Rule By Name ${rule_name_old} + Create A Rule With Existing Endpoint ${rule_name_old} pull nightly/a* image ${endpoint1} project${d} + Edit Replication Rule ${rule_name_old} # Change rule-name, source-registry, filter, trigger-mode for edition verification Clear Field Of Characters ${rule_name_input} 30 Retry Text Input ${rule_name_input} ${rule_name_new} @@ -122,7 +123,7 @@ Test Case - Replication Rule Edit Retry Text Input ${targetCron_id} ${cron_str} Retry Double Keywords When Error Retry Element Click ${rule_save_button} Retry Wait Until Page Not Contains Element ${rule_save_button} # verify all items were changed as expected - Edit Replication Rule By Name ${rule_name_new} + Edit Replication Rule ${rule_name_new} Retry Textfield Value Should Be ${rule_name_input} ${rule_name_new} Retry List Selection Should Be ${src_registry_dropdown_list} ${endpoint2}-https://${ip} Retry Textfield Value Should Be ${filter_name_id} project${d} @@ -131,7 +132,7 @@ Test Case - Replication Rule Edit Retry List Selection Should Be ${rule_trigger_select} ${mode} Retry Textfield Value Should Be ${targetCron_id} ${cron_str} Retry Element Click ${rule_cancel_btn} - Ensure Delete Replication Rule By Name ${rule_name_new} + Delete Replication Rule ${rule_name_new} Close Browser Test Case - Replication Rule Delete @@ -141,10 +142,10 @@ Test Case - Replication Rule Delete ${rule_name}= Set Variable rule_testabc${d} Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To Registries - Create A New Endpoint docker-hub ${endpoint1} https://hub.docker.com/ danfengliu Aa123456 Y + Create A New Endpoint harbor ${endpoint1} https://${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Y Switch To Replication Manage - Create A Rule With Existing Endpoint ${rule_name} pull danfengliu/* image ${endpoint1} project${d} - Ensure Delete Replication Rule By Name ${rule_name} + Create A Rule With Existing Endpoint ${rule_name} pull ${DOCKER_USER}/* image ${endpoint1} project${d} + Delete Replication Rule ${rule_name} Close Browser Test Case - Replication Of Pull Images from DockerHub To Self @@ -154,11 +155,11 @@ Test Case - Replication Of Pull Images from DockerHub To Self Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Create An New Project And Go Into Project project${d} Switch To Registries - Create A New Endpoint docker-hub e${d} https://hub.docker.com/ danfengliu Aa123456 Y + Create A New Endpoint docker-hub e${d} https://hub.docker.com/ ${DOCKER_USER} ${DOCKER_PWD} Y Switch To Replication Manage - Create A Rule With Existing Endpoint rule${d} pull danfengliu/{cent*,mariadb} image e${d} project${d} + Create A Rule With Existing Endpoint rule${d} pull ${DOCKER_USER}/{cent*,mariadb} image e${d} project${d} Select Rule And Replicate rule${d} - #In docker-hub, under repository danfengliu, there're only 2 images: centos,mariadb. + #In docker-hub, under repository ${DOCKER_USER}, there're only 2 images: centos,mariadb. Image Should Be Replicated To Project project${d} centos Image Should Be Replicated To Project project${d} mariadb Close Browser @@ -275,7 +276,7 @@ Test Case - Replication Of Pull Images from Google-GCR To Self Close Browser Test Case - Replication Of Push Images to DockerHub Triggered By Event - Body Of Replication Of Push Images to Registry Triggered By Event docker-hub https://hub.docker.com/ danfengliu Aa123456 danfengliu + Body Of Replication Of Push Images to Registry Triggered By Event docker-hub https://hub.docker.com/ ${DOCKER_USER} ${DOCKER_PWD} ${DOCKER_USER} #Due to issue of delete event replication #Test Case - Replication Of Push Images to Google-GCR Triggered By Event diff --git a/tests/robot-cases/Group1-Nightly/Schedule.robot b/tests/robot-cases/Group1-Nightly/Schedule.robot index 742899cd7..91969e6d6 100644 --- a/tests/robot-cases/Group1-Nightly/Schedule.robot +++ b/tests/robot-cases/Group1-Nightly/Schedule.robot @@ -39,14 +39,15 @@ Test Case - Scan Schedule Job Retry Wait Until Page Contains Element ${not_scanned_icon} Switch To Vulnerability Page ${flag}= Set Variable ${false} - :FOR ${i} IN RANGE 999999 - \ ${minite}= Get Current Date result_format=%M - \ ${minite_int} = Convert To Integer ${minite} - \ ${left} = Evaluate ${minite_int}%10 - \ Log To Console ${i}/${left} - \ Sleep 55 - \ Run Keyword If ${left} <= 3 and ${left} != 0 Run Keywords Set Scan Schedule custom value=* */10 * * * * AND Set Suite Variable ${flag} ${true} - \ Exit For Loop If '${flag}' == '${true}' + FOR ${i} IN RANGE 999999 + ${minite}= Get Current Date result_format=%M + ${minite_int} = Convert To Integer ${minite} + ${left} = Evaluate ${minite_int}%10 + Log To Console ${i}/${left} + Sleep 55 + Run Keyword If ${left} <= 3 and ${left} != 0 Run Keywords Set Scan Schedule custom value=* */10 * * * * AND Set Suite Variable ${flag} ${true} + Exit For Loop If '${flag}' == '${true}' + END # After scan custom schedule is set, image should stay in unscanned status. Log To Console Sleep for 300 seconds...... Sleep 300 @@ -73,17 +74,18 @@ Test Case - Replication Schedule Job Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Create An New Project And Go Into Project ${project_name} Switch To Registries - Create A New Endpoint docker-hub e${d} https://hub.docker.com/ danfengliu Aa123456 Y + Create A New Endpoint harbor e${d} https://cicd.harbor.vmwarecna.net ${null} ${null} Y Switch To Replication Manage ${flag}= Set Variable ${false} - :FOR ${i} IN RANGE 999999 - \ ${minite}= Get Current Date result_format=%M - \ ${minite_int} = Convert To Integer ${minite} - \ ${left} = Evaluate ${minite_int}%10 - \ Log To Console ${i}/${left} - \ Run Keyword If ${left} <= 3 and ${left} != 0 Run Keywords Create A Rule With Existing Endpoint rule${d} pull danfengliu/* image e${d} ${project_name} mode=Scheduled cron=* */10 * * * * AND Set Suite Variable ${flag} ${true} - \ Sleep 40 - \ Exit For Loop If '${flag}' == '${true}' + FOR ${i} IN RANGE 999999 + ${minite}= Get Current Date result_format=%M + ${minite_int} = Convert To Integer ${minite} + ${left} = Evaluate ${minite_int}%10 + Log To Console ${i}/${left} + Run Keyword If ${left} <= 3 and ${left} != 0 Run Keywords Create A Rule With Existing Endpoint rule${d} pull nightly/{mariadb,centos} image e${d} ${project_name} mode=Scheduled cron=* */10 * * * * AND Set Suite Variable ${flag} ${true} + Sleep 40 + Exit For Loop If '${flag}' == '${true}' + END # After replication schedule is set, project should contain 2 images. Log To Console Sleep for 720 seconds...... diff --git a/tests/robot-cases/Group3-Upgrade/data.json b/tests/robot-cases/Group3-Upgrade/data.json index cc901e08a..d18155382 100644 --- a/tests/robot-cases/Group3-Upgrade/data.json +++ b/tests/robot-cases/Group3-Upgrade/data.json @@ -28,38 +28,15 @@ ], "endpoint":[ { - "url":"http://url", - "name":"endpoint1", - "user":"admin", - "pass":"Harbor12345", - "insecure":true, - "type":"harbor" - }, - { - "url":"https://hub.docker.com", + "url":"https://harbor-repo.vmware.com", "name":"endpoint_for_proxy_cache", "user":"", "pass":"", "insecure":false, - "type":"docker-hub" + "type":"harbor" } ], "replicationrule":[ - { - "project":"project1", - "endpoint":"endpoint1", - "trigger":"Manual", - "rulename":"rulename", - "is_src_registry":false, - "dest_namespace":"rule1-namespace", - "trigger_type":"event_based", - "cron":"", - "deletion":true, - "enabled":true, - "override":true, - "name_filters":"namefilter1", - "tag_filters":"tagfilter1" - }, { "project":"library", "endpoint":"endpoint_for_proxy_cache", @@ -203,7 +180,7 @@ "storage_unit_for_verify":"GB", "replications":{ "rulename":"ruleproject1", - "endpointname":"endpoint1", + "endpointname":"endpoint_for_proxy_cache", "trigger":"Manual" }, "labels":[ @@ -325,7 +302,7 @@ "storage_unit_for_verify":"TB", "replications":{ "rulename":"rulename1", - "endpointname":"endpoint1", + "endpointname":"endpoint_for_proxy_cache", "trigger":"Manual" }, "labels":[ @@ -392,7 +369,7 @@ "skip_cert_verify":false, "auth_header":"bbb", "enabled":true, - "notify_type":"http", + "notify_type":"slack", "name":"webhook28" }, "configuration":{ diff --git a/tests/robot-cases/Group3-Upgrade/prepare.py b/tests/robot-cases/Group3-Upgrade/prepare.py index 20f59168f..95d9ad3df 100644 --- a/tests/robot-cases/Group3-Upgrade/prepare.py +++ b/tests/robot-cases/Group3-Upgrade/prepare.py @@ -192,8 +192,6 @@ class HarborAPI: body=dict(body=payload) request(url+"targets", 'post', **body) elif kwargs["branch"] == 2: - if registry_type == "harbor": - endpointurl = endpoint_url payload = { "credential":{ "access_key":""+username+"", @@ -223,6 +221,7 @@ class HarborAPI: request(url+"policies/replication", 'post', **body) elif kwargs["branch"] == 2: r = request(url+"registries?name="+replicationrule["endpoint"]+"", 'get') + print("response:", r) targetid = r.json()[0]['id'] if replicationrule["is_src_registry"] is True: registry = r'"src_registry": { "id": '+str(targetid)+r'},' @@ -580,8 +579,6 @@ class HarborAPI: image_b = "busybox" repo_name_a, tag_a = push_image_to_project(project, args.endpoint, 'admin', 'Harbor12345', image_a, "latest") repo_name_b, tag_b = push_image_to_project(project, args.endpoint, 'admin', 'Harbor12345', image_b, "latest") - - #4. Push an index(IA) to Harbor by docker manifest CLI successfully; manifests = [args.endpoint+"/"+repo_name_a+":"+tag_a, args.endpoint+"/"+repo_name_b+":"+tag_b] index = args.endpoint+"/"+project+"/"+name+":"+tag docker_manifest_push_to_harbor(index, manifests, args.endpoint, 'admin', 'Harbor12345', cfg_file = args.libpath + "/update_docker_cfg.sh") @@ -640,6 +637,7 @@ def do_data_creation(): # Make sure to create endpoint first, it's for proxy cache project creation. for endpoint in data["endpoint"]: + print("endpoint:", endpoint) harborAPI.add_endpoint(endpoint["url"], endpoint["name"], endpoint["user"], endpoint["pass"], endpoint["insecure"], endpoint["type"], version=args.version) for distribution in data["distributions"]: @@ -648,8 +646,8 @@ def do_data_creation(): harborAPI.populate_projects(version=args.version) harborAPI.push_artifact_index(data["projects"][0]["name"], data["projects"][0]["artifact_index"]["name"], data["projects"][0]["artifact_index"]["tag"], version=args.version) + #pull_image("busybox", "redis", "haproxy", "alpine", "httpd:2") push_image_to_project(data["projects"][0]["name"], args.endpoint, 'admin', 'Harbor12345', "busybox", "latest") - push_image("busybox", data["projects"][0]["name"]) push_signed_image("alpine", data["projects"][0]["name"], "latest") for replicationrule in data["replicationrule"]: @@ -670,4 +668,3 @@ def do_data_creation(): harborAPI.add_sys_allowlist(data["configuration"]["deployment_security"], version=args.version) do_data_creation() - diff --git a/tests/robot-cases/Group3-Upgrade/run.sh b/tests/robot-cases/Group3-Upgrade/run.sh index b6266a704..142f0274c 100755 --- a/tests/robot-cases/Group3-Upgrade/run.sh +++ b/tests/robot-cases/Group3-Upgrade/run.sh @@ -1,7 +1,9 @@ #!/bin/bash IP=$1 HARBOR_VERSION=$2 +DOCKER_USER=$3 +DOCKER_PWD=$4 robot -v ip:$IP -v ip1: -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot cd /drone/tests/robot-cases/Group3-Upgrade -python ./prepare.py -e $IP -v $HARBOR_VERSION -l /drone/tests/apitests/python/ +DOCKER_USER=$DOCKER_USER DOCKER_PWD=$DOCKER_PWD python ./prepare.py -e $IP -v $HARBOR_VERSION -l /drone/tests/apitests/python/ diff --git a/tests/robot-cases/Group3-Upgrade/verify.robot b/tests/robot-cases/Group3-Upgrade/verify.robot index 1d967a6bd..b09253df2 100644 --- a/tests/robot-cases/Group3-Upgrade/verify.robot +++ b/tests/robot-cases/Group3-Upgrade/verify.robot @@ -49,7 +49,7 @@ Test Case - Upgrade Verify ${data}= Load Json From File ${CURDIR}${/}data.json Run Keyword Verify User ${data} Run Keyword Verify Project ${data} - #Run Keyword Verify Project Label ${data} + Run Keyword Verify Project Label ${data} Run Keyword Verify Project Metadata ${data} Run Keyword Verify Member Exist ${data} Run Keyword Verify Robot Account Exist ${data}