diff --git a/tests/apitests/python/library/helm.py b/tests/apitests/python/library/helm.py index 51ed13672..19002cfd7 100644 --- a/tests/apitests/python/library/helm.py +++ b/tests/apitests/python/library/helm.py @@ -35,7 +35,6 @@ def helm_chart_push_to_harbor(chart_file, archive, harbor_server, project, repo_ helm_save(archive, harbor_server, project, repo_name) return helm_push(harbor_server, project, repo_name, version) -# helm repo add --ca-file /ca/server.crt --username=${user} --password=${pwd} ${helm_repo_name} ${harbor_url}/chartrepo/${project_name} def helm2_add_repo(helm_repo_name, harbor_url, project, username, password): command = ["helm2", "repo", "add", "--username=" + username, "--password=" + password, helm_repo_name, harbor_url + "/chartrepo/" + project] print "Command: ", command diff --git a/tests/apitests/python/library/project.py b/tests/apitests/python/library/project.py index 9b5f9853c..aa572942b 100644 --- a/tests/apitests/python/library/project.py +++ b/tests/apitests/python/library/project.py @@ -183,7 +183,7 @@ class Project(base.Base): access_list = [] resource_by_project_id = "/project/"+str(project_id)+"/repository" resource_helm_by_project_id = "/project/"+str(project_id)+"/helm-chart" - resource_helm__create_by_project_id = "/project/"+str(project_id)+"/helm-chart-version" + resource_helm_create_by_project_id = "/project/"+str(project_id)+"/helm-chart-version" action_pull = "pull" action_push = "push" action_read = "read" @@ -198,7 +198,7 @@ class Project(base.Base): robotAccountAccess = swagger_client.RobotAccountAccess(resource = resource_helm_by_project_id, action = action_read) access_list.append(robotAccountAccess) if has_chart_create_right is True: - robotAccountAccess = swagger_client.RobotAccountAccess(resource = resource_helm__create_by_project_id, action = action_create) + robotAccountAccess = swagger_client.RobotAccountAccess(resource = resource_helm_create_by_project_id, action = action_create) access_list.append(robotAccountAccess) robotAccountCreate = swagger_client.RobotAccountCreate(robot_name, robot_desc, expires_at, access_list) diff --git a/tests/apitests/python/test_garbage_collection.py b/tests/apitests/python/test_garbage_collection.py index fd943b275..2ea0383aa 100644 --- a/tests/apitests/python/test_garbage_collection.py +++ b/tests/apitests/python/test_garbage_collection.py @@ -11,6 +11,8 @@ from library.repository import Repository from library.repository import push_image_to_project from testutils import harbor_server from library.base import _assert_status_code +from library.repository import push_special_image_to_project +from library.artifact import Artifact class TestProjects(unittest.TestCase): @classmethod @@ -19,12 +21,16 @@ class TestProjects(unittest.TestCase): self.project = Project() self.user = User() self.repo = Repository() + self.artifact = Artifact() + self.repo_name = "test_repo" + self.repo_name_untag = "test_untag" + self.tag = "v1.0" @classmethod def tearDown(self): print "Case completed" - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + @unittest.skipIf(TEARDOWN == True, "Test data won't be erased.") def test_ClearData(self): #2. Delete project(PA); self.project.delete_project(TestProjects.project_gc_id, **TestProjects.USER_GC_CLIENT) @@ -38,13 +44,17 @@ class TestProjects(unittest.TestCase): Garbage Collection Test step and expected result: 1. Create a new user(UA); - 2. Create a new project(PA) by user(UA); - 3. Push a new image(IA) in project(PA) by admin; - 4. Delete repository(RA) by user(UA); - 5. Get repository by user(UA), it should get nothing; - 6. Tigger garbage collection operation; - 7. Check garbage collection job was finished; - 8. Get garbage collection log, check there is number of files was deleted. + 2. Create project(PA) and project(PB) by user(UA); + 3. Push a image in project(PA) and then delete repository by admin; + 4. Get repository by user(UA), it should get nothing; + 5. Tigger garbage collection operation; + 6. Check garbage collection job was finished; + 7. Get garbage collection log, check there is a number of files was deleted; + 8. Push a image in project(PB) by admin and delete the only tag; + 9. Tigger garbage collection operation; + 10. Check garbage collection job was finished; + 11. Repository with untag image should be still there; + 12. But no any artifact in repository anymore. Tear down: 1. Delete project(PA); 2. Delete user(UA). @@ -59,27 +69,48 @@ class TestProjects(unittest.TestCase): TestProjects.USER_GC_CLIENT=dict(endpoint = url, username = user_gc_name, password = user_gc_password) - #2. Create a new project(PA) by user(UA); + #2. Create project(PA) and project(PB) by user(UA); TestProjects.project_gc_id, TestProjects.project_gc_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_GC_CLIENT) + TestProjects.project_gc_untag_id, TestProjects.project_gc_untag_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_GC_CLIENT) - #3. Push a new image(IA) in project(PA) by admin; - repo_name, _ = push_image_to_project(TestProjects.project_gc_name, harbor_server, admin_name, admin_password, "tomcat", "latest") + #3. Push a image in project(PA) and then delete repository by admin; + push_special_image_to_project(TestProjects.project_gc_name, harbor_server, admin_name, admin_password, self.repo_name, ["latest", "v1.2.3"]) + self.repo.delete_repoitory(TestProjects.project_gc_name, self.repo_name, **TestProjects.USER_GC_CLIENT) - #4. Delete repository(RA) by user(UA); - self.repo.delete_repoitory(TestProjects.project_gc_name, repo_name.split('/')[1], **TestProjects.USER_GC_CLIENT) - - #5. Get repository by user(UA), it should get nothing; + #4. Get repository by user(UA), it should get nothing; repo_data = self.repo.list_repositories(TestProjects.project_gc_name, **TestProjects.USER_GC_CLIENT) _assert_status_code(len(repo_data), 0) - #6. Tigger garbage collection operation; + #5. Tigger garbage collection operation; gc_id = self.system.gc_now(**ADMIN_CLIENT) - #7. Check garbage collection job was finished; + #6. Check garbage collection job was finished; self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT) - #8. Get garbage collection log, check there is number of files was deleted. + #7. Get garbage collection log, check there is a number of files was deleted; self.system.validate_deletion_success(gc_id, **ADMIN_CLIENT) + #8. Push a image in project(PB) by admin and delete the only tag; + push_special_image_to_project(TestProjects.project_gc_untag_name, harbor_server, admin_name, admin_password, self.repo_name_untag, [self.tag]) + self.artifact.delete_tag(TestProjects.project_gc_untag_name, self.repo_name_untag, self.tag, self.tag, **ADMIN_CLIENT) + + #9. Tigger garbage collection operation; + gc_id = self.system.gc_now(**ADMIN_CLIENT) + + #10. Check garbage collection job was finished; + self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT) + + #11. Repository with untag image should be still there; + repo_data_untag = self.repo.list_repositories(TestProjects.project_gc_untag_name, **TestProjects.USER_GC_CLIENT) + print "repo_data_untag:", repo_data_untag + _assert_status_code(len(repo_data_untag), 1) + self.assertEqual(TestProjects.project_gc_untag_name + "/" + self.repo_name_untag , repo_data_untag[0].name) + + #12. But no any artifact in repository anymore. + artifacts = self.artifact.list_artifacts(TestProjects.project_gc_untag_name, self.repo_name_untag, **TestProjects.USER_GC_CLIENT) + self.assertEqual(artifacts,[]) + + + if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/apitests/python/test_list_helm_charts.py b/tests/apitests/python/test_list_helm_charts.py index 0ae2a3fc8..a8db08923 100644 --- a/tests/apitests/python/test_list_helm_charts.py +++ b/tests/apitests/python/test_list_helm_charts.py @@ -11,14 +11,9 @@ from library.chart import Chart class TestProjects(unittest.TestCase): @classmethod def setUp(self): - chart = Chart() - self.chart= chart - - project = Project() - self.project= project - - user = User() - self.user= user + self.chart= Chart() + self.project= Project() + self.user= User() @classmethod def tearDown(self): diff --git a/tests/apitests/python/test_push_chart_by_helm2_helm3_with_robot_Account.py b/tests/apitests/python/test_push_chart_by_helm2_helm3_with_robot_Account.py index 785e588e8..0cb77f02b 100644 --- a/tests/apitests/python/test_push_chart_by_helm2_helm3_with_robot_Account.py +++ b/tests/apitests/python/test_push_chart_by_helm2_helm3_with_robot_Account.py @@ -5,26 +5,26 @@ import unittest import library.repository import library.helm -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, CHART_API_CLIENT from testutils import harbor_server from testutils import TEARDOWN from library.project import Project from library.user import User -from library.repository import Repository -from library.artifact import Artifact +from library.chart import Chart class TestProjects(unittest.TestCase): @classmethod def setUpClass(self): self.project= Project() self.user= User() - self.artifact = Artifact() - self.repo= Repository() + self.chart= Chart() self.url = ADMIN_CLIENT["endpoint"] + self.chart_api_url = CHART_API_CLIENT['endpoint'] self.user_push_chart_password = "Aa123456" self.chart_file = "https://storage.googleapis.com/harbor-builds/helm-chart-test-files/harbor-0.2.0.tgz" self.archive = "harbor/" + self.CHART_NAME=self.archive.replace("/", "") self.verion = "0.2.0" self.chart_repo_name = "chart_local" self.repo_name = "harbor_api_test" @@ -54,15 +54,15 @@ class TestProjects(unittest.TestCase): print "#1. Create user(UA);" TestProjects.user_id, user_name = self.user.create_user(user_password = self.user_push_chart_password, **ADMIN_CLIENT) - TestProjects.USER_RA_CLIENT=dict(endpoint = self.url, username = user_name, password = self.user_push_chart_password) - + TestProjects.USER_CLIENT=dict(endpoint = self.url, username = user_name, password = self.user_push_chart_password) + TestProjects.API_CHART_CLIENT=dict(endpoint = self.chart_api_url, username = user_name, password = self.user_push_chart_password) print "#2. Create private project(PA) with user(UA);" - TestProjects.project_id, TestProjects.project_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_RA_CLIENT) + TestProjects.project_id, TestProjects.project_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_CLIENT) print "#3. Create a new robot account(RA) with full priviliges in project(PA) with user(UA);" robot_id, robot_account = self.project.add_project_robot_account(TestProjects.project_id, TestProjects.project_name, - 2441000531 ,**TestProjects.USER_RA_CLIENT) + 2441000531 ,**TestProjects.USER_CLIENT) print robot_account.name print robot_account.token @@ -71,7 +71,7 @@ class TestProjects(unittest.TestCase): library.helm.helm2_push(self.chart_repo_name, self.chart_file, TestProjects.project_name, robot_account.name, robot_account.token) print "#5. Get chart repositry from project(PA) successfully;" - # Depend on issue #12252 + self.chart.chart_should_exist(TestProjects.project_name, self.CHART_NAME, **TestProjects.API_CHART_CLIENT) print "#6. Push chart to project(PA) by Helm3 CLI with robot account(RA);" chart_cli_ret = library.helm.helm_chart_push_to_harbor(self.chart_file, self.archive, harbor_server, TestProjects.project_name, self.repo_name, self.verion, robot_account.name, robot_account.token) diff --git a/tests/apitests/python/test_user_view_logs.py b/tests/apitests/python/test_user_view_logs.py index 6dddd3c7e..777b0c97b 100644 --- a/tests/apitests/python/test_user_view_logs.py +++ b/tests/apitests/python/test_user_view_logs.py @@ -90,6 +90,7 @@ class TestProjects(unittest.TestCase): format(user_user_view_logs_name, project_user_view_logs_name, "artifact", operation, log_count)) #4.1 Delete repository(RA) by user(UA); self.repo.delete_repoitory(project_user_view_logs_name, repo_name.split('/')[1], **TestProjects.USER_USER_VIEW_LOGS_CLIENT) + time.sleep(6) #4.2 In project(PA), there should be 1 'delete' log record; operation = "delete" diff --git a/tests/resources/Harbor-Pages/Project-Artifact.robot b/tests/resources/Harbor-Pages/Project-Artifact.robot index 8f8cb2da0..ae60f65c0 100644 --- a/tests/resources/Harbor-Pages/Project-Artifact.robot +++ b/tests/resources/Harbor-Pages/Project-Artifact.robot @@ -32,16 +32,15 @@ Should Not Contain Tag Add A New Tag [Arguments] ${tag} - Retry Element Click ${add_tag_button} + Retry Double Keywords When Error Retry Element Click ${add_tag_button} Retry Wait Element ${tag_name_xpath} Retry Text Input ${tag_name_xpath} ${tag} - Retry Element Click ${add_ok_button} + Retry Double Keywords When Error Retry Element Click ${add_ok_button} Should Contain Tag ${tag} Delete A Tag [Arguments] ${tag} Retry Element Click xpath=//clr-dg-row[contains(.,'${tag}')]//clr-checkbox-wrapper//label[contains(@class,'clr-control-label')] - Retry Element Click ${delete_tag_button} - Retry Wait Until Page Contains Element ${dialog_delete_button} - Retry Element Click ${dialog_delete_button} + Retry Double Keywords When Error Retry Element Click ${delete_tag_button} Retry Wait Until Page Contains Element ${dialog_delete_button} + Retry Double Keywords When Error Retry Element Click ${dialog_delete_button} Should Not Contain Tag ${tag} Should Contain Artifact Retry Wait Until Page Contains Element xpath=//artifact-list-tab//clr-dg-row//a[contains(.,'sha256')] diff --git a/tests/resources/Harbor-Pages/Project.robot b/tests/resources/Harbor-Pages/Project.robot index e00c5ae05..784e482a7 100644 --- a/tests/resources/Harbor-Pages/Project.robot +++ b/tests/resources/Harbor-Pages/Project.robot @@ -196,6 +196,18 @@ Do Log Advanced Search ${rc} = Get Element Count //audit-log//clr-dg-row Should Be Equal As Integers ${rc} 0 +Retry Click Repo Name + [Arguments] ${repo_name_element} + :For ${n} IN RANGE 1 10 + \ ${out} Run Keyword And Ignore Error Retry Double Keywords When Error Retry Element Click ${repo_name_element} Retry Wait Element ${tag_table_column_vulnerabilities} + \ Exit For Loop If '${out[0]}'=='PASS' + Should Be Equal As Strings '${out[0]}' 'PASS' + + :For ${n} IN RANGE 1 10 + \ ${out} Run Keyword And Ignore Error Retry Wait Until Page Not Contains Element ${repo_list_spinner} + \ Exit For Loop If '${out[0]}'=='PASS' + Should Be Equal As Strings '${out[0]}' 'PASS' + Go Into Repo [Arguments] ${repoName} Sleep 2 @@ -206,13 +218,12 @@ Go Into Repo \ Retry Clear Element Text ${repo_search_input} \ Retry Text Input ${repo_search_input} ${repoName} \ ${out} Run Keyword And Ignore Error Retry Wait Until Page Contains Element ${repo_name_element} - \ Exit For Loop If '${out[0]}'=='PASS' - \ Capture Page Screenshot gointo_${repoName}.png \ Sleep 2 - Retry Double Keywords When Error Retry Element Click ${repo_name_element} Retry Wait Until Page Not Contains Element ${repo_name_element} - Retry Wait Element ${tag_table_column_vulnerabilities} - Retry Wait Element ${tag_table_column_size} - Capture Page Screenshot gointo_${repoName}.png + \ Continue For Loop If '${out[0]}'=='FAIL' + \ ${out} Retry Click Repo Name ${repo_name_element} + \ Sleep 2 + \ Exit For Loop + Click Index Achieve [Arguments] ${tag_name} diff --git a/tests/robot-cases/Group0-BAT/API_DB.robot b/tests/robot-cases/Group0-BAT/API_DB.robot index 9052fccee..00bf02355 100644 --- a/tests/robot-cases/Group0-BAT/API_DB.robot +++ b/tests/robot-cases/Group0-BAT/API_DB.robot @@ -17,93 +17,123 @@ ${SERVER_API_ENDPOINT} ${SERVER_URL}/api &{SERVER_CONFIG} endpoint=${SERVER_API_ENDPOINT} verify_ssl=False *** Test Cases *** + Test Case - Garbage Collection [Tags] gc Harbor API Test ./tests/apitests/python/test_garbage_collection.py + Test Case - Add Private Project Member and Check User Can See It [Tags] private_member Harbor API Test ./tests/apitests/python/test_add_member_to_private_project.py + Test Case - Delete a Repository of a Certain Project Created by Normal User [Tags] del_repo Harbor API Test ./tests/apitests/python/test_del_repo.py + Test Case - Add a System Global Label to a Certain Tag [Tags] global_lbl Harbor API Test ./tests/apitests/python/test_add_sys_label_to_tag.py + Test Case - Add Replication Rule [Tags] replic_rule Harbor API Test ./tests/apitests/python/test_add_replication_rule.py + Test Case - Edit Project Creation [Tags] pro_creation Harbor API Test ./tests/apitests/python/test_edit_project_creation.py + Test Case - Manage Project Member [Tags] member Harbor API Test ./tests/apitests/python/test_manage_project_member.py + Test Case - Project Level Policy Content Trust [Tags] content_trust Harbor API Test ./tests/apitests/python/test_project_level_policy_content_trust.py + Test Case - User View Logs [Tags] view_logs Harbor API Test ./tests/apitests/python/test_user_view_logs.py + Test Case - List Helm Charts [Tags] list_helm_charts Harbor API Test ./tests/apitests/python/test_list_helm_charts.py + Test Case - Assign Sys Admin [Tags] assign_adin Harbor API Test ./tests/apitests/python/test_assign_sys_admin.py + Test Case - Copy Artifact Outside Project [Tags] copy_artifact Harbor API Test ./tests/apitests/python/test_copy_artifact_outside_project.py + Test Case - Robot Account [Tags] robot_account Harbor API Test ./tests/apitests/python/test_robot_account.py + Test Case - Sign A Image [Tags] sign_image Harbor API Test ./tests/apitests/python/test_sign_image.py + Test Case - Project Quota [Tags] quota Harbor API Test ./tests/apitests/python/test_project_quota.py + Test Case - System Level CVE Whitelist [Tags] sys_cve Harbor API Test ./tests/apitests/python/test_sys_cve_whitelists.py + Test Case - Project Level CVE Whitelist [Tags] pro_cve Harbor API Test ./tests/apitests/python/test_project_level_cve_whitelist.py + Test Case - Tag Retention [Tags] tag_retention Harbor API Test ./tests/apitests/python/test_retention.py + Test Case - Health Check [Tags] health Harbor API Test ./tests/apitests/python/test_health_check.py + Test Case - Push Index By Docker Manifest [Tags] push_index Harbor API Test ./tests/apitests/python/test_push_index_by_docker_manifest.py + Test Case - Push Chart By Helm3 Chart CLI [Tags] push_chart Harbor API Test ./tests/apitests/python/test_push_chart_by_helm3_chart_cli.py + Test Case - Push Cnab Bundle [Tags] push_cnab Harbor API Test ./tests/apitests/python/test_push_cnab_bundle.py + Test Case - Create/Delete tag [Tags] tag_cuid Harbor API Test ./tests/apitests/python/test_create_delete_tag.py + Test Case - Scan Image [Tags] scan Harbor API Test ./tests/apitests/python/test_scan_image_artifact.py + Test Case - Scan All Images [Tags] scan_all Harbor API Test ./tests/apitests/python/test_system_level_scan_all.py + Test Case - Registry API [Tags] reg_api Harbor API Test ./tests/apitests/python/test_registry_api.py + Test Case - Push Image With Special Name [Tags] special_repo_name Harbor API Test ./tests/apitests/python/test_push_image_with_special_name.py + Test Case - Push Artifact With ORAS CLI [Tags] oras Harbor API Test ./tests/apitests/python/test_push_files_by_oras.py + Test Case - Push Singularity file With Singularity CLI [Tags] singularity Harbor API Test ./tests/apitests/python/test_push_sif_by_singularity.py + Test Case - Push Chart File To Chart Repository By Helm V2 With Robot Account [Tags] helm2 Harbor API Test ./tests/apitests/python/test_push_chart_by_helm2_helm3_with_robot_Account.py diff --git a/tests/robot-cases/Group1-Nightly/Common.robot b/tests/robot-cases/Group1-Nightly/Common.robot index b9354969a..0af7b01de 100644 --- a/tests/robot-cases/Group1-Nightly/Common.robot +++ b/tests/robot-cases/Group1-Nightly/Common.robot @@ -526,11 +526,8 @@ Test Case - Tag CRUD Should Contain Tag latest # add more than one tag Add A New Tag 123 - Should Contain Tag 123 Add A New Tag 456 - Should Contain Tag 456 - Delete A Tag latest - Should Not Contain Tag latest + Delete A Tag latest Close Browser Test Case - Tag Retention