From 96a271f388048e361bd8e96326a0c9ab8635da99 Mon Sep 17 00:00:00 2001 From: "Danfeng Liu (c)" Date: Tue, 24 Sep 2019 18:35:19 +0800 Subject: [PATCH] In nightly migrate pipeline, after migration, should check all the data which were populated, for now, project meta data were populated, but were not been verified, so I add these verification in this PR. Signed-off-by: Danfeng Liu (c) --- tests/resources/Harbor-Pages/Project.robot | 8 +++- .../Harbor-Pages/Project_Elements.robot | 4 ++ tests/resources/Harbor-Pages/Verify.robot | 44 +++++++++++++++---- tests/robot-cases/Group3-Upgrade/data.json | 20 +++++---- tests/robot-cases/Group3-Upgrade/prepare.py | 5 ++- .../robot-cases/Group3-Upgrade/prepare_v17.py | 33 +++++++++----- 6 files changed, 81 insertions(+), 33 deletions(-) diff --git a/tests/resources/Harbor-Pages/Project.robot b/tests/resources/Harbor-Pages/Project.robot index 94ce7493e..5d1e8f36b 100644 --- a/tests/resources/Harbor-Pages/Project.robot +++ b/tests/resources/Harbor-Pages/Project.robot @@ -61,6 +61,10 @@ Switch To Replication Retry Element Click xpath=${project_replication_xpath} Sleep 1 +Switch To Project Configuration + Retry Element Click ${project_config_tabsheet} + Sleep 1 + Navigate To Projects Retry Element Click xpath=${projects_xpath} Sleep 2 @@ -82,7 +86,7 @@ Search Private Projects Make Project Private [Arguments] ${projectname} Go Into Project ${project name} - Retry Element Click ${project_config_tabsheet} + Switch To Project Configuration Retry Checkbox Should Be Selected ${project_config_public_checkbox} Retry Double Keywords When Error Retry Element Click ${project_config_public_checkbox_label} Retry Checkbox Should Not Be Selected ${project_config_public_checkbox} Retry Element Click //button[contains(.,'SAVE')] @@ -91,7 +95,7 @@ Make Project Private Make Project Public [Arguments] ${projectname} Go Into Project ${project name} - Retry Element Click ${project_config_tabsheet} + Switch To Project Configuration Retry Checkbox Should Not Be Selected ${project_config_public_checkbox} Retry Double Keywords When Error Retry Element Click ${project_config_public_checkbox_label} Retry Checkbox Should Be Selected ${project_config_public_checkbox} Retry Element Click //button[contains(.,'SAVE')] diff --git a/tests/resources/Harbor-Pages/Project_Elements.robot b/tests/resources/Harbor-Pages/Project_Elements.robot index a2200f66c..5fa922fb2 100644 --- a/tests/resources/Harbor-Pages/Project_Elements.robot +++ b/tests/resources/Harbor-Pages/Project_Elements.robot @@ -50,6 +50,10 @@ ${tag_images_btn} xpath=//hbr-repository//button[contains(.,'Images')] ${project_member_action_xpath} xpath=//*[@id='member-action'] ${project_member_set_role_xpath} xpath=//clr-dropdown-menu//label[contains(.,'SET ROLE')] ${project_config_public_checkbox} xpath=//input[@name='public'] +${project_config_content_trust_checkbox} xpath=//input[@name='content-trust'] +${project_config_scan_images_on_push_checkbox} xpath=//input[@name='scan-image-on-push'] +${project_config_prevent_vulnerable_images_from_running_checkbox} xpath=//input[@name='prevent-vulenrability-image-input'] +${project_config_severity_select} xpath=//select[@id='severity'] ${project_config_public_checkbox_label} xpath=//*[@id="clr-wrapper-public"]/div/clr-checkbox-wrapper/label ${project_config_prevent_vulenrability_checkbox_label} xpath=//*[@id='prevent-vulenrability-image']//clr-checkbox-wrapper//label ${project_config_system_wl_radio_input} xpath=//clr-radio-wrapper//label[contains(.,'System whitelist')] diff --git a/tests/resources/Harbor-Pages/Verify.robot b/tests/resources/Harbor-Pages/Verify.robot index 9337058be..00616e84a 100644 --- a/tests/resources/Harbor-Pages/Verify.robot +++ b/tests/resources/Harbor-Pages/Verify.robot @@ -27,8 +27,7 @@ Verify Project Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} :FOR ${project} IN @{project} \ Page Should Contain ${project} - #TO_DO: - #Verify project metadata. + Verify Project Metadata ${json} Close Browser Verify Image Tag @@ -40,11 +39,38 @@ Verify Image Tag \ @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image \ ${has_image} Set Variable If @{out_has_image}[0] == ${true} ${true} ${false} \ Go Into Project ${project} has_image=${has_image} - \ @{repo}= Get Value From Json ${json} $.projects[?(@name=${project})]..repo..name - \ Loop Image Repo @{repo} + \ @{repo}= Get Value From Json ${json} $.projects[?(@.name=${project})]..repo..name + \ Run Keyword If ${has_image} == ${true} Loop Image Repo @{repo} \ Navigate To Projects Close Browser +Verify Project Metadata + [Arguments] ${json} + @{project}= Get Value From Json ${json} $.projects.[*].name + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + :FOR ${project} IN @{project} + \ @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image + \ ${has_image} Set Variable If @{out_has_image}[0] == ${true} ${true} ${false} + \ Go Into Project ${project} has_image=${has_image} + \ Switch To Project Configuration + \ Verify Checkbox ${json} $.projects[?(@.name=${project})].configuration.public ${project_config_public_checkbox} + \ Verify Checkbox ${json} $.projects[?(@.name=${project})].configuration.enable_content_trust ${project_config_content_trust_checkbox} + \ Verify Checkbox ${json} $.projects[?(@.name=${project})].configuration.automatically_scan_images_on_push ${project_config_scan_images_on_push_checkbox} + \ Verify Checkbox ${json} $.projects[?(@.name=${project})].configuration.prevent_vulnerable_images_from_running ${project_config_prevent_vulnerable_images_from_running_checkbox} + \ ${ret} Get Selected List Value ${project_config_severity_select} + \ @{severity}= Get Value From Json ${json} $.projects[?(@.name=${project})].configuration.prevent_vlunerable_images_from_running_severity + \ Should Contain ${ret} @{severity}[0] + \ Navigate To Projects + Close Browser + +Verify Checkbox + [Arguments] ${json} ${key} ${checkbox} + @{out}= Get Value From Json ${json} ${key} + Run Keyword If '@{out}[0]'=='true' Checkbox Should Be Selected ${checkbox} + ... ELSE Checkbox Should Not Be Selected ${checkbox} + + Loop Image Repo [Arguments] @{repo} :For ${repo} In @{repo} @@ -60,7 +86,7 @@ Verify Member Exist \ ${has_image} Set Variable If @{out_has_image}[0] == ${true} ${true} ${false} \ Go Into Project ${project} has_image=${has_image} \ Switch To Member - \ @{members}= Get Value From Json ${json} $.projects[?(@name=${project})].member..name + \ @{members}= Get Value From Json ${json} $.projects[?(@.name=${project})].member..name \ Loop Member @{members} \ Navigate To Projects Close Browser @@ -76,10 +102,10 @@ Verify User System Admin Role Init Chrome Driver :FOR ${user} IN @{user} \ Sign In Harbor ${HARBOR_URL} ${user} ${HARBOR_PASSWORD} - \ Page Should Contain Administration + \ Page Should Contain Administration \ Logout Harbor Close Browser - + Verify System Label [Arguments] ${json} @{label}= Get Value From Json ${json} $..syslabel..name @@ -106,7 +132,7 @@ Verify Project Label \ \ Page Should Contain ${projectlabel} \ Navigate To Projects Close Browser - + Verify Endpoint [Arguments] ${json} @{endpoint}= Get Value From Json ${json} $.endpoint..name @@ -135,7 +161,7 @@ Verify Project Setting \ ${contenttrust}= Get Value From Json ${json} $.projects[?(@.name=${project})]..enable_content_trust \ ${preventrunning}= Get Value From Json ${json} $.projects[?(@.name=${project})]..prevent_vulnerable_images_from_running \ ${scanonpush}= Get Value From Json ${json} $.projects[?(@.name=${project})]..automatically_scan_images_on_push - \ Init Chrome Driver + \ Init Chrome Driver \ Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} \ @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image \ ${has_image} Set Variable If @{out_has_image}[0] == ${true} ${true} ${false} diff --git a/tests/robot-cases/Group3-Upgrade/data.json b/tests/robot-cases/Group3-Upgrade/data.json index 2bdb73941..07317ee3a 100644 --- a/tests/robot-cases/Group3-Upgrade/data.json +++ b/tests/robot-cases/Group3-Upgrade/data.json @@ -99,12 +99,12 @@ { "name":"busybox", "tag":"latest", - "signed":"False" + "signed":"false" }, { "name":"alpine", "tag":"latest", - "signed":"True" + "signed":"true" } ], "member":[ @@ -144,10 +144,11 @@ } ], "configuration":{ + "public":"true", "enable_content_trust":"true", "automatically_scan_images_on_push":"true", "prevent_vulnerable_images_from_running":"true", - "prevent_vlunerable_images_from_running_severity":"High" + "prevent_vlunerable_images_from_running_severity":"high" } }, { @@ -159,12 +160,12 @@ { "name":"busybox", "tag":"latest", - "signed":"False" + "signed":"false" }, { "name":"alpine", "tag":"latest", - "signed":"True" + "signed":"true" } ], "member":[ @@ -204,10 +205,11 @@ } ], "configuration":{ - "enable_content_trust":"True", - "automatically_scan_images_on_push":"True", - "prevent_vulnerable_images_from_running":"True", - "prevent_vlunerable_images_from_running_severity":"High" + "public":"false", + "enable_content_trust":"false", + "automatically_scan_images_on_push":"false", + "prevent_vulnerable_images_from_running":"true", + "prevent_vlunerable_images_from_running_severity":"medium" } } ] diff --git a/tests/robot-cases/Group3-Upgrade/prepare.py b/tests/robot-cases/Group3-Upgrade/prepare.py index bcb9407f1..852e1cb4d 100644 --- a/tests/robot-cases/Group3-Upgrade/prepare.py +++ b/tests/robot-cases/Group3-Upgrade/prepare.py @@ -68,13 +68,13 @@ class HarborAPI: body=dict(body=payload) request(url+"replication/policies", 'post', **body) - def update_project_setting(self, project, contenttrust, preventrunning, preventseverity, scanonpush): + def update_project_setting(self, project, public, contenttrust, preventrunning, preventseverity, scanonpush): r = request(url+"projects?name="+project+"", 'get') projectid = str(r.json()[0]['project_id']) payload = { "project_name": ""+project+"", "metadata": { - "public": "True", + "public": public, "enable_content_trust": contenttrust, "prevent_vulnerable_images_from_running": preventrunning, "prevent_vulnerable_images_from_running_severity": preventseverity, @@ -188,6 +188,7 @@ def do_data_creation(): replicationrule["rulename"]) for project in data["projects"]: harborAPI.update_project_setting(project["name"], + project["configuration"]["public"], project["configuration"]["enable_content_trust"], project["configuration"]["prevent_vulnerable_images_from_running"], project["configuration"]["prevent_vlunerable_images_from_running_severity"], diff --git a/tests/robot-cases/Group3-Upgrade/prepare_v17.py b/tests/robot-cases/Group3-Upgrade/prepare_v17.py index 8589f20d4..19b9fa2a6 100644 --- a/tests/robot-cases/Group3-Upgrade/prepare_v17.py +++ b/tests/robot-cases/Group3-Upgrade/prepare_v17.py @@ -58,19 +58,29 @@ class HarborAPI: body=dict(body=payload) request(url+"policies/replication", 'post', **body) - def update_project_setting(self, project, contenttrust, preventrunning, preventseverity, scanonpush): + def update_project_setting(self, project, public, contenttrust, preventrunning, preventseverity, scanonpush): r = request(url+"projects?name="+project+"", 'get') projectid = str(r.json()[0]['project_id']) - payload = { - "project_name": ""+project+"", - "metadata": { - "public": "True", - "enable_content_trust": contenttrust, - "prevent_vulnerable_images_from_running": preventrunning, - "prevent_vulnerable_images_from_running_severity": preventseverity, - "automatically_scan_images_on_push": scanonpush + if args.version == "1.6": + payload = { + "metadata": { + "public": public, + "enable_content_trust": contenttrust, + "prevent_vulnerable_images_from_running": preventrunning, + "prevent_vulnerable_images_from_running_severity": preventseverity, + "automatically_scan_images_on_push": scanonpush + } + } + else: + payload = { + "metadata": { + "public": public, + "enable_content_trust": contenttrust, + "prevent_vul": preventrunning, + "severity": preventseverity, + "auto_scan": scanonpush + } } - } body=dict(body=payload) request(url+"projects/"+projectid+"", 'put', **body) @@ -178,9 +188,10 @@ def do_data_creation(): replicationrule["rulename"]) for project in data["projects"]: harborAPI.update_project_setting(project["name"], + project["configuration"]["public"], project["configuration"]["enable_content_trust"], project["configuration"]["prevent_vulnerable_images_from_running"], - project["configuration"]["prevent_vlunerable_images_from_running_severity"], + project["configuration"]["prevent_vlunerable_images_from_running_severity"], project["configuration"]["automatically_scan_images_on_push"]) harborAPI.update_systemsetting(data["configuration"]["emailsetting"]["emailfrom"], data["configuration"]["emailsetting"]["emailserver"],