diff --git a/tests/resources/Harbor-Pages/Project-Members.robot b/tests/resources/Harbor-Pages/Project-Members.robot index b94fc36503..f8bb88c943 100644 --- a/tests/resources/Harbor-Pages/Project-Members.robot +++ b/tests/resources/Harbor-Pages/Project-Members.robot @@ -61,7 +61,7 @@ Change Project Member Role [Arguments] ${project} ${user} ${role} Retry Element Click xpath=//clr-dg-cell//a[contains(.,'${project}')] Retry Element Click xpath=${project_member_tag_xpath} - Retry Element Click xpath=//project-detail//clr-dg-row[contains(.,'${user}')]//label + Retry Element Click xpath=//project-detail//clr-dg-row[contains(.,'${user}')]//clr-checkbox-wrapper #change role Retry Element Click ${project_member_action_xpath} Retry Element Click //button[contains(.,'${role}')] diff --git a/tests/resources/Harbor-Pages/Replication.robot b/tests/resources/Harbor-Pages/Replication.robot index c8e1a3bd75..038a727b4d 100644 --- a/tests/resources/Harbor-Pages/Replication.robot +++ b/tests/resources/Harbor-Pages/Replication.robot @@ -201,7 +201,7 @@ Delete Endpoint Retry Element Click ${endpoint_filter_search} Retry Text Input ${endpoint_filter_input} ${name} #click checkbox before target endpoint - Retry Element Click //clr-dg-row[contains(.,'${name}')]//label + Retry Double Keywords When Error Retry Element Click //clr-dg-row[contains(.,'${name}')]//clr-checkbox-wrapper Retry Wait Element ${action_bar_delete} Retry Element Click ${action_bar_delete} Wait Until Page Contains Element ${dialog_delete} Retry Element Click ${dialog_delete} diff --git a/tests/resources/Harbor-Pages/ToolKit.robot b/tests/resources/Harbor-Pages/ToolKit.robot index b16ea233d6..e5c7f863e2 100644 --- a/tests/resources/Harbor-Pages/ToolKit.robot +++ b/tests/resources/Harbor-Pages/ToolKit.robot @@ -76,7 +76,7 @@ Multi-delete User Multi-delete Member [Arguments] @{obj} :For ${obj} in @{obj} - \ Retry Element Click //clr-dg-row[contains(.,'${obj}')]//label + \ Retry Element Click //clr-dg-row[contains(.,'${obj}')]//clr-checkbox-wrapper Retry Double Keywords When Error Retry Element Click ${member_action_xpath} Retry Wait Until Page Contains Element ${delete_action_xpath} Retry Double Keywords When Error Retry Element Click ${delete_action_xpath} Retry Wait Until Page Contains Element ${delete_btn} Retry Double Keywords When Error Retry Element Click ${delete_btn} Retry Wait Until Page Not Contains Element ${delete_btn} diff --git a/tests/robot-cases/Group1-Nightly/Chartmuseum.robot b/tests/robot-cases/Group1-Nightly/Chartmuseum.robot new file mode 100644 index 0000000000..82701d0c28 --- /dev/null +++ b/tests/robot-cases/Group1-Nightly/Chartmuseum.robot @@ -0,0 +1,35 @@ + +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +*** Settings *** +Documentation Harbor BATs +Resource ../../resources/Util.robot +Default Tags Nightly + +*** Variables *** +${HARBOR_URL} https://${ip} +${SSH_USER} root +${HARBOR_ADMIN} admin + +*** Test Cases *** +Test Case - List Helm Charts And Delete Chart Files + Body Of List Helm Charts + +Test Case - Helm CLI Push + Init Chrome Driver + ${user}= Set Variable user004 + ${pwd}= Set Variable Test1@34 + Sign In Harbor ${HARBOR_URL} ${user} ${pwd} + Helm CLI Push Without Sign In Harbor ${user} ${pwd} \ No newline at end of file diff --git a/tests/robot-cases/Group1-Nightly/Clair.robot b/tests/robot-cases/Group1-Nightly/Clair.robot new file mode 100644 index 0000000000..b98bf24519 --- /dev/null +++ b/tests/robot-cases/Group1-Nightly/Clair.robot @@ -0,0 +1,145 @@ + +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +*** Settings *** +Documentation Harbor BATs +Resource ../../resources/Util.robot +Default Tags Nightly + +*** Variables *** +${HARBOR_URL} https://${ip} +${SSH_USER} root +${HARBOR_ADMIN} admin + +*** Test Cases *** +Test Case - Vulnerability Data Not Ready +#This case must run before vulnerability db ready + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Go Into Project library has_image=${false} + Vulnerability Not Ready Project Hint + Switch To Vulnerability Page + Vulnerability Not Ready Config Hint + +Test Case - Disable Scan Schedule + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To Vulnerability Page + Disable Scan Schedule + Logout Harbor + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To Vulnerability Page + Retry Wait Until Page Contains None + Close Browser + +Test Case - Scan A Tag In The Repo + Body Of Scan A Tag In The Repo + +Test Case - Scan As An Unprivileged User + Init Chrome Driver + ${d}= get current date result_format=%m%s + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library hello-world + + Sign In Harbor ${HARBOR_URL} user024 Test1@34 + Go Into Project library + Go Into Repo hello-world + Select Object latest + Scan Is Disabled + Close Browser + +Test Case - Scan Image With Empty Vul + Init Chrome Driver + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library busybox + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Go Into Project library + Go Into Repo busybox + Scan Repo latest Succeed + Move To Summary Chart + Wait Until Page Contains Unknow + Close Browser + +Test Case - Manual Scan All + Init Chrome Driver + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library redis + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To Vulnerability Page + Trigger Scan Now + Navigate To Projects + Go Into Project library + Go Into Repo redis + Summary Chart Should Display latest + Close Browser + +Test Case - View Scan Error + Init Chrome Driver + ${d}= get current date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} user026 Test1@34 + Create An New Project project${d} + Push Image ${ip} user026 Test1@34 project${d} vmware/photon:1.0 + Go Into Project project${d} + Go Into Repo project${d}/vmware/photon + Scan Repo 1.0 Fail + View Scan Error Log + Close Browser + +Test Case - Scan Image On Push + [Tags] run-once + Wait Unitl Vul Data Ready ${HARBOR_URL} 7200 30 + Init Chrome Driver + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library hello-world + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Go Into Project library + Goto Project Config + Enable Scan On Push + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library memcached + Navigate To Projects + Go Into Project library + Go Into Repo memcached + Summary Chart Should Display latest + Close Browser + +Test Case - View Scan Results + [Tags] run-once + Init Chrome Driver + ${d}= get current date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} user025 Test1@34 + Create An New Project project${d} + Push Image ${ip} user025 Test1@34 project${d} tomcat + Go Into Project project${d} + Go Into Repo project${d}/tomcat + Scan Repo latest Succeed + Summary Chart Should Display latest + View Repo Scan Details + Close Browser + +Test Case - Project Level Image Serverity Policy + [Tags] run-once + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + ${d}= get current date result_format=%m%s + ${sha256}= Set Variable 68b49a280d2fbe9330c0031970ebb72015e1272dfa25f0ed7557514f9e5ad7b7 + ${image}= Set Variable postgres + Create An New Project project${d} + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} sha256=${sha256} tag=${sha256} + Go Into Project project${d} + Go Into Repo ${image} + Scan Repo ${sha256} Succeed + Navigate To Projects + Go Into Project project${d} + Set Vulnerabilty Serverity 0 + Cannot pull image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} tag=${sha256} + Close Browser diff --git a/tests/robot-cases/Group1-Nightly/Common.robot b/tests/robot-cases/Group1-Nightly/Common.robot index 2be78f4ce4..883c88aaab 100644 --- a/tests/robot-cases/Group1-Nightly/Common.robot +++ b/tests/robot-cases/Group1-Nightly/Common.robot @@ -29,45 +29,6 @@ Test Case - Sign With Admin Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Close Browser -Test Case - Vulnerability Data Not Ready -#This case must run before vulnerability db ready - Init Chrome Driver - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Go Into Project library has_image=${false} - Vulnerability Not Ready Project Hint - Switch To Vulnerability Page - Vulnerability Not Ready Config Hint - -Test Case - Garbage Collection - Init Chrome Driver - ${d}= Get Current Date result_format=%m%s - - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world - Sleep 2 - Go Into Project project${d} - Delete Repo project${d} - - Switch To Garbage Collection - Click GC Now - Logout Harbor - Sleep 2 - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Switch To Garbage Collection - Sleep 1 - Switch To GC History - Retry Wait Until Page Contains Finished - - ${rc} ${output}= Run And Return Rc And Output curl -u ${HARBOR_ADMIN}:${HARBOR_PASSWORD} -i --insecure -H "Content-Type: application/json" -X GET "https://${ip}/api/system/gc/1/log" - Log To Console ${output} - Should Be Equal As Integers ${rc} 0 - Should Contain ${output} 3 blobs and 0 manifests eligible for deletion - #Should Contain ${output} Deleting blob: - Should Contain ${output} success to run gc in job. - - Close Browser - Test Case - Create An New Project Init Chrome Driver ${d}= Get Current Date result_format=%m%s @@ -174,21 +135,6 @@ Test Case - Project Level Policy Public Project Should Be Public project${d} Close Browser -Test Case - Project Level Policy Content Trust - Init Chrome Driver - ${d}= Get Current Date result_format=%m%s - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world:latest - Go Into Project project${d} - Goto Project Config - Click Content Trust - Save Project Config - # Verify - Content Trust Should Be Selected - Cannot Pull Unsigned Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world:latest - Close Browser - Test Case - Verify Download Ca Link Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} @@ -256,17 +202,6 @@ Test Case - Delete Label Delete A Label label_${d} Close Browser -Test Case - Disable Scan Schedule - Init Chrome Driver - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Switch To Vulnerability Page - Disable Scan Schedule - Logout Harbor - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Switch To Vulnerability Page - Retry Wait Until Page Contains None - Close Browser - Test Case - User View Projects Init Chrome Driver ${d}= Get Current Date result_format=%m%s @@ -489,70 +424,6 @@ Test Case - Developer Operate Labels Page Should Not Contain Element xpath=//a[contains(.,'Labels')] Close Browser -Test Case - Scan A Tag In The Repo - Body Of Scan A Tag In The Repo - -Test Case - Scan As An Unprivileged User - Init Chrome Driver - ${d}= get current date result_format=%m%s - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library hello-world - - Sign In Harbor ${HARBOR_URL} user024 Test1@34 - Go Into Project library - Go Into Repo hello-world - Select Object latest - Scan Is Disabled - Close Browser - -Test Case - Scan Image With Empty Vul - Init Chrome Driver - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library busybox - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Go Into Project library - Go Into Repo busybox - Scan Repo latest Succeed - Move To Summary Chart - Wait Until Page Contains Unknow - Close Browser - -Test Case - Manual Scan All - Init Chrome Driver - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library redis - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Switch To Vulnerability Page - Trigger Scan Now - Navigate To Projects - Go Into Project library - Go Into Repo redis - Summary Chart Should Display latest - Close Browser - -Test Case - View Scan Error - Init Chrome Driver - ${d}= get current date result_format=%m%s - - Sign In Harbor ${HARBOR_URL} user026 Test1@34 - Create An New Project project${d} - Push Image ${ip} user026 Test1@34 project${d} vmware/photon:1.0 - Go Into Project project${d} - Go Into Repo project${d}/vmware/photon - Scan Repo 1.0 Fail - View Scan Error Log - Close Browser - -Test Case - List Helm Charts And Delete Chart Files - Body Of List Helm Charts - -Test Case - Helm CLI Push - Init Chrome Driver - ${user}= Set Variable user004 - ${pwd}= Set Variable Test1@34 - Sign In Harbor ${HARBOR_URL} ${user} ${pwd} - Helm CLI Push Without Sign In Harbor ${user} ${pwd} - -Test Case - Admin Push Signed Image - Body Of Admin Push Signed Image - Test Case - Retag A Image Tag Init Chrome Driver ${random_num1}= Get Current Date result_format=%m%s @@ -578,49 +449,3 @@ Test Case - Retag A Image Tag Page Should Contain Element xpath=${tag_value_xpath} Close Browser -Test Case - Scan Image On Push - Wait Unitl Vul Data Ready ${HARBOR_URL} 7200 30 - Init Chrome Driver - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library hello-world - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Go Into Project library - Goto Project Config - Enable Scan On Push - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library memcached - Navigate To Projects - Go Into Project library - Go Into Repo memcached - Summary Chart Should Display latest - Close Browser - -Test Case - View Scan Results - Init Chrome Driver - ${d}= get current date result_format=%m%s - - Sign In Harbor ${HARBOR_URL} user025 Test1@34 - Create An New Project project${d} - Push Image ${ip} user025 Test1@34 project${d} tomcat - Go Into Project project${d} - Go Into Repo project${d}/tomcat - Scan Repo latest Succeed - Summary Chart Should Display latest - View Repo Scan Details - Close Browser - -Test Case - Project Level Image Serverity Policy - Init Chrome Driver - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - ${d}= get current date result_format=%m%s - ${sha256}= Set Variable 68b49a280d2fbe9330c0031970ebb72015e1272dfa25f0ed7557514f9e5ad7b7 - ${image}= Set Variable postgres - Create An New Project project${d} - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} sha256=${sha256} tag=${sha256} - Go Into Project project${d} - Go Into Repo ${image} - Scan Repo ${sha256} Succeed - Navigate To Projects - Go Into Project project${d} - Set Vulnerabilty Serverity 0 - Cannot pull image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} tag=${sha256} - Close Browser - diff --git a/tests/robot-cases/Group1-Nightly/Common_GC.robot b/tests/robot-cases/Group1-Nightly/Common_GC.robot new file mode 100644 index 0000000000..c1a3ee9933 --- /dev/null +++ b/tests/robot-cases/Group1-Nightly/Common_GC.robot @@ -0,0 +1,55 @@ + +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +*** Settings *** +Documentation Harbor BATs +Resource ../../resources/Util.robot +Default Tags Nightly + +*** Variables *** +${HARBOR_URL} https://${ip} +${SSH_USER} root +${HARBOR_ADMIN} admin + +*** Test Cases *** +Test Case - Garbage Collection + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Create An New Project project${d} + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world + Sleep 2 + Go Into Project project${d} + Delete Repo project${d} + + Switch To Garbage Collection + Click GC Now + Logout Harbor + Sleep 2 + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To Garbage Collection + Sleep 1 + Switch To GC History + Retry Wait Until Page Contains Finished + + ${rc} ${output}= Run And Return Rc And Output curl -u ${HARBOR_ADMIN}:${HARBOR_PASSWORD} -i --insecure -H "Content-Type: application/json" -X GET "https://${ip}/api/system/gc/1/log" + Log To Console ${output} + Should Be Equal As Integers ${rc} 0 + Should Contain ${output} 3 blobs and 0 manifests eligible for deletion + #Should Contain ${output} Deleting blob: + Should Contain ${output} success to run gc in job. + + Close Browser \ No newline at end of file diff --git a/tests/robot-cases/Group1-Nightly/GC.robot b/tests/robot-cases/Group1-Nightly/GC.robot new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/robot-cases/Group1-Nightly/Notary.robot b/tests/robot-cases/Group1-Nightly/Notary.robot new file mode 100644 index 0000000000..bb361e19b5 --- /dev/null +++ b/tests/robot-cases/Group1-Nightly/Notary.robot @@ -0,0 +1,43 @@ + +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +*** Settings *** +Documentation Harbor BATs +Resource ../../resources/Util.robot +Default Tags Nightly + +*** Variables *** +${HARBOR_URL} https://${ip} +${SSH_USER} root +${HARBOR_ADMIN} admin + +*** Test Cases *** +Test Case - Project Level Policy Content Trust + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Create An New Project project${d} + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world:latest + Go Into Project project${d} + Goto Project Config + Click Content Trust + Save Project Config + # Verify + Content Trust Should Be Selected + Cannot Pull Unsigned Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world:latest + Close Browser + +Test Case - Admin Push Signed Image + Body Of Admin Push Signed Image \ No newline at end of file diff --git a/tests/robot-cases/Group1-Nightly/Replication.robot b/tests/robot-cases/Group1-Nightly/Replication.robot index be71be88af..dc606c3135 100644 --- a/tests/robot-cases/Group1-Nightly/Replication.robot +++ b/tests/robot-cases/Group1-Nightly/Replication.robot @@ -156,11 +156,13 @@ Test Case - Replication Of Pull Images from DockerHub To Self Switch To Replication Manage Create A Rule With Existing Endpoint rule${d} pull danfengliu/* image e${d} project${d} Select Rule And Replicate rule${d} - Sleep 20 + Sleep 30 Go Into Project project${d} Switch To Project Repo #In docker-hub, under repository danfengliu, there're only 2 images: centos,mariadb. Retry Wait Until Page Contains project${d}/centos + Go Into Project project${d} + Switch To Project Repo Retry Wait Until Page Contains project${d}/mariadb Close Browser