add-notary-resign-test-in-upgade-pipeline

Signed-off-by: danfengliu <danfengl@vmware.com>

tests/resources/Harbor-Pages/OIDC_Auth.robot

@@ -70,8 +70,8 @@ Able To Delete An OIDC User
     Switch to User Tag
     Retry Element Click   ${user_test7_checkbox}
     Retry Element Click   ${member_action_btn}
+    Retry Element Click   ${user_actions_del_btn}
     Retry Element Click   ${delete_btn}
-    Retry Element Click   ${confirm_delete_btn}
     Retry Wait Until Page Not Contains Element   ${user_test7_checkbox}
 
 Should Contain Target User

tests/resources/Harbor-Pages/OIDC_Auth_Elements.robot

@@ -32,6 +32,5 @@ ${more_btn}       //*[@id='hidden-generate-cli']
 ${reset_secret_btn}       //*[@id='reset-cli-btn']
 ${user_test7_checkbox}         //clr-dg-row[contains(.,'test7')]//div[contains(@class,'clr-checkbox-wrapper')]//label[contains(@class,'clr-control-label')]
 ${member_action_btn}   //*[@id='member-action']
-${delete_btn}          //*[@id='deleteUser']
+${user_actions_del_btn}          //*[@id='deleteUser']
-${confirm_delete_btn}  //clr-modal//button[contains(.,'DELETE')]
 ${user_test7_row}      //clr-dg-row[contains(.,'test7')]

tests/resources/Harbor-Pages/Project_Elements.robot

@@ -39,7 +39,6 @@ ${project_statistics_total_projects_icon}  xpath=//div[contains(@class, 'statist
 ${repo_delete_confirm_btn}  xpath=//clr-modal//button[2]
 ${repo_retag_confirm_dlg}  css=${modal-dialog}
 ${repo_delete_on_card_view_btn}  //clr-modal//button[contains(.,'DELETE')]
-${delete_btn}  //button[contains(.,'Delete')]
 ${repo_delete_btn}  xpath=//hbr-repository-gridview//button[contains(.,'Delete')]
 ${project_delete_btn}  xpath=//button[@id='delete-project']
 ${tag_delete_btn}  xpath=//tag-repository//clr-datagrid//button[contains(.,'Delete')]

tests/resources/Harbor-Pages/Verify.robot

@@ -569,3 +569,12 @@ Verify Quotas Display
         Should Match Regexp  ${storage_quota_ret}  ${str_expected}
     END
     Close Browser
+
+
+Verify Re-sign Image
+    [Arguments]    ${json}
+    Log To Console  "Verify Quotas Display..."
+    @{project}=  Get Value From Json  ${json}  $.notary_projects.[*].name
+    FOR    ${project}    IN    @{project}
+        Body Of Admin Push Signed Image  ${project}  alpine  new_tag  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  clear_trust_dir=${false}
+    END

tests/resources/Nightly-Util.robot

@@ -52,9 +52,20 @@ Nightly Test Setup In Ubuntu
     Prepare Test Tools
     Log To Console  Start Docker Daemon Locally ...
     Run Keyword  Start Docker Daemon Locally
-    #Log To Console  Start Containerd Daemon Locally ...
-    #Run Keyword  Start Containerd Daemon Locally
     Prepare Helm Plugin
+    #Docker login
+    Run Keyword If  '${DOCKER_USER}' != '${EMPTY}'  Docker Login  ""  ${DOCKER_USER}  ${DOCKER_PWD}
+
+Nightly Test Setup In Ubuntu For Upgrade
+    [Arguments]  ${ip}  ${HARBOR_PASSWORD}  ${ip1}==${EMPTY}
+    Get And Setup Harbor CA  ${ip}  ${HARBOR_PASSWORD}  CA Setup In ubuntu  ip1=${ip1}
+    Prepare Test Tools
+    Log To Console  Start Docker Daemon Locally ...
+    Run Keyword  Start Docker Daemon Locally
+    Prepare Helm Plugin
+    #For upgrade pipeline: get notary targets key from last execution.
+    ${rc}  ${output}=  Run And Return Rc And Output  [ -f "/key_store/private_keys_backup.tar.gz" ] && tar -zxvf /key_store/private_keys_backup.tar.gz -C /
+    #Docker login
     Run Keyword If  '${DOCKER_USER}' != '${EMPTY}'  Docker Login  ""  ${DOCKER_USER}  ${DOCKER_PWD}
 
 CA Setup In ubuntu

tests/resources/TestCaseBody.robot

@@ -167,10 +167,12 @@ Body Of Push Signed Image
     Close Browser
 
 Body Of Admin Push Signed Image
-    [Arguments]  ${project}  ${image}  ${tag}  ${user}  ${pwd}  ${with_remove}=${false}
+    [Arguments]  ${project}  ${image}  ${tag}  ${user}  ${pwd}  ${with_remove}=${false}  ${clear_trust_dir}=${true}
-    Wait Unitl Command Success  rm -rf ~/.docker/
+    Run Keyword If  ${clear_trust_dir}==${true}  Wait Unitl Command Success  rm -rf ~/.docker/
-    Docker Pull  ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/${image}
+    ${src_tag}=   Set Variable  latest
-    ${rc}  ${output}=  Run And Return Rc And Output  ./tests/robot-cases/Group0-Util/notary-push-image.sh ${ip} ${project} ${image} ${tag} ${notaryServerEndpoint} ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/${image}:${tag} ${user} ${pwd}
+    ${src_image}=   Set Variable  ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/${image}:${src_tag}
+    Docker Pull  ${src_image}
+    ${rc}  ${output}=  Run And Return Rc And Output  ./tests/robot-cases/Group0-Util/notary-push-image.sh ${ip} ${project} ${image} ${tag} ${notaryServerEndpoint} ${src_image} ${user} ${pwd}
 
     Log  ${output}
     Should Be Equal As Integers  ${rc}  0

tests/robot-cases/Group3-Upgrade/Setup.robot

@@ -0,0 +1,23 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+*** Settings ***
+Documentation  Harbor BATs
+Resource  ../../resources/Util.robot
+Default Tags  Nightly
+
+*** Test Cases ***
+Test Suites Setup For UI Test
+    [Tags]  setup
+    Nightly Test Setup In Ubuntu For Upgrade  ${ip}  ${HARBOR_PASSWORD}  ${ip1}

tests/robot-cases/Group3-Upgrade/data.json

@@ -136,6 +136,40 @@
          "insecure":true
       }
    ],
+   "notary_projects":[
+      {
+         "name":"projects_notary_1",
+         "registry_name":null,
+         "has_image":true,
+         "accesslevel":"public",
+         "repocounts":2,
+         "count_limit":1234,
+         "storage_limit":53687091200,
+         "repo":[
+            {
+               "name":"alpine",
+               "tag":"latest",
+               "signed":"false"
+            }
+         ]
+      },
+      {
+         "name":"projects_notary_2",
+         "registry_name":null,
+         "has_image":true,
+         "accesslevel":"public",
+         "repocounts":2,
+         "count_limit":1234,
+         "storage_limit":53687091200,
+         "repo":[
+            {
+               "name":"alpine",
+               "tag":"latest",
+               "signed":"false"
+            }
+         ]
+      }
+   ],
    "projects":[
       {
          "name":"project1194",

tests/robot-cases/Group3-Upgrade/prepare.py

@@ -13,6 +13,9 @@ parser = argparse.ArgumentParser(description='The script to generate data for ha
 parser.add_argument('--endpoint', '-e', dest='endpoint', required=True, help='The endpoint to harbor')
 parser.add_argument('--version', '-v', dest='version', required=False, help='The version to harbor')
 parser.add_argument('--libpath', '-l', dest='libpath', required=False, help='e2e library')
+parser.add_argument('--src-registry', '-g', dest='LOCAL_REGISTRY', required=False, help='Sample images registry')
+parser.add_argument('--src-repo', '-p', dest='LOCAL_REGISTRY_NAMESPACE', required=False, help='Sample images repo')
+
 args = parser.parse_args()
 
 from os import path
@@ -56,8 +59,8 @@ def get_feature_branch(func):
 
 class HarborAPI:
     @get_feature_branch
-    def populate_projects(self, **kwargs):
+    def populate_projects(self, key_name, create_project_only = False, **kwargs):
-        for project in data["projects"]:
+        for project in data[key_name]:
             if kwargs["branch"] in [1,2]:
                 if "registry_name" in project:
                     print("Populate proxy project...")
@@ -67,6 +70,8 @@ class HarborAPI:
             else:
                 raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
             self.create_project(project, version=args.version)
+            if create_project_only:
+                continue
             for member in project["member"]:
                 self.add_member(project["name"], member["name"], member["role"], version=args.version)
             for robot_account in project["robot_account"]:
@@ -621,7 +626,8 @@ def push_image(image, project):
     os.system("docker push "+args.endpoint+"/"+project+"/library/"+image)
 
 def push_signed_image(image, project, tag):
-    os.system("./sign_image.sh" + " " + args.endpoint + " " + project + " " + image + " " + tag)
+    print("LOCAL_REGISTRY:{} LOCAL_REGISTRY_NAMESPACE:{}".format(args.LOCAL_REGISTRY, args.LOCAL_REGISTRY_NAMESPACE))
+    os.system("./sign_image.sh" + " " + args.endpoint + " " + project + " " + image + " " + tag + " " + args.LOCAL_REGISTRY + " " + args.LOCAL_REGISTRY_NAMESPACE)
 
 @get_feature_branch
 def set_url(**kwargs):
@@ -650,13 +656,15 @@ def do_data_creation():
     for distribution in data["distributions"]:
         harborAPI.add_distribution(distribution, version=args.version)
 
-    harborAPI.populate_projects(version=args.version)
+    harborAPI.populate_projects("projects", version=args.version)
+    harborAPI.populate_projects("notary_projects", create_project_only=True, version=args.version)
     harborAPI.populate_quotas(version=args.version)
 
     harborAPI.push_artifact_index(data["projects"][0]["name"], data["projects"][0]["artifact_index"]["name"], data["projects"][0]["artifact_index"]["tag"], version=args.version)
     #pull_image("busybox", "redis", "haproxy", "alpine", "httpd:2")
     push_self_build_image_to_project(data["projects"][0]["name"], args.endpoint, 'admin', 'Harbor12345', "busybox", "latest")
-    push_signed_image("alpine", data["projects"][0]["name"], "latest")
+    for project in data["notary_projects"]:
+        push_signed_image("alpine", project["name"], "latest")
 
     for replicationrule in data["replicationrule"]:
         harborAPI.add_replication_rule(replicationrule, version=args.version)

tests/robot-cases/Group3-Upgrade/run.sh

@@ -3,7 +3,9 @@ IP=$1
 HARBOR_VERSION=$2
 DOCKER_USER=$3
 DOCKER_PWD=$4
-
+LOCAL_REGISTRY=$5
+LOCAL_REGISTRY_NAMESPACE=$6
+make swagger_client
 robot -v ip:$IP  -v ip1: -v HARBOR_PASSWORD:Harbor12345 -v DOCKER_USER:$DOCKER_USER -v DOCKER_PWD:$DOCKER_PWD -v http_get_ca:true /drone/tests/robot-cases/Group1-Nightly/Setup.robot
 cd /drone/tests/robot-cases/Group3-Upgrade
-DOCKER_USER=$DOCKER_USER DOCKER_PWD=$DOCKER_PWD python ./prepare.py -e $IP -v $HARBOR_VERSION -l /drone/tests/apitests/python/
+DOCKER_USER=$DOCKER_USER DOCKER_PWD=$DOCKER_PWD  python ./prepare.py -e $IP -v $HARBOR_VERSION -l /drone/tests/apitests/python/ -g $LOCAL_REGISTRY  -p $LOCAL_REGISTRY_NAMESPACE

tests/robot-cases/Group3-Upgrade/sign_image.sh

@@ -1,6 +1,10 @@
 #!/bin/bash
+set -x
+set -e
 
-docker pull $3:$4
+echo "registry:"$5
+echo "repo:"$6
+docker pull $5/$6/$3:$4
 
 IP=$1
 PASSHRASE='Harbor12345'
@@ -25,5 +29,7 @@ export DOCKER_CONTENT_TRUST_OFFLINE_PASSPHRASE=$PASSHRASE
 export DOCKER_CONTENT_TRUST_TAGGING_PASSPHRASE=$PASSHRASE
 
 docker login -u admin -p Harbor12345 $IP
-docker tag $3:$4 $IP/$2/$3:$4
+docker tag $5/$6/$3:$4 $IP/$2/$3:$4
-docker push $IP/$2/$3:$4
+docker push $IP/$2/$3:$4
+rm -rf /key_store/*
+umask 077; tar -zcvf /key_store/private_keys_backup.tar.gz ~/.docker/trust/private; umask 022

tests/robot-cases/Group3-Upgrade/verify.robot

@@ -88,6 +88,7 @@ Test Case - Upgrade Verify
     Run Keyword  Verify Trivy Is Default Scanner
     Run Keyword  Verify Artifact Index  ${data}
     Run Keyword  Verify Quotas Display  ${data}
+    Run Keyword  Verify Re-sign Image  ${data}
 
 Test Case - Upgrade Verify
     [Tags]  2.1-latest
@@ -115,3 +116,4 @@ Test Case - Upgrade Verify
     Run Keyword  Verify Distributions  ${data}
     Run Keyword  Verify P2P Preheat Policy  ${data}
     Run Keyword  Verify Quotas Display  ${data}
+    Run Keyword  Verify Re-sign Image  ${data}