Fix LDAP API test issue by upgrading python

Signed-off-by: danfengliu <danfengl@vmware.com>

.github/workflows/CI.yml

@@ -166,6 +166,14 @@ jobs:
           ./mconfig && \
           make -C builddir && \
           sudo make -C builddir install
+          python -V
+          sudo apt-get update -y && sudo apt-get install -y zbar-tools libzbar-dev python-zbar
+          sudo apt-get update -y
+          sudo apt-get install -y python3.6
+          sudo rm /usr/bin/python
+          sudo ln -s /usr/bin/python3.6 /usr/bin/python
+          sudo apt-get install -y python3-pip
+          python -V
       - name: install
         run: |
           cd src/github.com/goharbor/harbor

tests/apitests/python/library/Harbor.py

@@ -1,6 +1,4 @@
 # -*- coding: utf-8 -*-
-import site
-reload(site)
 import project
 import label
 import registry

tests/apitests/python/library/docker_api.py

@@ -103,7 +103,7 @@ class DockerAPI(object):
         caught_err = False
         ret = ""
         try:
-            ret = base._get_string_from_unicode(self.DCLIENT.pull(r'{}:{}'.format(image, _tag)))
+            self.DCLIENT.pull(r'{}:{}'.format(image, _tag))
             return ret
         except Exception as err:
             caught_err = True
@@ -137,7 +137,7 @@ class DockerAPI(object):
         if expected_error_message is "":
             expected_error_message = None
         try:
-            ret = base._get_string_from_unicode(self.DCLIENT.push(harbor_registry, tag, stream=True))
+            self.DCLIENT.push(harbor_registry, tag)
             return ret
         except Exception as err:
             caught_err = True

tests/apitests/python/library/sign.py

@@ -6,6 +6,6 @@ def sign_image(registry_ip, project_name, image, tag):
     try:
         ret = subprocess.check_output(["./tests/apitests/python/sign_image.sh", registry_ip, project_name, image, tag, notary_url], shell=False)
         print("sign_image return: ", ret)
-    except subprocess.CalledProcessError, exc:
+    except subprocess.CalledProcessError as exc:
         raise Exception("Failed to sign image error is {} {}.".format(exc.returncode, exc.output))
 

tests/apitests/python/test_assign_role_to_ldap_group.py

@@ -22,6 +22,7 @@ import docker
 
 import swagger_client
 
+from testutils import ADMIN_CLIENT
 from swagger_client.models.project import Project
 from swagger_client.models.project_req import ProjectReq
 from swagger_client.models.project_metadata import ProjectMetadata
@@ -29,8 +30,8 @@ from swagger_client.models.project_member import ProjectMember
 from swagger_client.models.user_group import UserGroup
 from swagger_client.models.configurations import Configurations
 from library.projectV2 import ProjectV2
-from testutils import ADMIN_CLIENT
 from library.base import _assert_status_code
+from library.base import _random_name
 
 
 from v2_swagger_client.rest import ApiException
@@ -45,6 +46,7 @@ class TestAssignRoleToLdapGroup(unittest.TestCase):
     repository_api = testutils.GetRepositoryApi("admin", "Harbor12345")
     project_id = 0
     docker_client = docker.from_env()
+    _project_name = _random_name("test_private")
 
     def setUp(self):
         self.projectv2= ProjectV2()
@@ -55,13 +57,13 @@ class TestAssignRoleToLdapGroup(unittest.TestCase):
         cfgs = self.product_api.configurations_get()
         pprint(cfgs)
         req = ProjectReq()
-        req.project_name = "ldap_group_test_prj"
+        req.project_name = self._project_name
         req.metadata = ProjectMetadata(public="false")
         result = self.product_api.projects_post(req)
         pprint(result)
 
-        projs = self.product_api.projects_get(name="ldap_group_test_prj")
+        projs = self.product_api.projects_get(name = self._project_name)
-        if projs.count>0 :
+        if len(projs)>0 :
             project = projs[0]
             self.project_id = project.project_id
 
@@ -93,9 +95,9 @@ class TestAssignRoleToLdapGroup(unittest.TestCase):
 
     def tearDown(self):
         #delete images in project
-        result = self.repository_api.delete_repository("ldap_group_test_prj", "busybox")
+        result = self.repository_api.delete_repository(self._project_name, "busybox")
         pprint(result)
-        result = self.repository_api.delete_repository("ldap_group_test_prj", "busyboxdev")
+        result = self.repository_api.delete_repository(self._project_name, "busyboxdev")
         pprint(result)
         if self.project_id > 0 :
               self.product_api.projects_project_id_delete(self.project_id)
@@ -104,19 +106,19 @@ class TestAssignRoleToLdapGroup(unittest.TestCase):
     def testAssignRoleToLdapGroup(self):
         """Test AssignRoleToLdapGroup"""
         admin_product_api = testutils.GetProductApi(username="admin_user", password="zhu88jie")
-        projects = admin_product_api.projects_get(name="ldap_group_test_prj")
+        projects = admin_product_api.projects_get(name=self._project_name)
-        self.assertTrue(projects.count > 1)
+        self.assertTrue(len(projects) == 1)
         self.assertEqual(1, projects[0].current_user_role_id)
 
 
         dev_product_api = testutils.GetProductApi("dev_user", "zhu88jie")
-        projects = dev_product_api.projects_get(name="ldap_group_test_prj")
+        projects = dev_product_api.projects_get(name=self._project_name)
-        self.assertTrue(projects.count > 1)
+        self.assertTrue(len(projects) == 1)
         self.assertEqual(2, projects[0].current_user_role_id)
 
         guest_product_api = testutils.GetProductApi("guest_user", "zhu88jie")
-        projects = guest_product_api.projects_get(name="ldap_group_test_prj")
+        projects = guest_product_api.projects_get(name=self._project_name)
-        self.assertTrue(projects.count > 1)
+        self.assertTrue(len(projects) == 1)
         self.assertEqual(3, projects[0].current_user_role_id)
 
         self.dockerCmdLoginAdmin(username="admin_user", password="zhu88jie")
@@ -136,19 +138,19 @@ class TestAssignRoleToLdapGroup(unittest.TestCase):
         self.docker_client.login(username=username, password=password, registry=self.harbor_host)
         self.docker_client.images.pull("busybox:latest")
         image = self.docker_client.images.get("busybox:latest")
-        image.tag(repository=self.harbor_host+"/ldap_group_test_prj/busybox", tag="latest")
+        image.tag(repository=self.harbor_host+"/"+self._project_name+"/busybox", tag="latest")
-        output = self.docker_client.images.push(repository=self.harbor_host+"/ldap_group_test_prj/busybox", tag="latest")
+        output = self.docker_client.images.push(repository=self.harbor_host+"/"+self._project_name+"/busybox", tag="latest")
         if output.find("error")>0 :
             self.fail("Should not fail to push image for admin_user")
-        self.docker_client.images.pull(repository=self.harbor_host+"/ldap_group_test_prj/busybox", tag="latest")
+        self.docker_client.images.pull(repository=self.harbor_host+"/"+self._project_name+"/busybox", tag="latest")
         pass
     # dev user can push, pull images
     def dockerCmdLoginDev(self, username, password, harbor_server=harbor_host):
         self.docker_client.login(username=username, password=password, registry=self.harbor_host)
         self.docker_client.images.pull("busybox:latest")
         image = self.docker_client.images.get("busybox:latest")
-        image.tag(repository=self.harbor_host+"/ldap_group_test_prj/busyboxdev", tag="latest")
+        image.tag(repository=self.harbor_host+"/"+self._project_name+"/busyboxdev", tag="latest")
-        output = self.docker_client.images.push(repository=self.harbor_host+"/ldap_group_test_prj/busyboxdev", tag="latest")
+        output = self.docker_client.images.push(repository=self.harbor_host+"/"+self._project_name+"/busyboxdev", tag="latest")
         if output.find("error") >0 :
             self.fail("Should not fail to push images for dev_user")
         pass
@@ -157,17 +159,17 @@ class TestAssignRoleToLdapGroup(unittest.TestCase):
         self.docker_client.login(username=username, password=password, registry=self.harbor_host)
         self.docker_client.images.pull("busybox:latest")
         image = self.docker_client.images.get("busybox:latest")
-        image.tag(repository=self.harbor_host+"/ldap_group_test_prj/busyboxguest", tag="latest")
+        image.tag(repository=self.harbor_host+"/"+self._project_name+"/busyboxguest", tag="latest")
-        output = self.docker_client.images.push(repository=self.harbor_host+"1/ldap_group_test_prj/busyboxguest", tag="latest")
+        output = self.docker_client.images.push(repository=self.harbor_host+"1/"+self._project_name+"/busyboxguest", tag="latest")
         if output.find("error")<0 :
             self.fail("Should failed to push image for guest user")
-        self.docker_client.images.pull(repository=self.harbor_host+"/ldap_group_test_prj/busybox", tag="latest")
+        self.docker_client.images.pull(repository=self.harbor_host+"/"+self._project_name+"/busybox", tag="latest")
         pass
     # check can see his log in current project
     def queryUserLogs(self, username, password, status_code=200):
         client=dict(endpoint = ADMIN_CLIENT["endpoint"], username = username, password = password)
         try:
-            logs = self.projectv2.get_project_log("ldap_group_test_prj", status_code, **client)
+            logs = self.projectv2.get_project_log(self._project_name, status_code, **client)
             count = 0
             for log in list(logs):
                 count = count + 1

tests/apitests/python/test_copy_artifact_outside_project.py

@@ -3,11 +3,10 @@ from __future__ import absolute_import
 
 import unittest
 
-from library.base import _assert_status_code
 from testutils import ADMIN_CLIENT
 from testutils import harbor_server
-
 from testutils import TEARDOWN
+from library.base import _assert_status_code
 from library.artifact import Artifact
 from library.project import Project
 from library.user import User

tests/apitests/python/test_create_delete_tag.py

@@ -3,13 +3,12 @@ from __future__ import absolute_import
 
 import unittest
 
+from testutils import ADMIN_CLIENT
+from testutils import harbor_server
+from testutils import TEARDOWN
 import library.repository
 import library.docker_api
 from library.base import _assert_status_code
-from testutils import ADMIN_CLIENT
-from testutils import harbor_server
-
-from testutils import TEARDOWN
 from library.project import Project
 from library.user import User
 from library.repository import Repository

tests/apitests/python/test_del_repo.py

@@ -3,11 +3,10 @@ from __future__ import absolute_import
 
 import unittest
 
-from library.base import _assert_status_code
 from testutils import ADMIN_CLIENT
 from testutils import harbor_server
-
 from testutils import TEARDOWN
+from library.base import _assert_status_code
 from library.project import Project
 from library.user import User
 from library.repository import Repository

tests/apitests/python/test_garbage_collection.py

@@ -5,12 +5,12 @@ import time
 
 from testutils import ADMIN_CLIENT
 from testutils import TEARDOWN
+from testutils import harbor_server
 from library.user import User
 from library.system import System
 from library.project import Project
 from library.repository import Repository
 from library.repository import push_image_to_project
-from testutils import harbor_server
 from library.base import _assert_status_code
 from library.repository import push_special_image_to_project
 from library.artifact import Artifact

tests/apitests/python/test_ldap_admin_role.py

@@ -20,10 +20,10 @@ sys.path.append(os.environ["SWAGGER_CLIENT_PATH"])
 import unittest
 import testutils
 import swagger_client
+from library.base import _random_name
 from swagger_client.models.project_req import ProjectReq
 from swagger_client.models.configurations import Configurations
 from swagger_client.rest import ApiException
-from swagger_client.models.configurations import Configurations 
 from pprint import pprint
 
 
@@ -44,16 +44,17 @@ class TestLdapAdminRole(unittest.TestCase):
 
     def testLdapAdminRole(self):
         """Test LdapAdminRole"""
+        _project_name = _random_name("test_private")
         result = self.product_api.configurations_put(configurations=Configurations(ldap_group_admin_dn="cn=harbor_users,ou=groups,dc=example,dc=com"))
-        pprint(result)
 
         # Create a private project
-        result = self.product_api.projects_post(project=ProjectReq(project_name="test_private"))
+        result = self.product_api.projects_post(project=ProjectReq(project_name= _project_name))
-        pprint(result)
 
         # query project with ldap user mike
-        projects = self.mike_product_api.projects_get(name="test_private")
+        projects = self.mike_product_api.projects_get(name=_project_name)
-        self.assertTrue(projects.count>1)
+
+        print("=================", projects)
+        self.assertTrue(len(projects) == 1)
         self.project_id = projects[0].project_id
 
         # check the mike is not admin in Database

tests/apitests/python/test_project_level_policy_content_trust.py

@@ -4,7 +4,6 @@ import unittest
 
 from testutils import ADMIN_CLIENT
 from testutils import harbor_server
-
 from testutils import TEARDOWN
 from library.artifact import Artifact
 from library.project import Project

tests/apitests/python/test_push_chart_by_helm2_helm3_with_robot_Account.py

@@ -3,12 +3,11 @@ from __future__ import absolute_import
 
 import unittest
 
-import library.repository
-import library.helm
 from testutils import ADMIN_CLIENT, CHART_API_CLIENT
 from testutils import harbor_server
-
 from testutils import TEARDOWN
+import library.repository
+import library.helm
 from library.project import Project
 from library.user import User
 from library.chart import Chart

tests/apitests/python/test_push_chart_by_helm3_chart_cli.py

@@ -3,12 +3,11 @@ from __future__ import absolute_import
 
 import unittest
 
-import library.repository
-import library.helm
 from testutils import ADMIN_CLIENT
 from testutils import harbor_server
-
 from testutils import TEARDOWN
+import library.repository
+import library.helm
 from library.project import Project
 from library.user import User
 from library.repository import Repository

tests/apitests/python/test_push_cnab_bundle.py

@@ -3,13 +3,11 @@ from __future__ import absolute_import
 
 import unittest
 
-import library.repository
-import library.cnab
-
 from testutils import ADMIN_CLIENT
 from testutils import harbor_server
-
 from testutils import TEARDOWN
+import library.repository
+import library.cnab
 from library.project import Project
 from library.user import User
 from library.repository import Repository

tests/apitests/python/test_push_files_by_oras.py

@@ -2,11 +2,11 @@ from __future__ import absolute_import
 import unittest
 import urllib
 
-import library.oras
-from library.sign import sign_image
 from testutils import ADMIN_CLIENT
 from testutils import harbor_server
 from testutils import TEARDOWN
+import library.oras
+from library.sign import sign_image
 from library.user import User
 from library.project import Project
 from library.repository import Repository

tests/apitests/python/test_push_image_with_special_name.py

@@ -2,10 +2,10 @@ from __future__ import absolute_import
 import unittest
 import urllib
 
-from library.sign import sign_image
 from testutils import ADMIN_CLIENT
 from testutils import harbor_server
 from testutils import TEARDOWN
+from library.sign import sign_image
 from library.artifact import Artifact
 from library.project import Project
 from library.user import User
@@ -77,9 +77,9 @@ class TestProjects(unittest.TestCase):
         TestProjects.repo_name, tag = push_image_to_project(TestProjects.project_sign_image_name, harbor_server, user_sign_image_name, user_001_password, image, src_tag, profix_for_image=profix)
 
         #7. Get signature of image with tag(TA), it should be exist.
-        full_name = urllib.quote(profix+"/"+image,'utf-8')
+        full_name = urllib.parse.quote(profix+"/"+image,'utf-8')
-        artifact = self.artifact.get_reference_info(TestProjects.project_sign_image_name, (str(full_name)).encode(), tag, **TestProjects.USER_sign_image_CLIENT)
 
+        artifact = self.artifact.get_reference_info(TestProjects.project_sign_image_name, full_name, tag, **TestProjects.USER_sign_image_CLIENT)
         self.assertEqual(artifact[0].type, 'IMAGE')
 
 if __name__ == '__main__':

tests/apitests/python/test_push_index_by_docker_manifest.py

@@ -3,14 +3,13 @@ from __future__ import absolute_import
 
 import unittest
 
+from testutils import ADMIN_CLIENT
+from testutils import harbor_server
+from testutils import TEARDOWN
 import library.repository
 import library.docker_api
 import library.containerd
 from library.base import _assert_status_code
-from testutils import ADMIN_CLIENT
-from testutils import harbor_server
-
-from testutils import TEARDOWN
 from library.project import Project
 from library.user import User
 from library.repository import Repository

tests/apitests/python/test_push_sif_by_singularity.py

@@ -2,11 +2,11 @@ from __future__ import absolute_import
 import unittest
 import urllib
 
-import library.singularity
-from library.sign import sign_image
 from testutils import ADMIN_CLIENT
 from testutils import harbor_server
 from testutils import TEARDOWN
+import library.singularity
+from library.sign import sign_image
 from library.user import User
 from library.project import Project
 from library.repository import Repository

tests/apitests/python/test_registry_api.py

@@ -13,6 +13,8 @@ from library.repository import Repository
 from library.repository import push_image_to_project
 from library.artifact import Artifact
 from library.scanner import Scanner
+from library.docker_api import list_image_tags
+from library.docker_api import list_repositories
 import os
 import library.base
 import json
@@ -84,24 +86,24 @@ class TestProjects(unittest.TestCase):
         for tag in create_tags:
             self.artifact.create_tag(TestProjects.project_Alice_name, self.repo_name, tag_c, tag, **USER_ALICE_CLIENT)
         #4. Call the image_list_tags API
-        tags = library.docker_api.list_image_tags(harbor_server,TestProjects.repo_c,user_Alice_name,user_common_password)
+        tags = list_image_tags(harbor_server,TestProjects.repo_c,user_Alice_name,user_common_password)
         for tag in create_tags:
             self.assertTrue(tags.count(tag)>0, "Expect tag: %s is not listed"%tag)
-        page_tags = library.docker_api.list_image_tags(harbor_server,TestProjects.repo_c,user_Alice_name,user_common_password,len(tags)/2+1)
+        page_tags = list_image_tags(harbor_server,TestProjects.repo_c,user_Alice_name,user_common_password,len(tags)/2+1)
-        page_tags += library.docker_api.list_image_tags(harbor_server,TestProjects.repo_c,user_Alice_name,user_common_password,len(tags)/2+1,tags[len(tags)/2])
+        page_tags += list_image_tags(harbor_server,TestProjects.repo_c,user_Alice_name,user_common_password,len(tags)/2+1,tags[int(len(tags)/2)])
         for tag in create_tags:
             self.assertTrue(page_tags.count(tag)>0, "Expect tag: %s is not listed by the pagination query"%tag)
         #5. Call the catalog API;
-        repos = library.docker_api.list_repositories(harbor_server,admin_user,admin_pwd)
+        repos = list_repositories(harbor_server,admin_user,admin_pwd)
         self.assertTrue(repos.count(TestProjects.repo_a)>0 and repos.count(TestProjects.repo_b)>0 and repos.count(TestProjects.repo_c)>0, "Expected repo not found")
         for repo in [TestProjects.repo_a,TestProjects.repo_b,TestProjects.repo_c]:
             self.assertTrue(repos.count(repo)>0,"Expected repo: %s is not listed"%repo)
-        page_repos = library.docker_api.list_repositories(harbor_server,admin_user,admin_pwd,len(repos)/2+1)
+        page_repos = list_repositories(harbor_server,admin_user,admin_pwd,len(repos)/2+1)
-        page_repos += library.docker_api.list_repositories(harbor_server,admin_user,admin_pwd,len(repos)/2+1,repos[len(repos)/2])
+        page_repos += list_repositories(harbor_server,admin_user,admin_pwd,len(repos)/2+1,repos[int(len(repos)/2)])
         for repo in [TestProjects.repo_a,TestProjects.repo_b,TestProjects.repo_c]:
             self.assertTrue(page_repos.count(repo)>0,"Expected repo: %s is not listed by the pagination query"%repo)
 
-        null_repos = library.docker_api.list_repositories(harbor_server,user_Alice_name,user_common_password)
+        null_repos = list_repositories(harbor_server,user_Alice_name,user_common_password)
         self.assertEqual(null_repos, "")
 
 if __name__ == '__main__':

tests/apitests/python/test_retention.py

@@ -8,7 +8,6 @@ from testutils import TEARDOWN
 from testutils import harbor_server
 from library.repository import push_special_image_to_project
 from library.docker_api import list_image_tags
-
 from library.retention import Retention
 from library.project import Project
 from library.repository import Repository

tests/apitests/python/test_robot_account.py

@@ -4,12 +4,12 @@ import unittest
 
 from testutils import ADMIN_CLIENT
 from testutils import TEARDOWN
+from testutils import harbor_server
 from library.user import User
 from library.project import Project
 from library.repository import Repository
 from library.repository import pull_harbor_image
 from library.repository import push_image_to_project
-from testutils import harbor_server
 from library.base import _assert_status_code
 
 class TestProjects(unittest.TestCase):

tests/apitests/python/test_sign_image.py

@@ -1,10 +1,10 @@
 from __future__ import absolute_import
 import unittest
 
-from library.sign import sign_image
 from testutils import ADMIN_CLIENT
 from testutils import harbor_server
 from testutils import TEARDOWN
+from library.sign import sign_image
 from library.artifact import Artifact
 from library.project import Project
 from library.user import User

tests/apitests/python/testutils.py

@@ -3,6 +3,13 @@ import os
 import sys
 
 sys.path.insert(0, os.environ["SWAGGER_CLIENT_PATH"])
+path=os.getcwd() + "/library"
+sys.path.insert(0, path)
+
+
+path=os.getcwd() + "/tests/apitests/python/library"
+sys.path.insert(0, path)
+
 import v2_swagger_client
 from swagger_client.rest import ApiException
 import swagger_client.models

tests/resources/Docker-Util.robot

@@ -165,7 +165,7 @@ Docker Pull
     [Arguments]  ${image}
     ${output}=  Retry Keyword N Times When Error  10  Wait Unitl Command Success  docker pull ${image}
     Log  ${output}
-    Log To Console  Docker Pull: \n ${output}
+    Log To Console  Docker Pull: ${output}
     [Return]  ${output}
 
 Docker Tag

tests/resources/Harbor-Util.robot

@@ -26,7 +26,7 @@ Install Harbor to Test Server
     Sleep  5s
     ${rc}  ${output}=  Run And Return Rc And Output  docker ps
     Should Be Equal As Integers  ${rc}  0
-    Log To Console  \n${output}
+    Log To Console  ${output}
     Log To Console  \nconfig harbor cfg
     Config Harbor cfg  http_proxy=https
     Prepare Cert
@@ -34,7 +34,7 @@ Install Harbor to Test Server
     Compile and Up Harbor With Source Code
     ${rc}  ${output}=  Run And Return Rc And Output  docker ps
     Should Be Equal As Integers  ${rc}  0
-    Log To Console  \n${output}
+    Log To Console  ${output}
     Generate Certificate Authority For Chrome
 
 Up Harbor
@@ -55,7 +55,7 @@ Package Harbor Offline
     [Arguments]  ${with_notary}=true  ${with_clair}=true  ${with_chartmuseum}=true  ${with_trivy}=true
     Log To Console  \nStart Docker Daemon
     Start Docker Daemon Locally
-    Log To Console  \n\nmake package_offline GOBUILDTAGS="include_oss include_gcs" BASEIMAGETAG=%{Harbor_Build_Base_Tag} NPM_REGISTRY=%{NPM_REGISTRY} VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} CHARTFLAG=${with_chartmuseum} TRIVYFLAG=${with_trivy} HTTPPROXY=
+    Log To Console  make package_offline GOBUILDTAGS="include_oss include_gcs" BASEIMAGETAG=%{Harbor_Build_Base_Tag} NPM_REGISTRY=%{NPM_REGISTRY} VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} CHARTFLAG=${with_chartmuseum} TRIVYFLAG=${with_trivy} HTTPPROXY=
     ${rc}  ${output}=  Run And Return Rc And Output  make package_offline GOBUILDTAGS="include_oss include_gcs" BASEIMAGETAG=%{Harbor_Build_Base_Tag} NPM_REGISTRY=%{NPM_REGISTRY} VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} CHARTFLAG=${with_chartmuseum} TRIVYFLAG=${with_trivy} HTTPPROXY=
     Log To Console  ${rc}
     Log To Console  ${output}