Add API E2E pytest of GC with untag flag enabled

1. Fix issue of keyword Go Into Repo, the verification logic could be more strict;
2. Add API E2E pytest of GC with untag flag enabled;
3. Add sleep in test_user_view_logs.py for delete log ocurred;
4. Test Case - Tag CRUD is not stable. Although add button was clicked, but the tag was'nt added successfully.

Signed-off-by: danfengliu <danfengl@vmware.com>
This commit is contained in:
danfengliu 2020-06-16 17:34:30 +08:00
parent c5e5e9ec9f
commit a8f159728b
10 changed files with 117 additions and 54 deletions

View File

@ -35,7 +35,6 @@ def helm_chart_push_to_harbor(chart_file, archive, harbor_server, project, repo_
helm_save(archive, harbor_server, project, repo_name)
return helm_push(harbor_server, project, repo_name, version)
# helm repo add --ca-file /ca/server.crt --username=${user} --password=${pwd} ${helm_repo_name} ${harbor_url}/chartrepo/${project_name}
def helm2_add_repo(helm_repo_name, harbor_url, project, username, password):
command = ["helm2", "repo", "add", "--username=" + username, "--password=" + password, helm_repo_name, harbor_url + "/chartrepo/" + project]
print "Command: ", command

View File

@ -183,7 +183,7 @@ class Project(base.Base):
access_list = []
resource_by_project_id = "/project/"+str(project_id)+"/repository"
resource_helm_by_project_id = "/project/"+str(project_id)+"/helm-chart"
resource_helm__create_by_project_id = "/project/"+str(project_id)+"/helm-chart-version"
resource_helm_create_by_project_id = "/project/"+str(project_id)+"/helm-chart-version"
action_pull = "pull"
action_push = "push"
action_read = "read"
@ -198,7 +198,7 @@ class Project(base.Base):
robotAccountAccess = swagger_client.RobotAccountAccess(resource = resource_helm_by_project_id, action = action_read)
access_list.append(robotAccountAccess)
if has_chart_create_right is True:
robotAccountAccess = swagger_client.RobotAccountAccess(resource = resource_helm__create_by_project_id, action = action_create)
robotAccountAccess = swagger_client.RobotAccountAccess(resource = resource_helm_create_by_project_id, action = action_create)
access_list.append(robotAccountAccess)
robotAccountCreate = swagger_client.RobotAccountCreate(robot_name, robot_desc, expires_at, access_list)

View File

@ -11,6 +11,8 @@ from library.repository import Repository
from library.repository import push_image_to_project
from testutils import harbor_server
from library.base import _assert_status_code
from library.repository import push_special_image_to_project
from library.artifact import Artifact
class TestProjects(unittest.TestCase):
@classmethod
@ -19,12 +21,16 @@ class TestProjects(unittest.TestCase):
self.project = Project()
self.user = User()
self.repo = Repository()
self.artifact = Artifact()
self.repo_name = "test_repo"
self.repo_name_untag = "test_untag"
self.tag = "v1.0"
@classmethod
def tearDown(self):
print "Case completed"
@unittest.skipIf(TEARDOWN == False, "Test data won't be erased.")
@unittest.skipIf(TEARDOWN == True, "Test data won't be erased.")
def test_ClearData(self):
#2. Delete project(PA);
self.project.delete_project(TestProjects.project_gc_id, **TestProjects.USER_GC_CLIENT)
@ -38,13 +44,17 @@ class TestProjects(unittest.TestCase):
Garbage Collection
Test step and expected result:
1. Create a new user(UA);
2. Create a new project(PA) by user(UA);
3. Push a new image(IA) in project(PA) by admin;
4. Delete repository(RA) by user(UA);
5. Get repository by user(UA), it should get nothing;
6. Tigger garbage collection operation;
7. Check garbage collection job was finished;
8. Get garbage collection log, check there is number of files was deleted.
2. Create project(PA) and project(PB) by user(UA);
3. Push a image in project(PA) and then delete repository by admin;
4. Get repository by user(UA), it should get nothing;
5. Tigger garbage collection operation;
6. Check garbage collection job was finished;
7. Get garbage collection log, check there is a number of files was deleted;
8. Push a image in project(PB) by admin and delete the only tag;
9. Tigger garbage collection operation;
10. Check garbage collection job was finished;
11. Repository with untag image should be still there;
12. But no any artifact in repository anymore.
Tear down:
1. Delete project(PA);
2. Delete user(UA).
@ -59,27 +69,48 @@ class TestProjects(unittest.TestCase):
TestProjects.USER_GC_CLIENT=dict(endpoint = url, username = user_gc_name, password = user_gc_password)
#2. Create a new project(PA) by user(UA);
#2. Create project(PA) and project(PB) by user(UA);
TestProjects.project_gc_id, TestProjects.project_gc_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_GC_CLIENT)
TestProjects.project_gc_untag_id, TestProjects.project_gc_untag_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_GC_CLIENT)
#3. Push a new image(IA) in project(PA) by admin;
repo_name, _ = push_image_to_project(TestProjects.project_gc_name, harbor_server, admin_name, admin_password, "tomcat", "latest")
#3. Push a image in project(PA) and then delete repository by admin;
push_special_image_to_project(TestProjects.project_gc_name, harbor_server, admin_name, admin_password, self.repo_name, ["latest", "v1.2.3"])
self.repo.delete_repoitory(TestProjects.project_gc_name, self.repo_name, **TestProjects.USER_GC_CLIENT)
#4. Delete repository(RA) by user(UA);
self.repo.delete_repoitory(TestProjects.project_gc_name, repo_name.split('/')[1], **TestProjects.USER_GC_CLIENT)
#5. Get repository by user(UA), it should get nothing;
#4. Get repository by user(UA), it should get nothing;
repo_data = self.repo.list_repositories(TestProjects.project_gc_name, **TestProjects.USER_GC_CLIENT)
_assert_status_code(len(repo_data), 0)
#6. Tigger garbage collection operation;
#5. Tigger garbage collection operation;
gc_id = self.system.gc_now(**ADMIN_CLIENT)
#7. Check garbage collection job was finished;
#6. Check garbage collection job was finished;
self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT)
#8. Get garbage collection log, check there is number of files was deleted.
#7. Get garbage collection log, check there is a number of files was deleted;
self.system.validate_deletion_success(gc_id, **ADMIN_CLIENT)
#8. Push a image in project(PB) by admin and delete the only tag;
push_special_image_to_project(TestProjects.project_gc_untag_name, harbor_server, admin_name, admin_password, self.repo_name_untag, [self.tag])
self.artifact.delete_tag(TestProjects.project_gc_untag_name, self.repo_name_untag, self.tag, self.tag, **ADMIN_CLIENT)
#9. Tigger garbage collection operation;
gc_id = self.system.gc_now(**ADMIN_CLIENT)
#10. Check garbage collection job was finished;
self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT)
#11. Repository with untag image should be still there;
repo_data_untag = self.repo.list_repositories(TestProjects.project_gc_untag_name, **TestProjects.USER_GC_CLIENT)
print "repo_data_untag:", repo_data_untag
_assert_status_code(len(repo_data_untag), 1)
self.assertEqual(TestProjects.project_gc_untag_name + "/" + self.repo_name_untag , repo_data_untag[0].name)
#12. But no any artifact in repository anymore.
artifacts = self.artifact.list_artifacts(TestProjects.project_gc_untag_name, self.repo_name_untag, **TestProjects.USER_GC_CLIENT)
self.assertEqual(artifacts,[])
if __name__ == '__main__':
unittest.main()

View File

@ -11,14 +11,9 @@ from library.chart import Chart
class TestProjects(unittest.TestCase):
@classmethod
def setUp(self):
chart = Chart()
self.chart= chart
project = Project()
self.project= project
user = User()
self.user= user
self.chart= Chart()
self.project= Project()
self.user= User()
@classmethod
def tearDown(self):

View File

@ -5,26 +5,26 @@ import unittest
import library.repository
import library.helm
from testutils import ADMIN_CLIENT
from testutils import ADMIN_CLIENT, CHART_API_CLIENT
from testutils import harbor_server
from testutils import TEARDOWN
from library.project import Project
from library.user import User
from library.repository import Repository
from library.artifact import Artifact
from library.chart import Chart
class TestProjects(unittest.TestCase):
@classmethod
def setUpClass(self):
self.project= Project()
self.user= User()
self.artifact = Artifact()
self.repo= Repository()
self.chart= Chart()
self.url = ADMIN_CLIENT["endpoint"]
self.chart_api_url = CHART_API_CLIENT['endpoint']
self.user_push_chart_password = "Aa123456"
self.chart_file = "https://storage.googleapis.com/harbor-builds/helm-chart-test-files/harbor-0.2.0.tgz"
self.archive = "harbor/"
self.CHART_NAME=self.archive.replace("/", "")
self.verion = "0.2.0"
self.chart_repo_name = "chart_local"
self.repo_name = "harbor_api_test"
@ -54,15 +54,15 @@ class TestProjects(unittest.TestCase):
print "#1. Create user(UA);"
TestProjects.user_id, user_name = self.user.create_user(user_password = self.user_push_chart_password, **ADMIN_CLIENT)
TestProjects.USER_RA_CLIENT=dict(endpoint = self.url, username = user_name, password = self.user_push_chart_password)
TestProjects.USER_CLIENT=dict(endpoint = self.url, username = user_name, password = self.user_push_chart_password)
TestProjects.API_CHART_CLIENT=dict(endpoint = self.chart_api_url, username = user_name, password = self.user_push_chart_password)
print "#2. Create private project(PA) with user(UA);"
TestProjects.project_id, TestProjects.project_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_RA_CLIENT)
TestProjects.project_id, TestProjects.project_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_CLIENT)
print "#3. Create a new robot account(RA) with full priviliges in project(PA) with user(UA);"
robot_id, robot_account = self.project.add_project_robot_account(TestProjects.project_id, TestProjects.project_name,
2441000531 ,**TestProjects.USER_RA_CLIENT)
2441000531 ,**TestProjects.USER_CLIENT)
print robot_account.name
print robot_account.token
@ -71,7 +71,7 @@ class TestProjects(unittest.TestCase):
library.helm.helm2_push(self.chart_repo_name, self.chart_file, TestProjects.project_name, robot_account.name, robot_account.token)
print "#5. Get chart repositry from project(PA) successfully;"
# Depend on issue #12252
self.chart.chart_should_exist(TestProjects.project_name, self.CHART_NAME, **TestProjects.API_CHART_CLIENT)
print "#6. Push chart to project(PA) by Helm3 CLI with robot account(RA);"
chart_cli_ret = library.helm.helm_chart_push_to_harbor(self.chart_file, self.archive, harbor_server, TestProjects.project_name, self.repo_name, self.verion, robot_account.name, robot_account.token)

View File

@ -90,6 +90,7 @@ class TestProjects(unittest.TestCase):
format(user_user_view_logs_name, project_user_view_logs_name, "artifact", operation, log_count))
#4.1 Delete repository(RA) by user(UA);
self.repo.delete_repoitory(project_user_view_logs_name, repo_name.split('/')[1], **TestProjects.USER_USER_VIEW_LOGS_CLIENT)
time.sleep(6)
#4.2 In project(PA), there should be 1 'delete' log record;
operation = "delete"

View File

@ -32,16 +32,15 @@ Should Not Contain Tag
Add A New Tag
[Arguments] ${tag}
Retry Element Click ${add_tag_button}
Retry Double Keywords When Error Retry Element Click ${add_tag_button} Retry Wait Element ${tag_name_xpath}
Retry Text Input ${tag_name_xpath} ${tag}
Retry Element Click ${add_ok_button}
Retry Double Keywords When Error Retry Element Click ${add_ok_button} Should Contain Tag ${tag}
Delete A Tag
[Arguments] ${tag}
Retry Element Click xpath=//clr-dg-row[contains(.,'${tag}')]//clr-checkbox-wrapper//label[contains(@class,'clr-control-label')]
Retry Element Click ${delete_tag_button}
Retry Wait Until Page Contains Element ${dialog_delete_button}
Retry Element Click ${dialog_delete_button}
Retry Double Keywords When Error Retry Element Click ${delete_tag_button} Retry Wait Until Page Contains Element ${dialog_delete_button}
Retry Double Keywords When Error Retry Element Click ${dialog_delete_button} Should Not Contain Tag ${tag}
Should Contain Artifact
Retry Wait Until Page Contains Element xpath=//artifact-list-tab//clr-dg-row//a[contains(.,'sha256')]

View File

@ -196,6 +196,18 @@ Do Log Advanced Search
${rc} = Get Element Count //audit-log//clr-dg-row
Should Be Equal As Integers ${rc} 0
Retry Click Repo Name
[Arguments] ${repo_name_element}
:For ${n} IN RANGE 1 10
\ ${out} Run Keyword And Ignore Error Retry Double Keywords When Error Retry Element Click ${repo_name_element} Retry Wait Element ${tag_table_column_vulnerabilities}
\ Exit For Loop If '${out[0]}'=='PASS'
Should Be Equal As Strings '${out[0]}' 'PASS'
:For ${n} IN RANGE 1 10
\ ${out} Run Keyword And Ignore Error Retry Wait Until Page Not Contains Element ${repo_list_spinner}
\ Exit For Loop If '${out[0]}'=='PASS'
Should Be Equal As Strings '${out[0]}' 'PASS'
Go Into Repo
[Arguments] ${repoName}
Sleep 2
@ -206,13 +218,12 @@ Go Into Repo
\ Retry Clear Element Text ${repo_search_input}
\ Retry Text Input ${repo_search_input} ${repoName}
\ ${out} Run Keyword And Ignore Error Retry Wait Until Page Contains Element ${repo_name_element}
\ Exit For Loop If '${out[0]}'=='PASS'
\ Capture Page Screenshot gointo_${repoName}.png
\ Sleep 2
Retry Double Keywords When Error Retry Element Click ${repo_name_element} Retry Wait Until Page Not Contains Element ${repo_name_element}
Retry Wait Element ${tag_table_column_vulnerabilities}
Retry Wait Element ${tag_table_column_size}
Capture Page Screenshot gointo_${repoName}.png
\ Continue For Loop If '${out[0]}'=='FAIL'
\ ${out} Retry Click Repo Name ${repo_name_element}
\ Sleep 2
\ Exit For Loop
Click Index Achieve
[Arguments] ${tag_name}

View File

@ -17,93 +17,123 @@ ${SERVER_API_ENDPOINT} ${SERVER_URL}/api
&{SERVER_CONFIG} endpoint=${SERVER_API_ENDPOINT} verify_ssl=False
*** Test Cases ***
Test Case - Garbage Collection
[Tags] gc
Harbor API Test ./tests/apitests/python/test_garbage_collection.py
Test Case - Add Private Project Member and Check User Can See It
[Tags] private_member
Harbor API Test ./tests/apitests/python/test_add_member_to_private_project.py
Test Case - Delete a Repository of a Certain Project Created by Normal User
[Tags] del_repo
Harbor API Test ./tests/apitests/python/test_del_repo.py
Test Case - Add a System Global Label to a Certain Tag
[Tags] global_lbl
Harbor API Test ./tests/apitests/python/test_add_sys_label_to_tag.py
Test Case - Add Replication Rule
[Tags] replic_rule
Harbor API Test ./tests/apitests/python/test_add_replication_rule.py
Test Case - Edit Project Creation
[Tags] pro_creation
Harbor API Test ./tests/apitests/python/test_edit_project_creation.py
Test Case - Manage Project Member
[Tags] member
Harbor API Test ./tests/apitests/python/test_manage_project_member.py
Test Case - Project Level Policy Content Trust
[Tags] content_trust
Harbor API Test ./tests/apitests/python/test_project_level_policy_content_trust.py
Test Case - User View Logs
[Tags] view_logs
Harbor API Test ./tests/apitests/python/test_user_view_logs.py
Test Case - List Helm Charts
[Tags] list_helm_charts
Harbor API Test ./tests/apitests/python/test_list_helm_charts.py
Test Case - Assign Sys Admin
[Tags] assign_adin
Harbor API Test ./tests/apitests/python/test_assign_sys_admin.py
Test Case - Copy Artifact Outside Project
[Tags] copy_artifact
Harbor API Test ./tests/apitests/python/test_copy_artifact_outside_project.py
Test Case - Robot Account
[Tags] robot_account
Harbor API Test ./tests/apitests/python/test_robot_account.py
Test Case - Sign A Image
[Tags] sign_image
Harbor API Test ./tests/apitests/python/test_sign_image.py
Test Case - Project Quota
[Tags] quota
Harbor API Test ./tests/apitests/python/test_project_quota.py
Test Case - System Level CVE Whitelist
[Tags] sys_cve
Harbor API Test ./tests/apitests/python/test_sys_cve_whitelists.py
Test Case - Project Level CVE Whitelist
[Tags] pro_cve
Harbor API Test ./tests/apitests/python/test_project_level_cve_whitelist.py
Test Case - Tag Retention
[Tags] tag_retention
Harbor API Test ./tests/apitests/python/test_retention.py
Test Case - Health Check
[Tags] health
Harbor API Test ./tests/apitests/python/test_health_check.py
Test Case - Push Index By Docker Manifest
[Tags] push_index
Harbor API Test ./tests/apitests/python/test_push_index_by_docker_manifest.py
Test Case - Push Chart By Helm3 Chart CLI
[Tags] push_chart
Harbor API Test ./tests/apitests/python/test_push_chart_by_helm3_chart_cli.py
Test Case - Push Cnab Bundle
[Tags] push_cnab
Harbor API Test ./tests/apitests/python/test_push_cnab_bundle.py
Test Case - Create/Delete tag
[Tags] tag_cuid
Harbor API Test ./tests/apitests/python/test_create_delete_tag.py
Test Case - Scan Image
[Tags] scan
Harbor API Test ./tests/apitests/python/test_scan_image_artifact.py
Test Case - Scan All Images
[Tags] scan_all
Harbor API Test ./tests/apitests/python/test_system_level_scan_all.py
Test Case - Registry API
[Tags] reg_api
Harbor API Test ./tests/apitests/python/test_registry_api.py
Test Case - Push Image With Special Name
[Tags] special_repo_name
Harbor API Test ./tests/apitests/python/test_push_image_with_special_name.py
Test Case - Push Artifact With ORAS CLI
[Tags] oras
Harbor API Test ./tests/apitests/python/test_push_files_by_oras.py
Test Case - Push Singularity file With Singularity CLI
[Tags] singularity
Harbor API Test ./tests/apitests/python/test_push_sif_by_singularity.py
Test Case - Push Chart File To Chart Repository By Helm V2 With Robot Account
[Tags] helm2
Harbor API Test ./tests/apitests/python/test_push_chart_by_helm2_helm3_with_robot_Account.py

View File

@ -526,11 +526,8 @@ Test Case - Tag CRUD
Should Contain Tag latest
# add more than one tag
Add A New Tag 123
Should Contain Tag 123
Add A New Tag 456
Should Contain Tag 456
Delete A Tag latest
Should Not Contain Tag latest
Delete A Tag latest
Close Browser
Test Case - Tag Retention