Add metrics test case (#17795)

Add more metrics validation

Signed-off-by: Yang Jiao <jiaoya@vmware.com>

tests/apitests/python/library/base.py

@@ -31,7 +31,8 @@ def _create_client(server, credential, debug, api_type="products"):
     cfg = None
     if api_type in ('projectv2', 'artifact', 'repository', 'scanner', 'scan', 'scanall', 'preheat', 'quota',
                     'replication', 'registry', 'robot', 'gc', 'retention', 'immutable', 'system_cve_allowlist',
-                    'configure', 'user', 'member', 'health', 'label', 'webhook', 'purge', 'audit_log', 'scan_data_export'):
+                    'configure', 'user', 'member', 'health', 'label', 'webhook', 'purge', 'audit_log', 'scan_data_export',
+                    'statistic', "system_info"):
         cfg = v2_swagger_client.Configuration()
     else:
         cfg = swagger_client.Configuration()
@@ -79,7 +80,9 @@ def _create_client(server, credential, debug, api_type="products"):
         "webhook": v2_swagger_client.WebhookApi(v2_swagger_client.ApiClient(cfg)),
         "purge": v2_swagger_client.PurgeApi(v2_swagger_client.ApiClient(cfg)),
         "audit_log": v2_swagger_client.AuditlogApi(v2_swagger_client.ApiClient(cfg)),
-        "scan_data_export": v2_swagger_client.ScanDataExportApi(v2_swagger_client.ApiClient(cfg))
+        "scan_data_export": v2_swagger_client.ScanDataExportApi(v2_swagger_client.ApiClient(cfg)),
+        "statistic": v2_swagger_client.StatisticApi(v2_swagger_client.ApiClient(cfg)),
+        "system_info": v2_swagger_client.SysteminfoApi(v2_swagger_client.ApiClient(cfg))
     }.get(api_type,'Error: Wrong API type')
 
 def _assert_status_code(expect_code, return_code, err_msg = r"HTTPS status code s not as we expected. Expected {}, while actual HTTPS status code is {}."):

tests/apitests/python/library/statistic.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+
+import base
+import v2_swagger_client
+from v2_swagger_client.rest import ApiException
+
+
+class Statistic(base.Base):
+
+    def __init__(self):
+        super(Statistic, self).__init__(api_type="statistic")
+
+    def get_statistic(self, expect_status_code=200, expect_response_body=None, **kwargs):
+        try:
+            return_data, status_code, _ = self._get_client(**kwargs).get_statistic_with_http_info()
+        except ApiException as e:
+            base._assert_status_code(expect_status_code, e.status)
+            if expect_response_body is not None:
+                base._assert_status_body(expect_response_body, e.body)
+            return
+        base._assert_status_code(expect_status_code, status_code)
+        return return_data

tests/apitests/python/library/system_info.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+
+import base
+from v2_swagger_client.rest import ApiException
+
+
+class System_info(base.Base):
+
+    def __init__(self):
+        super(System_info, self).__init__(api_type="system_info")
+
+    def get_system_info(self, expect_status_code=200, expect_response_body=None, **kwargs):
+        try:
+            return_data, status_code, _ = self._get_client(**kwargs).get_system_info_with_http_info()
+        except ApiException as e:
+            base._assert_status_code(expect_status_code, e.status)
+            if expect_response_body is not None:
+                base._assert_status_body(expect_response_body, e.body)
+            return
+        base._assert_status_code(expect_status_code, status_code)
+        return return_data

tests/apitests/python/test_verify_metrics_enabled.py

@@ -6,7 +6,16 @@ import unittest
 import requests
 import testutils
 
+from testutils import suppress_urllib3_warning
+from library.statistic import Statistic
+
 class TestMetricsExist(unittest.TestCase):
+
+    @suppress_urllib3_warning
+    def setUp(self):
+        statistic = Statistic()
+        self.statistic_data = statistic.get_statistic()
+
     golang_basic_metrics = ["go_gc_duration_seconds", "go_goroutines", "go_info", "go_memstats_alloc_bytes"]
 
     metrics = {
@@ -14,7 +23,21 @@ class TestMetricsExist(unittest.TestCase):
             "harbor_core_http_request_total",
             "harbor_core_http_request_duration_seconds",
             "harbor_core_http_inflight_requests"],
-        'registry': golang_basic_metrics + ["registry_http_in_flight_requests"],
+        'registry': golang_basic_metrics + [
+            "registry_http_in_flight_requests",
+            "registry_http_request_duration_seconds_bucket",
+            "registry_http_request_duration_seconds_sum",
+            "registry_http_request_duration_seconds_count",
+            "registry_http_request_size_bytes_bucket",
+            "registry_http_request_size_bytes_sum",
+            "registry_http_request_size_bytes_count",
+            "registry_http_requests_total",
+            "registry_http_response_size_bytes_bucket",
+            "registry_http_response_size_bytes_sum",
+            "registry_http_response_size_bytes_count",
+            "registry_storage_action_seconds_bucket",
+            "registry_storage_action_seconds_sum",
+            "registry_storage_action_seconds_count"],
         'exporter': golang_basic_metrics + [
             "artifact_pulled",
             "harbor_project_artifact_total",
@@ -26,7 +49,12 @@ class TestMetricsExist(unittest.TestCase):
             "harbor_task_concurrency",
             "harbor_task_queue_latency",
             "harbor_task_queue_size",
-            "harbor_task_scheduled_total"],
+            "harbor_task_scheduled_total",
+            "harbor_project_quota_usage_byte",
+            "harbor_artifact_pulled",
+            "harbor_health",
+            "harbor_system_info",
+            "harbor_up"],
         'jobservice': golang_basic_metrics + [
             "harbor_jobservice_info",
             "harbor_jobservice_task_process_time_seconds",
@@ -43,12 +71,18 @@ class TestMetricsExist(unittest.TestCase):
 
     def testMetricsExist(self):
         for k, metric_text in self.get_metrics():
-
-
-
             for metric_name in self.metrics[k]:
                 print("Metric {} should exist in {} ".format(metric_name, k))
+                self.verifyMetrics(metric_name, metric_text)
+
+    def verifyMetrics(self, metric_name, metric_text):
+        if metric_name == "harbor_project_total":
+            self.assertTrue('harbor_project_total{public="false"} ' + str(self.statistic_data.private_project_count) in metric_text)
+            self.assertTrue('harbor_project_total{public="true"} ' + str(self.statistic_data.public_project_count) in metric_text)
+        else:
             self.assertTrue(metric_name in metric_text)
 
+
+
 if __name__ == '__main__':
     unittest.main()