Merge pull request #16138 from stonezdj/21dec2_purge_upload

Add upload purge config to registry/config.yml

make/harbor.yml.tmpl

@@ -228,3 +228,12 @@ proxy:
 #   #   compression: false
 #   #   insecure: true
 #   #   timeout: 10s
+
+# enable purge _upload directories
+upload_purging:
+  enabled: true
+  # remove files in _upload directories which exist for a period of time, default is one week.
+  age: 168h
+  # the interval of the purge operations
+  interval: 24h
+  dryrun: false

make/photon/prepare/models.py

@@ -204,3 +204,33 @@ class Trace:
             self.jaeger.validate()
         elif self.otel.enabled:
             self.otel.validate()
+
+class PurgeUpload:
+    def __init__(self, config: dict):
+        if not config:
+            self.enabled = False
+        self.enabled = config.get('enabled')
+        self.age = config.get('age') or '168h'
+        self.interval = config.get('interval') or '24h'
+        self.dryrun = config.get('dryrun') or False
+        return
+
+    def validate(self):
+        if not self.enabled:
+            return
+        # age should end with h
+        if not isinstance(self.age, str) or not self.age.endswith('h'):
+            raise Exception('purge upload age should set with with nh, n is the number of hour')
+        # interval should larger than 2h
+        age = self.age[:-1]
+        if not age.isnumeric() or int(age) < 2 :
+            raise Exception('purge upload age should set with with nh, n is the number of hour and n should not be less than 2')
+        
+        # interval should end with h
+        if not isinstance(self.interval, str) or not self.interval.endswith('h'):
+            raise Exception('purge upload interval should set with with nh, n is the number of hour')
+        # interval should larger than 2h
+        interval = self.interval[:-1]
+        if not interval.isnumeric() or int(interval) < 2 :
+            raise Exception('purge upload interval should set with with nh, n is the number of hour and n should not beless than 2')
+        return

make/photon/prepare/templates/registry/config.yml.jinja

@@ -9,7 +9,18 @@ storage:
   {{storage_provider_info}}
   maintenance:
     uploadpurging:
+{% if purge_upload.enabled %}
+      enabled: true
+      age: {{ purge_upload.age }}
+      interval: {{ purge_upload.interval }}
+      {% if purge_upload.dryrun %}
+      dryrun: true
+      {% else %}
+      dryrun: false
+      {% endif %}
+{% else %}
       enabled: false
+{% endif %}
   delete:
     enabled: true
 {% if storage_redirect_disabled %}

make/photon/prepare/test_purgeuploads.py

@@ -0,0 +1,88 @@
+
+import unittest
+from models import PurgeUpload
+
+class TestPurgeUploadsDefault(unittest.TestCase):
+    def test_validate_config(self):
+        purge_config = dict([
+            ('enabled',True),
+            ('age','168h'),
+            ('interval','24h'),
+            ('dryrun', False),
+        ])
+        cfg = PurgeUpload(purge_config)
+        cfg.validate()
+
+    def test_validate_config(self):
+        purge_config = dict([
+            ('enabled','false'),
+            ('age','168h'),
+            ('interval','24h'),
+            ('dryrun', 'false'),
+        ])
+        cfg = PurgeUpload(purge_config)
+        cfg.validate()
+
+    def test_validate_config_2hour(self):
+        purge_config = dict([
+            ('enabled',True),
+            ('age','2h'),
+            ('interval','2h'),
+            ('dryrun', False),
+        ])
+        cfg = PurgeUpload(purge_config)
+        cfg.validate()        
+
+    def test_validate_config_1hour(self):
+        purge_config = dict([
+            ('enabled',True),
+            ('age','1h'),
+            ('interval','1h'),
+            ('dryrun', False),
+        ])
+        cfg = PurgeUpload(purge_config)
+        with self.assertRaises(Exception):
+            cfg.validate()
+
+    def test_validate_config_invalid_format(self):
+        purge_config = dict([
+            ('enabled',True),
+            ('age','1s'),
+            ('interval','1s'),
+            ('dryrun', False),
+        ])
+        cfg = PurgeUpload(purge_config)
+        with self.assertRaises(Exception):
+            cfg.validate()      
+
+    def test_validate_config_invalid_format(self):
+        purge_config = dict([
+            ('enabled',True),
+            ('age',168),
+            ('interval',24),
+            ('dryrun', False),
+        ])
+        cfg = PurgeUpload(purge_config)
+        with self.assertRaises(Exception):
+            cfg.validate()  
+
+    def test_validate_config_disabled_invalid_format(self):
+        purge_config = dict([
+            ('enabled',"false"),
+            ('age','ssh'),
+            ('interval','ssh'),
+            ('dryrun', False),
+        ])
+        cfg = PurgeUpload(purge_config)
+        cfg.validate()  
+
+    def test_validate_config_invalid_string(self):
+        purge_config = dict([
+            ('enabled',True),
+            ('age','ssh'),
+            ('interval','ssh'),
+            ('dryrun', False),
+        ])
+        cfg = PurgeUpload(purge_config)
+        with self.assertRaises(Exception):
+            cfg.validate()

make/photon/prepare/utils/configs.py

@@ -3,7 +3,7 @@ import os
 import yaml
 from urllib.parse import urlencode
 from g import versions_file_path, host_root_dir, DEFAULT_UID, INTERNAL_NO_PROXY_DN
-from models import InternalTLS, Metric, Trace
+from models import InternalTLS, Metric, Trace, PurgeUpload
 from utils.misc import generate_random_string, owner_can_read, other_can_read
 
 default_db_max_idle_conns = 2  # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns
@@ -78,6 +78,9 @@ def validate(conf: dict, **kwargs):
 
     if conf.get('trace'):
         conf['trace'].validate()
+        
+    if conf.get('purge_upload'):
+        conf['purge_upload'].validate()
 
 def parse_versions():
     if not versions_file_path.is_file():
@@ -346,6 +349,10 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
         # config_dict['notary_url'] = 'http://notary-server:4443'
         config_dict['chart_repository_url'] = 'https://chartmuseum:9443'
 
+    # purge upload configs
+    purge_upload_config = configs.get('upload_purging')
+    config_dict['purge_upload'] = PurgeUpload(purge_upload_config or {})
+
     return config_dict