diff --git a/make/photon/prepare/commands/migrate.py b/make/photon/prepare/commands/migrate.py index 3abc0c8a8..c6fd0d34b 100644 --- a/make/photon/prepare/commands/migrate.py +++ b/make/photon/prepare/commands/migrate.py @@ -14,6 +14,9 @@ from migrations import accept_versions def migrate(input_, output, target): """ migrate command will migrate config file style to specific version + :input_: is the path of the original config file + :output: is the destination path of config file, the generated configs will storage in it + :target: is the the target version of config file will upgrade to """ if target not in accept_versions: click.echo('target version {} not supported'.format(target)) diff --git a/make/photon/prepare/migrations/version_1_10_0/__init__.py b/make/photon/prepare/migrations/version_1_10_0/__init__.py index 4d383c937..84b89c0a1 100644 --- a/make/photon/prepare/migrations/version_1_10_0/__init__.py +++ b/make/photon/prepare/migrations/version_1_10_0/__init__.py @@ -3,7 +3,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined from utils.migration import read_conf revision = '1.10.0' -down_revision = '1.9.0' +down_revisions = ['1.9.0'] def migrate(input_cfg, output_cfg): config_dict = read_conf(input_cfg) diff --git a/make/photon/prepare/migrations/version_1_9_0/__init__.py b/make/photon/prepare/migrations/version_1_9_0/__init__.py index 041dcf4f7..1e9cdd61f 100644 --- a/make/photon/prepare/migrations/version_1_9_0/__init__.py +++ b/make/photon/prepare/migrations/version_1_9_0/__init__.py @@ -3,14 +3,14 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined from utils.migration import read_conf revision = '1.9.0' -down_revision = None +down_revisions = [] def migrate(input_cfg, output_cfg): config_dict = read_conf(input_cfg) - this_dir = os.path.dirname(__file__) + current_dir = os.path.dirname(__file__) tpl = Environment( - loader=FileSystemLoader(this_dir), + loader=FileSystemLoader(current_dir), undefined=StrictUndefined, trim_blocks=True, lstrip_blocks=True diff --git a/make/photon/prepare/migrations/version_1_9_0/harbor.yml.jinja b/make/photon/prepare/migrations/version_1_9_0/harbor.yml.jinja index c8867486a..6d6b9750b 100644 --- a/make/photon/prepare/migrations/version_1_9_0/harbor.yml.jinja +++ b/make/photon/prepare/migrations/version_1_9_0/harbor.yml.jinja @@ -48,7 +48,7 @@ harbor_admin_password: {{ harbor_admin_password }} # Harbor DB configuration database: # The password for the root user of Harbor DB. Change this before any production use. - password: {{ database.password}} + password: {{ database.password }} # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained. max_idle_conns: 50 # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections. diff --git a/make/photon/prepare/migrations/version_2_0_0/__init__.py b/make/photon/prepare/migrations/version_2_0_0/__init__.py index 344ce5b63..9dd4c135d 100644 --- a/make/photon/prepare/migrations/version_2_0_0/__init__.py +++ b/make/photon/prepare/migrations/version_2_0_0/__init__.py @@ -3,7 +3,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined from utils.migration import read_conf revision = '2.0.0' -down_revision = '1.10.0' +down_revisions = ['1.10.0'] def migrate(input_cfg, output_cfg): config_dict = read_conf(input_cfg) diff --git a/make/photon/prepare/tests/migrations/utils_test.py b/make/photon/prepare/tests/migrations/utils_test.py index 5fd15b5c6..df74c5399 100644 --- a/make/photon/prepare/tests/migrations/utils_test.py +++ b/make/photon/prepare/tests/migrations/utils_test.py @@ -1,36 +1,46 @@ import pytest import importlib -from utils.migration import search +from utils.migration import search, MigratioNotFound class mockModule: - def __init__(self, revision, down_revision): + def __init__(self, revision: str, down_revisions: list): self.revision = revision - self.down_revision = down_revision + self.down_revisions = down_revisions def mock_import_module_loop(module_path: str): - loop_modules = { - 'migration.versions.1_9_0': mockModule('1.9.0', None), - 'migration.versions.1_10_0': mockModule('1.10.0', '2.0.0'), - 'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0') + modules = { + 'migrations.version_1_9_0': mockModule('1.9.0', []), + 'migrations.version_1_10_0': mockModule('1.10.0', ['2.0.0']), + 'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0']) } - return loop_modules[module_path] + return modules[module_path] def mock_import_module_mission(module_path: str): - loop_modules = { - 'migration.versions.1_9_0': mockModule('1.9.0', None), - 'migration.versions.1_10_0': mockModule('1.10.0', None), - 'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0') + modules = { + 'migrations.version_1_9_0': mockModule('1.9.0', []), + 'migrations.version_1_10_0': mockModule('1.10.0', []), + 'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0']) } - return loop_modules[module_path] + return modules[module_path] def mock_import_module_success(module_path: str): - loop_modules = { - 'migration.versions.1_9_0': mockModule('1.9.0', None), - 'migration.versions.1_10_0': mockModule('1.10.0', '1.9.0'), - 'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0') + modules = { + 'migrations.version_1_9_0': mockModule('1.9.0', []), + 'migrations.version_1_10_0': mockModule('1.10.0', ['1.9.0']), + 'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0']) } - return loop_modules[module_path] + return modules[module_path] + +def mock_import_module_success_multi_downversion(module_path: str): + modules = { + 'migrations.version_1_9_0': mockModule('1.9.0', []), + 'migrations.version_1_10_0': mockModule('1.10.0', ['1.9.0']), + 'migrations.version_1_10_1': mockModule('1.10.1', ['1.9.0']), + 'migrations.version_1_10_2': mockModule('1.10.2', ['1.9.0']), + 'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0', '1.10.1', '1.10.2']) + } + return modules[module_path] @pytest.fixture def mock_import_module_with_loop(monkeypatch): @@ -44,15 +54,25 @@ def mock_import_module_with_mission(monkeypatch): def mock_import_module_with_success(monkeypatch): monkeypatch.setattr(importlib, "import_module", mock_import_module_success) +@pytest.fixture +def mock_import_module_with_success_multi_downversion(monkeypatch): + monkeypatch.setattr(importlib, "import_module", mock_import_module_success_multi_downversion) + def test_search_loop(mock_import_module_with_loop): with pytest.raises(Exception): search('1.9.0', '2.0.0') def test_search_mission(mock_import_module_with_mission): - with pytest.raises(Exception): + with pytest.raises(MigratioNotFound): search('1.9.0', '2.0.0') -def test_search_success(): +def test_search_success(mock_import_module_with_success): migration_path = search('1.9.0', '2.0.0') assert migration_path[0].revision == '1.10.0' assert migration_path[1].revision == '2.0.0' + +def test_search_success_multi_downversion(mock_import_module_with_success_multi_downversion): + migration_path = search('1.9.0', '2.0.0') + print(migration_path) + assert migration_path[0].revision == '1.10.2' + assert migration_path[1].revision == '2.0.0' diff --git a/make/photon/prepare/utils/migration.py b/make/photon/prepare/utils/migration.py index 778b7e496..1389bae45 100644 --- a/make/photon/prepare/utils/migration.py +++ b/make/photon/prepare/utils/migration.py @@ -4,7 +4,24 @@ import importlib import os from collections import deque -from migrations import MIGRATION_BASE_DIR +class MigratioNotFound(Exception): ... + +class MigrationVersion: + ''' + The version used to migration + + Arttribute: + name(str): version name like `1.0.0` + module: the python module object for a specific migration which contains migrate info, codes and templates + down_versions(list): previous versions that can migrated to this version + ''' + def __init__(self, version: str): + self.name = version + self.module = importlib.import_module("migrations.version_{}".format(version.replace(".","_"))) + + @property + def down_versions(self): + return self.module.down_revisions def read_conf(path): with open(path) as f: @@ -15,29 +32,35 @@ def read_conf(path): exit(-1) return d -def _to_module_path(ver): - return "migrations.version_{}".format(ver.replace(".","_")) +def search(input_version: str, target_version: str) -> list : + """ + Find the migration path by BFS + Args: + input_version(str): The version migration start from + target_version(str): The target version migrated to + Returns: + list: the module of migrations in the upgrade path + """ + upgrade_path = [] + next_version, visited, q = {}, set(), deque() + q.append(target_version) + found = False + while q: # BFS to find a valid path + version = MigrationVersion(q.popleft()) + visited.add(version.name) + if version.name == input_version: + found = True + break # break loop cause migration path found + for v in version.down_versions: + next_version[v] = version.name + if v not in (visited.union(q)): + q.append(v) -def search(input_ver: str, target_ver: str) -> deque : - """ - Search accept a input version and the target version. - Returns the module of migrations in the upgrade path - """ - upgrade_path, visited = deque(), set() - while True: - module_path = _to_module_path(target_ver) - visited.add(target_ver) # mark current version for loop finding - if os.path.isdir(os.path.join(MIGRATION_BASE_DIR, 'version_{}'.format(target_ver.replace(".","_")))): - module = importlib.import_module(module_path) - if module.revision == input_ver: # migration path found - break - elif module.down_revision is None: # migration path not found - raise Exception('no migration path found') - else: - upgrade_path.appendleft(module) - target_ver = module.down_revision - if target_ver in visited: # version visited before, loop found - raise Exception('find a loop caused by {} on migration path'.format(target_ver)) - else: - raise Exception('{} not dir'.format(os.path.join(MIGRATION_BASE_DIR, 'versions', target_ver.replace(".","_")))) - return upgrade_path + if not found: + raise MigratioNotFound('no migration path found to target version') + + current_version = MigrationVersion(input_version) + while current_version.name != target_version: + current_version = MigrationVersion(next_version[current_version.name]) + upgrade_path.append(current_version) + return list(map(lambda x: x.module, upgrade_path)) \ No newline at end of file