diff --git a/.travis.yml b/.travis.yml index 604d2cfff..1b0160898 100644 --- a/.travis.yml +++ b/.travis.yml @@ -93,7 +93,7 @@ script: - sudo ./ldapprepare.sh - sudo ./admiral.sh - cd .. - - go test -i ./src/ui ./src/adminserver ./src/jobservice + - go test -race -i ./src/ui ./src/adminserver ./src/jobservice - sudo -E env "PATH=$PATH" ./tests/coverage4gotest.sh - goveralls -coverprofile=profile.cov -service=travis-ci diff --git a/tools/migration/import b/tools/migration/import index f8706c573..16bd81435 100755 --- a/tools/migration/import +++ b/tools/migration/import @@ -15,6 +15,7 @@ class Parameters(object): self.admiral_endpoint = '' self.tokenfile = '' self.projectsfile = '' + self.mapprojectsfile = '' self.init_from_input() @staticmethod @@ -24,17 +25,20 @@ class Parameters(object): parser.add_option("-a", "--admiralendpoint", dest="admiral_endpoint", help="admiral endpoint") parser.add_option("-t", "--tokenfile", dest="tokenfile", help="the path of token file") parser.add_option("-f", "--projectsfile", dest="projectsfile", help="the path of exported json file") + parser.add_option("-m", "--mapprojectsfile", dest="mapprojectsfile", help="the path of output projects file for mapping project id") (options, args) = parser.parse_args() - return (options.admiral_endpoint, options.tokenfile, options.projectsfile) + return (options.admiral_endpoint, options.tokenfile, options.projectsfile, options.mapprojectsfile) def init_from_input(self): - (self.admiral_endpoint, self.tokenfile, self.projectsfile) = Parameters.parse_input() + (self.admiral_endpoint, self.tokenfile, self.projectsfile, self.mapprojectsfile) = Parameters.parse_input() class Project: - def __init__(self, name, public): + def __init__(self, project_id, name, public): + self.project_id = project_id self.project_name = name self.public = public + self.index_id = '' class Admiral: def __init__(self, admiral_url, token): @@ -52,7 +56,9 @@ class Admiral: request.add_header('Content-Length', data_len) try: - urllib2.urlopen(request, context=ssl._create_unverified_context()) + response = urllib2.urlopen(request, context=ssl._create_unverified_context()) + response_obj = response.read() + project.index_id = json.loads(response_obj)['customProperties']['__projectIndex'] except Exception, e: if not retry: logger.error("failed to import project: %s, admiral_endpoint: %s, error: %s " % (project.project_name, self.admiral_url, str(e))) @@ -86,13 +92,16 @@ def main(): projects_import_list = [] for item in project_data['projects']: - projects_import_list.append(Project(item['project_name'], item['public'])) + projects_import_list.append(Project(item['project_id'], item['project_name'], item['public'])) admiral.import_project(projects_import_list) + + with open(commandline_input.mapprojectsfile, 'w') as outfile: + json.dump({'map_projects': [project.__dict__ for project in projects_import_list]}, outfile, sort_keys=True, indent=4) except Exception, e: logger.error("failed to import project, admiral_endpoint: %s, error: %s " % (commandline_input.admiral_endpoint, str(e))) sys.exit(1) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/tools/migration/mapprojects b/tools/migration/mapprojects new file mode 100755 index 000000000..898919053 --- /dev/null +++ b/tools/migration/mapprojects @@ -0,0 +1,226 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import json +import fileinput +from optparse import OptionParser +import os +import MySQLdb +import sys + + +class Parameters(object): + + def __init__(self): + self.dbuser = '' + self.dbpwd = '' + self.mapprojectsfile = '' + self.init_from_input() + + @staticmethod + def parse_input(): + usage = \ + 'usage: %prog [options] ' + parser = OptionParser(usage) + parser.add_option('-u', '--dbuser', dest='dbuser', + help='db user') + parser.add_option('-p', '--dbpwd', dest='dbpwd', + help='db password') + parser.add_option('-m', '--mapprojectsfile', + dest='mapprojectsfile', + help='the path of mapping projects file') + + (options, args) = parser.parse_args() + return (options.dbuser, options.dbpwd, options.mapprojectsfile) + + def init_from_input(self): + (self.dbuser, self.dbpwd, self.mapprojectsfile) = \ + Parameters.parse_input() + + +class AccessLog: + + def __init__(self, log_id, project_id): + self.log_id = log_id + self.project_id = project_id + + +class Repository: + + def __init__(self, repository_id, project_id): + self.repository_id = repository_id + self.project_id = project_id + + +class ReplicationPolicy: + + def __init__(self, replication_policy_id, project_id): + self.replication_policy_id = replication_policy_id + self.project_id = project_id + + +class Project: + + def __init__( + self, + project_id, + name, + index_id, + ): + + self.project_id = project_id + self.project_name = name + self.index_id = index_id + + +class HarborUtil: + + def __init__(self, dbuser, dbpwd): + self.serverName = 'localhost' + self.user = dbuser + self.password = dbpwd + self.port = '3306' + self.subDB = 'registry' + self.db = None + self.cursor = None + + def connect(self): + try: + self.db = MySQLdb.connect(host=self.serverName, + user=self.user, passwd=self.password, db=self.subDB) + self.db.autocommit(False) + self.cursor = self.db.cursor() + except Exception, e: + raise Exception(e) + + def close(self): + try: + self.db.commit() + self.cursor.close() + self.db.close() + except Exception, e: + print str(e) + + def enable_foreign_key_check(self): + try: + self.cursor.execute('SET FOREIGN_KEY_CHECKS=1') + except Exception, e: + print str(e) + + def disable_foreign_key_check(self): + try: + self.cursor.execute('SET FOREIGN_KEY_CHECKS=0') + except Exception, e: + print str(e) + + def get_index_id(self, projects, project_id): + for project in projects: + if project.project_id == project_id: + return project.index_id + return '' + + def update_access_log_table(self, projects): + access_logs = [] + try: + query_sccess_log = \ + 'SELECT log_id, project_id from registry.access_log' + self.cursor.execute(query_sccess_log) + self.cursor.fetchall() + for result in self.cursor: + access_logs.append(AccessLog(result[0], result[1])) + except Exception, e: + raise Exception(e) + + for item in access_logs: + index_id = self.get_index_id(projects, item.project_id) + if index_id != '': + try: + update_access_log_project_id = \ + 'UPDATE registry.access_log SET project_id=%s where log_id=%s' \ + % (index_id, item.log_id) + self.cursor.execute(update_access_log_project_id) + except Exception, e: + raise Exception(e) + + def update_repository_table(self, projects): + repositories = [] + try: + query_repository = \ + 'SELECT repository_id, project_id from registry.repository' + self.cursor.execute(query_repository) + self.cursor.fetchall() + for result in self.cursor: + repositories.append(Repository(result[0], result[1])) + except Exception, e: + raise Exception(e) + + for item in repositories: + index_id = self.get_index_id(projects, item.project_id) + if index_id != '': + try: + update_repository_project_id = \ + 'UPDATE registry.repository SET project_id=%s where repository_id=%s' \ + % (index_id, item.repository_id) + self.cursor.execute(update_repository_project_id) + except Exception, e: + raise Exception(e) + + def update_replication_policy_table(self, projects): + replication_policies = [] + try: + query_replication_policy = \ + 'SELECT id, project_id from registry.replication_policy' + self.cursor.execute(query_replication_policy) + self.cursor.fetchall() + for result in self.cursor: + replication_policies.append(ReplicationPolicy(result[0], + result[1])) + except Exception, e: + raise Exception(e) + + for item in replication_policies: + index_id = self.get_index_id(projects, item.project_id) + if index_id != '': + try: + update_replication_policy_id = \ + 'UPDATE registry.replication_policy SET project_id=%s where id=%s' \ + % (index_id, item.replication_policy_id) + self.cursor.execute(update_replication_policy_id) + except Exception, e: + raise Exception(e) + + +def main(): + commandline_input = Parameters() + harbor = HarborUtil(commandline_input.dbuser, + commandline_input.dbpwd) + + try: + harbor.connect() + harbor.disable_foreign_key_check() + + with open(commandline_input.mapprojectsfile, 'r') as \ + project_mapping_file: + project_mapping_data = json.load(project_mapping_file) + + projects_mapping_list = [] + for item in project_mapping_data['map_projects']: + projects_mapping_list.append(Project(item['project_id'], + item['project_name'], item['index_id'])) + + harbor.update_access_log_table(projects_mapping_list) + harbor.update_repository_table(projects_mapping_list) + harbor.update_replication_policy_table(projects_mapping_list) + except Exception, e: + print e + sys.exit(1) + finally: + harbor.enable_foreign_key_check() + harbor.close() + + +if __name__ == '__main__': + main() + + + diff --git a/tools/migration/run.sh b/tools/migration/run.sh index 17767d532..c4141b87f 100755 --- a/tools/migration/run.sh +++ b/tools/migration/run.sh @@ -110,6 +110,14 @@ export) echo $rc exit $rc ;; +mapprojects) + echo "Performing map projects..." + ./mapprojects --dbuser ${DB_USR} --dbpwd ${DB_PWD} --mapprojectsfile ${MAPPROJECTFILE} + rc="$?" + echo "Map projects performed." + echo $rc + exit $rc + ;; restore) echo "Performing restore..." mysql $DBCNF < ./backup/registry.sql