diff --git a/tools/migration/changelog.md b/tools/migration/changelog.md index 20ea6d8e2..cfcb21912 100644 --- a/tools/migration/changelog.md +++ b/tools/migration/changelog.md @@ -45,4 +45,7 @@ Changelog for harbor database schema - delete foreign key (user_id) references user(user_id)from table `access_log` - delete foreign key (project_id) references project(project_id)from table `access_log` - add column `username` varchar (32) to table `access_log` - - alter column `realname` on table `user`: varchar(20)->varchar(255) \ No newline at end of file + - alter column `realname` on table `user`: varchar(20)->varchar(255) + - create table `img_scan_job` + - create table `img_scan_overview` + - create table `clair_vuln_timestamp` \ No newline at end of file diff --git a/tools/migration/db_meta.py b/tools/migration/db_meta.py index ac7d45bf2..248667fe7 100644 --- a/tools/migration/db_meta.py +++ b/tools/migration/db_meta.py @@ -169,10 +169,18 @@ class ImageScanJob(Base): class ImageScanOverview(Base): __tablename__ = "img_scan_overview" + id = sa.Column(sa.Integer, nullable=False, primary_key=True) + image_digest = sa.Column(sa.String(128), nullable=False) scan_job_id = sa.Column(sa.Integer, nullable=False) - image_digest = sa.Column(sa.String(128), nullable=False, primary_key=True) severity = sa.Column(sa.Integer, nullable=False, server_default=sa.text("'0'")) components_overview = sa.Column(sa.String(2048)) details_key = sa.Column(sa.String(128)) creation_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP")) - update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP")) \ No newline at end of file + update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP")) + +class ClairVulnTimestamp(Base): + __tablename__ = "clair_vuln_timestamp" + + id = sa.Column(sa.Integer, nullable=False, primary_key=True) + namespace = sa.Column(sa.String(128), nullable=False, unique=True) + last_update = sa.Column(mysql.TIMESTAMP) diff --git a/tools/migration/export b/tools/migration/export new file mode 100755 index 000000000..76f746193 --- /dev/null +++ b/tools/migration/export @@ -0,0 +1,109 @@ +#!/usr/bin/python +import json +import fileinput +from optparse import OptionParser +import os +import MySQLdb +import sys + +class Parameters(object): + def __init__(self): + self.dbuser = '' + self.dbpwd = '' + self.exportpath = '' + self.init_from_input() + + @staticmethod + def parse_input(): + usage = "usage: %prog [options] " + parser = OptionParser(usage) + parser.add_option("-u", "--dbuser", dest="dbuser", help="db user") + parser.add_option("-p", "--dbpwd", dest="dbpwd", help="db password") + parser.add_option("-o", "--exportpath", dest="exportpath", help="the path of exported json file") + + (options, args) = parser.parse_args() + return (options.dbuser, options.dbpwd, options.exportpath) + + def init_from_input(self): + (self.dbuser, self.dbpwd, self.exportpath) = Parameters.parse_input() + +class Project: + def __init__(self, project_id, name, public): + self.project_id = project_id + self.project_name = name + if public == 0: + self.public = "false" + elif public == 1: + self.public = "true" + else: + self.public = "false" + +class HarborUtil: + def __init__(self, dbuser, dbpwd): + self.serverName = 'localhost' + self.user = dbuser + self.password = dbpwd + self.port = '3306' + self.subDB = 'registry' + self.db = None + self.cursor = None + + def connect(self): + try: + self.db = MySQLdb.connect(host=self.serverName, user=self.user, + passwd=self.password, db=self.subDB) + self.cursor = self.db.cursor() + except Exception, e: + raise Exception(e) + + def close(self): + try: + self.cursor.close() + self.db.close() + except Exception, e: + print str(e) + + def get_projects(self): + projects = [] + try: + query = "SELECT project_id, name, public from registry.project where deleted=0" + self.cursor.execute(query) + self.cursor.fetchall() + for result in self.cursor: + projects.append(Project(int(result[0]), result[1], result[2])) + return projects + except Exception, e: + raise Exception(e) + +def delfile(src): + if not os.path.exists(src): + return + try: + os.remove(src) + except Exception, e: + raise Exception("unable to delete file: %s, error: %s" % (src, str(e))) + +def main(): + commandline_input = Parameters() + harbor = HarborUtil(commandline_input.dbuser, commandline_input.dbpwd) + + try: + harbor.connect() + projects = harbor.get_projects() + if len(projects) == 0: + return + + harbor_projects_json = commandline_input.exportpath + '/harbor_projects.json' + delfile(harbor_projects_json) + + with open(harbor_projects_json, 'w') as outfile: + json.dump({'projects': [project.__dict__ for project in projects]}, outfile, sort_keys=True, indent=4) + + except Exception, e: + print e + sys.exit(1) + finally: + harbor.close() + +if __name__ == '__main__': + main() diff --git a/tools/migration/import b/tools/migration/import new file mode 100755 index 000000000..af97830ec --- /dev/null +++ b/tools/migration/import @@ -0,0 +1,88 @@ +#!/usr/bin/python +import json +from optparse import OptionParser +import os +import urllib2 +import sys +import logging +import logging.config + +logging.basicConfig(filename="import_project.log", level=logging.INFO) +logger = logging.getLogger() + +class Parameters(object): + def __init__(self): + self.admiral_endpoint = '' + self.admiral_token = '' + self.projectsfile = '' + self.init_from_input() + + @staticmethod + def parse_input(): + usage = "usage: %prog [options] " + parser = OptionParser(usage) + parser.add_option("-a", "--admiralendpoint", dest="admiral_endpoint", help="admiral endpoint") + parser.add_option("-t", "--token", dest="admiral_token", help="admiral token") + parser.add_option("-f", "--projectsfile", dest="projectsfile", help="the path of exported json file") + + (options, args) = parser.parse_args() + return (options.admiral_endpoint, options.admiral_token, options.projectsfile) + + def init_from_input(self): + (self.admiral_endpoint, self.admiral_token, self.projectsfile) = Parameters.parse_input() + +class Project: + def __init__(self, name, public): + self.project_name = name + self.public = public + +class Admiral: + def __init__(self, admiral_url, token): + self.admiral_url = admiral_url + '/projects' + self.token = token + + def __import_project(self, project, retry=True): + project_data = json.dumps({ "name": project.project_name, "isPublic": project.public, + "customProperties": {"__enableContentTrust": False, "__preventVulnerableImagesFromRunning":False, + "__preventVulnerableImagesFromRunningSeverity":"high", "__automaticallyScanImagesOnPush":False }}) + data_len = len(project_data) + request = urllib2.Request(self.admiral_url, project_data) + request.add_header('x-xenon-auth-token', self.token) + request.add_header('Content-Type', 'application/json') + request.add_header('Content-Length', data_len) + + try: + urllib2.urlopen(request) + except Exception, e: + if not retry: + logger.error("failed to import project: %s, admiral_endpoint: %s, error: %s " % (project.project_name, self.admiral_url, str(e))) + return + self.__import_project(project, False) + + def import_project(self, projects): + for project in projects: + self.__import_project(project) + +def main(): + commandline_input = Parameters() + admiral = Admiral(commandline_input.admiral_endpoint, commandline_input.admiral_token) + + try: + if not os.path.exists(commandline_input.projectsfile): + raise Exception('Error: %s does not exist' % commandline_input.projectsfile) + + with open(commandline_input.projectsfile, 'r') as project_data_file: + project_data = json.load(project_data_file) + + projects_import_list = [] + for item in project_data['projects']: + projects_import_list.append(Project(item['project_name'], item['public'])) + + admiral.import_project(projects_import_list) + + except Exception, e: + logger.error("failed to import project, admiral_endpoint: %s, error: %s " % (commandline_input.admiral_endpoint, str(e))) + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/tools/migration/migration_harbor/versions/1_2_0.py b/tools/migration/migration_harbor/versions/1_2_0.py index 5c41509fd..a00654dcf 100644 --- a/tools/migration/migration_harbor/versions/1_2_0.py +++ b/tools/migration/migration_harbor/versions/1_2_0.py @@ -59,10 +59,11 @@ def upgrade(): op.drop_column("access_log", "user_id") op.drop_column("repository", "owner_id") - #create tables: img_scan_job, img_scan_overview + #create tables: img_scan_job, img_scan_overview, clair_vuln_timestamp ImageScanJob.__table__.create(bind) ImageScanOverview.__table__.create(bind) - + ClairVulnTimestamp.__table__.create(bind) + def downgrade(): """ Downgrade has been disabled. diff --git a/tools/migration/run.sh b/tools/migration/run.sh index 82fc09d87..be1935064 100755 --- a/tools/migration/run.sh +++ b/tools/migration/run.sh @@ -31,7 +31,7 @@ if [[ ( $1 = "up" || $1 = "upgrade" ) && ${SKIP_CONFIRM} != "y" ]]; then case $ans in [Yy]* ) ;; - [Nn]* ) + [Nn]* ) exit 0 ;; * ) echo "illegal answer: $ans. Upgrade abort!!" @@ -97,6 +97,14 @@ backup) mysqldump $DBCNF --add-drop-database --databases registry > ./backup/registry.sql echo "Backup performed." ;; +export) + echo "Performing export..." + ./export --dbuser ${DB_USR} --dbpwd ${DB_PWD} --exportpath ${EXPORTPATH} + rc="$?" + echo "Export performed." + echo $rc + exit $rc + ;; restore) echo "Performing restore..." mysql $DBCNF < ./backup/registry.sql