update DB Migrator according latest change. (#2763)

* update DB Migrator according latest change.

update format

update

add import

update

return the rc from container

update

update to false

* add user_id

tools/migration/changelog.md

@@ -45,4 +45,7 @@ Changelog for harbor database schema
   - delete foreign key (user_id) references user(user_id)from table `access_log`
   - delete foreign key (project_id) references project(project_id)from table `access_log`
   - add column `username` varchar (32) to table `access_log`
-  - alter column `realname` on table `user`: varchar(20)->varchar(255)
+  - alter column `realname` on table `user`: varchar(20)->varchar(255)
+  - create table `img_scan_job`
+  - create table `img_scan_overview`
+  - create table `clair_vuln_timestamp`

tools/migration/db_meta.py

@@ -169,10 +169,18 @@ class ImageScanJob(Base):
 class ImageScanOverview(Base):
     __tablename__ = "img_scan_overview"
 
+    id = sa.Column(sa.Integer, nullable=False, primary_key=True)
+    image_digest = sa.Column(sa.String(128), nullable=False)
     scan_job_id = sa.Column(sa.Integer, nullable=False)
-    image_digest = sa.Column(sa.String(128), nullable=False, primary_key=True)
     severity = sa.Column(sa.Integer, nullable=False, server_default=sa.text("'0'"))
     components_overview = sa.Column(sa.String(2048))
     details_key = sa.Column(sa.String(128))
     creation_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP"))
-    update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))
+    update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))
+	
+class ClairVulnTimestamp(Base):
+    __tablename__ = "clair_vuln_timestamp"
+
+    id = sa.Column(sa.Integer, nullable=False, primary_key=True)
+    namespace = sa.Column(sa.String(128), nullable=False, unique=True)
+    last_update = sa.Column(mysql.TIMESTAMP)

tools/migration/export

@@ -0,0 +1,109 @@
+#!/usr/bin/python
+import json
+import fileinput
+from optparse import OptionParser
+import os
+import MySQLdb
+import sys
+
+class Parameters(object):
+    def __init__(self):
+        self.dbuser = ''
+        self.dbpwd = ''
+        self.exportpath = ''
+        self.init_from_input()
+
+    @staticmethod
+    def parse_input():
+        usage = "usage: %prog [options] <dbuser> <dbpwd> <exportpath>"
+        parser = OptionParser(usage)
+        parser.add_option("-u", "--dbuser", dest="dbuser", help="db user")
+        parser.add_option("-p", "--dbpwd", dest="dbpwd", help="db password")
+        parser.add_option("-o", "--exportpath", dest="exportpath", help="the path of exported json file")
+
+        (options, args) = parser.parse_args()
+        return (options.dbuser, options.dbpwd, options.exportpath)
+
+    def init_from_input(self):
+        (self.dbuser, self.dbpwd, self.exportpath) = Parameters.parse_input()
+
+class Project:
+    def __init__(self, project_id, name, public):
+        self.project_id = project_id
+        self.project_name = name
+        if public == 0:
+            self.public = "false"
+        elif public == 1:
+            self.public = "true"
+        else:
+            self.public = "false"
+
+class HarborUtil:
+    def __init__(self, dbuser, dbpwd):
+        self.serverName = 'localhost'
+        self.user = dbuser
+        self.password = dbpwd
+        self.port = '3306'
+        self.subDB = 'registry'
+        self.db = None
+        self.cursor = None
+
+    def connect(self):
+        try:
+            self.db = MySQLdb.connect(host=self.serverName, user=self.user,
+                                      passwd=self.password, db=self.subDB)
+            self.cursor = self.db.cursor()
+        except Exception, e:
+            raise Exception(e)
+
+    def close(self):
+        try:
+            self.cursor.close()
+            self.db.close()
+        except Exception, e:
+            print str(e)
+
+    def get_projects(self):
+        projects = []
+        try:
+            query = "SELECT project_id, name, public from registry.project where deleted=0"
+            self.cursor.execute(query)
+            self.cursor.fetchall()
+            for result in self.cursor:
+                projects.append(Project(int(result[0]), result[1], result[2]))
+            return projects
+        except Exception, e:
+            raise Exception(e)
+
+def delfile(src):
+    if not os.path.exists(src):
+        return
+    try:
+        os.remove(src)
+    except Exception, e:
+        raise Exception("unable to delete file: %s, error: %s" % (src, str(e)))
+
+def main():
+    commandline_input = Parameters()
+    harbor = HarborUtil(commandline_input.dbuser, commandline_input.dbpwd)
+
+    try:
+        harbor.connect()
+        projects = harbor.get_projects()
+        if len(projects) == 0:
+            return
+
+        harbor_projects_json = commandline_input.exportpath + '/harbor_projects.json'
+        delfile(harbor_projects_json)
+
+        with open(harbor_projects_json, 'w') as outfile:
+            json.dump({'projects': [project.__dict__ for project in projects]}, outfile, sort_keys=True, indent=4)
+
+    except Exception, e:
+        print e
+        sys.exit(1)
+    finally:
+        harbor.close()
+
+if __name__ == '__main__':
+    main()

tools/migration/import

@@ -0,0 +1,88 @@
+#!/usr/bin/python
+import json
+from optparse import OptionParser
+import os
+import urllib2
+import sys
+import logging
+import logging.config
+
+logging.basicConfig(filename="import_project.log", level=logging.INFO)
+logger = logging.getLogger()
+
+class Parameters(object):
+    def __init__(self):
+        self.admiral_endpoint = ''
+        self.admiral_token = ''
+        self.projectsfile = ''
+        self.init_from_input()
+
+    @staticmethod
+    def parse_input():
+        usage = "usage: %prog [options] <admiralendpoint> <token> <projectsfile>"
+        parser = OptionParser(usage)
+        parser.add_option("-a", "--admiralendpoint", dest="admiral_endpoint", help="admiral endpoint")
+        parser.add_option("-t", "--token", dest="admiral_token", help="admiral token")
+        parser.add_option("-f", "--projectsfile", dest="projectsfile", help="the path of exported json file")
+
+        (options, args) = parser.parse_args()
+        return (options.admiral_endpoint, options.admiral_token, options.projectsfile)
+
+    def init_from_input(self):
+        (self.admiral_endpoint, self.admiral_token, self.projectsfile) = Parameters.parse_input()
+
+class Project:
+    def __init__(self, name, public):
+        self.project_name = name
+        self.public = public
+
+class Admiral:
+    def __init__(self, admiral_url, token):
+        self.admiral_url = admiral_url + '/projects'
+        self.token = token
+
+    def __import_project(self, project, retry=True):
+        project_data = json.dumps({ "name": project.project_name, "isPublic": project.public,
+                                    "customProperties": {"__enableContentTrust": False, "__preventVulnerableImagesFromRunning":False,
+                                    "__preventVulnerableImagesFromRunningSeverity":"high", "__automaticallyScanImagesOnPush":False }})
+        data_len = len(project_data)
+        request = urllib2.Request(self.admiral_url, project_data)
+        request.add_header('x-xenon-auth-token', self.token)
+        request.add_header('Content-Type', 'application/json')
+        request.add_header('Content-Length', data_len)
+
+        try:
+            urllib2.urlopen(request)
+        except Exception, e:
+            if not retry:
+                logger.error("failed to import project: %s, admiral_endpoint: %s, error: %s " % (project.project_name, self.admiral_url, str(e)))
+                return
+            self.__import_project(project, False)
+
+    def import_project(self, projects):
+        for project in projects:
+            self.__import_project(project)
+
+def main():
+    commandline_input = Parameters()
+    admiral = Admiral(commandline_input.admiral_endpoint, commandline_input.admiral_token)
+
+    try:
+        if not os.path.exists(commandline_input.projectsfile):
+            raise Exception('Error: %s does not exist' % commandline_input.projectsfile)
+
+        with open(commandline_input.projectsfile, 'r') as project_data_file:
+            project_data = json.load(project_data_file)
+
+        projects_import_list = []
+        for item in project_data['projects']:
+        	projects_import_list.append(Project(item['project_name'], item['public']))
+
+        admiral.import_project(projects_import_list)
+
+    except Exception, e:
+        logger.error("failed to import project, admiral_endpoint: %s, error: %s " % (commandline_input.admiral_endpoint, str(e)))
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()

tools/migration/migration_harbor/versions/1_2_0.py

@@ -59,10 +59,11 @@ def upgrade():
     op.drop_column("access_log", "user_id")
     op.drop_column("repository", "owner_id")
 
-    #create tables: img_scan_job, img_scan_overview
+    #create tables: img_scan_job, img_scan_overview, clair_vuln_timestamp
     ImageScanJob.__table__.create(bind)
     ImageScanOverview.__table__.create(bind)
-
+    ClairVulnTimestamp.__table__.create(bind)
+	
 def downgrade():
     """
     Downgrade has been disabled.

tools/migration/run.sh

@@ -31,7 +31,7 @@ if [[ ( $1 = "up" || $1 = "upgrade" ) && ${SKIP_CONFIRM} != "y" ]]; then
     case $ans in
         [Yy]* )
             ;;
-        [Nn]* ) 
+        [Nn]* )
             exit 0
             ;;
         * ) echo "illegal answer: $ans. Upgrade abort!!"
@@ -97,6 +97,14 @@ backup)
     mysqldump $DBCNF --add-drop-database --databases registry > ./backup/registry.sql
     echo "Backup performed."
     ;;
+export)
+    echo "Performing export..."
+    ./export --dbuser ${DB_USR} --dbpwd ${DB_PWD} --exportpath ${EXPORTPATH}
+    rc="$?"
+    echo "Export performed."
+    echo $rc
+    exit $rc
+    ;;
 restore)
     echo "Performing restore..."
     mysql $DBCNF < ./backup/registry.sql