Merge pull request #3172 from vmware/release-1.2.0

Release 1.2.0

.travis.yml

@@ -93,7 +93,7 @@ script:
   - sudo ./ldapprepare.sh 
   - sudo ./admiral.sh 
   - cd .. 
-  - go test -i ./src/ui ./src/adminserver ./src/jobservice
+  - go test -race -i ./src/ui ./src/adminserver ./src/jobservice
   - sudo -E env "PATH=$PATH" ./tests/coverage4gotest.sh
   - goveralls -coverprofile=profile.cov -service=travis-ci
 

tools/migration/import

@@ -15,6 +15,7 @@ class Parameters(object):
         self.admiral_endpoint = ''
         self.tokenfile = ''
         self.projectsfile = ''
+        self.mapprojectsfile = ''
         self.init_from_input()
 
     @staticmethod
@@ -24,17 +25,20 @@ class Parameters(object):
         parser.add_option("-a", "--admiralendpoint", dest="admiral_endpoint", help="admiral endpoint")
         parser.add_option("-t", "--tokenfile", dest="tokenfile", help="the path of token file")
         parser.add_option("-f", "--projectsfile", dest="projectsfile", help="the path of exported json file")
+        parser.add_option("-m", "--mapprojectsfile", dest="mapprojectsfile", help="the path of output projects file for mapping project id")
 
         (options, args) = parser.parse_args()
-        return (options.admiral_endpoint, options.tokenfile, options.projectsfile)
+        return (options.admiral_endpoint, options.tokenfile, options.projectsfile, options.mapprojectsfile)
 
     def init_from_input(self):
-        (self.admiral_endpoint, self.tokenfile, self.projectsfile) = Parameters.parse_input()
+        (self.admiral_endpoint, self.tokenfile, self.projectsfile, self.mapprojectsfile) = Parameters.parse_input()
 
 class Project:
-    def __init__(self, name, public):
+    def __init__(self, project_id, name, public):
+	self.project_id = project_id
         self.project_name = name
         self.public = public
+        self.index_id = ''
 
 class Admiral:
     def __init__(self, admiral_url, token):
@@ -52,7 +56,9 @@ class Admiral:
         request.add_header('Content-Length', data_len)
 
         try:
-            urllib2.urlopen(request, context=ssl._create_unverified_context())
+            response = urllib2.urlopen(request, context=ssl._create_unverified_context())
+            response_obj = response.read()
+            project.index_id = json.loads(response_obj)['customProperties']['__projectIndex']
         except Exception, e:
             if not retry:
                 logger.error("failed to import project: %s, admiral_endpoint: %s, error: %s " % (project.project_name, self.admiral_url, str(e)))
@@ -86,13 +92,16 @@ def main():
 
         projects_import_list = []
         for item in project_data['projects']:
-        	projects_import_list.append(Project(item['project_name'], item['public']))
+            projects_import_list.append(Project(item['project_id'], item['project_name'], item['public']))
 
         admiral.import_project(projects_import_list)
+			
+        with open(commandline_input.mapprojectsfile, 'w') as outfile:
+            json.dump({'map_projects': [project.__dict__ for project in projects_import_list]}, outfile, sort_keys=True, indent=4)			
 
     except Exception, e:
         logger.error("failed to import project, admiral_endpoint: %s, error: %s " % (commandline_input.admiral_endpoint, str(e)))
         sys.exit(1)
 
 if __name__ == '__main__':
-    main()
+    main()

tools/migration/mapprojects

@@ -0,0 +1,226 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+import json
+import fileinput
+from optparse import OptionParser
+import os
+import MySQLdb
+import sys
+
+
+class Parameters(object):
+
+    def __init__(self):
+        self.dbuser = ''
+        self.dbpwd = ''
+        self.mapprojectsfile = ''
+        self.init_from_input()
+
+    @staticmethod
+    def parse_input():
+        usage = \
+            'usage: %prog [options] <dbuser> <dbpwd> <mapprojectsfile>'
+        parser = OptionParser(usage)
+        parser.add_option('-u', '--dbuser', dest='dbuser',
+                          help='db user')
+        parser.add_option('-p', '--dbpwd', dest='dbpwd',
+                          help='db password')
+        parser.add_option('-m', '--mapprojectsfile',
+                          dest='mapprojectsfile',
+                          help='the path of mapping projects file')
+
+        (options, args) = parser.parse_args()
+        return (options.dbuser, options.dbpwd, options.mapprojectsfile)
+
+    def init_from_input(self):
+        (self.dbuser, self.dbpwd, self.mapprojectsfile) = \
+            Parameters.parse_input()
+
+
+class AccessLog:
+
+    def __init__(self, log_id, project_id):
+        self.log_id = log_id
+        self.project_id = project_id
+
+
+class Repository:
+
+    def __init__(self, repository_id, project_id):
+        self.repository_id = repository_id
+        self.project_id = project_id
+
+
+class ReplicationPolicy:
+
+    def __init__(self, replication_policy_id, project_id):
+        self.replication_policy_id = replication_policy_id
+        self.project_id = project_id
+
+
+class Project:
+
+    def __init__(
+        self,
+        project_id,
+        name,
+        index_id,
+        ):
+
+        self.project_id = project_id
+        self.project_name = name
+        self.index_id = index_id
+
+
+class HarborUtil:
+
+    def __init__(self, dbuser, dbpwd):
+        self.serverName = 'localhost'
+        self.user = dbuser
+        self.password = dbpwd
+        self.port = '3306'
+        self.subDB = 'registry'
+        self.db = None
+        self.cursor = None
+
+    def connect(self):
+        try:
+            self.db = MySQLdb.connect(host=self.serverName,
+                    user=self.user, passwd=self.password, db=self.subDB)
+            self.db.autocommit(False)
+            self.cursor = self.db.cursor()
+        except Exception, e:
+            raise Exception(e)
+
+    def close(self):
+        try:
+            self.db.commit()
+            self.cursor.close()
+            self.db.close()
+        except Exception, e:
+            print str(e)
+
+    def enable_foreign_key_check(self):
+        try:
+            self.cursor.execute('SET FOREIGN_KEY_CHECKS=1')
+        except Exception, e:
+            print str(e)
+
+    def disable_foreign_key_check(self):
+        try:
+            self.cursor.execute('SET FOREIGN_KEY_CHECKS=0')
+        except Exception, e:
+            print str(e)
+
+    def get_index_id(self, projects, project_id):
+        for project in projects:
+            if project.project_id == project_id:
+                return project.index_id
+        return ''
+
+    def update_access_log_table(self, projects):
+        access_logs = []
+        try:
+            query_sccess_log = \
+                'SELECT log_id, project_id from registry.access_log'
+            self.cursor.execute(query_sccess_log)
+            self.cursor.fetchall()
+            for result in self.cursor:
+                access_logs.append(AccessLog(result[0], result[1]))
+        except Exception, e:
+            raise Exception(e)
+
+        for item in access_logs:
+            index_id = self.get_index_id(projects, item.project_id)
+            if index_id != '':
+                try:
+                    update_access_log_project_id = \
+                        'UPDATE registry.access_log SET project_id=%s where log_id=%s' \
+                        % (index_id, item.log_id)
+                    self.cursor.execute(update_access_log_project_id)
+                except Exception, e:
+                    raise Exception(e)
+
+    def update_repository_table(self, projects):
+        repositories = []
+        try:
+            query_repository = \
+                'SELECT repository_id, project_id from registry.repository'
+            self.cursor.execute(query_repository)
+            self.cursor.fetchall()
+            for result in self.cursor:
+                repositories.append(Repository(result[0], result[1]))
+        except Exception, e:
+            raise Exception(e)
+
+        for item in repositories:
+            index_id = self.get_index_id(projects, item.project_id)
+            if index_id != '':
+                try:
+                    update_repository_project_id = \
+                        'UPDATE registry.repository SET project_id=%s where repository_id=%s' \
+                        % (index_id, item.repository_id)
+                    self.cursor.execute(update_repository_project_id)
+                except Exception, e:
+                    raise Exception(e)
+
+    def update_replication_policy_table(self, projects):
+        replication_policies = []
+        try:
+            query_replication_policy = \
+                'SELECT id, project_id from registry.replication_policy'
+            self.cursor.execute(query_replication_policy)
+            self.cursor.fetchall()
+            for result in self.cursor:
+                replication_policies.append(ReplicationPolicy(result[0],
+                                    result[1]))
+        except Exception, e:
+            raise Exception(e)
+
+        for item in replication_policies:
+            index_id = self.get_index_id(projects, item.project_id)
+            if index_id != '':
+                try:
+                    update_replication_policy_id = \
+                        'UPDATE registry.replication_policy SET project_id=%s where id=%s' \
+                        % (index_id, item.replication_policy_id)
+                    self.cursor.execute(update_replication_policy_id)
+                except Exception, e:
+                    raise Exception(e)
+
+
+def main():
+    commandline_input = Parameters()
+    harbor = HarborUtil(commandline_input.dbuser,
+                        commandline_input.dbpwd)
+
+    try:
+        harbor.connect()
+        harbor.disable_foreign_key_check()
+
+        with open(commandline_input.mapprojectsfile, 'r') as \
+            project_mapping_file:
+            project_mapping_data = json.load(project_mapping_file)
+
+        projects_mapping_list = []
+        for item in project_mapping_data['map_projects']:
+            projects_mapping_list.append(Project(item['project_id'],
+                    item['project_name'], item['index_id']))
+
+        harbor.update_access_log_table(projects_mapping_list)
+        harbor.update_repository_table(projects_mapping_list)
+        harbor.update_replication_policy_table(projects_mapping_list)
+    except Exception, e:
+        print e
+        sys.exit(1)
+    finally:
+        harbor.enable_foreign_key_check()
+        harbor.close()
+
+
+if __name__ == '__main__':
+    main()
+
+
+			

tools/migration/run.sh

@@ -110,6 +110,14 @@ export)
     echo $rc
     exit $rc
     ;;
+mapprojects)
+    echo "Performing map projects..."
+    ./mapprojects --dbuser ${DB_USR} --dbpwd ${DB_PWD} --mapprojectsfile ${MAPPROJECTFILE}
+    rc="$?"
+    echo "Map projects performed."
+    echo $rc
+    exit $rc
+    ;;
 restore)
     echo "Performing restore..."
     mysql $DBCNF < ./backup/registry.sql