Merge pull request #224 from saga92/master

fill sysadmin_flag&update_time&add test function

migration/Dockerfile

@@ -16,8 +16,6 @@ WORKDIR /harbor-migration
 
 COPY ./ ./
 
-COPY ./migration.cfg ./
-
 RUN ./prepare.sh
 
 ENTRYPOINT ["./run.sh"] 

migration/README.md

@@ -16,6 +16,10 @@ Migration is a module for migrating database schema between different version of
 - show instruction of harbor-migration
 
     ```docker run your-image-name help```
+    
+- test mysql connection in harbor-migration
+
+    ```docker run -v /data/database:/var/lib/mysql your-image-name test```
 
 - create backup file in `/path/to/backup`
 
@@ -33,15 +37,14 @@ Migration is a module for migrating database schema between different version of
 
     ```docker run -ti -v /data/database:/var/lib/mysql your-image-name up head```
 
-- perform database schema downgrade(downgrade has been disabled)
+you can use `-v /etc/localtime:/etc/localtime` to sync container timezone with host timezone.
-
-    ```docker run -v /data/database:/var/lib/mysql your-image-name down base```
 
+you may change `/data/database` to the mysql volumes path you set in docker-compose.yml.
 ###migration step
 - step 1: stop and remove harbor service
 
     ``` 
-    docker-compose stop && docker-compose rm -f
+    docker-compose down
     ```
 - step 2: perform migration operation
 - step 3: rebuild newest harbor images and restart service

migration/db_meta.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sqlalchemy as sa
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker, relationship
+
+Base = declarative_base()
+
+class User(Base):
+    __tablename__ = 'user'
+
+    user_id = sa.Column(sa.Integer, primary_key=True)
+    username = sa.Column(sa.String(15), unique=True)
+    email = sa.Column(sa.String(30), unique=True)
+    password = sa.Column(sa.String(40), nullable=False)
+    realname = sa.Column(sa.String(20), nullable=False)
+    comment = sa.Column(sa.String(30))
+    deleted = sa.Column(sa.Integer, nullable=False, server_default=sa.text("'0'"))
+    reset_uuid = sa.Column(sa.String(40))
+    salt = sa.Column(sa.String(40))
+    sysadmin_flag = sa.Column(sa.Integer)
+    creation_time = sa.Column(sa.DateTime)
+    update_time = sa.Column(sa.DateTime)
+
+class Properties(Base):
+    __tablename__ = 'properties'
+
+    k = sa.Column(sa.String(64), primary_key = True)
+    v = sa.Column(sa.String(128), nullable = False)
+
+class ProjectMember(Base):
+    __tablename__ = 'project_member'
+
+    project_id = sa.Column(sa.Integer(), primary_key = True)
+    user_id = sa.Column(sa.Integer(), primary_key = True)
+    role = sa.Column(sa.Integer(), nullable = False)
+    creation_time = sa.Column(sa.DateTime(), nullable = True)
+    update_time = sa.Column(sa.DateTime(), nullable = True)
+    sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'], ),
+    sa.ForeignKeyConstraint(['role'], [u'role.role_id'], ),
+    sa.ForeignKeyConstraint(['user_id'], [u'user.user_id'], ),
+
+class UserProjectRole(Base):
+    __tablename__ = 'user_project_role'
+
+    upr_id = sa.Column(sa.Integer(), primary_key = True)
+    user_id = sa.Column(sa.Integer(), sa.ForeignKey('user.user_id'))
+    pr_id = sa.Column(sa.Integer(), sa.ForeignKey('project_role.pr_id'))
+    project_role = relationship("ProjectRole")
+
+class ProjectRole(Base):
+    __tablename__ = 'project_role'
+
+    pr_id = sa.Column(sa.Integer(), primary_key = True)
+    project_id = sa.Column(sa.Integer(), nullable = False)
+    role_id = sa.Column(sa.Integer(), nullable = False)
+    sa.ForeignKeyConstraint(['role_id'], [u'role.role_id'])
+    sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'])
+
+class Access(Base):
+    __tablename__ = 'access'
+
+    access_id = sa.Column(sa.Integer(), primary_key = True)
+    access_code = sa.Column(sa.String(1))
+    comment = sa.Column(sa.String(30))
+
+class Role(Base):
+    __tablename__ = 'role'
+
+    role_id = sa.Column(sa.Integer, primary_key=True)
+    role_mask = sa.Column(sa.Integer, nullable=False, server_default=sa.text("'0'"))
+    role_code = sa.Column(sa.String(20))
+    name = sa.Column(sa.String(20))
+
+class Project(Base):
+    __tablename__ = 'project'
+
+    project_id = sa.Column(sa.Integer, primary_key=True)
+    owner_id = sa.Column(sa.ForeignKey(u'user.user_id'), nullable=False, index=True)
+    name = sa.Column(sa.String(30), nullable=False, unique=True)
+    creation_time = sa.Column(sa.DateTime)
+    update_time = sa.Column(sa.DateTime)
+    deleted = sa.Column(sa.Integer, nullable=False, server_default=sa.text("'0'"))
+    public = sa.Column(sa.Integer, nullable=False, server_default=sa.text("'0'"))
+    owner = relationship(u'User')

migration/migration_harbor/versions/0_1_1.py

@@ -27,58 +27,11 @@ branch_labels = None
 depends_on = None
 
 from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker, relationship
 from datetime import datetime
+from db_meta import *
 
 Session = sessionmaker()
 
-Base = declarative_base()
-
-class Properties(Base):
-    __tablename__ = 'properties'
-
-    k = sa.Column(sa.String(64), primary_key = True)
-    v = sa.Column(sa.String(128), nullable = False)
-
-class ProjectMember(Base):
-    __tablename__ = 'project_member'
-
-    project_id = sa.Column(sa.Integer(), primary_key = True)
-    user_id = sa.Column(sa.Integer(), primary_key = True)
-    role = sa.Column(sa.Integer(), nullable = False)
-    creation_time = sa.Column(sa.DateTime(), nullable = True)
-    update_time = sa.Column(sa.DateTime(), nullable = True)
-    sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'], ),
-    sa.ForeignKeyConstraint(['role'], [u'role.role_id'], ),
-    sa.ForeignKeyConstraint(['user_id'], [u'user.user_id'], ),
-
-class UserProjectRole(Base):
-    __tablename__ = 'user_project_role'
-
-    upr_id = sa.Column(sa.Integer(), primary_key = True)
-    user_id = sa.Column(sa.Integer(), sa.ForeignKey('user.user_id'))
-    pr_id = sa.Column(sa.Integer(), sa.ForeignKey('project_role.pr_id'))
-    project_role = relationship("ProjectRole")
-
-class ProjectRole(Base):
-    __tablename__ = 'project_role'
-
-    pr_id = sa.Column(sa.Integer(), primary_key = True)
-    project_id = sa.Column(sa.Integer(), nullable = False)
-    role_id = sa.Column(sa.Integer(), nullable = False)
-    sa.ForeignKeyConstraint(['role_id'], [u'role.role_id'])
-    sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'])
-
-class Access(Base):
-    __tablename__ = 'access'
-
-    access_id = sa.Column(sa.Integer(), primary_key = True)
-    access_code = sa.Column(sa.String(1))
-    comment = sa.Column(sa.String(30))
-
 def upgrade():
     """
     update schema&data
@@ -86,39 +39,60 @@ def upgrade():
     bind = op.get_bind()
     session = Session(bind=bind)
 
-    #delete M from table access
-    acc = session.query(Access).filter_by(access_id=1).first()
-    session.delete(acc)
-
     #create table property
     Properties.__table__.create(bind)
     session.add(Properties(k='schema_version', v='0.1.1'))
 
+    #add column to table user
+    op.add_column('user', sa.Column('creation_time', sa.DateTime(), nullable=True))
+    op.add_column('user', sa.Column('sysadmin_flag', sa.Integer(), nullable=True))
+    op.add_column('user', sa.Column('update_time', sa.DateTime(), nullable=True))
+
+    #fill update_time data into table user
+    session.query(User).update({User.update_time: datetime.now()})
+
+    #init all sysadmin_flag = 0
+    session.query(User).update({User.sysadmin_flag: 0})
+
     #create table project_member
     ProjectMember.__table__.create(bind)
 
-    #fill data
+    #fill data into project_member and user
     join_result = session.query(UserProjectRole).join(UserProjectRole.project_role).all()
     for result in join_result:
         session.add(ProjectMember(project_id=result.project_role.project_id, \
             user_id=result.user_id, role=result.project_role.role_id, \
             creation_time=datetime.now(), update_time=datetime.now()))
 
+    #update sysadmin_flag
+    sys_admin_result = session.query(UserProjectRole).\
+        join(UserProjectRole.project_role).filter(ProjectRole.role_id ==1).all()
+    for result in sys_admin_result:
+        session.query(User).filter(User.user_id == result.user_id).update({User.sysadmin_flag: 1})
+
+    #add column to table role
+    op.add_column('role', sa.Column('role_mask', sa.Integer(), server_default=sa.text(u"'0'"), nullable=False))
+
     #drop user_project_role table before drop project_role
     #because foreign key constraint
     op.drop_table('user_project_role')
     op.drop_table('project_role')
 
+    #delete sysadmin from table role
+    role = session.query(Role).filter_by(role_id=1).first()
+    session.delete(role)
+    session.query(Role).update({Role.role_id: Role.role_id - 1})
+
+    #delete M from table access
+    acc = session.query(Access).filter_by(access_id=1).first()
+    session.delete(acc)
+    session.query(Access).update({Access.access_id: Access.access_id - 1})
+
     #add column to table project
     op.add_column('project', sa.Column('update_time', sa.DateTime(), nullable=True))
 
-    #add column to table role
+    #fill update_time data into table project
-    op.add_column('role', sa.Column('role_mask', sa.Integer(), server_default=sa.text(u"'0'"), nullable=False))
+    session.query(Project).update({Project.update_time: datetime.now()})
-
-    #add column to table user
-    op.add_column('user', sa.Column('creation_time', sa.DateTime(), nullable=True))
-    op.add_column('user', sa.Column('sysadmin_flag', sa.Integer(), nullable=True))
-    op.add_column('user', sa.Column('update_time', sa.DateTime(), nullable=True))
     session.commit()
 
 def downgrade():

migration/run.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+export PYTHONPATH=$PYTHONPATH:/harbor-migration
+
 source ./migration.cfg
 
 WAITTIME=60
@@ -14,6 +16,7 @@ if [[ $1 = "help" || $1 = "h" || $# = 0 ]]; then
     echo "backup                perform database backup"
     echo "restore               perform database restore"
     echo "up,   upgrade         perform database schema upgrade"
+    echo "test                  test database connection"
     echo "h,    help            usage help"
     exit 0
 fi
@@ -36,7 +39,7 @@ fi
 
 echo 'Trying to start mysql server...'
 DBRUN=0
-nohup mysqld 2>&1 > ./nohup.log&
+nohup mysqld 2>&1 > ./mysqld.log&
 for i in $(seq 1 $WAITTIME); do
     echo "$(/usr/sbin/service mysql status)"
     if [[ "$(/usr/sbin/service mysql status)" =~ "not running" ]]; then
@@ -47,11 +50,19 @@ for i in $(seq 1 $WAITTIME); do
     fi
 done
 
-if [[ $DBRUN -eq 0 ]]; then
+if [[ $DBRUN -eq 0  ]]; then
     echo "timeout. Can't run mysql server."
+    if [[ $1 = "test" ]]; then
+        echo "test failed."
+    fi
     exit 1
 fi
 
+if [[ $1 = "test" ]]; then
+    echo "test passed."
+    exit 0
+fi
+
 key="$1"
 case $key in
 up|upgrade)