From d95f8bd20487b69d5fddb7c8e6bf7e5978109f10 Mon Sep 17 00:00:00 2001 From: saga92 Date: Fri, 13 May 2016 17:53:50 +0800 Subject: [PATCH 1/5] fill sysadmin_flag&update_time&add test function --- migration/Dockerfile | 2 - migration/db_meta.py | 86 ++++++++++++++++++ migration/migration_harbor/versions/0_1_1.py | 94 +++++++------------- migration/run.sh | 15 +++- 4 files changed, 133 insertions(+), 64 deletions(-) create mode 100644 migration/db_meta.py diff --git a/migration/Dockerfile b/migration/Dockerfile index 507342170e..5579bd95fd 100644 --- a/migration/Dockerfile +++ b/migration/Dockerfile @@ -16,8 +16,6 @@ WORKDIR /harbor-migration COPY ./ ./ -COPY ./migration.cfg ./ - RUN ./prepare.sh ENTRYPOINT ["./run.sh"] diff --git a/migration/db_meta.py b/migration/db_meta.py new file mode 100644 index 0000000000..e20dd924cb --- /dev/null +++ b/migration/db_meta.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sqlalchemy as sa +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker, relationship + +Base = declarative_base() + +class User(Base): + __tablename__ = 'user' + + user_id = sa.Column(sa.Integer, primary_key=True) + username = sa.Column(sa.String(15), unique=True) + email = sa.Column(sa.String(30), unique=True) + password = sa.Column(sa.String(40), nullable=False) + realname = sa.Column(sa.String(20), nullable=False) + comment = sa.Column(sa.String(30)) + deleted = sa.Column(sa.Integer, nullable=False, server_default=sa.text("'0'")) + reset_uuid = sa.Column(sa.String(40)) + salt = sa.Column(sa.String(40)) + sysadmin_flag = sa.Column(sa.Integer) + creation_time = sa.Column(sa.DateTime) + update_time = sa.Column(sa.DateTime) + +class Properties(Base): + __tablename__ = 'properties' + + k = sa.Column(sa.String(64), primary_key = True) + v = sa.Column(sa.String(128), nullable = False) + +class ProjectMember(Base): + __tablename__ = 'project_member' + + project_id = sa.Column(sa.Integer(), primary_key = True) + user_id = sa.Column(sa.Integer(), primary_key = True) + role = sa.Column(sa.Integer(), nullable = False) + creation_time = sa.Column(sa.DateTime(), nullable = True) + update_time = sa.Column(sa.DateTime(), nullable = True) + sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'], ), + sa.ForeignKeyConstraint(['role'], [u'role.role_id'], ), + sa.ForeignKeyConstraint(['user_id'], [u'user.user_id'], ), + +class UserProjectRole(Base): + __tablename__ = 'user_project_role' + + upr_id = sa.Column(sa.Integer(), primary_key = True) + user_id = sa.Column(sa.Integer(), sa.ForeignKey('user.user_id')) + pr_id = sa.Column(sa.Integer(), sa.ForeignKey('project_role.pr_id')) + project_role = relationship("ProjectRole") + +class ProjectRole(Base): + __tablename__ = 'project_role' + + pr_id = sa.Column(sa.Integer(), primary_key = True) + project_id = sa.Column(sa.Integer(), nullable = False) + role_id = sa.Column(sa.Integer(), nullable = False) + sa.ForeignKeyConstraint(['role_id'], [u'role.role_id']) + sa.ForeignKeyConstraint(['project_id'], [u'project.project_id']) + +class Access(Base): + __tablename__ = 'access' + + access_id = sa.Column(sa.Integer(), primary_key = True) + access_code = sa.Column(sa.String(1)) + comment = sa.Column(sa.String(30)) + +class Role(Base): + __tablename__ = 'role' + + role_id = sa.Column(sa.Integer, primary_key=True) + role_mask = sa.Column(sa.Integer, nullable=False, server_default=sa.text("'0'")) + role_code = sa.Column(sa.String(20)) + name = sa.Column(sa.String(20)) + +class Project(Base): + __tablename__ = 'project' + + project_id = sa.Column(sa.Integer, primary_key=True) + owner_id = sa.Column(sa.ForeignKey(u'user.user_id'), nullable=False, index=True) + name = sa.Column(sa.String(30), nullable=False, unique=True) + creation_time = sa.Column(sa.DateTime) + update_time = sa.Column(sa.DateTime) + deleted = sa.Column(sa.Integer, nullable=False, server_default=sa.text("'0'")) + public = sa.Column(sa.Integer, nullable=False, server_default=sa.text("'0'")) + owner = relationship(u'User') diff --git a/migration/migration_harbor/versions/0_1_1.py b/migration/migration_harbor/versions/0_1_1.py index 0f21b5436b..746446f39f 100644 --- a/migration/migration_harbor/versions/0_1_1.py +++ b/migration/migration_harbor/versions/0_1_1.py @@ -27,58 +27,11 @@ branch_labels = None depends_on = None from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker, relationship from datetime import datetime +from db_meta import * Session = sessionmaker() -Base = declarative_base() - -class Properties(Base): - __tablename__ = 'properties' - - k = sa.Column(sa.String(64), primary_key = True) - v = sa.Column(sa.String(128), nullable = False) - -class ProjectMember(Base): - __tablename__ = 'project_member' - - project_id = sa.Column(sa.Integer(), primary_key = True) - user_id = sa.Column(sa.Integer(), primary_key = True) - role = sa.Column(sa.Integer(), nullable = False) - creation_time = sa.Column(sa.DateTime(), nullable = True) - update_time = sa.Column(sa.DateTime(), nullable = True) - sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'], ), - sa.ForeignKeyConstraint(['role'], [u'role.role_id'], ), - sa.ForeignKeyConstraint(['user_id'], [u'user.user_id'], ), - -class UserProjectRole(Base): - __tablename__ = 'user_project_role' - - upr_id = sa.Column(sa.Integer(), primary_key = True) - user_id = sa.Column(sa.Integer(), sa.ForeignKey('user.user_id')) - pr_id = sa.Column(sa.Integer(), sa.ForeignKey('project_role.pr_id')) - project_role = relationship("ProjectRole") - -class ProjectRole(Base): - __tablename__ = 'project_role' - - pr_id = sa.Column(sa.Integer(), primary_key = True) - project_id = sa.Column(sa.Integer(), nullable = False) - role_id = sa.Column(sa.Integer(), nullable = False) - sa.ForeignKeyConstraint(['role_id'], [u'role.role_id']) - sa.ForeignKeyConstraint(['project_id'], [u'project.project_id']) - -class Access(Base): - __tablename__ = 'access' - - access_id = sa.Column(sa.Integer(), primary_key = True) - access_code = sa.Column(sa.String(1)) - comment = sa.Column(sa.String(30)) - def upgrade(): """ update schema&data @@ -86,39 +39,60 @@ def upgrade(): bind = op.get_bind() session = Session(bind=bind) - #delete M from table access - acc = session.query(Access).filter_by(access_id=1).first() - session.delete(acc) - #create table property Properties.__table__.create(bind) session.add(Properties(k='schema_version', v='0.1.1')) + #add column to table user + op.add_column('user', sa.Column('creation_time', sa.DateTime(), nullable=True)) + op.add_column('user', sa.Column('sysadmin_flag', sa.Integer(), nullable=True)) + op.add_column('user', sa.Column('update_time', sa.DateTime(), nullable=True)) + + #fill update_time data into table user + session.query(User).update({User.update_time: datetime.now()}) + + #init all sysadmin_flag = 0 + session.query(User).update({User.sysadmin_flag: 0}) + #create table project_member ProjectMember.__table__.create(bind) - #fill data + #fill data into project_member and user join_result = session.query(UserProjectRole).join(UserProjectRole.project_role).all() for result in join_result: session.add(ProjectMember(project_id=result.project_role.project_id, \ user_id=result.user_id, role=result.project_role.role_id, \ creation_time=datetime.now(), update_time=datetime.now())) + #update sysadmin_flag + sys_admin_result = session.query(UserProjectRole).\ + join(UserProjectRole.project_role).filter(ProjectRole.role_id ==1).all() + for result in sys_admin_result: + session.query(User).filter(User.user_id == result.user_id).update({User.sysadmin_flag: 1}) + + #add column to table role + op.add_column('role', sa.Column('role_mask', sa.Integer(), server_default=sa.text(u"'0'"), nullable=False)) + #drop user_project_role table before drop project_role #because foreign key constraint op.drop_table('user_project_role') op.drop_table('project_role') + #delete sysadmin from table role + role = session.query(Role).filter_by(role_id=1).first() + session.delete(role) + session.query(Role).update({Role.role_id: Role.role_id - 1}) + + #delete M from table access + acc = session.query(Access).filter_by(access_id=1).first() + session.delete(acc) + session.query(Access).update({Access.access_id: Access.access_id - 1}) + #add column to table project op.add_column('project', sa.Column('update_time', sa.DateTime(), nullable=True)) - #add column to table role - op.add_column('role', sa.Column('role_mask', sa.Integer(), server_default=sa.text(u"'0'"), nullable=False)) - - #add column to table user - op.add_column('user', sa.Column('creation_time', sa.DateTime(), nullable=True)) - op.add_column('user', sa.Column('sysadmin_flag', sa.Integer(), nullable=True)) - op.add_column('user', sa.Column('update_time', sa.DateTime(), nullable=True)) + #fill update_time data into table project + session.query(Project).update({Project.update_time: datetime.now()}) session.commit() def downgrade(): diff --git a/migration/run.sh b/migration/run.sh index 806378ff55..d3a5a5b8e6 100755 --- a/migration/run.sh +++ b/migration/run.sh @@ -1,5 +1,7 @@ #!/bin/bash +export PYTHONPATH=$PYTHONPATH:/harbor-migration + source ./migration.cfg WAITTIME=60 @@ -14,6 +16,7 @@ if [[ $1 = "help" || $1 = "h" || $# = 0 ]]; then echo "backup perform database backup" echo "restore perform database restore" echo "up, upgrade perform database schema upgrade" + echo "test test database connection" echo "h, help usage help" exit 0 fi @@ -36,7 +39,7 @@ fi echo 'Trying to start mysql server...' DBRUN=0 -nohup mysqld 2>&1 > ./nohup.log& +nohup mysqld 2>&1 > ./mysqld.log& for i in $(seq 1 $WAITTIME); do echo "$(/usr/sbin/service mysql status)" if [[ "$(/usr/sbin/service mysql status)" =~ "not running" ]]; then @@ -47,11 +50,19 @@ for i in $(seq 1 $WAITTIME); do fi done -if [[ $DBRUN -eq 0 ]]; then +if [[ $DBRUN -eq 0 ]]; then echo "timeout. Can't run mysql server." + if [[ $1 = "test" ]]; then + echo "test failed." + fi exit 1 fi +if [[ $1 = "test" ]]; then + echo "test passed." + exit 0 +fi + key="$1" case $key in up|upgrade) From 9e0812a1e6fdaa2682f0e2aa452ef98c7d0dc940 Mon Sep 17 00:00:00 2001 From: saga92 Date: Fri, 13 May 2016 18:05:27 +0800 Subject: [PATCH 2/5] Update README.md --- migration/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/migration/README.md b/migration/README.md index 1d013e47f5..41923c3020 100644 --- a/migration/README.md +++ b/migration/README.md @@ -16,6 +16,10 @@ Migration is a module for migrating database schema between different version of - show instruction of harbor-migration ```docker run your-image-name help``` + +- test mysql connection in harbor-migration + + ```docker run -v /data/database:/var/lib/mysql your-image-name test``` - create backup file in `/path/to/backup` @@ -33,15 +37,12 @@ Migration is a module for migrating database schema between different version of ```docker run -ti -v /data/database:/var/lib/mysql your-image-name up head``` -- perform database schema downgrade(downgrade has been disabled) - - ```docker run -v /data/database:/var/lib/mysql your-image-name down base``` - +you can use `-v /etc/localtime:/etc/localtime` to sync container timezone with host timezone. ###migration step - step 1: stop and remove harbor service ``` - docker-compose stop && docker-compose rm -f + docker-compose down ``` - step 2: perform migration operation - step 3: rebuild newest harbor images and restart service From 266d584a1cc7b3c4a994f52e0146b0d7c8dd1094 Mon Sep 17 00:00:00 2001 From: saga92 Date: Fri, 13 May 2016 18:08:22 +0800 Subject: [PATCH 3/5] Update README.md --- migration/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/migration/README.md b/migration/README.md index 41923c3020..400dfb8db9 100644 --- a/migration/README.md +++ b/migration/README.md @@ -38,6 +38,7 @@ Migration is a module for migrating database schema between different version of ```docker run -ti -v /data/database:/var/lib/mysql your-image-name up head``` you can use `-v /etc/localtime:/etc/localtime` to sync container timezone with host timezone. +you may change `/data/database` to the mysql volumes path you set in docker-compose.yml. ###migration step - step 1: stop and remove harbor service From 8153211842eeb9138f699e8d185b4bb6ebc83ff9 Mon Sep 17 00:00:00 2001 From: saga92 Date: Fri, 13 May 2016 18:08:52 +0800 Subject: [PATCH 4/5] Update README.md --- migration/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/migration/README.md b/migration/README.md index 400dfb8db9..dde8e00c9c 100644 --- a/migration/README.md +++ b/migration/README.md @@ -38,6 +38,7 @@ Migration is a module for migrating database schema between different version of ```docker run -ti -v /data/database:/var/lib/mysql your-image-name up head``` you can use `-v /etc/localtime:/etc/localtime` to sync container timezone with host timezone. + you may change `/data/database` to the mysql volumes path you set in docker-compose.yml. ###migration step - step 1: stop and remove harbor service From 4641f2b89f7bcb1df219072ef294721857921a49 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Mon, 16 May 2016 14:39:57 +0800 Subject: [PATCH 5/5] add authtication to repository API --- api/repository.go | 47 +++++++++++------------------------------------ 1 file changed, 11 insertions(+), 36 deletions(-) diff --git a/api/repository.go b/api/repository.go index b0361af3a6..a827d58bc6 100644 --- a/api/repository.go +++ b/api/repository.go @@ -29,7 +29,6 @@ import ( svc_utils "github.com/vmware/harbor/service/utils" "github.com/vmware/harbor/utils/log" "github.com/vmware/harbor/utils/registry" - "github.com/vmware/harbor/utils/registry/auth" "github.com/vmware/harbor/utils/registry/errors" ) @@ -39,22 +38,12 @@ import ( // the security of registry type RepositoryAPI struct { BaseAPI - userID int - username string + userID int } // Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission. func (ra *RepositoryAPI) Prepare() { - userID, ok := ra.GetSession("userId").(int) - if !ok { - userID = dao.NonExistUserID - } - ra.userID = userID - - username, ok := ra.GetSession("username").(string) - if ok { - ra.username = username - } + ra.userID = ra.ValidateUser() } // Get ... @@ -250,29 +239,15 @@ func (ra *RepositoryAPI) GetManifests() { } func (ra *RepositoryAPI) initializeRepositoryClient(repoName string) (r *registry.Repository, err error) { + u := models.User{ + UserID: ra.userID, + } + user, err := dao.GetUser(u) + if err != nil { + return nil, err + } + endpoint := os.Getenv("REGISTRY_URL") - //no session, use basic auth - if ra.userID == dao.NonExistUserID { - username, password, _ := ra.Ctx.Request.BasicAuth() - credential := auth.NewBasicAuthCredential(username, password) - - return registry.NewRepositoryWithCredential(repoName, endpoint, credential) - - } - - //session exists, use username - if len(ra.username) == 0 { - u := models.User{ - UserID: ra.userID, - } - user, err := dao.GetUser(u) - if err != nil { - return nil, err - } - - ra.username = user.Username - } - - return registry.NewRepositoryWithUsername(repoName, endpoint, ra.username) + return registry.NewRepositoryWithUsername(repoName, endpoint, user.Username) }