upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor synced 2025-04-15 07:45:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

support pg upgrade (#14846)

Browse Source 
1, use the pg source and photon spec to build postgres 9.6
2, install 9.6 on the photon 4.0
3, then leverage pg_upgrade to handle the pg major version migration

Signed-off-by: Wang Yan <wangyan@vmware.com>
This commit is contained in:
Wang Yan 2021-05-20 16:25:50 +08:00 committed by GitHub
parent a50f782beb
commit 86185989cf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 421 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Makefile
View File

@ -429,9 +429,12 @@ build_base_docker:
@for name in $(BUILDBASETARGET); do \
echo $$name ; \
sleep 30 ; \
if [ $$name == "db" ] ; then \
cd $(MAKEFILEPATH_PHOTON)/$$name && $(MAKEFILEPATH_PHOTON)/$$name/rpm_builder.sh && cd - ; \
fi; \
$(DOCKERBUILD) --pull --no-cache -f $(MAKEFILEPATH_PHOTON)/$$name/Dockerfile.base -t $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) --label base-build-date=$(date +"%Y%m%d") . && \
if [ -n "$(PUSHBASEIMAGE)" ] ; then \
$(PUSHSCRIPTPATH)/$(PUSHSCRIPTNAME) $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) $(REGISTRYUSER) $(REGISTRYPASSWORD) || exit 1; \
$(PUSHSCRIPTPATH)/$(PUSHSCRIPTNAME) $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) $(REGISTRYUSER) $(REGISTRYPASSWORD) || exit 1; \
fi ; \
done

4
make/photon/db/Dockerfile
View File

@ -5,6 +5,8 @@ FROM ${harbor_base_namespace}/harbor-db-base:${harbor_base_image_version}
VOLUME /var/lib/postgresql/data
COPY ./make/photon/db/docker-entrypoint.sh /docker-entrypoint.sh
COPY ./make/photon/db/initdb.sh /initdb.sh
COPY ./make/photon/db/upgrade.sh /upgrade.sh
COPY ./make/photon/db/docker-healthcheck.sh /docker-healthcheck.sh
COPY ./make/photon/db/initial-notaryserver.sql /docker-entrypoint-initdb.d/
COPY ./make/photon/db/initial-notarysigner.sql /docker-entrypoint-initdb.d/
@ -12,7 +14,7 @@ COPY ./make/photon/db/initial-registry.sql /docker-entrypoint-initdb.d/
RUN chown -R postgres:postgres /docker-entrypoint.sh /docker-healthcheck.sh /docker-entrypoint-initdb.d \
&& chmod u+x /docker-entrypoint.sh /docker-healthcheck.sh
ENTRYPOINT ["/docker-entrypoint.sh"]
ENTRYPOINT ["/docker-entrypoint.sh", "96", "13"]
HEALTHCHECK CMD ["/docker-healthcheck.sh"]
USER postgres

7
make/photon/db/Dockerfile.base
View File

@ -2,7 +2,12 @@ FROM photon:4.0
ENV PGDATA /var/lib/postgresql/data
RUN tdnf install -y shadow gzip postgresql >> /dev/null\
COPY ./make/photon/db/postgresql96-libs-9.6.21-1.ph4.x86_64.rpm /pg96/
COPY ./make/photon/db/postgresql96-9.6.21-1.ph4.x86_64.rpm /pg96/
RUN tdnf install -y /pg96/postgresql96-libs-9.6.21-1.ph4.x86_64.rpm /pg96/postgresql96-9.6.21-1.ph4.x86_64.rpm >> /dev/null \
&& rm -rf /pg96 \
&& tdnf install -y shadow gzip postgresql findutils bc >> /dev/null \
&& groupadd -r postgres --gid=999 \
&& useradd -m -r -g postgres --uid=999 postgres \
&& mkdir -p /docker-entrypoint-initdb.d \

141
make/photon/db/docker-entrypoint.sh
View File

@ -1,114 +1,48 @@
#!/bin/bash
set -e
# usage: file_env VAR [DEFAULT]
# ie: file_env 'XYZ_DB_PASSWORD' 'example'
# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
file_env() {
local var="$1"
local fileVar="${var}_FILE"
local def="${2:-}"
if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
exit 1
fi
local val="$def"
if [ "${!var:-}" ]; then
val="${!var}"
elif [ "${!fileVar:-}" ]; then
val="$(< "${!fileVar}")"
fi
export "$var"="$val"
unset "$fileVar"
}
source $PWD/initdb.sh
# look specifically for PG_VERSION, as it is expected in the DB dir
if [ ! -s "$PGDATA/PG_VERSION" ]; then
file_env 'POSTGRES_INITDB_ARGS'
if [ "$POSTGRES_INITDB_XLOGDIR" ]; then
export POSTGRES_INITDB_ARGS="$POSTGRES_INITDB_ARGS --xlogdir $POSTGRES_INITDB_XLOGDIR"
fi
initdb -D $PGDATA -U postgres -E UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8 $POSTGRES_INITDB_ARGS
# check password first so we can output the warning before postgres
# messes it up
file_env 'POSTGRES_PASSWORD'
if [ "$POSTGRES_PASSWORD" ]; then
pass="PASSWORD '$POSTGRES_PASSWORD'"
authMethod=md5
CUR=$PWD
PG_VERSION_OLD=$1
PG_VERSION_NEW=$2
PGBINOLD="/usr/local/pg${PG_VERSION_OLD}/bin"
PGDATAOLD=${PGDATA}/pg${PG_VERSION_OLD}
PGDATANEW=${PGDATA}/pg${PG_VERSION_NEW}
# to handle the PG 9.6 only
if [ -s $PGDATA/PG_VERSION ]; then
PGDATAOLD=$PGDATA
fi
#
# Init DB: $PGDATA is empty.
# Upgrade DB: 1, has $PGDATA\PG_VERSION. 2, has pg old version directory with PG_VERSION inside.
#
if [ "$(ls -A $PGDATA)" ]; then
if [ ! -d $PGDATANEW ]; then
if [ ! -d $PGDATAOLD ] || [ ! -s $PGDATAOLD/PG_VERSION ]; then
echo "incorrect data: $PGDATAOLD, make sure $PGDATAOLD is not empty and with PG_VERSION inside."
exit 1
fi
initPG $PGDATANEW false
# because of the 'set -e', the upgrade failure will break execution.
./$CUR/upgrade.sh --old-bindir $PGBINOLD --old-datadir $PGDATAOLD --new-datadir $PGDATANEW
echo "remove the $PGDATAOLD after upgrade success."
if [ "$PGDATAOLD" = "$PGDATA" ]; then
find $PGDATA/* -prune ! -name pg${PG_VERSION_NEW} -exec rm -rf {} \;
else
rm -rf $PGDATAOLD
fi
else
# The - option suppresses leading tabs but *not* spaces. :)
cat >&2 <<-EOF
****************************************************
WARNING: No password has been set for the database.
This will allow anyone with access to the
Postgres port to access your database. In
Docker's default configuration, this is
effectively any other container on the same
system.
Use "-e POSTGRES_PASSWORD=password" to set
it in "docker run".
****************************************************
EOF
pass=
authMethod=trust
echo "no need to upgrade postgres, launch it."
fi
{
echo
echo "host all all all $authMethod"
} >> "$PGDATA/pg_hba.conf"
echo `whoami`
# internal start of server in order to allow set-up using psql-client
# does not listen on external TCP/IP and waits until start finishes
pg_ctl -D "$PGDATA" -o "-c listen_addresses=''" -w start
file_env 'POSTGRES_USER' 'postgres'
file_env 'POSTGRES_DB' "$POSTGRES_USER"
psql=( psql -v ON_ERROR_STOP=1 )
if [ "$POSTGRES_DB" != 'postgres' ]; then
"${psql[@]}" --username postgres <<-EOSQL
CREATE DATABASE "$POSTGRES_DB" ;
EOSQL
echo
fi
if [ "$POSTGRES_USER" = 'postgres' ]; then
op='ALTER'
else
op='CREATE'
fi
"${psql[@]}" --username postgres <<-EOSQL
$op USER "$POSTGRES_USER" WITH SUPERUSER $pass ;
EOSQL
echo
psql+=( --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" )
echo
for f in /docker-entrypoint-initdb.d/*; do
case "$f" in
*.sh) echo "$0: running $f"; . "$f" ;;
*.sql) echo "$0: running $f"; "${psql[@]}" -f "$f"; echo ;;
*.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${psql[@]}"; echo ;;
*) echo "$0: ignoring $f" ;;
esac
echo
done
PGUSER="${PGUSER:-postgres}" \
pg_ctl -D "$PGDATA" -m fast -w stop
echo
echo 'PostgreSQL init process complete; ready for start up.'
echo
else
initPG $PGDATANEW true
fi
POSTGRES_PARAMETER=''
file_env 'POSTGRES_MAX_CONNECTIONS' '1024'
# The max value of 'max_connections' is 262143
if [ $POSTGRES_MAX_CONNECTIONS -le 0 ] || [ $POSTGRES_MAX_CONNECTIONS -gt 262143 ]; then
@ -116,5 +50,4 @@ if [ $POSTGRES_MAX_CONNECTIONS -le 0 ] || [ $POSTGRES_MAX_CONNECTIONS -gt 262143
fi
POSTGRES_PARAMETER="${POSTGRES_PARAMETER} -c max_connections=${POSTGRES_MAX_CONNECTIONS}"
exec postgres -D $PGDATA $POSTGRES_PARAMETER
exec postgres -D $PGDATANEW $POSTGRES_PARAMETER

114
make/photon/db/initdb.sh Executable file
View File

@ -0,0 +1,114 @@
#!/bin/bash
set -e
# usage: file_env VAR [DEFAULT]
# ie: file_env 'XYZ_DB_PASSWORD' 'example'
# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
function file_env() {
local var="$1"
local fileVar="${var}_FILE"
local def="${2:-}"
if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
exit 1
fi
local val="$def"
if [ "${!var:-}" ]; then
val="${!var}"
elif [ "${!fileVar:-}" ]; then
val="$(< "${!fileVar}")"
fi
export "$var"="$val"
unset "$fileVar"
}
# usage: initPG $Dir $initSql
# Use $Dir to index where to init the postgres db
# Use $initSql to indicate whether to execute the sql under docker-entrypoint-initdb.d, default is not.
function initPG() {
file_env 'POSTGRES_INITDB_ARGS'
if [ "$POSTGRES_INITDB_XLOGDIR" ]; then
export POSTGRES_INITDB_ARGS="$POSTGRES_INITDB_ARGS --xlogdir $POSTGRES_INITDB_XLOGDIR"
fi
initdb -D $1 -U postgres -E UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8 $POSTGRES_INITDB_ARGS
# check password first so we can output the warning before postgres
# messes it up
file_env 'POSTGRES_PASSWORD'
if [ "$POSTGRES_PASSWORD" ]; then
pass="PASSWORD '$POSTGRES_PASSWORD'"
authMethod=md5
else
# The - option suppresses leading tabs but *not* spaces. :)
cat >&2 <<-EOF
****************************************************
WARNING: No password has been set for the database.
This will allow anyone with access to the
Postgres port to access your database. In
Docker's default configuration, this is
effectively any other container on the same
system.
Use "-e POSTGRES_PASSWORD=password" to set
it in "docker run".
****************************************************
EOF
pass=
authMethod=trust
fi
{
echo
echo "host all all all $authMethod"
} >> "$1/pg_hba.conf"
echo `whoami`
# internal start of server in order to allow set-up using psql-client
# does not listen on external TCP/IP and waits until start finishes
pg_ctl -D "$1" -o "-c listen_addresses=''" -w start
file_env 'POSTGRES_USER' 'postgres'
file_env 'POSTGRES_DB' "$POSTGRES_USER"
psql=( psql -v ON_ERROR_STOP=1 )
if [ "$POSTGRES_DB" != 'postgres' ]; then
"${psql[@]}" --username postgres <<-EOSQL
CREATE DATABASE "$POSTGRES_DB" ;
EOSQL
echo
fi
if [ "$POSTGRES_USER" = 'postgres' ]; then
op='ALTER'
else
op='CREATE'
fi
"${psql[@]}" --username postgres <<-EOSQL
$op USER "$POSTGRES_USER" WITH SUPERUSER $pass ;
EOSQL
echo
psql+=( --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" )
echo
if [ $2 == "true" ]; then
for f in /docker-entrypoint-initdb.d/*; do
case "$f" in
*.sh) echo "$0: running $f"; . "$f" ;;
*.sql) echo "$0: running $f"; "${psql[@]}" -f "$f"; echo ;;
*.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${psql[@]}"; echo ;;
*) echo "$0: ignoring $f" ;;
esac
echo
done
fi
PGUSER="${PGUSER:-postgres}" \
pg_ctl -D "$1" -m fast -w stop
echo
echo 'PostgreSQL init process complete; ready for start up.'
echo
}

168
make/photon/db/postgres.spec Normal file
View File

@ -0,0 +1,168 @@
Summary: PostgreSQL database engine
Name: postgresql96
Version: 9.6.21
Release: 1%{?dist}
License: PostgreSQL
URL: www.postgresql.org
Group: Applications/Databases
Vendor: VMware, Inc.
Distribution: Photon
Source0: http://ftp.postgresql.org/pub/source/v%{version}/%{name}-%{version}.tar.bz2
%define sha1 postgresql=e24333824d361968958613f546ae06011d9d1dfc
# Customized location of pg96
%global pgbaseinstdir /usr/local/pg96
# Common libraries needed
BuildRequires: krb5-devel
BuildRequires: libxml2-devel
BuildRequires: openldap
BuildRequires: perl
BuildRequires: readline-devel
BuildRequires: openssl-devel
BuildRequires: zlib-devel
BuildRequires: tzdata
BuildRequires: bzip2
BuildRequires: sudo
Requires: krb5
Requires: libxml2
Requires: openldap
Requires: openssl
Requires: readline
Requires: zlib
Requires: tzdata
Requires: bzip2
Requires: sudo
Requires: %{name}-libs = %{version}-%{release}
%description
PostgreSQL is an object-relational database management system.
%package libs
Summary: Libraries for use with PostgreSQL
Group: Applications/Databases
%description libs
The postgresql-libs package provides the essential shared libraries for any
PostgreSQL client program or interface. You will need to install this package
to use any other PostgreSQL package or any clients that need to connect to a
PostgreSQL server.
%package devel
Summary: Development files for postgresql.
Group: Development/Libraries
Requires: postgresql = %{version}-%{release}
%description devel
The postgresql-devel package contains libraries and header files for
developing applications that use postgresql.
%prep
%setup -q
%build
ls -la
sed -i '/DEFAULT_PGSOCKET_DIR/s@/tmp@/run/postgresql@' src/include/pg_config_manual.h &&
./configure \
--prefix=%{pgbaseinstdir} \
--with-includes=%{pgbaseinstdir}/include \
--with-libraries=%{pgbaseinstdir}/lib \
--datarootdir=%{pgbaseinstdir}/share \
--enable-thread-safety \
--with-ldap \
--with-libxml \
--with-openssl \
--with-gssapi \
--with-readline \
--with-system-tzdata=%{_datadir}/zoneinfo \
--docdir=%{pgbaseinstdir}/doc/postgresql
make %{?_smp_mflags}
cd contrib && make %{?_smp_mflags}
%install
[ %{buildroot} != "/"] && rm -rf %{buildroot}/*
make install DESTDIR=%{buildroot}
cd contrib && make install DESTDIR=%{buildroot}
%{_fixperms} %{buildroot}/*
%check
sed -i '2219s/",/ ; EXIT_STATUS=$? ; sleep 5 ; exit $EXIT_STATUS",/g' src/test/regress/pg_regress.c
chown -Rv nobody .
sudo -u nobody -s /bin/bash -c "PATH=$PATH make -k check"
%post -p /sbin/ldconfig
%postun -p /sbin/ldconfig
%clean
rm -rf %{buildroot}/*
%files
%defattr(-,root,root)
%{pgbaseinstdir}/bin/initdb
%{pgbaseinstdir}/bin/oid2name
%{pgbaseinstdir}/bin/pg_archivecleanup
%{pgbaseinstdir}/bin/pg_basebackup
%{pgbaseinstdir}/bin/pg_controldata
%{pgbaseinstdir}/bin/pg_ctl
%{pgbaseinstdir}/bin/pg_receivexlog
%{pgbaseinstdir}/bin/pg_recvlogical
%{pgbaseinstdir}/bin/pg_resetxlog
%{pgbaseinstdir}/bin/pg_rewind
%{pgbaseinstdir}/bin/pg_standby
%{pgbaseinstdir}/bin/pg_test_fsync
%{pgbaseinstdir}/bin/pg_test_timing
%{pgbaseinstdir}/bin/pg_upgrade
%{pgbaseinstdir}/bin/pg_xlogdump
%{pgbaseinstdir}/bin/pgbench
%{pgbaseinstdir}/bin/postgres
%{pgbaseinstdir}/bin/postmaster
%{pgbaseinstdir}/bin/vacuumlo
%{pgbaseinstdir}/share/postgresql/*
%{pgbaseinstdir}/lib/postgresql/*
%{pgbaseinstdir}/doc/postgresql/extension/*.example
%exclude %{pgbaseinstdir}/share/postgresql/pg_service.conf.sample
%exclude %{pgbaseinstdir}/share/postgresql/psqlrc.sample
%files libs
%{pgbaseinstdir}/bin/clusterdb
%{pgbaseinstdir}/bin/createdb
%{pgbaseinstdir}/bin/createlang
%{pgbaseinstdir}/bin/createuser
%{pgbaseinstdir}/bin/dropdb
%{pgbaseinstdir}/bin/droplang
%{pgbaseinstdir}/bin/dropuser
%{pgbaseinstdir}/bin/ecpg
%{pgbaseinstdir}/bin/pg_config
%{pgbaseinstdir}/bin/pg_dump
%{pgbaseinstdir}/bin/pg_dumpall
%{pgbaseinstdir}/bin/pg_isready
%{pgbaseinstdir}/bin/pg_restore
%{pgbaseinstdir}/bin/psql
%{pgbaseinstdir}/bin/reindexdb
%{pgbaseinstdir}/bin/vacuumdb
%{pgbaseinstdir}/lib/libecpg*.so.*
%{pgbaseinstdir}/lib/libpgtypes*.so.*
%{pgbaseinstdir}/lib/libpq*.so.*
%{pgbaseinstdir}/share/postgresql/pg_service.conf.sample
%{pgbaseinstdir}/share/postgresql/psqlrc.sample
%files devel
%defattr(-,root,root)
%{pgbaseinstdir}/include/*
%{pgbaseinstdir}/lib/pkgconfig/*
%{pgbaseinstdir}/lib/libecpg*.so
%{pgbaseinstdir}/lib/libpgtypes*.so
%{pgbaseinstdir}/lib/libpq*.so
%{pgbaseinstdir}/lib/libpgcommon.a
%{pgbaseinstdir}/lib/libpgfeutils.a
%{pgbaseinstdir}/lib/libpgport.a
%{pgbaseinstdir}/lib/libpq.a
%{pgbaseinstdir}/lib/libecpg.a
%{pgbaseinstdir}/lib/libecpg_compat.a
%{pgbaseinstdir}/lib/libpgtypes.a
%changelog
* Yan Wang <wangyan@vmware.com>
- Customize postgres 96 from original spec

43
make/photon/db/rpm_builder.sh Executable file
View File

@ -0,0 +1,43 @@
#!/bin/bash
set -e
name='postgresql'
version='9.6.21'
function checkdep {
if ! wget --version &> /dev/null
then
echo "Need to install wget first and run this script again."
exit 1
fi
if ! bzip2 --version &> /dev/null
then
echo "Need to install bzip2 first and run this script again."
exit 1
fi
}
checkdep
cur=$PWD
workDir=`mktemp -d ${TMPDIR-/tmp}/$name.XXXXXX`
mkdir -p $workDir && cd $workDir
# step 1: get source code of pg 9.6, and rename the code directory from postgres to postgres96
wget http://ftp.postgresql.org/pub/source/v$version/$name-$version.tar.bz2
bzip2 -d ./$name-$version.tar.bz2 && tar -xvf ./$name-$version.tar
mkdir -p ${name}96-$version && cp -r ./$name-$version/* ./${name}96-$version/ && rm -rf ./$name-$version
tar -cvjSf ${name}96-$version.tar.bz2 ${name}96-$version
# step 2: get spec builder script, and replace version to 4, then to build the pg96 rpm packages
wget https://raw.githubusercontent.com/vmware/photon/4.0/tools/scripts/build_spec.sh
sed "s|VERSION=3|VERSION=4|g" -i build_spec.sh
chmod 655 ./build_spec.sh && cp $cur/postgres.spec .
./build_spec.sh ./postgres.spec
cp ./stage/RPMS/x86_64/${name}96-libs-$version-1.ph4.x86_64.rpm $cur
cp ./stage/RPMS/x86_64/${name}96-$version-1.ph4.x86_64.rpm $cur
# clean
cd $cur && rm -rf $workDir

46
make/photon/db/upgrade.sh Executable file
View File

@ -0,0 +1,46 @@
#!/bin/bash
PGBINOLD="/usr/local/pg96/bin/"
PGBINNEW="/usr/bin"
PGDATAOLD=""
PGDATANEW=""
while [[ "$#" -gt 0 ]]; do
case $1 in
-b|--old-datadir) PGDATAOLD="$2"; shift ;;
-B|--new-datadir) PGDATANEW="$2"; shift ;;
-d|--old-bindir) PGBINOLD="$2"; shift ;;
-D|--new-bindir) PGBINNEW="$2"; shift ;;
*) echo "Unknown parameter passed: $1"; exit 1 ;;
esac
shift
done
if [ "$PGDATAOLD" = "" ] || [ "$PGDATANEW" = "" ]; then
echo "required parameter is missing: $PGDATAOLD, $PGDATANEW"
exit 1
fi
export PGDATAOLD=$PGDATAOLD
export PGDATANEW=$PGDATANEW
export PGBINNEW=$PGBINNEW
export PGBINOLD=$PGBINOLD
echo 'start to upgrade.'
cd /tmp
${PGBINNEW}/pg_upgrade \
--old-datadir=$PGDATAOLD \
--new-datadir=$PGDATANEW \
--old-bindir=$PGBINOLD \
--new-bindir=$PGBINNEW \
--old-options '-c config_file=$PGDATAOLD/postgresql.conf' \
--new-options '-c config_file=$PGDATANEW/postgresql.conf'
if [ $? -ne 0 ]; then
echo 'fail to upgrade.'
cat /tmp/pg_upgrade_internal.log
exit 1
fi
cp $PGDATAOLD/pg_hba.conf $PGDATANEW/pg_hba.conf
echo 'success to upgrade.'