added master/slave modes; added migrate.py

Dockerfile

@@ -1,19 +1,32 @@
 FROM robinthoni/postgres-multiarch:9.6
 
+RUN mv -v /usr/share/postgresql/$PG_MAJOR/pg_hba.conf.sample /usr/share/postgresql/ \
+              && ln -sv ../pg_hba.conf.sample /usr/share/postgresql/$PG_MAJOR/
+
 RUN apt-get update && apt-get -y install\
-        cron &&\
+        cron\
+        python3\
+        python3-pip\
+        &&\
         apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
+RUN pip3 install -U\
+      argparse
+
+RUN mkdir /docker-entrypoint-initdb-core.d
+
+COPY ./docker-entrypoint-initdb-core.d/ /docker-entrypoint-initdb-core.d
+
 COPY ./backup/pg_backup.config /etc/pg_backup.config
 
 COPY ./backup/pg_backup_rotated.sh /usr/local/bin/pg_backup_rotated.sh
 
 COPY ./backup/pg_backup_rotated.cron.sh /etc/cron.daily/pg_backup_rotated
 
-COPY ./run.sh /run.sh
+COPY ./docker-entrypoint.sh /usr/local/bin/
 
-VOLUME "/var/lib/postgresql/backup"
+COPY ./migrate.py /usr/local/bin/
 
-ENTRYPOINT ["/run.sh"]
+RUN rm -rf /var/log/*
 
-CMD ["postgres"]
+VOLUME "/var/lib/postgresql/backup"

backup/pg_backup_rotated.cron.sh

@@ -1,3 +1,7 @@
 #! /usr/bin/env bash
 
+if [ "${POSTGRES_NO_BACKUP}" == 1 ]
+then
+  exit 0
+fi
 pg_backup_rotated.sh -c /etc/pg_backup.config > /dev/null

docker-entrypoint-initdb-core.d/1.0.0_01_create_db.sh

@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+
+if [ "${POSTGRES_SLAVE_MODE}" == 1 ]
+then
+  echo "Database is in slave mode. Exiting."
+  exit 0
+fi
+
+if [ "${POSTGRES_USER}" != 'postgres' ]
+then
+  psql <<-EOF
+  CREATE USER "${POSTGRES_USER}" WITH SUPERUSER PASSWORD '${POSTGRES_PASSWORD}';
+EOF
+else
+  psql <<-EOF
+  ALTER USER "${POSTGRES_USER}" WITH SUPERUSER PASSWORD '${POSTGRES_PASSWORD}';
+EOF
+fi

docker-entrypoint-initdb-core.d/1.0.0_02_create_user.sh

@@ -0,0 +1,14 @@
+#! /usr/bin/env bash
+
+if [ "${POSTGRES_SLAVE_MODE}" == 1 ]
+then
+  echo "Database is in slave mode. Exiting."
+  exit 0
+fi
+
+if [ "${POSTGRES_DB}" != 'postgres' ]
+then
+  psql <<-EOF
+  CREATE DATABASE "${POSTGRES_DB}";
+EOF
+fi

docker-entrypoint-initdb-core.d/1.1.0_01_master_create_rep_user.sh

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+if [ "${POSTGRES_MASTER_MODE}" != 1 ]
+then
+  echo "Database is not in master mode. Exiting."
+  exit 0
+fi
+
+psql <<-EOF
+  CREATE USER "${POSTGRES_REP_USER}" REPLICATION LOGIN ENCRYPTED PASSWORD '${POSTGRES_REP_PASSWORD}';
+EOF

docker-entrypoint-initdb-core.d/1.1.0_02_master_create_ro_user.sh

@@ -0,0 +1,25 @@
+#! /usr/bin/env bash
+
+if [ "${POSTGRES_MASTER_MODE}" != 1 ]
+then
+  echo "Database is not in master mode. Exiting."
+  exit 0
+fi
+
+psql <<-EOF
+  CREATE USER "${POSTGRES_RO_USER}"
+    NOSUPERUSER
+    NOCREATEDB
+    NOCREATEROLE
+    LOGIN
+    ENCRYPTED PASSWORD '${POSTGRES_RO_PASSWORD}';
+
+  REVOKE ALL ON DATABASE ${POSTGRES_DB} FROM ${POSTGRES_RO_USER};
+  GRANT CONNECT ON DATABASE ${POSTGRES_DB} TO ${POSTGRES_RO_USER};
+
+  REVOKE ALL ON SCHEMA public FROM ${POSTGRES_RO_USER};
+  GRANT USAGE ON SCHEMA public TO ${POSTGRES_RO_USER};
+
+  GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${POSTGRES_RO_USER};
+  ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO ${POSTGRES_RO_USER};
+EOF

docker-entrypoint-initdb-core.d/1.1.0_03_master_edit_postgres_conf.sh

@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+
+if [ "${POSTGRES_MASTER_MODE}" != 1 ]
+then
+  echo "Database is not in master mode. Exiting."
+  exit 0
+fi
+
+replace_pg_conf()
+{
+  var_name="${1}"
+  var_value="${2}"
+  sed -ri "s!^#?(${var_name})\s*=\s*\S+.*!\1 = ${var_value}!" /var/lib/postgresql/data/postgresql.conf
+}
+
+replace_pg_conf wal_level replica
+replace_pg_conf max_wal_senders 3
+replace_pg_conf wal_keep_segments 8

docker-entrypoint-initdb-core.d/1.1.0_04_master_edit_pg_hba_conf.sh

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+if [ "${POSTGRES_MASTER_MODE}" != 1 ]
+then
+  echo "Database is not in master mode. Exiting."
+  exit 0
+fi
+
+cat <<-EOF >> /var/lib/postgresql/data/pg_hba.conf
+  host    replication     ${POSTGRES_REP_USER}      ${POSTGRES_REP_ALLOWED_HOST}           md5
+EOF

docker-entrypoint-initdb-core.d/1.1.0_05_slave_edit_postgres_conf.sh

@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+
+if [ "${POSTGRES_SLAVE_MODE}" != 1 ]
+then
+  echo "Database is not in slave mode. Exiting."
+  exit 0
+fi
+
+replace_pg_conf()
+{
+  var_name="${1}"
+  var_value="${2}"
+  sed -ri "s!^#?(${var_name})\s*=\s*\S+.*!\1 = ${var_value}!" /var/lib/postgresql/data/postgresql.conf
+}
+
+replace_pg_conf wal_level replica
+replace_pg_conf max_wal_senders 3
+replace_pg_conf wal_keep_segments 8
+replace_pg_conf hot_standby on

docker-entrypoint-initdb-core.d/1.1.0_06_slave_edit_recovery_conf.sh

@@ -0,0 +1,14 @@
+#! /usr/bin/env bash
+
+if [ "${POSTGRES_SLAVE_MODE}" != 1 ]
+then
+  echo "Database is not in slave mode. Exiting."
+  exit 0
+fi
+
+
+cat <<-EOF > "${PGDATA}/recovery.conf"
+  standby_mode = 'on'
+  primary_conninfo = 'host=${POSTGRES_REP_HOST} port=${POSTGRES_REP_PORT} user=${POSTGRES_REP_USER} password=${POSTGRES_REP_PASSWORD}'
+  trigger_file = '/tmp/postgresql.trigger'
+EOF

docker-entrypoint.sh

@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+set -e
+
+if [ "${POSTGRES_INDEX}" != "" ]
+then
+  set_var_value()
+  {
+    var_name="${1}"_"${POSTGRES_INDEX}"
+    export "${1}"="${!var_name}"
+  }
+
+  set_var_value POSTGRES_DISABLED
+  set_var_value POSTGRES_SLAVE_MODE
+  set_var_value POSTGRES_MASTER_MODE
+  set_var_value POSTGRES_NO_BACKUP
+  set_var_value POSTGRES_HOST
+  set_var_value POSTGRES_USER
+  set_var_value POSTGRES_PASSWORD
+  set_var_value POSTGRES_DB
+  set_var_value POSTGRES_REP_HOST
+  set_var_value POSTGRES_REP_PORT
+  set_var_value POSTGRES_REP_USER
+  set_var_value POSTGRES_REP_PASSWORD
+  set_var_value POSTGRES_REP_ALLOWED_HOST
+  set_var_value POSTGRES_RO_USER
+  set_var_value POSTGRES_RO_PASSWORD
+fi
+
+if [ "${POSTGRES_DISABLED}" == 1 ]
+then
+  echo "Database ${POSTGRES_INDEX} is disabled. Exiting."
+  exit 0
+fi
+
+if [ "${POSTGRES_SLAVE_MODE}" == 1 ] && [ "${POSTGRES_MASTER_MODE}" == 1 ]
+then
+  echo "Misconfiguration: master and slave mode are both enabled. Exiting."
+  exit 64
+fi
+
+if [ "$(id -u)" == "0" ]
+then
+  cron
+fi
+
+# usage: file_env VAR [DEFAULT]
+#    ie: file_env 'XYZ_DB_PASSWORD' 'example'
+# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
+#  "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
+file_env() {
+	local var="$1"
+	local fileVar="${var}_FILE"
+	local def="${2:-}"
+	if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+		echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+		exit 1
+	fi
+	local val="$def"
+	if [ "${!var:-}" ]; then
+		val="${!var}"
+	elif [ "${!fileVar:-}" ]; then
+		val="$(< "${!fileVar}")"
+	fi
+	export "$var"="$val"
+	unset "$fileVar"
+}
+
+if [ "${1:0:1}" = '-' ]; then
+	set -- postgres "$@"
+fi
+
+# allow the container to be started with `--user`
+if [ "$1" = 'postgres' ] && [ "$(id -u)" = '0' ]; then
+	mkdir -p "$PGDATA"
+	chown -R postgres "$PGDATA"
+	chmod 700 "$PGDATA"
+
+	mkdir -p /var/run/postgresql
+	chown -R postgres /var/run/postgresql
+	chmod 775 /var/run/postgresql
+
+	# Create the transaction log directory before initdb is run (below) so the directory is owned by the correct user
+	if [ "$POSTGRES_INITDB_XLOGDIR" ]; then
+		mkdir -p "$POSTGRES_INITDB_XLOGDIR"
+		chown -R postgres "$POSTGRES_INITDB_XLOGDIR"
+		chmod 700 "$POSTGRES_INITDB_XLOGDIR"
+	fi
+
+	exec gosu postgres "$BASH_SOURCE" "$@"
+fi
+
+if [ "$1" = 'postgres' ]; then
+	mkdir -p "$PGDATA"
+	chown -R "$(id -u)" "$PGDATA" 2>/dev/null || :
+	chmod 700 "$PGDATA" 2>/dev/null || :
+
+        if [ "${POSTGRES_SLAVE_MODE}" != 1 ]
+        then
+
+	    # look specifically for PG_VERSION, as it is expected in the DB dir
+	    if [ ! -s "$PGDATA/PG_VERSION" ]; then
+		file_env 'POSTGRES_INITDB_ARGS'
+		if [ "$POSTGRES_INITDB_XLOGDIR" ]; then
+			export POSTGRES_INITDB_ARGS="$POSTGRES_INITDB_ARGS --xlogdir $POSTGRES_INITDB_XLOGDIR"
+		fi
+		eval "initdb --username=postgres $POSTGRES_INITDB_ARGS"
+
+		# check password first so we can output the warning before postgres
+		# messes it up
+		file_env 'POSTGRES_PASSWORD'
+		if [ "$POSTGRES_PASSWORD" ]; then
+			pass="PASSWORD '$POSTGRES_PASSWORD'"
+			authMethod=md5
+		else
+			# The - option suppresses leading tabs but *not* spaces. :)
+			cat >&2 <<-'EOWARN'
+				****************************************************
+				WARNING: No password has been set for the database.
+				         This will allow anyone with access to the
+				         Postgres port to access your database. In
+				         Docker's default configuration, this is
+				         effectively any other container on the same
+				         system.
+
+				         Use "-e POSTGRES_PASSWORD=password" to set
+				         it in "docker run".
+				****************************************************
+			EOWARN
+
+			pass=
+			authMethod=trust
+		fi
+
+		{
+			echo
+			echo "host all all all $authMethod"
+		} >> "$PGDATA/pg_hba.conf"
+
+		# internal start of server in order to allow set-up using psql-client
+		# does not listen on external TCP/IP and waits until start finishes
+		PGUSER="${PGUSER:-postgres}" \
+		pg_ctl -D "$PGDATA" \
+			-o "-c listen_addresses='localhost'" \
+			-w start
+
+		file_env 'POSTGRES_USER' 'postgres'
+		file_env 'POSTGRES_DB' "$POSTGRES_USER"
+
+		echo
+
+                export PGUSER="postgres"
+                export PGDATABASE="postgres"
+                echo "Running core migrate"
+                migrate.py --folder /docker-entrypoint-initdb-core.d/ --init
+
+                export PGUSER="${POSTGRES_USER}"
+                export PGDATABASE="${POSTGRES_DB}"
+                echo "Running user migrate"
+                migrate.py --folder /docker-entrypoint-initdb.d/ --init
+
+		PGUSER="${PGUSER:-postgres}" \
+		pg_ctl -D "$PGDATA" -m fast -w stop
+
+		echo
+		echo 'PostgreSQL init process complete; ready for start up.'
+		echo
+	    fi
+        else
+            echo "Setting up slave..."
+
+            rm -rf "${PGDATA}"/*
+
+            PGPASSWORD="${POSTGRES_REP_PASSWORD}" pg_basebackup -h "${POSTGRES_REP_HOST}" -p "${POSTGRES_REP_PORT}" -D "${PGDATA}" -U "${POSTGRES_REP_USER}" -v
+
+            export PGUSER="postgres"
+            export PGDATABASE="postgres"
+            echo "Running core migrate"
+            migrate.py --folder /docker-entrypoint-initdb-core.d/ --init
+
+            echo
+            echo 'PostgreSQL init process complete; ready for start up.'
+            echo
+        fi
+fi
+
+exec "$@"

migrate.py

@@ -0,0 +1,140 @@
+#! /usr/bin/env python3
+
+import argparse
+import sys
+import os
+import os.path
+import re
+import distutils.version
+import subprocess
+
+
+def file_runner_pgsql(folder, file):
+    subprocess.call(['psql', '-f', os.path.join(folder, file)])
+
+def file_runner_pgsql_gz(folder, file):
+    p1 = subprocess.Popen(['gunzip', '-c', os.path.join(folder, file)], stdout=subprocess.PIPE)
+    p2 = subprocess.Popen(["psql"], stdin=p1.stdout)
+    p1.stdout.close()
+    p2.communicate()
+
+def file_runner_exec(folder, file):
+    subprocess.call([os.path.join(folder, file)])
+
+
+def list_files(folder):
+    files = [f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))]
+    files.sort()
+    return files
+
+def sort_versions(versions):
+    versions.sort(key=distutils.version.StrictVersion)
+
+def list_versions(files):
+    versions = []
+    for file in files:
+        version = file.split('_')[0]
+        if re.match('^[0-9]+\.[0-9]+\.[0-9]+$', version):
+            if version not in versions:
+                versions.append(version)
+    sort_versions(versions)
+    return versions
+
+def run_file(folder, file, no_run, file_runners):
+    can_run = False
+    for file_runner_ext in file_runners:
+        if file.endswith(file_runner_ext):
+            print('Running file %s' % (file))
+            can_run = True
+            if not no_run:
+                try:
+                    sys.stdout.flush()
+                    file_runners[file_runner_ext](folder, file)
+                except Exception as e:
+                    print('Failed to run file: %s' % (e))
+    if not can_run:
+        print('Ignoring file %s' % (file))
+
+def run_migration(folder, files, version, no_run, file_runners):
+    for file in files:
+        if file.startswith('%s_' % (version)):
+            run_file(folder, file, no_run, file_runners)
+
+def run_migrations(folder, version_from, version_to, no_run, file_runners):
+    files = list_files(folder)
+    versions = list_versions(files)
+
+    if not versions:
+        print('No migration available. Exiting.')
+        return 0
+
+    if version_to is None:
+        version_to = versions[-1]
+        print('Defaulting VERSION_TO to %s' % (version_to))
+    if version_to not in versions:
+        print('Could not find VERSION_TO %s' % (version_to))
+        return 1
+    if version_from is not None and version_from not in versions:
+        print('Could not find VERSION_FROM %s' % (version_from))
+        return 1
+
+    if no_run:
+        print('Not running because --no-run was specified')
+
+    if version_from is None:
+        print('Initialising from %s (inclusive) to %s (inclusive)' % (versions[0], version_to))
+    else:
+        print('Migrating from %s (exclusive) to %s (inclusive)' % (version_from, version_to))
+
+    version_from_obj = distutils.version.StrictVersion('0.0.0' if version_from is None else version_from)
+    version_to_obj = distutils.version.StrictVersion(version_to)
+
+    last_version = version_from
+    for version in versions:
+        version_obj = distutils.version.StrictVersion(version)
+        if version_from_obj < version_obj and version_obj <= version_to_obj:
+            if last_version is None:
+                print('Initialising %s' % (version))
+            else:
+                print('Migrating from %s to %s' % (last_version, version))
+            run_migration(folder, files, version, no_run, file_runners)
+            last_version = version
+
+    return 0
+
+def main():
+    parser = argparse.ArgumentParser(description='Migrate container configuration/data')
+    parser.add_argument('--folder', dest='folder', default='/docker-entrypoint-initdb.d/', help='Folder to use for migration')
+    parser.add_argument('--init', dest='is_init', default=False, action='store_true', help='Run all migrations')
+    parser.add_argument('--migrate', dest='is_migrate', default=False, action='store_true', help='Run all migrations between VERSION_FROM (exclusive) and VERSION_TO (inclusive)')
+    parser.add_argument('--no-run', dest='no_run', default=False, action='store_true', help='Do not run migration, just print')
+    parser.add_argument('--version-from', dest='version_from', default=None, help='Current version, required if using --migrate, can not be used --init')
+    parser.add_argument('--version-to', dest='version_to', default=None, help='Final version, default to last available version is not given')
+
+    args = parser.parse_args()
+
+    file_runners = {
+            'sql': file_runner_pgsql,
+            'sql.gz': file_runner_pgsql_gz,
+            'sh': file_runner_exec
+            }
+
+    if args.is_init and args.is_migrate:
+        print('--init and --migrate can not be used together')
+        return 64
+    if args.is_init:
+        if args.version_from is not None:
+            print('--version-from can not be used with --init')
+            return 64
+        return run_migrations(args.folder, None, args.version_to, args.no_run, file_runners)
+    elif args.is_migrate:
+        if args.version_from is None:
+            print('--version-from is required. Use --init to run all migrations')
+            return 64
+        return run_migrations(args.folder, args.version_from, args.version_to, args.no_run, file_runners)
+    else:
+        print('Missing --init or --migrate')
+        return 64
+
+if __name__ == '__main__':
+    sys.exit(main())

run.sh

@@ -1,5 +0,0 @@
-#! /usr/bin/env bash
-
-cron -f &
-
-/docker-entrypoint.sh $*