Refactor repository layout and convert build system to Mage.
This commit implements a massive refactor of the repository, and moves the build system over to use Mage (magefile.org) which should allow seamless building across multiple platforms.
This commit is contained in:
Will Rouesnel
7
cmd/postgres_exporter/tests/docker-postgres-replication/Dockerfile
Executable file
7
cmd/postgres_exporter/tests/docker-postgres-replication/Dockerfile
Executable file
@@ -0,0 +1,7 @@
|
||||
FROM postgres:10
|
||||
MAINTAINER Daniel Dent (https://www.danieldent.com)
|
||||
ENV PG_MAX_WAL_SENDERS 8
|
||||
ENV PG_WAL_KEEP_SEGMENTS 8
|
||||
COPY setup-replication.sh /docker-entrypoint-initdb.d/
|
||||
COPY docker-entrypoint.sh /docker-entrypoint.sh
|
||||
RUN chmod +x /docker-entrypoint-initdb.d/setup-replication.sh /docker-entrypoint.sh
|
||||
@@ -0,0 +1,7 @@
|
||||
FROM postgres:{{VERSION}}
|
||||
MAINTAINER Daniel Dent (https://www.danieldent.com)
|
||||
ENV PG_MAX_WAL_SENDERS 8
|
||||
ENV PG_WAL_KEEP_SEGMENTS 8
|
||||
COPY setup-replication.sh /docker-entrypoint-initdb.d/
|
||||
COPY docker-entrypoint.sh /docker-entrypoint.sh
|
||||
RUN chmod +x /docker-entrypoint-initdb.d/setup-replication.sh /docker-entrypoint.sh
|
||||
@@ -0,0 +1,11 @@
|
||||
# Replicated postgres cluster in docker.
|
||||
|
||||
Upstream is forked from https://github.com/DanielDent/docker-postgres-replication
|
||||
|
||||
My version lives at https://github.com/wrouesnel/docker-postgres-replication
|
||||
|
||||
This very simple docker-compose file lets us stand up a replicated postgres
|
||||
cluster so we can test streaming.
|
||||
|
||||
# TODO:
|
||||
Pull in p2 and template the Dockerfile so we can test multiple versions.
|
||||
@@ -0,0 +1,32 @@
|
||||
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
pg-master:
|
||||
build: '.'
|
||||
image: 'danieldent/postgres-replication'
|
||||
restart: 'always'
|
||||
environment:
|
||||
POSTGRES_USER: 'postgres'
|
||||
POSTGRES_PASSWORD: 'postgres'
|
||||
PGDATA: '/var/lib/postgresql/data/pgdata'
|
||||
volumes:
|
||||
- '/var/lib/postgresql/data'
|
||||
expose:
|
||||
- '5432'
|
||||
|
||||
pg-slave:
|
||||
build: '.'
|
||||
image: 'danieldent/postgres-replication'
|
||||
restart: 'always'
|
||||
environment:
|
||||
POSTGRES_USER: 'postgres'
|
||||
POSTGRES_PASSWORD: 'postgres'
|
||||
PGDATA: '/var/lib/postgresql/data/pgdata'
|
||||
REPLICATE_FROM: 'pg-master'
|
||||
volumes:
|
||||
- '/var/lib/postgresql/data'
|
||||
expose:
|
||||
- '5432'
|
||||
links:
|
||||
- 'pg-master'
|
||||
140
cmd/postgres_exporter/tests/docker-postgres-replication/docker-entrypoint.sh
Executable file
140
cmd/postgres_exporter/tests/docker-postgres-replication/docker-entrypoint.sh
Executable file
@@ -0,0 +1,140 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Backwards compatibility for old variable names (deprecated)
|
||||
if [ "x$PGUSER" != "x" ]; then
|
||||
POSTGRES_USER=$PGUSER
|
||||
fi
|
||||
if [ "x$PGPASSWORD" != "x" ]; then
|
||||
POSTGRES_PASSWORD=$PGPASSWORD
|
||||
fi
|
||||
|
||||
# Forwards-compatibility for old variable names (pg_basebackup uses them)
|
||||
if [ "x$PGPASSWORD" = "x" ]; then
|
||||
export PGPASSWORD=$POSTGRES_PASSWORD
|
||||
fi
|
||||
|
||||
# Based on official postgres package's entrypoint script (https://hub.docker.com/_/postgres/)
|
||||
# Modified to be able to set up a slave. The docker-entrypoint-initdb.d hook provided is inadequate.
|
||||
|
||||
set -e
|
||||
|
||||
if [ "${1:0:1}" = '-' ]; then
|
||||
set -- postgres "$@"
|
||||
fi
|
||||
|
||||
if [ "$1" = 'postgres' ]; then
|
||||
mkdir -p "$PGDATA"
|
||||
chmod 700 "$PGDATA"
|
||||
chown -R postgres "$PGDATA"
|
||||
|
||||
mkdir -p /run/postgresql
|
||||
chmod g+s /run/postgresql
|
||||
chown -R postgres /run/postgresql
|
||||
|
||||
# look specifically for PG_VERSION, as it is expected in the DB dir
|
||||
if [ ! -s "$PGDATA/PG_VERSION" ]; then
|
||||
if [ "x$REPLICATE_FROM" == "x" ]; then
|
||||
eval "gosu postgres initdb $POSTGRES_INITDB_ARGS"
|
||||
else
|
||||
until ping -c 1 -W 1 ${REPLICATE_FROM}
|
||||
do
|
||||
echo "Waiting for master to ping..."
|
||||
sleep 1s
|
||||
done
|
||||
until gosu postgres pg_basebackup -h ${REPLICATE_FROM} -D ${PGDATA} -U ${POSTGRES_USER} -vP -w
|
||||
do
|
||||
echo "Waiting for master to connect..."
|
||||
sleep 1s
|
||||
done
|
||||
fi
|
||||
|
||||
# check password first so we can output the warning before postgres
|
||||
# messes it up
|
||||
if [ ! -z "$POSTGRES_PASSWORD" ]; then
|
||||
pass="PASSWORD '$POSTGRES_PASSWORD'"
|
||||
authMethod=md5
|
||||
else
|
||||
# The - option suppresses leading tabs but *not* spaces. :)
|
||||
cat >&2 <<-'EOWARN'
|
||||
****************************************************
|
||||
WARNING: No password has been set for the database.
|
||||
This will allow anyone with access to the
|
||||
Postgres port to access your database. In
|
||||
Docker's default configuration, this is
|
||||
effectively any other container on the same
|
||||
system.
|
||||
|
||||
Use "-e POSTGRES_PASSWORD=password" to set
|
||||
it in "docker run".
|
||||
****************************************************
|
||||
EOWARN
|
||||
|
||||
pass=
|
||||
authMethod=trust
|
||||
fi
|
||||
|
||||
if [ "x$REPLICATE_FROM" == "x" ]; then
|
||||
|
||||
{ echo; echo "host replication all 0.0.0.0/0 $authMethod"; } | gosu postgres tee -a "$PGDATA/pg_hba.conf" > /dev/null
|
||||
{ echo; echo "host all all 0.0.0.0/0 $authMethod"; } | gosu postgres tee -a "$PGDATA/pg_hba.conf" > /dev/null
|
||||
|
||||
# internal start of server in order to allow set-up using psql-client
|
||||
# does not listen on external TCP/IP and waits until start finishes
|
||||
gosu postgres pg_ctl -D "$PGDATA" \
|
||||
-o "-c listen_addresses='localhost'" \
|
||||
-w start
|
||||
|
||||
: ${POSTGRES_USER:=postgres}
|
||||
: ${POSTGRES_DB:=$POSTGRES_USER}
|
||||
export POSTGRES_USER POSTGRES_DB
|
||||
|
||||
psql=( psql -v ON_ERROR_STOP=1 )
|
||||
|
||||
if [ "$POSTGRES_DB" != 'postgres' ]; then
|
||||
"${psql[@]}" --username postgres <<-EOSQL
|
||||
CREATE DATABASE "$POSTGRES_DB" ;
|
||||
EOSQL
|
||||
echo
|
||||
fi
|
||||
|
||||
if [ "$POSTGRES_USER" = 'postgres' ]; then
|
||||
op='ALTER'
|
||||
else
|
||||
op='CREATE'
|
||||
fi
|
||||
"${psql[@]}" --username postgres <<-EOSQL
|
||||
$op USER "$POSTGRES_USER" WITH SUPERUSER $pass ;
|
||||
EOSQL
|
||||
echo
|
||||
|
||||
fi
|
||||
|
||||
psql+=( --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" )
|
||||
|
||||
echo
|
||||
for f in /docker-entrypoint-initdb.d/*; do
|
||||
case "$f" in
|
||||
*.sh) echo "$0: running $f"; . "$f" ;;
|
||||
*.sql) echo "$0: running $f"; "${psql[@]}" < "$f"; echo ;;
|
||||
*.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${psql[@]}"; echo ;;
|
||||
*) echo "$0: ignoring $f" ;;
|
||||
esac
|
||||
echo
|
||||
done
|
||||
|
||||
if [ "x$REPLICATE_FROM" == "x" ]; then
|
||||
gosu postgres pg_ctl -D "$PGDATA" -m fast -w stop
|
||||
fi
|
||||
|
||||
echo
|
||||
echo 'PostgreSQL init process complete; ready for start up.'
|
||||
echo
|
||||
fi
|
||||
|
||||
# We need this health check so we know when it's started up.
|
||||
touch /tmp/.postgres_init_complete
|
||||
|
||||
exec gosu postgres "$@"
|
||||
fi
|
||||
|
||||
exec "$@"
|
||||
22
cmd/postgres_exporter/tests/docker-postgres-replication/setup-replication.sh
Executable file
22
cmd/postgres_exporter/tests/docker-postgres-replication/setup-replication.sh
Executable file
@@ -0,0 +1,22 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ "x$REPLICATE_FROM" == "x" ]; then
|
||||
|
||||
cat >> ${PGDATA}/postgresql.conf <<EOF
|
||||
wal_level = hot_standby
|
||||
max_wal_senders = $PG_MAX_WAL_SENDERS
|
||||
wal_keep_segments = $PG_WAL_KEEP_SEGMENTS
|
||||
hot_standby = on
|
||||
EOF
|
||||
|
||||
else
|
||||
|
||||
cat > ${PGDATA}/recovery.conf <<EOF
|
||||
standby_mode = on
|
||||
primary_conninfo = 'host=${REPLICATE_FROM} port=5432 user=${POSTGRES_USER} password=${POSTGRES_PASSWORD}'
|
||||
trigger_file = '/tmp/touch_me_to_promote_to_me_master'
|
||||
EOF
|
||||
chown postgres ${PGDATA}/recovery.conf
|
||||
chmod 600 ${PGDATA}/recovery.conf
|
||||
|
||||
fi
|
||||
169
cmd/postgres_exporter/tests/test-smoke
Executable file
169
cmd/postgres_exporter/tests/test-smoke
Executable file
@@ -0,0 +1,169 @@
|
||||
#!/bin/bash
|
||||
# Basic integration tests with postgres. Requires docker to work.
|
||||
|
||||
SOURCE="${BASH_SOURCE[0]}"
|
||||
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
|
||||
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
||||
SOURCE="$(readlink "$SOURCE")"
|
||||
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
|
||||
done
|
||||
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
||||
|
||||
METRICS_DIR=$(pwd)
|
||||
|
||||
# Read the absolute path to the exporter
|
||||
postgres_exporter="$1"
|
||||
test_binary="$2"
|
||||
export POSTGRES_PASSWORD=postgres
|
||||
exporter_port=9187
|
||||
|
||||
echo "Exporter Binary: $postgres_exporter" 1>&2
|
||||
echo "Test Binary: $test_binary" 1>&2
|
||||
|
||||
[ -z "$postgres_exporter" ] && echo "Missing exporter binary" && exit 1
|
||||
[ -z "$test_binary" ] && echo "Missing test binary" && exit 1
|
||||
|
||||
cd $DIR
|
||||
|
||||
VERSIONS=( \
|
||||
9.1 \
|
||||
9.2 \
|
||||
9.3 \
|
||||
9.4 \
|
||||
9.5 \
|
||||
9.6 \
|
||||
10 \
|
||||
)
|
||||
|
||||
wait_for_postgres(){
|
||||
local ip=$1
|
||||
local port=$2
|
||||
if [ -z $ip ]; then
|
||||
echo "No IP specified." 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z $port ]; then
|
||||
echo "No port specified." 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local wait_start=$(date +%s)
|
||||
echo "Waiting for postgres to start listening..."
|
||||
while ! pg_isready --host=$ip --port=$port &> /dev/null; do
|
||||
if [ $(( $(date +%s) - $wait_start )) -gt $TIMEOUT ]; then
|
||||
echo "Timed out waiting for postgres to start!" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_exporter() {
|
||||
local wait_start=$(date +%s)
|
||||
echo "Waiting for exporter to start..."
|
||||
while ! nc -z localhost $exporter_port ; do
|
||||
if [ $(( $(date +%s) - $wait_start )) -gt $TIMEOUT ]; then
|
||||
echo "Timed out waiting for exporter!" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
smoketest_postgres() {
|
||||
local version=$1
|
||||
local CONTAINER_NAME=postgres_exporter-test-smoke
|
||||
local TIMEOUT=30
|
||||
local IMAGE_NAME=postgres
|
||||
|
||||
local CUR_IMAGE=$IMAGE_NAME:$version
|
||||
|
||||
echo "#######################"
|
||||
echo "Standalone Postgres $version"
|
||||
echo "#######################"
|
||||
local docker_cmd="docker run -d -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD $CUR_IMAGE"
|
||||
echo "Docker Cmd: $docker_cmd"
|
||||
|
||||
CONTAINER_NAME=$($docker_cmd)
|
||||
standalone_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $CONTAINER_NAME)
|
||||
trap "docker logs $CONTAINER_NAME ; docker kill $CONTAINER_NAME ; docker rm -v $CONTAINER_NAME; exit 1" EXIT INT TERM
|
||||
wait_for_postgres $standalone_ip 5432
|
||||
|
||||
|
||||
# Run the test binary.
|
||||
DATA_SOURCE_NAME="postgresql://postgres:$POSTGRES_PASSWORD@$standalone_ip:5432/?sslmode=disable" $test_binary || exit $?
|
||||
|
||||
# Extract a raw metric list.
|
||||
DATA_SOURCE_NAME="postgresql://postgres:$POSTGRES_PASSWORD@$standalone_ip:5432/?sslmode=disable" $postgres_exporter --log.level=debug --web.listen-address=:$exporter_port &
|
||||
exporter_pid=$!
|
||||
trap "docker logs $CONTAINER_NAME ; docker kill $CONTAINER_NAME ; docker rm -v $CONTAINER_NAME; kill $exporter_pid; exit 1" EXIT INT TERM
|
||||
wait_for_exporter
|
||||
|
||||
# Dump the metrics to a file.
|
||||
wget -q -O - http://localhost:$exporter_port/metrics 1> $METRICS_DIR/.metrics.single.$version.prom
|
||||
if [ "$?" != "0" ]; then
|
||||
echo "Failed on postgres $version ($DOCKER_IMAGE)" 1>&2
|
||||
kill $exporter_pid
|
||||
exit 1
|
||||
fi
|
||||
|
||||
kill $exporter_pid
|
||||
docker kill $CONTAINER_NAME
|
||||
docker rm -v $CONTAINER_NAME
|
||||
trap - EXIT INT TERM
|
||||
|
||||
echo "#######################"
|
||||
echo "Replicated Postgres $version"
|
||||
echo "#######################"
|
||||
old_pwd=$(pwd)
|
||||
cd docker-postgres-replication
|
||||
|
||||
VERSION=$version p2 -t Dockerfile.p2 -o Dockerfile
|
||||
if [ "$?" != "0" ]; then
|
||||
echo "Templating failed" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
trap "docker-compose logs; docker-compose down ; docker-compose rm -v; exit 1" EXIT INT TERM
|
||||
local compose_cmd="POSTGRES_PASSWORD=$POSTGRES_PASSWORD docker-compose up -d --force-recreate --build"
|
||||
echo "Compose Cmd: $compose_cmd"
|
||||
eval $compose_cmd
|
||||
|
||||
master_container=$(docker-compose ps -q pg-master)
|
||||
slave_container=$(docker-compose ps -q pg-slave)
|
||||
master_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $master_container)
|
||||
slave_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $slave_container)
|
||||
echo "Got master IP: $master_ip"
|
||||
wait_for_postgres $master_ip 5432
|
||||
wait_for_postgres $slave_ip 5432
|
||||
|
||||
DATA_SOURCE_NAME="postgresql://postgres:$POSTGRES_PASSWORD@$master_ip:5432/?sslmode=disable" $test_binary || exit $?
|
||||
|
||||
DATA_SOURCE_NAME="postgresql://postgres:$POSTGRES_PASSWORD@$master_ip:5432/?sslmode=disable" $postgres_exporter --log.level=debug --web.listen-address=:$exporter_port &
|
||||
exporter_pid=$!
|
||||
trap "docker-compose logs; docker-compose down ; docker-compose rm -v ; kill $exporter_pid; exit 1" EXIT INT TERM
|
||||
wait_for_exporter
|
||||
|
||||
wget -q -O - http://localhost:$exporter_port/metrics 1> $METRICS_DIR/.metrics.replicated.$version.prom
|
||||
if [ "$?" != "0" ]; then
|
||||
echo "Failed on postgres $version ($DOCKER_IMAGE)" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
kill $exporter_pid
|
||||
docker-compose down
|
||||
docker-compose rm -v
|
||||
trap - EXIT INT TERM
|
||||
|
||||
cd $old_pwd
|
||||
}
|
||||
|
||||
# Start pulling the docker images in advance
|
||||
for version in ${VERSIONS[@]}; do
|
||||
docker pull postgres:$version > /dev/null &
|
||||
done
|
||||
|
||||
for version in ${VERSIONS[@]}; do
|
||||
echo "Testing postgres version $version"
|
||||
smoketest_postgres $version
|
||||
done
|
||||
1
cmd/postgres_exporter/tests/username_file
Normal file
1
cmd/postgres_exporter/tests/username_file
Normal file
@@ -0,0 +1 @@
|
||||
custom_username
|
||||
1
cmd/postgres_exporter/tests/userpass_file
Normal file
1
cmd/postgres_exporter/tests/userpass_file
Normal file
@@ -0,0 +1 @@
|
||||
custom_password
|
||||
Reference in New Issue
Block a user