Browse Source

new: [postgres] cleaned the charm for docker

postgres
Valentin Lab 6 years ago
parent
commit
055012c2d6
  1. 70
      postgres/actions/load
  2. 55
      postgres/actions/relations/postgres-database/load-db
  3. 53
      postgres/hooks/init
  4. 55
      postgres/hooks/install
  5. 47
      postgres/hooks/postgres_database-relation-joined
  6. 49
      postgres/hooks/schedule_command-relation-joined
  7. 120
      postgres/lib/common
  8. 9
      postgres/metadata.yml
  9. 36
      postgres/resources/bin/pg-backup

70
postgres/actions/load

@ -0,0 +1,70 @@
#!/bin/bash
## Load action gets a first argument a DIRECTORY holding the necessary files.
##
##
if [ -z "$SERVICE_DATASTORE" ]; then
echo "This script is meant to be run through 'compose' to work properly." >&2
exit 1
fi
usage="$exname [-h|--help] [--with-postgis] [--with-unaccent] SOURCE DBNAME"
dbname=
postgis=
while [ "$1" ]; do
case "$1" in
"--help"|"-h")
print_usage
exit 0
;;
"--with-postgis")
postgis=true
;;
"--with-unaccent")
unaccent=true
;;
*)
[ -z "$SOURCE" ] && { SOURCE=$1 ; shift ; continue ; }
[ -z "$dbname" ] && { dbname=$1 ; shift ; continue ; }
err "Unexpected argument '$1'."
exit 1
;;
esac
shift
done
if [ -z "$SOURCE" ]; then
err "You must provide a source file as first argument."
print_usage
exit 1
fi
if [ -z "$dbname" ]; then
err "You must provide a destination database name as second argument."
print_usage
exit 1
fi
if [[ "$dbname" == *"@"* ]]; then
IFS="@" read user dbname < <(echo "$dbname")
fi
. lib/common
## This can work only if ~/.pgpass is correctly created by init.
set -e
db_drop "$dbname"
POSTGIS=$postgis UNACCENT=$unaccent db_create "$dbname"
ddb "$dbname" > /dev/null < <(get_file "$SOURCE")
[ "$user" ] &&
db_grant_rights "$dbname" "$user"
info "Loaded '$SOURCE' into database '$dbname'."

55
postgres/actions/relations/postgres-database/load-db

@ -0,0 +1,55 @@
#!/bin/bash
## Load action gets a first argument a FILE/DIRECTORY/URL holding the necessary files.
##
##
if [ -z "$SERVICE_DATASTORE" ]; then
echo "This script is meant to be run through 'compose' to work properly." >&2
exit 1
fi
usage="$exname [-h|--help] SOURCE"
dbname=
while [ "$1" ]; do
case "$1" in
"--help"|"-h")
print_usage
exit 0
;;
*)
[ -z "$SOURCE" ] && { SOURCE=$1 ; shift ; continue ; }
err "Unexpected argument '$1'."
exit 1
;;
esac
shift
done
if [ -z "$SOURCE" ]; then
err "You must provide a source file as first argument."
print_usage
exit 1
fi
include parse
include pretty
set -e
USER=$(relation-get user)
DBNAME=$(relation-get dbname)
POSTGIS=$(relation-get postgis 2>/dev/null) || true
UNACCENT=$(relation-get unaccent 2>/dev/null) || true
opts=()
if [ "$POSTGIS" ]; then
opts+=("--with-postgis")
fi
if [ "$UNACCENT" ]; then
opts+=("--with-unaccent")
fi
run_service_action "$RELATION_TARGET_CHARM" load "$SOURCE" "$USER@$DBNAME" "${opts[@]}" "$@"

53
postgres/hooks/init

@ -0,0 +1,53 @@
#!/bin/bash
## Init is run on host
## For now it is run every time the script is launched, but
## it should be launched only once after build.
## Accessible variables are:
## - SERVICE_NAME Name of current service
## - DOCKER_BASE_IMAGE Base image from which this service might be built if any
## - SERVICE_DATASTORE Location on host of the DATASTORE of this service
## - SERVICE_CONFIGSTORE Location on host of the CONFIGSTORE of this service
# Please note that postgres detect on its own if its datadir needs to be populated
[ -e ~/.pgpass ] && exit 0
. lib/common
set -e
POSTGRES_ROOT_PASSWORD="$(gen_password)"
##
## Setting up access from host
##
ddb < <(echo "ALTER USER postgres WITH ENCRYPTED password '$POSTGRES_ROOT_PASSWORD'")
sed -ri 's%^host all all 0\.0\.0\.0/0 trust$%host all all 0.0.0.0/0 md5%g' \
"$SERVICE_DATASTORE/var/lib/postgresql/data/pg_hba.conf"
docker restart "$container_id"
## XXXvlab: this won't help support multiple project running on the
## same host
cat <<EOF > ~/.pgpass
*:*:*:postgres:$POSTGRES_ROOT_PASSWORD
EOF
chmod 600 ~/.pgpass
##
## pgm
##
echo 'prefix_pg_local_command=" " ## otherwise, will default to sudo -u postgres ' > /root/.pgm.rc
info "New root password for postgres. "

55
postgres/hooks/install

@ -1,55 +0,0 @@
#!/bin/bash
set -eux # -x for verbose logging to juju debug-log
## 0k git remote options
GIT_0K_CLONE_OPTIONS=${GIT_0K_CLONE_OPTIONS:-""}
## 0k git remote path
GIT_0K_BASE=${GIT_0K_BASE:-"git.0k.io:/var/git"}
apt-get install -y --force-yes cron kal-scripts
if [ "$(lsb_release -c -s)" == "trusty" ]; then
apt-get install -y --force-yes postgresql-9.3
else
apt-get install -y --force-yes postgresql-9.1
fi
cat <<EOF > /etc/cron.d/pgbackup
SHELL=/bin/sh
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
32 1 * * * root ansi_colors=no dayold=2 nbold=5 pgdump_to_dir /var/backups
EOF
##
## Install 0k-manage for all the pg_* tools
##
(
if ! [ -d "/opt/apps/0k-manage" ]; then
cd /opt/apps &&
git clone $GIT_0K_CLONE_OPTIONS "$GIT_0K_BASE/0k/0k-manage.git" &&
cd /opt/apps/0k-manage &&
git checkout 0k/prod/master
fi
ln -sf /opt/apps/0k-manage/src/bin/* /usr/local/bin/
)
## XXXvlab: Shouldn't we use ldap to create a key for each client and
## remove the clear password from here ?
#echo "CREATE USER openerp WITH PASSWORD 'xxxx' CREATEDB NOCREATEROLE;" | sudo -u postgres psql
## add this to pghba
#host all all 172.32.0.0/12 md5
#host all all 172.33.0.0/12 md5
## modify listen_addresses in postgresql.conf
#listen_addresses='*'

47
postgres/hooks/postgres_database-relation-joined

@ -0,0 +1,47 @@
#!/bin/bash
## When writing relation script, remember:
## - they should be idempotents
## - they can be launched while the dockers is already up
## - they are launched from the host
## - the target of the link is launched first, and get a chance to ``relation-set``
## - both side of the scripts get to use ``relation-get``.
## could generate this also if not set
DBNAME=$(relation-get dbname)
[ "$(relation-get password 2>/dev/null)" ] && exit 0
. lib/common
set -e
USER=$(relation-get user)
PASSWORD="$(gen_password)"
POSTGIS=$(relation-get postgis 2>/dev/null) || true
UNACCENT=$(relation-get unaccent 2>/dev/null) || true
ensure_db_docker_running
db_has_database "$DBNAME" || UNACCENT="$UNACCENT" POSTGIS="$POSTGIS" db_create "$DBNAME"
if ! db_has_user "$USER"; then
info "Creating a new user $USER."
db_create_user "$USER" "$PASSWORD"
else
info "Updating password of user $USER."
db_change_password "$USER" "$PASSWORD"
fi
db_grant_rights "$DBNAME" "$USER"
pgpass_line="*:*:*:$USER:$PASSWORD"
pgpass_file="$CONFIGSTORE/$BASE_CHARM_NAME/root/.pgpass"
if [ -e "$pgpass_file" ]; then
sed -ri "/^.+:.+:.+:$USER:.*$/d" "$pgpass_file"
fi
mkdir -p "$(dirname "$pgpass_file")"
echo "$pgpass_line" >> "$pgpass_file"
chmod 600 "$pgpass_file"
relation-set password "$PASSWORD"

49
postgres/hooks/schedule_command-relation-joined

@ -0,0 +1,49 @@
#!/bin/bash
## When writing relation script, remember:
## - they should be idempotents
## - they can be launched while the dockers is already up
## - they are launched from the host
## - the target of the link is launched first, and get a chance to ``relation-set``
## - both side of the scripts get to use ``relation-get``.
set -e
## XXXvlab: should use container name here so that it could support
## multiple postgres
label=pg-backup-$SERVICE_NAME
DST=$CONFIGSTORE/$TARGET_CHARM_NAME/etc/cron/$label
schedule=$(relation-get schedule)
if ! echo "$schedule" | egrep '^\s*(([0-9/,*-]+\s+){4,4}[0-9/,*-]+|@[a-z]+)\s*$' >/dev/null 2>&1; then
err "Unrecognized schedule '$schedule'."
exit 1
fi
## XXXvlab: how is this meant to works if there's multiples postgres (about PGHOST=?) ?
## Warning: using '\' in heredoc will be removed in the final cron file, which
## is totally wanted: cron does not support multilines.
exclude_dbs=$(relation-get exclude-dbs 2>/dev/null) || true
exclude_dbs=$(echo "$exclude_dbs" | shyaml get-values 2>/dev/null | xargs echo) || true
## Warning: 'docker -v' will use HOST directory even if launched from
## 'cron' container.
file_put "$DST" <<EOF
$schedule root lock $label -D -p 10 -c "\
docker run --rm \
-e PGHOST=\$POSTGRES_PORT_5432_TCP_ADDR \
-e exclude_dbs=\"$exclude_dbs\" \
-v /root/.pgpass:/root/.pgpass \
-v \"$BASE_CHARM_PATH/resources/bin/pg-backup:/usr/sbin/pg-backup\" \
-v \"$SERVICE_DATASTORE/var/backups/pg:/var/backups/pg\" \
--entrypoint pg-backup \
\"$DOCKER_BASE_IMAGE\"" 2>&1 | ts '\%F \%T \%Z' >> /var/log/cron/pg-backup_script.log
EOF
chmod +x "$DST"
config-add "\
$MASTER_TARGET_CHARM_NAME:
links:
- $MASTER_BASE_CHARM_NAME
"

120
postgres/lib/common

@ -0,0 +1,120 @@
# -*- mode: shell-script -*-
include pretty
export DB_NAME="postgres"
export DB_DATADIR=/var/lib/postgresql/data
export DB_PASSFILE=/root/.pgpass
is_db_locked() {
require lsof || apt-get install -y lsof </dev/null || {
err "Couldn't install command 'lsof'."
return 1
}
# "$lsof" +D "$host_db_working_dir" || true
## We need:
## 1- to protect against set -e
## 2- to detect if errorlevel 1 because lsof failed for good reason: permissions error, directory not found
## 3- to detect if errorlevel 1 because no process found
[ -d "$host_db_working_dir" ] || {
err "Directory '$host_db_working_dir' not found. Might be normal at this stage."
return 1
}
## We choose to ignore (2), and use stdout emptyness to ensure (3)
open_files=$(lsof +D "$host_db_working_dir") || true
[ "$open_files" ]
}
_set_db_params() {
local docker_ip="$1" docker_network="$2"
export db_docker_opts="-e PGHOST=$docker_ip -e PGUSER=postgres"
export db_cmd_opts=
}
ddb () { dcmd psql -qAt "$@"; }
##
## Entrypoints
##
db_drop () {
local dbname="$1"
dcmd dropdb --if-exists "$1"
}
db_create () {
local dbname="$1"
dcmd createdb "$dbname" || return 1
info "Database '$dbname' created."
if [ "$POSTGIS" ]; then
ddb -d "$dbname" < <(echo "CREATE EXTENSION postgis; CREATE EXTENSION postgis_topology;") || return 1
dcmd /bin/bash -c "psql -d '$dbname' -f /usr/share/postgresql/*/contrib/postgis-2.1/legacy.sql" || return 1
info "Installed postgis extensions on database '$dbname'."
fi
if [ "$UNACCENT" ]; then
ddb -d "$dbname" < <(echo "CREATE EXTENSION IF NOT EXISTS unaccent;") || return 1
info "Installed unaccent extension on database '$dbname'."
fi
}
## XXXvlab: if launched first, it'll fail handling correctly the open/close of docker
db_has_database() {
local dbname="$1"
if [ "$(ddb < <(echo "SELECT 1 FROM pg_database WHERE datname = '$dbname'"))" ]; then
debug "Database $dbname exists."
return 0
else
debug "Database $dbname exists."
return 1
fi
}
## XXXvlab: if launched first, it'll fail handling correctly the open/close of docker
db_has_user() {
local user="$1" users
users=$(ddb < <(echo "select u.usename from pg_catalog.pg_user u")) || {
err "Failed to get user list"
return 1
}
echo "$users" | grep "^${user}$" >/dev/null 2>&1
}
db_create_user() {
local user="$1" password="$2"
## Using this instead of pipes is important so that trap works
ddb < <(echo "CREATE USER $user WITH PASSWORD '$password' CREATEDB NOCREATEROLE;")
}
db_change_password() {
local user="$1" password="$2"
## Using this instead of pipes is important so that trap works
ddb < <(echo "ALTER USER $user WITH PASSWORD '$password';")
}
db_grant_rights () {
local dbname="$1" user="$2"
require psql || apt-get install -y postgresql-client </dev/null
require pgm || {
(
cd /opt/apps
git clone https://github.com/0k/pgm.git
ln -sf /opt/apps/pgm/bin/* /usr/local/bin/
apt-get install -y --force-yes pv buffer < /dev/null
# cd /opt/apps/0k-docker
# git checkout master
)
}
debug PGHOST="$DOCKER_IP" PGUSER=postgres pgm chown "$user" "$dbname"
PGHOST="$DOCKER_IP" PGUSER=postgres prefix_pg_local_command=" " pgm chown "$user" "$dbname"
}

9
postgres/metadata.yml

@ -1,10 +1,5 @@
name: postgres
summary: "Postgres server" summary: "Postgres server"
maintainer: "Valentin Lab <valentin.lab@kalysto.org>" maintainer: "Valentin Lab <valentin.lab@kalysto.org>"
inherit: base-0k
description: |
Postgres server
docker-image: docker.0k.io/postgis
data-resources: data-resources:
- /var/backups/pg
config-resources:
- /etc/postgresql
- /var/lib/postgresql/data

36
postgres/resources/bin/pg-backup

@ -0,0 +1,36 @@
#!/bin/bash
export PGUSER=postgres
pg_databases() {
echo "SELECT datname FROM pg_catalog.pg_database" | psql -At
}
dbs=$(pg_databases)
mkdir -p /var/backups/pg/
for db in $dbs; do
if [[ "$db" == "template"* || "$db" == "postgres" ]]; then
continue
fi
for exclude_db in $exclude_dbs; do
if [ "$exclude_db" == "$db" ]; then
echo "No backup for database $db as it is listed in 'exclude-dbs'."
continue 2
fi
done
dst=/var/backups/pg/$db.sql.gz
(( start = SECONDS ))
echo "Dumping database $db..." >&2
pg_dump -Ox "$db" | gzip --rsyncable > "$dst.inprogress"
errlvl="$?"
## Atomic replace
(( elapsed = SECONDS - start ))
if [ "$errlvl" != "0" ]; then
echo " !! Error when dumping database $db." >&2
else
echo " ..dumped $db to $dst ($(du -sh "$dst" | cut -f 1) in ${elapsed}s)" >&2
fi
mv "$dst.inprogress" "$dst"
done
Loading…
Cancel
Save