9 changed files with 432 additions and 62 deletions
-
70postgres/actions/load
-
55postgres/actions/relations/postgres-database/load-db
-
53postgres/hooks/init
-
55postgres/hooks/install
-
47postgres/hooks/postgres_database-relation-joined
-
49postgres/hooks/schedule_command-relation-joined
-
120postgres/lib/common
-
9postgres/metadata.yml
-
36postgres/resources/bin/pg-backup
@ -0,0 +1,70 @@ |
|||||
|
#!/bin/bash |
||||
|
|
||||
|
## Load action gets a first argument a DIRECTORY holding the necessary files. |
||||
|
## |
||||
|
## |
||||
|
|
||||
|
if [ -z "$SERVICE_DATASTORE" ]; then |
||||
|
echo "This script is meant to be run through 'compose' to work properly." >&2 |
||||
|
exit 1 |
||||
|
fi |
||||
|
|
||||
|
usage="$exname [-h|--help] [--with-postgis] [--with-unaccent] SOURCE DBNAME" |
||||
|
|
||||
|
dbname= |
||||
|
postgis= |
||||
|
while [ "$1" ]; do |
||||
|
case "$1" in |
||||
|
"--help"|"-h") |
||||
|
print_usage |
||||
|
exit 0 |
||||
|
;; |
||||
|
"--with-postgis") |
||||
|
postgis=true |
||||
|
;; |
||||
|
"--with-unaccent") |
||||
|
unaccent=true |
||||
|
;; |
||||
|
*) |
||||
|
[ -z "$SOURCE" ] && { SOURCE=$1 ; shift ; continue ; } |
||||
|
[ -z "$dbname" ] && { dbname=$1 ; shift ; continue ; } |
||||
|
err "Unexpected argument '$1'." |
||||
|
exit 1 |
||||
|
;; |
||||
|
esac |
||||
|
shift |
||||
|
done |
||||
|
|
||||
|
if [ -z "$SOURCE" ]; then |
||||
|
err "You must provide a source file as first argument." |
||||
|
print_usage |
||||
|
exit 1 |
||||
|
fi |
||||
|
|
||||
|
if [ -z "$dbname" ]; then |
||||
|
err "You must provide a destination database name as second argument." |
||||
|
print_usage |
||||
|
exit 1 |
||||
|
fi |
||||
|
|
||||
|
if [[ "$dbname" == *"@"* ]]; then |
||||
|
IFS="@" read user dbname < <(echo "$dbname") |
||||
|
fi |
||||
|
|
||||
|
. lib/common |
||||
|
|
||||
|
## This can work only if ~/.pgpass is correctly created by init. |
||||
|
|
||||
|
set -e |
||||
|
|
||||
|
|
||||
|
db_drop "$dbname" |
||||
|
POSTGIS=$postgis UNACCENT=$unaccent db_create "$dbname" |
||||
|
|
||||
|
|
||||
|
ddb "$dbname" > /dev/null < <(get_file "$SOURCE") |
||||
|
|
||||
|
[ "$user" ] && |
||||
|
db_grant_rights "$dbname" "$user" |
||||
|
|
||||
|
info "Loaded '$SOURCE' into database '$dbname'." |
||||
@ -0,0 +1,55 @@ |
|||||
|
#!/bin/bash |
||||
|
|
||||
|
## Load action gets a first argument a FILE/DIRECTORY/URL holding the necessary files. |
||||
|
## |
||||
|
## |
||||
|
|
||||
|
if [ -z "$SERVICE_DATASTORE" ]; then |
||||
|
echo "This script is meant to be run through 'compose' to work properly." >&2 |
||||
|
exit 1 |
||||
|
fi |
||||
|
|
||||
|
usage="$exname [-h|--help] SOURCE" |
||||
|
|
||||
|
dbname= |
||||
|
while [ "$1" ]; do |
||||
|
case "$1" in |
||||
|
"--help"|"-h") |
||||
|
print_usage |
||||
|
exit 0 |
||||
|
;; |
||||
|
*) |
||||
|
[ -z "$SOURCE" ] && { SOURCE=$1 ; shift ; continue ; } |
||||
|
err "Unexpected argument '$1'." |
||||
|
exit 1 |
||||
|
;; |
||||
|
esac |
||||
|
shift |
||||
|
done |
||||
|
|
||||
|
if [ -z "$SOURCE" ]; then |
||||
|
err "You must provide a source file as first argument." |
||||
|
print_usage |
||||
|
exit 1 |
||||
|
fi |
||||
|
|
||||
|
include parse |
||||
|
include pretty |
||||
|
|
||||
|
set -e |
||||
|
|
||||
|
USER=$(relation-get user) |
||||
|
DBNAME=$(relation-get dbname) |
||||
|
POSTGIS=$(relation-get postgis 2>/dev/null) || true |
||||
|
UNACCENT=$(relation-get unaccent 2>/dev/null) || true |
||||
|
|
||||
|
opts=() |
||||
|
if [ "$POSTGIS" ]; then |
||||
|
opts+=("--with-postgis") |
||||
|
fi |
||||
|
|
||||
|
if [ "$UNACCENT" ]; then |
||||
|
opts+=("--with-unaccent") |
||||
|
fi |
||||
|
|
||||
|
run_service_action "$RELATION_TARGET_CHARM" load "$SOURCE" "$USER@$DBNAME" "${opts[@]}" "$@" |
||||
@ -0,0 +1,53 @@ |
|||||
|
#!/bin/bash |
||||
|
|
||||
|
## Init is run on host |
||||
|
## For now it is run every time the script is launched, but |
||||
|
## it should be launched only once after build. |
||||
|
|
||||
|
## Accessible variables are: |
||||
|
## - SERVICE_NAME Name of current service |
||||
|
## - DOCKER_BASE_IMAGE Base image from which this service might be built if any |
||||
|
## - SERVICE_DATASTORE Location on host of the DATASTORE of this service |
||||
|
## - SERVICE_CONFIGSTORE Location on host of the CONFIGSTORE of this service |
||||
|
|
||||
|
|
||||
|
|
||||
|
# Please note that postgres detect on its own if its datadir needs to be populated |
||||
|
|
||||
|
|
||||
|
[ -e ~/.pgpass ] && exit 0 |
||||
|
|
||||
|
. lib/common |
||||
|
|
||||
|
set -e |
||||
|
|
||||
|
POSTGRES_ROOT_PASSWORD="$(gen_password)" |
||||
|
|
||||
|
## |
||||
|
## Setting up access from host |
||||
|
## |
||||
|
|
||||
|
ddb < <(echo "ALTER USER postgres WITH ENCRYPTED password '$POSTGRES_ROOT_PASSWORD'") |
||||
|
|
||||
|
sed -ri 's%^host all all 0\.0\.0\.0/0 trust$%host all all 0.0.0.0/0 md5%g' \ |
||||
|
"$SERVICE_DATASTORE/var/lib/postgresql/data/pg_hba.conf" |
||||
|
|
||||
|
docker restart "$container_id" |
||||
|
|
||||
|
|
||||
|
## XXXvlab: this won't help support multiple project running on the |
||||
|
## same host |
||||
|
cat <<EOF > ~/.pgpass |
||||
|
*:*:*:postgres:$POSTGRES_ROOT_PASSWORD |
||||
|
EOF |
||||
|
|
||||
|
chmod 600 ~/.pgpass |
||||
|
|
||||
|
## |
||||
|
## pgm |
||||
|
## |
||||
|
|
||||
|
echo 'prefix_pg_local_command=" " ## otherwise, will default to sudo -u postgres ' > /root/.pgm.rc |
||||
|
|
||||
|
info "New root password for postgres. " |
||||
|
|
||||
@ -1,55 +0,0 @@ |
|||||
#!/bin/bash |
|
||||
|
|
||||
set -eux # -x for verbose logging to juju debug-log |
|
||||
|
|
||||
## 0k git remote options |
|
||||
GIT_0K_CLONE_OPTIONS=${GIT_0K_CLONE_OPTIONS:-""} |
|
||||
|
|
||||
## 0k git remote path |
|
||||
GIT_0K_BASE=${GIT_0K_BASE:-"git.0k.io:/var/git"} |
|
||||
|
|
||||
apt-get install -y --force-yes cron kal-scripts |
|
||||
|
|
||||
if [ "$(lsb_release -c -s)" == "trusty" ]; then |
|
||||
apt-get install -y --force-yes postgresql-9.3 |
|
||||
else |
|
||||
apt-get install -y --force-yes postgresql-9.1 |
|
||||
fi |
|
||||
|
|
||||
cat <<EOF > /etc/cron.d/pgbackup |
|
||||
|
|
||||
SHELL=/bin/sh |
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin |
|
||||
|
|
||||
32 1 * * * root ansi_colors=no dayold=2 nbold=5 pgdump_to_dir /var/backups |
|
||||
|
|
||||
EOF |
|
||||
|
|
||||
## |
|
||||
## Install 0k-manage for all the pg_* tools |
|
||||
## |
|
||||
|
|
||||
( |
|
||||
if ! [ -d "/opt/apps/0k-manage" ]; then |
|
||||
cd /opt/apps && |
|
||||
git clone $GIT_0K_CLONE_OPTIONS "$GIT_0K_BASE/0k/0k-manage.git" && |
|
||||
cd /opt/apps/0k-manage && |
|
||||
git checkout 0k/prod/master |
|
||||
fi |
|
||||
|
|
||||
ln -sf /opt/apps/0k-manage/src/bin/* /usr/local/bin/ |
|
||||
) |
|
||||
|
|
||||
|
|
||||
## XXXvlab: Shouldn't we use ldap to create a key for each client and |
|
||||
## remove the clear password from here ? |
|
||||
#echo "CREATE USER openerp WITH PASSWORD 'xxxx' CREATEDB NOCREATEROLE;" | sudo -u postgres psql |
|
||||
|
|
||||
|
|
||||
## add this to pghba |
|
||||
#host all all 172.32.0.0/12 md5 |
|
||||
#host all all 172.33.0.0/12 md5 |
|
||||
|
|
||||
## modify listen_addresses in postgresql.conf |
|
||||
#listen_addresses='*' |
|
||||
|
|
||||
@ -0,0 +1,47 @@ |
|||||
|
#!/bin/bash |
||||
|
|
||||
|
## When writing relation script, remember: |
||||
|
## - they should be idempotents |
||||
|
## - they can be launched while the dockers is already up |
||||
|
## - they are launched from the host |
||||
|
## - the target of the link is launched first, and get a chance to ``relation-set`` |
||||
|
## - both side of the scripts get to use ``relation-get``. |
||||
|
|
||||
|
## could generate this also if not set |
||||
|
DBNAME=$(relation-get dbname) |
||||
|
|
||||
|
[ "$(relation-get password 2>/dev/null)" ] && exit 0 |
||||
|
|
||||
|
. lib/common |
||||
|
|
||||
|
set -e |
||||
|
|
||||
|
USER=$(relation-get user) |
||||
|
PASSWORD="$(gen_password)" |
||||
|
POSTGIS=$(relation-get postgis 2>/dev/null) || true |
||||
|
UNACCENT=$(relation-get unaccent 2>/dev/null) || true |
||||
|
|
||||
|
|
||||
|
ensure_db_docker_running |
||||
|
|
||||
|
db_has_database "$DBNAME" || UNACCENT="$UNACCENT" POSTGIS="$POSTGIS" db_create "$DBNAME" |
||||
|
if ! db_has_user "$USER"; then |
||||
|
info "Creating a new user $USER." |
||||
|
db_create_user "$USER" "$PASSWORD" |
||||
|
else |
||||
|
info "Updating password of user $USER." |
||||
|
db_change_password "$USER" "$PASSWORD" |
||||
|
fi |
||||
|
db_grant_rights "$DBNAME" "$USER" |
||||
|
|
||||
|
pgpass_line="*:*:*:$USER:$PASSWORD" |
||||
|
pgpass_file="$CONFIGSTORE/$BASE_CHARM_NAME/root/.pgpass" |
||||
|
|
||||
|
if [ -e "$pgpass_file" ]; then |
||||
|
sed -ri "/^.+:.+:.+:$USER:.*$/d" "$pgpass_file" |
||||
|
fi |
||||
|
mkdir -p "$(dirname "$pgpass_file")" |
||||
|
echo "$pgpass_line" >> "$pgpass_file" |
||||
|
chmod 600 "$pgpass_file" |
||||
|
|
||||
|
relation-set password "$PASSWORD" |
||||
@ -0,0 +1,49 @@ |
|||||
|
#!/bin/bash |
||||
|
|
||||
|
## When writing relation script, remember: |
||||
|
## - they should be idempotents |
||||
|
## - they can be launched while the dockers is already up |
||||
|
## - they are launched from the host |
||||
|
## - the target of the link is launched first, and get a chance to ``relation-set`` |
||||
|
## - both side of the scripts get to use ``relation-get``. |
||||
|
|
||||
|
set -e |
||||
|
|
||||
|
## XXXvlab: should use container name here so that it could support |
||||
|
## multiple postgres |
||||
|
label=pg-backup-$SERVICE_NAME |
||||
|
DST=$CONFIGSTORE/$TARGET_CHARM_NAME/etc/cron/$label |
||||
|
schedule=$(relation-get schedule) |
||||
|
|
||||
|
if ! echo "$schedule" | egrep '^\s*(([0-9/,*-]+\s+){4,4}[0-9/,*-]+|@[a-z]+)\s*$' >/dev/null 2>&1; then |
||||
|
err "Unrecognized schedule '$schedule'." |
||||
|
exit 1 |
||||
|
fi |
||||
|
|
||||
|
## XXXvlab: how is this meant to works if there's multiples postgres (about PGHOST=?) ? |
||||
|
## Warning: using '\' in heredoc will be removed in the final cron file, which |
||||
|
## is totally wanted: cron does not support multilines. |
||||
|
|
||||
|
exclude_dbs=$(relation-get exclude-dbs 2>/dev/null) || true |
||||
|
exclude_dbs=$(echo "$exclude_dbs" | shyaml get-values 2>/dev/null | xargs echo) || true |
||||
|
|
||||
|
## Warning: 'docker -v' will use HOST directory even if launched from |
||||
|
## 'cron' container. |
||||
|
file_put "$DST" <<EOF |
||||
|
$schedule root lock $label -D -p 10 -c "\ |
||||
|
docker run --rm \ |
||||
|
-e PGHOST=\$POSTGRES_PORT_5432_TCP_ADDR \ |
||||
|
-e exclude_dbs=\"$exclude_dbs\" \ |
||||
|
-v /root/.pgpass:/root/.pgpass \ |
||||
|
-v \"$BASE_CHARM_PATH/resources/bin/pg-backup:/usr/sbin/pg-backup\" \ |
||||
|
-v \"$SERVICE_DATASTORE/var/backups/pg:/var/backups/pg\" \ |
||||
|
--entrypoint pg-backup \ |
||||
|
\"$DOCKER_BASE_IMAGE\"" 2>&1 | ts '\%F \%T \%Z' >> /var/log/cron/pg-backup_script.log |
||||
|
EOF |
||||
|
chmod +x "$DST" |
||||
|
|
||||
|
config-add "\ |
||||
|
$MASTER_TARGET_CHARM_NAME: |
||||
|
links: |
||||
|
- $MASTER_BASE_CHARM_NAME |
||||
|
" |
||||
@ -0,0 +1,120 @@ |
|||||
|
# -*- mode: shell-script -*- |
||||
|
|
||||
|
include pretty |
||||
|
|
||||
|
export DB_NAME="postgres" |
||||
|
export DB_DATADIR=/var/lib/postgresql/data |
||||
|
export DB_PASSFILE=/root/.pgpass |
||||
|
|
||||
|
|
||||
|
is_db_locked() { |
||||
|
require lsof || apt-get install -y lsof </dev/null || { |
||||
|
err "Couldn't install command 'lsof'." |
||||
|
return 1 |
||||
|
} |
||||
|
|
||||
|
# "$lsof" +D "$host_db_working_dir" || true |
||||
|
|
||||
|
## We need: |
||||
|
## 1- to protect against set -e |
||||
|
## 2- to detect if errorlevel 1 because lsof failed for good reason: permissions error, directory not found |
||||
|
## 3- to detect if errorlevel 1 because no process found |
||||
|
|
||||
|
[ -d "$host_db_working_dir" ] || { |
||||
|
err "Directory '$host_db_working_dir' not found. Might be normal at this stage." |
||||
|
return 1 |
||||
|
} |
||||
|
|
||||
|
## We choose to ignore (2), and use stdout emptyness to ensure (3) |
||||
|
open_files=$(lsof +D "$host_db_working_dir") || true |
||||
|
|
||||
|
[ "$open_files" ] |
||||
|
} |
||||
|
|
||||
|
|
||||
|
_set_db_params() { |
||||
|
local docker_ip="$1" docker_network="$2" |
||||
|
|
||||
|
export db_docker_opts="-e PGHOST=$docker_ip -e PGUSER=postgres" |
||||
|
export db_cmd_opts= |
||||
|
} |
||||
|
|
||||
|
ddb () { dcmd psql -qAt "$@"; } |
||||
|
|
||||
|
|
||||
|
## |
||||
|
## Entrypoints |
||||
|
## |
||||
|
|
||||
|
|
||||
|
db_drop () { |
||||
|
local dbname="$1" |
||||
|
dcmd dropdb --if-exists "$1" |
||||
|
} |
||||
|
|
||||
|
db_create () { |
||||
|
local dbname="$1" |
||||
|
dcmd createdb "$dbname" || return 1 |
||||
|
info "Database '$dbname' created." |
||||
|
if [ "$POSTGIS" ]; then |
||||
|
ddb -d "$dbname" < <(echo "CREATE EXTENSION postgis; CREATE EXTENSION postgis_topology;") || return 1 |
||||
|
dcmd /bin/bash -c "psql -d '$dbname' -f /usr/share/postgresql/*/contrib/postgis-2.1/legacy.sql" || return 1 |
||||
|
info "Installed postgis extensions on database '$dbname'." |
||||
|
fi |
||||
|
if [ "$UNACCENT" ]; then |
||||
|
ddb -d "$dbname" < <(echo "CREATE EXTENSION IF NOT EXISTS unaccent;") || return 1 |
||||
|
info "Installed unaccent extension on database '$dbname'." |
||||
|
fi |
||||
|
} |
||||
|
|
||||
|
|
||||
|
## XXXvlab: if launched first, it'll fail handling correctly the open/close of docker |
||||
|
db_has_database() { |
||||
|
local dbname="$1" |
||||
|
if [ "$(ddb < <(echo "SELECT 1 FROM pg_database WHERE datname = '$dbname'"))" ]; then |
||||
|
debug "Database $dbname exists." |
||||
|
return 0 |
||||
|
else |
||||
|
debug "Database $dbname exists." |
||||
|
return 1 |
||||
|
fi |
||||
|
} |
||||
|
|
||||
|
## XXXvlab: if launched first, it'll fail handling correctly the open/close of docker |
||||
|
db_has_user() { |
||||
|
local user="$1" users |
||||
|
users=$(ddb < <(echo "select u.usename from pg_catalog.pg_user u")) || { |
||||
|
err "Failed to get user list" |
||||
|
return 1 |
||||
|
} |
||||
|
echo "$users" | grep "^${user}$" >/dev/null 2>&1 |
||||
|
} |
||||
|
|
||||
|
db_create_user() { |
||||
|
local user="$1" password="$2" |
||||
|
## Using this instead of pipes is important so that trap works |
||||
|
ddb < <(echo "CREATE USER $user WITH PASSWORD '$password' CREATEDB NOCREATEROLE;") |
||||
|
} |
||||
|
|
||||
|
db_change_password() { |
||||
|
local user="$1" password="$2" |
||||
|
## Using this instead of pipes is important so that trap works |
||||
|
ddb < <(echo "ALTER USER $user WITH PASSWORD '$password';") |
||||
|
} |
||||
|
|
||||
|
db_grant_rights () { |
||||
|
local dbname="$1" user="$2" |
||||
|
require psql || apt-get install -y postgresql-client </dev/null |
||||
|
require pgm || { |
||||
|
( |
||||
|
cd /opt/apps |
||||
|
git clone https://github.com/0k/pgm.git |
||||
|
ln -sf /opt/apps/pgm/bin/* /usr/local/bin/ |
||||
|
apt-get install -y --force-yes pv buffer < /dev/null |
||||
|
# cd /opt/apps/0k-docker |
||||
|
# git checkout master |
||||
|
) |
||||
|
} |
||||
|
debug PGHOST="$DOCKER_IP" PGUSER=postgres pgm chown "$user" "$dbname" |
||||
|
PGHOST="$DOCKER_IP" PGUSER=postgres prefix_pg_local_command=" " pgm chown "$user" "$dbname" |
||||
|
} |
||||
@ -1,10 +1,5 @@ |
|||||
name: postgres |
|
||||
summary: "Postgres server" |
summary: "Postgres server" |
||||
maintainer: "Valentin Lab <valentin.lab@kalysto.org>" |
maintainer: "Valentin Lab <valentin.lab@kalysto.org>" |
||||
inherit: base-0k |
|
||||
description: | |
|
||||
Postgres server |
|
||||
|
docker-image: docker.0k.io/postgis |
||||
data-resources: |
data-resources: |
||||
- /var/backups/pg |
|
||||
config-resources: |
|
||||
- /etc/postgresql |
|
||||
|
- /var/lib/postgresql/data |
||||
@ -0,0 +1,36 @@ |
|||||
|
#!/bin/bash |
||||
|
|
||||
|
export PGUSER=postgres |
||||
|
|
||||
|
pg_databases() { |
||||
|
echo "SELECT datname FROM pg_catalog.pg_database" | psql -At |
||||
|
} |
||||
|
|
||||
|
dbs=$(pg_databases) |
||||
|
|
||||
|
mkdir -p /var/backups/pg/ |
||||
|
for db in $dbs; do |
||||
|
if [[ "$db" == "template"* || "$db" == "postgres" ]]; then |
||||
|
continue |
||||
|
fi |
||||
|
for exclude_db in $exclude_dbs; do |
||||
|
if [ "$exclude_db" == "$db" ]; then |
||||
|
echo "No backup for database $db as it is listed in 'exclude-dbs'." |
||||
|
continue 2 |
||||
|
fi |
||||
|
done |
||||
|
dst=/var/backups/pg/$db.sql.gz |
||||
|
(( start = SECONDS )) |
||||
|
echo "Dumping database $db..." >&2 |
||||
|
pg_dump -Ox "$db" | gzip --rsyncable > "$dst.inprogress" |
||||
|
errlvl="$?" |
||||
|
## Atomic replace |
||||
|
(( elapsed = SECONDS - start )) |
||||
|
if [ "$errlvl" != "0" ]; then |
||||
|
echo " !! Error when dumping database $db." >&2 |
||||
|
else |
||||
|
echo " ..dumped $db to $dst ($(du -sh "$dst" | cut -f 1) in ${elapsed}s)" >&2 |
||||
|
fi |
||||
|
mv "$dst.inprogress" "$dst" |
||||
|
done |
||||
|
|
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue