9 changed files with 432 additions and 62 deletions
-
70postgres/actions/load
-
55postgres/actions/relations/postgres-database/load-db
-
53postgres/hooks/init
-
55postgres/hooks/install
-
47postgres/hooks/postgres_database-relation-joined
-
49postgres/hooks/schedule_command-relation-joined
-
120postgres/lib/common
-
9postgres/metadata.yml
-
36postgres/resources/bin/pg-backup
@ -0,0 +1,70 @@ |
|||
#!/bin/bash |
|||
|
|||
## Load action gets a first argument a DIRECTORY holding the necessary files. |
|||
## |
|||
## |
|||
|
|||
if [ -z "$SERVICE_DATASTORE" ]; then |
|||
echo "This script is meant to be run through 'compose' to work properly." >&2 |
|||
exit 1 |
|||
fi |
|||
|
|||
usage="$exname [-h|--help] [--with-postgis] [--with-unaccent] SOURCE DBNAME" |
|||
|
|||
dbname= |
|||
postgis= |
|||
while [ "$1" ]; do |
|||
case "$1" in |
|||
"--help"|"-h") |
|||
print_usage |
|||
exit 0 |
|||
;; |
|||
"--with-postgis") |
|||
postgis=true |
|||
;; |
|||
"--with-unaccent") |
|||
unaccent=true |
|||
;; |
|||
*) |
|||
[ -z "$SOURCE" ] && { SOURCE=$1 ; shift ; continue ; } |
|||
[ -z "$dbname" ] && { dbname=$1 ; shift ; continue ; } |
|||
err "Unexpected argument '$1'." |
|||
exit 1 |
|||
;; |
|||
esac |
|||
shift |
|||
done |
|||
|
|||
if [ -z "$SOURCE" ]; then |
|||
err "You must provide a source file as first argument." |
|||
print_usage |
|||
exit 1 |
|||
fi |
|||
|
|||
if [ -z "$dbname" ]; then |
|||
err "You must provide a destination database name as second argument." |
|||
print_usage |
|||
exit 1 |
|||
fi |
|||
|
|||
if [[ "$dbname" == *"@"* ]]; then |
|||
IFS="@" read user dbname < <(echo "$dbname") |
|||
fi |
|||
|
|||
. lib/common |
|||
|
|||
## This can work only if ~/.pgpass is correctly created by init. |
|||
|
|||
set -e |
|||
|
|||
|
|||
db_drop "$dbname" |
|||
POSTGIS=$postgis UNACCENT=$unaccent db_create "$dbname" |
|||
|
|||
|
|||
ddb "$dbname" > /dev/null < <(get_file "$SOURCE") |
|||
|
|||
[ "$user" ] && |
|||
db_grant_rights "$dbname" "$user" |
|||
|
|||
info "Loaded '$SOURCE' into database '$dbname'." |
|||
@ -0,0 +1,55 @@ |
|||
#!/bin/bash |
|||
|
|||
## Load action gets a first argument a FILE/DIRECTORY/URL holding the necessary files. |
|||
## |
|||
## |
|||
|
|||
if [ -z "$SERVICE_DATASTORE" ]; then |
|||
echo "This script is meant to be run through 'compose' to work properly." >&2 |
|||
exit 1 |
|||
fi |
|||
|
|||
usage="$exname [-h|--help] SOURCE" |
|||
|
|||
dbname= |
|||
while [ "$1" ]; do |
|||
case "$1" in |
|||
"--help"|"-h") |
|||
print_usage |
|||
exit 0 |
|||
;; |
|||
*) |
|||
[ -z "$SOURCE" ] && { SOURCE=$1 ; shift ; continue ; } |
|||
err "Unexpected argument '$1'." |
|||
exit 1 |
|||
;; |
|||
esac |
|||
shift |
|||
done |
|||
|
|||
if [ -z "$SOURCE" ]; then |
|||
err "You must provide a source file as first argument." |
|||
print_usage |
|||
exit 1 |
|||
fi |
|||
|
|||
include parse |
|||
include pretty |
|||
|
|||
set -e |
|||
|
|||
USER=$(relation-get user) |
|||
DBNAME=$(relation-get dbname) |
|||
POSTGIS=$(relation-get postgis 2>/dev/null) || true |
|||
UNACCENT=$(relation-get unaccent 2>/dev/null) || true |
|||
|
|||
opts=() |
|||
if [ "$POSTGIS" ]; then |
|||
opts+=("--with-postgis") |
|||
fi |
|||
|
|||
if [ "$UNACCENT" ]; then |
|||
opts+=("--with-unaccent") |
|||
fi |
|||
|
|||
run_service_action "$RELATION_TARGET_CHARM" load "$SOURCE" "$USER@$DBNAME" "${opts[@]}" "$@" |
|||
@ -0,0 +1,53 @@ |
|||
#!/bin/bash |
|||
|
|||
## Init is run on host |
|||
## For now it is run every time the script is launched, but |
|||
## it should be launched only once after build. |
|||
|
|||
## Accessible variables are: |
|||
## - SERVICE_NAME Name of current service |
|||
## - DOCKER_BASE_IMAGE Base image from which this service might be built if any |
|||
## - SERVICE_DATASTORE Location on host of the DATASTORE of this service |
|||
## - SERVICE_CONFIGSTORE Location on host of the CONFIGSTORE of this service |
|||
|
|||
|
|||
|
|||
# Please note that postgres detect on its own if its datadir needs to be populated |
|||
|
|||
|
|||
[ -e ~/.pgpass ] && exit 0 |
|||
|
|||
. lib/common |
|||
|
|||
set -e |
|||
|
|||
POSTGRES_ROOT_PASSWORD="$(gen_password)" |
|||
|
|||
## |
|||
## Setting up access from host |
|||
## |
|||
|
|||
ddb < <(echo "ALTER USER postgres WITH ENCRYPTED password '$POSTGRES_ROOT_PASSWORD'") |
|||
|
|||
sed -ri 's%^host all all 0\.0\.0\.0/0 trust$%host all all 0.0.0.0/0 md5%g' \ |
|||
"$SERVICE_DATASTORE/var/lib/postgresql/data/pg_hba.conf" |
|||
|
|||
docker restart "$container_id" |
|||
|
|||
|
|||
## XXXvlab: this won't help support multiple project running on the |
|||
## same host |
|||
cat <<EOF > ~/.pgpass |
|||
*:*:*:postgres:$POSTGRES_ROOT_PASSWORD |
|||
EOF |
|||
|
|||
chmod 600 ~/.pgpass |
|||
|
|||
## |
|||
## pgm |
|||
## |
|||
|
|||
echo 'prefix_pg_local_command=" " ## otherwise, will default to sudo -u postgres ' > /root/.pgm.rc |
|||
|
|||
info "New root password for postgres. " |
|||
|
|||
@ -1,55 +0,0 @@ |
|||
#!/bin/bash |
|||
|
|||
set -eux # -x for verbose logging to juju debug-log |
|||
|
|||
## 0k git remote options |
|||
GIT_0K_CLONE_OPTIONS=${GIT_0K_CLONE_OPTIONS:-""} |
|||
|
|||
## 0k git remote path |
|||
GIT_0K_BASE=${GIT_0K_BASE:-"git.0k.io:/var/git"} |
|||
|
|||
apt-get install -y --force-yes cron kal-scripts |
|||
|
|||
if [ "$(lsb_release -c -s)" == "trusty" ]; then |
|||
apt-get install -y --force-yes postgresql-9.3 |
|||
else |
|||
apt-get install -y --force-yes postgresql-9.1 |
|||
fi |
|||
|
|||
cat <<EOF > /etc/cron.d/pgbackup |
|||
|
|||
SHELL=/bin/sh |
|||
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin |
|||
|
|||
32 1 * * * root ansi_colors=no dayold=2 nbold=5 pgdump_to_dir /var/backups |
|||
|
|||
EOF |
|||
|
|||
## |
|||
## Install 0k-manage for all the pg_* tools |
|||
## |
|||
|
|||
( |
|||
if ! [ -d "/opt/apps/0k-manage" ]; then |
|||
cd /opt/apps && |
|||
git clone $GIT_0K_CLONE_OPTIONS "$GIT_0K_BASE/0k/0k-manage.git" && |
|||
cd /opt/apps/0k-manage && |
|||
git checkout 0k/prod/master |
|||
fi |
|||
|
|||
ln -sf /opt/apps/0k-manage/src/bin/* /usr/local/bin/ |
|||
) |
|||
|
|||
|
|||
## XXXvlab: Shouldn't we use ldap to create a key for each client and |
|||
## remove the clear password from here ? |
|||
#echo "CREATE USER openerp WITH PASSWORD 'xxxx' CREATEDB NOCREATEROLE;" | sudo -u postgres psql |
|||
|
|||
|
|||
## add this to pghba |
|||
#host all all 172.32.0.0/12 md5 |
|||
#host all all 172.33.0.0/12 md5 |
|||
|
|||
## modify listen_addresses in postgresql.conf |
|||
#listen_addresses='*' |
|||
|
|||
@ -0,0 +1,47 @@ |
|||
#!/bin/bash |
|||
|
|||
## When writing relation script, remember: |
|||
## - they should be idempotents |
|||
## - they can be launched while the dockers is already up |
|||
## - they are launched from the host |
|||
## - the target of the link is launched first, and get a chance to ``relation-set`` |
|||
## - both side of the scripts get to use ``relation-get``. |
|||
|
|||
## could generate this also if not set |
|||
DBNAME=$(relation-get dbname) |
|||
|
|||
[ "$(relation-get password 2>/dev/null)" ] && exit 0 |
|||
|
|||
. lib/common |
|||
|
|||
set -e |
|||
|
|||
USER=$(relation-get user) |
|||
PASSWORD="$(gen_password)" |
|||
POSTGIS=$(relation-get postgis 2>/dev/null) || true |
|||
UNACCENT=$(relation-get unaccent 2>/dev/null) || true |
|||
|
|||
|
|||
ensure_db_docker_running |
|||
|
|||
db_has_database "$DBNAME" || UNACCENT="$UNACCENT" POSTGIS="$POSTGIS" db_create "$DBNAME" |
|||
if ! db_has_user "$USER"; then |
|||
info "Creating a new user $USER." |
|||
db_create_user "$USER" "$PASSWORD" |
|||
else |
|||
info "Updating password of user $USER." |
|||
db_change_password "$USER" "$PASSWORD" |
|||
fi |
|||
db_grant_rights "$DBNAME" "$USER" |
|||
|
|||
pgpass_line="*:*:*:$USER:$PASSWORD" |
|||
pgpass_file="$CONFIGSTORE/$BASE_CHARM_NAME/root/.pgpass" |
|||
|
|||
if [ -e "$pgpass_file" ]; then |
|||
sed -ri "/^.+:.+:.+:$USER:.*$/d" "$pgpass_file" |
|||
fi |
|||
mkdir -p "$(dirname "$pgpass_file")" |
|||
echo "$pgpass_line" >> "$pgpass_file" |
|||
chmod 600 "$pgpass_file" |
|||
|
|||
relation-set password "$PASSWORD" |
|||
@ -0,0 +1,49 @@ |
|||
#!/bin/bash |
|||
|
|||
## When writing relation script, remember: |
|||
## - they should be idempotents |
|||
## - they can be launched while the dockers is already up |
|||
## - they are launched from the host |
|||
## - the target of the link is launched first, and get a chance to ``relation-set`` |
|||
## - both side of the scripts get to use ``relation-get``. |
|||
|
|||
set -e |
|||
|
|||
## XXXvlab: should use container name here so that it could support |
|||
## multiple postgres |
|||
label=pg-backup-$SERVICE_NAME |
|||
DST=$CONFIGSTORE/$TARGET_CHARM_NAME/etc/cron/$label |
|||
schedule=$(relation-get schedule) |
|||
|
|||
if ! echo "$schedule" | egrep '^\s*(([0-9/,*-]+\s+){4,4}[0-9/,*-]+|@[a-z]+)\s*$' >/dev/null 2>&1; then |
|||
err "Unrecognized schedule '$schedule'." |
|||
exit 1 |
|||
fi |
|||
|
|||
## XXXvlab: how is this meant to works if there's multiples postgres (about PGHOST=?) ? |
|||
## Warning: using '\' in heredoc will be removed in the final cron file, which |
|||
## is totally wanted: cron does not support multilines. |
|||
|
|||
exclude_dbs=$(relation-get exclude-dbs 2>/dev/null) || true |
|||
exclude_dbs=$(echo "$exclude_dbs" | shyaml get-values 2>/dev/null | xargs echo) || true |
|||
|
|||
## Warning: 'docker -v' will use HOST directory even if launched from |
|||
## 'cron' container. |
|||
file_put "$DST" <<EOF |
|||
$schedule root lock $label -D -p 10 -c "\ |
|||
docker run --rm \ |
|||
-e PGHOST=\$POSTGRES_PORT_5432_TCP_ADDR \ |
|||
-e exclude_dbs=\"$exclude_dbs\" \ |
|||
-v /root/.pgpass:/root/.pgpass \ |
|||
-v \"$BASE_CHARM_PATH/resources/bin/pg-backup:/usr/sbin/pg-backup\" \ |
|||
-v \"$SERVICE_DATASTORE/var/backups/pg:/var/backups/pg\" \ |
|||
--entrypoint pg-backup \ |
|||
\"$DOCKER_BASE_IMAGE\"" 2>&1 | ts '\%F \%T \%Z' >> /var/log/cron/pg-backup_script.log |
|||
EOF |
|||
chmod +x "$DST" |
|||
|
|||
config-add "\ |
|||
$MASTER_TARGET_CHARM_NAME: |
|||
links: |
|||
- $MASTER_BASE_CHARM_NAME |
|||
" |
|||
@ -0,0 +1,120 @@ |
|||
# -*- mode: shell-script -*- |
|||
|
|||
include pretty |
|||
|
|||
export DB_NAME="postgres" |
|||
export DB_DATADIR=/var/lib/postgresql/data |
|||
export DB_PASSFILE=/root/.pgpass |
|||
|
|||
|
|||
is_db_locked() { |
|||
require lsof || apt-get install -y lsof </dev/null || { |
|||
err "Couldn't install command 'lsof'." |
|||
return 1 |
|||
} |
|||
|
|||
# "$lsof" +D "$host_db_working_dir" || true |
|||
|
|||
## We need: |
|||
## 1- to protect against set -e |
|||
## 2- to detect if errorlevel 1 because lsof failed for good reason: permissions error, directory not found |
|||
## 3- to detect if errorlevel 1 because no process found |
|||
|
|||
[ -d "$host_db_working_dir" ] || { |
|||
err "Directory '$host_db_working_dir' not found. Might be normal at this stage." |
|||
return 1 |
|||
} |
|||
|
|||
## We choose to ignore (2), and use stdout emptyness to ensure (3) |
|||
open_files=$(lsof +D "$host_db_working_dir") || true |
|||
|
|||
[ "$open_files" ] |
|||
} |
|||
|
|||
|
|||
_set_db_params() { |
|||
local docker_ip="$1" docker_network="$2" |
|||
|
|||
export db_docker_opts="-e PGHOST=$docker_ip -e PGUSER=postgres" |
|||
export db_cmd_opts= |
|||
} |
|||
|
|||
ddb () { dcmd psql -qAt "$@"; } |
|||
|
|||
|
|||
## |
|||
## Entrypoints |
|||
## |
|||
|
|||
|
|||
db_drop () { |
|||
local dbname="$1" |
|||
dcmd dropdb --if-exists "$1" |
|||
} |
|||
|
|||
db_create () { |
|||
local dbname="$1" |
|||
dcmd createdb "$dbname" || return 1 |
|||
info "Database '$dbname' created." |
|||
if [ "$POSTGIS" ]; then |
|||
ddb -d "$dbname" < <(echo "CREATE EXTENSION postgis; CREATE EXTENSION postgis_topology;") || return 1 |
|||
dcmd /bin/bash -c "psql -d '$dbname' -f /usr/share/postgresql/*/contrib/postgis-2.1/legacy.sql" || return 1 |
|||
info "Installed postgis extensions on database '$dbname'." |
|||
fi |
|||
if [ "$UNACCENT" ]; then |
|||
ddb -d "$dbname" < <(echo "CREATE EXTENSION IF NOT EXISTS unaccent;") || return 1 |
|||
info "Installed unaccent extension on database '$dbname'." |
|||
fi |
|||
} |
|||
|
|||
|
|||
## XXXvlab: if launched first, it'll fail handling correctly the open/close of docker |
|||
db_has_database() { |
|||
local dbname="$1" |
|||
if [ "$(ddb < <(echo "SELECT 1 FROM pg_database WHERE datname = '$dbname'"))" ]; then |
|||
debug "Database $dbname exists." |
|||
return 0 |
|||
else |
|||
debug "Database $dbname exists." |
|||
return 1 |
|||
fi |
|||
} |
|||
|
|||
## XXXvlab: if launched first, it'll fail handling correctly the open/close of docker |
|||
db_has_user() { |
|||
local user="$1" users |
|||
users=$(ddb < <(echo "select u.usename from pg_catalog.pg_user u")) || { |
|||
err "Failed to get user list" |
|||
return 1 |
|||
} |
|||
echo "$users" | grep "^${user}$" >/dev/null 2>&1 |
|||
} |
|||
|
|||
db_create_user() { |
|||
local user="$1" password="$2" |
|||
## Using this instead of pipes is important so that trap works |
|||
ddb < <(echo "CREATE USER $user WITH PASSWORD '$password' CREATEDB NOCREATEROLE;") |
|||
} |
|||
|
|||
db_change_password() { |
|||
local user="$1" password="$2" |
|||
## Using this instead of pipes is important so that trap works |
|||
ddb < <(echo "ALTER USER $user WITH PASSWORD '$password';") |
|||
} |
|||
|
|||
db_grant_rights () { |
|||
local dbname="$1" user="$2" |
|||
require psql || apt-get install -y postgresql-client </dev/null |
|||
require pgm || { |
|||
( |
|||
cd /opt/apps |
|||
git clone https://github.com/0k/pgm.git |
|||
ln -sf /opt/apps/pgm/bin/* /usr/local/bin/ |
|||
apt-get install -y --force-yes pv buffer < /dev/null |
|||
# cd /opt/apps/0k-docker |
|||
# git checkout master |
|||
) |
|||
} |
|||
debug PGHOST="$DOCKER_IP" PGUSER=postgres pgm chown "$user" "$dbname" |
|||
PGHOST="$DOCKER_IP" PGUSER=postgres prefix_pg_local_command=" " pgm chown "$user" "$dbname" |
|||
} |
|||
@ -1,10 +1,5 @@ |
|||
name: postgres |
|||
summary: "Postgres server" |
|||
maintainer: "Valentin Lab <valentin.lab@kalysto.org>" |
|||
inherit: base-0k |
|||
description: | |
|||
Postgres server |
|||
docker-image: docker.0k.io/postgis |
|||
data-resources: |
|||
- /var/backups/pg |
|||
config-resources: |
|||
- /etc/postgresql |
|||
- /var/lib/postgresql/data |
|||
@ -0,0 +1,36 @@ |
|||
#!/bin/bash |
|||
|
|||
export PGUSER=postgres |
|||
|
|||
pg_databases() { |
|||
echo "SELECT datname FROM pg_catalog.pg_database" | psql -At |
|||
} |
|||
|
|||
dbs=$(pg_databases) |
|||
|
|||
mkdir -p /var/backups/pg/ |
|||
for db in $dbs; do |
|||
if [[ "$db" == "template"* || "$db" == "postgres" ]]; then |
|||
continue |
|||
fi |
|||
for exclude_db in $exclude_dbs; do |
|||
if [ "$exclude_db" == "$db" ]; then |
|||
echo "No backup for database $db as it is listed in 'exclude-dbs'." |
|||
continue 2 |
|||
fi |
|||
done |
|||
dst=/var/backups/pg/$db.sql.gz |
|||
(( start = SECONDS )) |
|||
echo "Dumping database $db..." >&2 |
|||
pg_dump -Ox "$db" | gzip --rsyncable > "$dst.inprogress" |
|||
errlvl="$?" |
|||
## Atomic replace |
|||
(( elapsed = SECONDS - start )) |
|||
if [ "$errlvl" != "0" ]; then |
|||
echo " !! Error when dumping database $db." >&2 |
|||
else |
|||
echo " ..dumped $db to $dst ($(du -sh "$dst" | cut -f 1) in ${elapsed}s)" >&2 |
|||
fi |
|||
mv "$dst.inprogress" "$dst" |
|||
done |
|||
|
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue