diff --git a/postgres/actions/load b/postgres/actions/load new file mode 100755 index 00000000..199b02c6 --- /dev/null +++ b/postgres/actions/load @@ -0,0 +1,70 @@ +#!/bin/bash + +## Load action gets a first argument a DIRECTORY holding the necessary files. +## +## + +if [ -z "$SERVICE_DATASTORE" ]; then + echo "This script is meant to be run through 'compose' to work properly." >&2 + exit 1 +fi + +usage="$exname [-h|--help] [--with-postgis] [--with-unaccent] SOURCE DBNAME" + +dbname= +postgis= +while [ "$1" ]; do + case "$1" in + "--help"|"-h") + print_usage + exit 0 + ;; + "--with-postgis") + postgis=true + ;; + "--with-unaccent") + unaccent=true + ;; + *) + [ -z "$SOURCE" ] && { SOURCE=$1 ; shift ; continue ; } + [ -z "$dbname" ] && { dbname=$1 ; shift ; continue ; } + err "Unexpected argument '$1'." + exit 1 + ;; + esac + shift +done + +if [ -z "$SOURCE" ]; then + err "You must provide a source file as first argument." + print_usage + exit 1 +fi + +if [ -z "$dbname" ]; then + err "You must provide a destination database name as second argument." + print_usage + exit 1 +fi + +if [[ "$dbname" == *"@"* ]]; then + IFS="@" read user dbname < <(echo "$dbname") +fi + +. lib/common + +## This can work only if ~/.pgpass is correctly created by init. + +set -e + + +db_drop "$dbname" +POSTGIS=$postgis UNACCENT=$unaccent db_create "$dbname" + + +ddb "$dbname" > /dev/null < <(get_file "$SOURCE") + +[ "$user" ] && + db_grant_rights "$dbname" "$user" + +info "Loaded '$SOURCE' into database '$dbname'." diff --git a/postgres/actions/relations/postgres-database/load-db b/postgres/actions/relations/postgres-database/load-db new file mode 100755 index 00000000..353ae0e5 --- /dev/null +++ b/postgres/actions/relations/postgres-database/load-db @@ -0,0 +1,55 @@ +#!/bin/bash + +## Load action gets a first argument a FILE/DIRECTORY/URL holding the necessary files. +## +## + +if [ -z "$SERVICE_DATASTORE" ]; then + echo "This script is meant to be run through 'compose' to work properly." >&2 + exit 1 +fi + +usage="$exname [-h|--help] SOURCE" + +dbname= +while [ "$1" ]; do + case "$1" in + "--help"|"-h") + print_usage + exit 0 + ;; + *) + [ -z "$SOURCE" ] && { SOURCE=$1 ; shift ; continue ; } + err "Unexpected argument '$1'." + exit 1 + ;; + esac + shift +done + +if [ -z "$SOURCE" ]; then + err "You must provide a source file as first argument." + print_usage + exit 1 +fi + +include parse +include pretty + +set -e + +USER=$(relation-get user) +DBNAME=$(relation-get dbname) +POSTGIS=$(relation-get postgis 2>/dev/null) || true +UNACCENT=$(relation-get unaccent 2>/dev/null) || true + +opts=() +if [ "$POSTGIS" ]; then + opts+=("--with-postgis") +fi + +if [ "$UNACCENT" ]; then + opts+=("--with-unaccent") +fi + +run_service_action "$RELATION_TARGET_CHARM" load "$SOURCE" "$USER@$DBNAME" "${opts[@]}" "$@" diff --git a/postgres/hooks/init b/postgres/hooks/init new file mode 100755 index 00000000..bc12e68d --- /dev/null +++ b/postgres/hooks/init @@ -0,0 +1,53 @@ +#!/bin/bash + +## Init is run on host +## For now it is run every time the script is launched, but +## it should be launched only once after build. + +## Accessible variables are: +## - SERVICE_NAME Name of current service +## - DOCKER_BASE_IMAGE Base image from which this service might be built if any +## - SERVICE_DATASTORE Location on host of the DATASTORE of this service +## - SERVICE_CONFIGSTORE Location on host of the CONFIGSTORE of this service + + + +# Please note that postgres detect on its own if its datadir needs to be populated + + +[ -e ~/.pgpass ] && exit 0 + +. lib/common + +set -e + +POSTGRES_ROOT_PASSWORD="$(gen_password)" + +## +## Setting up access from host +## + +ddb < <(echo "ALTER USER postgres WITH ENCRYPTED password '$POSTGRES_ROOT_PASSWORD'") + +sed -ri 's%^host all all 0\.0\.0\.0/0 trust$%host all all 0.0.0.0/0 md5%g' \ + "$SERVICE_DATASTORE/var/lib/postgresql/data/pg_hba.conf" + +docker restart "$container_id" + + +## XXXvlab: this won't help support multiple project running on the +## same host +cat < ~/.pgpass +*:*:*:postgres:$POSTGRES_ROOT_PASSWORD +EOF + +chmod 600 ~/.pgpass + +## +## pgm +## + +echo 'prefix_pg_local_command=" " ## otherwise, will default to sudo -u postgres ' > /root/.pgm.rc + +info "New root password for postgres. " + diff --git a/postgres/hooks/install b/postgres/hooks/install deleted file mode 100755 index ff6178a6..00000000 --- a/postgres/hooks/install +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -set -eux # -x for verbose logging to juju debug-log - -## 0k git remote options -GIT_0K_CLONE_OPTIONS=${GIT_0K_CLONE_OPTIONS:-""} - -## 0k git remote path -GIT_0K_BASE=${GIT_0K_BASE:-"git.0k.io:/var/git"} - -apt-get install -y --force-yes cron kal-scripts - -if [ "$(lsb_release -c -s)" == "trusty" ]; then - apt-get install -y --force-yes postgresql-9.3 -else - apt-get install -y --force-yes postgresql-9.1 -fi - -cat < /etc/cron.d/pgbackup - -SHELL=/bin/sh -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -32 1 * * * root ansi_colors=no dayold=2 nbold=5 pgdump_to_dir /var/backups - -EOF - -## -## Install 0k-manage for all the pg_* tools -## - -( - if ! [ -d "/opt/apps/0k-manage" ]; then - cd /opt/apps && - git clone $GIT_0K_CLONE_OPTIONS "$GIT_0K_BASE/0k/0k-manage.git" && - cd /opt/apps/0k-manage && - git checkout 0k/prod/master - fi - - ln -sf /opt/apps/0k-manage/src/bin/* /usr/local/bin/ -) - - -## XXXvlab: Shouldn't we use ldap to create a key for each client and -## remove the clear password from here ? -#echo "CREATE USER openerp WITH PASSWORD 'xxxx' CREATEDB NOCREATEROLE;" | sudo -u postgres psql - - -## add this to pghba -#host all all 172.32.0.0/12 md5 -#host all all 172.33.0.0/12 md5 - -## modify listen_addresses in postgresql.conf -#listen_addresses='*' - diff --git a/postgres/hooks/postgres_database-relation-joined b/postgres/hooks/postgres_database-relation-joined new file mode 100755 index 00000000..950a8734 --- /dev/null +++ b/postgres/hooks/postgres_database-relation-joined @@ -0,0 +1,47 @@ +#!/bin/bash + +## When writing relation script, remember: +## - they should be idempotents +## - they can be launched while the dockers is already up +## - they are launched from the host +## - the target of the link is launched first, and get a chance to ``relation-set`` +## - both side of the scripts get to use ``relation-get``. + +## could generate this also if not set +DBNAME=$(relation-get dbname) + +[ "$(relation-get password 2>/dev/null)" ] && exit 0 + +. lib/common + +set -e + +USER=$(relation-get user) +PASSWORD="$(gen_password)" +POSTGIS=$(relation-get postgis 2>/dev/null) || true +UNACCENT=$(relation-get unaccent 2>/dev/null) || true + + +ensure_db_docker_running + +db_has_database "$DBNAME" || UNACCENT="$UNACCENT" POSTGIS="$POSTGIS" db_create "$DBNAME" +if ! db_has_user "$USER"; then + info "Creating a new user $USER." + db_create_user "$USER" "$PASSWORD" +else + info "Updating password of user $USER." + db_change_password "$USER" "$PASSWORD" +fi +db_grant_rights "$DBNAME" "$USER" + +pgpass_line="*:*:*:$USER:$PASSWORD" +pgpass_file="$CONFIGSTORE/$BASE_CHARM_NAME/root/.pgpass" + +if [ -e "$pgpass_file" ]; then + sed -ri "/^.+:.+:.+:$USER:.*$/d" "$pgpass_file" +fi +mkdir -p "$(dirname "$pgpass_file")" +echo "$pgpass_line" >> "$pgpass_file" +chmod 600 "$pgpass_file" + +relation-set password "$PASSWORD" \ No newline at end of file diff --git a/postgres/hooks/schedule_command-relation-joined b/postgres/hooks/schedule_command-relation-joined new file mode 100755 index 00000000..68cecb97 --- /dev/null +++ b/postgres/hooks/schedule_command-relation-joined @@ -0,0 +1,49 @@ +#!/bin/bash + +## When writing relation script, remember: +## - they should be idempotents +## - they can be launched while the dockers is already up +## - they are launched from the host +## - the target of the link is launched first, and get a chance to ``relation-set`` +## - both side of the scripts get to use ``relation-get``. + +set -e + +## XXXvlab: should use container name here so that it could support +## multiple postgres +label=pg-backup-$SERVICE_NAME +DST=$CONFIGSTORE/$TARGET_CHARM_NAME/etc/cron/$label +schedule=$(relation-get schedule) + +if ! echo "$schedule" | egrep '^\s*(([0-9/,*-]+\s+){4,4}[0-9/,*-]+|@[a-z]+)\s*$' >/dev/null 2>&1; then + err "Unrecognized schedule '$schedule'." + exit 1 +fi + +## XXXvlab: how is this meant to works if there's multiples postgres (about PGHOST=?) ? +## Warning: using '\' in heredoc will be removed in the final cron file, which +## is totally wanted: cron does not support multilines. + +exclude_dbs=$(relation-get exclude-dbs 2>/dev/null) || true +exclude_dbs=$(echo "$exclude_dbs" | shyaml get-values 2>/dev/null | xargs echo) || true + +## Warning: 'docker -v' will use HOST directory even if launched from +## 'cron' container. +file_put "$DST" <&1 | ts '\%F \%T \%Z' >> /var/log/cron/pg-backup_script.log +EOF +chmod +x "$DST" + +config-add "\ +$MASTER_TARGET_CHARM_NAME: + links: + - $MASTER_BASE_CHARM_NAME +" diff --git a/postgres/lib/common b/postgres/lib/common new file mode 100644 index 00000000..d8139cdc --- /dev/null +++ b/postgres/lib/common @@ -0,0 +1,120 @@ +# -*- mode: shell-script -*- + +include pretty + +export DB_NAME="postgres" +export DB_DATADIR=/var/lib/postgresql/data +export DB_PASSFILE=/root/.pgpass + + +is_db_locked() { + require lsof || apt-get install -y lsof /dev/null 2>&1 +} + +db_create_user() { + local user="$1" password="$2" + ## Using this instead of pipes is important so that trap works + ddb < <(echo "CREATE USER $user WITH PASSWORD '$password' CREATEDB NOCREATEROLE;") +} + +db_change_password() { + local user="$1" password="$2" + ## Using this instead of pipes is important so that trap works + ddb < <(echo "ALTER USER $user WITH PASSWORD '$password';") +} + +db_grant_rights () { + local dbname="$1" user="$2" + require psql || apt-get install -y postgresql-client " -inherit: base-0k -description: | - Postgres server +docker-image: docker.0k.io/postgis data-resources: - - /var/backups/pg -config-resources: - - /etc/postgresql + - /var/lib/postgresql/data diff --git a/postgres/resources/bin/pg-backup b/postgres/resources/bin/pg-backup new file mode 100755 index 00000000..daeffea1 --- /dev/null +++ b/postgres/resources/bin/pg-backup @@ -0,0 +1,36 @@ +#!/bin/bash + +export PGUSER=postgres + +pg_databases() { + echo "SELECT datname FROM pg_catalog.pg_database" | psql -At +} + +dbs=$(pg_databases) + +mkdir -p /var/backups/pg/ +for db in $dbs; do + if [[ "$db" == "template"* || "$db" == "postgres" ]]; then + continue + fi + for exclude_db in $exclude_dbs; do + if [ "$exclude_db" == "$db" ]; then + echo "No backup for database $db as it is listed in 'exclude-dbs'." + continue 2 + fi + done + dst=/var/backups/pg/$db.sql.gz + (( start = SECONDS )) + echo "Dumping database $db..." >&2 + pg_dump -Ox "$db" | gzip --rsyncable > "$dst.inprogress" + errlvl="$?" + ## Atomic replace + (( elapsed = SECONDS - start )) + if [ "$errlvl" != "0" ]; then + echo " !! Error when dumping database $db." >&2 + else + echo " ..dumped $db to $dst ($(du -sh "$dst" | cut -f 1) in ${elapsed}s)" >&2 + fi + mv "$dst.inprogress" "$dst" +done +