new: [mysql,rsync-backup] added host install script

Signed-off-by: Valentin Lab <valentin.lab@kalysto.org>

mysql/hooks/install.d/60-backup.sh

@@ -0,0 +1,86 @@
+
+set -eux  ## important for unbound variable ?
+
+## Require these to be set
+# MYSQL_ROOT_PASSWORD=
+# MYSQL_CONTAINER=
+
+[ "${MYSQL_ROOT_PASSWORD}" ] || {
+    echo "Error: you must set \$MYSQL_ROOT_PASSWORD prior to running this script." >&2
+    exit 1
+}
+
+[ "${MYSQL_CONTAINER}" ] || {
+    echo "Error: you must set \$MYSQL_CONTAINER prior to running this script." >&2
+    exit 1
+}
+
+
+##
+## Init, to setup passwordless connection to mysql
+##
+
+type -p mysql >/dev/null || apt-get install -y mysql-client </dev/null
+
+if ! [ -e "/root/.my.cnf" ]; then
+    cat <<EOF > ~/.my.cnf
+[client]
+password=${MYSQL_ROOT_PASSWORD}
+EOF
+    chmod 600 ~/.my.cnf
+fi
+
+##
+## installation of the mysql-backup script
+##
+
+
+apt-get install -y kal-shlib-{core,pretty,common} </dev/null
+cp -f resources/bin/mysql-backup /usr/local/sbin/mysql-backup
+
+
+##
+## Connection to cron
+##
+
+
+depends cron
+cat <<EOF > /etc/cron.d/mysql-backup
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+0 * * * *   root   /usr/local/sbin/mysql-backup --host \$(docker-ip "$MYSQL_CONTAINER" 2>/dev/null | sed -r 's/ +/ /g' | cut -f 3 -d " ") | logger -t mysql-backup
+
+EOF
+
+
+##
+## Connection with backup
+##
+
+if type -p mirror-dir >/dev/null 2>&1; then
+    [ -d "/etc/mirror-dir" ] || {
+        echo "'mirror-dir' is installed but no '/etc/mirror-dir' was found." >&2
+        exit 1
+    }
+    depends shyaml
+
+    if ! sources=$(shyaml get-values default.sources < /etc/mirror-dir/config.yml); then
+        echo "Couldn't query 'default.sources' in '/etc/mirror-dir/config.yml'." >&2
+        exit 1
+    fi
+
+    if ! echo "$sources" | grep "^/var/backups/mysql$" 2>/dev/null; then
+        sed -i '/sources:/a\  - /var/backups/mysql' /etc/mirror-dir/config.yml
+        cat <<EOF >> /etc/mirror-dir/config.yml
+/var/backups/mysql:
+  exclude:
+    - "/*.inprogress"
+EOF
+    fi
+else
+    echo "warn: 'mirror-dir' not installed, backup won't be sent" >&2
+fi
+
+
+

mysql/resources/bin/mysql-backup

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+. /etc/shlib
+
+include common
+include pretty
+
+usage="$exname [--host HOST] [DATABASE...]"
+
+
+DBS=()
+host=
+while [ "$1" ]; do
+    case "$1" in
+        "--help"|"-h")
+            print_usage
+            exit 0
+        ;;
+        "--host")
+            host="$2"
+            shift
+        ;;
+        *)
+            DBS+=("$1")
+        ;;
+    esac
+    shift
+done
+
+
+mysql_opts=()
+if [ "$host" ]; then
+    mysql_opts+=(-h "$host")
+fi
+
+m() {
+    mysql "${mysql_opts[@]}" -Bs "$@"
+}
+
+md() {
+    mysqldump "${mysql_opts[@]}" "$@"
+}
+
+mysql_databases() {
+     echo "SHOW DATABASES" | m
+}
+
+mysql_tables() {
+    local db="$1"
+     echo "SHOW TABLES" | m "$db"
+}
+
+
+if [ "${#DBS[@]}" == 0 ]; then
+    DBS=($(mysql_databases)) || exit 1
+fi
+
+mkdir -p /var/backups/mysql
+
+for db in "${DBS[@]}"; do
+    if [[ "$db" == "information_schema" || "$db" == "performance_schema" || "$db" == "mysql" ]]; then
+        continue
+    fi
+    echo "Dumping database $db..." >&2
+    # omitting all the rotation logic
+    dst=/"var/backups/mysql/$db"
+    [ -d "$dst.old" ] && rm -rf "$dst.old"
+    [ -d "$dst" ] && mv "$dst" "$dst.old"
+    mkdir -p "$dst.inprogress"
+    (( start = SECONDS ))
+    md "$db" --routines --no-data --add-drop-database --database "$db" | gzip --rsyncable > "$dst.inprogress/schema.sql.gz"
+    tables=$(mysql_tables "$db")
+    for table in $tables; do
+        backup_file="$dst.inprogress/${table}.sql.gz"
+        echo "  Dumping $table into ${backup_file}"
+        md "$db" "$table" | gzip --rsyncable > "$backup_file" || break
+    done
+    mv "$dst.inprogress" "$dst"
+    [ -d "$dst.old" ] && rm -rf "$dst.old"
+    (( elapsed = SECONDS - start ))
+    echo "  ..dumped $db to $dst ($(du -sh "$dst" | cut -f 1) in ${elapsed}s)" >&2
+done

rsync-backup-target/resources/bin/compose-add-rsync-key

@@ -0,0 +1,144 @@
+#!/bin/bash
+
+
+. /etc/shlib
+
+[[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true
+
+include common
+include pretty
+include cmdline
+
+desc='Adds YAML key quite crudely in given compose.yml.'
+help="\
+
+$WHITE$exname$NORMAL will find a add or replace SSH key to
+given identifier for rsync-backup-target charm.
+
+For now, it only use detection of '## INSERTION-POINT' to manage
+edition of 'compose.yml', and verification it didn't break anything
+before overwriting.
+
+"
+
+
+[ "$SOURCED" ] && return 0
+
+##
+## Command line processing
+##
+
+
+# remove all lines from regex to regex (not included).
+remove_lines_from_to() {
+    local from="$1" to="$2"
+    sed -r "/$from/,/$to/{/$to/"'!'"d};/$from/d"
+}
+
+check_valid_yaml() {
+    shyaml get-value >/dev/null 2>&1;
+}
+
+
+cmdline.spec.gnu
+cmdline.spec.reporting
+
+service_name=${SERVICE_NAME:-rsync-backup-target}
+compose_file=${COMPOSE_FILE:-/etc/compose/compose.yml}
+
+cmdline.spec::valued:--compose-file,-f:usage() {
+    echo "Compose file location. Defaults to '/etc/compose/compose.yml'"; }
+cmdline.spec::valued:--compose-file,-f:run() { compose_file="$1"; }
+
+cmdline.spec::valued:--service-name,-s:usage() {
+    echo "YAML service name in compose file to check for existence of key. Defaults to 'rsync-backup-target'"; }
+cmdline.spec::valued:--service-name,-s:run() { service_name="$1"; }
+
+cmdline.spec::cmd:__main__:run() {
+
+    : :posarg: DOMAIN           'domain identifier'
+    : :posarg: SSH_PUBLIC_KEY   'ssh public key'
+
+    if ! existing_domains=$(shyaml keys "${service_name//./\\.}.options.keys" < "$compose_file"); then
+        err "Couldn't query file '$compose_file' for keys of" \
+            "service ${DARKYELLOW}${service_name}${NORMAL}."
+        exit 1
+    fi
+
+    content=$(cat "$compose_file")
+    if echo "$existing_domains" | grep "^${DOMAIN}$" >/dev/null 2>&1; then
+
+        if ! prev_key=$(shyaml get-value "${service_name//./\\.}.options.keys.${DOMAIN//./\\.}" \
+                        < "$compose_file"); then
+            err "Couldn't query file '$compose_file' for key of domain '$DOMAIN'."
+            exit 1
+        fi
+
+        if [ "${prev_key}" == "$SSH_PUBLIC_KEY" ]; then
+            echo "Key was already setup."
+            exit 0
+        fi
+
+        content=$(echo "$content" | remove_lines_from_to '^      '"${DOMAIN//./\\.}"': ".*\\$' \
+                                                         '^      ([A-Za-z0-9.-]+: "|## END MARKER)')
+
+        if [ -z "$content" ]; then
+            err "Didn't manage to remove key to compose file '$DOMAIN' in '$compose_file'."
+            exit 1
+        fi
+
+        if [ "$content" == "$(cat "$compose_file")" ]; then
+            err "Couldn't remove previous key for '$DOMAIN' in '$compose_file'."
+            exit 1
+        fi
+        ## check we didn't break yaml
+        if ! echo "$content" | check_valid_yaml; then
+            err "Couldn't safely remove previous key for '$DOMAIN' in '$compose_file'."
+            exit 1
+        fi
+    fi
+
+    excerpt=$(cat <<EOF
+      $DOMAIN: "$(
+    echo "$SSH_PUBLIC_KEY" |
+        fold -w 67 | sed -r 's/$/\\/g;2,$ s/^/        /g;$,$ s/\\$//g')"
+
+EOF
+           )
+    excerpt="${excerpt//\\/\\\\\\}"
+    content="$(echo "$content" | sed -r '/^      ## INSERTION-POINT/a\'"${excerpt}")"
+
+    if [ -z "$content" ]; then
+        err "Didn't manage to add key to compose file '$DOMAIN' in '$compose_file'."
+        exit 1
+    fi
+
+    if ! echo "$content" | check_valid_yaml; then
+        err "Couldn't safely add key to compose file '$DOMAIN' in '$compose_file'."
+        exit 1
+    fi
+
+    if diff=$(diff -u "$compose_file" <(echo "$content")); then
+        err "No difference, this is not expected, and something went wrong."
+        exit 1
+    fi
+    echo "${WHITE}Applying these changes:${NORMAL}"
+    echo "$diff"
+
+    cp "$compose_file" "${compose_file}.old"
+    echo "$content" > "$compose_file"
+
+    ## reloading (could be much faster)
+    compose --debug down && compose --debug up
+
+    if [ "$?" == 0 ]; then
+        echo "Added key, and restarted service ${DARKYELLOW}$service_name${NORMAL}."
+    else
+        echo "something went wrong ! Should check the state of '$DOMAIN' !!"
+        exit 1
+    fi
+}
+
+
+cmdline::parse "$@"
+

rsync-backup/hooks/install.d/60-install.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+
+set -eux
+
+
+[ "${DOMAIN}" ] || {
+    echo "Error: you must set \$DOMAIN prior to running this script." >&2
+    exit 1
+}
+
+[ "${BACKUP_SERVER}" ] || {
+    echo "Error: you must set \$BACKUP_SERVER prior to running this script." >&2
+    exit 1
+}
+
+
+## rsync
+type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null
+
+## creating rsync user
+mkdir -p /var/lib/rsync
+
+getent group rsync >/dev/null ||
+    groupadd -r rsync
+
+getent passwd rsync >/dev/null ||
+    useradd -r rsync -d /var/lib/rsync -g rsync
+
+chown rsync:rsync /var/lib/rsync
+
+## rsync ssh key creation
+[ -e /var/lib/rsync/.ssh/id_rsa ] ||
+    su -c 'ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa -q -C rsync@'"$DOMAIN" - rsync
+
+dest="$BACKUP_SERVER"
+if [[ "$dest" == *"/"* ]]; then
+    dest="${dest%/*}"
+fi
+
+if [[ "$dest" == *":"* ]]; then
+    ssh_options+=("-p" "${dest#*:}")
+    dest="${dest%%:*}"
+fi
+
+ssh-keyscan "${ssh_options[@]}" -H "${dest}" > /var/lib/rsync/.ssh/known_hosts
+
+apt-get install kal-shlib-process </dev/null
+ln -sf "$PWD/resources/bin/mirror-dir" /usr/local/sbin/mirror-dir
+
+
+
+if ! [ -e /etc/mirror-dir/config.yml ]; then
+    mkdir -p /etc/mirror-dir
+    cat <<EOF > /etc/mirror-dir/config.yml
+default:
+  sources:
+  - /etc
+EOF
+fi
+
+cat <<EOF > /etc/cron.d/mirror-dir
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+$((RANDOM % 60)) * * * *   root   mirror-dir -h "$DOMAIN" -d "$BACKUP_SERVER" -u rsync 2>&1 | logger -t mirror-dir
+
+EOF

rsync-backup/resources/bin/mirror-dir

@@ -0,0 +1,196 @@
+#!/bin/bash
+
+
+#:-
+. /etc/shlib
+#:-
+
+include common
+include parse
+include process
+
+depends shyaml lock
+
+[ "$UID" != "0" ] && echo "You must be root." && exit 1
+
+##
+## Here's an example crontab:
+##
+##   SHELL=/bin/sh
+##   PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+##
+##   49 */2 * * *   root   mirror-dir -d core-05.0k.io:10023 -u rsync /etc /home /opt/apps 2>&1 | logger -t mirror-dir
+##
+
+
+usage="usage: $exname -d DEST1 [-d DEST2 [...]] [-u USER] [DIR1 [DIR2 ...]]
+
+Preserve as much as possible the source structure, keeping hard-links, acl,
+exact numerical uids and gids, and being able to resume in very large files.
+
+Options:
+     DIR1 ... DIRn
+	        Local directories that should be mirrored on destination(s).
+
+	        examples: /etc /home /var/backups
+
+            If no directories are provided, the config file root
+            entries will be used all as destination to copy.
+
+     -d DESTn
+            Can be repeated. Specifies host destination towards which
+        	files will be send. Note that you can specify port number after
+            a colon and a bandwidth limit for rsync after a '/'.
+
+        	examples: -d liszt.musicalta:10022 -d 10.8.0.19/200
+
+     -u USER (default: 'backuppc')
+
+        	Local AND destination user to log as at both ends to transfer file.
+        	This local user need to have a NOPASSWD ssh login towards it's
+        	account on destination. This destination account should have
+        	full permissions access without passwd to write with rsync-server
+        	in the destination directory.
+
+     -h STORE (default is taken of the hostname file)
+
+            Set the destination store, this is the name of the directory where
+            the files will all directories will be copied. Beware ! if 2 hosts
+            use the same store, this means they'll conflictingly update the
+            same destination directory. Only use this if you know what you
+            are doing.
+
+"
+
+dests=()
+source_dirs=()
+hostname=
+while [ "$#" != 0 ]; do
+    case "$1" in
+	    "-d")
+		    dests+=("$2")
+		    shift
+	        ;;
+        "-h")
+            hostname="$2"
+            shift
+            ;;
+	    "-u")
+	        user="$2"
+	        shift
+            ;;
+	    *)
+            source_dirs+=("$1")
+	        ;;
+    esac
+    shift
+done
+
+
+if test -z "$hostname"; then
+    hostname=$(hostname)
+fi
+
+if test -z "$hostname"; then
+    die "Couldn't figure a valid hostname. Please specify one with \`\`-h STORENAME\`\`."
+fi
+
+user=${user:-backuppc}
+dest_path=/var/mirror/$hostname
+
+config_file="/etc/$exname/config.yml"
+
+if [ "${#source_dirs[@]}" == 0 ]; then
+    if [ -e "$config_file" ]; then
+        echo "No source provided on command line.. "
+        echo "   ..so reading '$config_file' for default sources..."
+        source_dirs=($(eval echo $(shyaml get-values default.sources < "$config_file")))
+    fi
+    if [ "${#source_dirs[@]}" == 0 ]; then
+        err "You must specify at least one source directory to mirror" \
+             "on command line (or in a config file)."
+        print_usage
+        exit 1
+    fi
+fi
+echo "Sources directories are: ${source_dirs[@]}"
+
+if [ "${#dests[@]}" == 0 ]; then
+    err "You must specify at least a destination."
+    print_usage
+    exit 1
+fi
+
+rsync_options=(${RSYNC_OPTIONS:-})
+ssh_options=(${SSH_OPTIONS:-})
+
+get_exclude_patterns() {
+    local dir="$1"
+    [ -e "$config_file" ] || return
+    cat "$config_file" | shyaml get-values-0 "$(echo "$dir" | sed -r 's%\.%\\.%g').exclude"
+}
+
+for dest in "${dests[@]}"; do
+    for d in "${source_dirs[@]}"; do
+        current_rsync_options=("${rsync_options[@]}")
+
+        if [[ "$dest" == *"/"* ]]; then
+            current_rsync_options+=("--bwlimit" "${dest##*/}")
+            dest="${dest%/*}"
+        fi
+
+        if [[ "$dest" == *":"* ]]; then
+            ssh_options+=("-p" "${dest#*:}")
+            dest="${dest%%:*}"
+        fi
+
+        dirpath="$(dirname "$d")"
+        if [ "$dirpath" == "/" ]; then
+            dir="/$(basename "$d")"
+        else
+            dir="$dirpath/$(basename "$d")"
+        fi
+
+        [ -d "$dir" ] || {
+            warn "ignoring '$dir' as it is not existing."
+            continue
+        }
+
+        lock_label=$exname-$hostname-$(echo "$dest" | md5_compat | cut -f 1 -d " ")
+
+        exclude_patterns="$(get_exclude_patterns "$dir")"
+
+        tmp_exclude_patterns=/tmp/${lock_label}.$(echo "$d" | md5_compat | cut -f 1 -d " ").exclude_patterns.tmp
+        if [ "$exclude_patterns" ]; then
+            echo "Adding exclude patterns..."
+
+            ## Adding the base of the dir if required... seems necessary with
+            ## the rsync option that replicate the full path.
+            while read-0 exclude_dir; do
+                if [[ "$exclude_dir" == "/"* ]]; then
+                    echo -en "$dir""$(echo "$exclude_dir" | cut -c 1-)\0"
+                else
+                    echo -en "$exclude_dir\0"
+                fi
+            done < <(get_exclude_patterns "$dir") > "$tmp_exclude_patterns"
+            cat "$tmp_exclude_patterns" | xargs -0 -n 1 echo
+            current_rsync_options=("-0" "--exclude-from"="$tmp_exclude_patterns" "${current_rsync_options[@]}")
+        else
+            echo "No exclude patterns for '$dir'."
+        fi
+
+        echo ---------------------------------
+        date
+
+        echo nice -n 15 rsync "${current_rsync_options[@]}" -azvARH -e "'sudo -u $user ssh ${ssh_options[*]}'" --delete --delete-excluded --partial --partial-dir .rsync-partial --numeric-ids "$dir/" "$user@$dest":"$dest_path"
+
+        lock "$lock_label" -v -D -k -- \
+             nice -n 15 \
+             rsync "${current_rsync_options[@]}" -azvARH \
+                 -e "sudo -u $user ssh ${ssh_options[*]}" \
+                 --delete --delete-excluded --partial --partial-dir .rsync-partial \
+                 --numeric-ids "$dir/" "$user@$dest":"$dest_path"
+
+        rm -fv "$tmp_exclude_patterns"
+    done
+done