From bea7a24e80566ec8bb2a4422dc802c32168fba20 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Wed, 10 Jun 2020 18:29:36 +0200 Subject: [PATCH] new: [mysql,rsync-backup] added host install script Signed-off-by: Valentin Lab --- mysql/hooks/install.d/60-backup.sh | 86 ++++++++ mysql/resources/bin/mysql-backup | 82 ++++++++ .../resources/bin/compose-add-rsync-key | 144 +++++++++++++ rsync-backup/hooks/install.d/60-install.sh | 68 ++++++ rsync-backup/resources/bin/mirror-dir | 196 ++++++++++++++++++ 5 files changed, 576 insertions(+) create mode 100644 mysql/hooks/install.d/60-backup.sh create mode 100755 mysql/resources/bin/mysql-backup create mode 100755 rsync-backup-target/resources/bin/compose-add-rsync-key create mode 100644 rsync-backup/hooks/install.d/60-install.sh create mode 100755 rsync-backup/resources/bin/mirror-dir diff --git a/mysql/hooks/install.d/60-backup.sh b/mysql/hooks/install.d/60-backup.sh new file mode 100644 index 0000000..c07659a --- /dev/null +++ b/mysql/hooks/install.d/60-backup.sh @@ -0,0 +1,86 @@ + +set -eux ## important for unbound variable ? + +## Require these to be set +# MYSQL_ROOT_PASSWORD= +# MYSQL_CONTAINER= + +[ "${MYSQL_ROOT_PASSWORD}" ] || { + echo "Error: you must set \$MYSQL_ROOT_PASSWORD prior to running this script." >&2 + exit 1 +} + +[ "${MYSQL_CONTAINER}" ] || { + echo "Error: you must set \$MYSQL_CONTAINER prior to running this script." >&2 + exit 1 +} + + +## +## Init, to setup passwordless connection to mysql +## + +type -p mysql >/dev/null || apt-get install -y mysql-client ~/.my.cnf +[client] +password=${MYSQL_ROOT_PASSWORD} +EOF + chmod 600 ~/.my.cnf +fi + +## +## installation of the mysql-backup script +## + + +apt-get install -y kal-shlib-{core,pretty,common} /etc/cron.d/mysql-backup +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin + +0 * * * * root /usr/local/sbin/mysql-backup --host \$(docker-ip "$MYSQL_CONTAINER" 2>/dev/null | sed -r 's/ +/ /g' | cut -f 3 -d " ") | logger -t mysql-backup + +EOF + + +## +## Connection with backup +## + +if type -p mirror-dir >/dev/null 2>&1; then + [ -d "/etc/mirror-dir" ] || { + echo "'mirror-dir' is installed but no '/etc/mirror-dir' was found." >&2 + exit 1 + } + depends shyaml + + if ! sources=$(shyaml get-values default.sources < /etc/mirror-dir/config.yml); then + echo "Couldn't query 'default.sources' in '/etc/mirror-dir/config.yml'." >&2 + exit 1 + fi + + if ! echo "$sources" | grep "^/var/backups/mysql$" 2>/dev/null; then + sed -i '/sources:/a\ - /var/backups/mysql' /etc/mirror-dir/config.yml + cat <> /etc/mirror-dir/config.yml +/var/backups/mysql: + exclude: + - "/*.inprogress" +EOF + fi +else + echo "warn: 'mirror-dir' not installed, backup won't be sent" >&2 +fi + + + diff --git a/mysql/resources/bin/mysql-backup b/mysql/resources/bin/mysql-backup new file mode 100755 index 0000000..139366d --- /dev/null +++ b/mysql/resources/bin/mysql-backup @@ -0,0 +1,82 @@ +#!/bin/bash + +. /etc/shlib + +include common +include pretty + +usage="$exname [--host HOST] [DATABASE...]" + + +DBS=() +host= +while [ "$1" ]; do + case "$1" in + "--help"|"-h") + print_usage + exit 0 + ;; + "--host") + host="$2" + shift + ;; + *) + DBS+=("$1") + ;; + esac + shift +done + + +mysql_opts=() +if [ "$host" ]; then + mysql_opts+=(-h "$host") +fi + +m() { + mysql "${mysql_opts[@]}" -Bs "$@" +} + +md() { + mysqldump "${mysql_opts[@]}" "$@" +} + +mysql_databases() { + echo "SHOW DATABASES" | m +} + +mysql_tables() { + local db="$1" + echo "SHOW TABLES" | m "$db" +} + + +if [ "${#DBS[@]}" == 0 ]; then + DBS=($(mysql_databases)) || exit 1 +fi + +mkdir -p /var/backups/mysql + +for db in "${DBS[@]}"; do + if [[ "$db" == "information_schema" || "$db" == "performance_schema" || "$db" == "mysql" ]]; then + continue + fi + echo "Dumping database $db..." >&2 + # omitting all the rotation logic + dst=/"var/backups/mysql/$db" + [ -d "$dst.old" ] && rm -rf "$dst.old" + [ -d "$dst" ] && mv "$dst" "$dst.old" + mkdir -p "$dst.inprogress" + (( start = SECONDS )) + md "$db" --routines --no-data --add-drop-database --database "$db" | gzip --rsyncable > "$dst.inprogress/schema.sql.gz" + tables=$(mysql_tables "$db") + for table in $tables; do + backup_file="$dst.inprogress/${table}.sql.gz" + echo " Dumping $table into ${backup_file}" + md "$db" "$table" | gzip --rsyncable > "$backup_file" || break + done + mv "$dst.inprogress" "$dst" + [ -d "$dst.old" ] && rm -rf "$dst.old" + (( elapsed = SECONDS - start )) + echo " ..dumped $db to $dst ($(du -sh "$dst" | cut -f 1) in ${elapsed}s)" >&2 +done diff --git a/rsync-backup-target/resources/bin/compose-add-rsync-key b/rsync-backup-target/resources/bin/compose-add-rsync-key new file mode 100755 index 0000000..dbd8acf --- /dev/null +++ b/rsync-backup-target/resources/bin/compose-add-rsync-key @@ -0,0 +1,144 @@ +#!/bin/bash + + +. /etc/shlib + +[[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true + +include common +include pretty +include cmdline + +desc='Adds YAML key quite crudely in given compose.yml.' +help="\ + +$WHITE$exname$NORMAL will find a add or replace SSH key to +given identifier for rsync-backup-target charm. + +For now, it only use detection of '## INSERTION-POINT' to manage +edition of 'compose.yml', and verification it didn't break anything +before overwriting. + +" + + +[ "$SOURCED" ] && return 0 + +## +## Command line processing +## + + +# remove all lines from regex to regex (not included). +remove_lines_from_to() { + local from="$1" to="$2" + sed -r "/$from/,/$to/{/$to/"'!'"d};/$from/d" +} + +check_valid_yaml() { + shyaml get-value >/dev/null 2>&1; +} + + +cmdline.spec.gnu +cmdline.spec.reporting + +service_name=${SERVICE_NAME:-rsync-backup-target} +compose_file=${COMPOSE_FILE:-/etc/compose/compose.yml} + +cmdline.spec::valued:--compose-file,-f:usage() { + echo "Compose file location. Defaults to '/etc/compose/compose.yml'"; } +cmdline.spec::valued:--compose-file,-f:run() { compose_file="$1"; } + +cmdline.spec::valued:--service-name,-s:usage() { + echo "YAML service name in compose file to check for existence of key. Defaults to 'rsync-backup-target'"; } +cmdline.spec::valued:--service-name,-s:run() { service_name="$1"; } + +cmdline.spec::cmd:__main__:run() { + + : :posarg: DOMAIN 'domain identifier' + : :posarg: SSH_PUBLIC_KEY 'ssh public key' + + if ! existing_domains=$(shyaml keys "${service_name//./\\.}.options.keys" < "$compose_file"); then + err "Couldn't query file '$compose_file' for keys of" \ + "service ${DARKYELLOW}${service_name}${NORMAL}." + exit 1 + fi + + content=$(cat "$compose_file") + if echo "$existing_domains" | grep "^${DOMAIN}$" >/dev/null 2>&1; then + + if ! prev_key=$(shyaml get-value "${service_name//./\\.}.options.keys.${DOMAIN//./\\.}" \ + < "$compose_file"); then + err "Couldn't query file '$compose_file' for key of domain '$DOMAIN'." + exit 1 + fi + + if [ "${prev_key}" == "$SSH_PUBLIC_KEY" ]; then + echo "Key was already setup." + exit 0 + fi + + content=$(echo "$content" | remove_lines_from_to '^ '"${DOMAIN//./\\.}"': ".*\\$' \ + '^ ([A-Za-z0-9.-]+: "|## END MARKER)') + + if [ -z "$content" ]; then + err "Didn't manage to remove key to compose file '$DOMAIN' in '$compose_file'." + exit 1 + fi + + if [ "$content" == "$(cat "$compose_file")" ]; then + err "Couldn't remove previous key for '$DOMAIN' in '$compose_file'." + exit 1 + fi + ## check we didn't break yaml + if ! echo "$content" | check_valid_yaml; then + err "Couldn't safely remove previous key for '$DOMAIN' in '$compose_file'." + exit 1 + fi + fi + + excerpt=$(cat < "$compose_file" + + ## reloading (could be much faster) + compose --debug down && compose --debug up + + if [ "$?" == 0 ]; then + echo "Added key, and restarted service ${DARKYELLOW}$service_name${NORMAL}." + else + echo "something went wrong ! Should check the state of '$DOMAIN' !!" + exit 1 + fi +} + + +cmdline::parse "$@" + diff --git a/rsync-backup/hooks/install.d/60-install.sh b/rsync-backup/hooks/install.d/60-install.sh new file mode 100644 index 0000000..7b08031 --- /dev/null +++ b/rsync-backup/hooks/install.d/60-install.sh @@ -0,0 +1,68 @@ +#!/bin/bash + + +set -eux + + +[ "${DOMAIN}" ] || { + echo "Error: you must set \$DOMAIN prior to running this script." >&2 + exit 1 +} + +[ "${BACKUP_SERVER}" ] || { + echo "Error: you must set \$BACKUP_SERVER prior to running this script." >&2 + exit 1 +} + + +## rsync +type -p rsync >/dev/null 2>&1 || apt-get install -y rsync /dev/null || + groupadd -r rsync + +getent passwd rsync >/dev/null || + useradd -r rsync -d /var/lib/rsync -g rsync + +chown rsync:rsync /var/lib/rsync + +## rsync ssh key creation +[ -e /var/lib/rsync/.ssh/id_rsa ] || + su -c 'ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa -q -C rsync@'"$DOMAIN" - rsync + +dest="$BACKUP_SERVER" +if [[ "$dest" == *"/"* ]]; then + dest="${dest%/*}" +fi + +if [[ "$dest" == *":"* ]]; then + ssh_options+=("-p" "${dest#*:}") + dest="${dest%%:*}" +fi + +ssh-keyscan "${ssh_options[@]}" -H "${dest}" > /var/lib/rsync/.ssh/known_hosts + +apt-get install kal-shlib-process /etc/mirror-dir/config.yml +default: + sources: + - /etc +EOF +fi + +cat < /etc/cron.d/mirror-dir +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin + +$((RANDOM % 60)) * * * * root mirror-dir -h "$DOMAIN" -d "$BACKUP_SERVER" -u rsync 2>&1 | logger -t mirror-dir + +EOF diff --git a/rsync-backup/resources/bin/mirror-dir b/rsync-backup/resources/bin/mirror-dir new file mode 100755 index 0000000..725cabc --- /dev/null +++ b/rsync-backup/resources/bin/mirror-dir @@ -0,0 +1,196 @@ +#!/bin/bash + + +#:- +. /etc/shlib +#:- + +include common +include parse +include process + +depends shyaml lock + +[ "$UID" != "0" ] && echo "You must be root." && exit 1 + +## +## Here's an example crontab: +## +## SHELL=/bin/sh +## PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin +## +## 49 */2 * * * root mirror-dir -d core-05.0k.io:10023 -u rsync /etc /home /opt/apps 2>&1 | logger -t mirror-dir +## + + +usage="usage: $exname -d DEST1 [-d DEST2 [...]] [-u USER] [DIR1 [DIR2 ...]] + +Preserve as much as possible the source structure, keeping hard-links, acl, +exact numerical uids and gids, and being able to resume in very large files. + +Options: + DIR1 ... DIRn + Local directories that should be mirrored on destination(s). + + examples: /etc /home /var/backups + + If no directories are provided, the config file root + entries will be used all as destination to copy. + + -d DESTn + Can be repeated. Specifies host destination towards which + files will be send. Note that you can specify port number after + a colon and a bandwidth limit for rsync after a '/'. + + examples: -d liszt.musicalta:10022 -d 10.8.0.19/200 + + -u USER (default: 'backuppc') + + Local AND destination user to log as at both ends to transfer file. + This local user need to have a NOPASSWD ssh login towards it's + account on destination. This destination account should have + full permissions access without passwd to write with rsync-server + in the destination directory. + + -h STORE (default is taken of the hostname file) + + Set the destination store, this is the name of the directory where + the files will all directories will be copied. Beware ! if 2 hosts + use the same store, this means they'll conflictingly update the + same destination directory. Only use this if you know what you + are doing. + +" + +dests=() +source_dirs=() +hostname= +while [ "$#" != 0 ]; do + case "$1" in + "-d") + dests+=("$2") + shift + ;; + "-h") + hostname="$2" + shift + ;; + "-u") + user="$2" + shift + ;; + *) + source_dirs+=("$1") + ;; + esac + shift +done + + +if test -z "$hostname"; then + hostname=$(hostname) +fi + +if test -z "$hostname"; then + die "Couldn't figure a valid hostname. Please specify one with \`\`-h STORENAME\`\`." +fi + +user=${user:-backuppc} +dest_path=/var/mirror/$hostname + +config_file="/etc/$exname/config.yml" + +if [ "${#source_dirs[@]}" == 0 ]; then + if [ -e "$config_file" ]; then + echo "No source provided on command line.. " + echo " ..so reading '$config_file' for default sources..." + source_dirs=($(eval echo $(shyaml get-values default.sources < "$config_file"))) + fi + if [ "${#source_dirs[@]}" == 0 ]; then + err "You must specify at least one source directory to mirror" \ + "on command line (or in a config file)." + print_usage + exit 1 + fi +fi +echo "Sources directories are: ${source_dirs[@]}" + +if [ "${#dests[@]}" == 0 ]; then + err "You must specify at least a destination." + print_usage + exit 1 +fi + +rsync_options=(${RSYNC_OPTIONS:-}) +ssh_options=(${SSH_OPTIONS:-}) + +get_exclude_patterns() { + local dir="$1" + [ -e "$config_file" ] || return + cat "$config_file" | shyaml get-values-0 "$(echo "$dir" | sed -r 's%\.%\\.%g').exclude" +} + +for dest in "${dests[@]}"; do + for d in "${source_dirs[@]}"; do + current_rsync_options=("${rsync_options[@]}") + + if [[ "$dest" == *"/"* ]]; then + current_rsync_options+=("--bwlimit" "${dest##*/}") + dest="${dest%/*}" + fi + + if [[ "$dest" == *":"* ]]; then + ssh_options+=("-p" "${dest#*:}") + dest="${dest%%:*}" + fi + + dirpath="$(dirname "$d")" + if [ "$dirpath" == "/" ]; then + dir="/$(basename "$d")" + else + dir="$dirpath/$(basename "$d")" + fi + + [ -d "$dir" ] || { + warn "ignoring '$dir' as it is not existing." + continue + } + + lock_label=$exname-$hostname-$(echo "$dest" | md5_compat | cut -f 1 -d " ") + + exclude_patterns="$(get_exclude_patterns "$dir")" + + tmp_exclude_patterns=/tmp/${lock_label}.$(echo "$d" | md5_compat | cut -f 1 -d " ").exclude_patterns.tmp + if [ "$exclude_patterns" ]; then + echo "Adding exclude patterns..." + + ## Adding the base of the dir if required... seems necessary with + ## the rsync option that replicate the full path. + while read-0 exclude_dir; do + if [[ "$exclude_dir" == "/"* ]]; then + echo -en "$dir""$(echo "$exclude_dir" | cut -c 1-)\0" + else + echo -en "$exclude_dir\0" + fi + done < <(get_exclude_patterns "$dir") > "$tmp_exclude_patterns" + cat "$tmp_exclude_patterns" | xargs -0 -n 1 echo + current_rsync_options=("-0" "--exclude-from"="$tmp_exclude_patterns" "${current_rsync_options[@]}") + else + echo "No exclude patterns for '$dir'." + fi + + echo --------------------------------- + date + + echo nice -n 15 rsync "${current_rsync_options[@]}" -azvARH -e "'sudo -u $user ssh ${ssh_options[*]}'" --delete --delete-excluded --partial --partial-dir .rsync-partial --numeric-ids "$dir/" "$user@$dest":"$dest_path" + + lock "$lock_label" -v -D -k -- \ + nice -n 15 \ + rsync "${current_rsync_options[@]}" -azvARH \ + -e "sudo -u $user ssh ${ssh_options[*]}" \ + --delete --delete-excluded --partial --partial-dir .rsync-partial \ + --numeric-ids "$dir/" "$user@$dest":"$dest_path" + + rm -fv "$tmp_exclude_patterns" + done +done