0k-charms/rsync-backup/resources/bin/mirror-dir

#!/bin/bash


#:-
. /etc/shlib
#:-

include common
include parse
include process

depends shyaml lock

[ "$UID" != "0" ] && echo "You must be root." && exit 1

##
## Here's an example crontab:
##
##   SHELL=/bin/sh
##   PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
##
##   49 */2 * * *   root   mirror-dir -d core-05.0k.io:10023 -u rsync /etc /home /opt/apps 2>&1 | logger -t mirror-dir
##


usage="usage: $exname -d DEST1 [-d DEST2 [...]] [-u USER] [DIR1 [DIR2 ...]]

Preserve as much as possible the source structure, keeping hard-links, acl,
exact numerical uids and gids, and being able to resume in very large files.

Options:
     DIR1 ... DIRn
	        Local directories that should be mirrored on destination(s).

	        examples: /etc /home /var/backups

            If no directories are provided, the config file root
            entries will be used all as destination to copy.

     -d DESTn
            Can be repeated. Specifies host destination towards which
        	files will be send. Note that you can specify port number after
            a colon and a bandwidth limit for rsync after a '/'.

        	examples: -d liszt.musicalta:10022 -d 10.8.0.19/200

     -u USER (default: 'rsync')

        	Local AND destination user to log as at both ends to transfer file.
        	This local user need to have a NOPASSWD ssh login towards it's
        	account on destination. This destination account should have
        	full permissions access without passwd to write with rsync-server
        	in the destination directory.

     -h STORE (default is taken of the hostname file)

            Set the destination store, this is the name of the directory where
            the files will all directories will be copied. Beware ! if 2 hosts
            use the same store, this means they'll conflictingly update the
            same destination directory. Only use this if you know what you
            are doing.

"

dests=()
source_dirs=()
hostname=
while [ "$#" != 0 ]; do
    case "$1" in
	    "-d")
		    dests+=("$2")
		    shift
	        ;;
        "-h")
            hostname="$2"
            shift
            ;;
	    "-u")
	        user="$2"
	        shift
            ;;
	    *)
            source_dirs+=("$1")
	        ;;
    esac
    shift
done


if test -z "$hostname"; then
    hostname=$(hostname)
fi

if test -z "$hostname"; then
    die "Couldn't figure a valid hostname. Please specify one with \`\`-h STORENAME\`\`."
fi

user=${user:-rsync}
dest_path=/var/mirror/$hostname

config_file="/etc/$exname/config.yml"

if [ "${#source_dirs[@]}" == 0 ]; then
    if [ -e "$config_file" ]; then
        echo "No source provided on command line.. "
        echo "   ..so reading '$config_file' for default sources..."
        source_dirs=($(eval echo $(shyaml get-values default.sources < "$config_file")))
    fi
    if [ "${#source_dirs[@]}" == 0 ]; then
        err "You must specify at least one source directory to mirror" \
             "on command line (or in a config file)."
        print_usage
        exit 1
    fi
fi
echo "Sources directories are: ${source_dirs[@]}"

if [ "${#dests[@]}" == 0 ]; then
    err "You must specify at least a destination."
    print_usage
    exit 1
fi

state_dir=/var/run/mirror-dir
mkdir -p "$state_dir"
rsync_options=(${RSYNC_OPTIONS:-})
ssh_options=(${SSH_OPTIONS:-})


get_exclude_patterns() {
    local dir="$1"
    [ -e "$config_file" ] || return
    cat "$config_file" | shyaml get-values-0 "${dir//.\\./}.exclude" 2>/dev/null
}

append_trim() {
    local f="$1"
    shift
    e "$(date --rfc-3339=s) $*"$'\n' >> "$f" &&
    tail -n 5000 "$f" > "$f".tmp &&
    mv "$f"{.tmp,}
}


for dest in "${dests[@]}"; do
    dest_rsync_options=("${rsync_options[@]}")
    if [[ "$dest" == *"/"* ]]; then
        dest_rsync_options+=("--bwlimit" "${dest##*/}")
        dest="${dest%/*}"
    fi
    dest_for_session="$dest"

    for d in "${source_dirs[@]}"; do

        current_rsync_options=("${dest_rsync_options[@]}")

        session_id="$(echo "${dest_for_session}$d" | md5_compat)"
        session_id="${session_id:1:8}"

        if [[ "$dest" == *":"* ]]; then
            ssh_options+=("-p" "${dest#*:}")
            dest="${dest%%:*}"
        fi

        dirpath="$(dirname "$d")"
        if [ "$dirpath" == "/" ]; then
            dir="/$(basename "$d")"
        else
            dir="$dirpath/$(basename "$d")"
        fi

        [ -d "$dir" ] || {
            warn "ignoring '$dir' as it is not existing."
            continue
        }

        lock_label=$exname-$hostname-${session_id}

        tmp_exclude_patterns=/tmp/${lock_label}.exclude_patterns.tmp
        ## Adding the base of the dir if required... seems necessary with
        ## the rsync option that replicate the full path.
        has_exclude_pattern=
        while read-0 exclude_dir; do
            if [ -z "$has_exclude_pattern" ]; then
                echo "Adding exclude patterns for source '$dir':" >&2
                has_exclude_pattern=1
            fi
            if [[ "$exclude_dir" == "/"* ]]; then
                exclude_dir="$dir${exclude_dir}"
            fi
            echo "  - $exclude_dir" >&2
            p0 "$exclude_dir"
        done < <(get_exclude_patterns "$dir") > "$tmp_exclude_patterns"
        if [ -n "$has_exclude_pattern" ]; then
            current_rsync_options+=("-0" "--exclude-from"="$tmp_exclude_patterns")
        else
            echo "No exclude patterns for '$dir'."
        fi
        echo ---------------------------------
        echo "Starting rsync: $d -> $dest ($(date))"

        echo nice -n 15 \
             rsync "${current_rsync_options[@]}" -azvARH \
             -e "'sudo -u $user ssh ${ssh_options[*]}'" \
             --delete --delete-excluded \
             --partial --partial-dir .rsync-partial \
             --numeric-ids "$dir/" "$user@$dest":"$dest_path"

        start="$SECONDS"
        retry=1
        errlvls=()
        while true; do
            lock "$lock_label" -v -D -k -- \
                 nice -n 15 \
                 rsync "${current_rsync_options[@]}" -azvARH \
                 -e "sudo -u $user ssh ${ssh_options[*]}" \
                 --delete --delete-excluded \
                 --partial --partial-dir .rsync-partial \
                 --numeric-ids "$dir/" "$user@$dest":"$dest_path"
            errlvl="$?"
            case "$errlvl" in
                20)  ## Received SIGUSR1, SIGINTT
                    echo "!! Rsync received SIGUSR1 or SIGINT."
                    echo "   .. Full interruption while $d -> $dest and after $((SECONDS - start))s"
                    append_trim "${state_dir}/${session_id}-fail" \
                                "$dest $d $((SECONDS - start)) signal SIGUSR1, SIGINT or SIGHUP"
                    break 2
                    ;;
                137|143)  ## killed SIGKILL, SIGTERM, SIGINT
                    echo "!! Rsync received $(kill -l "$errlvl")"
                    echo "   .. Full interruption while $d -> $dest and after $((SECONDS - start))s"
                    append_trim "${state_dir}/${session_id}-fail" \
                                "$dest $d $((SECONDS - start)) signal: $(kill -l "$errlvl")"
                    break 2
                    ;;
                0)
                    echo "Rsync finished with success $d -> $dest in $((SECONDS - start))s"
                    append_trim "${state_dir}/${session_id}-success" \
                                "$dest $d $((SECONDS - start)) OK"
                    break
                    ;;
                *)
                    errlvls+=("$errlvl")
                    echo "!! Rsync failed with an errorlevel $errlvl after $((SECONDS - start))s since start."
                    if [ "$retry" -lt 3 ]; then
                        echo "!! Triggering a retry ($((++retry))/3)"
                        continue
                    else
                        echo "!! Tried 3 times, bailing out."
                        echo "   .. interruption of $d -> $dest after $((SECONDS - start))s"
                        append_trim "${state_dir}/${session_id}-fail" \
                                    "$dest $d $((SECONDS - start))" \
                                    "Failed after 3 retries (errorlevels: ${errlvls[@]})"
                        break
                    fi
                    ;;
            esac
        done
        if [ -n "$has_exclude_pattern" ]; then
            rm -fv "$tmp_exclude_patterns"
        fi
    done
done