diff --git a/rsync-backup/hooks/install.d/60-install.sh b/rsync-backup/hooks/install.d/60-install.sh index 7b080316..271dfec3 100755 --- a/rsync-backup/hooks/install.d/60-install.sh +++ b/rsync-backup/hooks/install.d/60-install.sh @@ -14,6 +14,11 @@ set -eux exit 1 } +MIRROR_DIR_PATH="${MIRROR_DIR_PATH:-$PWD/resources/bin/mirror-dir}" +[ -e "$MIRROR_DIR_PATH" ] || { + echo "Error: you must set \$MIRROR_DIR_PATH or be the root of the charm to run this script." >&2 + exit 1 +} ## rsync type -p rsync >/dev/null 2>&1 || apt-get install -y rsync /var/lib/rsync/.ssh/known_hosts -apt-get install kal-shlib-process /dev/null || awk --version) in + "mawk 1.3.3"*) + ## Not good, it is from 1996, and we still find it on Debian 10 + apt-get install -y gawk /etc/cron.d/mirror-dir SHELL=/bin/bash PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -$((RANDOM % 60)) * * * * root mirror-dir -h "$DOMAIN" -d "$BACKUP_SERVER" -u rsync 2>&1 | logger -t mirror-dir +$((RANDOM % 60)) $((RANDOM % 4))-23/4 * * * root mirror-dir backup -q -h "$DOMAIN" -d "$BACKUP_SERVER" EOF diff --git a/rsync-backup/resources/bin/mirror-dir b/rsync-backup/resources/bin/mirror-dir index 909f980f..7ffb44cf 100755 --- a/rsync-backup/resources/bin/mirror-dir +++ b/rsync-backup/resources/bin/mirror-dir @@ -1,5 +1,14 @@ #!/bin/bash +## +## Here's an example crontab: +## +## SHELL=/bin/sh +## PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin +## +## 49 */2 * * * root mirror-dir run -d core-05.0k.io:10023 -u rsync /etc /home /opt/apps 2>&1 | logger -t mirror-dir +## + #:- . /etc/shlib @@ -8,256 +17,570 @@ include common include parse include process +include cmdline +include array depends shyaml lock -[ "$UID" != "0" ] && echo "You must be root." && exit 1 ## -## Here's an example crontab: +## Functions ## -## SHELL=/bin/sh -## PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin + + +MIRROR_DIR_LOG=/var/log/mirror-dir.log +MIRROR_DIR_REPORT_MAX_READ_LINE=1000000 +R_DATE='[0-9]{4,4}-[01][0-9]-[0-3][0-9] [012][0-9]:[0-5][0-9]:[0-5][0-9][+-][01][0-9][0-5][0-9]' + +mirror-dir:report() { + + local s1 s2 s3 d1 d2 d3 host source sent received rate + + while read s1 s2 d1 d2 host source sent received rate; do + s=$(date -d"$s1 $s2" --rfc-3339=seconds) + s_s=$(date -d"$s1 $s2" +%s) + d_s=$(date -d"$d1 $d2" +%s) + duration=$((d_s - s_s)) + printf "%s %-15s %-30s | %s %s %s %10s\n" \ + "$s" "$host" "$source" "$sent" "$received" "$rate" "$(print_duration "$duration")" + done < <( + tail "$MIRROR_DIR_LOG" -n "$MIRROR_DIR_REPORT_MAX_READ_LINE" | + egrep "^${R_DATE} (Starting|sent)" | + sed -r 's/\s*\(.*\)$//g + s/ (([0-9]{1,3},)*[0-9]{1,3})(\.[0-9]{2,2})? bytes(\/sec)?/:\1/g + s/,//g + s/ :([0-9]+)$/ rate:\1/g' | + grep -v "^--$" | + sed -r "/Starting/N; + {s/\n(${R_DATE} )(.*)sent/ \1 sent/g}" | + sed -r "s/^(${R_DATE} )Starting rsync: ([^ ]+) -> ([^ ]+) (${R_DATE} )/\1\4\3 \2/g + s/ +/ /g + s/ [a-z]+:/ /g" | + egrep "^${R_DATE} ${R_DATE} [^ ]+ /[^ ]+ [0-9]+ [0-9]+ [0-9]+$" + ) | + numfmt --field=6,7 --to=iec-i --suffix=B --padding=8 | + numfmt --field=8 --to=iec-i --suffix=B/s --padding=10 | + sed -r 's/ \| / /g' +} + + +mirror-dir:run() { + + local hostname="$1" dests="$2" source_dirs + shift 2 + + dests=($dests) ## individual dests can't use any space-like separators + source_dirs=("$@") + + dest_path=/var/mirror/$hostname + state_dir=/var/run/mirror-dir + mkdir -p "$state_dir" + rsync_options=( + ${RSYNC_OPTIONS:-} --stats --out-format='%i %o %f %l %b') + ssh_options=(${SSH_OPTIONS:-}) + + for dest in "${dests[@]}"; do + dest_rsync_options=("${rsync_options[@]}") + if [[ "$dest" == *"/"* ]]; then + dest_rsync_options+=("--bwlimit" "${dest##*/}") + dest="${dest%/*}" + fi + dest_for_session="$dest" + + for d in "${source_dirs[@]}"; do + + current_rsync_options=("${dest_rsync_options[@]}") + + session_id="$(echo "${dest_for_session}$d" | md5_compat)" + session_id="${session_id:1:8}" + + if [[ "$dest" == *":"* ]]; then + ssh_options+=("-p" "${dest#*:}") + dest="${dest%%:*}" + fi + + dirpath="$(dirname "$d")" + if [ "$dirpath" == "/" ]; then + dir="/$(basename "$d")" + else + dir="$dirpath/$(basename "$d")" + fi + + [ -d "$dir" ] || { + warn "ignoring '$dir' as it is not existing." + continue + } + + lock_label=$exname-$hostname-${session_id} + + tmp_exclude_patterns=/tmp/${lock_label}.exclude_patterns.tmp + ## Adding the base of the dir if required... seems necessary with + ## the rsync option that replicate the full path. + has_exclude_pattern= + while read-0 exclude_dir; do + if [ -z "$has_exclude_pattern" ]; then + echo "Adding exclude patterns for source '$dir':" >&2 + has_exclude_pattern=1 + fi + if [[ "$exclude_dir" == "/"* ]]; then + exclude_dir="$dir${exclude_dir}" + fi + echo " - $exclude_dir" >&2 + p0 "$exclude_dir" + done < <(get_exclude_patterns "$dir") > "$tmp_exclude_patterns" + if [ -n "$has_exclude_pattern" ]; then + current_rsync_options+=("-0" "--exclude-from"="$tmp_exclude_patterns") + else + echo "No exclude patterns for '$dir'." + fi + echo --------------------------------- + echo "Starting rsync: $d -> $dest ($(date))" + cmd=( + nice -n 15 \ + rsync "${current_rsync_options[@]}" -azvARH \ + -e "sudo -u $user ssh ${ssh_options[*]}" \ + --delete --delete-excluded \ + --partial --partial-dir .rsync-partial \ + --numeric-ids "$dir/" "$user@$dest":"$dest_path" + ) + echo "${cmd[@]}" + start="$SECONDS" + retry=1 + errlvls=() + while true; do + lock "$lock_label" -v -D -k -- "${cmd[@]}" + errlvl="$?" + case "$errlvl" in + 20) ## Received SIGUSR1, SIGINTT + echo "!! Rsync received SIGUSR1 or SIGINT." + echo " .. Full interruption while $d -> $dest and after $((SECONDS - start))s" + append_trim "${state_dir}/${session_id}-fail" \ + "$dest $d $((SECONDS - start)) signal SIGUSR1, SIGINT or SIGHUP" + break 2 + ;; + 137|143) ## killed SIGKILL, SIGTERM + echo "!! Rsync received $(kill -l "$errlvl")" + echo " .. Full interruption while $d -> $dest and after $((SECONDS - start))s" + append_trim "${state_dir}/${session_id}-fail" \ + "$dest $d $((SECONDS - start)) signal: $(kill -l "$errlvl")" + break 2 + ;; + 0) + echo "Rsync finished with success $d -> $dest in $((SECONDS - start))s" + append_trim "${state_dir}/${session_id}-success" \ + "$dest $d $((SECONDS - start)) OK" + break + ;; + *) + errlvls+=("$errlvl") + echo "!! Rsync failed with an errorlevel $errlvl after $((SECONDS - start))s since start." + if [ "$retry" -lt 3 ]; then + echo "!! Triggering a retry ($((++retry))/3)" + continue + else + echo "!! Tried 3 times, bailing out." + echo " .. interruption of $d -> $dest after $((SECONDS - start))s" + append_trim "${state_dir}/${session_id}-fail" \ + "$dest $d $((SECONDS - start))" \ + "Failed after 3 retries (errorlevels: ${errlvls[@]})" + break + fi + ;; + esac + done + if [ -n "$has_exclude_pattern" ]; then + rm -fv "$tmp_exclude_patterns" + fi + done + done + +} + + + +get_exclude_patterns() { + local dir="$1" + [ -e "$config_file" ] || return + cat "$config_file" | shyaml get-values-0 "${dir//.\\./}.exclude" 2>/dev/null +} + +append_trim() { + local f="$1" + shift + e "$(date --rfc-3339=s) $*"$'\n' >> "$f" && + tail -n 5000 "$f" > "$f".tmp && + mv "$f"{.tmp,} +} + + +log_tee() { tee -a "$MIRROR_DIR_LOG"; } +log_file() { cat >> "$MIRROR_DIR_LOG"; } + + +get_ids() { + local session_id id_done + declare -A id_done + for file in "$state_dir"/*{-fail,-success}; do + session_id=${file%-*} + [ "${id_done["$session_id"]}" ] && continue + id_done["$session_id"]=1 + echo "${session_id##*/}" + done +} + + +mirror-dir:_get_sources() { + local DIR=("$@") + + config_file="/etc/$exname/config.yml" + + if [ "${#DIR[@]}" == 0 ]; then + if [ -e "$config_file" ]; then + info "No source provided on command line," \ + "reading '$config_file' for default sources" + DIR=($(eval echo $(shyaml get-values default.sources < "$config_file"))) + fi + fi + array_values_to_stdin DIR +} + + +[[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true + +version=0.1 +desc='Manage mirroring of local directory to distant hosts' +help="" + + + ## -## 49 */2 * * * root mirror-dir -d core-05.0k.io:10023 -u rsync /etc /home /opt/apps 2>&1 | logger -t mirror-dir +## Code ## -usage="usage: $exname -d DEST1 [-d DEST2 [...]] [-u USER] [DIR1 [DIR2 ...]] -Preserve as much as possible the source structure, keeping hard-links, acl, -exact numerical uids and gids, and being able to resume in very large files. +cmdline.spec.gnu -Options: - DIR1 ... DIRn - Local directories that should be mirrored on destination(s). - examples: /etc /home /var/backups +cmdline.spec.gnu backup - If no directories are provided, the config file root - entries will be used all as destination to copy. +cmdline.spec:backup:valued:-d,--dest:run() { + dests+=("$1") +} - -d DESTn - Can be repeated. Specifies host destination towards which - files will be send. Note that you can specify port number after - a colon and a bandwidth limit for rsync after a '/'. +dests=() - examples: -d liszt.musicalta:10022 -d 10.8.0.19/200 +cmdline.spec::cmd:backup:run() { - -u USER (default: 'rsync') +# usage="usage: $exname -d DEST1 [-d DEST2 [...]] [-u USER] [DIR1 [DIR2 ...]] - Local AND destination user to log as at both ends to transfer file. - This local user need to have a NOPASSWD ssh login towards it's - account on destination. This destination account should have - full permissions access without passwd to write with rsync-server - in the destination directory. +# Preserve as much as possible the source structure, keeping hard-links, acl, +# exact numerical uids and gids, and being able to resume in very large files. - -h STORE (default is taken of the hostname file) - Set the destination store, this is the name of the directory where - the files will all directories will be copied. Beware ! if 2 hosts - use the same store, this means they'll conflictingly update the - same destination directory. Only use this if you know what you - are doing. +# " -" + : :posarg: [DIR...] 'Local directories that should be mirrored + on destination(s). -dests=() -source_dirs=() -hostname= -while [ "$#" != 0 ]; do - case "$1" in - "-d") - dests+=("$2") - shift - ;; - "-h") - hostname="$2" - shift - ;; - "-u") - user="$2" - shift - ;; - *) - source_dirs+=("$1") - ;; - esac - shift -done + Examples: /etc /home /var/backups + + If no directories are provided, the config + file root entries will be used all as + destination to copy.' + + : :optval: -d,--dest 'Can be repeated. Specifies host + destination towards which files will be + send. Note that you can specify port + number after a colon and a bandwidth limit + for rsync after a '/'. + + Examples: -d liszt.musicalta:10022 + -d 10.8.0.19/200' + + + : :optval: -u,--user "(default: 'rsync') + + Local AND destination system user to log + as at both ends to transfer file. This + local user need to have a no password ssh + login to it's own account on destination. + This destination account should have full + permissions access without passwd to write + with rsync-server in the destination + directory." + + : :optval: -h,--hostname "(default is taken of the hostname file) + Set the destination store, this is the + name of the directory where the files + will all directories will be copied. + Beware ! if 2 hosts use the same store, + this means they'll conflictingly update + the same destination directory. Only + use this if you know what you are + doing." -if test -z "$hostname"; then - hostname=$(hostname) -fi + : :optfla: -q,--quiet "Prevent output on stderr. Please note that + output is always written in log file." -if test -z "$hostname"; then - die "Couldn't figure a valid hostname. Please specify one with \`\`-h STORENAME\`\`." -fi -user=${user:-rsync} -dest_path=/var/mirror/$hostname + [ "$UID" != "0" ] && echo "You must be root." && exit 1 -config_file="/etc/$exname/config.yml" + [ -n "$opt_hostname" ] || opt_hostname=$(hostname) -if [ "${#source_dirs[@]}" == 0 ]; then - if [ -e "$config_file" ]; then - echo "No source provided on command line.. " - echo " ..so reading '$config_file' for default sources..." - source_dirs=($(eval echo $(shyaml get-values default.sources < "$config_file"))) + if [ -n "$opt_quiet" ]; then + log_facility=log_file + else + log_facility=log_tee fi - if [ "${#source_dirs[@]}" == 0 ]; then + + if [ -z "$opt_hostname" ]; then + err "Couldn't figure a valid hostname. Please specify one with \`\`-h STORENAME\`\`." + return 1 + fi + + user=${opt_user:-rsync} + + config_file="/etc/$exname/config.yml" + + array_read-0 DIR < <( + { + { + mirror-dir:_get_sources "${DIR[@]}" + } 3>&1 1>&2 2>&3 | "$log_facility" + } 3>&1 1>&2 2>&3 + ) + + if [ "${#DIR[@]}" == 0 ]; then err "You must specify at least one source directory to mirror" \ - "on command line (or in a config file)." - print_usage + "on command line (or in a config file)." + echo "$usage" >&2 exit 1 fi -fi -echo "Sources directories are: ${source_dirs[@]}" + info "Source directories are: ${DIR[@]}" 2>&1 | "$log_facility" -if [ "${#dests[@]}" == 0 ]; then - err "You must specify at least a destination." - print_usage - exit 1 -fi + if [ "${#dests[@]}" == 0 ]; then + err "You must specify at least a destination (using \`\`-d\`\` or \`\`--dest\`\`)." + echo "$usage" >&2 + return 1 + fi -state_dir=/var/run/mirror-dir -mkdir -p "$state_dir" -rsync_options=(${RSYNC_OPTIONS:-}) -ssh_options=(${SSH_OPTIONS:-}) + ## XXXvlab: note that we use here a special version of awk supporting + ## ``strftime``. This is only to prefix a date to the logs. Yes, we know + ## about ``--out-format`` and its ``%t`` which would be ideal, but it + ## doesn't output proper UTC time (it is system time, no timezone info). + mirror-dir:run "$opt_hostname" "${dests[*]}" "${DIR[@]}" 2>&1 | + awk -W interactive '{ print strftime("%Y-%m-%d %H:%M:%S%z"), $0 }' | + "$log_facility" +} -get_exclude_patterns() { - local dir="$1" - [ -e "$config_file" ] || return - cat "$config_file" | shyaml get-values-0 "${dir//.\\./}.exclude" 2>/dev/null + + +cmdline.spec.gnu report +cmdline.spec::cmd:report:run() { + mirror-dir:report } -append_trim() { - local f="$1" - shift - e "$(date --rfc-3339=s) $*"$'\n' >> "$f" && - tail -n 5000 "$f" > "$f".tmp && - mv "$f"{.tmp,} + + +cmdline.spec:check:valued:-d,--dest:run() { + dests+=("$1") } +cmdline.spec.gnu check +cmdline.spec::cmd:check:run() { -for dest in "${dests[@]}"; do - dest_rsync_options=("${rsync_options[@]}") - if [[ "$dest" == *"/"* ]]; then - dest_rsync_options+=("--bwlimit" "${dest##*/}") - dest="${dest%/*}" - fi - dest_for_session="$dest" +# usage="usage: $exname -d DEST1 [-d DEST2 [...]] [DIR1 [DIR2 ...]] + +# Checks that mirror-dir did it's job. Will send an email if not. +# " + + + : :posarg: [DIR...] 'Local directories that should be mirrored + on destination(s). + + Examples: /etc /home /var/backups + + If no directories are provided, the config + file root entries will be used all as + destination to copy.' + + : :optval: -d,--dest 'Can be repeated. Specifies host + destination towards which files will be + send. Note that you can specify port + number after a colon and a bandwidth limit + for rsync after a '/'. + + Examples: -d liszt.musicalta:10022 + -d 10.8.0.19/200' + + : :optval: -n,--time-spec "Give a full English time spec about how + old the last full run of rsync should + be at most. Defaults to '12 hours'. + + Examples: -n '12 hours' + -n '1 day'" + + : :optfla: -m,--mail-alert "Send alert via email. This is intended to + use in cron." - for d in "${source_dirs[@]}"; do - current_rsync_options=("${dest_rsync_options[@]}") + [ "$UID" != "0" ] && echo "You must be root." && exit 1 - session_id="$(echo "${dest_for_session}$d" | md5_compat)" - session_id="${session_id:1:8}" + if [ "${#dests[@]}" == 0 ]; then + err "You must specify at least a destination (using \`\`-d\`\` or \`\`--dest\`\`)." + echo "$usage" >&2 + return 1 + fi + + if [ -n "$opt_mail_alert" ]; then + CHECK_DEFAULT_SOURCE=/etc/default/alerting + [ -f "$CHECK_DEFAULT_SOURCE" ] && . "$CHECK_DEFAULT_SOURCE" - if [[ "$dest" == *":"* ]]; then - ssh_options+=("-p" "${dest#*:}") - dest="${dest%%:*}" + if [ "${#MAIL_DESTS[@]}" == 0 ]; then + echo "You must set at least one recipient destination for mails." >&2 + echo " You can do that in '$CHECK_DEFAULT_SOURCE', using the variable" >&2 + echo " '\$MAIL_DESTS'. Note this is a bash array variable." >&2 + exit 1 fi + fi - dirpath="$(dirname "$d")" - if [ "$dirpath" == "/" ]; then - dir="/$(basename "$d")" - else - dir="$dirpath/$(basename "$d")" + array_read-0 DIR < <(mirror-dir:_get_sources "${DIR[@]}") + + if [ "${#DIR[@]}" == 0 ]; then + err "You must specify at least one source directory to mirror" \ + "on command line (or in a config file)." + echo "$usage" >&2 + exit 1 + fi + + time_spec="${opt_time_spec:-12 hours}" + + state_dir=/var/run/mirror-dir + + ## Getting max string length of source + dir_max_len=0 + for d in "${DIR[@]}"; do + [ "$dir_max_len" -lt "${#d}" ] && + dir_max_len="${#d}" + done + + ## Getting max string length of dests + dest_max_len=0 + for d in "${dests[@]}"; do + [ "$dest_max_len" -lt "${#d}" ] && + dest_max_len="${#d}" + done + + declare -A sessions=() + bad_sessions=() + msg=() + for dest in "${dests[@]}"; do + if [[ "$dest" == *"/"* ]]; then + current_rsync_options+=("--bwlimit" "${dest##*/}") + dest="${dest%/*}" fi - [ -d "$dir" ] || { - warn "ignoring '$dir' as it is not existing." - continue - } - - lock_label=$exname-$hostname-${session_id} - - tmp_exclude_patterns=/tmp/${lock_label}.exclude_patterns.tmp - ## Adding the base of the dir if required... seems necessary with - ## the rsync option that replicate the full path. - has_exclude_pattern= - while read-0 exclude_dir; do - if [ -z "$has_exclude_pattern" ]; then - echo "Adding exclude patterns for source '$dir':" >&2 - has_exclude_pattern=1 + for d in "${DIR[@]}"; do + session_id="$(echo "$dest$d" | md5_compat)" + session_id="${session_id:1:8}" + sessions["$session_id"]="$dest $d" + f=$(find "$state_dir" \ + -maxdepth 1 -newermt "-$time_spec" \ + -type f -name "${session_id}-success") + if [ -z "$f" ]; then + if [ -e "$state_dir/${session_id}-success" ]; then + msg+=("$(printf "%-${dest_max_len}s %-${dir_max_len}s last full sync %s" \ + "$dest" "$d" \ + "$(stat -c %y "$state_dir/${session_id}-success" | + sed -r 's/\.[0-9]{9,9} / /g')")") + else + msg+=("$(printf "%-${dest_max_len}s %-${dir_max_len}s never finished yet" \ + "$dest" "$d")") + fi + bad_sessions+=("$session_id") fi - if [[ "$exclude_dir" == "/"* ]]; then - exclude_dir="$dir${exclude_dir}" + done + done + + [ "${#msg[@]}" == 0 ] && return 0 + + if [ -z "$opt_mail_alert" ]; then + echo + echo "${DARKRED}These destination/source directory were" \ + "last synced more than $time_spec ago:${NORMAL}" + for m in "${msg[@]}"; do + printf " %s\n" "$m" + done + echo + echo "${DARKRED}Last failed logs:${NORMAL}" + for m in "${bad_sessions[@]}"; do + if [ -e "${state_dir}"/$m-fail ]; then + echo " ${sessions[$m]}:" + tail -n 5 "${state_dir}"/$m-fail | cut -f 1,2,5- -d " " | sed -r "s/^/ /g" + echo + else + echo " ${sessions[$m]}: no fail log available" fi - echo " - $exclude_dir" >&2 - p0 "$exclude_dir" - done < <(get_exclude_patterns "$dir") > "$tmp_exclude_patterns" - if [ -n "$has_exclude_pattern" ]; then - current_rsync_options+=("-0" "--exclude-from"="$tmp_exclude_patterns") - else - echo "No exclude patterns for '$dir'." - fi - echo --------------------------------- - echo "Starting rsync: $d -> $dest ($(date))" - - echo nice -n 15 \ - rsync "${current_rsync_options[@]}" -azvARH \ - -e "'sudo -u $user ssh ${ssh_options[*]}'" \ - --delete --delete-excluded \ - --partial --partial-dir .rsync-partial \ - --numeric-ids "$dir/" "$user@$dest":"$dest_path" - - start="$SECONDS" - retry=1 - errlvls=() - while true; do - lock "$lock_label" -v -D -k -- \ - nice -n 15 \ - rsync "${current_rsync_options[@]}" -azvARH \ - -e "sudo -u $user ssh ${ssh_options[*]}" \ - --delete --delete-excluded \ - --partial --partial-dir .rsync-partial \ - --numeric-ids "$dir/" "$user@$dest":"$dest_path" - errlvl="$?" - case "$errlvl" in - 20) ## Received SIGUSR1, SIGINTT - echo "!! Rsync received SIGUSR1 or SIGINT." - echo " .. Full interruption while $d -> $dest and after $((SECONDS - start))s" - append_trim "${state_dir}/${session_id}-fail" \ - "$dest $d $((SECONDS - start)) signal SIGUSR1, SIGINT or SIGHUP" - break 2 - ;; - 137|143) ## killed SIGKILL, SIGTERM, SIGINT - echo "!! Rsync received $(kill -l "$errlvl")" - echo " .. Full interruption while $d -> $dest and after $((SECONDS - start))s" - append_trim "${state_dir}/${session_id}-fail" \ - "$dest $d $((SECONDS - start)) signal: $(kill -l "$errlvl")" - break 2 - ;; - 0) - echo "Rsync finished with success $d -> $dest in $((SECONDS - start))s" - append_trim "${state_dir}/${session_id}-success" \ - "$dest $d $((SECONDS - start)) OK" - break - ;; - *) - errlvls+=("$errlvl") - echo "!! Rsync failed with an errorlevel $errlvl after $((SECONDS - start))s since start." - if [ "$retry" -lt 3 ]; then - echo "!! Triggering a retry ($((++retry))/3)" - continue - else - echo "!! Tried 3 times, bailing out." - echo " .. interruption of $d -> $dest after $((SECONDS - start))s" - append_trim "${state_dir}/${session_id}-fail" \ - "$dest $d $((SECONDS - start))" \ - "Failed after 3 retries (errorlevels: ${errlvls[@]})" - break - fi - ;; - esac done - if [ -n "$has_exclude_pattern" ]; then - rm -fv "$tmp_exclude_patterns" + return 1 + fi + + + ## + ## Mail + ## + + if [ "${#msg[@]}" != 0 ]; then + + cat <&2 - echo " You can do that in '$CHECK_DEFAULT_SOURCE', using the variable" >&2 - echo " '\$MAIL_DESTS'. Note this is a bash array variable." >&2 - exit 1 -fi - - -usage="usage: $exname -d DEST1 [-d DEST2 [...]] [DIR1 [DIR2 ...]] - -Checks that mirror-dir did it's job. Will send an email if not. - -Options: - DIR1 ... DIRn - Local directories that should be mirrored on destination(s). - - examples: /etc /home /var/backups - - If no directories are provided, the config file root - entries will be used all as destination to copy. - - -d DESTn - Can be repeated. Specifies host destination towards which - files will be send. Note that you can specify port number after - a colon and a bandwidth limit for rsync after a '/'. - - examples: -d liszt.musicalta:10022 -d 10.8.0.19/200 - - -n TIME_SPEC - Give a full english time spec about how old the last full - run of rsync should be at most. Defaults to '12 hours'. - - examples: -n '12 hours' -" - - -dests=() -source_dirs=() -time_spec='12 hours' -while [ "$#" != 0 ]; do - case "$1" in - "-d") - dests+=("$2") - shift - ;; - "-n") - time_spec="$2" - shift - ;; - *) - source_dirs+=("$1") - ;; - esac - shift -done - -config_file="/etc/mirror-dir/config.yml" - -if [ "${#source_dirs[@]}" == 0 ]; then - if [ -e "$config_file" ]; then - source_dirs=($(eval echo $(shyaml get-values default.sources < "$config_file"))) - fi >&2 - if [ "${#source_dirs[@]}" == 0 ]; then - err "You must specify at least one source directory to mirror" \ - "on command line (or in a config file)." - print_usage - exit 1 - fi -fi - -if [ "${#dests[@]}" == 0 ]; then - err "You must specify at least a destination." - print_usage - exit 1 -fi - -state_dir=/var/run/mirror-dir - -get_ids() { - local session_id id_done - declare -A id_done - for file in "$state_dir"/*{-fail,-success}; do - session_id=${file%-*} - [ "${id_done["$session_id"]}" ] && continue - id_done["$session_id"]=1 - echo "${session_id##*/}" - done -} - - -dir_max_len=0 -for d in "${source_dirs[@]}"; do - [ "$dir_max_len" -lt "${#d}" ] && - dir_max_len="${#d}" -done - - -declare -A sessions=() -bad_sessions=() -for dest in "${dests[@]}"; do - if [[ "$dest" == *"/"* ]]; then - current_rsync_options+=("--bwlimit" "${dest##*/}") - dest="${dest%/*}" - fi - - msg=() - for d in "${source_dirs[@]}"; do - - session_id="$(echo "$dest$d" | md5_compat)" - session_id="${session_id:1:8}" - sessions["$session_id"]="$dest $d" - f=$(find "$state_dir" \ - -maxdepth 1 -newermt "-$time_spec" \ - -type f -name "${session_id}-success") - if [ -z "$f" ]; then - if [ -e "$state_dir/${session_id}-success" ]; then - msg+=("$(printf "%s %-${dir_max_len}s last full sync %s" \ - "$dest" \ - "$d" \ - "$(stat -c %y "$state_dir/${session_id}-success" | - sed -r 's/\.[0-9]{9,9} / /g')")") - else - msg+=("$(printf "%s %-${dir_max_len}s never finished yet" \ - "$dest" \ - "$d")") - fi - bad_sessions+=("$session_id") - fi - done -done - - -if [ "${#msg[@]}" != 0 ]; then - - cat <