From 5ab6a48a31b09adeedc95bfbb2f03e6bd754422e Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Thu, 18 Mar 2021 07:52:58 +0100 Subject: [PATCH] chg: [rsync-backup] better logs and code Signed-off-by: Valentin Lab --- rsync-backup/resources/bin/mirror-dir | 57 +++++++++++++++------------ 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/rsync-backup/resources/bin/mirror-dir b/rsync-backup/resources/bin/mirror-dir index 470b700..279c8f8 100755 --- a/rsync-backup/resources/bin/mirror-dir +++ b/rsync-backup/resources/bin/mirror-dir @@ -124,10 +124,11 @@ fi rsync_options=(${RSYNC_OPTIONS:-}) ssh_options=(${SSH_OPTIONS:-}) + get_exclude_patterns() { local dir="$1" [ -e "$config_file" ] || return - cat "$config_file" | shyaml get-values-0 "$(echo "$dir" | sed -r 's%\.%\\.%g').exclude" + cat "$config_file" | shyaml get-values-0 "${dir//.\\./}.exclude" 2>/dev/null } for dest in "${dests[@]}"; do @@ -156,31 +157,31 @@ for dest in "${dests[@]}"; do continue } - lock_label=$exname-$hostname-$(echo "$dest" | md5_compat | cut -f 1 -d " ") - - exclude_patterns="$(get_exclude_patterns "$dir")" - - tmp_exclude_patterns=/tmp/${lock_label}.$(echo "$d" | md5_compat | cut -f 1 -d " ").exclude_patterns.tmp - if [ "$exclude_patterns" ]; then - echo "Adding exclude patterns..." - - ## Adding the base of the dir if required... seems necessary with - ## the rsync option that replicate the full path. - while read-0 exclude_dir; do - if [[ "$exclude_dir" == "/"* ]]; then - echo -en "$dir""$(echo "$exclude_dir" | cut -c 1-)\0" - else - echo -en "$exclude_dir\0" - fi - done < <(get_exclude_patterns "$dir") > "$tmp_exclude_patterns" - cat "$tmp_exclude_patterns" | xargs -0 -n 1 echo - current_rsync_options=("-0" "--exclude-from"="$tmp_exclude_patterns" "${current_rsync_options[@]}") + lock_label=$exname-$hostname-$(echo "$dest$d" | md5_compat | cut -c 1-6) + + tmp_exclude_patterns=/tmp/${lock_label}.exclude_patterns.tmp + ## Adding the base of the dir if required... seems necessary with + ## the rsync option that replicate the full path. + has_exclude_pattern= + while read-0 exclude_dir; do + if [ -z "$has_exclude_pattern" ]; then + echo "Adding exclude patterns..." >&2 + has_exclude_pattern=1 + fi + if [[ "$exclude_dir" == "/"* ]]; then + exclude_dir="$dir${exclude_dir}" + fi + echo " - $exclude_dir" >&2 + p0 "$exclude_dir" + done < <(get_exclude_patterns "$dir") > "$tmp_exclude_patterns" + if [ -n "$has_exclude_pattern" ]; then + current_rsync_options+=("-0" "--exclude-from"="$tmp_exclude_patterns") else echo "No exclude patterns for '$dir'." fi echo --------------------------------- - date + echo "Starting rsync: $d -> $dest ($(date))" echo nice -n 15 \ rsync "${current_rsync_options[@]}" -azvARH \ @@ -189,6 +190,7 @@ for dest in "${dests[@]}"; do --partial --partial-dir .rsync-partial \ --numeric-ids "$dir/" "$user@$dest":"$dest_path" + start="$SECONDS" retry=1 while true; do lock "$lock_label" -v -D -k -- \ @@ -201,29 +203,34 @@ for dest in "${dests[@]}"; do errlvl="$?" case "$errlvl" in 20) ## Received SIGUSR1, SIGINTT - echo "!! Rsync received SIGUSR1 or SIGINT" + echo "!! Rsync received SIGUSR1 or SIGINT." + echo " .. Full interruption while $d -> $dest and after $((SECONDS - start))s" break 2 ;; 137|143) ## killed SIGKILL, SIGTERM, SIGINT echo "!! Rsync received $(kill -l "$errlvl")" + echo " .. Full interruption while $d -> $dest and after $((SECONDS - start))s" break 2 ;; 0) - echo "Rsync finished with success." + echo "Rsync finished with success $d -> $dest in $((SECONDS - start))s" break ;; *) - echo "!! Rsync failed with an errorlevel $errlvl." + echo "!! Rsync failed with an errorlevel $errlvl after $((SECONDS - start))s since start." if [ "$retry" -lt 3 ]; then echo "!! Triggering a retry ($((++retry))/3)" continue else echo "!! Tried 3 times, bailing out." + echo " .. interruption of $d -> $dest after $((SECONDS - start))s" break fi ;; esac done - rm -fv "$tmp_exclude_patterns" + if [ -n "$has_exclude_pattern" ]; then + rm -fv "$tmp_exclude_patterns" + fi done done