From cca9f30830d7d81b8834d8e9b090a6240172159a Mon Sep 17 00:00:00 2001
From: Valentin Lab <valentin.lab@kalysto.org>
Date: Thu, 18 Mar 2021 06:52:18 +0100
Subject: [PATCH] new: [rsync-backup] add retry 3 times mecanism in
 ``mirror-dir``

Signed-off-by: Valentin Lab <valentin.lab@kalysto.org>
---
 rsync-backup/resources/bin/mirror-dir | 49 ++++++++++++++++++++++-----
 1 file changed, 41 insertions(+), 8 deletions(-)

diff --git a/rsync-backup/resources/bin/mirror-dir b/rsync-backup/resources/bin/mirror-dir
index 725cabc4..470b700f 100755
--- a/rsync-backup/resources/bin/mirror-dir
+++ b/rsync-backup/resources/bin/mirror-dir
@@ -44,7 +44,7 @@ Options:
 
         	examples: -d liszt.musicalta:10022 -d 10.8.0.19/200
 
-     -u USER (default: 'backuppc')
+     -u USER (default: 'rsync')
 
         	Local AND destination user to log as at both ends to transfer file.
         	This local user need to have a NOPASSWD ssh login towards it's
@@ -95,7 +95,7 @@ if test -z "$hostname"; then
     die "Couldn't figure a valid hostname. Please specify one with \`\`-h STORENAME\`\`."
 fi
 
-user=${user:-backuppc}
+user=${user:-rsync}
 dest_path=/var/mirror/$hostname
 
 config_file="/etc/$exname/config.yml"
@@ -182,15 +182,48 @@ for dest in "${dests[@]}"; do
         echo ---------------------------------
         date
 
-        echo nice -n 15 rsync "${current_rsync_options[@]}" -azvARH -e "'sudo -u $user ssh ${ssh_options[*]}'" --delete --delete-excluded --partial --partial-dir .rsync-partial --numeric-ids "$dir/" "$user@$dest":"$dest_path"
-
-        lock "$lock_label" -v -D -k -- \
-             nice -n 15 \
+        echo nice -n 15 \
              rsync "${current_rsync_options[@]}" -azvARH \
+             -e "'sudo -u $user ssh ${ssh_options[*]}'" \
+             --delete --delete-excluded \
+             --partial --partial-dir .rsync-partial \
+             --numeric-ids "$dir/" "$user@$dest":"$dest_path"
+
+        retry=1
+        while true; do
+            lock "$lock_label" -v -D -k -- \
+                 nice -n 15 \
+                 rsync "${current_rsync_options[@]}" -azvARH \
                  -e "sudo -u $user ssh ${ssh_options[*]}" \
-                 --delete --delete-excluded --partial --partial-dir .rsync-partial \
+                 --delete --delete-excluded \
+                 --partial --partial-dir .rsync-partial \
                  --numeric-ids "$dir/" "$user@$dest":"$dest_path"
-
+            errlvl="$?"
+            case "$errlvl" in
+                20)  ## Received SIGUSR1, SIGINTT
+                    echo "!! Rsync received SIGUSR1 or SIGINT"
+                    break 2
+                    ;;
+                137|143)  ## killed SIGKILL, SIGTERM, SIGINT
+                    echo "!! Rsync received $(kill -l "$errlvl")"
+                    break 2
+                    ;;
+                0)
+                    echo "Rsync finished with success."
+                    break
+                    ;;
+                *)
+                    echo "!! Rsync failed with an errorlevel $errlvl."
+                    if [ "$retry" -lt 3 ]; then
+                        echo "!! Triggering a retry ($((++retry))/3)"
+                        continue
+                    else
+                        echo "!! Tried 3 times, bailing out."
+                        break
+                    fi
+                    ;;
+            esac
+        done
         rm -fv "$tmp_exclude_patterns"
     done
 done