new: [rsync-backup] add retry 3 times mecanism in ``mirror-dir``

Signed-off-by: Valentin Lab <valentin.lab@kalysto.org>

rsync-backup/resources/bin/mirror-dir

@@ -44,7 +44,7 @@ Options:
 
         	examples: -d liszt.musicalta:10022 -d 10.8.0.19/200
 
-     -u USER (default: 'backuppc')
+     -u USER (default: 'rsync')
 
         	Local AND destination user to log as at both ends to transfer file.
         	This local user need to have a NOPASSWD ssh login towards it's
@@ -95,7 +95,7 @@ if test -z "$hostname"; then
     die "Couldn't figure a valid hostname. Please specify one with \`\`-h STORENAME\`\`."
 fi
 
-user=${user:-backuppc}
+user=${user:-rsync}
 dest_path=/var/mirror/$hostname
 
 config_file="/etc/$exname/config.yml"
@@ -182,15 +182,48 @@ for dest in "${dests[@]}"; do
         echo ---------------------------------
         date
 
-        echo nice -n 15 rsync "${current_rsync_options[@]}" -azvARH -e "'sudo -u $user ssh ${ssh_options[*]}'" --delete --delete-excluded --partial --partial-dir .rsync-partial --numeric-ids "$dir/" "$user@$dest":"$dest_path"
-
-        lock "$lock_label" -v -D -k -- \
-             nice -n 15 \
+        echo nice -n 15 \
              rsync "${current_rsync_options[@]}" -azvARH \
+             -e "'sudo -u $user ssh ${ssh_options[*]}'" \
+             --delete --delete-excluded \
+             --partial --partial-dir .rsync-partial \
+             --numeric-ids "$dir/" "$user@$dest":"$dest_path"
+
+        retry=1
+        while true; do
+            lock "$lock_label" -v -D -k -- \
+                 nice -n 15 \
+                 rsync "${current_rsync_options[@]}" -azvARH \
                  -e "sudo -u $user ssh ${ssh_options[*]}" \
-                 --delete --delete-excluded --partial --partial-dir .rsync-partial \
+                 --delete --delete-excluded \
+                 --partial --partial-dir .rsync-partial \
                  --numeric-ids "$dir/" "$user@$dest":"$dest_path"
-
+            errlvl="$?"
+            case "$errlvl" in
+                20)  ## Received SIGUSR1, SIGINTT
+                    echo "!! Rsync received SIGUSR1 or SIGINT"
+                    break 2
+                    ;;
+                137|143)  ## killed SIGKILL, SIGTERM, SIGINT
+                    echo "!! Rsync received $(kill -l "$errlvl")"
+                    break 2
+                    ;;
+                0)
+                    echo "Rsync finished with success."
+                    break
+                    ;;
+                *)
+                    echo "!! Rsync failed with an errorlevel $errlvl."
+                    if [ "$retry" -lt 3 ]; then
+                        echo "!! Triggering a retry ($((++retry))/3)"
+                        continue
+                    else
+                        echo "!! Tried 3 times, bailing out."
+                        break
+                    fi
+                    ;;
+            esac
+        done
         rm -fv "$tmp_exclude_patterns"
     done
 done