From 5236bbe3387729195e920ba9dea60d4991f4ac9e Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Thu, 18 Mar 2021 15:58:46 +0100 Subject: [PATCH] new: [rsync-backup] new state files and check capability for ``mirror-dir`` Signed-off-by: Valentin Lab --- rsync-backup/resources/bin/mirror-dir | 43 ++++- rsync-backup/resources/bin/mirror-dir-check | 194 ++++++++++++++++++++ 2 files changed, 229 insertions(+), 8 deletions(-) create mode 100755 rsync-backup/resources/bin/mirror-dir-check diff --git a/rsync-backup/resources/bin/mirror-dir b/rsync-backup/resources/bin/mirror-dir index 279c8f8..909f980 100755 --- a/rsync-backup/resources/bin/mirror-dir +++ b/rsync-backup/resources/bin/mirror-dir @@ -121,6 +121,8 @@ if [ "${#dests[@]}" == 0 ]; then exit 1 fi +state_dir=/var/run/mirror-dir +mkdir -p "$state_dir" rsync_options=(${RSYNC_OPTIONS:-}) ssh_options=(${SSH_OPTIONS:-}) @@ -131,14 +133,29 @@ get_exclude_patterns() { cat "$config_file" | shyaml get-values-0 "${dir//.\\./}.exclude" 2>/dev/null } +append_trim() { + local f="$1" + shift + e "$(date --rfc-3339=s) $*"$'\n' >> "$f" && + tail -n 5000 "$f" > "$f".tmp && + mv "$f"{.tmp,} +} + + for dest in "${dests[@]}"; do + dest_rsync_options=("${rsync_options[@]}") + if [[ "$dest" == *"/"* ]]; then + dest_rsync_options+=("--bwlimit" "${dest##*/}") + dest="${dest%/*}" + fi + dest_for_session="$dest" + for d in "${source_dirs[@]}"; do - current_rsync_options=("${rsync_options[@]}") - if [[ "$dest" == *"/"* ]]; then - current_rsync_options+=("--bwlimit" "${dest##*/}") - dest="${dest%/*}" - fi + current_rsync_options=("${dest_rsync_options[@]}") + + session_id="$(echo "${dest_for_session}$d" | md5_compat)" + session_id="${session_id:1:8}" if [[ "$dest" == *":"* ]]; then ssh_options+=("-p" "${dest#*:}") @@ -157,7 +174,7 @@ for dest in "${dests[@]}"; do continue } - lock_label=$exname-$hostname-$(echo "$dest$d" | md5_compat | cut -c 1-6) + lock_label=$exname-$hostname-${session_id} tmp_exclude_patterns=/tmp/${lock_label}.exclude_patterns.tmp ## Adding the base of the dir if required... seems necessary with @@ -165,7 +182,7 @@ for dest in "${dests[@]}"; do has_exclude_pattern= while read-0 exclude_dir; do if [ -z "$has_exclude_pattern" ]; then - echo "Adding exclude patterns..." >&2 + echo "Adding exclude patterns for source '$dir':" >&2 has_exclude_pattern=1 fi if [[ "$exclude_dir" == "/"* ]]; then @@ -179,7 +196,6 @@ for dest in "${dests[@]}"; do else echo "No exclude patterns for '$dir'." fi - echo --------------------------------- echo "Starting rsync: $d -> $dest ($(date))" @@ -192,6 +208,7 @@ for dest in "${dests[@]}"; do start="$SECONDS" retry=1 + errlvls=() while true; do lock "$lock_label" -v -D -k -- \ nice -n 15 \ @@ -205,18 +222,25 @@ for dest in "${dests[@]}"; do 20) ## Received SIGUSR1, SIGINTT echo "!! Rsync received SIGUSR1 or SIGINT." echo " .. Full interruption while $d -> $dest and after $((SECONDS - start))s" + append_trim "${state_dir}/${session_id}-fail" \ + "$dest $d $((SECONDS - start)) signal SIGUSR1, SIGINT or SIGHUP" break 2 ;; 137|143) ## killed SIGKILL, SIGTERM, SIGINT echo "!! Rsync received $(kill -l "$errlvl")" echo " .. Full interruption while $d -> $dest and after $((SECONDS - start))s" + append_trim "${state_dir}/${session_id}-fail" \ + "$dest $d $((SECONDS - start)) signal: $(kill -l "$errlvl")" break 2 ;; 0) echo "Rsync finished with success $d -> $dest in $((SECONDS - start))s" + append_trim "${state_dir}/${session_id}-success" \ + "$dest $d $((SECONDS - start)) OK" break ;; *) + errlvls+=("$errlvl") echo "!! Rsync failed with an errorlevel $errlvl after $((SECONDS - start))s since start." if [ "$retry" -lt 3 ]; then echo "!! Triggering a retry ($((++retry))/3)" @@ -224,6 +248,9 @@ for dest in "${dests[@]}"; do else echo "!! Tried 3 times, bailing out." echo " .. interruption of $d -> $dest after $((SECONDS - start))s" + append_trim "${state_dir}/${session_id}-fail" \ + "$dest $d $((SECONDS - start))" \ + "Failed after 3 retries (errorlevels: ${errlvls[@]})" break fi ;; diff --git a/rsync-backup/resources/bin/mirror-dir-check b/rsync-backup/resources/bin/mirror-dir-check new file mode 100755 index 0000000..8e8b9fd --- /dev/null +++ b/rsync-backup/resources/bin/mirror-dir-check @@ -0,0 +1,194 @@ +#!/bin/bash + + +#:- +. /etc/shlib +#:- + +include common +include parse +include process + +depends shyaml lock + +[ "$UID" != "0" ] && echo "You must be root." && exit 1 + + +CHECK_DEFAULT_SOURCE=/etc/default/alerting +[ -f "$CHECK_DEFAULT_SOURCE" ] && . "$CHECK_DEFAULT_SOURCE" + +if [ "${#MAIL_DESTS[@]}" == 0 ]; then + echo "You must set at least one recipient destination for mails." >&2 + echo " You can do that in '$CHECK_DEFAULT_SOURCE', using the variable" >&2 + echo " '\$MAIL_DESTS'. Note this is a bash array variable." >&2 + exit 1 +fi + + +usage="usage: $exname -d DEST1 [-d DEST2 [...]] [DIR1 [DIR2 ...]] + +Checks that mirror-dir did it's job. Will send an email if not. + +Options: + DIR1 ... DIRn + Local directories that should be mirrored on destination(s). + + examples: /etc /home /var/backups + + If no directories are provided, the config file root + entries will be used all as destination to copy. + + -d DESTn + Can be repeated. Specifies host destination towards which + files will be send. Note that you can specify port number after + a colon and a bandwidth limit for rsync after a '/'. + + examples: -d liszt.musicalta:10022 -d 10.8.0.19/200 + + -n TIME_SPEC + Give a full english time spec about how old the last full + run of rsync should be at most. Defaults to '12 hours'. + + examples: -n '12 hours' +" + + +dests=() +source_dirs=() +time_spec='12 hours' +while [ "$#" != 0 ]; do + case "$1" in + "-d") + dests+=("$2") + shift + ;; + "-n") + time_spec="$2" + shift + ;; + *) + source_dirs+=("$1") + ;; + esac + shift +done + +config_file="/etc/mirror-dir/config.yml" + +if [ "${#source_dirs[@]}" == 0 ]; then + if [ -e "$config_file" ]; then + source_dirs=($(eval echo $(shyaml get-values default.sources < "$config_file"))) + fi >&2 + if [ "${#source_dirs[@]}" == 0 ]; then + err "You must specify at least one source directory to mirror" \ + "on command line (or in a config file)." + print_usage + exit 1 + fi +fi + +if [ "${#dests[@]}" == 0 ]; then + err "You must specify at least a destination." + print_usage + exit 1 +fi + +state_dir=/var/run/mirror-dir + +get_ids() { + local session_id id_done + declare -A id_done + for file in "$state_dir"/*{-fail,-success}; do + session_id=${file%-*} + [ "${id_done["$session_id"]}" ] && continue + id_done["$session_id"]=1 + echo "${session_id##*/}" + done +} + + +dir_max_len=0 +for d in "${source_dirs[@]}"; do + [ "$dir_max_len" -lt "${#d}" ] && + dir_max_len="${#d}" +done + + +declare -A sessions=() +bad_sessions=() +for dest in "${dests[@]}"; do + if [[ "$dest" == *"/"* ]]; then + current_rsync_options+=("--bwlimit" "${dest##*/}") + dest="${dest%/*}" + fi + + msg=() + for d in "${source_dirs[@]}"; do + + session_id="$(echo "$dest$d" | md5_compat)" + session_id="${session_id:1:8}" + sessions["$session_id"]="$dest $d" + f=$(find "$state_dir" \ + -maxdepth 1 -newermt "-$time_spec" \ + -type f -name "${session_id}-success") + if [ -z "$f" ]; then + if [ -e "$state_dir/${session_id}-success" ]; then + msg+=("$(printf "%s %-${dir_max_len}s last full sync %s" \ + "$dest" \ + "$d" \ + "$(stat -c %y "$state_dir/${session_id}-success" | + sed -r 's/\.[0-9]{9,9} / /g')")") + else + msg+=("$(printf "%s %-${dir_max_len}s never finished yet" \ + "$dest" \ + "$d")") + fi + bad_sessions+=("$session_id") + fi + done +done + + +if [ "${#msg[@]}" != 0 ]; then + + cat <