Browse Source

new: [rsync-backup] new state files and check capability for ``mirror-dir``

Signed-off-by: Valentin Lab <valentin.lab@kalysto.org>
backup
Valentin Lab 4 years ago
parent
commit
5236bbe338
  1. 43
      rsync-backup/resources/bin/mirror-dir
  2. 194
      rsync-backup/resources/bin/mirror-dir-check

43
rsync-backup/resources/bin/mirror-dir

@ -121,6 +121,8 @@ if [ "${#dests[@]}" == 0 ]; then
exit 1
fi
state_dir=/var/run/mirror-dir
mkdir -p "$state_dir"
rsync_options=(${RSYNC_OPTIONS:-})
ssh_options=(${SSH_OPTIONS:-})
@ -131,14 +133,29 @@ get_exclude_patterns() {
cat "$config_file" | shyaml get-values-0 "${dir//.\\./}.exclude" 2>/dev/null
}
append_trim() {
local f="$1"
shift
e "$(date --rfc-3339=s) $*"$'\n' >> "$f" &&
tail -n 5000 "$f" > "$f".tmp &&
mv "$f"{.tmp,}
}
for dest in "${dests[@]}"; do
dest_rsync_options=("${rsync_options[@]}")
if [[ "$dest" == *"/"* ]]; then
dest_rsync_options+=("--bwlimit" "${dest##*/}")
dest="${dest%/*}"
fi
dest_for_session="$dest"
for d in "${source_dirs[@]}"; do
current_rsync_options=("${rsync_options[@]}")
if [[ "$dest" == *"/"* ]]; then
current_rsync_options+=("--bwlimit" "${dest##*/}")
dest="${dest%/*}"
fi
current_rsync_options=("${dest_rsync_options[@]}")
session_id="$(echo "${dest_for_session}$d" | md5_compat)"
session_id="${session_id:1:8}"
if [[ "$dest" == *":"* ]]; then
ssh_options+=("-p" "${dest#*:}")
@ -157,7 +174,7 @@ for dest in "${dests[@]}"; do
continue
}
lock_label=$exname-$hostname-$(echo "$dest$d" | md5_compat | cut -c 1-6)
lock_label=$exname-$hostname-${session_id}
tmp_exclude_patterns=/tmp/${lock_label}.exclude_patterns.tmp
## Adding the base of the dir if required... seems necessary with
@ -165,7 +182,7 @@ for dest in "${dests[@]}"; do
has_exclude_pattern=
while read-0 exclude_dir; do
if [ -z "$has_exclude_pattern" ]; then
echo "Adding exclude patterns..." >&2
echo "Adding exclude patterns for source '$dir':" >&2
has_exclude_pattern=1
fi
if [[ "$exclude_dir" == "/"* ]]; then
@ -179,7 +196,6 @@ for dest in "${dests[@]}"; do
else
echo "No exclude patterns for '$dir'."
fi
echo ---------------------------------
echo "Starting rsync: $d -> $dest ($(date))"
@ -192,6 +208,7 @@ for dest in "${dests[@]}"; do
start="$SECONDS"
retry=1
errlvls=()
while true; do
lock "$lock_label" -v -D -k -- \
nice -n 15 \
@ -205,18 +222,25 @@ for dest in "${dests[@]}"; do
20) ## Received SIGUSR1, SIGINTT
echo "!! Rsync received SIGUSR1 or SIGINT."
echo " .. Full interruption while $d -> $dest and after $((SECONDS - start))s"
append_trim "${state_dir}/${session_id}-fail" \
"$dest $d $((SECONDS - start)) signal SIGUSR1, SIGINT or SIGHUP"
break 2
;;
137|143) ## killed SIGKILL, SIGTERM, SIGINT
echo "!! Rsync received $(kill -l "$errlvl")"
echo " .. Full interruption while $d -> $dest and after $((SECONDS - start))s"
append_trim "${state_dir}/${session_id}-fail" \
"$dest $d $((SECONDS - start)) signal: $(kill -l "$errlvl")"
break 2
;;
0)
echo "Rsync finished with success $d -> $dest in $((SECONDS - start))s"
append_trim "${state_dir}/${session_id}-success" \
"$dest $d $((SECONDS - start)) OK"
break
;;
*)
errlvls+=("$errlvl")
echo "!! Rsync failed with an errorlevel $errlvl after $((SECONDS - start))s since start."
if [ "$retry" -lt 3 ]; then
echo "!! Triggering a retry ($((++retry))/3)"
@ -224,6 +248,9 @@ for dest in "${dests[@]}"; do
else
echo "!! Tried 3 times, bailing out."
echo " .. interruption of $d -> $dest after $((SECONDS - start))s"
append_trim "${state_dir}/${session_id}-fail" \
"$dest $d $((SECONDS - start))" \
"Failed after 3 retries (errorlevels: ${errlvls[@]})"
break
fi
;;

194
rsync-backup/resources/bin/mirror-dir-check

@ -0,0 +1,194 @@
#!/bin/bash
#:-
. /etc/shlib
#:-
include common
include parse
include process
depends shyaml lock
[ "$UID" != "0" ] && echo "You must be root." && exit 1
CHECK_DEFAULT_SOURCE=/etc/default/alerting
[ -f "$CHECK_DEFAULT_SOURCE" ] && . "$CHECK_DEFAULT_SOURCE"
if [ "${#MAIL_DESTS[@]}" == 0 ]; then
echo "You must set at least one recipient destination for mails." >&2
echo " You can do that in '$CHECK_DEFAULT_SOURCE', using the variable" >&2
echo " '\$MAIL_DESTS'. Note this is a bash array variable." >&2
exit 1
fi
usage="usage: $exname -d DEST1 [-d DEST2 [...]] [DIR1 [DIR2 ...]]
Checks that mirror-dir did it's job. Will send an email if not.
Options:
DIR1 ... DIRn
Local directories that should be mirrored on destination(s).
examples: /etc /home /var/backups
If no directories are provided, the config file root
entries will be used all as destination to copy.
-d DESTn
Can be repeated. Specifies host destination towards which
files will be send. Note that you can specify port number after
a colon and a bandwidth limit for rsync after a '/'.
examples: -d liszt.musicalta:10022 -d 10.8.0.19/200
-n TIME_SPEC
Give a full english time spec about how old the last full
run of rsync should be at most. Defaults to '12 hours'.
examples: -n '12 hours'
"
dests=()
source_dirs=()
time_spec='12 hours'
while [ "$#" != 0 ]; do
case "$1" in
"-d")
dests+=("$2")
shift
;;
"-n")
time_spec="$2"
shift
;;
*)
source_dirs+=("$1")
;;
esac
shift
done
config_file="/etc/mirror-dir/config.yml"
if [ "${#source_dirs[@]}" == 0 ]; then
if [ -e "$config_file" ]; then
source_dirs=($(eval echo $(shyaml get-values default.sources < "$config_file")))
fi >&2
if [ "${#source_dirs[@]}" == 0 ]; then
err "You must specify at least one source directory to mirror" \
"on command line (or in a config file)."
print_usage
exit 1
fi
fi
if [ "${#dests[@]}" == 0 ]; then
err "You must specify at least a destination."
print_usage
exit 1
fi
state_dir=/var/run/mirror-dir
get_ids() {
local session_id id_done
declare -A id_done
for file in "$state_dir"/*{-fail,-success}; do
session_id=${file%-*}
[ "${id_done["$session_id"]}" ] && continue
id_done["$session_id"]=1
echo "${session_id##*/}"
done
}
dir_max_len=0
for d in "${source_dirs[@]}"; do
[ "$dir_max_len" -lt "${#d}" ] &&
dir_max_len="${#d}"
done
declare -A sessions=()
bad_sessions=()
for dest in "${dests[@]}"; do
if [[ "$dest" == *"/"* ]]; then
current_rsync_options+=("--bwlimit" "${dest##*/}")
dest="${dest%/*}"
fi
msg=()
for d in "${source_dirs[@]}"; do
session_id="$(echo "$dest$d" | md5_compat)"
session_id="${session_id:1:8}"
sessions["$session_id"]="$dest $d"
f=$(find "$state_dir" \
-maxdepth 1 -newermt "-$time_spec" \
-type f -name "${session_id}-success")
if [ -z "$f" ]; then
if [ -e "$state_dir/${session_id}-success" ]; then
msg+=("$(printf "%s %-${dir_max_len}s last full sync %s" \
"$dest" \
"$d" \
"$(stat -c %y "$state_dir/${session_id}-success" |
sed -r 's/\.[0-9]{9,9} / /g')")")
else
msg+=("$(printf "%s %-${dir_max_len}s never finished yet" \
"$dest" \
"$d")")
fi
bad_sessions+=("$session_id")
fi
done
done
if [ "${#msg[@]}" != 0 ]; then
cat <<EOF | mail -s "[$(hostname)] mirror backup failing" "${MAIL_DESTS[@]}"
Hi,
Some configured mirroring targets have not finished gracefully in
the last $time_spec. Please see for yourself:
$(
for m in "${msg[@]}"; do
echo " $m"
done
)
You might want to find these following information of some use:
$(
for m in "${bad_sessions[@]}"; do
if [ -e "${state_dir}"/$m-fail ]; then
echo " ${sessions[$m]}:"
tail -n 5 "${state_dir}"/$m-fail | cut -f 1,2,5- -d " " | sed -r "s/^/ /g"
echo
else
echo " ${sessions[$m]}: no fail log available"
fi
done
)
Hoping all this will help you sort out the issue...
Yours sincerly,
--
mirror-dir-check
PS: You received this email because your email is listed in
\$MAIL_DESTS of '$CHECK_DEFAULT_SOURCE' of '$(hostname)'
(also known as $(cat /etc/mailname)).
EOF
fi
|||||||
100:0
Loading…
Cancel
Save