forked from 0k/0k-charms
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
263 lines
8.3 KiB
263 lines
8.3 KiB
#!/bin/bash
|
|
|
|
|
|
#:-
|
|
. /etc/shlib
|
|
#:-
|
|
|
|
include common
|
|
include parse
|
|
include process
|
|
|
|
depends shyaml lock
|
|
|
|
[ "$UID" != "0" ] && echo "You must be root." && exit 1
|
|
|
|
##
|
|
## Here's an example crontab:
|
|
##
|
|
## SHELL=/bin/sh
|
|
## PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
|
|
##
|
|
## 49 */2 * * * root mirror-dir -d core-05.0k.io:10023 -u rsync /etc /home /opt/apps 2>&1 | logger -t mirror-dir
|
|
##
|
|
|
|
|
|
usage="usage: $exname -d DEST1 [-d DEST2 [...]] [-u USER] [DIR1 [DIR2 ...]]
|
|
|
|
Preserve as much as possible the source structure, keeping hard-links, acl,
|
|
exact numerical uids and gids, and being able to resume in very large files.
|
|
|
|
Options:
|
|
DIR1 ... DIRn
|
|
Local directories that should be mirrored on destination(s).
|
|
|
|
examples: /etc /home /var/backups
|
|
|
|
If no directories are provided, the config file root
|
|
entries will be used all as destination to copy.
|
|
|
|
-d DESTn
|
|
Can be repeated. Specifies host destination towards which
|
|
files will be send. Note that you can specify port number after
|
|
a colon and a bandwidth limit for rsync after a '/'.
|
|
|
|
examples: -d liszt.musicalta:10022 -d 10.8.0.19/200
|
|
|
|
-u USER (default: 'rsync')
|
|
|
|
Local AND destination user to log as at both ends to transfer file.
|
|
This local user need to have a NOPASSWD ssh login towards it's
|
|
account on destination. This destination account should have
|
|
full permissions access without passwd to write with rsync-server
|
|
in the destination directory.
|
|
|
|
-h STORE (default is taken of the hostname file)
|
|
|
|
Set the destination store, this is the name of the directory where
|
|
the files will all directories will be copied. Beware ! if 2 hosts
|
|
use the same store, this means they'll conflictingly update the
|
|
same destination directory. Only use this if you know what you
|
|
are doing.
|
|
|
|
"
|
|
|
|
dests=()
|
|
source_dirs=()
|
|
hostname=
|
|
while [ "$#" != 0 ]; do
|
|
case "$1" in
|
|
"-d")
|
|
dests+=("$2")
|
|
shift
|
|
;;
|
|
"-h")
|
|
hostname="$2"
|
|
shift
|
|
;;
|
|
"-u")
|
|
user="$2"
|
|
shift
|
|
;;
|
|
*)
|
|
source_dirs+=("$1")
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
|
|
if test -z "$hostname"; then
|
|
hostname=$(hostname)
|
|
fi
|
|
|
|
if test -z "$hostname"; then
|
|
die "Couldn't figure a valid hostname. Please specify one with \`\`-h STORENAME\`\`."
|
|
fi
|
|
|
|
user=${user:-rsync}
|
|
dest_path=/var/mirror/$hostname
|
|
|
|
config_file="/etc/$exname/config.yml"
|
|
|
|
if [ "${#source_dirs[@]}" == 0 ]; then
|
|
if [ -e "$config_file" ]; then
|
|
echo "No source provided on command line.. "
|
|
echo " ..so reading '$config_file' for default sources..."
|
|
source_dirs=($(eval echo $(shyaml get-values default.sources < "$config_file")))
|
|
fi
|
|
if [ "${#source_dirs[@]}" == 0 ]; then
|
|
err "You must specify at least one source directory to mirror" \
|
|
"on command line (or in a config file)."
|
|
print_usage
|
|
exit 1
|
|
fi
|
|
fi
|
|
echo "Sources directories are: ${source_dirs[@]}"
|
|
|
|
if [ "${#dests[@]}" == 0 ]; then
|
|
err "You must specify at least a destination."
|
|
print_usage
|
|
exit 1
|
|
fi
|
|
|
|
state_dir=/var/run/mirror-dir
|
|
mkdir -p "$state_dir"
|
|
rsync_options=(${RSYNC_OPTIONS:-})
|
|
ssh_options=(${SSH_OPTIONS:-})
|
|
|
|
|
|
get_exclude_patterns() {
|
|
local dir="$1"
|
|
[ -e "$config_file" ] || return
|
|
cat "$config_file" | shyaml get-values-0 "${dir//.\\./}.exclude" 2>/dev/null
|
|
}
|
|
|
|
append_trim() {
|
|
local f="$1"
|
|
shift
|
|
e "$(date --rfc-3339=s) $*"$'\n' >> "$f" &&
|
|
tail -n 5000 "$f" > "$f".tmp &&
|
|
mv "$f"{.tmp,}
|
|
}
|
|
|
|
|
|
for dest in "${dests[@]}"; do
|
|
dest_rsync_options=("${rsync_options[@]}")
|
|
if [[ "$dest" == *"/"* ]]; then
|
|
dest_rsync_options+=("--bwlimit" "${dest##*/}")
|
|
dest="${dest%/*}"
|
|
fi
|
|
dest_for_session="$dest"
|
|
|
|
for d in "${source_dirs[@]}"; do
|
|
|
|
current_rsync_options=("${dest_rsync_options[@]}")
|
|
|
|
session_id="$(echo "${dest_for_session}$d" | md5_compat)"
|
|
session_id="${session_id:1:8}"
|
|
|
|
if [[ "$dest" == *":"* ]]; then
|
|
ssh_options+=("-p" "${dest#*:}")
|
|
dest="${dest%%:*}"
|
|
fi
|
|
|
|
dirpath="$(dirname "$d")"
|
|
if [ "$dirpath" == "/" ]; then
|
|
dir="/$(basename "$d")"
|
|
else
|
|
dir="$dirpath/$(basename "$d")"
|
|
fi
|
|
|
|
[ -d "$dir" ] || {
|
|
warn "ignoring '$dir' as it is not existing."
|
|
continue
|
|
}
|
|
|
|
lock_label=$exname-$hostname-${session_id}
|
|
|
|
tmp_exclude_patterns=/tmp/${lock_label}.exclude_patterns.tmp
|
|
## Adding the base of the dir if required... seems necessary with
|
|
## the rsync option that replicate the full path.
|
|
has_exclude_pattern=
|
|
while read-0 exclude_dir; do
|
|
if [ -z "$has_exclude_pattern" ]; then
|
|
echo "Adding exclude patterns for source '$dir':" >&2
|
|
has_exclude_pattern=1
|
|
fi
|
|
if [[ "$exclude_dir" == "/"* ]]; then
|
|
exclude_dir="$dir${exclude_dir}"
|
|
fi
|
|
echo " - $exclude_dir" >&2
|
|
p0 "$exclude_dir"
|
|
done < <(get_exclude_patterns "$dir") > "$tmp_exclude_patterns"
|
|
if [ -n "$has_exclude_pattern" ]; then
|
|
current_rsync_options+=("-0" "--exclude-from"="$tmp_exclude_patterns")
|
|
else
|
|
echo "No exclude patterns for '$dir'."
|
|
fi
|
|
echo ---------------------------------
|
|
echo "Starting rsync: $d -> $dest ($(date))"
|
|
|
|
echo nice -n 15 \
|
|
rsync "${current_rsync_options[@]}" -azvARH \
|
|
-e "'sudo -u $user ssh ${ssh_options[*]}'" \
|
|
--delete --delete-excluded \
|
|
--partial --partial-dir .rsync-partial \
|
|
--numeric-ids "$dir/" "$user@$dest":"$dest_path"
|
|
|
|
start="$SECONDS"
|
|
retry=1
|
|
errlvls=()
|
|
while true; do
|
|
lock "$lock_label" -v -D -k -- \
|
|
nice -n 15 \
|
|
rsync "${current_rsync_options[@]}" -azvARH \
|
|
-e "sudo -u $user ssh ${ssh_options[*]}" \
|
|
--delete --delete-excluded \
|
|
--partial --partial-dir .rsync-partial \
|
|
--numeric-ids "$dir/" "$user@$dest":"$dest_path"
|
|
errlvl="$?"
|
|
case "$errlvl" in
|
|
20) ## Received SIGUSR1, SIGINTT
|
|
echo "!! Rsync received SIGUSR1 or SIGINT."
|
|
echo " .. Full interruption while $d -> $dest and after $((SECONDS - start))s"
|
|
append_trim "${state_dir}/${session_id}-fail" \
|
|
"$dest $d $((SECONDS - start)) signal SIGUSR1, SIGINT or SIGHUP"
|
|
break 2
|
|
;;
|
|
137|143) ## killed SIGKILL, SIGTERM, SIGINT
|
|
echo "!! Rsync received $(kill -l "$errlvl")"
|
|
echo " .. Full interruption while $d -> $dest and after $((SECONDS - start))s"
|
|
append_trim "${state_dir}/${session_id}-fail" \
|
|
"$dest $d $((SECONDS - start)) signal: $(kill -l "$errlvl")"
|
|
break 2
|
|
;;
|
|
0)
|
|
echo "Rsync finished with success $d -> $dest in $((SECONDS - start))s"
|
|
append_trim "${state_dir}/${session_id}-success" \
|
|
"$dest $d $((SECONDS - start)) OK"
|
|
break
|
|
;;
|
|
*)
|
|
errlvls+=("$errlvl")
|
|
echo "!! Rsync failed with an errorlevel $errlvl after $((SECONDS - start))s since start."
|
|
if [ "$retry" -lt 3 ]; then
|
|
echo "!! Triggering a retry ($((++retry))/3)"
|
|
continue
|
|
else
|
|
echo "!! Tried 3 times, bailing out."
|
|
echo " .. interruption of $d -> $dest after $((SECONDS - start))s"
|
|
append_trim "${state_dir}/${session_id}-fail" \
|
|
"$dest $d $((SECONDS - start))" \
|
|
"Failed after 3 retries (errorlevels: ${errlvls[@]})"
|
|
break
|
|
fi
|
|
;;
|
|
esac
|
|
done
|
|
if [ -n "$has_exclude_pattern" ]; then
|
|
rm -fv "$tmp_exclude_patterns"
|
|
fi
|
|
done
|
|
done
|