#!/bin/bash #:- . /etc/shlib #:- include common include parse include process depends shyaml lock [ "$UID" != "0" ] && echo "You must be root." && exit 1 ## ## Here's an example crontab: ## ## SHELL=/bin/sh ## PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin ## ## 49 */2 * * * root mirror-dir -d core-05.0k.io:10023 -u rsync /etc /home /opt/apps 2>&1 | logger -t mirror-dir ## usage="usage: $exname -d DEST1 [-d DEST2 [...]] [-u USER] [DIR1 [DIR2 ...]] Preserve as much as possible the source structure, keeping hard-links, acl, exact numerical uids and gids, and being able to resume in very large files. Options: DIR1 ... DIRn Local directories that should be mirrored on destination(s). examples: /etc /home /var/backups If no directories are provided, the config file root entries will be used all as destination to copy. -d DESTn Can be repeated. Specifies host destination towards which files will be send. Note that you can specify port number after a colon and a bandwidth limit for rsync after a '/'. examples: -d liszt.musicalta:10022 -d 10.8.0.19/200 -u USER (default: 'rsync') Local AND destination user to log as at both ends to transfer file. This local user need to have a NOPASSWD ssh login towards it's account on destination. This destination account should have full permissions access without passwd to write with rsync-server in the destination directory. -h STORE (default is taken of the hostname file) Set the destination store, this is the name of the directory where the files will all directories will be copied. Beware ! if 2 hosts use the same store, this means they'll conflictingly update the same destination directory. Only use this if you know what you are doing. " dests=() source_dirs=() hostname= while [ "$#" != 0 ]; do case "$1" in "-d") dests+=("$2") shift ;; "-h") hostname="$2" shift ;; "-u") user="$2" shift ;; *) source_dirs+=("$1") ;; esac shift done if test -z "$hostname"; then hostname=$(hostname) fi if test -z "$hostname"; then die "Couldn't figure a valid hostname. Please specify one with \`\`-h STORENAME\`\`." fi user=${user:-rsync} dest_path=/var/mirror/$hostname config_file="/etc/$exname/config.yml" if [ "${#source_dirs[@]}" == 0 ]; then if [ -e "$config_file" ]; then echo "No source provided on command line.. " echo " ..so reading '$config_file' for default sources..." source_dirs=($(eval echo $(shyaml get-values default.sources < "$config_file"))) fi if [ "${#source_dirs[@]}" == 0 ]; then err "You must specify at least one source directory to mirror" \ "on command line (or in a config file)." print_usage exit 1 fi fi echo "Sources directories are: ${source_dirs[@]}" if [ "${#dests[@]}" == 0 ]; then err "You must specify at least a destination." print_usage exit 1 fi rsync_options=(${RSYNC_OPTIONS:-}) ssh_options=(${SSH_OPTIONS:-}) get_exclude_patterns() { local dir="$1" [ -e "$config_file" ] || return cat "$config_file" | shyaml get-values-0 "${dir//.\\./}.exclude" 2>/dev/null } for dest in "${dests[@]}"; do for d in "${source_dirs[@]}"; do current_rsync_options=("${rsync_options[@]}") if [[ "$dest" == *"/"* ]]; then current_rsync_options+=("--bwlimit" "${dest##*/}") dest="${dest%/*}" fi if [[ "$dest" == *":"* ]]; then ssh_options+=("-p" "${dest#*:}") dest="${dest%%:*}" fi dirpath="$(dirname "$d")" if [ "$dirpath" == "/" ]; then dir="/$(basename "$d")" else dir="$dirpath/$(basename "$d")" fi [ -d "$dir" ] || { warn "ignoring '$dir' as it is not existing." continue } lock_label=$exname-$hostname-$(echo "$dest$d" | md5_compat | cut -c 1-6) tmp_exclude_patterns=/tmp/${lock_label}.exclude_patterns.tmp ## Adding the base of the dir if required... seems necessary with ## the rsync option that replicate the full path. has_exclude_pattern= while read-0 exclude_dir; do if [ -z "$has_exclude_pattern" ]; then echo "Adding exclude patterns..." >&2 has_exclude_pattern=1 fi if [[ "$exclude_dir" == "/"* ]]; then exclude_dir="$dir${exclude_dir}" fi echo " - $exclude_dir" >&2 p0 "$exclude_dir" done < <(get_exclude_patterns "$dir") > "$tmp_exclude_patterns" if [ -n "$has_exclude_pattern" ]; then current_rsync_options+=("-0" "--exclude-from"="$tmp_exclude_patterns") else echo "No exclude patterns for '$dir'." fi echo --------------------------------- echo "Starting rsync: $d -> $dest ($(date))" echo nice -n 15 \ rsync "${current_rsync_options[@]}" -azvARH \ -e "'sudo -u $user ssh ${ssh_options[*]}'" \ --delete --delete-excluded \ --partial --partial-dir .rsync-partial \ --numeric-ids "$dir/" "$user@$dest":"$dest_path" start="$SECONDS" retry=1 while true; do lock "$lock_label" -v -D -k -- \ nice -n 15 \ rsync "${current_rsync_options[@]}" -azvARH \ -e "sudo -u $user ssh ${ssh_options[*]}" \ --delete --delete-excluded \ --partial --partial-dir .rsync-partial \ --numeric-ids "$dir/" "$user@$dest":"$dest_path" errlvl="$?" case "$errlvl" in 20) ## Received SIGUSR1, SIGINTT echo "!! Rsync received SIGUSR1 or SIGINT." echo " .. Full interruption while $d -> $dest and after $((SECONDS - start))s" break 2 ;; 137|143) ## killed SIGKILL, SIGTERM, SIGINT echo "!! Rsync received $(kill -l "$errlvl")" echo " .. Full interruption while $d -> $dest and after $((SECONDS - start))s" break 2 ;; 0) echo "Rsync finished with success $d -> $dest in $((SECONDS - start))s" break ;; *) echo "!! Rsync failed with an errorlevel $errlvl after $((SECONDS - start))s since start." if [ "$retry" -lt 3 ]; then echo "!! Triggering a retry ($((++retry))/3)" continue else echo "!! Tried 3 times, bailing out." echo " .. interruption of $d -> $dest after $((SECONDS - start))s" break fi ;; esac done if [ -n "$has_exclude_pattern" ]; then rm -fv "$tmp_exclude_patterns" fi done done