You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

229 lines
6.9 KiB

  1. #!/bin/bash
  2. #:-
  3. . /etc/shlib
  4. #:-
  5. include common
  6. include parse
  7. include process
  8. depends shyaml lock
  9. [ "$UID" != "0" ] && echo "You must be root." && exit 1
  10. ##
  11. ## Here's an example crontab:
  12. ##
  13. ## SHELL=/bin/sh
  14. ## PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
  15. ##
  16. ## 49 */2 * * * root mirror-dir -d core-05.0k.io:10023 -u rsync /etc /home /opt/apps 2>&1 | logger -t mirror-dir
  17. ##
  18. usage="usage: $exname -d DEST1 [-d DEST2 [...]] [-u USER] [DIR1 [DIR2 ...]]
  19. Preserve as much as possible the source structure, keeping hard-links, acl,
  20. exact numerical uids and gids, and being able to resume in very large files.
  21. Options:
  22. DIR1 ... DIRn
  23. Local directories that should be mirrored on destination(s).
  24. examples: /etc /home /var/backups
  25. If no directories are provided, the config file root
  26. entries will be used all as destination to copy.
  27. -d DESTn
  28. Can be repeated. Specifies host destination towards which
  29. files will be send. Note that you can specify port number after
  30. a colon and a bandwidth limit for rsync after a '/'.
  31. examples: -d liszt.musicalta:10022 -d 10.8.0.19/200
  32. -u USER (default: 'rsync')
  33. Local AND destination user to log as at both ends to transfer file.
  34. This local user need to have a NOPASSWD ssh login towards it's
  35. account on destination. This destination account should have
  36. full permissions access without passwd to write with rsync-server
  37. in the destination directory.
  38. -h STORE (default is taken of the hostname file)
  39. Set the destination store, this is the name of the directory where
  40. the files will all directories will be copied. Beware ! if 2 hosts
  41. use the same store, this means they'll conflictingly update the
  42. same destination directory. Only use this if you know what you
  43. are doing.
  44. "
  45. dests=()
  46. source_dirs=()
  47. hostname=
  48. while [ "$#" != 0 ]; do
  49. case "$1" in
  50. "-d")
  51. dests+=("$2")
  52. shift
  53. ;;
  54. "-h")
  55. hostname="$2"
  56. shift
  57. ;;
  58. "-u")
  59. user="$2"
  60. shift
  61. ;;
  62. *)
  63. source_dirs+=("$1")
  64. ;;
  65. esac
  66. shift
  67. done
  68. if test -z "$hostname"; then
  69. hostname=$(hostname)
  70. fi
  71. if test -z "$hostname"; then
  72. die "Couldn't figure a valid hostname. Please specify one with \`\`-h STORENAME\`\`."
  73. fi
  74. user=${user:-rsync}
  75. dest_path=/var/mirror/$hostname
  76. config_file="/etc/$exname/config.yml"
  77. if [ "${#source_dirs[@]}" == 0 ]; then
  78. if [ -e "$config_file" ]; then
  79. echo "No source provided on command line.. "
  80. echo " ..so reading '$config_file' for default sources..."
  81. source_dirs=($(eval echo $(shyaml get-values default.sources < "$config_file")))
  82. fi
  83. if [ "${#source_dirs[@]}" == 0 ]; then
  84. err "You must specify at least one source directory to mirror" \
  85. "on command line (or in a config file)."
  86. print_usage
  87. exit 1
  88. fi
  89. fi
  90. echo "Sources directories are: ${source_dirs[@]}"
  91. if [ "${#dests[@]}" == 0 ]; then
  92. err "You must specify at least a destination."
  93. print_usage
  94. exit 1
  95. fi
  96. rsync_options=(${RSYNC_OPTIONS:-})
  97. ssh_options=(${SSH_OPTIONS:-})
  98. get_exclude_patterns() {
  99. local dir="$1"
  100. [ -e "$config_file" ] || return
  101. cat "$config_file" | shyaml get-values-0 "$(echo "$dir" | sed -r 's%\.%\\.%g').exclude"
  102. }
  103. for dest in "${dests[@]}"; do
  104. for d in "${source_dirs[@]}"; do
  105. current_rsync_options=("${rsync_options[@]}")
  106. if [[ "$dest" == *"/"* ]]; then
  107. current_rsync_options+=("--bwlimit" "${dest##*/}")
  108. dest="${dest%/*}"
  109. fi
  110. if [[ "$dest" == *":"* ]]; then
  111. ssh_options+=("-p" "${dest#*:}")
  112. dest="${dest%%:*}"
  113. fi
  114. dirpath="$(dirname "$d")"
  115. if [ "$dirpath" == "/" ]; then
  116. dir="/$(basename "$d")"
  117. else
  118. dir="$dirpath/$(basename "$d")"
  119. fi
  120. [ -d "$dir" ] || {
  121. warn "ignoring '$dir' as it is not existing."
  122. continue
  123. }
  124. lock_label=$exname-$hostname-$(echo "$dest" | md5_compat | cut -f 1 -d " ")
  125. exclude_patterns="$(get_exclude_patterns "$dir")"
  126. tmp_exclude_patterns=/tmp/${lock_label}.$(echo "$d" | md5_compat | cut -f 1 -d " ").exclude_patterns.tmp
  127. if [ "$exclude_patterns" ]; then
  128. echo "Adding exclude patterns..."
  129. ## Adding the base of the dir if required... seems necessary with
  130. ## the rsync option that replicate the full path.
  131. while read-0 exclude_dir; do
  132. if [[ "$exclude_dir" == "/"* ]]; then
  133. echo -en "$dir""$(echo "$exclude_dir" | cut -c 1-)\0"
  134. else
  135. echo -en "$exclude_dir\0"
  136. fi
  137. done < <(get_exclude_patterns "$dir") > "$tmp_exclude_patterns"
  138. cat "$tmp_exclude_patterns" | xargs -0 -n 1 echo
  139. current_rsync_options=("-0" "--exclude-from"="$tmp_exclude_patterns" "${current_rsync_options[@]}")
  140. else
  141. echo "No exclude patterns for '$dir'."
  142. fi
  143. echo ---------------------------------
  144. date
  145. echo nice -n 15 \
  146. rsync "${current_rsync_options[@]}" -azvARH \
  147. -e "'sudo -u $user ssh ${ssh_options[*]}'" \
  148. --delete --delete-excluded \
  149. --partial --partial-dir .rsync-partial \
  150. --numeric-ids "$dir/" "$user@$dest":"$dest_path"
  151. retry=1
  152. while true; do
  153. lock "$lock_label" -v -D -k -- \
  154. nice -n 15 \
  155. rsync "${current_rsync_options[@]}" -azvARH \
  156. -e "sudo -u $user ssh ${ssh_options[*]}" \
  157. --delete --delete-excluded \
  158. --partial --partial-dir .rsync-partial \
  159. --numeric-ids "$dir/" "$user@$dest":"$dest_path"
  160. errlvl="$?"
  161. case "$errlvl" in
  162. 20) ## Received SIGUSR1, SIGINTT
  163. echo "!! Rsync received SIGUSR1 or SIGINT"
  164. break 2
  165. ;;
  166. 137|143) ## killed SIGKILL, SIGTERM, SIGINT
  167. echo "!! Rsync received $(kill -l "$errlvl")"
  168. break 2
  169. ;;
  170. 0)
  171. echo "Rsync finished with success."
  172. break
  173. ;;
  174. *)
  175. echo "!! Rsync failed with an errorlevel $errlvl."
  176. if [ "$retry" -lt 3 ]; then
  177. echo "!! Triggering a retry ($((++retry))/3)"
  178. continue
  179. else
  180. echo "!! Tried 3 times, bailing out."
  181. break
  182. fi
  183. ;;
  184. esac
  185. done
  186. rm -fv "$tmp_exclude_patterns"
  187. done
  188. done