You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1195 lines
38 KiB

  1. #!/bin/bash
  2. . /etc/shlib
  3. include common
  4. include parse
  5. include cmdline
  6. include config
  7. [[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true
  8. version=0.1
  9. desc='Manage 0k related installs'
  10. help=""
  11. ##
  12. ## Functions
  13. ##
  14. is-port-open() {
  15. local host="$1" port="$2" timeout=5
  16. start="$SECONDS"
  17. debug "Testing if $host's port $2 is open ..."
  18. while true; do
  19. timeout 1 bash -c "</dev/tcp/${host}/${port}" >/dev/null 2>&1 && break
  20. sleep 0.2
  21. if [ "$((SECONDS - start))" -gt "$timeout" ]; then
  22. return 1
  23. fi
  24. done
  25. debug " .. $host's port $2 is open."
  26. return 0
  27. }
  28. resolve() {
  29. local ent hostname="$1"
  30. debug "Resolving $1 ..."
  31. if ent=$(getent ahosts "$hostname"); then
  32. ent=$(echo "$ent" | egrep ^"[0-9]+.[0-9]+.[0-9]+.[0-9]+\s+" | \
  33. head -n 1 | awk '{ print $1 }')
  34. debug " .. resolved $1 to $ent."
  35. echo "$ent"
  36. else
  37. debug " .. couldn't resolve $1."
  38. return 1
  39. fi
  40. }
  41. set_errlvl() { return "${1:-1}"; }
  42. export master_pid=$$
  43. ssh:open() {
  44. local hostname ssh_cmd ssh_options
  45. ssh_cmd=(ssh)
  46. ssh_options=()
  47. while [ "$#" != 0 ]; do
  48. case "$1" in
  49. "--stdin-password")
  50. ssh_cmd=(sshpass "${ssh_cmd[@]}")
  51. ;;
  52. -o)
  53. ssh_options+=("$1" "$2")
  54. shift
  55. ;;
  56. *)
  57. [ -z "$hostname" ] && hostname="$1" || {
  58. err "Surnumerous positional argument '$1'. Expecting only hostname."
  59. return 1
  60. }
  61. ;;
  62. esac
  63. shift
  64. done
  65. full_cmd=(
  66. "${ssh_cmd[@]}"
  67. -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
  68. -o ControlMaster=auto -o ControlPersist=900 \
  69. -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
  70. "${ssh_options[@]}" \
  71. "$hostname" "$@" -- true)
  72. "${full_cmd[@]}" >/dev/null 2>&1 || {
  73. err "Failed: ${full_cmd[*]}"
  74. return 1
  75. }
  76. trap_add EXIT,INT 'ssh:quit "$hostname"'
  77. }
  78. ssh:open-try() {
  79. local opts hostnames
  80. opts=()
  81. hostnames=()
  82. while [ "$#" != 0 ]; do
  83. case "$1" in
  84. -o)
  85. opts+=("$1" "$2")
  86. shift
  87. ;;
  88. *)
  89. hostnames+=("$1")
  90. ;;
  91. esac
  92. shift
  93. done
  94. password=''
  95. for host in "${hostnames[@]}"; do
  96. debug "Trying $host with publickey."
  97. ssh:open -o PreferredAuthentications=publickey \
  98. "${opts[@]}" \
  99. "$host" >/dev/null 2>&1 && {
  100. echo "$host"$'\n'"$password"$'\n'
  101. return 0
  102. }
  103. debug " .. failed connecting to $host with publickey."
  104. done
  105. local times=0 password
  106. while [ "$((++times))" -le 3 ]; do
  107. read -sp "$HOST's password: " password
  108. errlvl="$?"
  109. echo >&2
  110. if [ "$errlvl" -gt 0 ]; then
  111. exit 1
  112. fi
  113. for host in "${hostnames[@]}"; do
  114. debug "Trying $host with password ($times/3)"
  115. echo "$password" | ssh:open -o PreferredAuthentications=password \
  116. --stdin-password \
  117. "${opts[@]}" \
  118. "$host" >/dev/null 2>&1 && {
  119. echo "$host"$'\n'"$password"$'\n'
  120. return 0
  121. }
  122. debug " .. failed connecting to $host with password."
  123. done
  124. err "login failed. Try again... ($((times+1))/3)"
  125. done
  126. return 1
  127. }
  128. ssh:run() {
  129. local hostname="$1" ssh_options cmd
  130. shift
  131. ssh_options=()
  132. cmd=()
  133. while [ "$#" != 0 ]; do
  134. case "$1" in
  135. "--")
  136. shift
  137. cmd+=("$@")
  138. break
  139. ;;
  140. *)
  141. ssh_options+=("$1")
  142. ;;
  143. esac
  144. shift
  145. done
  146. ## XXXvlab: keeping in case we need some debug
  147. # debug "$DARKCYAN$hostname$NORMAL $WHITE\$$NORMAL" "$@"
  148. # debug "Running cmd: ${cmd[*]}"
  149. # for arg in "${cmd[@]}"; do
  150. # debug "$arg"
  151. # done
  152. {
  153. {
  154. ssh -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
  155. -o ControlMaster=auto -o ControlPersist=900 \
  156. -o "StrictHostKeyChecking=no" \
  157. "${ssh_options[@]}" "$hostname" -- "${cmd[@]}"
  158. } 3>&1 1>&2 2>&3 ## | sed -r "s/^/$DARKCYAN$hostname$NORMAL $DARKRED\!$NORMAL /g"
  159. set_errlvl "${PIPESTATUS[0]}"
  160. } 3>&1 1>&2 2>&3
  161. }
  162. ssh:quit() {
  163. local hostname="$1"
  164. shift
  165. ssh -o ControlPath=/tmp/ssh-control-master-${master_pid} \
  166. -o ControlMaster=auto -o ControlPersist=900 -O exit \
  167. "$hostname" 2>/dev/null
  168. }
  169. is_ovh_domain_name() {
  170. local domain="$1"
  171. [[ "$domain" == *.ovh.net ]] && return 0
  172. [[ "$domain" == "ns"*".ip-"*".eu" ]] && return 0
  173. return 1
  174. }
  175. is_ovh_hostname() {
  176. local domain="$1"
  177. [[ "$domain" =~ ^vps-[0-9a-f]*$ ]] && return 0
  178. [[ "$domain" =~ ^vps[0-9]*$ ]] && return 0
  179. return 1
  180. }
  181. vps_connection_check() {
  182. local vps="$1"
  183. ip=$(resolve "$vps") ||
  184. { echo "${DARKRED}no-resolve${NORMAL}"; return 1; }
  185. is-port-open "$ip" "22" </dev/null ||
  186. { echo "${DARKRED}no-port-22-open${NORMAL}"; return 1; }
  187. ssh:open -o ConnectTimeout=10 -o PreferredAuthentications=publickey \
  188. "root@$vps" >/dev/null 2>&1 ||
  189. { echo "${DARKRED}no-ssh-root-access${NORMAL}"; return 1; }
  190. }
  191. vps_check() {
  192. local vps="$1"
  193. vps_connection_check "$vps" </dev/null || return 1
  194. if size=$(
  195. echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 5 -d ' ' | cut -f 1 -d %" |
  196. ssh:run "root@$vps" -- bash); then
  197. if [ "$size" -gt "90" ]; then
  198. echo "${DARKRED}above-90%-disk-usage${NORMAL}"
  199. elif [ "$size" -gt "75" ]; then
  200. echo "${DARKYELLOW}above-75%-disk-usage${NORMAL}"
  201. fi
  202. else
  203. echo "${DARKRED}no-size${NORMAL}"
  204. fi </dev/null
  205. compose_content=$(ssh:run "root@$vps" -- cat /opt/apps/myc-deploy/compose.yml </dev/null) ||
  206. { echo "${DARKRED}no-compose${NORMAL}"; return 1; }
  207. echo "$compose_content" | grep backup >/dev/null 2>&1 ||
  208. { echo "${DARKRED}no-backup${NORMAL}"; return 1; }
  209. }
  210. backup:setup-rsync() {
  211. local admin="$1" vps="$2" server="$3" id="$4"
  212. [ -z "${BACKUP_SSH_SERVER}" ] || {
  213. err "Unexpected error: '\$BACKUP_SSH_SERVER' is already set in '$FUNCNAME'."
  214. return 1
  215. }
  216. BACKUP_SSH_OPTIONS=(-o StrictHostKeyChecking=no)
  217. if [[ "$server" == *":"* ]]; then
  218. BACKUP_SSH_OPTIONS+=(-p "${server#*:}")
  219. BACKUP_SSH_SERVER=${server%%:*}
  220. else
  221. BACKUP_SSH_SERVER="$server"
  222. fi
  223. if ! private_key=$(ssh "${BACKUP_SSH_OPTIONS[@]}" \
  224. "$admin"@"${BACKUP_SSH_SERVER}" request-recovery-key "$id"); then
  225. err "Couldn't request a recovery key for '$id' with account '$admin'."
  226. return 1
  227. fi
  228. if ! VPS_TMP_DIR=$(echo "mktemp -d" | ssh:run "root@$vps" -- bash); then
  229. err "Couldn't create a temporary directory on vps"
  230. return 1
  231. fi
  232. cat <<EOF | ssh:run "root@$vps" -- bash || return 1
  233. touch "$VPS_TMP_DIR/recover_key" &&
  234. chmod go-rwx "$VPS_TMP_DIR/recover_key" &&
  235. printf "%s\n" "$private_key" >> "$VPS_TMP_DIR/recover_key"
  236. EOF
  237. BACKUP_SSH_OPTIONS+=(-i "$VPS_TMP_DIR/recover_key" -l rsync)
  238. BACKUP_VPS_TARGET="$vps"
  239. BACKUP_IDENT="$id"
  240. echo "type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null" |
  241. ssh:run "root@$vps" -- bash || return 1
  242. }
  243. backup:rsync() {
  244. local ssh_options
  245. [ -n "${BACKUP_SSH_SERVER}" ] || {
  246. err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
  247. return 1
  248. }
  249. rsync_options=()
  250. while [[ "$1" == "-"* ]]; do
  251. rsync_options+=("$1")
  252. shift
  253. done
  254. local src="$1" dst="$2"
  255. cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash
  256. rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
  257. -azvArH --delete --delete-excluded \
  258. --partial --partial-dir .rsync-partial \
  259. --numeric-ids ${rsync_options[*]} \
  260. "${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "${dst}"
  261. EOF
  262. }
  263. backup:path_exists() {
  264. local src="$1"
  265. [ -n "${BACKUP_SSH_SERVER}" ] || {
  266. err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
  267. return 1
  268. }
  269. cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash >/dev/null 2>&1
  270. rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
  271. -nazvArH --numeric-ids \
  272. "${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "/tmp/dummy"
  273. EOF
  274. }
  275. file:vps_backup_recover() {
  276. local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"
  277. backup:rsync "${path}" "${vps_path}" || return 1
  278. if [[ "$path" == *"/" ]]; then
  279. if [ "$path" == "$vps_path"/ ]; then
  280. msg_target="Directory '$path'"
  281. else
  282. msg_target="Directory '$path' -> '$vps_path'"
  283. fi
  284. else
  285. if [ "$path" == "$vps_path" ]; then
  286. msg_target="File '$path'"
  287. else
  288. msg_target="File '$path' -> '$vps_path'"
  289. fi
  290. fi
  291. info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."
  292. }
  293. mailcow:vps_backup_recover() {
  294. local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"
  295. if ! compose_yml_files=$(cat <<EOF | ssh:run "root@$vps" -- bash
  296. urn=com.docker.compose.project
  297. docker ps -f "label=\$urn=mailcowdockerized" \
  298. --format="{{.Label \"\$urn.working_dir\"}}/{{.Label \"\$urn.config_files\"}}" |
  299. uniq
  300. EOF
  301. ); then
  302. err "Couldn't get list of running projects"
  303. return 1
  304. fi
  305. stopped_containers=
  306. if [ -n "$compose_yml_files" ]; then
  307. echo "Found running mailcowdockerized containers" >&2
  308. if [[ "$compose_yml_files" == *$'\n'* ]]; then
  309. err "Running containers are confusing, did not find only one mailcowdockerized project."
  310. return 1
  311. fi
  312. if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
  313. ## For some reason, sometimes $urn.config_files holds an absolute path
  314. compose_yml_files=/${compose_yml_files#*//}
  315. if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
  316. err "Running containers are confusing, they don't point to an existing docker-compose.yml."
  317. return 1
  318. fi
  319. fi
  320. echo "Containers where launched from '$compose_yml_files'" >&2
  321. COMPOSE_FILE="$compose_yml_files"
  322. ENV_FILE="${COMPOSE_FILE%/*}/.env"
  323. if ! echo "[ -e \"${ENV_FILE}\" ]" | ssh:run "root@$vps" -- bash ; then
  324. err "Running containers are confusing, docker-compose.yml has no '.env' next to it."
  325. return 1
  326. fi
  327. echo "${WHITE}Bringing mailcowdockerized down${NORMAL}"
  328. echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
  329. ssh:run "root@$vps" -- bash
  330. stopped_containers=1
  331. fi
  332. if [[ "$path" == "/"* ]]; then
  333. ##
  334. ## Additional intelligence to simple file copy
  335. ##
  336. if [[ "$path" == "/var/lib/docker/volumes/mailcowdockerized_*-vol-1/_data"* ]]; then
  337. volume_name=${path#/var/lib/docker/volumes/}
  338. volume_name=${volume_name%%/*}
  339. volume_dir=${path%%"$volume_name"*}
  340. ## Create volumes if not existent
  341. if ! ssh:run "root@$vps" -- "
  342. [ -d '${volume_dir}' ] ||
  343. docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
  344. [ -d '${volume_dir}' ]
  345. "; then
  346. err "Couldn't find nor create '${volume_dir}'."
  347. return 1
  348. fi
  349. fi
  350. echo "${WHITE}Sync from backup ${path} to VPS ${vps_path}${NORMAL}" >&2
  351. backup:rsync "${path}" "${vps_path}" || return 1
  352. if [[ "$path" == *"/" ]]; then
  353. if [ "$path" == "$vps_path"/ ]; then
  354. msg_target="Directory '$path'"
  355. else
  356. msg_target="Directory '$path' -> '$vps_path'"
  357. fi
  358. else
  359. if [ "$path" == "$vps_path" ]; then
  360. msg_target="File '$path'"
  361. else
  362. msg_target="File '$path' -> '$vps_path'"
  363. fi
  364. fi
  365. else
  366. ALL_TARGETS=(mailcow postfix rspamd redis crypt vmail{,-attachments} mysql)
  367. if [[ -n "$path" ]]; then
  368. targets=()
  369. bad_targets=()
  370. for target in ${path//,/ }; do
  371. if [[ " ${ALL_TARGETS[*]} " != *" $target "* ]]; then
  372. bad_targets+=("$target")
  373. fi
  374. targets+=("$target")
  375. done
  376. if [ "${#bad_targets[@]}" -gt 0 ]; then
  377. bad_target_msg=$(printf "%s, " "${bad_targets[@]}")
  378. err "Unknown components: ${bad_target_msg%, }. These are allowed components:"
  379. printf " - %s\n" "${ALL_TARGETS[@]}" >&2
  380. return 1
  381. fi
  382. msg_target="Partial mailcow backup"
  383. else
  384. targets=("${ALL_TARGETS[@]}")
  385. msg_target="Full mailcow backup"
  386. fi
  387. for target in "${targets[@]}"; do
  388. case "$target" in
  389. postfix|rspamd|redis|crypt|vmail|vmail-attachments)
  390. volume_name="mailcowdockerized_${target}-vol-1"
  391. volume_dir="/var/lib/docker/volumes/${volume_name}/_data"
  392. if ! backup:path_exists "${volume_dir}/"; then
  393. warn "No '$volume_name' in backup. This might be expected."
  394. continue
  395. fi
  396. ## Create volumes if not existent
  397. if ! ssh:run "root@$vps" -- "
  398. [ -d '${volume_dir}' ] ||
  399. docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
  400. [ -d '${volume_dir}' ]
  401. "; then
  402. err "Couldn't find nor create '${volume_dir}'."
  403. return 1
  404. fi
  405. echo "${WHITE}Downloading of $volume_name${NORMAL}"
  406. backup:rsync "${volume_dir}/" "${volume_dir}" || return 1
  407. ;;
  408. mailcow)
  409. ## Mailcow git base
  410. COMPOSE_FILE=
  411. for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
  412. backup:path_exists "${mailcow_dir}/" || continue
  413. ## this possibly change last value
  414. COMPOSE_FILE="$mailcow_dir/docker-compose.yml"
  415. ENV_FILE="$mailcow_dir/.env"
  416. echo "${WHITE}Download of $mailcow_dir${NORMAL}"
  417. backup:rsync "${mailcow_dir}"/ "${mailcow_dir}" || return 1
  418. break
  419. done
  420. if [ -z "$COMPOSE_FILE" ]; then
  421. err "Can't find mailcow base installation path in backup."
  422. return 1
  423. fi
  424. ;;
  425. mysql)
  426. if [ -z "$COMPOSE_FILE" ]; then
  427. ## Mailcow git base
  428. compose_files=()
  429. for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
  430. ssh:run "root@$vps" -- "[ -e \"$mailcow_dir/docker-compose.yml\" ]" || continue
  431. ## this possibly change last value
  432. compose_files+=("$mailcow_dir/docker-compose.yml")
  433. done
  434. if [ "${#compose_files[@]}" == 0 ]; then
  435. err "No compose file found for mailcow installation."
  436. return 1
  437. elif [ "${#compose_files[@]}" -gt 1 ]; then
  438. err "Multiple compose files for mailcow found:"
  439. for f in "${compose_files[@]}"; do
  440. echo " - $f" >&2
  441. done
  442. echo "Can't decide which to use for mounting mysql container." >&2
  443. return 1
  444. fi
  445. COMPOSE_FILE="${compose_files[0]}"
  446. ENV_FILE="${COMPOSE_FILE%/*}/.env"
  447. if ! ssh:run "root@$vps" -- "[ -e \"${COMPOSE_FILE%/*}/.env\" ]"; then
  448. err "No env file in '$ENV_FILE' found."
  449. return 1
  450. fi
  451. fi
  452. ## Mysql database
  453. echo "${WHITE}Downloading last backup of mysql backups${NORMAL}"
  454. backup:rsync "/var/backups/mysql/" "/var/backups/mysql" || return 1
  455. if ! env_content=$(echo "cat '$ENV_FILE'" | ssh:run "root@$vps" -- bash); then
  456. err "Can't access env file: '$ENV_FILE'."
  457. return 1
  458. fi
  459. root_password=$(printf "%s\n" "$env_content" | grep ^DBROOT= | cut -f 2 -d =)
  460. echo "${WHITE}Bringing mysql-mailcow up${NORMAL}"
  461. if ! image=$(cat <<EOF | ssh:run "root@$vps" -- bash
  462. shyaml get-value services.mysql-mailcow.image < "${COMPOSE_FILE}"
  463. EOF
  464. ); then
  465. err "Failed to get image name of service 'mysql-mailcow' in 'compose.yml'."
  466. return 1
  467. fi
  468. if [ -z "$(ssh:run "root@$vps" -- docker images -q "$image")" ]; then
  469. info "Image '$image' not available, pull it."
  470. if ! ssh:run "root@$vps" -- \
  471. docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
  472. pull mysql-mailcow; then
  473. err "Failed to pull image of service 'mysql-mailcow'."
  474. return 1
  475. fi
  476. fi
  477. if ! container_id=$(cat <<EOF | ssh:run "root@$vps" -- bash
  478. echo "[client]
  479. password=$root_password" > "$VPS_TMP_DIR/my.cnf"
  480. docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
  481. run -d \
  482. -v "$VPS_TMP_DIR/my.cnf:/root/.my.cnf:ro" \
  483. mysql-mailcow
  484. EOF
  485. ); then
  486. err "Failed to bring up mysql-mailcow"
  487. return 1
  488. fi
  489. START="$SECONDS"
  490. retries=0
  491. timeout=600
  492. while true; do
  493. ((retries++))
  494. echo " waiting for mysql db..." \
  495. "(retry $retries, $(($SECONDS - $START))s elapsed, timeout is ${timeout}s)" >&2
  496. cat <<EOF | ssh:run "root@$vps" -- bash && break
  497. echo "SELECT 1;" | docker exec -i "$container_id" mysql >/dev/null 2>&1
  498. EOF
  499. if (($SECONDS - $START > $timeout)); then
  500. err "Failed to connect to mysql-mailcow."
  501. echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
  502. ssh:run "root@$vps" -- bash
  503. return 1
  504. fi
  505. sleep 0.4
  506. done
  507. DBUSER=$(printf "%s\n" "$env_content" | grep ^DBUSER= | cut -f 2 -d =)
  508. DBPASS=$(printf "%s\n" "$env_content" | grep ^DBPASS= | cut -f 2 -d =)
  509. echo "${WHITE}Uploading mysql dump${NORMAL}"
  510. cat <<EOF | ssh:run "root@$vps" -- bash
  511. echo "
  512. DROP DATABASE IF EXISTS mailcow;
  513. CREATE DATABASE mailcow;
  514. GRANT ALL PRIVILEGES ON mailcow.* TO '$DBUSER'@'%' IDENTIFIED BY '$DBPASS';
  515. " | docker exec -i "$container_id" mysql
  516. zcat /var/backups/mysql/mailcow/*.gz | docker exec -i "$container_id" mysql mailcow
  517. EOF
  518. if [ "$?" != 0 ]; then
  519. err "Failed to load mysql dump."
  520. echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
  521. ssh:run "root@$vps" -- bash
  522. return 1
  523. fi
  524. echo "${WHITE}Bringing mysql-mailcow down${NORMAL}"
  525. echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
  526. ssh:run "root@$vps" -- bash
  527. ;;
  528. *)
  529. err "Unknown component '$target'. Bailing out."
  530. return 1
  531. esac
  532. done
  533. ssh:run "root@$vps" -- "rm -rf '$VPS_TMP_DIR'"
  534. fi
  535. if [ -n "$stopped_containers" ]; then
  536. echo "${WHITE}Starting mailcow${NORMAL}" >&2
  537. echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" up -d" |
  538. ssh:run "root@$vps" -- bash
  539. fi
  540. info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."
  541. }
  542. vps_backup_recover() {
  543. local vps="$1" admin server id path rtype force type
  544. read-0 admin server id path rtype force
  545. if [[ "$vps" == *":"* ]]; then
  546. vps_path=${vps#*:}
  547. vps=${vps%%:*}
  548. else
  549. vps_path=
  550. fi
  551. vps_connection_check "$vps" </dev/null || {
  552. err "Failed to access '$vps'."
  553. return 1
  554. }
  555. if type=$(ssh:run "root@$vps" -- vps get-type); then
  556. info "VPS $vps seems to be of ${WHITE}$type${NORMAL} type"
  557. else
  558. warn "Couldn't find type of vps '$vps' (command 'vps get-type' failed on vps)."
  559. fi
  560. if [ -z "$path" ]; then
  561. if [ -n "$vps_path" ]; then
  562. err "You can't provide a VPS with path as destination if you don't provide a path in backup source."
  563. return 1
  564. fi
  565. info "No path provided in backup, so we assume you want ${WHITE}full recovery${NORMAL}."
  566. if [ "$rtype" != "$type" ]; then
  567. if [ -n "$force" ]; then
  568. warn "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type."
  569. else
  570. err "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type. (use \`\`-f\`\` to force)"
  571. return 1
  572. fi
  573. fi
  574. else
  575. if [ "$path" == "/" ]; then
  576. if [ -z "$vps_path" ]; then
  577. err "Recovery of '/' (full backup files) requires that you provide a vps path also."
  578. return 1
  579. fi
  580. if [ "$vps_path" == "/" ]; then
  581. err "Recovery of '/' (full backup files) requires that you provide" \
  582. "a vps path different from '/' also."
  583. return 1
  584. fi
  585. fi
  586. fi
  587. ## Sets VPS and internal global variable to allow rsync to work
  588. ## from vps to backup server.
  589. backup:setup-rsync "$admin" "$vps" "$server" "$id" || return 1
  590. if [[ "$path" == "/"* ]]; then
  591. if ! backup:path_exists "${path}"; then
  592. err "File or directory '$path' not found in backup."
  593. return 1
  594. fi
  595. if [ -z "$vps_path" ]; then
  596. if [[ "$path" != *"/" ]] && backup:path_exists "${path}"/ ; then
  597. path="$path/"
  598. fi
  599. vps_path=${path%/}
  600. vps_path=${vps_path:-/}
  601. fi
  602. fi
  603. case "$rtype-$type" in
  604. mailcow-*)
  605. ## Supports having $path and $vps_path set or unset, with additional behavior
  606. mailcow:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
  607. ;;
  608. *-*)
  609. if [[ "$path" == "/"* ]]; then
  610. ## For now, will require having $path and $vps_path set, no additional behaviors
  611. file:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
  612. else
  613. if [ -n "$path" ]; then
  614. err "Partial component recover of ${rtype:-unknown} backup type on" \
  615. "${type:-unknown} type VPS is not yet implemented."
  616. return 1
  617. else
  618. err "Full recover of ${rtype:-unknown} backup type on" \
  619. "${type:-unknown} type VPS is not yet implemented."
  620. return 1
  621. fi
  622. fi
  623. ;;
  624. esac
  625. }
  626. vps_install_backup() {
  627. local vps="$1" admin server
  628. vps_connection_check "$vps" </dev/null || return 1
  629. read-0 admin server
  630. if ! type=$(ssh:run "root@$vps" -- vps get-type); then
  631. err "Could not get type."
  632. return 1
  633. fi
  634. if ! out=$(ssh:run "root@$vps" -- vps install backup "$server" 2>&1); then
  635. err "Command 'vps install backup $server' on $vps failed:"
  636. echo "$out" | prefix " ${DARKGRAY}|${NORMAL} " >&2
  637. return 1
  638. fi
  639. out="${out%$'\n'}"
  640. out="${out#*$'\n'}"
  641. key="${out%\'*}"
  642. key="${key##*\'}"
  643. if ! [[ "$key" =~ ^"ssh-rsa "[a-zA-Z0-9/+]+" "[a-zA-Z0-9._-]+"@"[a-zA-Z0-9._-]+$ ]]; then
  644. err "Unexpected output from 'vps install backup $server'. Can't find key."
  645. echo "$out" | prefix " ${GRAY}|$NORMAL " >&2
  646. echo " Extracted key:" >&2
  647. echo "$key" | prefix " ${GRAY}|$NORMAL " >&2
  648. return 1
  649. fi
  650. if [ "$type" == "compose" ]; then
  651. if ! ssh:run "root@$vps" -- \
  652. docker exec myc_cron_1 \
  653. cat /etc/cron.d/rsync-backup >/dev/null 2>&1; then
  654. ssh:run "root@$vps" -- compose --debug up || {
  655. err "Command 'compose --debug up' failed."
  656. return 1
  657. }
  658. if ! ssh:run "root@$vps" -- \
  659. docker exec myc_cron_1 \
  660. cat /etc/cron.d/rsync-backup >/dev/null 2>&1; then
  661. err "Launched 'compose up' successfully but ${YELLOW}cron${NORMAL} container is not setup as expected."
  662. echo " Was waiting for existence of '/etc/cron.d/rsync-backup' in it." >&2
  663. return 1
  664. fi
  665. fi
  666. fi
  667. dest="$server"
  668. dest="${dest%/*}"
  669. ssh_options=()
  670. if [[ "$dest" == *":"* ]]; then
  671. port="${dest##*:}"
  672. dest="${dest%%:*}"
  673. ssh_options=(-p "$port")
  674. else
  675. port=""
  676. dest="${dest%%:*}"
  677. fi
  678. cmd=(ssh "${ssh_options[@]}" "$admin"@"$dest" ssh-key add "$key")
  679. echo "${WHITE}Launching:${NORMAL} ${cmd[@]}"
  680. "${cmd[@]}" || {
  681. err "Failed add key to backup server '$dest'."
  682. return 1
  683. }
  684. echo "${WHITE}Launching backup${NORMAL} from '$vps'"
  685. ssh:run "root@$vps" -- vps backup || {
  686. err "First backup failed to run."
  687. return 1
  688. }
  689. echo "Backup is ${GREEN}up and running${NORMAL}."
  690. }
  691. vps_udpate() {
  692. local vps="$1"
  693. vps_connection_check "$vps" || return 1
  694. ssh:run "root@$vps" -- myc-update </dev/null
  695. }
  696. vps_bash() {
  697. local vps="$1"
  698. vps_connection_check "$vps" </dev/null || return 1
  699. ssh:run "root@$vps" -- bash
  700. }
  701. vps_mux() {
  702. local fn="$1" vps_done VPS max_size vps
  703. shift
  704. VPS=($(printf "%s\n" "$@" | sort))
  705. max_size=0
  706. declare -A vps_done;
  707. new_vps=()
  708. for name in "${VPS[@]}"; do
  709. [ -n "${vps_done[$name]}" ] && {
  710. warn "duplicate vps '$name' provided. Ignoring."
  711. continue
  712. }
  713. vps_done["$name"]=1
  714. new_vps+=("$name")
  715. size_name="${#name}"
  716. [ "$max_size" -lt "${size_name}" ] &&
  717. max_size="$size_name"
  718. done
  719. settmpdir "_0KM_TMP_DIR"
  720. cat > "$_0KM_TMP_DIR/code"
  721. for vps in "${new_vps[@]}"; do
  722. label=$(printf "%-${max_size}s" "$vps")
  723. (
  724. {
  725. {
  726. "$fn" "$vps" < "$_0KM_TMP_DIR/code"
  727. } 3>&1 1>&2 2>&3 | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKRED\!$NORMAL /g"
  728. set_errlvl "${PIPESTATUS[0]}"
  729. } 3>&1 1>&2 2>&3 | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKGRAY\|$NORMAL /g"
  730. set_errlvl "${PIPESTATUS[0]}"
  731. ) &
  732. done
  733. wait
  734. }
  735. [ "$SOURCED" ] && return 0
  736. ##
  737. ## Command line processing
  738. ##
  739. cmdline.spec.gnu
  740. cmdline.spec.reporting
  741. cmdline.spec.gnu vps-setup
  742. cmdline.spec::cmd:vps-setup:run() {
  743. : :posarg: HOST 'Target host to check/fix ssh-access'
  744. : :optfla: --force,-f "Will force domain name change, even if
  745. current hostname was not recognized as
  746. an ovh domain name. "
  747. depends sshpass shyaml
  748. KEY_PATH="ssh-access.public-keys"
  749. local keys=$(config get-value -y "ssh-access.public-keys") || true
  750. if [ -z "$keys" ]; then
  751. err "No ssh publickeys configured in config file."
  752. echo " Looking for keys in ${WHITE}${KEY_PATH}${NORMAL}" \
  753. "in config file." >&2
  754. config:exists --message 2>&1 | prefix " "
  755. if [ "${PIPESTATUS[0]}" == "0" ]; then
  756. echo " Config file found in $(config:filename)"
  757. fi
  758. return 1
  759. fi
  760. local tkey=$(e "$keys" | shyaml get-type)
  761. if [ "$tkey" != "sequence" ]; then
  762. err "Value type of ${WHITE}${KEY_PATH}${NORMAL} unexpected (is $tkey, expecting sequence)."
  763. echo " Check content of $(config:filename), and make sure to use a sequence." >&2
  764. return 1
  765. fi
  766. local IP NAME keys host_pass_connected
  767. if ! IP=$(resolve "$HOST"); then
  768. err "'$HOST' name unresolvable."
  769. exit 1
  770. fi
  771. NAME="$HOST"
  772. if [ "$IP" != "$HOST" ]; then
  773. NAME="$HOST ($IP)"
  774. fi
  775. if ! is-port-open "$IP" "22"; then
  776. err "$NAME unreachable or port 22 closed."
  777. exit 1
  778. fi
  779. debug "Host $IP's port 22 is open."
  780. if ! host_pass_connected=$(ssh:open-try \
  781. {root,debian}@"$HOST"); then
  782. err "Could not connect to {root,debian}@$HOST with publickey nor password."
  783. exit 1
  784. fi
  785. read-0a host password <<<"$host_pass_connected"
  786. sudo_if_necessary=
  787. if [ "$password" -o "${host%%@*}" != "root" ]; then
  788. if ! ssh:run "$host" -- sudo -n true >/dev/null 2>&1; then
  789. err "Couldn't do a password-less sudo from $host."
  790. echo " This is not yet supported."
  791. exit 1
  792. else
  793. sudo_if_necessary=sudo
  794. fi
  795. fi
  796. Section Checking access
  797. while read-0 key; do
  798. prefix="${key%% *}"
  799. if [ "$prefix" != "ssh-rsa" ]; then
  800. err "Unsupported key:"$'\n'"$key"
  801. return 1
  802. fi
  803. label="${key##* }"
  804. Elt "considering adding key ${DARKYELLOW}$label${NORMAL}"
  805. dest="/root/.ssh/authorized_keys"
  806. if ssh:run "$host" -- $sudo_if_necessary grep "\"$key\"" "$dest" >/dev/null 2>&1 </dev/null; then
  807. print_info "already present"
  808. print_status noop
  809. Feed
  810. else
  811. if echo "$key" | ssh:run "$host" -- $sudo_if_necessary tee -a "$dest" >/dev/null; then
  812. print_info added
  813. else
  814. echo
  815. Feedback failure
  816. return 1
  817. fi
  818. Feedback success
  819. fi
  820. done < <(e "$keys" | shyaml get-values-0)
  821. Section Checking ovh hostname file
  822. Elt "Checking /etc/ovh-hostname"
  823. if ! ssh:run "$host" -- $sudo_if_necessary [ -e "/etc/ovh-hostname" ]; then
  824. print_info "creating"
  825. ssh:run "$host" -- $sudo_if_necessary cp /etc/hostname /etc/ovh-hostname
  826. ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
  827. Elt "Checking /etc/ovh-hostname: $ovhname"
  828. Feedback || return 1
  829. else
  830. ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
  831. Elt "Checking /etc/ovh-hostname: $ovhname"
  832. print_info "already present"
  833. print_status noop
  834. Feed
  835. fi
  836. if ! is_ovh_domain_name "$HOST" && ! str_is_ipv4 "$HOST"; then
  837. Section Checking hostname
  838. Elt "Checking /etc/hostname..."
  839. old_etc_hostname="$(ssh:run "$host" -- $sudo_if_necessary cat /etc/hostname)"
  840. if [ "$old_etc_hostname" != "$HOST" ]; then
  841. Elt "/etc/hostname is '$old_etc_hostname'"
  842. if is_ovh_hostname "$old_etc_hostname" || [ -n "$opt_force" ]; then
  843. Elt "Hostname '$old_etc_hostname' --> '$HOST'"
  844. print_info "creating"
  845. echo "$HOST" | ssh:run "$host" -- $sudo_if_necessary tee /etc/hostname >/dev/null &&
  846. ssh:run "$host" -- $sudo_if_necessary hostname "$HOST"
  847. Feedback || return 1
  848. else
  849. Elt "Hostname '$old_etc_hostname' isn't an ovh domain"
  850. print_info "no change"
  851. print_status noop
  852. Feed
  853. warn "Domain name was not changed because it was already set"
  854. echo " (use \`-f\` or \`--force\`) to force domain name change to $HOST." >&2
  855. fi
  856. else
  857. print_info "already set"
  858. print_status noop
  859. Feed
  860. fi
  861. Elt "Checking consistency between /etc/hostname and \`hostname\`..."
  862. etc_hostname="$(ssh:run "$host" -- $sudo_if_necessary cat /etc/hostname)"
  863. transient_hostname="$(ssh:run "$host" -- $sudo_if_necessary hostname)"
  864. if [ "$etc_hostname" != "$transient_hostname" ]; then
  865. print_info "change"
  866. ssh:run "$host" -- $sudo_if_necessary hostname "$etc_hostname"
  867. Feedback || return 1
  868. else
  869. print_info "consistent"
  870. print_status noop
  871. Feed
  872. fi
  873. else
  874. info "Not changing domain as '$HOST' doesn't seem to be final domain."
  875. fi
  876. }
  877. cmdline.spec.gnu vps-check
  878. cmdline.spec::cmd:vps-check:run() {
  879. : :posarg: [VPS...] 'Target host(s) to check'
  880. echo "" |
  881. vps_mux vps_check "${VPS[@]}"
  882. }
  883. cmdline.spec.gnu vps-install
  884. cmdline.spec::cmd:vps-install:run() {
  885. :
  886. }
  887. cmdline.spec.gnu backup
  888. cmdline.spec:vps-install:cmd:backup:run() {
  889. : :posarg: BACKUP_TARGET 'Backup target.
  890. (ie: myadmin@backup.domain.org:10023/256)'
  891. : :posarg: [VPS...] 'Target host(s) to check'
  892. if [ "${#VPS[@]}" == 0 ]; then
  893. warn "VPS list provided in command line is empty. Nothing will be done."
  894. return 0
  895. fi
  896. if ! [[ "$BACKUP_TARGET" == *"@"* ]]; then
  897. err "Missing admin account identifier in backup target."
  898. echo " Have you forgottent to specify an account, ie 'myadmin@<MYBACKUP_SERVER>' ?)"
  899. return 1
  900. fi
  901. admin=${BACKUP_TARGET%%@*}
  902. server=${BACKUP_TARGET#*@}
  903. p0 "$admin" "$server" |
  904. vps_mux vps_install_backup "${VPS[@]}"
  905. }
  906. cmdline.spec.gnu vps-backup
  907. cmdline.spec::cmd:vps-backup:run() {
  908. :
  909. }
  910. cmdline.spec.gnu ls
  911. cmdline.spec:vps-backup:cmd:ls:run() {
  912. : :posarg: BACKUP_ID 'Backup id.
  913. (ie: myadmin@backup.domain.org:10023)'
  914. if ! [[ "$BACKUP_ID" == *"@"* ]]; then
  915. err "Missing admin account identifier in backup id."
  916. echo " Have you forgottent to specify an admin account ?" \
  917. "ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
  918. return 1
  919. fi
  920. id=${BACKUP_ID##*#}
  921. BACKUP_TARGET=${BACKUP_ID%#*}
  922. admin=${BACKUP_TARGET%%@*}
  923. server=${BACKUP_TARGET#*@}
  924. ## XXXvlab: in this first implementation we expect to have access
  925. ## to the server main ssh port 22, so we won't use the provided port.
  926. ssh_options=()
  927. if [[ "$server" == *":"* ]]; then
  928. ssh_options+=(-p "${server#*:}")
  929. server=${server%%:*}
  930. fi
  931. ssh "${ssh_options[@]}" "$admin"@"$server" ssh-key ls
  932. }
  933. cmdline.spec.gnu recover
  934. cmdline.spec:vps-backup:cmd:recover:run() {
  935. : :posarg: BACKUP_ID 'Backup id.
  936. (ie: myadmin@backup.domain.org:10023#mx.myvps.org
  937. myadmin@ark-01.org#myid:/a/path
  938. admin@ark-02.io#myid:myqsl,mailcow)'
  939. : :posarg: VPS_PATH 'Target host(s) to check.
  940. (ie: myvps.com
  941. myvps.com:/a/path)'
  942. : :optval: --date,-D '"last", or label of version to recover. (Default: "last").'
  943. : :optfla: --force,-f 'Will allow you to bypass some checks.'
  944. if ! [[ "$BACKUP_ID" == *"@"* ]]; then
  945. err "Missing admin account identifier in backup id."
  946. echo " Have you forgottent to specify an admin account ?" \
  947. "ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
  948. return 1
  949. fi
  950. if ! [[ "$BACKUP_ID" == *"@"*"#"* ]]; then
  951. err "Missing backup label identifier in backup id."
  952. echo " Have you forgottent to specify a backup label identifier ?" \
  953. "ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
  954. return 1
  955. fi
  956. id_path=${BACKUP_ID#*#}
  957. if [[ "$id_path" == *":"* ]]; then
  958. id=${id_path%%:*}
  959. path=${id_path#*:}
  960. else
  961. id="$id_path"
  962. path=
  963. fi
  964. BACKUP_TARGET=${BACKUP_ID%#*}
  965. admin=${BACKUP_TARGET%%@*}
  966. server=${BACKUP_TARGET#*@}
  967. ssh_options=()
  968. if [[ "$server" == *":"* ]]; then
  969. ssh_options+=(-p "${server#*:}")
  970. ssh_server=${server%%:*}
  971. fi
  972. BACKUP_PATH="/srv/datastore/data/rsync-backup-target/var/mirror"
  973. if ! content=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key ls 2>/dev/null); then
  974. err "Access denied to '$admin@${server}'."
  975. return 1
  976. fi
  977. idents=$(echo "$content" | sed -r "s/"$'\e'"\[[0-9]+(;[0-9]+)*m//g" | cut -f 2 -d " ")
  978. if ! [[ $'\n'"$idents"$'\n' == *$'\n'"$id"$'\n'* ]]; then
  979. err "Given backup id '$id' not found in $admin@${server}'s idents."
  980. return 1
  981. fi
  982. rtype=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key get-type "$id" ) &&
  983. info "Backup archive matches ${WHITE}${rtype}${NORMAL} type"
  984. p0 "$admin" "$server" "$id" "$path" "$rtype" "$opt_force" |
  985. vps_backup_recover "${VPS_PATH}"
  986. }
  987. cmdline.spec.gnu vps-update
  988. cmdline.spec::cmd:vps-update:run() {
  989. : :posarg: [VPS...] 'Target host to check'
  990. echo "" |
  991. vps_mux vps_update "${VPS[@]}"
  992. }
  993. cmdline.spec.gnu vps-mux
  994. cmdline.spec::cmd:vps-mux:run() {
  995. : :posarg: [VPS...] 'Target host(s) to check'
  996. cat | vps_mux vps_bash "${VPS[@]}"
  997. }
  998. cmdline.spec.gnu vps-space
  999. cmdline.spec::cmd:vps-space:run() {
  1000. : :posarg: [VPS...] 'Target host(s) to check'
  1001. echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 2-5 -d ' '" |
  1002. vps_mux vps_bash "${VPS[@]}"
  1003. }
  1004. cmdline::parse "$@"