You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1899 lines
64 KiB

  1. #!/bin/bash
  2. . /etc/shlib
  3. include common
  4. include parse
  5. include cmdline
  6. include config
  7. [[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true
  8. version=0.1
  9. desc='Manage 0k related installs'
  10. help=""
  11. ##
  12. ## Functions
  13. ##
  14. is-port-open() {
  15. local host="$1" port="$2" timeout=5
  16. start="$SECONDS"
  17. debug "Testing if $host's port $2 is open ..."
  18. while true; do
  19. timeout 1 bash -c "</dev/tcp/${host}/${port}" >/dev/null 2>&1 && break
  20. sleep 0.2
  21. if [ "$((SECONDS - start))" -gt "$timeout" ]; then
  22. return 1
  23. fi
  24. done
  25. debug " .. $host's port $2 is open."
  26. return 0
  27. }
  28. resolve() {
  29. local ent hostname="$1"
  30. debug "Resolving $1 ..."
  31. if ent=$(getent ahosts "$hostname"); then
  32. ent=$(echo "$ent" | egrep ^"[0-9]+.[0-9]+.[0-9]+.[0-9]+\s+" | \
  33. head -n 1 | awk '{ print $1 }')
  34. debug " .. resolved $1 to $ent."
  35. echo "$ent"
  36. else
  37. debug " .. couldn't resolve $1."
  38. return 1
  39. fi
  40. }
  41. set_errlvl() { return "${1:-1}"; }
  42. export master_pid=$$
  43. ssh:open() {
  44. local hostname ssh_cmd ssh_options
  45. ssh_cmd=(ssh)
  46. ssh_options=()
  47. while [ "$#" != 0 ]; do
  48. case "$1" in
  49. "--stdin-password")
  50. ssh_cmd=(sshpass "${ssh_cmd[@]}")
  51. ;;
  52. -o)
  53. ssh_options+=("$1" "$2")
  54. shift
  55. ;;
  56. *)
  57. [ -z "$hostname" ] && hostname="$1" || {
  58. err "Surnumerous positional argument '$1'. Expecting only hostname."
  59. return 1
  60. }
  61. ;;
  62. esac
  63. shift
  64. done
  65. full_cmd=(
  66. "${ssh_cmd[@]}"
  67. -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
  68. -o ControlMaster=auto -o ControlPersist=900 \
  69. -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
  70. "${ssh_options[@]}" \
  71. "$hostname" "$@" -- true)
  72. "${full_cmd[@]}" >/dev/null 2>&1 || {
  73. err "Failed: ${full_cmd[*]}"
  74. return 1
  75. }
  76. trap_add EXIT,INT 'ssh:quit "$hostname"'
  77. }
  78. ssh:rsync() {
  79. local src="$1" dst="$2"
  80. hostname=${src%%:*}
  81. hostname=${hostname#*@}
  82. local rsync_ssh_options=(
  83. -o ControlPath="/tmp/ssh-control-master-${master_pid}-$hostname"
  84. -o ControlMaster=auto
  85. -o ControlPersist=900
  86. -o ConnectTimeout=10
  87. -o StrictHostKeyChecking=no
  88. )
  89. if ! ssh:run "root@$hostname" -- type -p rsync </dev/null >/dev/null; then
  90. info "No 'rsync' available on '$hostname'. Requesting installation..."
  91. ssh:run "root@$hostname" -- apt-get install rsync -y || {
  92. err "Installation of 'rsync' failed on '$hostname'"
  93. return 1
  94. }
  95. fi
  96. local cmd=(
  97. rsync -e "ssh ${rsync_ssh_options[*]}"
  98. -azvArH --delete --delete-excluded
  99. --partial --partial-dir .rsync-partial
  100. "$src" "$dst"
  101. )
  102. "${cmd[@]}"
  103. }
  104. ssh:open-try() {
  105. local opts hostnames
  106. opts=()
  107. hostnames=()
  108. while [ "$#" != 0 ]; do
  109. case "$1" in
  110. -o)
  111. opts+=("$1" "$2")
  112. shift
  113. ;;
  114. *)
  115. hostnames+=("$1")
  116. ;;
  117. esac
  118. shift
  119. done
  120. password=''
  121. for host in "${hostnames[@]}"; do
  122. debug "Trying $host with publickey."
  123. ssh:open -o PreferredAuthentications=publickey \
  124. "${opts[@]}" \
  125. "$host" >/dev/null 2>&1 && {
  126. echo "$host"$'\n'"$password"$'\n'
  127. return 0
  128. }
  129. debug " .. failed connecting to $host with publickey."
  130. done
  131. local times=0 password
  132. while [ "$((++times))" -le 3 ]; do
  133. read -sp "$HOST's password: " password
  134. errlvl="$?"
  135. echo >&2
  136. if [ "$errlvl" -gt 0 ]; then
  137. exit 1
  138. fi
  139. for host in "${hostnames[@]}"; do
  140. debug "Trying $host with password ($times/3)"
  141. echo "$password" | ssh:open -o PreferredAuthentications=password \
  142. --stdin-password \
  143. "${opts[@]}" \
  144. "$host" >/dev/null 2>&1 && {
  145. echo "$host"$'\n'"$password"$'\n'
  146. return 0
  147. }
  148. debug " .. failed connecting to $host with password."
  149. done
  150. err "login failed. Try again... ($((times+1))/3)"
  151. done
  152. return 1
  153. }
  154. ssh:run() {
  155. local hostname="$1" ssh_options cmd
  156. shift
  157. ssh_options=()
  158. cmd=()
  159. while [ "$#" != 0 ]; do
  160. case "$1" in
  161. "--")
  162. shift
  163. cmd+=("$@")
  164. break
  165. ;;
  166. *)
  167. ssh_options+=("$1")
  168. ;;
  169. esac
  170. shift
  171. done
  172. ## XXXvlab: keeping in case we need some debug
  173. # debug "$DARKCYAN$hostname$NORMAL $WHITE\$$NORMAL" "$@"
  174. # debug "Running cmd: ${cmd[*]}"
  175. # for arg in "${cmd[@]}"; do
  176. # debug "$arg"
  177. # done
  178. {
  179. {
  180. ssh -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
  181. -o ControlMaster=auto -o ControlPersist=900 \
  182. -o "StrictHostKeyChecking=no" \
  183. "${ssh_options[@]}" "$hostname" -- "${cmd[@]}"
  184. } 3>&1 1>&2 2>&3 ## | sed -r "s/^/$DARKCYAN$hostname$NORMAL $DARKRED\!$NORMAL /g"
  185. set_errlvl "${PIPESTATUS[0]}"
  186. } 3>&1 1>&2 2>&3
  187. }
  188. ssh:quit() {
  189. local hostname="$1"
  190. shift
  191. ssh -o ControlPath=/tmp/ssh-control-master-${master_pid} \
  192. -o ControlMaster=auto -o ControlPersist=900 -O exit \
  193. "$hostname" 2>/dev/null
  194. }
  195. is_ovh_domain_name() {
  196. local domain="$1"
  197. [[ "$domain" == *.ovh.net ]] && return 0
  198. [[ "$domain" == "ns"*".ip-"*".eu" ]] && return 0
  199. return 1
  200. }
  201. is_ovh_hostname() {
  202. local domain="$1"
  203. [[ "$domain" =~ ^vps-[0-9a-f]*$ ]] && return 0
  204. [[ "$domain" =~ ^vps[0-9]*$ ]] && return 0
  205. return 1
  206. }
  207. vps_connection_check() {
  208. local vps="$1"
  209. ip=$(resolve "$vps") ||
  210. { echo "${DARKRED}no-resolve${NORMAL}"; return 1; }
  211. is-port-open "$ip" "22" </dev/null ||
  212. { echo "${DARKRED}no-port-22-open${NORMAL}"; return 1; }
  213. ssh:open -o ConnectTimeout=10 -o PreferredAuthentications=publickey \
  214. "root@$vps" >/dev/null 2>&1 ||
  215. { echo "${DARKRED}no-ssh-root-access${NORMAL}"; return 1; }
  216. }
  217. vps_check() {
  218. local vps="$1"
  219. vps_connection_check "$vps" </dev/null || return 1
  220. if size=$(
  221. echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 5 -d ' ' | cut -f 1 -d %" |
  222. ssh:run "root@$vps" -- bash); then
  223. if [ "$size" -gt "90" ]; then
  224. echo "${DARKRED}above-90%-disk-usage${NORMAL}"
  225. elif [ "$size" -gt "75" ]; then
  226. echo "${DARKYELLOW}above-75%-disk-usage${NORMAL}"
  227. fi
  228. else
  229. echo "${DARKRED}no-size${NORMAL}"
  230. fi </dev/null
  231. compose_content=$(ssh:run "root@$vps" -- cat /opt/apps/myc-deploy/compose.yml </dev/null) ||
  232. { echo "${DARKRED}no-compose${NORMAL}"; return 1; }
  233. echo "$compose_content" | yq -e ".rsync-backup" >/dev/null 2>&1 ||
  234. { echo "${DARKRED}no-backup${NORMAL}"; return 1; }
  235. }
  236. backup:setup-rsync() {
  237. local admin="$1" vps="$2" server="$3" id="$4"
  238. [ -z "${BACKUP_SSH_SERVER}" ] || {
  239. err "Unexpected error: '\$BACKUP_SSH_SERVER' is already set in '$FUNCNAME'."
  240. return 1
  241. }
  242. BACKUP_SSH_OPTIONS=(-o StrictHostKeyChecking=no)
  243. if [[ "$server" == *":"* ]]; then
  244. BACKUP_SSH_OPTIONS+=(-p "${server#*:}")
  245. BACKUP_SSH_SERVER=${server%%:*}
  246. else
  247. BACKUP_SSH_SERVER="$server"
  248. fi
  249. if ! private_key=$(ssh "${BACKUP_SSH_OPTIONS[@]}" \
  250. "$admin"@"${BACKUP_SSH_SERVER}" request-recovery-key "$id"); then
  251. err "Couldn't request a recovery key for '$id' with account '$admin'."
  252. return 1
  253. fi
  254. if ! VPS_TMP_DIR=$(echo "mktemp -d" | ssh:run "root@$vps" -- bash); then
  255. err "Couldn't create a temporary directory on vps"
  256. return 1
  257. fi
  258. cat <<EOF | ssh:run "root@$vps" -- bash || return 1
  259. touch "$VPS_TMP_DIR/recover_key" &&
  260. chmod go-rwx "$VPS_TMP_DIR/recover_key" &&
  261. printf "%s\n" "$private_key" >> "$VPS_TMP_DIR/recover_key"
  262. EOF
  263. BACKUP_SSH_OPTIONS+=(-i "$VPS_TMP_DIR/recover_key" -l rsync)
  264. BACKUP_VPS_TARGET="$vps"
  265. BACKUP_IDENT="$id"
  266. echo "type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null" |
  267. ssh:run "root@$vps" -- bash || return 1
  268. }
  269. backup:rsync() {
  270. local ssh_options
  271. [ -n "${BACKUP_SSH_SERVER}" ] || {
  272. err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
  273. return 1
  274. }
  275. rsync_options=()
  276. while [[ "$1" == "-"* ]]; do
  277. rsync_options+=("$1")
  278. shift
  279. done
  280. local src="$1" dst="$2"
  281. cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash
  282. rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
  283. -azvArH --delete --delete-excluded \
  284. --partial --partial-dir .rsync-partial \
  285. --numeric-ids ${rsync_options[*]} \
  286. "${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "${dst}"
  287. EOF
  288. }
  289. backup:path_exists() {
  290. local src="$1"
  291. [ -n "${BACKUP_SSH_SERVER}" ] || {
  292. err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
  293. return 1
  294. }
  295. cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash >/dev/null 2>&1
  296. rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
  297. -nazvArH --numeric-ids \
  298. "${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "/tmp/dummy"
  299. EOF
  300. }
  301. file:vps_backup_recover() {
  302. local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"
  303. backup:rsync "${path}" "${vps_path}" || return 1
  304. if [[ "$path" == *"/" ]]; then
  305. if [ "$path" == "$vps_path"/ ]; then
  306. msg_target="Directory '$path'"
  307. else
  308. msg_target="Directory '$path' -> '$vps_path'"
  309. fi
  310. else
  311. if [ "$path" == "$vps_path" ]; then
  312. msg_target="File '$path'"
  313. else
  314. msg_target="File '$path' -> '$vps_path'"
  315. fi
  316. fi
  317. info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."
  318. }
  319. mailcow:vps_backup_recover() {
  320. local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"
  321. if ! compose_yml_files=$(cat <<EOF | ssh:run "root@$vps" -- bash
  322. urn=com.docker.compose.project
  323. docker ps -f "label=\$urn=mailcowdockerized" \
  324. --format="{{.Label \"\$urn.working_dir\"}}/{{.Label \"\$urn.config_files\"}}" |
  325. uniq
  326. EOF
  327. ); then
  328. err "Couldn't get list of running projects"
  329. return 1
  330. fi
  331. stopped_containers=
  332. if [ -n "$compose_yml_files" ]; then
  333. echo "Found running mailcowdockerized containers" >&2
  334. if [[ "$compose_yml_files" == *$'\n'* ]]; then
  335. err "Running containers are confusing, did not find only one mailcowdockerized project."
  336. return 1
  337. fi
  338. if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
  339. ## For some reason, sometimes $urn.config_files holds an absolute path
  340. compose_yml_files=/${compose_yml_files#*//}
  341. if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
  342. err "Running containers are confusing, they don't point to an existing docker-compose.yml."
  343. return 1
  344. fi
  345. fi
  346. echo "Containers where launched from '$compose_yml_files'" >&2
  347. COMPOSE_FILE="$compose_yml_files"
  348. ENV_FILE="${COMPOSE_FILE%/*}/.env"
  349. if ! echo "[ -e \"${ENV_FILE}\" ]" | ssh:run "root@$vps" -- bash ; then
  350. err "Running containers are confusing, docker-compose.yml has no '.env' next to it."
  351. return 1
  352. fi
  353. echo "${WHITE}Bringing mailcowdockerized down${NORMAL}"
  354. echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
  355. ssh:run "root@$vps" -- bash
  356. stopped_containers=1
  357. fi
  358. if [[ "$path" == "/"* ]]; then
  359. ##
  360. ## Additional intelligence to simple file copy
  361. ##
  362. if [[ "$path" == "/var/lib/docker/volumes/mailcowdockerized_*-vol-1/_data"* ]]; then
  363. volume_name=${path#/var/lib/docker/volumes/}
  364. volume_name=${volume_name%%/*}
  365. volume_dir=${path%%"$volume_name"*}
  366. ## Create volumes if not existent
  367. if ! ssh:run "root@$vps" -- "
  368. [ -d '${volume_dir}' ] ||
  369. docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
  370. [ -d '${volume_dir}' ]
  371. "; then
  372. err "Couldn't find nor create '${volume_dir}'."
  373. return 1
  374. fi
  375. fi
  376. echo "${WHITE}Sync from backup ${path} to VPS ${vps_path}${NORMAL}" >&2
  377. backup:rsync "${path}" "${vps_path}" || return 1
  378. if [[ "$path" == *"/" ]]; then
  379. if [ "$path" == "$vps_path"/ ]; then
  380. msg_target="Directory '$path'"
  381. else
  382. msg_target="Directory '$path' -> '$vps_path'"
  383. fi
  384. else
  385. if [ "$path" == "$vps_path" ]; then
  386. msg_target="File '$path'"
  387. else
  388. msg_target="File '$path' -> '$vps_path'"
  389. fi
  390. fi
  391. else
  392. ALL_TARGETS=(mailcow postfix rspamd redis crypt vmail{,-attachments} mysql)
  393. if [[ -n "$path" ]]; then
  394. targets=()
  395. bad_targets=()
  396. for target in ${path//,/ }; do
  397. if [[ " ${ALL_TARGETS[*]} " != *" $target "* ]]; then
  398. bad_targets+=("$target")
  399. fi
  400. targets+=("$target")
  401. done
  402. if [ "${#bad_targets[@]}" -gt 0 ]; then
  403. bad_target_msg=$(printf "%s, " "${bad_targets[@]}")
  404. err "Unknown components: ${bad_target_msg%, }. These are allowed components:"
  405. printf " - %s\n" "${ALL_TARGETS[@]}" >&2
  406. return 1
  407. fi
  408. msg_target="Partial mailcow backup"
  409. else
  410. targets=("${ALL_TARGETS[@]}")
  411. msg_target="Full mailcow backup"
  412. fi
  413. for target in "${targets[@]}"; do
  414. case "$target" in
  415. postfix|rspamd|redis|crypt|vmail|vmail-attachments)
  416. volume_name="mailcowdockerized_${target}-vol-1"
  417. volume_dir="/var/lib/docker/volumes/${volume_name}/_data"
  418. if ! backup:path_exists "${volume_dir}/"; then
  419. warn "No '$volume_name' in backup. This might be expected."
  420. continue
  421. fi
  422. ## Create volumes if not existent
  423. if ! ssh:run "root@$vps" -- "
  424. [ -d '${volume_dir}' ] ||
  425. docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
  426. [ -d '${volume_dir}' ]
  427. "; then
  428. err "Couldn't find nor create '${volume_dir}'."
  429. return 1
  430. fi
  431. echo "${WHITE}Downloading of $volume_name${NORMAL}"
  432. backup:rsync "${volume_dir}/" "${volume_dir}" || return 1
  433. ;;
  434. mailcow)
  435. ## Mailcow git base
  436. COMPOSE_FILE=
  437. for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
  438. backup:path_exists "${mailcow_dir}/" || continue
  439. ## this possibly change last value
  440. COMPOSE_FILE="$mailcow_dir/docker-compose.yml"
  441. ENV_FILE="$mailcow_dir/.env"
  442. echo "${WHITE}Download of $mailcow_dir${NORMAL}"
  443. backup:rsync "${mailcow_dir}"/ "${mailcow_dir}" || return 1
  444. break
  445. done
  446. if [ -z "$COMPOSE_FILE" ]; then
  447. err "Can't find mailcow base installation path in backup."
  448. return 1
  449. fi
  450. ;;
  451. mysql)
  452. if [ -z "$COMPOSE_FILE" ]; then
  453. ## Mailcow git base
  454. compose_files=()
  455. for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
  456. ssh:run "root@$vps" -- "[ -e \"$mailcow_dir/docker-compose.yml\" ]" || continue
  457. ## this possibly change last value
  458. compose_files+=("$mailcow_dir/docker-compose.yml")
  459. done
  460. if [ "${#compose_files[@]}" == 0 ]; then
  461. err "No compose file found for mailcow installation."
  462. return 1
  463. elif [ "${#compose_files[@]}" -gt 1 ]; then
  464. err "Multiple compose files for mailcow found:"
  465. for f in "${compose_files[@]}"; do
  466. echo " - $f" >&2
  467. done
  468. echo "Can't decide which to use for mounting mysql container." >&2
  469. return 1
  470. fi
  471. COMPOSE_FILE="${compose_files[0]}"
  472. ENV_FILE="${COMPOSE_FILE%/*}/.env"
  473. if ! ssh:run "root@$vps" -- "[ -e \"${COMPOSE_FILE%/*}/.env\" ]"; then
  474. err "No env file in '$ENV_FILE' found."
  475. return 1
  476. fi
  477. fi
  478. ## Mysql database
  479. echo "${WHITE}Downloading last backup of mysql backups${NORMAL}"
  480. backup:rsync "/var/backups/mysql/" "/var/backups/mysql" || return 1
  481. if ! env_content=$(echo "cat '$ENV_FILE'" | ssh:run "root@$vps" -- bash); then
  482. err "Can't access env file: '$ENV_FILE'."
  483. return 1
  484. fi
  485. root_password=$(printf "%s\n" "$env_content" | grep ^DBROOT= | cut -f 2 -d =)
  486. echo "${WHITE}Bringing mysql-mailcow up${NORMAL}"
  487. if ! image=$(cat <<EOF | ssh:run "root@$vps" -- bash
  488. shyaml get-value services.mysql-mailcow.image < "${COMPOSE_FILE}"
  489. EOF
  490. ); then
  491. err "Failed to get image name of service 'mysql-mailcow' in 'compose.yml'."
  492. return 1
  493. fi
  494. if [ -z "$(ssh:run "root@$vps" -- docker images -q "$image")" ]; then
  495. info "Image '$image' not available, pull it."
  496. if ! ssh:run "root@$vps" -- \
  497. docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
  498. pull mysql-mailcow; then
  499. err "Failed to pull image of service 'mysql-mailcow'."
  500. return 1
  501. fi
  502. fi
  503. if ! container_id=$(cat <<EOF | ssh:run "root@$vps" -- bash
  504. echo "[client]
  505. password=$root_password" > "$VPS_TMP_DIR/my.cnf"
  506. docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
  507. run -d \
  508. -v "$VPS_TMP_DIR/my.cnf:/root/.my.cnf:ro" \
  509. mysql-mailcow
  510. EOF
  511. ); then
  512. err "Failed to bring up mysql-mailcow"
  513. return 1
  514. fi
  515. START="$SECONDS"
  516. retries=0
  517. timeout=600
  518. while true; do
  519. ((retries++))
  520. echo " waiting for mysql db..." \
  521. "(retry $retries, $(($SECONDS - $START))s elapsed, timeout is ${timeout}s)" >&2
  522. cat <<EOF | ssh:run "root@$vps" -- bash && break
  523. echo "SELECT 1;" | docker exec -i "$container_id" mysql >/dev/null 2>&1
  524. EOF
  525. if (($SECONDS - $START > $timeout)); then
  526. err "Failed to connect to mysql-mailcow."
  527. echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
  528. ssh:run "root@$vps" -- bash
  529. return 1
  530. fi
  531. sleep 0.4
  532. done
  533. DBUSER=$(printf "%s\n" "$env_content" | grep ^DBUSER= | cut -f 2 -d =)
  534. DBPASS=$(printf "%s\n" "$env_content" | grep ^DBPASS= | cut -f 2 -d =)
  535. echo "${WHITE}Uploading mysql dump${NORMAL}"
  536. cat <<EOF | ssh:run "root@$vps" -- bash
  537. echo "
  538. DROP DATABASE IF EXISTS mailcow;
  539. CREATE DATABASE mailcow;
  540. GRANT ALL PRIVILEGES ON mailcow.* TO '$DBUSER'@'%' IDENTIFIED BY '$DBPASS';
  541. " | docker exec -i "$container_id" mysql
  542. zcat /var/backups/mysql/mailcow/*.gz | docker exec -i "$container_id" mysql mailcow
  543. EOF
  544. if [ "$?" != 0 ]; then
  545. err "Failed to load mysql dump."
  546. echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
  547. ssh:run "root@$vps" -- bash
  548. return 1
  549. fi
  550. echo "${WHITE}Bringing mysql-mailcow down${NORMAL}"
  551. echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
  552. ssh:run "root@$vps" -- bash
  553. ;;
  554. *)
  555. err "Unknown component '$target'. Bailing out."
  556. return 1
  557. esac
  558. done
  559. ssh:run "root@$vps" -- "rm -rf '$VPS_TMP_DIR'"
  560. fi
  561. if [ -n "$stopped_containers" ]; then
  562. echo "${WHITE}Starting mailcow${NORMAL}" >&2
  563. echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" up -d" |
  564. ssh:run "root@$vps" -- bash
  565. fi
  566. info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."
  567. }
  568. NTFY_TOPIC_FILE="/etc/ntfy/topics.yml"
  569. NTFY_CONFIG_FILE="/etc/ntfy/ntfy.conf"
  570. subscribe:ntfy:topic-file-exists() {
  571. local vps="$1"
  572. if ! out=$(echo "[ -f \"$NTFY_TOPIC_FILE\" ] && echo ok || true" | \
  573. ssh:run "root@$vps" -- bash); then
  574. err "Unable to check for existence of '$NTFY_TOPIC_FILE'."
  575. fi
  576. if [ -z "$out" ]; then
  577. err "File '$NTFY_TOPIC_FILE' not found on $vps."
  578. return 1
  579. fi
  580. }
  581. subscribe:ntfy:config-file-exists() {
  582. local vps="$1"
  583. if ! out=$(echo "[ -f \"$NTFY_CONFIG_FILE\" ] && echo ok || true" | \
  584. ssh:run "root@$vps" -- bash); then
  585. err "Unable to check for existence of '$NTFY_CONFIG_FILE'."
  586. fi
  587. if [ -z "$out" ]; then
  588. err "File '$NTFY_CONFIG_FILE' not found on $vps."
  589. return 1
  590. fi
  591. }
  592. ntfy:rm() {
  593. local channel="$1" topic="$2" vps="$3"
  594. subscribe:ntfy:topic-file-exists "$vps" || return 1
  595. if ! out=$(echo "yq -i 'del(.[\"$channel\"][] | select(. == \"$TOPIC\"))' \"$NTFY_TOPIC_FILE\"" | \
  596. ssh:run "root@$vps" -- bash); then
  597. err "Failed to remove channel '$channel' from '$NTFY_TOPIC_FILE'."
  598. return 1
  599. fi
  600. info "Channel '$channel' removed from '$NTFY_TOPIC_FILE' on $vps."
  601. ssh:run "root@$vps" -- cat "$NTFY_TOPIC_FILE"
  602. }
  603. ntfy:add() {
  604. local channel="$1" topic="$2" vps="$3"
  605. vps_connection_check "$vps" </dev/null || return 1
  606. subscribe:ntfy:topic-file-exists "$vps" || return 1
  607. if ! out=$(echo "yq '. | has(\"$channel\")' \"$NTFY_TOPIC_FILE\"" | \
  608. ssh:run "root@$vps" -- bash); then
  609. err "Failed to check if channel '$channel' with topic '$topic' is already in '$NTFY_TOPIC_FILE'."
  610. return 1
  611. fi
  612. if [ "$out" != "true" ]; then
  613. ## Channel does not exist
  614. if ! out=$(echo "yq -i '.[\"$channel\"] = []' \"$NTFY_TOPIC_FILE\"" | \
  615. ssh:run "root@$vps" -- bash); then
  616. err "Failed to create a new channel '$channel' entry in '$NTFY_TOPIC_FILE'."
  617. return 1
  618. fi
  619. else
  620. ## Channel exists
  621. if ! out=$(echo "yq '.[\"$channel\"] | any_c(. == \"$topic\")' \"$NTFY_TOPIC_FILE\"" | \
  622. ssh:run "root@$vps" -- bash); then
  623. err "Failed to check if channel '$channel' with topic '$topic' is already in '$NTFY_TOPIC_FILE'."
  624. return 1
  625. fi
  626. if [ "$out" == "true" ]; then
  627. info "Channel '$channel' with topic '$topic' already exists in '$NTFY_TOPIC_FILE'."
  628. return 0
  629. fi
  630. fi
  631. if ! out=$(echo "yq -i '.[\"$channel\"] += [\"$topic\"]' \"$NTFY_TOPIC_FILE\"" | \
  632. ssh:run "root@$vps" -- bash); then
  633. err "Failed to add channel '$channel' with topic '$topic' to '$NTFY_TOPIC_FILE'."
  634. return 1
  635. fi
  636. info "Channel '$channel' added with topic '$topic' to '$NTFY_TOPIC_FILE' on $vps."
  637. }
  638. NTFY_BROKER_SERVER="ntfy.0k.io"
  639. ntfy:topic-access() {
  640. local action="$1" topic="$2" vps="$3"
  641. subscribe:ntfy:config-file-exists "$vps" || return 1
  642. local user
  643. user=$(ntfy:get-login "$vps") || return 1
  644. case "$action" in
  645. "write")
  646. ssh "ntfy@$NTFY_BROKER_SERVER" "topic-access" \
  647. "$user" "$topic" "write-only" </dev/null || {
  648. err "Failed to grant write access to '$user' for topic '$topic'."
  649. return 1
  650. }
  651. info "Granted write access for '$user' to topic '$topic'."
  652. ;;
  653. "remove")
  654. ssh "ntfy@$NTFY_BROKER_SERVER" "topic-access" -r "$user" "$topic" </dev/null || {
  655. err "Failed to reset access of '$user' for topic '$topic'."
  656. return 1
  657. }
  658. info "Access for '$user' to topic '$topic' was resetted successfully."
  659. ;;
  660. *)
  661. err "Invalid action '$action'."
  662. return 1
  663. ;;
  664. esac
  665. }
  666. ntfy:get-login() {
  667. local vps="$1"
  668. if ! out=$(echo ". \"$NTFY_CONFIG_FILE\" && echo \"\$LOGIN\"" | \
  669. ssh:run "root@$vps" -- bash); then
  670. err "Failed to get ntfy login from '$NTFY_CONFIG_FILE'."
  671. return 1
  672. fi
  673. if [ -z "$out" ]; then
  674. err "Unexpected empty login retrieved from sourcing '$NTFY_CONFIG_FILE'."
  675. return 1
  676. fi
  677. echo "$out"
  678. }
  679. subscribe:add() {
  680. local vps="$1"
  681. read-0 channel topic || {
  682. err "Couldn't read CHANNEL and TOPIC arguments."
  683. return 1
  684. }
  685. vps_connection_check "$vps" </dev/null || return 1
  686. ntfy:topic-access "write" "$topic" "$vps" </dev/null || return 1
  687. ntfy:add "$channel" "$topic" "$vps" || {
  688. err "Failed to add channel '$channel' with topic '$topic' to '$NTFY_TOPIC_FILE'."
  689. echo " Removing topic access." >&2
  690. ntfy:topic-access "remove" "$topic" "$vps" </dev/null
  691. return 1
  692. }
  693. }
  694. subscribe:rm() {
  695. local vps="$1"
  696. read-0 channel topic || {
  697. err "Couldn't read CHANNEL and TOPIC arguments."
  698. return 1
  699. }
  700. vps_connection_check "$vps" </dev/null || return 1
  701. ntfy:rm "$channel" "$topic" "$vps" || return 1
  702. ntfy:topic-access "remove" "$topic" "$vps" </dev/null || {
  703. err "Failed to remove topic access for '$topic' on '$vps'."
  704. return 1
  705. }
  706. }
  707. vps_backup_recover() {
  708. local vps="$1" admin server id path rtype force type
  709. read-0 admin server id path rtype force
  710. if [[ "$vps" == *":"* ]]; then
  711. vps_path=${vps#*:}
  712. vps=${vps%%:*}
  713. else
  714. vps_path=
  715. fi
  716. vps_connection_check "$vps" </dev/null || {
  717. err "Failed to access '$vps'."
  718. return 1
  719. }
  720. if type=$(ssh:run "root@$vps" -- vps get-type); then
  721. info "VPS $vps seems to be of ${WHITE}$type${NORMAL} type"
  722. else
  723. warn "Couldn't find type of vps '$vps' (command 'vps get-type' failed on vps)."
  724. fi
  725. if [ -z "$path" ]; then
  726. if [ -n "$vps_path" ]; then
  727. err "You can't provide a VPS with path as destination if you don't provide a path in backup source."
  728. return 1
  729. fi
  730. info "No path provided in backup, so we assume you want ${WHITE}full recovery${NORMAL}."
  731. if [ "$rtype" != "$type" ]; then
  732. if [ -n "$force" ]; then
  733. warn "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type."
  734. else
  735. err "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type. (use \`\`-f\`\` to force)"
  736. return 1
  737. fi
  738. fi
  739. else
  740. if [ "$path" == "/" ]; then
  741. if [ -z "$vps_path" ]; then
  742. err "Recovery of '/' (full backup files) requires that you provide a vps path also."
  743. return 1
  744. fi
  745. if [ "$vps_path" == "/" ]; then
  746. err "Recovery of '/' (full backup files) requires that you provide" \
  747. "a vps path different from '/' also."
  748. return 1
  749. fi
  750. fi
  751. fi
  752. ## Sets VPS and internal global variable to allow rsync to work
  753. ## from vps to backup server.
  754. backup:setup-rsync "$admin" "$vps" "$server" "$id" || return 1
  755. if [[ "$path" == "/"* ]]; then
  756. if ! backup:path_exists "${path}"; then
  757. err "File or directory '$path' not found in backup."
  758. return 1
  759. fi
  760. if [ -z "$vps_path" ]; then
  761. if [[ "$path" != *"/" ]] && backup:path_exists "${path}"/ ; then
  762. path="$path/"
  763. fi
  764. vps_path=${path%/}
  765. vps_path=${vps_path:-/}
  766. fi
  767. fi
  768. case "$rtype-$type" in
  769. mailcow-*)
  770. ## Supports having $path and $vps_path set or unset, with additional behavior
  771. mailcow:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
  772. ;;
  773. *-*)
  774. if [[ "$path" == "/"* ]]; then
  775. ## For now, will require having $path and $vps_path set, no additional behaviors
  776. file:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
  777. else
  778. if [ -n "$path" ]; then
  779. err "Partial component recover of ${rtype:-unknown} backup type on" \
  780. "${type:-unknown} type VPS is not yet implemented."
  781. return 1
  782. else
  783. err "Full recover of ${rtype:-unknown} backup type on" \
  784. "${type:-unknown} type VPS is not yet implemented."
  785. return 1
  786. fi
  787. fi
  788. ;;
  789. esac
  790. }
  791. vps_install_backup() {
  792. local vps="$1" admin server
  793. vps_connection_check "$vps" </dev/null || return 1
  794. read-0 admin server
  795. if ! type=$(ssh:run "root@$vps" -- vps get-type </dev/null); then
  796. err "Could not get type."
  797. return 1
  798. fi
  799. backup_opts=()
  800. local opt
  801. while read-0 opt; do
  802. case "$opt" in
  803. --ignore-domain-check|--ignore-ping-check)
  804. backup_opts+=("$opt")
  805. ;;
  806. *)
  807. err "Unknown option '$opt'."
  808. return 1
  809. ;;
  810. esac
  811. done
  812. if ! out=$(ssh:run "root@$vps" -- vps install backup "$server" "${backup_opts[@]}" 2>&1); then
  813. err "Command 'vps install backup $server ${backup_opts[@]}' on $vps failed:"
  814. echo "$out" | prefix " ${DARKGRAY}|${NORMAL} " >&2
  815. return 1
  816. fi
  817. ## Format of output:
  818. ##
  819. ## II Entry for service rsync-backup is already present in '/opt/apps/myc-deploy/compose.yml'.
  820. ## II You can run this following command from an host having admin access to core-07.0k.io:
  821. ## (Or send it to a backup admin of core-07.0k.io)
  822. ## ssh -p 10023 myadmin@core-07.0k.io ssh-key add 'ssh-rsa AAAAAAAD...QCJ\
  823. ## 8HH6pVgEpu1twyxpr9xTt7eh..WaJdgPoxmiEwGfjMMNGxs39ggOTKUuSFSmOv8TiA1fzY\
  824. ## s85hF...dKP1qByJU1k= compose@odoo.sikle.fr'
  825. key="ssh-rsa ${out##*\'ssh-rsa }" ## remove everything before last "'ssh-rsa"
  826. key="${key%\'*}" ## remove everything after last '
  827. if ! [[ "$key" =~ ^"ssh-rsa "[a-zA-Z0-9/+=]+" "[a-zA-Z0-9._-]+"@"[a-zA-Z0-9._-]+$ ]]; then
  828. err "Unexpected output from 'vps install backup $server'. Can't find key."
  829. echo "$out" | prefix " ${GRAY}|$NORMAL " >&2
  830. echo " Extracted key:" >&2
  831. echo "$key" | prefix " ${GRAY}|$NORMAL " >&2
  832. return 1
  833. fi
  834. if [ "$type" == "compose" ]; then
  835. if ! ssh:run "root@$vps" -- \
  836. docker exec myc_cron_1 \
  837. grep rsync-backup /etc/crontabs/root >/dev/null 2>&1; then
  838. ssh:run "root@$vps" -- compose --debug up || {
  839. err "Command 'compose --debug up' failed."
  840. return 1
  841. }
  842. if ! ssh:run "root@$vps" -- \
  843. docker exec myc_cron_1 \
  844. grep rsync-backup /etc/crontabs/root >/dev/null 2>&1; then
  845. err "Launched 'compose up' successfully but ${YELLOW}cron${NORMAL} container is not setup as expected."
  846. echo " Was waiting for existence of a line mentionning 'rsync-backup' in '/etc/crontabs/root' in it." >&2
  847. return 1
  848. fi
  849. fi
  850. fi
  851. dest="$server"
  852. dest="${dest%/*}"
  853. ssh_options=()
  854. if [[ "$dest" == *":"* ]]; then
  855. port="${dest##*:}"
  856. dest="${dest%%:*}"
  857. ssh_options=(-p "$port")
  858. else
  859. port=""
  860. dest="${dest%%:*}"
  861. fi
  862. cmd=(ssh "${ssh_options[@]}" "$admin"@"$dest" ssh-key add "$key")
  863. echo "${WHITE}Launching:${NORMAL} ${cmd[@]}"
  864. "${cmd[@]}" || {
  865. err "Failed add key to backup server '$dest'."
  866. return 1
  867. }
  868. echo "${WHITE}Launching backup${NORMAL} from '$vps'"
  869. ssh:run "root@$vps" -- vps backup || {
  870. err "First backup failed to run."
  871. return 1
  872. }
  873. echo "Backup is ${GREEN}up and running${NORMAL}."
  874. }
  875. vps_update() {
  876. local vps="$1"
  877. vps_connection_check "$vps" || return 1
  878. ssh:run "root@$vps" -- myc-update </dev/null
  879. }
  880. vps_bash() {
  881. local vps="$1"
  882. vps_connection_check "$vps" </dev/null || return 1
  883. ssh:run "root@$vps" -- bash
  884. }
  885. vps_mux() {
  886. local fn="$1" vps_done VPS max_size vps
  887. shift
  888. VPS=($(printf "%s\n" "$@" | sort))
  889. max_size=0
  890. declare -A vps_done;
  891. new_vps=()
  892. for name in "${VPS[@]}"; do
  893. [ -n "${vps_done[$name]}" ] && {
  894. warn "duplicate vps '$name' provided. Ignoring."
  895. continue
  896. }
  897. vps_done["$name"]=1
  898. new_vps+=("$name")
  899. size_name="${#name}"
  900. [ "$max_size" -lt "${size_name}" ] &&
  901. max_size="$size_name"
  902. done
  903. settmpdir "_0KM_TMP_DIR"
  904. cat > "$_0KM_TMP_DIR/code"
  905. for vps in "${new_vps[@]}"; do
  906. label=$(printf "%-${max_size}s" "$vps")
  907. (
  908. {
  909. {
  910. "$fn" "$vps" < "$_0KM_TMP_DIR/code"
  911. } 3>&1 1>&2 2>&3 | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKRED\!$NORMAL /g"
  912. set_errlvl "${PIPESTATUS[0]}"
  913. } 3>&1 1>&2 2>&3 | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKGRAY\|$NORMAL /g"
  914. set_errlvl "${PIPESTATUS[0]}"
  915. ) &
  916. done
  917. wait
  918. }
  919. [ "$SOURCED" ] && return 0
  920. ##
  921. ## Command line processing
  922. ##
  923. cmdline.spec.gnu
  924. cmdline.spec.reporting
  925. cmdline.spec.gnu vps-setup
  926. cmdline.spec::cmd:vps-setup:run() {
  927. : :posarg: HOST 'Target host to check/fix ssh-access'
  928. : :optfla: --force,-f "Will force domain name change, even if
  929. current hostname was not recognized as
  930. an ovh domain name. "
  931. depends sshpass shyaml
  932. KEY_PATH="ssh-access.public-keys"
  933. local keys=$(config get-value -y "ssh-access.public-keys") || true
  934. if [ -z "$keys" ]; then
  935. err "No ssh publickeys configured in config file."
  936. echo " Looking for keys in ${WHITE}${KEY_PATH}${NORMAL}" \
  937. "in config file." >&2
  938. config:exists --message 2>&1 | prefix " "
  939. if [ "${PIPESTATUS[0]}" == "0" ]; then
  940. echo " Config file found in $(config:filename)"
  941. fi
  942. return 1
  943. fi
  944. local tkey=$(e "$keys" | shyaml get-type)
  945. if [ "$tkey" != "sequence" ]; then
  946. err "Value type of ${WHITE}${KEY_PATH}${NORMAL} unexpected (is $tkey, expecting sequence)."
  947. echo " Check content of $(config:filename), and make sure to use a sequence." >&2
  948. return 1
  949. fi
  950. local IP NAME keys host_pass_connected
  951. if ! IP=$(resolve "$HOST"); then
  952. err "'$HOST' name unresolvable."
  953. exit 1
  954. fi
  955. NAME="$HOST"
  956. if [ "$IP" != "$HOST" ]; then
  957. NAME="$HOST ($IP)"
  958. fi
  959. if ! is-port-open "$IP" "22"; then
  960. err "$NAME unreachable or port 22 closed."
  961. exit 1
  962. fi
  963. debug "Host $IP's port 22 is open."
  964. if ! host_pass_connected=$(ssh:open-try \
  965. {root,debian}@"$HOST"); then
  966. err "Could not connect to {root,debian}@$HOST with publickey nor password."
  967. exit 1
  968. fi
  969. read-0a host password <<<"$host_pass_connected"
  970. sudo_if_necessary=
  971. if [ "$password" -o "${host%%@*}" != "root" ]; then
  972. if ! ssh:run "$host" -- sudo -n true >/dev/null 2>&1; then
  973. err "Couldn't do a password-less sudo from $host."
  974. echo " This is not yet supported."
  975. exit 1
  976. else
  977. sudo_if_necessary=sudo
  978. fi
  979. fi
  980. SUPPORTED_KEY_TYPE=(ssh-rsa ssh-ed25519)
  981. Section Checking access
  982. while read-0 key; do
  983. prefix="${key%% *}"
  984. if ! [[ " ${SUPPORTED_KEY_TYPE[*]} " == *" $prefix "* ]]; then
  985. err "Unsupported key:"$'\n'"$key"
  986. echo " Please use only key of the following type:" >&2
  987. printf " - %s\n" "${SUPPORTED_KEY_TYPE[@]}" >&2
  988. return 1
  989. fi
  990. label="${key##* }"
  991. Elt "considering adding key ${DARKYELLOW}$label${NORMAL}"
  992. dest="/root/.ssh/authorized_keys"
  993. if ssh:run "$host" -- $sudo_if_necessary grep "\"$key\"" "$dest" >/dev/null 2>&1 </dev/null; then
  994. print_info "already present"
  995. print_status noop
  996. Feed
  997. else
  998. if echo "$key" | ssh:run "$host" -- $sudo_if_necessary tee -a "$dest" >/dev/null; then
  999. print_info added
  1000. else
  1001. echo
  1002. Feedback failure
  1003. return 1
  1004. fi
  1005. Feedback success
  1006. fi
  1007. done < <(e "$keys" | shyaml get-values-0)
  1008. Section Checking ovh hostname file
  1009. Elt "Checking /etc/ovh-hostname"
  1010. if ! ssh:run "$host" -- $sudo_if_necessary [ -e "/etc/ovh-hostname" ]; then
  1011. print_info "creating"
  1012. ssh:run "$host" -- $sudo_if_necessary cp /etc/hostname /etc/ovh-hostname
  1013. ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
  1014. Elt "Checking /etc/ovh-hostname: $ovhname"
  1015. Feedback || return 1
  1016. else
  1017. ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
  1018. Elt "Checking /etc/ovh-hostname: $ovhname"
  1019. print_info "already present"
  1020. print_status noop
  1021. Feed
  1022. fi
  1023. if ! is_ovh_domain_name "$HOST" && ! str_is_ipv4 "$HOST"; then
  1024. Section Checking hostname
  1025. Elt "Checking /etc/hostname..."
  1026. old_etc_hostname="$(ssh:run "$host" -- $sudo_if_necessary cat /etc/hostname)"
  1027. if [ "$old_etc_hostname" != "$HOST" ]; then
  1028. Elt "/etc/hostname is '$old_etc_hostname'"
  1029. if is_ovh_hostname "$old_etc_hostname" || [ -n "$opt_force" ]; then
  1030. Elt "Hostname '$old_etc_hostname' --> '$HOST'"
  1031. print_info "creating"
  1032. echo "$HOST" | ssh:run "$host" -- $sudo_if_necessary tee /etc/hostname >/dev/null &&
  1033. ssh:run "$host" -- $sudo_if_necessary hostname "$HOST"
  1034. Feedback || return 1
  1035. else
  1036. Elt "Hostname '$old_etc_hostname' isn't an ovh domain"
  1037. print_info "no change"
  1038. print_status noop
  1039. Feed
  1040. warn "Domain name was not changed because it was already set"
  1041. echo " (use \`-f\` or \`--force\`) to force domain name change to $HOST." >&2
  1042. fi
  1043. else
  1044. print_info "already set"
  1045. print_status noop
  1046. Feed
  1047. fi
  1048. Elt "Checking consistency between /etc/hostname and \`hostname\`..."
  1049. etc_hostname="$(ssh:run "$host" -- $sudo_if_necessary cat /etc/hostname)"
  1050. transient_hostname="$(ssh:run "$host" -- $sudo_if_necessary hostname)"
  1051. if [ "$etc_hostname" != "$transient_hostname" ]; then
  1052. print_info "change"
  1053. ssh:run "$host" -- $sudo_if_necessary hostname "$etc_hostname"
  1054. Feedback || return 1
  1055. else
  1056. print_info "consistent"
  1057. print_status noop
  1058. Feed
  1059. fi
  1060. else
  1061. info "Not changing domain as '$HOST' doesn't seem to be final domain."
  1062. fi
  1063. }
  1064. cmdline.spec.gnu vps-check
  1065. cmdline.spec::cmd:vps-check:run() {
  1066. : :posarg: [VPS...] 'Target host(s) to check'
  1067. echo "" |
  1068. vps_mux vps_check "${VPS[@]}"
  1069. }
  1070. cmdline.spec.gnu vps-install
  1071. cmdline.spec::cmd:vps-install:run() {
  1072. :
  1073. }
  1074. cmdline.spec.gnu backup
  1075. cmdline.spec:vps-install:cmd:backup:run() {
  1076. : :posarg: BACKUP_TARGET 'Backup target.
  1077. (ie: myadmin@backup.domain.org:10023/256)'
  1078. : :optfla: --ignore-domain-check \
  1079. "Allow to bypass the domain check in
  1080. compose file (only used in compose
  1081. installation)."
  1082. : :optfla: --ignore-ping-check "Allow to bypass the ping check of host."
  1083. : :posarg: [VPS...] 'Target host(s) to check'
  1084. if [ "${#VPS[@]}" == 0 ]; then
  1085. warn "VPS list provided in command line is empty. Nothing will be done."
  1086. return 0
  1087. fi
  1088. if ! [[ "$BACKUP_TARGET" == *"@"* ]]; then
  1089. err "Missing admin account identifier in backup target."
  1090. echo " Have you forgottent to specify an account, ie 'myadmin@<MYBACKUP_SERVER>' ?)"
  1091. return 1
  1092. fi
  1093. admin=${BACKUP_TARGET%%@*}
  1094. server=${BACKUP_TARGET#*@}
  1095. opts=()
  1096. [ -n "$opt_ignore_ping_check" ] &&
  1097. opts+=("--ignore-ping-check")
  1098. [ -n "$opt_ignore_domain_check" ] &&
  1099. opts+=("--ignore-domain-check")
  1100. p0 "$admin" "$server" "${opts[@]}" |
  1101. vps_mux vps_install_backup "${VPS[@]}"
  1102. }
  1103. cmdline.spec.gnu vps-backup
  1104. cmdline.spec::cmd:vps-backup:run() {
  1105. :
  1106. }
  1107. cmdline.spec.gnu ls
  1108. cmdline.spec:vps-backup:cmd:ls:run() {
  1109. : :posarg: BACKUP_ID 'Backup id.
  1110. (ie: myadmin@backup.domain.org:10023)'
  1111. if ! [[ "$BACKUP_ID" == *"@"* ]]; then
  1112. err "Missing admin account identifier in backup id."
  1113. echo " Have you forgottent to specify an admin account ?" \
  1114. "ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
  1115. return 1
  1116. fi
  1117. id=${BACKUP_ID##*#}
  1118. BACKUP_TARGET=${BACKUP_ID%#*}
  1119. admin=${BACKUP_TARGET%%@*}
  1120. server=${BACKUP_TARGET#*@}
  1121. ## XXXvlab: in this first implementation we expect to have access
  1122. ## to the server main ssh port 22, so we won't use the provided port.
  1123. ssh_options=()
  1124. if [[ "$server" == *":"* ]]; then
  1125. ssh_options+=(-p "${server#*:}")
  1126. server=${server%%:*}
  1127. fi
  1128. ssh "${ssh_options[@]}" "$admin"@"$server" ssh-key ls
  1129. }
  1130. cmdline.spec.gnu recover
  1131. cmdline.spec:vps-backup:cmd:recover:run() {
  1132. : :posarg: BACKUP_ID 'Backup id.
  1133. (ie: myadmin@backup.domain.org:10023#mx.myvps.org
  1134. myadmin@ark-01.org#myid:/a/path
  1135. admin@ark-02.io#myid:myqsl,mailcow)'
  1136. : :posarg: VPS_PATH 'Target host(s) to check.
  1137. (ie: myvps.com
  1138. myvps.com:/a/path)'
  1139. : :optval: --date,-D '"last", or label of version to recover. (Default: "last").'
  1140. : :optfla: --force,-f 'Will allow you to bypass some checks.'
  1141. if ! [[ "$BACKUP_ID" == *"@"* ]]; then
  1142. err "Missing admin account identifier in backup id."
  1143. echo " Have you forgottent to specify an admin account ?" \
  1144. "ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
  1145. return 1
  1146. fi
  1147. if ! [[ "$BACKUP_ID" == *"@"*"#"* ]]; then
  1148. err "Missing backup label identifier in backup id."
  1149. echo " Have you forgottent to specify a backup label identifier ?" \
  1150. "ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
  1151. return 1
  1152. fi
  1153. id_path=${BACKUP_ID#*#}
  1154. if [[ "$id_path" == *":"* ]]; then
  1155. id=${id_path%%:*}
  1156. path=${id_path#*:}
  1157. else
  1158. id="$id_path"
  1159. path=
  1160. fi
  1161. BACKUP_TARGET=${BACKUP_ID%#*}
  1162. admin=${BACKUP_TARGET%%@*}
  1163. server=${BACKUP_TARGET#*@}
  1164. ssh_options=()
  1165. if [[ "$server" == *":"* ]]; then
  1166. ssh_options+=(-p "${server#*:}")
  1167. ssh_server=${server%%:*}
  1168. fi
  1169. BACKUP_PATH="/srv/datastore/data/rsync-backup-target/var/mirror"
  1170. if ! content=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key ls 2>/dev/null); then
  1171. err "Access denied to '$admin@${server}'."
  1172. return 1
  1173. fi
  1174. idents=$(echo "$content" | sed -r "s/"$'\e'"\[[0-9]+(;[0-9]+)*m//g" | cut -f 2 -d " ")
  1175. if ! [[ $'\n'"$idents"$'\n' == *$'\n'"$id"$'\n'* ]]; then
  1176. err "Given backup id '$id' not found in $admin@${server}'s idents."
  1177. return 1
  1178. fi
  1179. rtype=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key get-type "$id" ) &&
  1180. info "Backup archive matches ${WHITE}${rtype}${NORMAL} type"
  1181. p0 "$admin" "$server" "$id" "$path" "$rtype" "$opt_force" |
  1182. vps_backup_recover "${VPS_PATH}"
  1183. }
  1184. cmdline.spec.gnu vps-update
  1185. cmdline.spec::cmd:vps-update:run() {
  1186. : :posarg: [VPS...] 'Target host to check'
  1187. echo "" |
  1188. vps_mux vps_update "${VPS[@]}"
  1189. }
  1190. cmdline.spec.gnu vps-mux
  1191. cmdline.spec::cmd:vps-mux:run() {
  1192. : :posarg: [VPS...] 'Target host(s) to check'
  1193. cat | vps_mux vps_bash "${VPS[@]}"
  1194. }
  1195. cmdline.spec.gnu vps-space
  1196. cmdline.spec::cmd:vps-space:run() {
  1197. : :posarg: [VPS...] 'Target host(s) to check'
  1198. : :optfla: --nextcloud,-n 'target space used by nextcloud and total size of trashfiles'
  1199. : :optfla: --docker,-d 'target space used by docker'
  1200. echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 2-5 -d ' '" |
  1201. vps_mux vps_bash "${VPS[@]}"
  1202. if [ -n "$opt_nextcloud" ]; then
  1203. echo "
  1204. NEXTCLOUD_TRASH_DIR=\"/srv/datastore/data/nextcloud/var/lib/nextcloud/data/*/files_trashbin\"
  1205. echo \"---- Nextcloud data size : \"
  1206. du -sh /srv/datastore/data/nextcloud/
  1207. echo \"---- Nextcloud thrashbin files size (MB) : \"
  1208. du -sk /srv/datastore/data/nextcloud/var/lib/nextcloud/data/*/files_trashbin | awk '{sum += \$1} END {print sum / 1024}'
  1209. echo \"---- Trashbin group files (MB) : \"
  1210. du -sk /srv/datastore/data/nextcloud/var/lib/nextcloud/data/*/files_trashbin/* | awk '{sum += \$1} END {print sum / 1024}'
  1211. " |
  1212. vps_mux vps_bash "${VPS[@]}"
  1213. fi
  1214. if [ -n "$opt_docker" ]; then
  1215. echo "
  1216. echo \"---- Docker sytem disk usage ---- \"
  1217. docker system df
  1218. echo \"---- Docker containers logs size : \"
  1219. du -h max-depth=1 /var/lib/docker/containers/* | sort -h
  1220. " |
  1221. vps_mux vps_bash "${VPS[@]}"
  1222. fi
  1223. }
  1224. cmdline.spec.gnu vps-stats
  1225. cmdline.spec::cmd:vps-stats:run() {
  1226. : :posarg: [VPS...] 'Target host(s) to get stats'
  1227. : :optfla: --follow,-f 'Refresh graph every 2m'
  1228. : :optval: --timespan,-t 'timespan START[..END]'
  1229. : :optval: --resource,-r 'resource(s) separated with a comma'
  1230. : :optval: --interval,-i 'refersh interval (default: 60s)'
  1231. local opts_rrdfetch=( -a )
  1232. if [ -n "${opt_timespan}" ]; then
  1233. start=${opt_timespan%\.\.*}
  1234. opts_rrdfetch+=(-s "$start")
  1235. if [ "$start" != "${opt_timespan}" ]; then
  1236. end=${opt_timespan#*..}
  1237. opts_rrdfetch+=(-e "$end")
  1238. fi
  1239. fi
  1240. local resources=(c.memory c.network load_avg disk)
  1241. if [ -n "${opt_resource}" ]; then
  1242. resources=(${opt_resource//,/ })
  1243. fi
  1244. local not_found=()
  1245. for resource in "${resources[@]}"; do
  1246. if ! fn.exists "graph:def:$resource"; then
  1247. not_found+=("$resource")
  1248. fi
  1249. done
  1250. if [[ "${#not_found[@]}" -gt 0 ]]; then
  1251. not_found_msg=$(printf "%s, " "${not_found[@]}")
  1252. not_found_msg=${not_found_msg%, }
  1253. err "Unsupported resource(s) provided: ${not_found_msg}"
  1254. echo " resource must be one-of:" >&2
  1255. declare -F | egrep 'graph:def:[a-zA-Z_.]+$' | cut -f 3- -d " " | cut -f 3- -d ":" | prefix " - " >&2
  1256. return 1
  1257. fi
  1258. if [ "${#VPS[@]}" == 0 ]; then
  1259. err "You must provide a VPS list as positional arguments"
  1260. return 1
  1261. fi
  1262. include cache
  1263. if [ -z "$VAR_DIR" ]; then
  1264. err "Unset \$VAR_DIR, can't downlowd rrd graph"
  1265. return 1
  1266. fi
  1267. mkdir -p "$VAR_DIR/rrd"
  1268. if ! [ -d "$VAR_DIR/rrd" ]; then
  1269. err "Invalid \$VAR_DIR: '$VAR_DIR/rrd' is not a directory"
  1270. return 1
  1271. fi
  1272. (
  1273. for vps in "${VPS[@]}"; do
  1274. (
  1275. {
  1276. {
  1277. ssh:open "root@$vps" 2>/dev/null || {
  1278. err "Can't open connection $vps."
  1279. return 1
  1280. }
  1281. while true; do
  1282. if ssh:run "root@$vps" -- "[ -d '/var/lib/vps/rrd' ]"; then
  1283. echo "${WHITE}Collecting stats${NORMAL}..."
  1284. {
  1285. {
  1286. ssh:rsync "root@$vps:/var/lib/vps/rrd/" "${VAR_DIR}/rrd/${vps}"
  1287. } 3>&1 1>&2 2>&3 | prefix " ${DARKRED}\!${NORMAL} "
  1288. set_errlvl "${PIPESTATUS[0]}"
  1289. } 3>&1 1>&2 2>&3 | prefix " ${GRAY}|${NORMAL} "
  1290. echo " ${GRAY}..${NORMAL} ${DARKGREEN}done${NORMAL} collecting stats"
  1291. else
  1292. warn "No stats found. Did you run 'myc-update' on the vps ?."
  1293. fi
  1294. [ -z "$opt_follow" ] && break
  1295. echo "${WHITE}Sleeping ${DARKYELLOW}${opt_interval:-60}${NORMAL}s..."
  1296. sleep "${opt_interval:-60}"
  1297. echo " ${GRAY}..${NORMAL} ${DARKGREEN}done${NORMAL} sleeping"
  1298. done
  1299. } 3>&1 1>&2 2>&3 | prefix " ${DARKRED}\!${GRAY} collect(${DARKCYAN}$vps${GRAY})${NORMAL} "
  1300. set_errlvl "${PIPESTATUS[0]}"
  1301. } 3>&1 1>&2 2>&3 | prefix " ${GRAY}| collect(${DARKCYAN}$vps${GRAY})${NORMAL} " >&2
  1302. ) &
  1303. done
  1304. wait
  1305. ) &
  1306. collect_pid="$!"
  1307. if [ -z "$opt_follow" ]; then
  1308. echo "${WHITE}Fetching last stats${NORMAL}${GRAY}..${NORMAL}" >&2
  1309. wait
  1310. echo " ${GRAY}..${DARKGREEN} done${NORMAL} fetching stats" >&2
  1311. else
  1312. collect_end_msg=" ${GRAY}..${NORMAL} ${DARKGREEN}stop${NORMAL} collecting daemon (pid: ${DARKYELLOW}$collect_pid${NORMAL})"
  1313. trap_add EXIT \
  1314. "printf '%s\n' \"$collect_end_msg\" && kill $collect_pid"
  1315. echo "${WHITE}Start collecting daemon${NORMAL} (pid: ${DARKYELLOW}$collect_pid${NORMAL}) ${GRAY}..${NORMAL}" >&2
  1316. fi
  1317. ( depends gnuplot ) || {
  1318. echo ""
  1319. echo " Gnuplot is required to display graphs..." \
  1320. "You might want to try to install ${WHITE}gnuplot${NORMAL} with:"
  1321. echo ""
  1322. echo " apt install gnuplot"
  1323. echo ""
  1324. return 1
  1325. } >&2
  1326. ( depends rrdtool ) || {
  1327. echo ""
  1328. echo " Rrdtool is required to display graphs..." \
  1329. "You might want to try to install ${WHITE}rrdtool${NORMAL} with:"
  1330. echo ""
  1331. echo " apt install rrdtool"
  1332. echo ""
  1333. return 1
  1334. } >&2
  1335. export GNUTERM=qt
  1336. ## rrdtool fetch will use comma for floating point depending on some locals !
  1337. export LC_ALL=C
  1338. exec {PFD}> >(exec gnuplot 2>/dev/null)
  1339. gnuplot_pid="$!"
  1340. if [ -z "$opt_follow" ]; then
  1341. echo "${WHITE}Draw gnuplot graph${GRAY}..${NORMAL}" >&2
  1342. else
  1343. gnuplot_end_msg=" ${GRAY}..${NORMAL} ${DARKGREEN}stop${NORMAL} gnuplot process (pid: $gnuplot_pid)"
  1344. trap_add EXIT \
  1345. "printf '%s\n' \"$gnuplot_end_msg\" && kill $gnuplot_pid"
  1346. echo "${WHITE}Start gnuplot process${NORMAL} (pid: $gnuplot_pid) ${GRAY}..${NORMAL}" >&2
  1347. fi
  1348. echo "set term qt noraise replotonresize" >&$PFD
  1349. while true; do
  1350. {
  1351. i=0
  1352. data_start_ts=
  1353. data_stop_ts=
  1354. for resource in "${resources[@]}"; do
  1355. for vps in "${VPS[@]}"; do
  1356. ((i++))
  1357. {
  1358. {
  1359. rrd_vps_path="$VAR_DIR/rrd/$vps"
  1360. if [ -d "$rrd_vps_path" ]; then
  1361. graph:def:"${resource}" "$vps" "$i" "${opts_rrdfetch[@]}"
  1362. else
  1363. warn "No data yet, ignoring..."
  1364. fi
  1365. } 3>&1 1>&2 2>&3 | prefix " ${DARKRED}\!${GRAY} graph(${DARKCYAN}$vps${GRAY}:${WHITE}$resource${NORMAL})${NORMAL} "
  1366. set_errlvl "${PIPESTATUS[0]}"
  1367. } 3>&1 1>&2 2>&3 || continue 1
  1368. done
  1369. done
  1370. } >&$PFD
  1371. if [ -z "$opt_follow" ]; then
  1372. echo " ${GRAY}..${DARKGREEN} done${NORMAL} gnuplot graphing" >&2
  1373. break
  1374. else
  1375. {
  1376. echo "${WHITE}Sleeping ${DARKYELLOW}${opt_interval:-60}${NORMAL}s..."
  1377. sleep "${opt_interval:-60}"
  1378. echo " ${GRAY}..${NORMAL} ${DARKGREEN}done${NORMAL} sleeping"
  1379. } | prefix " ${GRAY}| gnuplot:${NORMAL} " >&2
  1380. fi
  1381. done
  1382. if [ -n "$opt_follow" ]; then
  1383. echo "Waiting for child process to finish.." >&2
  1384. wait
  1385. echo " ..done" >&2
  1386. else
  1387. echo "pause mouse close" >&$PFD
  1388. fi
  1389. }
  1390. graph:def:c.memory() {
  1391. local vps="$1" i="$2"
  1392. shift 2
  1393. local opts_rrdfetch=("$@")
  1394. local resource="memory"
  1395. rrd_vps_path="$VAR_DIR/rrd/$vps"
  1396. [ -d "$rrd_vps_path/containers" ] || {
  1397. warn "No containers data yet for vps '$vps'... Ignoring"
  1398. return 0
  1399. }
  1400. containers=(
  1401. $(
  1402. cd "$rrd_vps_path/containers";
  1403. find -maxdepth 3 -mindepth 3 -name "${resource}.rrd" -type f |
  1404. sed -r 's%^./([^/]+/[^/]+)/[^/]+.rrd$%\1%g'
  1405. )
  1406. )
  1407. gnuplot_line_config=(
  1408. "set term qt $i title \"$vps $resource\" replotonresize noraise"
  1409. "set title '$vps'"
  1410. "set xdata time"
  1411. "set timefmt '%s'"
  1412. "set ylabel '$resource Usage'"
  1413. "set format y '%s'"
  1414. "set ytics format ' %g'"
  1415. "set mouse mouseformat 6"
  1416. "set yrange [0:*] "
  1417. "set border behind"
  1418. )
  1419. printf "%s\n" "${gnuplot_line_config[@]}"
  1420. first=1
  1421. for container in "${containers[@]}"; do
  1422. rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/containers/$container/$resource.rrd\""
  1423. rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
  1424. rrdfetch_cmd+=" tail -n +2 | \\"$'\n'
  1425. rrdfetch_cmd+=" egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
  1426. rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
  1427. rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
  1428. first_ts=
  1429. first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ")
  1430. if [ -z "$first_ts" ]; then
  1431. warn "No data for $container on vps $vps, skipping..."
  1432. continue
  1433. fi
  1434. last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1 | cut -f 1 -d " ")
  1435. if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
  1436. data_start_ts="$first_ts"
  1437. fi
  1438. if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
  1439. data_stop_ts="$last_ts"
  1440. fi
  1441. if [ -n "$first" ]; then
  1442. first=
  1443. echo "plot \\"
  1444. else
  1445. echo ", \\"
  1446. fi
  1447. container="${container//\'/}"
  1448. container="${container//@/\\@}"
  1449. echo -n " ${rrdfetch_cmd} u 1:(\$3/(1000*1000*1000)) w lines title '${container//_/\\_}'"
  1450. done
  1451. echo
  1452. }
  1453. graph:def:c.network() {
  1454. local vps="$1" i="$2"
  1455. shift 2
  1456. local opts_rrdfetch=("$@")
  1457. local resource="network"
  1458. rrd_vps_path="$VAR_DIR/rrd/$vps"
  1459. [ -d "$rrd_vps_path/containers" ] || {
  1460. warn "No containers data yet for vps '$vps'... Ignoring"
  1461. return 0
  1462. }
  1463. containers=(
  1464. $(
  1465. cd "$rrd_vps_path/containers";
  1466. find -maxdepth 3 -mindepth 3 -name "${resource}.rrd" -type f |
  1467. sed -r 's%^./([^/]+/[^/]+)/[^/]+.rrd$%\1%g'
  1468. )
  1469. )
  1470. gnuplot_line_config=(
  1471. "set term qt $i title \"$vps $resource\" replotonresize noraise"
  1472. "set title '$vps'"
  1473. "set xdata time"
  1474. "set timefmt '%s'"
  1475. "set ylabel '$resource Usage'"
  1476. "set format y '%s'"
  1477. "set ytics format ' %.2f MiB/s'"
  1478. "set mouse mouseformat 6"
  1479. "set yrange [0:*] "
  1480. "set border behind"
  1481. )
  1482. printf "%s\n" "${gnuplot_line_config[@]}"
  1483. first=1
  1484. for container in "${containers[@]}"; do
  1485. rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/containers/$container/$resource.rrd\""
  1486. rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
  1487. rrdfetch_cmd+=" tail -n +2 | \\"$'\n'
  1488. rrdfetch_cmd+=" egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
  1489. rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
  1490. rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
  1491. first_ts=
  1492. first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ")
  1493. if [ -z "$first_ts" ]; then
  1494. warn "No data for $container on vps $vps, skipping..."
  1495. continue
  1496. fi
  1497. last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1 | cut -f 1 -d " ")
  1498. if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
  1499. data_start_ts="$first_ts"
  1500. fi
  1501. if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
  1502. data_stop_ts="$last_ts"
  1503. fi
  1504. if [ -n "$first" ]; then
  1505. first=
  1506. echo "plot \\"
  1507. else
  1508. echo ", \\"
  1509. fi
  1510. container="${container//\'/}"
  1511. container="${container//@/\\@}"
  1512. echo -n " ${rrdfetch_cmd} u 1:((\$3 / 1024) / 1024) w lines title '${container//_/\\_}'"
  1513. done
  1514. echo
  1515. }
  1516. graph:def:load_avg() {
  1517. local vps="$1" i="$2"
  1518. shift 2
  1519. local opts_rrdfetch=("$@")
  1520. rrd_vps_path="$VAR_DIR/rrd/$vps"
  1521. [ -f "$rrd_vps_path/$resource.rrd" ] || {
  1522. warn "No containers data yet for vps '$vps'... Ignoring"
  1523. return 0
  1524. }
  1525. gnuplot_line_config=(
  1526. "set term qt $i title \"$vps $resource\" replotonresize noraise"
  1527. "set title '$vps'"
  1528. "set xdata time"
  1529. "set timefmt '%s'"
  1530. "set ylabel '${resource//_/\\_} Usage'"
  1531. "set format y '%s'"
  1532. "set ytics format '%g'"
  1533. "set mouse mouseformat 6"
  1534. "set yrange [0:*] "
  1535. "set border behind"
  1536. )
  1537. printf "%s\n" "${gnuplot_line_config[@]}"
  1538. first=1
  1539. for value in 1m:2 5m:3 15m:4; do
  1540. label="${value%:*}"
  1541. col_num="${value#*:}"
  1542. rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/$resource.rrd\""
  1543. rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
  1544. rrdfetch_cmd+=" tail -n +2 | \\"$'\n'
  1545. rrdfetch_cmd+=" egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
  1546. rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
  1547. rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
  1548. first_ts=
  1549. first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ")
  1550. if [ -z "$first_ts" ]; then
  1551. warn "No data for $resource on vps $vps, skipping..."
  1552. continue
  1553. fi
  1554. last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1 | cut -f 1 -d " ")
  1555. if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
  1556. data_start_ts="$first_ts"
  1557. fi
  1558. if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
  1559. data_stop_ts="$last_ts"
  1560. fi
  1561. if [ -n "$first" ]; then
  1562. first=
  1563. echo "plot \\"
  1564. else
  1565. echo ", \\"
  1566. fi
  1567. container="${container//\'/}"
  1568. container="${container//@/\\@}"
  1569. echo -n " ${rrdfetch_cmd} u 1:$col_num w lines title '${label}'"
  1570. done
  1571. echo
  1572. }
  1573. graph:def:disk() {
  1574. local vps="$1" i="$2"
  1575. shift 2
  1576. local opts_rrdfetch=("$@")
  1577. rrd_vps_path="$VAR_DIR/rrd/$vps"
  1578. [ -f "$rrd_vps_path/$resource.rrd" ] || {
  1579. warn "No containers data yet for vps '$vps'... Ignoring"
  1580. return 0
  1581. }
  1582. gnuplot_line_config=(
  1583. "set term qt $i title \"$vps $resource\" replotonresize noraise"
  1584. "set title '$vps'"
  1585. "set xdata time"
  1586. "set timefmt '%s'"
  1587. "set ylabel '${resource//_/\\_} Usage'"
  1588. "set format y '%s'"
  1589. "set ytics format '%g GiB'"
  1590. "set mouse mouseformat 6"
  1591. "set yrange [0:*] "
  1592. "set border behind"
  1593. )
  1594. printf "%s\n" "${gnuplot_line_config[@]}"
  1595. first=1
  1596. for value in used:2 size:3; do
  1597. label="${value%:*}"
  1598. col_num="${value#*:}"
  1599. rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/$resource.rrd\""
  1600. rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
  1601. rrdfetch_cmd+=" tail -n +2 | \\"$'\n'
  1602. rrdfetch_cmd+=" egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
  1603. rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
  1604. rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
  1605. first_ts=
  1606. first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ")
  1607. if [ -z "$first_ts" ]; then
  1608. warn "No data for $resource on vps $vps, skipping..."
  1609. continue
  1610. fi
  1611. last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1 | cut -f 1 -d " ")
  1612. if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
  1613. data_start_ts="$first_ts"
  1614. fi
  1615. if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
  1616. data_stop_ts="$last_ts"
  1617. fi
  1618. if [ -n "$first" ]; then
  1619. first=
  1620. echo "plot \\"
  1621. else
  1622. echo ", \\"
  1623. fi
  1624. container="${container//\'/}"
  1625. container="${container//@/\\@}"
  1626. echo -n " ${rrdfetch_cmd} u 1:(\$${col_num}/(1024*1024)) w lines title '${label}'"
  1627. done
  1628. echo
  1629. }
  1630. cmdline.spec.gnu vps-subscribe
  1631. cmdline.spec::cmd:vps-subscribe:run() {
  1632. :
  1633. }
  1634. cmdline.spec.gnu add
  1635. cmdline.spec:vps-subscribe:cmd:add:run() {
  1636. : :posarg: CHANNEL 'Channel which will be sent to given topic'
  1637. : :posarg: TOPIC 'Ntfy topic to recieve messages of given channel
  1638. (format: "[MYSERVER:]MYTOPICS"
  1639. Examples: "ntfy.0k.io:main,storage,alerts",
  1640. "main{1,3,7}"
  1641. )'
  1642. : :posarg: [VPS...] 'Target host(s) to get stats'
  1643. printf "%s\0" "$CHANNEL" "$TOPIC" |
  1644. vps_mux subscribe:add "${VPS[@]}"
  1645. }
  1646. cmdline.spec.gnu rm
  1647. cmdline.spec:vps-subscribe:cmd:rm:run() {
  1648. : :posarg: CHANNEL 'Channel which will be sent to given topic'
  1649. : :posarg: TOPIC 'Ntfy topic to recieve messages of given channel
  1650. (format: "[MYSERVER:]MYTOPICS"
  1651. Examples: "ntfy.0k.io:main,storage,alerts",
  1652. "main{1,3,7}"
  1653. )'
  1654. : :posarg: [VPS...] 'Target host(s) to get stats'
  1655. printf "%s\0" "$CHANNEL" "$TOPIC" |
  1656. vps_mux subscribe:rm "${VPS[@]}"
  1657. }
  1658. cmdline::parse "$@"