bgallet
/
myc-manage
forked from Myceliandre/myc-manage
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
157 Commits
5 Branches
0 Tags
2.4 MiB
 
 
Tree: b87b36303d
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'b87b36303d'
${ noResults }
myc-manage/bin/0km

1872 lines
62 KiB
Raw Blame History

#!/bin/bash
. /etc/shlib
include common
include parse
include cmdline
include config
[[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true
version=0.1
desc='Manage 0k related installs'
help=""
##
## Functions
##
is-port-open() {
local host="$1" port="$2" timeout=5
start="$SECONDS"
debug "Testing if $host's port $2 is open ..."
while true; do
timeout 1 bash -c "</dev/tcp/${host}/${port}" >/dev/null 2>&1 && break
sleep 0.2
if [ "$((SECONDS - start))" -gt "$timeout" ]; then
return 1
fi
done
debug " .. $host's port $2 is open."
return 0
}
resolve() {
local ent hostname="$1"
debug "Resolving $1 ..."
if ent=$(getent ahosts "$hostname"); then
ent=$(echo "$ent" | egrep ^"[0-9]+.[0-9]+.[0-9]+.[0-9]+\s+" | \
head -n 1 | awk '{ print $1 }')
debug " .. resolved $1 to $ent."
echo "$ent"
else
debug " .. couldn't resolve $1."
return 1
fi
}
set_errlvl() { return "${1:-1}"; }
export master_pid=$$
ssh:open() {
local hostname ssh_cmd ssh_options
ssh_cmd=(ssh)
ssh_options=()
while [ "$#" != 0 ]; do
case "$1" in
"--stdin-password")
ssh_cmd=(sshpass "${ssh_cmd[@]}")
;;
-o)
ssh_options+=("$1" "$2")
shift
;;
*)
[ -z "$hostname" ] && hostname="$1" || {
err "Surnumerous positional argument '$1'. Expecting only hostname."
return 1
}
;;
esac
shift
done
full_cmd=(
"${ssh_cmd[@]}"
-o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
-o ControlMaster=auto -o ControlPersist=900 \
-o ConnectTimeout=5 -o StrictHostKeyChecking=no \
"${ssh_options[@]}" \
"$hostname" "$@" -- true)
"${full_cmd[@]}" >/dev/null 2>&1 || {
err "Failed: ${full_cmd[*]}"
return 1
}
trap_add EXIT,INT 'ssh:quit "$hostname"'
}
ssh:rsync() {
local src="$1" dst="$2"
hostname=${src%%:*}
hostname=${hostname#*@}
local rsync_ssh_options=(
-o ControlPath="/tmp/ssh-control-master-${master_pid}-$hostname"
-o ControlMaster=auto
-o ControlPersist=900
-o ConnectTimeout=10
-o StrictHostKeyChecking=no
)
if ! ssh:run "root@$hostname" -- type -p rsync </dev/null >/dev/null; then
info "No 'rsync' available on '$hostname'. Requesting installation..."
ssh:run "root@$hostname" -- apt-get install rsync -y || {
err "Installation of 'rsync' failed on '$hostname'"
return 1
}
fi
local cmd=(
rsync -e "ssh ${rsync_ssh_options[*]}"
-azvArH --delete --delete-excluded
--partial --partial-dir .rsync-partial
"$src" "$dst"
)
"${cmd[@]}"
}
ssh:open-try() {
local opts hostnames
opts=()
hostnames=()
while [ "$#" != 0 ]; do
case "$1" in
-o)
opts+=("$1" "$2")
shift
;;
*)
hostnames+=("$1")
;;
esac
shift
done
password=''
for host in "${hostnames[@]}"; do
debug "Trying $host with publickey."
ssh:open -o PreferredAuthentications=publickey \
"${opts[@]}" \
"$host" >/dev/null 2>&1 && {
echo "$host"$'\n'"$password"$'\n'
return 0
}
debug " .. failed connecting to $host with publickey."
done
local times=0 password
while [ "$((++times))" -le 3 ]; do
read -sp "$HOST's password: " password
errlvl="$?"
echo >&2
if [ "$errlvl" -gt 0 ]; then
exit 1
fi
for host in "${hostnames[@]}"; do
debug "Trying $host with password ($times/3)"
echo "$password" | ssh:open -o PreferredAuthentications=password \
--stdin-password \
"${opts[@]}" \
"$host" >/dev/null 2>&1 && {
echo "$host"$'\n'"$password"$'\n'
return 0
}
debug " .. failed connecting to $host with password."
done
err "login failed. Try again... ($((times+1))/3)"
done
return 1
}
ssh:run() {
local hostname="$1" ssh_options cmd
shift
ssh_options=()
cmd=()
while [ "$#" != 0 ]; do
case "$1" in
"--")
shift
cmd+=("$@")
break
;;
*)
ssh_options+=("$1")
;;
esac
shift
done
## XXXvlab: keeping in case we need some debug
# debug "$DARKCYAN$hostname$NORMAL $WHITE\$$NORMAL" "$@"
# debug "Running cmd: ${cmd[*]}"
# for arg in "${cmd[@]}"; do
# debug "$arg"
# done
{
{
ssh -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
-o ControlMaster=auto -o ControlPersist=900 \
-o "StrictHostKeyChecking=no" \
"${ssh_options[@]}" "$hostname" -- "${cmd[@]}"
} 3>&1 1>&2 2>&3 ## | sed -r "s/^/$DARKCYAN$hostname$NORMAL $DARKRED\!$NORMAL /g"
set_errlvl "${PIPESTATUS[0]}"
} 3>&1 1>&2 2>&3
}
ssh:quit() {
local hostname="$1"
shift
ssh -o ControlPath=/tmp/ssh-control-master-${master_pid} \
-o ControlMaster=auto -o ControlPersist=900 -O exit \
"$hostname" 2>/dev/null
}
is_ovh_domain_name() {
local domain="$1"
[[ "$domain" == *.ovh.net ]] && return 0
[[ "$domain" == "ns"*".ip-"*".eu" ]] && return 0
return 1
}
is_ovh_hostname() {
local domain="$1"
[[ "$domain" =~ ^vps-[0-9a-f]*$ ]] && return 0
[[ "$domain" =~ ^vps[0-9]*$ ]] && return 0
return 1
}
vps_connection_check() {
local vps="$1"
ip=$(resolve "$vps") ||
{ echo "${DARKRED}no-resolve${NORMAL}"; return 1; }
is-port-open "$ip" "22" </dev/null ||
{ echo "${DARKRED}no-port-22-open${NORMAL}"; return 1; }
ssh:open -o ConnectTimeout=10 -o PreferredAuthentications=publickey \
"root@$vps" >/dev/null 2>&1 ||
{ echo "${DARKRED}no-ssh-root-access${NORMAL}"; return 1; }
}
vps_check() {
local vps="$1"
vps_connection_check "$vps" </dev/null || return 1
if size=$(
echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 5 -d ' ' | cut -f 1 -d %" |
ssh:run "root@$vps" -- bash); then
if [ "$size" -gt "90" ]; then
echo "${DARKRED}above-90%-disk-usage${NORMAL}"
elif [ "$size" -gt "75" ]; then
echo "${DARKYELLOW}above-75%-disk-usage${NORMAL}"
fi
else
echo "${DARKRED}no-size${NORMAL}"
fi </dev/null
compose_content=$(ssh:run "root@$vps" -- cat /opt/apps/myc-deploy/compose.yml </dev/null) ||
{ echo "${DARKRED}no-compose${NORMAL}"; return 1; }
echo "$compose_content" | yq -e ".rsync-backup" >/dev/null 2>&1 ||
{ echo "${DARKRED}no-backup${NORMAL}"; return 1; }
}
backup:setup-rsync() {
local admin="$1" vps="$2" server="$3" id="$4"
[ -z "${BACKUP_SSH_SERVER}" ] || {
err "Unexpected error: '\$BACKUP_SSH_SERVER' is already set in '$FUNCNAME'."
return 1
}
BACKUP_SSH_OPTIONS=(-o StrictHostKeyChecking=no)
if [[ "$server" == *":"* ]]; then
BACKUP_SSH_OPTIONS+=(-p "${server#*:}")
BACKUP_SSH_SERVER=${server%%:*}
else
BACKUP_SSH_SERVER="$server"
fi
if ! private_key=$(ssh "${BACKUP_SSH_OPTIONS[@]}" \
"$admin"@"${BACKUP_SSH_SERVER}" request-recovery-key "$id"); then
err "Couldn't request a recovery key for '$id' with account '$admin'."
return 1
fi
if ! VPS_TMP_DIR=$(echo "mktemp -d" | ssh:run "root@$vps" -- bash); then
err "Couldn't create a temporary directory on vps"
return 1
fi
cat <<EOF | ssh:run "root@$vps" -- bash || return 1
touch "$VPS_TMP_DIR/recover_key" &&
chmod go-rwx "$VPS_TMP_DIR/recover_key" &&
printf "%s\n" "$private_key" >> "$VPS_TMP_DIR/recover_key"
EOF
BACKUP_SSH_OPTIONS+=(-i "$VPS_TMP_DIR/recover_key" -l rsync)
BACKUP_VPS_TARGET="$vps"
BACKUP_IDENT="$id"
echo "type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null" |
ssh:run "root@$vps" -- bash || return 1
}
backup:rsync() {
local ssh_options
[ -n "${BACKUP_SSH_SERVER}" ] || {
err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
return 1
}
rsync_options=()
while [[ "$1" == "-"* ]]; do
rsync_options+=("$1")
shift
done
local src="$1" dst="$2"
cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash
rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
-azvArH --delete --delete-excluded \
--partial --partial-dir .rsync-partial \
--numeric-ids ${rsync_options[*]} \
"${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "${dst}"
EOF
}
backup:path_exists() {
local src="$1"
[ -n "${BACKUP_SSH_SERVER}" ] || {
err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
return 1
}
cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash >/dev/null 2>&1
rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
-nazvArH --numeric-ids \
"${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "/tmp/dummy"
EOF
}
file:vps_backup_recover() {
local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"
backup:rsync "${path}" "${vps_path}" || return 1
if [[ "$path" == *"/" ]]; then
if [ "$path" == "$vps_path"/ ]; then
msg_target="Directory '$path'"
else
msg_target="Directory '$path' -> '$vps_path'"
fi
else
if [ "$path" == "$vps_path" ]; then
msg_target="File '$path'"
else
msg_target="File '$path' -> '$vps_path'"
fi
fi
info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."
}
mailcow:vps_backup_recover() {
local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"
if ! compose_yml_files=$(cat <<EOF | ssh:run "root@$vps" -- bash
urn=com.docker.compose.project
docker ps -f "label=\$urn=mailcowdockerized" \
--format="{{.Label \"\$urn.working_dir\"}}/{{.Label \"\$urn.config_files\"}}" |
uniq
EOF
); then
err "Couldn't get list of running projects"
return 1
fi
stopped_containers=
if [ -n "$compose_yml_files" ]; then
echo "Found running mailcowdockerized containers" >&2
if [[ "$compose_yml_files" == *$'\n'* ]]; then
err "Running containers are confusing, did not find only one mailcowdockerized project."
return 1
fi
if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
## For some reason, sometimes $urn.config_files holds an absolute path
compose_yml_files=/${compose_yml_files#*//}
if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
err "Running containers are confusing, they don't point to an existing docker-compose.yml."
return 1
fi
fi
echo "Containers where launched from '$compose_yml_files'" >&2
COMPOSE_FILE="$compose_yml_files"
ENV_FILE="${COMPOSE_FILE%/*}/.env"
if ! echo "[ -e \"${ENV_FILE}\" ]" | ssh:run "root@$vps" -- bash ; then
err "Running containers are confusing, docker-compose.yml has no '.env' next to it."
return 1
fi
echo "${WHITE}Bringing mailcowdockerized down${NORMAL}"
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
ssh:run "root@$vps" -- bash
stopped_containers=1
fi
if [[ "$path" == "/"* ]]; then
##
## Additional intelligence to simple file copy
##
if [[ "$path" == "/var/lib/docker/volumes/mailcowdockerized_*-vol-1/_data"* ]]; then
volume_name=${path#/var/lib/docker/volumes/}
volume_name=${volume_name%%/*}
volume_dir=${path%%"$volume_name"*}
## Create volumes if not existent
if ! ssh:run "root@$vps" -- "
[ -d '${volume_dir}' ] ||
docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
[ -d '${volume_dir}' ]
"; then
err "Couldn't find nor create '${volume_dir}'."
return 1
fi
fi
echo "${WHITE}Sync from backup ${path} to VPS ${vps_path}${NORMAL}" >&2
backup:rsync "${path}" "${vps_path}" || return 1
if [[ "$path" == *"/" ]]; then
if [ "$path" == "$vps_path"/ ]; then
msg_target="Directory '$path'"
else
msg_target="Directory '$path' -> '$vps_path'"
fi
else
if [ "$path" == "$vps_path" ]; then
msg_target="File '$path'"
else
msg_target="File '$path' -> '$vps_path'"
fi
fi
else
ALL_TARGETS=(mailcow postfix rspamd redis crypt vmail{,-attachments} mysql)
if [[ -n "$path" ]]; then
targets=()
bad_targets=()
for target in ${path//,/ }; do
if [[ " ${ALL_TARGETS[*]} " != *" $target "* ]]; then
bad_targets+=("$target")
fi
targets+=("$target")
done
if [ "${#bad_targets[@]}" -gt 0 ]; then
bad_target_msg=$(printf "%s, " "${bad_targets[@]}")
err "Unknown components: ${bad_target_msg%, }. These are allowed components:"
printf " - %s\n" "${ALL_TARGETS[@]}" >&2
return 1
fi
msg_target="Partial mailcow backup"
else
targets=("${ALL_TARGETS[@]}")
msg_target="Full mailcow backup"
fi
for target in "${targets[@]}"; do
case "$target" in
postfix|rspamd|redis|crypt|vmail|vmail-attachments)
volume_name="mailcowdockerized_${target}-vol-1"
volume_dir="/var/lib/docker/volumes/${volume_name}/_data"
if ! backup:path_exists "${volume_dir}/"; then
warn "No '$volume_name' in backup. This might be expected."
continue
fi
## Create volumes if not existent
if ! ssh:run "root@$vps" -- "
[ -d '${volume_dir}' ] ||
docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
[ -d '${volume_dir}' ]
"; then
err "Couldn't find nor create '${volume_dir}'."
return 1
fi
echo "${WHITE}Downloading of $volume_name${NORMAL}"
backup:rsync "${volume_dir}/" "${volume_dir}" || return 1
;;
mailcow)
## Mailcow git base
COMPOSE_FILE=
for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
backup:path_exists "${mailcow_dir}/" || continue
## this possibly change last value
COMPOSE_FILE="$mailcow_dir/docker-compose.yml"
ENV_FILE="$mailcow_dir/.env"
echo "${WHITE}Download of $mailcow_dir${NORMAL}"
backup:rsync "${mailcow_dir}"/ "${mailcow_dir}" || return 1
break
done
if [ -z "$COMPOSE_FILE" ]; then
err "Can't find mailcow base installation path in backup."
return 1
fi
;;
mysql)
if [ -z "$COMPOSE_FILE" ]; then
## Mailcow git base
compose_files=()
for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
ssh:run "root@$vps" -- "[ -e \"$mailcow_dir/docker-compose.yml\" ]" || continue
## this possibly change last value
compose_files+=("$mailcow_dir/docker-compose.yml")
done
if [ "${#compose_files[@]}" == 0 ]; then
err "No compose file found for mailcow installation."
return 1
elif [ "${#compose_files[@]}" -gt 1 ]; then
err "Multiple compose files for mailcow found:"
for f in "${compose_files[@]}"; do
echo " - $f" >&2
done
echo "Can't decide which to use for mounting mysql container." >&2
return 1
fi
COMPOSE_FILE="${compose_files[0]}"
ENV_FILE="${COMPOSE_FILE%/*}/.env"
if ! ssh:run "root@$vps" -- "[ -e \"${COMPOSE_FILE%/*}/.env\" ]"; then
err "No env file in '$ENV_FILE' found."
return 1
fi
fi
## Mysql database
echo "${WHITE}Downloading last backup of mysql backups${NORMAL}"
backup:rsync "/var/backups/mysql/" "/var/backups/mysql" || return 1
if ! env_content=$(echo "cat '$ENV_FILE'" | ssh:run "root@$vps" -- bash); then
err "Can't access env file: '$ENV_FILE'."
return 1
fi
root_password=$(printf "%s\n" "$env_content" | grep ^DBROOT= | cut -f 2 -d =)
echo "${WHITE}Bringing mysql-mailcow up${NORMAL}"
if ! image=$(cat <<EOF | ssh:run "root@$vps" -- bash
shyaml get-value services.mysql-mailcow.image < "${COMPOSE_FILE}"
EOF
); then
err "Failed to get image name of service 'mysql-mailcow' in 'compose.yml'."
return 1
fi
if [ -z "$(ssh:run "root@$vps" -- docker images -q "$image")" ]; then
info "Image '$image' not available, pull it."
if ! ssh:run "root@$vps" -- \
docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
pull mysql-mailcow; then
err "Failed to pull image of service 'mysql-mailcow'."
return 1
fi
fi
if ! container_id=$(cat <<EOF | ssh:run "root@$vps" -- bash
echo "[client]
password=$root_password" > "$VPS_TMP_DIR/my.cnf"
docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
run -d \
-v "$VPS_TMP_DIR/my.cnf:/root/.my.cnf:ro" \
mysql-mailcow
EOF
); then
err "Failed to bring up mysql-mailcow"
return 1
fi
START="$SECONDS"
retries=0
timeout=600
while true; do
((retries++))
echo " waiting for mysql db..." \
"(retry $retries, $(($SECONDS - $START))s elapsed, timeout is ${timeout}s)" >&2
cat <<EOF | ssh:run "root@$vps" -- bash && break
echo "SELECT 1;" | docker exec -i "$container_id" mysql >/dev/null 2>&1
EOF
if (($SECONDS - $START > $timeout)); then
err "Failed to connect to mysql-mailcow."
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
ssh:run "root@$vps" -- bash
return 1
fi
sleep 0.4
done
DBUSER=$(printf "%s\n" "$env_content" | grep ^DBUSER= | cut -f 2 -d =)
DBPASS=$(printf "%s\n" "$env_content" | grep ^DBPASS= | cut -f 2 -d =)
echo "${WHITE}Uploading mysql dump${NORMAL}"
cat <<EOF | ssh:run "root@$vps" -- bash
echo "
DROP DATABASE IF EXISTS mailcow;
CREATE DATABASE mailcow;
GRANT ALL PRIVILEGES ON mailcow.* TO '$DBUSER'@'%' IDENTIFIED BY '$DBPASS';
" | docker exec -i "$container_id" mysql
zcat /var/backups/mysql/mailcow/*.gz | docker exec -i "$container_id" mysql mailcow
EOF
if [ "$?" != 0 ]; then
err "Failed to load mysql dump."
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
ssh:run "root@$vps" -- bash
return 1
fi
echo "${WHITE}Bringing mysql-mailcow down${NORMAL}"
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
ssh:run "root@$vps" -- bash
;;
*)
err "Unknown component '$target'. Bailing out."
return 1
esac
done
ssh:run "root@$vps" -- "rm -rf '$VPS_TMP_DIR'"
fi
if [ -n "$stopped_containers" ]; then
echo "${WHITE}Starting mailcow${NORMAL}" >&2
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" up -d" |
ssh:run "root@$vps" -- bash
fi
info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."
}
NTFY_TOPIC_FILE="/etc/ntfy/topics.yml"
NTFY_CONFIG_FILE="/etc/ntfy/ntfy.conf"
subscribe:ntfy:topic-file-exists() {
local vps="$1"
if ! out=$(echo "[ -f \"$NTFY_TOPIC_FILE\" ] && echo ok || true" | \
ssh:run "root@$vps" -- bash); then
err "Unable to check for existence of '$NTFY_TOPIC_FILE'."
fi
if [ -z "$out" ]; then
err "File '$NTFY_TOPIC_FILE' not found on $vps."
return 1
fi
}
subscribe:ntfy:config-file-exists() {
local vps="$1"
if ! out=$(echo "[ -f \"$NTFY_CONFIG_FILE\" ] && echo ok || true" | \
ssh:run "root@$vps" -- bash); then
err "Unable to check for existence of '$NTFY_CONFIG_FILE'."
fi
if [ -z "$out" ]; then
err "File '$NTFY_CONFIG_FILE' not found on $vps."
return 1
fi
}
ntfy:rm() {
local channel="$1" topic="$2" vps="$3"
subscribe:ntfy:topic-file-exists "$vps" || return 1
if ! out=$(echo "yq -i 'del(.[\"$channel\"][] | select(. == \"$TOPIC\"))' \"$NTFY_TOPIC_FILE\"" | \
ssh:run "root@$vps" -- bash); then
err "Failed to remove channel '$channel' from '$NTFY_TOPIC_FILE'."
return 1
fi
info "Channel '$channel' removed from '$NTFY_TOPIC_FILE' on $vps."
ssh:run "root@$vps" -- cat "$NTFY_TOPIC_FILE"
}
ntfy:add() {
local channel="$1" topic="$2" vps="$3"
vps_connection_check "$vps" </dev/null || return 1
subscribe:ntfy:topic-file-exists "$vps" || return 1
if ! out=$(echo "yq '. | has(\"$channel\")' \"$NTFY_TOPIC_FILE\"" | \
ssh:run "root@$vps" -- bash); then
err "Failed to check if channel '$channel' with topic '$topic' is already in '$NTFY_TOPIC_FILE'."
return 1
fi
if [ "$out" != "true" ]; then
## Channel does not exist
if ! out=$(echo "yq -i '.[\"$channel\"] = []' \"$NTFY_TOPIC_FILE\"" | \
ssh:run "root@$vps" -- bash); then
err "Failed to create a new channel '$channel' entry in '$NTFY_TOPIC_FILE'."
return 1
fi
else
## Channel exists
if ! out=$(echo "yq '.[\"$channel\"] | any_c(. == \"$topic\")' \"$NTFY_TOPIC_FILE\"" | \
ssh:run "root@$vps" -- bash); then
err "Failed to check if channel '$channel' with topic '$topic' is already in '$NTFY_TOPIC_FILE'."
return 1
fi
if [ "$out" == "true" ]; then
info "Channel '$channel' with topic '$topic' already exists in '$NTFY_TOPIC_FILE'."
return 0
fi
fi
if ! out=$(echo "yq -i '.[\"$channel\"] += [\"$topic\"]' \"$NTFY_TOPIC_FILE\"" | \
ssh:run "root@$vps" -- bash); then
err "Failed to add channel '$channel' with topic '$topic' to '$NTFY_TOPIC_FILE'."
return 1
fi
info "Channel '$channel' added with topic '$topic' to '$NTFY_TOPIC_FILE' on $vps."
}
NTFY_BROKER_SERVER="ntfy.0k.io"
ntfy:topic-access() {
local action="$1" topic="$2" vps="$3"
subscribe:ntfy:config-file-exists "$vps" || return 1
local user
user=$(ntfy:get-login "$vps") || return 1
case "$action" in
"write")
ssh "ntfy@$NTFY_BROKER_SERVER" "topic-access" \
"$user" "$topic" "write-only" </dev/null || {
err "Failed to grant write access to '$user' for topic '$topic'."
return 1
}
info "Granted write access for '$user' to topic '$topic'."
;;
"remove")
ssh "ntfy@$NTFY_BROKER_SERVER" "topic-access" -r "$user" "$topic" </dev/null || {
err "Failed to reset access of '$user' for topic '$topic'."
return 1
}
info "Access for '$user' to topic '$topic' was resetted successfully."
;;
*)
err "Invalid action '$action'."
return 1
;;
esac
}
ntfy:get-login() {
local vps="$1"
if ! out=$(echo ". \"$NTFY_CONFIG_FILE\" && echo \"\$LOGIN\"" | \
ssh:run "root@$vps" -- bash); then
err "Failed to get ntfy login from '$NTFY_CONFIG_FILE'."
return 1
fi
if [ -z "$out" ]; then
err "Unexpected empty login retrieved from sourcing '$NTFY_CONFIG_FILE'."
return 1
fi
echo "$out"
}
subscribe:add() {
local vps="$1"
read-0 channel topic || {
err "Couldn't read CHANNEL and TOPIC arguments."
return 1
}
vps_connection_check "$vps" </dev/null || return 1
ntfy:topic-access "write" "$topic" "$vps" </dev/null || return 1
ntfy:add "$channel" "$topic" "$vps" || {
err "Failed to add channel '$channel' with topic '$topic' to '$NTFY_TOPIC_FILE'."
echo " Removing topic access." >&2
ntfy:topic-access "remove" "$topic" "$vps" </dev/null
return 1
}
}
subscribe:rm() {
local vps="$1"
read-0 channel topic || {
err "Couldn't read CHANNEL and TOPIC arguments."
return 1
}
vps_connection_check "$vps" </dev/null || return 1
ntfy:rm "$channel" "$topic" "$vps" || return 1
ntfy:topic-access "remove" "$topic" "$vps" </dev/null || {
err "Failed to remove topic access for '$topic' on '$vps'."
return 1
}
}
vps_backup_recover() {
local vps="$1" admin server id path rtype force type
read-0 admin server id path rtype force
if [[ "$vps" == *":"* ]]; then
vps_path=${vps#*:}
vps=${vps%%:*}
else
vps_path=
fi
vps_connection_check "$vps" </dev/null || {
err "Failed to access '$vps'."
return 1
}
if type=$(ssh:run "root@$vps" -- vps get-type); then
info "VPS $vps seems to be of ${WHITE}$type${NORMAL} type"
else
warn "Couldn't find type of vps '$vps' (command 'vps get-type' failed on vps)."
fi
if [ -z "$path" ]; then
if [ -n "$vps_path" ]; then
err "You can't provide a VPS with path as destination if you don't provide a path in backup source."
return 1
fi
info "No path provided in backup, so we assume you want ${WHITE}full recovery${NORMAL}."
if [ "$rtype" != "$type" ]; then
if [ -n "$force" ]; then
warn "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type."
else
err "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type. (use \`\`-f\`\` to force)"
return 1
fi
fi
else
if [ "$path" == "/" ]; then
if [ -z "$vps_path" ]; then
err "Recovery of '/' (full backup files) requires that you provide a vps path also."
return 1
fi
if [ "$vps_path" == "/" ]; then
err "Recovery of '/' (full backup files) requires that you provide" \
"a vps path different from '/' also."
return 1
fi
fi
fi
## Sets VPS and internal global variable to allow rsync to work
## from vps to backup server.
backup:setup-rsync "$admin" "$vps" "$server" "$id" || return 1
if [[ "$path" == "/"* ]]; then
if ! backup:path_exists "${path}"; then
err "File or directory '$path' not found in backup."
return 1
fi
if [ -z "$vps_path" ]; then
if [[ "$path" != *"/" ]] && backup:path_exists "${path}"/ ; then
path="$path/"
fi
vps_path=${path%/}
vps_path=${vps_path:-/}
fi
fi
case "$rtype-$type" in
mailcow-*)
## Supports having $path and $vps_path set or unset, with additional behavior
mailcow:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
;;
*-*)
if [[ "$path" == "/"* ]]; then
## For now, will require having $path and $vps_path set, no additional behaviors
file:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
else
if [ -n "$path" ]; then
err "Partial component recover of ${rtype:-unknown} backup type on" \
"${type:-unknown} type VPS is not yet implemented."
return 1
else
err "Full recover of ${rtype:-unknown} backup type on" \
"${type:-unknown} type VPS is not yet implemented."
return 1
fi
fi
;;
esac
}
vps_install_backup() {
local vps="$1" admin server
vps_connection_check "$vps" </dev/null || return 1
read-0 admin server
if ! type=$(ssh:run "root@$vps" -- vps get-type </dev/null); then
err "Could not get type."
return 1
fi
backup_opts=()
local opt
while read-0 opt; do
case "$opt" in
--ignore-domain-check|--ignore-ping-check)
backup_opts+=("$opt")
;;
*)
err "Unknown option '$opt'."
return 1
;;
esac
done
if ! out=$(ssh:run "root@$vps" -- vps install backup "$server" "${backup_opts[@]}" 2>&1); then
err "Command 'vps install backup $server ${backup_opts[@]}' on $vps failed:"
echo "$out" | prefix " ${DARKGRAY}|${NORMAL} " >&2
return 1
fi
## Format of output:
##
## II Entry for service rsync-backup is already present in '/opt/apps/myc-deploy/compose.yml'.
## II You can run this following command from an host having admin access to core-07.0k.io:
## (Or send it to a backup admin of core-07.0k.io)
## ssh -p 10023 myadmin@core-07.0k.io ssh-key add 'ssh-rsa AAAAAAAD...QCJ\
## 8HH6pVgEpu1twyxpr9xTt7eh..WaJdgPoxmiEwGfjMMNGxs39ggOTKUuSFSmOv8TiA1fzY\
## s85hF...dKP1qByJU1k= compose@odoo.sikle.fr'
key="ssh-rsa ${out##*\'ssh-rsa }" ## remove everything before last "'ssh-rsa"
key="${key%\'*}" ## remove everything after last '
if ! [[ "$key" =~ ^"ssh-rsa "[a-zA-Z0-9/+=]+" "[a-zA-Z0-9._-]+"@"[a-zA-Z0-9._-]+$ ]]; then
err "Unexpected output from 'vps install backup $server'. Can't find key."
echo "$out" | prefix " ${GRAY}|$NORMAL " >&2
echo " Extracted key:" >&2
echo "$key" | prefix " ${GRAY}|$NORMAL " >&2
return 1
fi
if [ "$type" == "compose" ]; then
if ! ssh:run "root@$vps" -- \
docker exec myc_cron_1 \
cat /etc/cron.d/rsync-backup >/dev/null 2>&1; then
ssh:run "root@$vps" -- compose --debug up || {
err "Command 'compose --debug up' failed."
return 1
}
if ! ssh:run "root@$vps" -- \
docker exec myc_cron_1 \
cat /etc/cron.d/rsync-backup >/dev/null 2>&1; then
err "Launched 'compose up' successfully but ${YELLOW}cron${NORMAL} container is not setup as expected."
echo " Was waiting for existence of '/etc/cron.d/rsync-backup' in it." >&2
return 1
fi
fi
fi
dest="$server"
dest="${dest%/*}"
ssh_options=()
if [[ "$dest" == *":"* ]]; then
port="${dest##*:}"
dest="${dest%%:*}"
ssh_options=(-p "$port")
else
port=""
dest="${dest%%:*}"
fi
cmd=(ssh "${ssh_options[@]}" "$admin"@"$dest" ssh-key add "$key")
echo "${WHITE}Launching:${NORMAL} ${cmd[@]}"
"${cmd[@]}" || {
err "Failed add key to backup server '$dest'."
return 1
}
echo "${WHITE}Launching backup${NORMAL} from '$vps'"
ssh:run "root@$vps" -- vps backup || {
err "First backup failed to run."
return 1
}
echo "Backup is ${GREEN}up and running${NORMAL}."
}
vps_update() {
local vps="$1"
vps_connection_check "$vps" || return 1
ssh:run "root@$vps" -- myc-update </dev/null
}
vps_bash() {
local vps="$1"
vps_connection_check "$vps" </dev/null || return 1
ssh:run "root@$vps" -- bash
}
vps_mux() {
local fn="$1" vps_done VPS max_size vps
shift
VPS=($(printf "%s\n" "$@" | sort))
max_size=0
declare -A vps_done;
new_vps=()
for name in "${VPS[@]}"; do
[ -n "${vps_done[$name]}" ] && {
warn "duplicate vps '$name' provided. Ignoring."
continue
}
vps_done["$name"]=1
new_vps+=("$name")
size_name="${#name}"
[ "$max_size" -lt "${size_name}" ] &&
max_size="$size_name"
done
settmpdir "_0KM_TMP_DIR"
cat > "$_0KM_TMP_DIR/code"
for vps in "${new_vps[@]}"; do
label=$(printf "%-${max_size}s" "$vps")
(
{
{
"$fn" "$vps" < "$_0KM_TMP_DIR/code"
} 3>&1 1>&2 2>&3 | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKRED\!$NORMAL /g"
set_errlvl "${PIPESTATUS[0]}"
} 3>&1 1>&2 2>&3 | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKGRAY\|$NORMAL /g"
set_errlvl "${PIPESTATUS[0]}"
) &
done
wait
}
[ "$SOURCED" ] && return 0
##
## Command line processing
##
cmdline.spec.gnu
cmdline.spec.reporting
cmdline.spec.gnu vps-setup
cmdline.spec::cmd:vps-setup:run() {
: :posarg: HOST 'Target host to check/fix ssh-access'
: :optfla: --force,-f "Will force domain name change, even if
current hostname was not recognized as
an ovh domain name. "
depends sshpass shyaml
KEY_PATH="ssh-access.public-keys"
local keys=$(config get-value -y "ssh-access.public-keys") || true
if [ -z "$keys" ]; then
err "No ssh publickeys configured in config file."
echo " Looking for keys in ${WHITE}${KEY_PATH}${NORMAL}" \
"in config file." >&2
config:exists --message 2>&1 | prefix " "
if [ "${PIPESTATUS[0]}" == "0" ]; then
echo " Config file found in $(config:filename)"
fi
return 1
fi
local tkey=$(e "$keys" | shyaml get-type)
if [ "$tkey" != "sequence" ]; then
err "Value type of ${WHITE}${KEY_PATH}${NORMAL} unexpected (is $tkey, expecting sequence)."
echo " Check content of $(config:filename), and make sure to use a sequence." >&2
return 1
fi
local IP NAME keys host_pass_connected
if ! IP=$(resolve "$HOST"); then
err "'$HOST' name unresolvable."
exit 1
fi
NAME="$HOST"
if [ "$IP" != "$HOST" ]; then
NAME="$HOST ($IP)"
fi
if ! is-port-open "$IP" "22"; then
err "$NAME unreachable or port 22 closed."
exit 1
fi
debug "Host $IP's port 22 is open."
if ! host_pass_connected=$(ssh:open-try \
{root,debian}@"$HOST"); then
err "Could not connect to {root,debian}@$HOST with publickey nor password."
exit 1
fi
read-0a host password <<<"$host_pass_connected"
sudo_if_necessary=
if [ "$password" -o "${host%%@*}" != "root" ]; then
if ! ssh:run "$host" -- sudo -n true >/dev/null 2>&1; then
err "Couldn't do a password-less sudo from $host."
echo " This is not yet supported."
exit 1
else
sudo_if_necessary=sudo
fi
fi
SUPPORTED_KEY_TYPE=(ssh-rsa ssh-ed25519)
Section Checking access
while read-0 key; do
prefix="${key%% *}"
if ! [[ " ${SUPPORTED_KEY_TYPE[*]} " == *" $prefix "* ]]; then
err "Unsupported key:"$'\n'"$key"
echo " Please use only key of the following type:" >&2
printf " - %s\n" "${SUPPORTED_KEY_TYPE[@]}" >&2
return 1
fi
label="${key##* }"
Elt "considering adding key ${DARKYELLOW}$label${NORMAL}"
dest="/root/.ssh/authorized_keys"
if ssh:run "$host" -- $sudo_if_necessary grep "\"$key\"" "$dest" >/dev/null 2>&1 </dev/null; then
print_info "already present"
print_status noop
Feed
else
if echo "$key" | ssh:run "$host" -- $sudo_if_necessary tee -a "$dest" >/dev/null; then
print_info added
else
echo
Feedback failure
return 1
fi
Feedback success
fi
done < <(e "$keys" | shyaml get-values-0)
Section Checking ovh hostname file
Elt "Checking /etc/ovh-hostname"
if ! ssh:run "$host" -- $sudo_if_necessary [ -e "/etc/ovh-hostname" ]; then
print_info "creating"
ssh:run "$host" -- $sudo_if_necessary cp /etc/hostname /etc/ovh-hostname
ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
Elt "Checking /etc/ovh-hostname: $ovhname"
Feedback || return 1
else
ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
Elt "Checking /etc/ovh-hostname: $ovhname"
print_info "already present"
print_status noop
Feed
fi
if ! is_ovh_domain_name "$HOST" && ! str_is_ipv4 "$HOST"; then
Section Checking hostname
Elt "Checking /etc/hostname..."
old_etc_hostname="$(ssh:run "$host" -- $sudo_if_necessary cat /etc/hostname)"
if [ "$old_etc_hostname" != "$HOST" ]; then
Elt "/etc/hostname is '$old_etc_hostname'"
if is_ovh_hostname "$old_etc_hostname" || [ -n "$opt_force" ]; then
Elt "Hostname '$old_etc_hostname' --> '$HOST'"
print_info "creating"
echo "$HOST" | ssh:run "$host" -- $sudo_if_necessary tee /etc/hostname >/dev/null &&
ssh:run "$host" -- $sudo_if_necessary hostname "$HOST"
Feedback || return 1
else
Elt "Hostname '$old_etc_hostname' isn't an ovh domain"
print_info "no change"
print_status noop
Feed
warn "Domain name was not changed because it was already set"
echo " (use \`-f\` or \`--force\`) to force domain name change to $HOST." >&2
fi
else
print_info "already set"
print_status noop
Feed
fi
Elt "Checking consistency between /etc/hostname and \`hostname\`..."
etc_hostname="$(ssh:run "$host" -- $sudo_if_necessary cat /etc/hostname)"
transient_hostname="$(ssh:run "$host" -- $sudo_if_necessary hostname)"
if [ "$etc_hostname" != "$transient_hostname" ]; then
print_info "change"
ssh:run "$host" -- $sudo_if_necessary hostname "$etc_hostname"
Feedback || return 1
else
print_info "consistent"
print_status noop
Feed
fi
else
info "Not changing domain as '$HOST' doesn't seem to be final domain."
fi
}
cmdline.spec.gnu vps-check
cmdline.spec::cmd:vps-check:run() {
: :posarg: [VPS...] 'Target host(s) to check'
echo "" |
vps_mux vps_check "${VPS[@]}"
}
cmdline.spec.gnu vps-install
cmdline.spec::cmd:vps-install:run() {
:
}
cmdline.spec.gnu backup
cmdline.spec:vps-install:cmd:backup:run() {
: :posarg: BACKUP_TARGET 'Backup target.
(ie: myadmin@backup.domain.org:10023/256)'
: :optfla: --ignore-domain-check \
"Allow to bypass the domain check in
compose file (only used in compose
installation)."
: :optfla: --ignore-ping-check "Allow to bypass the ping check of host."
: :posarg: [VPS...] 'Target host(s) to check'
if [ "${#VPS[@]}" == 0 ]; then
warn "VPS list provided in command line is empty. Nothing will be done."
return 0
fi
if ! [[ "$BACKUP_TARGET" == *"@"* ]]; then
err "Missing admin account identifier in backup target."
echo " Have you forgottent to specify an account, ie 'myadmin@<MYBACKUP_SERVER>' ?)"
return 1
fi
admin=${BACKUP_TARGET%%@*}
server=${BACKUP_TARGET#*@}
opts=()
[ -n "$opt_ignore_ping_check" ] &&
opts+=("--ignore-ping-check")
[ -n "$opt_ignore_domain_check" ] &&
opts+=("--ignore-domain-check")
p0 "$admin" "$server" "${opts[@]}" |
vps_mux vps_install_backup "${VPS[@]}"
}
cmdline.spec.gnu vps-backup
cmdline.spec::cmd:vps-backup:run() {
:
}
cmdline.spec.gnu ls
cmdline.spec:vps-backup:cmd:ls:run() {
: :posarg: BACKUP_ID 'Backup id.
(ie: myadmin@backup.domain.org:10023)'
if ! [[ "$BACKUP_ID" == *"@"* ]]; then
err "Missing admin account identifier in backup id."
echo " Have you forgottent to specify an admin account ?" \
"ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
return 1
fi
id=${BACKUP_ID##*#}
BACKUP_TARGET=${BACKUP_ID%#*}
admin=${BACKUP_TARGET%%@*}
server=${BACKUP_TARGET#*@}
## XXXvlab: in this first implementation we expect to have access
## to the server main ssh port 22, so we won't use the provided port.
ssh_options=()
if [[ "$server" == *":"* ]]; then
ssh_options+=(-p "${server#*:}")
server=${server%%:*}
fi
ssh "${ssh_options[@]}" "$admin"@"$server" ssh-key ls
}
cmdline.spec.gnu recover
cmdline.spec:vps-backup:cmd:recover:run() {
: :posarg: BACKUP_ID 'Backup id.
(ie: myadmin@backup.domain.org:10023#mx.myvps.org
myadmin@ark-01.org#myid:/a/path
admin@ark-02.io#myid:myqsl,mailcow)'
: :posarg: VPS_PATH 'Target host(s) to check.
(ie: myvps.com
myvps.com:/a/path)'
: :optval: --date,-D '"last", or label of version to recover. (Default: "last").'
: :optfla: --force,-f 'Will allow you to bypass some checks.'
if ! [[ "$BACKUP_ID" == *"@"* ]]; then
err "Missing admin account identifier in backup id."
echo " Have you forgottent to specify an admin account ?" \
"ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
return 1
fi
if ! [[ "$BACKUP_ID" == *"@"*"#"* ]]; then
err "Missing backup label identifier in backup id."
echo " Have you forgottent to specify a backup label identifier ?" \
"ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
return 1
fi
id_path=${BACKUP_ID#*#}
if [[ "$id_path" == *":"* ]]; then
id=${id_path%%:*}
path=${id_path#*:}
else
id="$id_path"
path=
fi
BACKUP_TARGET=${BACKUP_ID%#*}
admin=${BACKUP_TARGET%%@*}
server=${BACKUP_TARGET#*@}
ssh_options=()
if [[ "$server" == *":"* ]]; then
ssh_options+=(-p "${server#*:}")
ssh_server=${server%%:*}
fi
BACKUP_PATH="/srv/datastore/data/rsync-backup-target/var/mirror"
if ! content=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key ls 2>/dev/null); then
err "Access denied to '$admin@${server}'."
return 1
fi
idents=$(echo "$content" | sed -r "s/"$'\e'"\[[0-9]+(;[0-9]+)*m//g" | cut -f 2 -d " ")
if ! [[ $'\n'"$idents"$'\n' == *$'\n'"$id"$'\n'* ]]; then
err "Given backup id '$id' not found in $admin@${server}'s idents."
return 1
fi
rtype=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key get-type "$id" ) &&
info "Backup archive matches ${WHITE}${rtype}${NORMAL} type"
p0 "$admin" "$server" "$id" "$path" "$rtype" "$opt_force" |
vps_backup_recover "${VPS_PATH}"
}
cmdline.spec.gnu vps-update
cmdline.spec::cmd:vps-update:run() {
: :posarg: [VPS...] 'Target host to check'
echo "" |
vps_mux vps_update "${VPS[@]}"
}
cmdline.spec.gnu vps-mux
cmdline.spec::cmd:vps-mux:run() {
: :posarg: [VPS...] 'Target host(s) to check'
cat | vps_mux vps_bash "${VPS[@]}"
}
cmdline.spec.gnu vps-space
cmdline.spec::cmd:vps-space:run() {
: :posarg: [VPS...] 'Target host(s) to check'
echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 2-5 -d ' '" |
vps_mux vps_bash "${VPS[@]}"
}
cmdline.spec.gnu vps-stats
cmdline.spec::cmd:vps-stats:run() {
: :posarg: [VPS...] 'Target host(s) to get stats'
: :optfla: --follow,-f 'Refresh graph every 2m'
: :optval: --timespan,-t 'timespan START[..END]'
: :optval: --resource,-r 'resource(s) separated with a comma'
: :optval: --interval,-i 'refersh interval (default: 60s)'
local opts_rrdfetch=( -a )
if [ -n "${opt_timespan}" ]; then
start=${opt_timespan%\.\.*}
opts_rrdfetch+=(-s "$start")
if [ "$start" != "${opt_timespan}" ]; then
end=${opt_timespan#*..}
opts_rrdfetch+=(-e "$end")
fi
fi
local resources=(c.memory c.network load_avg disk)
if [ -n "${opt_resource}" ]; then
resources=(${opt_resource//,/ })
fi
local not_found=()
for resource in "${resources[@]}"; do
if ! fn.exists "graph:def:$resource"; then
not_found+=("$resource")
fi
done
if [[ "${#not_found[@]}" -gt 0 ]]; then
not_found_msg=$(printf "%s, " "${not_found[@]}")
not_found_msg=${not_found_msg%, }
err "Unsupported resource(s) provided: ${not_found_msg}"
echo " resource must be one-of:" >&2
declare -F | egrep 'graph:def:[a-zA-Z_.]+$' | cut -f 3- -d " " | cut -f 3- -d ":" | prefix " - " >&2
return 1
fi
if [ "${#VPS[@]}" == 0 ]; then
err "You must provide a VPS list as positional arguments"
return 1
fi
include cache
if [ -z "$VAR_DIR" ]; then
err "Unset \$VAR_DIR, can't downlowd rrd graph"
return 1
fi
mkdir -p "$VAR_DIR/rrd"
if ! [ -d "$VAR_DIR/rrd" ]; then
err "Invalid \$VAR_DIR: '$VAR_DIR/rrd' is not a directory"
return 1
fi
(
for vps in "${VPS[@]}"; do
(
{
{
ssh:open "root@$vps" 2>/dev/null || {
err "Can't open connection $vps."
return 1
}
while true; do
if ssh:run "root@$vps" -- "[ -d '/var/lib/vps/rrd' ]"; then
echo "${WHITE}Collecting stats${NORMAL}..."
{
{
ssh:rsync "root@$vps:/var/lib/vps/rrd/" "${VAR_DIR}/rrd/${vps}"
} 3>&1 1>&2 2>&3 | prefix " ${DARKRED}\!${NORMAL} "
set_errlvl "${PIPESTATUS[0]}"
} 3>&1 1>&2 2>&3 | prefix " ${GRAY}|${NORMAL} "
echo " ${GRAY}..${NORMAL} ${DARKGREEN}done${NORMAL} collecting stats"
else
warn "No stats found. Did you run 'myc-update' on the vps ?."
fi
[ -z "$opt_follow" ] && break
echo "${WHITE}Sleeping ${DARKYELLOW}${opt_interval:-60}${NORMAL}s..."
sleep "${opt_interval:-60}"
echo " ${GRAY}..${NORMAL} ${DARKGREEN}done${NORMAL} sleeping"
done
} 3>&1 1>&2 2>&3 | prefix " ${DARKRED}\!${GRAY} collect(${DARKCYAN}$vps${GRAY})${NORMAL} "
set_errlvl "${PIPESTATUS[0]}"
} 3>&1 1>&2 2>&3 | prefix " ${GRAY}| collect(${DARKCYAN}$vps${GRAY})${NORMAL} " >&2
) &
done
wait
) &
collect_pid="$!"
if [ -z "$opt_follow" ]; then
echo "${WHITE}Fetching last stats${NORMAL}${GRAY}..${NORMAL}" >&2
wait
echo " ${GRAY}..${DARKGREEN} done${NORMAL} fetching stats" >&2
else
collect_end_msg=" ${GRAY}..${NORMAL} ${DARKGREEN}stop${NORMAL} collecting daemon (pid: ${DARKYELLOW}$collect_pid${NORMAL})"
trap_add EXIT \
"printf '%s\n' \"$collect_end_msg\" && kill $collect_pid"
echo "${WHITE}Start collecting daemon${NORMAL} (pid: ${DARKYELLOW}$collect_pid${NORMAL}) ${GRAY}..${NORMAL}" >&2
fi
( depends gnuplot ) || {
echo ""
echo " Gnuplot is required to display graphs..." \
"You might want to try to install ${WHITE}gnuplot${NORMAL} with:"
echo ""
echo " apt install gnuplot"
echo ""
return 1
} >&2
( depends rrdtool ) || {
echo ""
echo " Rrdtool is required to display graphs..." \
"You might want to try to install ${WHITE}rrdtool${NORMAL} with:"
echo ""
echo " apt install rrdtool"
echo ""
return 1
} >&2
export GNUTERM=qt
## rrdtool fetch will use comma for floating point depending on some locals !
export LC_ALL=C
exec {PFD}> >(exec gnuplot 2>/dev/null)
gnuplot_pid="$!"
if [ -z "$opt_follow" ]; then
echo "${WHITE}Draw gnuplot graph${GRAY}..${NORMAL}" >&2
else
gnuplot_end_msg=" ${GRAY}..${NORMAL} ${DARKGREEN}stop${NORMAL} gnuplot process (pid: $gnuplot_pid)"
trap_add EXIT \
"printf '%s\n' \"$gnuplot_end_msg\" && kill $gnuplot_pid"
echo "${WHITE}Start gnuplot process${NORMAL} (pid: $gnuplot_pid) ${GRAY}..${NORMAL}" >&2
fi
echo "set term qt noraise replotonresize" >&$PFD
while true; do
{
i=0
data_start_ts=
data_stop_ts=
for resource in "${resources[@]}"; do
for vps in "${VPS[@]}"; do
((i++))
{
{
rrd_vps_path="$VAR_DIR/rrd/$vps"
if [ -d "$rrd_vps_path" ]; then
graph:def:"${resource}" "$vps" "$i" "${opts_rrdfetch[@]}"
else
warn "No data yet, ignoring..."
fi
} 3>&1 1>&2 2>&3 | prefix " ${DARKRED}\!${GRAY} graph(${DARKCYAN}$vps${GRAY}:${WHITE}$resource${NORMAL})${NORMAL} "
set_errlvl "${PIPESTATUS[0]}"
} 3>&1 1>&2 2>&3 || continue 1
done
done
} >&$PFD
if [ -z "$opt_follow" ]; then
echo " ${GRAY}..${DARKGREEN} done${NORMAL} gnuplot graphing" >&2
break
else
{
echo "${WHITE}Sleeping ${DARKYELLOW}${opt_interval:-60}${NORMAL}s..."
sleep "${opt_interval:-60}"
echo " ${GRAY}..${NORMAL} ${DARKGREEN}done${NORMAL} sleeping"
} | prefix " ${GRAY}| gnuplot:${NORMAL} " >&2
fi
done
if [ -n "$opt_follow" ]; then
echo "Waiting for child process to finish.." >&2
wait
echo " ..done" >&2
else
echo "pause mouse close" >&$PFD
fi
}
graph:def:c.memory() {
local vps="$1" i="$2"
shift 2
local opts_rrdfetch=("$@")
local resource="memory"
rrd_vps_path="$VAR_DIR/rrd/$vps"
[ -d "$rrd_vps_path/containers" ] || {
warn "No containers data yet for vps '$vps'... Ignoring"
return 0
}
containers=(
$(
cd "$rrd_vps_path/containers";
find -maxdepth 3 -mindepth 3 -name "${resource}.rrd" -type f |
sed -r 's%^./([^/]+/[^/]+)/[^/]+.rrd$%\1%g'
)
)
gnuplot_line_config=(
"set term qt $i title \"$vps $resource\" replotonresize noraise"
"set title '$vps'"
"set xdata time"
"set timefmt '%s'"
"set ylabel '$resource Usage'"
"set format y '%s'"
"set ytics format ' %g'"
"set mouse mouseformat 6"
"set yrange [0:*] "
"set border behind"
)
printf "%s\n" "${gnuplot_line_config[@]}"
first=1
for container in "${containers[@]}"; do
rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/containers/$container/$resource.rrd\""
rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
rrdfetch_cmd+=" tail -n +2 | \\"$'\n'
rrdfetch_cmd+=" egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
first_ts=
first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ")
if [ -z "$first_ts" ]; then
warn "No data for $container on vps $vps, skipping..."
continue
fi
last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1 | cut -f 1 -d " ")
if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
data_start_ts="$first_ts"
fi
if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
data_stop_ts="$last_ts"
fi
if [ -n "$first" ]; then
first=
echo "plot \\"
else
echo ", \\"
fi
container="${container//\'/}"
container="${container//@/\\@}"
echo -n " ${rrdfetch_cmd} u 1:((\$3 - \$2)/1000000000) w lines title '${container//_/\\_}'"
done
echo
}
graph:def:c.network() {
local vps="$1" i="$2"
shift 2
local opts_rrdfetch=("$@")
local resource="network"
rrd_vps_path="$VAR_DIR/rrd/$vps"
[ -d "$rrd_vps_path/containers" ] || {
warn "No containers data yet for vps '$vps'... Ignoring"
return 0
}
containers=(
$(
cd "$rrd_vps_path/containers";
find -maxdepth 3 -mindepth 3 -name "${resource}.rrd" -type f |
sed -r 's%^./([^/]+/[^/]+)/[^/]+.rrd$%\1%g'
)
)
gnuplot_line_config=(
"set term qt $i title \"$vps $resource\" replotonresize noraise"
"set title '$vps'"
"set xdata time"
"set timefmt '%s'"
"set ylabel '$resource Usage'"
"set format y '%s'"
"set ytics format ' %.2f MiB/s'"
"set mouse mouseformat 6"
"set yrange [0:*] "
"set border behind"
)
printf "%s\n" "${gnuplot_line_config[@]}"
first=1
for container in "${containers[@]}"; do
rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/containers/$container/$resource.rrd\""
rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
rrdfetch_cmd+=" tail -n +2 | \\"$'\n'
rrdfetch_cmd+=" egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
first_ts=
first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ")
if [ -z "$first_ts" ]; then
warn "No data for $container on vps $vps, skipping..."
continue
fi
last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1 | cut -f 1 -d " ")
if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
data_start_ts="$first_ts"
fi
if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
data_stop_ts="$last_ts"
fi
if [ -n "$first" ]; then
first=
echo "plot \\"
else
echo ", \\"
fi
container="${container//\'/}"
container="${container//@/\\@}"
echo -n " ${rrdfetch_cmd} u 1:((\$3 / 1024) / 1024) w lines title '${container//_/\\_}'"
done
echo
}
graph:def:load_avg() {
local vps="$1" i="$2"
shift 2
local opts_rrdfetch=("$@")
rrd_vps_path="$VAR_DIR/rrd/$vps"
[ -f "$rrd_vps_path/$resource.rrd" ] || {
warn "No containers data yet for vps '$vps'... Ignoring"
return 0
}
gnuplot_line_config=(
"set term qt $i title \"$vps $resource\" replotonresize noraise"
"set title '$vps'"
"set xdata time"
"set timefmt '%s'"
"set ylabel '${resource//_/\\_} Usage'"
"set format y '%s'"
"set ytics format '%g'"
"set mouse mouseformat 6"
"set yrange [0:*] "
"set border behind"
)
printf "%s\n" "${gnuplot_line_config[@]}"
first=1
for value in 1m:2 5m:3 15m:4; do
label="${value%:*}"
col_num="${value#*:}"
rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/$resource.rrd\""
rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
rrdfetch_cmd+=" tail -n +2 | \\"$'\n'
rrdfetch_cmd+=" egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
first_ts=
first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ")
if [ -z "$first_ts" ]; then
warn "No data for $resource on vps $vps, skipping..."
continue
fi
last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1 | cut -f 1 -d " ")
if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
data_start_ts="$first_ts"
fi
if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
data_stop_ts="$last_ts"
fi
if [ -n "$first" ]; then
first=
echo "plot \\"
else
echo ", \\"
fi
container="${container//\'/}"
container="${container//@/\\@}"
echo -n " ${rrdfetch_cmd} u 1:$col_num w lines title '${label}'"
done
echo
}
graph:def:disk() {
local vps="$1" i="$2"
shift 2
local opts_rrdfetch=("$@")
rrd_vps_path="$VAR_DIR/rrd/$vps"
[ -f "$rrd_vps_path/$resource.rrd" ] || {
warn "No containers data yet for vps '$vps'... Ignoring"
return 0
}
gnuplot_line_config=(
"set term qt $i title \"$vps $resource\" replotonresize noraise"
"set title '$vps'"
"set xdata time"
"set timefmt '%s'"
"set ylabel '${resource//_/\\_} Usage'"
"set format y '%s'"
"set ytics format '%g GiB'"
"set mouse mouseformat 6"
"set yrange [0:*] "
"set border behind"
)
printf "%s\n" "${gnuplot_line_config[@]}"
first=1
for value in used:2 size:3; do
label="${value%:*}"
col_num="${value#*:}"
rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/$resource.rrd\""
rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
rrdfetch_cmd+=" tail -n +2 | \\"$'\n'
rrdfetch_cmd+=" egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
first_ts=
first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ")
if [ -z "$first_ts" ]; then
warn "No data for $resource on vps $vps, skipping..."
continue
fi
last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1 | cut -f 1 -d " ")
if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
data_start_ts="$first_ts"
fi
if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
data_stop_ts="$last_ts"
fi
if [ -n "$first" ]; then
first=
echo "plot \\"
else
echo ", \\"
fi
container="${container//\'/}"
container="${container//@/\\@}"
echo -n " ${rrdfetch_cmd} u 1:(\$${col_num}/(1024*1024)) w lines title '${label}'"
done
echo
}
cmdline.spec.gnu vps-subscribe
cmdline.spec::cmd:vps-subscribe:run() {
:
}
cmdline.spec.gnu add
cmdline.spec:vps-subscribe:cmd:add:run() {
: :posarg: CHANNEL 'Channel which will be sent to given topic'
: :posarg: TOPIC 'Ntfy topic to recieve messages of given channel
(format: "[MYSERVER:]MYTOPICS"
Examples: "ntfy.0k.io:main,storage,alerts",
"main{1,3,7}"
)'
: :posarg: [VPS...] 'Target host(s) to get stats'
printf "%s\0" "$CHANNEL" "$TOPIC" |
vps_mux subscribe:add "${VPS[@]}"
}
cmdline.spec.gnu rm
cmdline.spec:vps-subscribe:cmd:rm:run() {
: :posarg: CHANNEL 'Channel which will be sent to given topic'
: :posarg: TOPIC 'Ntfy topic to recieve messages of given channel
(format: "[MYSERVER:]MYTOPICS"
Examples: "ntfy.0k.io:main,storage,alerts",
"main{1,3,7}"
)'
: :posarg: [VPS...] 'Target host(s) to get stats'
printf "%s\0" "$CHANNEL" "$TOPIC" |
vps_mux subscribe:rm "${VPS[@]}"
}
cmdline::parse "$@"