You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2041 lines
69 KiB
2041 lines
69 KiB
#!/bin/bash
|
|
|
|
|
|
. /etc/shlib
|
|
|
|
include common
|
|
include parse
|
|
include cmdline
|
|
include config
|
|
|
|
|
|
[[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true
|
|
|
|
version=0.1
|
|
desc='Manage 0k related installs'
|
|
help=""
|
|
|
|
|
|
##
|
|
## Functions
|
|
##
|
|
|
|
is-port-open() {
|
|
local host="$1" port="$2" timeout=5
|
|
start="$SECONDS"
|
|
debug "Testing if $host's port $2 is open ..."
|
|
while true; do
|
|
timeout 1 bash -c "</dev/tcp/${host}/${port}" >/dev/null 2>&1 && break
|
|
sleep 0.2
|
|
if [ "$((SECONDS - start))" -gt "$timeout" ]; then
|
|
return 1
|
|
fi
|
|
done
|
|
debug " .. $host's port $2 is open."
|
|
return 0
|
|
}
|
|
|
|
resolve() {
|
|
local ent hostname="$1"
|
|
debug "Resolving $1 ..."
|
|
if ent=$(getent ahosts "$hostname"); then
|
|
ent=$(echo "$ent" | egrep ^"[0-9]+.[0-9]+.[0-9]+.[0-9]+\s+" | \
|
|
head -n 1 | awk '{ print $1 }')
|
|
debug " .. resolved $1 to $ent."
|
|
echo "$ent"
|
|
else
|
|
debug " .. couldn't resolve $1."
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
|
|
set_errlvl() { return "${1:-1}"; }
|
|
|
|
export master_pid=$$
|
|
ssh:open() {
|
|
local hostname ssh_cmd ssh_options
|
|
ssh_cmd=(ssh)
|
|
ssh_options=()
|
|
while [ "$#" != 0 ]; do
|
|
case "$1" in
|
|
"--stdin-password")
|
|
ssh_cmd=(sshpass "${ssh_cmd[@]}")
|
|
;;
|
|
-o)
|
|
ssh_options+=("$1" "$2")
|
|
shift
|
|
;;
|
|
*)
|
|
[ -z "$hostname" ] && hostname="$1" || {
|
|
err "Surnumerous positional argument '$1'. Expecting only hostname."
|
|
return 1
|
|
}
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
full_cmd=(
|
|
"${ssh_cmd[@]}"
|
|
-o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
|
|
-o ControlMaster=auto -o ControlPersist=900 \
|
|
-o ConnectTimeout=5 -o StrictHostKeyChecking=no \
|
|
"${ssh_options[@]}" \
|
|
"$hostname" "$@" -- true)
|
|
"${full_cmd[@]}" >/dev/null 2>&1 || {
|
|
err "Failed: ${full_cmd[*]}"
|
|
return 1
|
|
}
|
|
trap_add EXIT,INT 'ssh:quit "$hostname"'
|
|
}
|
|
|
|
ssh:rsync() {
|
|
local src="$1" dst="$2"
|
|
hostname=${src%%:*}
|
|
hostname=${hostname#*@}
|
|
local rsync_ssh_options=(
|
|
-o ControlPath="/tmp/ssh-control-master-${master_pid}-$hostname"
|
|
-o ControlMaster=auto
|
|
-o ControlPersist=900
|
|
-o ConnectTimeout=10
|
|
-o StrictHostKeyChecking=no
|
|
)
|
|
if ! ssh:run "root@$hostname" -- type -p rsync </dev/null >/dev/null; then
|
|
info "No 'rsync' available on '$hostname'. Requesting installation..."
|
|
ssh:run "root@$hostname" -- apt-get install rsync -y || {
|
|
err "Installation of 'rsync' failed on '$hostname'"
|
|
return 1
|
|
}
|
|
fi
|
|
|
|
local cmd=(
|
|
rsync -e "ssh ${rsync_ssh_options[*]}"
|
|
-azvArH --delete --delete-excluded
|
|
--partial --partial-dir .rsync-partial
|
|
"$src" "$dst"
|
|
)
|
|
"${cmd[@]}"
|
|
}
|
|
|
|
ssh:open-try() {
|
|
local opts hostnames
|
|
opts=()
|
|
hostnames=()
|
|
while [ "$#" != 0 ]; do
|
|
case "$1" in
|
|
-o)
|
|
opts+=("$1" "$2")
|
|
shift
|
|
;;
|
|
*)
|
|
hostnames+=("$1")
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
password=''
|
|
for host in "${hostnames[@]}"; do
|
|
debug "Trying $host with publickey."
|
|
ssh:open -o PreferredAuthentications=publickey \
|
|
"${opts[@]}" \
|
|
"$host" >/dev/null 2>&1 && {
|
|
echo "$host"$'\n'"$password"$'\n'
|
|
return 0
|
|
}
|
|
debug " .. failed connecting to $host with publickey."
|
|
done
|
|
local times=0 password
|
|
while [ "$((++times))" -le 3 ]; do
|
|
read -sp "$HOST's password: " password
|
|
errlvl="$?"
|
|
echo >&2
|
|
if [ "$errlvl" -gt 0 ]; then
|
|
exit 1
|
|
fi
|
|
for host in "${hostnames[@]}"; do
|
|
debug "Trying $host with password ($times/3)"
|
|
echo "$password" | ssh:open -o PreferredAuthentications=password \
|
|
--stdin-password \
|
|
"${opts[@]}" \
|
|
"$host" >/dev/null 2>&1 && {
|
|
echo "$host"$'\n'"$password"$'\n'
|
|
return 0
|
|
}
|
|
debug " .. failed connecting to $host with password."
|
|
done
|
|
err "login failed. Try again... ($((times+1))/3)"
|
|
done
|
|
return 1
|
|
}
|
|
|
|
rscolcat:check-install() {
|
|
type -p rscolcat >/dev/null || {
|
|
err "'rscolcat' is not installed. Please install it."
|
|
echo " sudo wget https://docker.0k.io/downloads/rscolcat-0.1.0 -O /usr/local/bin/rscolcat && " >&2
|
|
echo " sudo chmod +x /usr/local/bin/rscolcat" >&2
|
|
return 1
|
|
}
|
|
}
|
|
|
|
ssh:run() {
|
|
local hostname="$1" ssh_options cmd
|
|
shift
|
|
|
|
ssh_options=()
|
|
cmd=()
|
|
while [ "$#" != 0 ]; do
|
|
case "$1" in
|
|
"--")
|
|
shift
|
|
cmd+=("$@")
|
|
break
|
|
;;
|
|
*)
|
|
ssh_options+=("$1")
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
## XXXvlab: keeping in case we need some debug
|
|
# debug "$DARKCYAN$hostname$NORMAL $WHITE\$$NORMAL" "$@"
|
|
# debug "Running cmd: ${cmd[*]}"
|
|
# for arg in "${cmd[@]}"; do
|
|
# debug "$arg"
|
|
# done
|
|
{
|
|
{
|
|
ssh -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
|
|
-o ControlMaster=auto -o ControlPersist=900 \
|
|
-o "StrictHostKeyChecking=no" \
|
|
"${ssh_options[@]}" "$hostname" -- "${cmd[@]}"
|
|
} 3>&1 1>&2 2>&3 ## | sed -r "s/^/$DARKCYAN$hostname$NORMAL $DARKRED\!$NORMAL /g"
|
|
set_errlvl "${PIPESTATUS[0]}"
|
|
} 3>&1 1>&2 2>&3
|
|
}
|
|
|
|
ssh:quit() {
|
|
local hostname="$1"
|
|
shift
|
|
ssh -o ControlPath=/tmp/ssh-control-master-${master_pid} \
|
|
-o ControlMaster=auto -o ControlPersist=900 -O exit \
|
|
"$hostname" 2>/dev/null
|
|
}
|
|
|
|
|
|
is_ovh_domain_name() {
|
|
local domain="$1"
|
|
|
|
[[ "$domain" == *.ovh.net ]] && return 0
|
|
[[ "$domain" == "ns"*".ip-"*".eu" ]] && return 0
|
|
return 1
|
|
}
|
|
|
|
is_ovh_hostname() {
|
|
local domain="$1"
|
|
|
|
[[ "$domain" =~ ^vps-[0-9a-f]*$ ]] && return 0
|
|
[[ "$domain" =~ ^vps[0-9]*$ ]] && return 0
|
|
return 1
|
|
}
|
|
|
|
vps_connection_check() {
|
|
local vps="$1"
|
|
ip=$(resolve "$vps") ||
|
|
{ echo "${DARKRED}no-resolve${NORMAL}"; return 1; }
|
|
|
|
is-port-open "$ip" "22" </dev/null ||
|
|
{ echo "${DARKRED}no-port-22-open${NORMAL}"; return 1; }
|
|
|
|
ssh:open -o ConnectTimeout=10 -o PreferredAuthentications=publickey \
|
|
"root@$vps" >/dev/null 2>&1 ||
|
|
{ echo "${DARKRED}no-ssh-root-access${NORMAL}"; return 1; }
|
|
}
|
|
|
|
vps_check() {
|
|
local vps="$1"
|
|
vps_connection_check "$vps" </dev/null || return 1
|
|
if size=$(
|
|
echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 5 -d ' ' | cut -f 1 -d %" |
|
|
ssh:run "root@$vps" -- bash); then
|
|
if [ "$size" -gt "90" ]; then
|
|
echo "${DARKRED}above-90%-disk-usage${NORMAL}"
|
|
elif [ "$size" -gt "75" ]; then
|
|
echo "${DARKYELLOW}above-75%-disk-usage${NORMAL}"
|
|
fi
|
|
else
|
|
echo "${DARKRED}no-size${NORMAL}"
|
|
fi </dev/null
|
|
compose_content=$(ssh:run "root@$vps" -- cat /opt/apps/myc-deploy/compose.yml </dev/null) ||
|
|
{ echo "${DARKRED}no-compose${NORMAL}"; return 1; }
|
|
echo "$compose_content" | yq -e ".rsync-backup" >/dev/null 2>&1 ||
|
|
{ echo "${DARKRED}no-backup${NORMAL}"; return 1; }
|
|
}
|
|
|
|
|
|
|
|
backup:setup-rsync() {
|
|
local admin="$1" vps="$2" server="$3" id="$4"
|
|
|
|
[ -z "${BACKUP_SSH_SERVER}" ] || {
|
|
err "Unexpected error: '\$BACKUP_SSH_SERVER' is already set in '$FUNCNAME'."
|
|
return 1
|
|
}
|
|
|
|
BACKUP_SSH_OPTIONS=(-o StrictHostKeyChecking=no)
|
|
if [[ "$server" == *":"* ]]; then
|
|
BACKUP_SSH_OPTIONS+=(-p "${server#*:}")
|
|
BACKUP_SSH_SERVER=${server%%:*}
|
|
else
|
|
BACKUP_SSH_SERVER="$server"
|
|
fi
|
|
|
|
if ! private_key=$(ssh "${BACKUP_SSH_OPTIONS[@]}" \
|
|
"$admin"@"${BACKUP_SSH_SERVER}" request-recovery-key "$id"); then
|
|
err "Couldn't request a recovery key for '$id' with account '$admin'."
|
|
return 1
|
|
fi
|
|
|
|
if ! VPS_TMP_DIR=$(echo "mktemp -d" | ssh:run "root@$vps" -- bash); then
|
|
err "Couldn't create a temporary directory on vps"
|
|
return 1
|
|
fi
|
|
|
|
cat <<EOF | ssh:run "root@$vps" -- bash || return 1
|
|
touch "$VPS_TMP_DIR/recover_key" &&
|
|
chmod go-rwx "$VPS_TMP_DIR/recover_key" &&
|
|
printf "%s\n" "$private_key" >> "$VPS_TMP_DIR/recover_key"
|
|
EOF
|
|
|
|
BACKUP_SSH_OPTIONS+=(-i "$VPS_TMP_DIR/recover_key" -l rsync)
|
|
BACKUP_VPS_TARGET="$vps"
|
|
BACKUP_IDENT="$id"
|
|
|
|
echo "type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null" |
|
|
ssh:run "root@$vps" -- bash || return 1
|
|
|
|
}
|
|
|
|
|
|
backup:rsync() {
|
|
local ssh_options
|
|
|
|
[ -n "${BACKUP_SSH_SERVER}" ] || {
|
|
err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
|
|
return 1
|
|
}
|
|
|
|
rsync_options=()
|
|
while [[ "$1" == "-"* ]]; do
|
|
rsync_options+=("$1")
|
|
shift
|
|
done
|
|
local src="$1" dst="$2"
|
|
|
|
cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash
|
|
rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
|
|
-azvArH --delete --delete-excluded \
|
|
--partial --partial-dir .rsync-partial \
|
|
--numeric-ids ${rsync_options[*]} \
|
|
"${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "${dst}"
|
|
EOF
|
|
|
|
}
|
|
|
|
|
|
backup:path_exists() {
|
|
local src="$1"
|
|
|
|
[ -n "${BACKUP_SSH_SERVER}" ] || {
|
|
err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
|
|
return 1
|
|
}
|
|
|
|
cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash >/dev/null 2>&1
|
|
rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
|
|
-nazvArH --numeric-ids \
|
|
"${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "/tmp/dummy"
|
|
EOF
|
|
|
|
}
|
|
|
|
file:vps_backup_recover() {
|
|
local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"
|
|
|
|
backup:rsync "${path}" "${vps_path}" || return 1
|
|
|
|
if [[ "$path" == *"/" ]]; then
|
|
if [ "$path" == "$vps_path"/ ]; then
|
|
msg_target="Directory '$path'"
|
|
else
|
|
msg_target="Directory '$path' -> '$vps_path'"
|
|
fi
|
|
else
|
|
if [ "$path" == "$vps_path" ]; then
|
|
msg_target="File '$path'"
|
|
else
|
|
msg_target="File '$path' -> '$vps_path'"
|
|
fi
|
|
fi
|
|
|
|
info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."
|
|
|
|
}
|
|
|
|
mailcow:vps_backup_recover() {
|
|
local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"
|
|
|
|
|
|
if ! compose_yml_files=$(cat <<EOF | ssh:run "root@$vps" -- bash
|
|
urn=com.docker.compose.project
|
|
docker ps -f "label=\$urn=mailcowdockerized" \
|
|
--format="{{.Label \"\$urn.working_dir\"}}/{{.Label \"\$urn.config_files\"}}" |
|
|
uniq
|
|
EOF
|
|
); then
|
|
err "Couldn't get list of running projects"
|
|
return 1
|
|
fi
|
|
|
|
stopped_containers=
|
|
if [ -n "$compose_yml_files" ]; then
|
|
echo "Found running mailcowdockerized containers" >&2
|
|
if [[ "$compose_yml_files" == *$'\n'* ]]; then
|
|
err "Running containers are confusing, did not find only one mailcowdockerized project."
|
|
return 1
|
|
fi
|
|
if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
|
|
## For some reason, sometimes $urn.config_files holds an absolute path
|
|
compose_yml_files=/${compose_yml_files#*//}
|
|
if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
|
|
err "Running containers are confusing, they don't point to an existing docker-compose.yml."
|
|
return 1
|
|
fi
|
|
fi
|
|
echo "Containers where launched from '$compose_yml_files'" >&2
|
|
COMPOSE_FILE="$compose_yml_files"
|
|
ENV_FILE="${COMPOSE_FILE%/*}/.env"
|
|
if ! echo "[ -e \"${ENV_FILE}\" ]" | ssh:run "root@$vps" -- bash ; then
|
|
err "Running containers are confusing, docker-compose.yml has no '.env' next to it."
|
|
return 1
|
|
fi
|
|
echo "${WHITE}Bringing mailcowdockerized down${NORMAL}"
|
|
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
|
|
ssh:run "root@$vps" -- bash
|
|
stopped_containers=1
|
|
fi
|
|
|
|
if [[ "$path" == "/"* ]]; then
|
|
|
|
##
|
|
## Additional intelligence to simple file copy
|
|
##
|
|
|
|
if [[ "$path" == "/var/lib/docker/volumes/mailcowdockerized_*-vol-1/_data"* ]]; then
|
|
volume_name=${path#/var/lib/docker/volumes/}
|
|
volume_name=${volume_name%%/*}
|
|
volume_dir=${path%%"$volume_name"*}
|
|
|
|
## Create volumes if not existent
|
|
if ! ssh:run "root@$vps" -- "
|
|
[ -d '${volume_dir}' ] ||
|
|
docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
|
|
[ -d '${volume_dir}' ]
|
|
"; then
|
|
err "Couldn't find nor create '${volume_dir}'."
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
echo "${WHITE}Sync from backup ${path} to VPS ${vps_path}${NORMAL}" >&2
|
|
backup:rsync "${path}" "${vps_path}" || return 1
|
|
|
|
if [[ "$path" == *"/" ]]; then
|
|
if [ "$path" == "$vps_path"/ ]; then
|
|
msg_target="Directory '$path'"
|
|
else
|
|
msg_target="Directory '$path' -> '$vps_path'"
|
|
fi
|
|
else
|
|
if [ "$path" == "$vps_path" ]; then
|
|
msg_target="File '$path'"
|
|
else
|
|
msg_target="File '$path' -> '$vps_path'"
|
|
fi
|
|
fi
|
|
else
|
|
ALL_TARGETS=(mailcow postfix rspamd redis crypt vmail{,-attachments} mysql)
|
|
|
|
if [[ -n "$path" ]]; then
|
|
targets=()
|
|
bad_targets=()
|
|
for target in ${path//,/ }; do
|
|
if [[ " ${ALL_TARGETS[*]} " != *" $target "* ]]; then
|
|
bad_targets+=("$target")
|
|
fi
|
|
targets+=("$target")
|
|
done
|
|
if [ "${#bad_targets[@]}" -gt 0 ]; then
|
|
bad_target_msg=$(printf "%s, " "${bad_targets[@]}")
|
|
err "Unknown components: ${bad_target_msg%, }. These are allowed components:"
|
|
printf " - %s\n" "${ALL_TARGETS[@]}" >&2
|
|
return 1
|
|
fi
|
|
msg_target="Partial mailcow backup"
|
|
else
|
|
targets=("${ALL_TARGETS[@]}")
|
|
msg_target="Full mailcow backup"
|
|
fi
|
|
|
|
|
|
for target in "${targets[@]}"; do
|
|
case "$target" in
|
|
postfix|rspamd|redis|crypt|vmail|vmail-attachments)
|
|
|
|
volume_name="mailcowdockerized_${target}-vol-1"
|
|
volume_dir="/var/lib/docker/volumes/${volume_name}/_data"
|
|
if ! backup:path_exists "${volume_dir}/"; then
|
|
warn "No '$volume_name' in backup. This might be expected."
|
|
continue
|
|
fi
|
|
## Create volumes if not existent
|
|
if ! ssh:run "root@$vps" -- "
|
|
[ -d '${volume_dir}' ] ||
|
|
docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
|
|
[ -d '${volume_dir}' ]
|
|
"; then
|
|
err "Couldn't find nor create '${volume_dir}'."
|
|
return 1
|
|
fi
|
|
|
|
echo "${WHITE}Downloading of $volume_name${NORMAL}"
|
|
backup:rsync "${volume_dir}/" "${volume_dir}" || return 1
|
|
;;
|
|
mailcow)
|
|
## Mailcow git base
|
|
COMPOSE_FILE=
|
|
for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
|
|
backup:path_exists "${mailcow_dir}/" || continue
|
|
## this possibly change last value
|
|
COMPOSE_FILE="$mailcow_dir/docker-compose.yml"
|
|
ENV_FILE="$mailcow_dir/.env"
|
|
echo "${WHITE}Download of $mailcow_dir${NORMAL}"
|
|
backup:rsync "${mailcow_dir}"/ "${mailcow_dir}" || return 1
|
|
break
|
|
done
|
|
if [ -z "$COMPOSE_FILE" ]; then
|
|
err "Can't find mailcow base installation path in backup."
|
|
return 1
|
|
fi
|
|
|
|
;;
|
|
mysql)
|
|
if [ -z "$COMPOSE_FILE" ]; then
|
|
## Mailcow git base
|
|
compose_files=()
|
|
for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
|
|
ssh:run "root@$vps" -- "[ -e \"$mailcow_dir/docker-compose.yml\" ]" || continue
|
|
## this possibly change last value
|
|
compose_files+=("$mailcow_dir/docker-compose.yml")
|
|
done
|
|
if [ "${#compose_files[@]}" == 0 ]; then
|
|
err "No compose file found for mailcow installation."
|
|
return 1
|
|
elif [ "${#compose_files[@]}" -gt 1 ]; then
|
|
err "Multiple compose files for mailcow found:"
|
|
for f in "${compose_files[@]}"; do
|
|
echo " - $f" >&2
|
|
done
|
|
echo "Can't decide which to use for mounting mysql container." >&2
|
|
return 1
|
|
fi
|
|
COMPOSE_FILE="${compose_files[0]}"
|
|
ENV_FILE="${COMPOSE_FILE%/*}/.env"
|
|
if ! ssh:run "root@$vps" -- "[ -e \"${COMPOSE_FILE%/*}/.env\" ]"; then
|
|
err "No env file in '$ENV_FILE' found."
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
## Mysql database
|
|
echo "${WHITE}Downloading last backup of mysql backups${NORMAL}"
|
|
backup:rsync "/var/backups/mysql/" "/var/backups/mysql" || return 1
|
|
|
|
if ! env_content=$(echo "cat '$ENV_FILE'" | ssh:run "root@$vps" -- bash); then
|
|
err "Can't access env file: '$ENV_FILE'."
|
|
return 1
|
|
fi
|
|
|
|
root_password=$(printf "%s\n" "$env_content" | grep ^DBROOT= | cut -f 2 -d =)
|
|
|
|
echo "${WHITE}Bringing mysql-mailcow up${NORMAL}"
|
|
if ! image=$(cat <<EOF | ssh:run "root@$vps" -- bash
|
|
shyaml get-value services.mysql-mailcow.image < "${COMPOSE_FILE}"
|
|
EOF
|
|
); then
|
|
err "Failed to get image name of service 'mysql-mailcow' in 'compose.yml'."
|
|
return 1
|
|
fi
|
|
if [ -z "$(ssh:run "root@$vps" -- docker images -q "$image")" ]; then
|
|
info "Image '$image' not available, pull it."
|
|
if ! ssh:run "root@$vps" -- \
|
|
docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
|
|
pull mysql-mailcow; then
|
|
|
|
err "Failed to pull image of service 'mysql-mailcow'."
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
if ! container_id=$(cat <<EOF | ssh:run "root@$vps" -- bash
|
|
echo "[client]
|
|
password=$root_password" > "$VPS_TMP_DIR/my.cnf"
|
|
|
|
docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
|
|
run -d \
|
|
-v "$VPS_TMP_DIR/my.cnf:/root/.my.cnf:ro" \
|
|
mysql-mailcow
|
|
|
|
EOF
|
|
); then
|
|
err "Failed to bring up mysql-mailcow"
|
|
return 1
|
|
fi
|
|
|
|
START="$SECONDS"
|
|
retries=0
|
|
timeout=600
|
|
while true; do
|
|
((retries++))
|
|
echo " waiting for mysql db..." \
|
|
"(retry $retries, $(($SECONDS - $START))s elapsed, timeout is ${timeout}s)" >&2
|
|
cat <<EOF | ssh:run "root@$vps" -- bash && break
|
|
echo "SELECT 1;" | docker exec -i "$container_id" mysql >/dev/null 2>&1
|
|
EOF
|
|
if (($SECONDS - $START > $timeout)); then
|
|
err "Failed to connect to mysql-mailcow."
|
|
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
|
|
ssh:run "root@$vps" -- bash
|
|
return 1
|
|
fi
|
|
sleep 0.4
|
|
done
|
|
|
|
DBUSER=$(printf "%s\n" "$env_content" | grep ^DBUSER= | cut -f 2 -d =)
|
|
DBPASS=$(printf "%s\n" "$env_content" | grep ^DBPASS= | cut -f 2 -d =)
|
|
|
|
echo "${WHITE}Uploading mysql dump${NORMAL}"
|
|
cat <<EOF | ssh:run "root@$vps" -- bash
|
|
|
|
echo "
|
|
DROP DATABASE IF EXISTS mailcow;
|
|
CREATE DATABASE mailcow;
|
|
GRANT ALL PRIVILEGES ON mailcow.* TO '$DBUSER'@'%' IDENTIFIED BY '$DBPASS';
|
|
" | docker exec -i "$container_id" mysql
|
|
|
|
zcat /var/backups/mysql/mailcow/*.gz | docker exec -i "$container_id" mysql mailcow
|
|
|
|
EOF
|
|
if [ "$?" != 0 ]; then
|
|
err "Failed to load mysql dump."
|
|
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
|
|
ssh:run "root@$vps" -- bash
|
|
return 1
|
|
fi
|
|
|
|
echo "${WHITE}Bringing mysql-mailcow down${NORMAL}"
|
|
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
|
|
ssh:run "root@$vps" -- bash
|
|
;;
|
|
*)
|
|
err "Unknown component '$target'. Bailing out."
|
|
return 1
|
|
esac
|
|
done
|
|
|
|
ssh:run "root@$vps" -- "rm -rf '$VPS_TMP_DIR'"
|
|
fi
|
|
|
|
if [ -n "$stopped_containers" ]; then
|
|
echo "${WHITE}Starting mailcow${NORMAL}" >&2
|
|
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" up -d" |
|
|
ssh:run "root@$vps" -- bash
|
|
fi
|
|
info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."
|
|
|
|
}
|
|
|
|
NTFY_TOPIC_FILE="/etc/ntfy/topics.yml"
|
|
NTFY_CONFIG_FILE="/etc/ntfy/ntfy.conf"
|
|
subscribe:ntfy:topic-file-exists() {
|
|
local vps="$1"
|
|
if ! out=$(echo "[ -f \"$NTFY_TOPIC_FILE\" ] && echo ok || true" | \
|
|
ssh:run "root@$vps" -- bash); then
|
|
err "Unable to check for existence of '$NTFY_TOPIC_FILE'."
|
|
fi
|
|
if [ -z "$out" ]; then
|
|
err "File '$NTFY_TOPIC_FILE' not found on $vps."
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
subscribe:ntfy:config-file-exists() {
|
|
local vps="$1"
|
|
if ! out=$(echo "[ -f \"$NTFY_CONFIG_FILE\" ] && echo ok || true" | \
|
|
ssh:run "root@$vps" -- bash); then
|
|
err "Unable to check for existence of '$NTFY_CONFIG_FILE'."
|
|
fi
|
|
if [ -z "$out" ]; then
|
|
err "File '$NTFY_CONFIG_FILE' not found on $vps."
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
|
|
ntfy:rm() {
|
|
local channel="$1" topic="$2" vps="$3"
|
|
subscribe:ntfy:topic-file-exists "$vps" || return 1
|
|
if ! out=$(echo "yq -i 'del(.[\"$channel\"][] | select(. == \"$TOPIC\"))' \"$NTFY_TOPIC_FILE\"" | \
|
|
ssh:run "root@$vps" -- bash); then
|
|
err "Failed to remove channel '$channel' from '$NTFY_TOPIC_FILE'."
|
|
return 1
|
|
fi
|
|
info "Channel '$channel' removed from '$NTFY_TOPIC_FILE' on $vps."
|
|
ssh:run "root@$vps" -- cat "$NTFY_TOPIC_FILE"
|
|
}
|
|
|
|
ntfy:add() {
|
|
local channel="$1" topic="$2" vps="$3"
|
|
vps_connection_check "$vps" </dev/null || return 1
|
|
subscribe:ntfy:topic-file-exists "$vps" || return 1
|
|
if ! out=$(echo "yq '. | has(\"$channel\")' \"$NTFY_TOPIC_FILE\"" | \
|
|
ssh:run "root@$vps" -- bash); then
|
|
err "Failed to check if channel '$channel' with topic '$topic' is already in '$NTFY_TOPIC_FILE'."
|
|
return 1
|
|
fi
|
|
if [ "$out" != "true" ]; then
|
|
## Channel does not exist
|
|
if ! out=$(echo "yq -i '.[\"$channel\"] = []' \"$NTFY_TOPIC_FILE\"" | \
|
|
ssh:run "root@$vps" -- bash); then
|
|
err "Failed to create a new channel '$channel' entry in '$NTFY_TOPIC_FILE'."
|
|
return 1
|
|
fi
|
|
else
|
|
## Channel exists
|
|
if ! out=$(echo "yq '.[\"$channel\"] | any_c(. == \"$topic\")' \"$NTFY_TOPIC_FILE\"" | \
|
|
ssh:run "root@$vps" -- bash); then
|
|
err "Failed to check if channel '$channel' with topic '$topic' is already in '$NTFY_TOPIC_FILE'."
|
|
return 1
|
|
fi
|
|
if [ "$out" == "true" ]; then
|
|
info "Channel '$channel' with topic '$topic' already exists in '$NTFY_TOPIC_FILE'."
|
|
return 0
|
|
fi
|
|
fi
|
|
if ! out=$(echo "yq -i '.[\"$channel\"] += [\"$topic\"]' \"$NTFY_TOPIC_FILE\"" | \
|
|
ssh:run "root@$vps" -- bash); then
|
|
err "Failed to add channel '$channel' with topic '$topic' to '$NTFY_TOPIC_FILE'."
|
|
return 1
|
|
fi
|
|
info "Channel '$channel' added with topic '$topic' to '$NTFY_TOPIC_FILE' on $vps."
|
|
}
|
|
|
|
NTFY_BROKER_SERVER="ntfy.0k.io"
|
|
ntfy:topic-access() {
|
|
local action="$1" topic="$2" vps="$3"
|
|
subscribe:ntfy:config-file-exists "$vps" || return 1
|
|
|
|
local user
|
|
user=$(ntfy:get-login "$vps") || return 1
|
|
case "$action" in
|
|
"write")
|
|
ssh "ntfy@$NTFY_BROKER_SERVER" "topic-access" \
|
|
"$user" "$topic" "write-only" </dev/null || {
|
|
err "Failed to grant write access to '$user' for topic '$topic'."
|
|
return 1
|
|
}
|
|
info "Granted write access for '$user' to topic '$topic'."
|
|
;;
|
|
"remove")
|
|
ssh "ntfy@$NTFY_BROKER_SERVER" "topic-access" -r "$user" "$topic" </dev/null || {
|
|
err "Failed to reset access of '$user' for topic '$topic'."
|
|
return 1
|
|
}
|
|
info "Access for '$user' to topic '$topic' was resetted successfully."
|
|
;;
|
|
*)
|
|
err "Invalid action '$action'."
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
ntfy:get-login() {
|
|
local vps="$1"
|
|
if ! out=$(echo ". \"$NTFY_CONFIG_FILE\" && echo \"\$LOGIN\"" | \
|
|
ssh:run "root@$vps" -- bash); then
|
|
err "Failed to get ntfy login from '$NTFY_CONFIG_FILE'."
|
|
return 1
|
|
fi
|
|
if [ -z "$out" ]; then
|
|
err "Unexpected empty login retrieved from sourcing '$NTFY_CONFIG_FILE'."
|
|
return 1
|
|
fi
|
|
echo "$out"
|
|
}
|
|
|
|
subscribe:add() {
|
|
local vps="$1"
|
|
read-0 channel topic || {
|
|
err "Couldn't read CHANNEL and TOPIC arguments."
|
|
return 1
|
|
}
|
|
vps_connection_check "$vps" </dev/null || return 1
|
|
ntfy:topic-access "write" "$topic" "$vps" </dev/null || return 1
|
|
ntfy:add "$channel" "$topic" "$vps" || {
|
|
err "Failed to add channel '$channel' with topic '$topic' to '$NTFY_TOPIC_FILE'."
|
|
echo " Removing topic access." >&2
|
|
ntfy:topic-access "remove" "$topic" "$vps" </dev/null
|
|
return 1
|
|
}
|
|
}
|
|
|
|
subscribe:rm() {
|
|
local vps="$1"
|
|
read-0 channel topic || {
|
|
err "Couldn't read CHANNEL and TOPIC arguments."
|
|
return 1
|
|
}
|
|
vps_connection_check "$vps" </dev/null || return 1
|
|
|
|
ntfy:rm "$channel" "$topic" "$vps" || return 1
|
|
ntfy:topic-access "remove" "$topic" "$vps" </dev/null || {
|
|
err "Failed to remove topic access for '$topic' on '$vps'."
|
|
return 1
|
|
}
|
|
}
|
|
|
|
vps_backup_recover() {
|
|
local vps="$1" admin server id path rtype force type
|
|
|
|
read-0 admin server id path rtype force
|
|
|
|
if [[ "$vps" == *":"* ]]; then
|
|
vps_path=${vps#*:}
|
|
vps=${vps%%:*}
|
|
else
|
|
vps_path=
|
|
fi
|
|
|
|
vps_connection_check "$vps" </dev/null || {
|
|
err "Failed to access '$vps'."
|
|
return 1
|
|
}
|
|
|
|
if type=$(ssh:run "root@$vps" -- vps get-type); then
|
|
info "VPS $vps seems to be of ${WHITE}$type${NORMAL} type"
|
|
else
|
|
warn "Couldn't find type of vps '$vps' (command 'vps get-type' failed on vps)."
|
|
fi
|
|
|
|
if [ -z "$path" ]; then
|
|
if [ -n "$vps_path" ]; then
|
|
err "You can't provide a VPS with path as destination if you don't provide a path in backup source."
|
|
return 1
|
|
fi
|
|
info "No path provided in backup, so we assume you want ${WHITE}full recovery${NORMAL}."
|
|
if [ "$rtype" != "$type" ]; then
|
|
if [ -n "$force" ]; then
|
|
warn "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type."
|
|
else
|
|
err "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type. (use \`\`-f\`\` to force)"
|
|
return 1
|
|
fi
|
|
fi
|
|
else
|
|
if [ "$path" == "/" ]; then
|
|
if [ -z "$vps_path" ]; then
|
|
err "Recovery of '/' (full backup files) requires that you provide a vps path also."
|
|
return 1
|
|
fi
|
|
if [ "$vps_path" == "/" ]; then
|
|
err "Recovery of '/' (full backup files) requires that you provide" \
|
|
"a vps path different from '/' also."
|
|
return 1
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
## Sets VPS and internal global variable to allow rsync to work
|
|
## from vps to backup server.
|
|
backup:setup-rsync "$admin" "$vps" "$server" "$id" || return 1
|
|
|
|
if [[ "$path" == "/"* ]]; then
|
|
if ! backup:path_exists "${path}"; then
|
|
err "File or directory '$path' not found in backup."
|
|
return 1
|
|
fi
|
|
if [ -z "$vps_path" ]; then
|
|
if [[ "$path" != *"/" ]] && backup:path_exists "${path}"/ ; then
|
|
path="$path/"
|
|
fi
|
|
vps_path=${path%/}
|
|
vps_path=${vps_path:-/}
|
|
fi
|
|
fi
|
|
|
|
|
|
case "$rtype-$type" in
|
|
mailcow-*)
|
|
## Supports having $path and $vps_path set or unset, with additional behavior
|
|
mailcow:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
|
|
;;
|
|
*-*)
|
|
if [[ "$path" == "/"* ]]; then
|
|
## For now, will require having $path and $vps_path set, no additional behaviors
|
|
file:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
|
|
else
|
|
if [ -n "$path" ]; then
|
|
err "Partial component recover of ${rtype:-unknown} backup type on" \
|
|
"${type:-unknown} type VPS is not yet implemented."
|
|
return 1
|
|
else
|
|
err "Full recover of ${rtype:-unknown} backup type on" \
|
|
"${type:-unknown} type VPS is not yet implemented."
|
|
return 1
|
|
fi
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
}
|
|
|
|
|
|
vps_install_backup() {
|
|
local vps="$1" admin server
|
|
vps_connection_check "$vps" </dev/null || return 1
|
|
|
|
read-0 admin server
|
|
if ! type=$(ssh:run "root@$vps" -- vps get-type </dev/null); then
|
|
err "Could not get type."
|
|
return 1
|
|
fi
|
|
backup_opts=()
|
|
local opt
|
|
while read-0 opt; do
|
|
case "$opt" in
|
|
--ignore-domain-check|--ignore-ping-check)
|
|
backup_opts+=("$opt")
|
|
;;
|
|
*)
|
|
err "Unknown option '$opt'."
|
|
return 1
|
|
;;
|
|
esac
|
|
done
|
|
if ! out=$(ssh:run "root@$vps" -- vps install backup "$server" "${backup_opts[@]}" 2>&1); then
|
|
err "Command 'vps install backup $server ${backup_opts[@]}' on $vps failed:"
|
|
echo "$out" | prefix " ${DARKGRAY}|${NORMAL} " >&2
|
|
return 1
|
|
fi
|
|
|
|
## Format of output:
|
|
##
|
|
## II Entry for service rsync-backup is already present in '/opt/apps/myc-deploy/compose.yml'.
|
|
## II You can run this following command from an host having admin access to core-07.0k.io:
|
|
## (Or send it to a backup admin of core-07.0k.io)
|
|
## ssh -p 10023 myadmin@core-07.0k.io ssh-key add 'ssh-rsa AAAAAAAD...QCJ\
|
|
## 8HH6pVgEpu1twyxpr9xTt7eh..WaJdgPoxmiEwGfjMMNGxs39ggOTKUuSFSmOv8TiA1fzY\
|
|
## s85hF...dKP1qByJU1k= compose@odoo.sikle.fr'
|
|
|
|
key="ssh-rsa ${out##*\'ssh-rsa }" ## remove everything before last "'ssh-rsa"
|
|
key="${key%\'*}" ## remove everything after last '
|
|
|
|
if ! [[ "$key" =~ ^"ssh-rsa "[a-zA-Z0-9/+=]+" "[a-zA-Z0-9._-]+"@"[a-zA-Z0-9._-]+$ ]]; then
|
|
err "Unexpected output from 'vps install backup $server'. Can't find key."
|
|
echo "$out" | prefix " ${GRAY}|$NORMAL " >&2
|
|
echo " Extracted key:" >&2
|
|
echo "$key" | prefix " ${GRAY}|$NORMAL " >&2
|
|
return 1
|
|
fi
|
|
|
|
if [ "$type" == "compose" ]; then
|
|
if ! ssh:run "root@$vps" -- \
|
|
docker exec myc_cron_1 \
|
|
grep rsync-backup /etc/crontabs/root >/dev/null 2>&1; then
|
|
ssh:run "root@$vps" -- compose --debug up || {
|
|
err "Command 'compose --debug up' failed."
|
|
return 1
|
|
}
|
|
if ! ssh:run "root@$vps" -- \
|
|
docker exec myc_cron_1 \
|
|
grep rsync-backup /etc/crontabs/root >/dev/null 2>&1; then
|
|
err "Launched 'compose up' successfully but ${YELLOW}cron${NORMAL} container is not setup as expected."
|
|
echo " Was waiting for existence of a line mentionning 'rsync-backup' in '/etc/crontabs/root' in it." >&2
|
|
return 1
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
dest="$server"
|
|
dest="${dest%/*}"
|
|
ssh_options=()
|
|
if [[ "$dest" == *":"* ]]; then
|
|
port="${dest##*:}"
|
|
dest="${dest%%:*}"
|
|
ssh_options=(-p "$port")
|
|
else
|
|
port=""
|
|
dest="${dest%%:*}"
|
|
fi
|
|
|
|
cmd=(ssh "${ssh_options[@]}" "$admin"@"$dest" ssh-key add "$key")
|
|
echo "${WHITE}Launching:${NORMAL} ${cmd[@]}"
|
|
|
|
"${cmd[@]}" || {
|
|
err "Failed add key to backup server '$dest'."
|
|
return 1
|
|
}
|
|
echo "${WHITE}Launching backup${NORMAL} from '$vps'"
|
|
|
|
ssh:run "root@$vps" -- vps backup || {
|
|
err "First backup failed to run."
|
|
return 1
|
|
}
|
|
|
|
echo "Backup is ${GREEN}up and running${NORMAL}."
|
|
}
|
|
|
|
|
|
vps_update() {
|
|
local vps="$1"
|
|
vps_connection_check "$vps" || return 1
|
|
ssh:run "root@$vps" -- myc-update </dev/null
|
|
}
|
|
|
|
|
|
vps_bash() {
|
|
local vps="$1"
|
|
vps_connection_check "$vps" </dev/null || return 1
|
|
ssh:run "root@$vps" -- bash
|
|
}
|
|
|
|
vps_mux() {
|
|
local fn="$1" vps_done VPS max_size vps
|
|
shift
|
|
VPS=($(printf "%s\n" "$@" | sort))
|
|
max_size=0
|
|
declare -A vps_done;
|
|
new_vps=()
|
|
for name in "${VPS[@]}"; do
|
|
[ -n "${vps_done[$name]}" ] && {
|
|
warn "duplicate vps '$name' provided. Ignoring."
|
|
continue
|
|
}
|
|
vps_done["$name"]=1
|
|
new_vps+=("$name")
|
|
size_name="${#name}"
|
|
[ "$max_size" -lt "${size_name}" ] &&
|
|
max_size="$size_name"
|
|
done
|
|
settmpdir "_0KM_TMP_DIR"
|
|
cat > "$_0KM_TMP_DIR/code"
|
|
for vps in "${new_vps[@]}"; do
|
|
label=$(printf "%-${max_size}s" "$vps")
|
|
(
|
|
{
|
|
{
|
|
"$fn" "$vps" < "$_0KM_TMP_DIR/code"
|
|
} 3>&1 1>&2 2>&3 | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKRED\!$NORMAL /g"
|
|
set_errlvl "${PIPESTATUS[0]}"
|
|
} 3>&1 1>&2 2>&3 | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKGRAY\|$NORMAL /g"
|
|
set_errlvl "${PIPESTATUS[0]}"
|
|
) &
|
|
done
|
|
wait
|
|
}
|
|
|
|
|
|
[ "$SOURCED" ] && return 0
|
|
|
|
##
|
|
## Command line processing
|
|
##
|
|
|
|
|
|
cmdline.spec.gnu
|
|
cmdline.spec.reporting
|
|
|
|
cmdline.spec.gnu vps-setup
|
|
|
|
cmdline.spec::cmd:vps-setup:run() {
|
|
|
|
: :posarg: HOST 'Target host to check/fix ssh-access'
|
|
|
|
: :optfla: --force,-f "Will force domain name change, even if
|
|
current hostname was not recognized as
|
|
an ovh domain name. "
|
|
|
|
depends sshpass shyaml
|
|
|
|
KEY_PATH="ssh-access.public-keys"
|
|
local keys=$(config get-value -y "ssh-access.public-keys") || true
|
|
if [ -z "$keys" ]; then
|
|
err "No ssh publickeys configured in config file."
|
|
echo " Looking for keys in ${WHITE}${KEY_PATH}${NORMAL}" \
|
|
"in config file." >&2
|
|
config:exists --message 2>&1 | prefix " "
|
|
if [ "${PIPESTATUS[0]}" == "0" ]; then
|
|
echo " Config file found in $(config:filename)"
|
|
fi
|
|
return 1
|
|
fi
|
|
local tkey=$(e "$keys" | shyaml get-type)
|
|
if [ "$tkey" != "sequence" ]; then
|
|
err "Value type of ${WHITE}${KEY_PATH}${NORMAL} unexpected (is $tkey, expecting sequence)."
|
|
echo " Check content of $(config:filename), and make sure to use a sequence." >&2
|
|
return 1
|
|
fi
|
|
|
|
local IP NAME keys host_pass_connected
|
|
if ! IP=$(resolve "$HOST"); then
|
|
err "'$HOST' name unresolvable."
|
|
exit 1
|
|
fi
|
|
|
|
NAME="$HOST"
|
|
if [ "$IP" != "$HOST" ]; then
|
|
NAME="$HOST ($IP)"
|
|
fi
|
|
|
|
if ! is-port-open "$IP" "22"; then
|
|
err "$NAME unreachable or port 22 closed."
|
|
exit 1
|
|
fi
|
|
debug "Host $IP's port 22 is open."
|
|
if ! host_pass_connected=$(ssh:open-try \
|
|
{root,debian}@"$HOST"); then
|
|
err "Could not connect to {root,debian}@$HOST with publickey nor password."
|
|
exit 1
|
|
fi
|
|
|
|
read-0a host password <<<"$host_pass_connected"
|
|
|
|
sudo_if_necessary=
|
|
if [ "$password" -o "${host%%@*}" != "root" ]; then
|
|
if ! ssh:run "$host" -- sudo -n true >/dev/null 2>&1; then
|
|
err "Couldn't do a password-less sudo from $host."
|
|
echo " This is not yet supported."
|
|
exit 1
|
|
else
|
|
sudo_if_necessary=sudo
|
|
fi
|
|
fi
|
|
|
|
SUPPORTED_KEY_TYPE=(ssh-rsa ssh-ed25519)
|
|
Section Checking access
|
|
while read-0 key; do
|
|
prefix="${key%% *}"
|
|
if ! [[ " ${SUPPORTED_KEY_TYPE[*]} " == *" $prefix "* ]]; then
|
|
err "Unsupported key:"$'\n'"$key"
|
|
echo " Please use only key of the following type:" >&2
|
|
printf " - %s\n" "${SUPPORTED_KEY_TYPE[@]}" >&2
|
|
return 1
|
|
fi
|
|
label="${key##* }"
|
|
Elt "considering adding key ${DARKYELLOW}$label${NORMAL}"
|
|
dest="/root/.ssh/authorized_keys"
|
|
if ssh:run "$host" -- $sudo_if_necessary grep "\"$key\"" "$dest" >/dev/null 2>&1 </dev/null; then
|
|
print_info "already present"
|
|
print_status noop
|
|
Feed
|
|
else
|
|
if echo "$key" | ssh:run "$host" -- $sudo_if_necessary tee -a "$dest" >/dev/null; then
|
|
print_info added
|
|
else
|
|
echo
|
|
Feedback failure
|
|
return 1
|
|
fi
|
|
Feedback success
|
|
fi
|
|
done < <(e "$keys" | shyaml get-values-0)
|
|
|
|
Section Checking ovh hostname file
|
|
Elt "Checking /etc/ovh-hostname"
|
|
if ! ssh:run "$host" -- $sudo_if_necessary [ -e "/etc/ovh-hostname" ]; then
|
|
print_info "creating"
|
|
ssh:run "$host" -- $sudo_if_necessary cp /etc/hostname /etc/ovh-hostname
|
|
ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
|
|
Elt "Checking /etc/ovh-hostname: $ovhname"
|
|
Feedback || return 1
|
|
else
|
|
ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
|
|
Elt "Checking /etc/ovh-hostname: $ovhname"
|
|
print_info "already present"
|
|
print_status noop
|
|
Feed
|
|
fi
|
|
|
|
if ! is_ovh_domain_name "$HOST" && ! str_is_ipv4 "$HOST"; then
|
|
Section Checking hostname
|
|
Elt "Checking /etc/hostname..."
|
|
old_etc_hostname="$(ssh:run "$host" -- $sudo_if_necessary cat /etc/hostname)"
|
|
if [ "$old_etc_hostname" != "$HOST" ]; then
|
|
Elt "/etc/hostname is '$old_etc_hostname'"
|
|
if is_ovh_hostname "$old_etc_hostname" || [ -n "$opt_force" ]; then
|
|
Elt "Hostname '$old_etc_hostname' --> '$HOST'"
|
|
print_info "creating"
|
|
echo "$HOST" | ssh:run "$host" -- $sudo_if_necessary tee /etc/hostname >/dev/null &&
|
|
ssh:run "$host" -- $sudo_if_necessary hostname "$HOST"
|
|
Feedback || return 1
|
|
else
|
|
Elt "Hostname '$old_etc_hostname' isn't an ovh domain"
|
|
print_info "no change"
|
|
print_status noop
|
|
Feed
|
|
warn "Domain name was not changed because it was already set"
|
|
echo " (use \`-f\` or \`--force\`) to force domain name change to $HOST." >&2
|
|
fi
|
|
|
|
else
|
|
print_info "already set"
|
|
print_status noop
|
|
Feed
|
|
fi
|
|
Elt "Checking consistency between /etc/hostname and \`hostname\`..."
|
|
etc_hostname="$(ssh:run "$host" -- $sudo_if_necessary cat /etc/hostname)"
|
|
transient_hostname="$(ssh:run "$host" -- $sudo_if_necessary hostname)"
|
|
if [ "$etc_hostname" != "$transient_hostname" ]; then
|
|
print_info "change"
|
|
ssh:run "$host" -- $sudo_if_necessary hostname "$etc_hostname"
|
|
Feedback || return 1
|
|
else
|
|
print_info "consistent"
|
|
print_status noop
|
|
Feed
|
|
fi
|
|
else
|
|
info "Not changing domain as '$HOST' doesn't seem to be final domain."
|
|
fi
|
|
|
|
}
|
|
|
|
|
|
cmdline.spec.gnu vps-check
|
|
cmdline.spec::cmd:vps-check:run() {
|
|
|
|
: :posarg: [VPS...] 'Target host(s) to check'
|
|
|
|
|
|
echo "" |
|
|
vps_mux vps_check "${VPS[@]}"
|
|
}
|
|
|
|
|
|
cmdline.spec.gnu vps-install
|
|
cmdline.spec::cmd:vps-install:run() {
|
|
:
|
|
}
|
|
|
|
cmdline.spec.gnu backup
|
|
cmdline.spec:vps-install:cmd:backup:run() {
|
|
|
|
: :posarg: BACKUP_TARGET 'Backup target.
|
|
(ie: myadmin@backup.domain.org:10023/256)'
|
|
: :optfla: --ignore-domain-check \
|
|
"Allow to bypass the domain check in
|
|
compose file (only used in compose
|
|
installation)."
|
|
: :optfla: --ignore-ping-check "Allow to bypass the ping check of host."
|
|
: :posarg: [VPS...] 'Target host(s) to check'
|
|
|
|
|
|
if [ "${#VPS[@]}" == 0 ]; then
|
|
warn "VPS list provided in command line is empty. Nothing will be done."
|
|
return 0
|
|
fi
|
|
|
|
if ! [[ "$BACKUP_TARGET" == *"@"* ]]; then
|
|
err "Missing admin account identifier in backup target."
|
|
echo " Have you forgottent to specify an account, ie 'myadmin@<MYBACKUP_SERVER>' ?)"
|
|
return 1
|
|
fi
|
|
|
|
admin=${BACKUP_TARGET%%@*}
|
|
server=${BACKUP_TARGET#*@}
|
|
opts=()
|
|
|
|
[ -n "$opt_ignore_ping_check" ] &&
|
|
opts+=("--ignore-ping-check")
|
|
|
|
[ -n "$opt_ignore_domain_check" ] &&
|
|
opts+=("--ignore-domain-check")
|
|
|
|
p0 "$admin" "$server" "${opts[@]}" |
|
|
vps_mux vps_install_backup "${VPS[@]}"
|
|
}
|
|
|
|
|
|
|
|
cmdline.spec.gnu vps-backup
|
|
cmdline.spec::cmd:vps-backup:run() {
|
|
:
|
|
}
|
|
|
|
|
|
cmdline.spec.gnu ls
|
|
cmdline.spec:vps-backup:cmd:ls:run() {
|
|
: :posarg: BACKUP_ID 'Backup id.
|
|
(ie: myadmin@backup.domain.org:10023)'
|
|
|
|
if ! [[ "$BACKUP_ID" == *"@"* ]]; then
|
|
err "Missing admin account identifier in backup id."
|
|
echo " Have you forgottent to specify an admin account ?" \
|
|
"ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
|
|
return 1
|
|
fi
|
|
|
|
id=${BACKUP_ID##*#}
|
|
BACKUP_TARGET=${BACKUP_ID%#*}
|
|
admin=${BACKUP_TARGET%%@*}
|
|
server=${BACKUP_TARGET#*@}
|
|
|
|
## XXXvlab: in this first implementation we expect to have access
|
|
## to the server main ssh port 22, so we won't use the provided port.
|
|
ssh_options=()
|
|
if [[ "$server" == *":"* ]]; then
|
|
ssh_options+=(-p "${server#*:}")
|
|
server=${server%%:*}
|
|
fi
|
|
|
|
ssh "${ssh_options[@]}" "$admin"@"$server" ssh-key ls
|
|
}
|
|
|
|
|
|
cmdline.spec.gnu recover
|
|
cmdline.spec:vps-backup:cmd:recover:run() {
|
|
|
|
: :posarg: BACKUP_ID 'Backup id.
|
|
(ie: myadmin@backup.domain.org:10023#mx.myvps.org
|
|
myadmin@ark-01.org#myid:/a/path
|
|
admin@ark-02.io#myid:myqsl,mailcow)'
|
|
: :posarg: VPS_PATH 'Target host(s) to check.
|
|
(ie: myvps.com
|
|
myvps.com:/a/path)'
|
|
|
|
: :optval: --date,-D '"last", or label of version to recover. (Default: "last").'
|
|
: :optfla: --force,-f 'Will allow you to bypass some checks.'
|
|
|
|
|
|
if ! [[ "$BACKUP_ID" == *"@"* ]]; then
|
|
err "Missing admin account identifier in backup id."
|
|
echo " Have you forgottent to specify an admin account ?" \
|
|
"ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
|
|
return 1
|
|
fi
|
|
if ! [[ "$BACKUP_ID" == *"@"*"#"* ]]; then
|
|
err "Missing backup label identifier in backup id."
|
|
echo " Have you forgottent to specify a backup label identifier ?" \
|
|
"ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
|
|
return 1
|
|
fi
|
|
|
|
id_path=${BACKUP_ID#*#}
|
|
if [[ "$id_path" == *":"* ]]; then
|
|
id=${id_path%%:*}
|
|
path=${id_path#*:}
|
|
else
|
|
id="$id_path"
|
|
path=
|
|
fi
|
|
BACKUP_TARGET=${BACKUP_ID%#*}
|
|
admin=${BACKUP_TARGET%%@*}
|
|
server=${BACKUP_TARGET#*@}
|
|
|
|
ssh_options=()
|
|
if [[ "$server" == *":"* ]]; then
|
|
ssh_options+=(-p "${server#*:}")
|
|
ssh_server=${server%%:*}
|
|
fi
|
|
|
|
BACKUP_PATH="/srv/datastore/data/rsync-backup-target/var/mirror"
|
|
|
|
if ! content=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key ls 2>/dev/null); then
|
|
err "Access denied to '$admin@${server}'."
|
|
return 1
|
|
fi
|
|
|
|
idents=$(echo "$content" | sed -r "s/"$'\e'"\[[0-9]+(;[0-9]+)*m//g" | cut -f 2 -d " ")
|
|
if ! [[ $'\n'"$idents"$'\n' == *$'\n'"$id"$'\n'* ]]; then
|
|
err "Given backup id '$id' not found in $admin@${server}'s idents."
|
|
return 1
|
|
fi
|
|
|
|
rtype=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key get-type "$id" ) &&
|
|
info "Backup archive matches ${WHITE}${rtype}${NORMAL} type"
|
|
|
|
p0 "$admin" "$server" "$id" "$path" "$rtype" "$opt_force" |
|
|
vps_backup_recover "${VPS_PATH}"
|
|
}
|
|
|
|
|
|
|
|
|
|
cmdline.spec.gnu vps-update
|
|
cmdline.spec::cmd:vps-update:run() {
|
|
|
|
: :posarg: [VPS...] 'Target host to check'
|
|
|
|
echo "" |
|
|
vps_mux vps_update "${VPS[@]}"
|
|
}
|
|
|
|
|
|
cmdline.spec.gnu vps-mux
|
|
cmdline.spec::cmd:vps-mux:run() {
|
|
|
|
: :posarg: [VPS...] 'Target host(s) to check'
|
|
|
|
cat | vps_mux vps_bash "${VPS[@]}"
|
|
}
|
|
|
|
|
|
cmdline.spec.gnu vps-space
|
|
cmdline.spec::cmd:vps-space:run() {
|
|
|
|
: :posarg: [VPS...] 'Target host(s) to check'
|
|
|
|
echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 2-5 -d ' '" |
|
|
vps_mux vps_bash "${VPS[@]}"
|
|
}
|
|
|
|
|
|
cmdline.spec.gnu vps-stats
|
|
cmdline.spec::cmd:vps-stats:run() {
|
|
|
|
: :posarg: [VPS...] 'Target host(s) to get stats'
|
|
|
|
: :optfla: --follow,-f 'Refresh graph every 2m'
|
|
: :optval: --timespan,-t 'timespan START[..END]'
|
|
: :optval: --resource,-r 'resource(s) separated with a comma'
|
|
: :optval: --interval,-i 'refersh interval (default: 60s)'
|
|
|
|
local opts_rrdfetch=( -a )
|
|
if [ -n "${opt_timespan}" ]; then
|
|
start=${opt_timespan%\.\.*}
|
|
opts_rrdfetch+=(-s "$start")
|
|
if [ "$start" != "${opt_timespan}" ]; then
|
|
end=${opt_timespan#*..}
|
|
opts_rrdfetch+=(-e "$end")
|
|
fi
|
|
fi
|
|
local resources=(c.memory c.network load_avg disk)
|
|
if [ -n "${opt_resource}" ]; then
|
|
resources=(${opt_resource//,/ })
|
|
fi
|
|
|
|
local not_found=()
|
|
for resource in "${resources[@]}"; do
|
|
if ! fn.exists "graph:def:$resource"; then
|
|
not_found+=("$resource")
|
|
fi
|
|
done
|
|
|
|
if [[ "${#not_found[@]}" -gt 0 ]]; then
|
|
not_found_msg=$(printf "%s, " "${not_found[@]}")
|
|
not_found_msg=${not_found_msg%, }
|
|
err "Unsupported resource(s) provided: ${not_found_msg}"
|
|
echo " resource must be one-of:" >&2
|
|
declare -F | egrep 'graph:def:[a-zA-Z_.]+$' | cut -f 3- -d " " | cut -f 3- -d ":" | prefix " - " >&2
|
|
return 1
|
|
fi
|
|
|
|
if [ "${#VPS[@]}" == 0 ]; then
|
|
err "You must provide a VPS list as positional arguments"
|
|
return 1
|
|
fi
|
|
|
|
include cache
|
|
if [ -z "$VAR_DIR" ]; then
|
|
err "Unset \$VAR_DIR, can't downlowd rrd graph"
|
|
return 1
|
|
fi
|
|
mkdir -p "$VAR_DIR/rrd"
|
|
if ! [ -d "$VAR_DIR/rrd" ]; then
|
|
err "Invalid \$VAR_DIR: '$VAR_DIR/rrd' is not a directory"
|
|
return 1
|
|
fi
|
|
|
|
(
|
|
for vps in "${VPS[@]}"; do
|
|
(
|
|
{
|
|
{
|
|
ssh:open "root@$vps" 2>/dev/null || {
|
|
err "Can't open connection $vps."
|
|
return 1
|
|
}
|
|
while true; do
|
|
if ssh:run "root@$vps" -- "[ -d '/var/lib/vps/rrd' ]"; then
|
|
echo "${WHITE}Collecting stats${NORMAL}..."
|
|
{
|
|
{
|
|
ssh:rsync "root@$vps:/var/lib/vps/rrd/" "${VAR_DIR}/rrd/${vps}"
|
|
} 3>&1 1>&2 2>&3 | prefix " ${DARKRED}\!${NORMAL} "
|
|
set_errlvl "${PIPESTATUS[0]}"
|
|
} 3>&1 1>&2 2>&3 | prefix " ${GRAY}|${NORMAL} "
|
|
echo " ${GRAY}..${NORMAL} ${DARKGREEN}done${NORMAL} collecting stats"
|
|
else
|
|
warn "No stats found. Did you run 'myc-update' on the vps ?."
|
|
fi
|
|
[ -z "$opt_follow" ] && break
|
|
echo "${WHITE}Sleeping ${DARKYELLOW}${opt_interval:-60}${NORMAL}s..."
|
|
sleep "${opt_interval:-60}"
|
|
echo " ${GRAY}..${NORMAL} ${DARKGREEN}done${NORMAL} sleeping"
|
|
done
|
|
} 3>&1 1>&2 2>&3 | prefix " ${DARKRED}\!${GRAY} collect(${DARKCYAN}$vps${GRAY})${NORMAL} "
|
|
set_errlvl "${PIPESTATUS[0]}"
|
|
} 3>&1 1>&2 2>&3 | prefix " ${GRAY}| collect(${DARKCYAN}$vps${GRAY})${NORMAL} " >&2
|
|
) &
|
|
done
|
|
wait
|
|
) &
|
|
collect_pid="$!"
|
|
if [ -z "$opt_follow" ]; then
|
|
echo "${WHITE}Fetching last stats${NORMAL}${GRAY}..${NORMAL}" >&2
|
|
wait
|
|
echo " ${GRAY}..${DARKGREEN} done${NORMAL} fetching stats" >&2
|
|
else
|
|
collect_end_msg=" ${GRAY}..${NORMAL} ${DARKGREEN}stop${NORMAL} collecting daemon (pid: ${DARKYELLOW}$collect_pid${NORMAL})"
|
|
trap_add EXIT \
|
|
"printf '%s\n' \"$collect_end_msg\" && kill $collect_pid"
|
|
echo "${WHITE}Start collecting daemon${NORMAL} (pid: ${DARKYELLOW}$collect_pid${NORMAL}) ${GRAY}..${NORMAL}" >&2
|
|
fi
|
|
( depends gnuplot ) || {
|
|
echo ""
|
|
echo " Gnuplot is required to display graphs..." \
|
|
"You might want to try to install ${WHITE}gnuplot${NORMAL} with:"
|
|
echo ""
|
|
echo " apt install gnuplot"
|
|
echo ""
|
|
return 1
|
|
} >&2
|
|
( depends rrdtool ) || {
|
|
echo ""
|
|
echo " Rrdtool is required to display graphs..." \
|
|
"You might want to try to install ${WHITE}rrdtool${NORMAL} with:"
|
|
echo ""
|
|
echo " apt install rrdtool"
|
|
echo ""
|
|
return 1
|
|
} >&2
|
|
rrd_tmpdir=$(mktemp -d -t rrd.XXXXXX)
|
|
export rrd_tmpdir
|
|
export GNUTERM=qt
|
|
## rrdtool fetch will use comma for floating point depending on some locals !
|
|
export LC_ALL=C
|
|
exec {PFD}> >(exec gnuplot 2>/dev/null)
|
|
gnuplot_pid="$!"
|
|
if [ -z "$opt_follow" ]; then
|
|
echo "${WHITE}Draw gnuplot graph${GRAY}..${NORMAL}" >&2
|
|
else
|
|
gnuplot_end_msg=" ${GRAY}..${NORMAL} ${DARKGREEN}stop${NORMAL} gnuplot process (pid: $gnuplot_pid)"
|
|
trap_add EXIT \
|
|
"printf '%s\n' \"$gnuplot_end_msg\" && kill $gnuplot_pid"
|
|
echo "${WHITE}Start gnuplot process${NORMAL} (pid: $gnuplot_pid) ${GRAY}..${NORMAL}" >&2
|
|
fi
|
|
echo "set term qt noraise replotonresize" >&$PFD
|
|
while true; do
|
|
{
|
|
i=0
|
|
data_start_ts=
|
|
data_stop_ts=
|
|
for resource in "${resources[@]}"; do
|
|
for vps in "${VPS[@]}"; do
|
|
((i++))
|
|
{
|
|
{
|
|
rrd_vps_path="$VAR_DIR/rrd/$vps"
|
|
if [ -d "$rrd_vps_path" ]; then
|
|
graph:def:"${resource}" "$vps" "$i" "${opts_rrdfetch[@]}"
|
|
else
|
|
warn "No data yet, ignoring..."
|
|
fi
|
|
} 3>&1 1>&2 2>&3 | prefix " ${DARKRED}\!${GRAY} graph(${DARKCYAN}$vps${GRAY}:${WHITE}$resource${NORMAL})${NORMAL} "
|
|
set_errlvl "${PIPESTATUS[0]}"
|
|
} 3>&1 1>&2 2>&3 || continue 1
|
|
done
|
|
done
|
|
} >&$PFD
|
|
if [ -z "$opt_follow" ]; then
|
|
echo " ${GRAY}..${DARKGREEN} done${NORMAL} gnuplot graphing" >&2
|
|
break
|
|
else
|
|
{
|
|
echo "${WHITE}Sleeping ${DARKYELLOW}${opt_interval:-60}${NORMAL}s..."
|
|
sleep "${opt_interval:-60}"
|
|
echo " ${GRAY}..${NORMAL} ${DARKGREEN}done${NORMAL} sleeping"
|
|
} | prefix " ${GRAY}| gnuplot:${NORMAL} " >&2
|
|
fi
|
|
done
|
|
|
|
if [ -n "$opt_follow" ]; then
|
|
echo "Waiting for child process to finish.." >&2
|
|
wait
|
|
echo " ..done" >&2
|
|
else
|
|
echo "pause mouse close" >&$PFD
|
|
fi
|
|
}
|
|
|
|
|
|
graph:def:c.memory() {
|
|
local vps="$1" i="$2"
|
|
shift 2
|
|
local opts_rrdfetch=("$@")
|
|
local resource="memory"
|
|
rrd_vps_path="$VAR_DIR/rrd/$vps"
|
|
[ -d "$rrd_vps_path/containers" ] || {
|
|
warn "No containers data yet for vps '$vps'... Ignoring"
|
|
return 0
|
|
}
|
|
containers=(
|
|
$(
|
|
cd "$rrd_vps_path/containers";
|
|
find -maxdepth 3 -mindepth 3 -name "${resource}.rrd" -type f |
|
|
sed -r 's%^./([^/]+/[^/]+)/[^/]+.rrd$%\1%g'
|
|
)
|
|
)
|
|
gnuplot_line_config=(
|
|
"set term qt $i title \"$vps $resource\" replotonresize noraise"
|
|
"set title '$vps'"
|
|
"set xdata time"
|
|
"set timefmt '%s'"
|
|
"set ylabel '$resource Usage'"
|
|
"set format y '%s'"
|
|
"set ytics format ' %g'"
|
|
"set mouse mouseformat 6"
|
|
"set yrange [0:*] "
|
|
"set border behind"
|
|
)
|
|
printf "%s\n" "${gnuplot_line_config[@]}"
|
|
first=1
|
|
for container in "${containers[@]}"; do
|
|
rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/containers/$container/$resource.rrd\""
|
|
rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
|
|
rrdfetch_cmd+=" tail -n +2 | \\"$'\n'
|
|
rrdfetch_cmd+=" egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
|
|
rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
|
|
rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
|
|
first_ts=
|
|
first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ")
|
|
if [ -z "$first_ts" ]; then
|
|
warn "No data for $container on vps $vps, skipping..."
|
|
continue
|
|
fi
|
|
last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1 | cut -f 1 -d " ")
|
|
if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
|
|
data_start_ts="$first_ts"
|
|
fi
|
|
if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
|
|
data_stop_ts="$last_ts"
|
|
fi
|
|
if [ -n "$first" ]; then
|
|
first=
|
|
echo "plot \\"
|
|
else
|
|
echo ", \\"
|
|
fi
|
|
container="${container//\'/}"
|
|
container="${container//@/\\@}"
|
|
echo -n " ${rrdfetch_cmd} u 1:(\$3/(1000*1000*1000)) w lines title '${container//_/\\_}'"
|
|
done
|
|
echo
|
|
}
|
|
|
|
|
|
graph:def:c.network() {
|
|
local vps="$1" i="$2"
|
|
shift 2
|
|
local opts_rrdfetch=("$@")
|
|
local resource="network"
|
|
rrd_vps_path="$VAR_DIR/rrd/$vps"
|
|
[ -d "$rrd_vps_path/containers" ] || {
|
|
warn "No containers data yet for vps '$vps'... Ignoring"
|
|
return 0
|
|
}
|
|
containers=(
|
|
$(
|
|
cd "$rrd_vps_path/containers";
|
|
find -maxdepth 3 -mindepth 3 -name "${resource}.rrd" -type f |
|
|
sed -r 's%^./([^/]+/[^/]+)/[^/]+.rrd$%\1%g'
|
|
)
|
|
)
|
|
gnuplot_line_config=(
|
|
"set term qt $i title \"$vps $resource\" replotonresize noraise"
|
|
"set title '$vps'"
|
|
"set xdata time"
|
|
"set timefmt '%s'"
|
|
"set ylabel '$resource Usage'"
|
|
"set format y '%s'"
|
|
"set ytics format ' %.2f MiB/s'"
|
|
"set mouse mouseformat 6"
|
|
"set yrange [0:*] "
|
|
"set border behind"
|
|
)
|
|
printf "%s\n" "${gnuplot_line_config[@]}"
|
|
first=1
|
|
for container in "${containers[@]}"; do
|
|
rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/containers/$container/$resource.rrd\""
|
|
rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
|
|
rrdfetch_cmd+=" tail -n +2 | \\"$'\n'
|
|
rrdfetch_cmd+=" egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
|
|
rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
|
|
rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
|
|
first_ts=
|
|
first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ")
|
|
if [ -z "$first_ts" ]; then
|
|
warn "No data for $container on vps $vps, skipping..."
|
|
continue
|
|
fi
|
|
last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1 | cut -f 1 -d " ")
|
|
if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
|
|
data_start_ts="$first_ts"
|
|
fi
|
|
if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
|
|
data_stop_ts="$last_ts"
|
|
fi
|
|
if [ -n "$first" ]; then
|
|
first=
|
|
echo "plot \\"
|
|
else
|
|
echo ", \\"
|
|
fi
|
|
container="${container//\'/}"
|
|
container="${container//@/\\@}"
|
|
echo -n " ${rrdfetch_cmd} u 1:((\$3 / 1024) / 1024) w lines title '${container//_/\\_}'"
|
|
done
|
|
echo
|
|
}
|
|
|
|
|
|
graph:def:load_avg() {
|
|
local vps="$1" i="$2"
|
|
shift 2
|
|
local opts_rrdfetch=("$@")
|
|
rrd_vps_path="$VAR_DIR/rrd/$vps"
|
|
[ -f "$rrd_vps_path/$resource.rrd" ] || {
|
|
warn "No containers data yet for vps '$vps'... Ignoring"
|
|
return 0
|
|
}
|
|
gnuplot_line_config=(
|
|
"set term qt $i title \"$vps $resource\" replotonresize noraise"
|
|
"set title '$vps'"
|
|
"set xdata time"
|
|
"set timefmt '%s'"
|
|
"set ylabel '${resource//_/\\_} Usage'"
|
|
"set format y '%s'"
|
|
"set ytics format '%g'"
|
|
"set mouse mouseformat 6"
|
|
"set yrange [0:*] "
|
|
"set border behind"
|
|
)
|
|
printf "%s\n" "${gnuplot_line_config[@]}"
|
|
first=1
|
|
for value in 1m:2 5m:3 15m:4; do
|
|
label="${value%:*}"
|
|
col_num="${value#*:}"
|
|
rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/$resource.rrd\""
|
|
rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
|
|
rrdfetch_cmd+=" tail -n +2 | \\"$'\n'
|
|
rrdfetch_cmd+=" egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
|
|
rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
|
|
rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
|
|
first_ts=
|
|
first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ")
|
|
if [ -z "$first_ts" ]; then
|
|
warn "No data for $resource on vps $vps, skipping..."
|
|
continue
|
|
fi
|
|
last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1 | cut -f 1 -d " ")
|
|
if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
|
|
data_start_ts="$first_ts"
|
|
fi
|
|
if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
|
|
data_stop_ts="$last_ts"
|
|
fi
|
|
if [ -n "$first" ]; then
|
|
first=
|
|
echo "plot \\"
|
|
else
|
|
echo ", \\"
|
|
fi
|
|
container="${container//\'/}"
|
|
container="${container//@/\\@}"
|
|
echo -n " ${rrdfetch_cmd} u 1:$col_num w lines title '${label}'"
|
|
done
|
|
echo
|
|
}
|
|
|
|
graph:def:disk() {
|
|
local vps="$1" i="$2"
|
|
shift 2
|
|
local opts_rrdfetch=("$@")
|
|
rrd_vps_path="$VAR_DIR/rrd/$vps"
|
|
[ -f "$rrd_vps_path/$resource.rrd" ] || {
|
|
warn "No containers data yet for vps '$vps'... Ignoring"
|
|
return 0
|
|
}
|
|
gnuplot_line_config=(
|
|
"set term qt $i title \"$vps $resource\" replotonresize noraise"
|
|
"set title '$vps'"
|
|
"set xdata time"
|
|
"set timefmt '%s'"
|
|
"set ylabel '${resource//_/\\_} Usage'"
|
|
"set format y '%s'"
|
|
"set ytics format '%g GiB'"
|
|
"set mouse mouseformat 6"
|
|
"set yrange [0:*] "
|
|
"set border behind"
|
|
)
|
|
printf "%s\n" "${gnuplot_line_config[@]}"
|
|
first=1
|
|
for value in used:2 size:3; do
|
|
label="${value%:*}"
|
|
col_num="${value#*:}"
|
|
rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/$resource.rrd\""
|
|
rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
|
|
rrdfetch_cmd+=" tail -n +2 | \\"$'\n'
|
|
rrdfetch_cmd+=" egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
|
|
rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
|
|
rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
|
|
first_ts=
|
|
first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ")
|
|
if [ -z "$first_ts" ]; then
|
|
warn "No data for $resource on vps $vps, skipping..."
|
|
continue
|
|
fi
|
|
last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1 | cut -f 1 -d " ")
|
|
if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
|
|
data_start_ts="$first_ts"
|
|
fi
|
|
if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
|
|
data_stop_ts="$last_ts"
|
|
fi
|
|
if [ -n "$first" ]; then
|
|
first=
|
|
echo "plot \\"
|
|
else
|
|
echo ", \\"
|
|
fi
|
|
container="${container//\'/}"
|
|
container="${container//@/\\@}"
|
|
echo -n " ${rrdfetch_cmd} u 1:(\$${col_num}/(1024*1024)) w lines title '${label}'"
|
|
done
|
|
echo
|
|
}
|
|
|
|
|
|
graph:def:disk-2() {
|
|
local vps="$1" i="$2"
|
|
shift 2
|
|
local opts_rrdfetch=("$@")
|
|
rrd_vps_path="$VAR_DIR/rrd/$vps"
|
|
rscolcat:check-install || return 1
|
|
folders=(
|
|
$(
|
|
cd "$rrd_vps_path/$resource/data";
|
|
find -maxdepth 2 -mindepth 2 -name "size.rrd" -type f |
|
|
sed -r 's%^./([^/]+)/[^/]+.rrd$%data/\1%g'
|
|
)
|
|
)
|
|
gnuplot_line_config=(
|
|
"set term qt $i title \"$vps $resource\" replotonresize noraise"
|
|
"set title '$vps'"
|
|
"set xdata time"
|
|
"set timefmt '%s'"
|
|
"set ylabel '${resource//_/\\_} Usage'"
|
|
"set format y '%s'"
|
|
"set ytics format '%g GiB'"
|
|
"set mouse mouseformat 6"
|
|
"set yrange [0:*] "
|
|
"set border behind"
|
|
'set key outside top right'
|
|
)
|
|
declare -A colors
|
|
colors[host/other]="#000000"
|
|
colors[docker/build_cache]="#87CEEB"
|
|
colors[docker/containers]="#0047AB"
|
|
colors[docker/images]="#00FFFF"
|
|
colors[docker/local_volumes]="#6495ED"
|
|
colors[docker/logs]="#000080"
|
|
|
|
|
|
printf "%s\n" "${gnuplot_line_config[@]}"
|
|
first=1
|
|
rrdfetch_cmd="'< rscolcat concat"
|
|
lines_def=()
|
|
legend_def=()
|
|
label="${value%:*}"
|
|
idx=1
|
|
previous_sum=0
|
|
values=(
|
|
host/other
|
|
docker/{build_cache,local_volumes,logs,containers,images}
|
|
"${folders[@]}"
|
|
)
|
|
declare -A values_colnb
|
|
for subvalue in "${values[@]}" ; do
|
|
idx=$((idx + 1))
|
|
values_colnb["$subvalue"]="$idx"
|
|
[ -e "$rrd_vps_path/$resource/$subvalue/size.rrd" ] || {
|
|
warn "No data yet for $subvalue on vps $vps, ignoring..."
|
|
continue
|
|
}
|
|
rrdfetch_cmd+=" \\"$'\n'
|
|
rrdfetch_cmd+=" <(rrdtool fetch \"$rrd_vps_path/$resource/$subvalue/size.rrd\""
|
|
rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
|
|
rrdfetch_cmd+=" tail -n +2 | \\"$'\n'
|
|
rrdfetch_cmd+=" egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\")"
|
|
|
|
done
|
|
rrdfetch_cmd+="'"
|
|
rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
|
|
rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
|
|
eval "$rrdfetch_cmd_bash" > "$rrd_tmpdir/data.txt" || return 1
|
|
first_ts=
|
|
first_ts=$(cat "$rrd_tmpdir/data.txt" | head -n 1 | cut -f 1 -d " ")
|
|
if [ -z "$first_ts" ]; then
|
|
err "No data for $resource on vps $vps, skipping..."
|
|
return 1
|
|
fi
|
|
last_ts=$(cat "$rrd_tmpdir/data.txt" | tail -n 1 | cut -f 1 -d " ")
|
|
last_valued_line=$(cat "$rrd_tmpdir/data.txt" | egrep -v '^[0-9]+( -)+' | tail -n 1) || return 1
|
|
if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
|
|
data_start_ts="$first_ts"
|
|
fi
|
|
if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
|
|
data_stop_ts="$last_ts"
|
|
fi
|
|
last_values=(${last_valued_line#* })
|
|
idx=0
|
|
data_sorted_values=$(
|
|
for last_value in "${last_values[@]}"; do
|
|
[[ "${values[$idx]}" != "data/"* ]] && {
|
|
idx=$((idx + 1))
|
|
continue
|
|
}
|
|
if [ "$last_value" == "-" ]; then
|
|
last_value=0
|
|
fi
|
|
printf "%s %.0f\n" "${values[$idx]}" "$last_value"
|
|
idx=$((idx + 1))
|
|
done | sort -k 2 -n | cut -f 1 -d " "
|
|
)
|
|
data_sorted_values=(${data_sorted_values})
|
|
color_step=$(( 255 / ${#data_sorted_values[@]} ))
|
|
toggle=10
|
|
for label in "${data_sorted_values[@]}"; do
|
|
if [ "$toggle" -eq 10 ]; then
|
|
toggle=240
|
|
else
|
|
toggle=10
|
|
fi
|
|
colors[$label]=$(printf "#%02x%02x%02x" "$toggle" $(( 255 - (color_step * idx) )) $(( color_step * idx)) )
|
|
idx=$((idx + 1))
|
|
done
|
|
echo "plot \\"
|
|
first=1
|
|
line_def=""
|
|
ordered_values=(
|
|
host/other
|
|
docker/{build_cache,local_volumes,logs,containers,images}
|
|
"${data_sorted_values[@]}"
|
|
)
|
|
idx=0
|
|
for subvalue in "${ordered_values[@]}" ; do
|
|
colnb="${values_colnb[$subvalue]}"
|
|
sum_str="(valid(${colnb}) ? column(${colnb}) : 0)"
|
|
line_def=""
|
|
if [ -n "$first" ]; then
|
|
current_sum+="${sum_str}"
|
|
first=
|
|
line_def="u 1:(0):(1/(1/((${current_sum})/(1024*1024)))) w filledcurves title '${subvalue//_/\\_}'"
|
|
else
|
|
echo -n ", \\"$'\n'
|
|
current_sum+=" + $sum_str"
|
|
line_def="u 1:(1/(1/((${previous_sum})/(1024*1024)))):(1/(1/((${current_sum})/(1024*1024)))) w filledcurves title '${subvalue//_/\\_}'"
|
|
fi
|
|
previous_sum="$current_sum"
|
|
if [[ -n "${colors[$subvalue]}" ]]; then
|
|
line_def+=" lc rgb '${colors[$subvalue]}'"
|
|
fi
|
|
echo -n " '$rrd_tmpdir/data.txt' ${line_def}"
|
|
done
|
|
## Add line with "available" value from host/available
|
|
echo -n ", \\"$'\n'
|
|
echo -n " '< rrdtool fetch \"$rrd_vps_path/$resource/host/available/size.rrd\""
|
|
echo -n " AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
|
|
echo -n " tail -n +2 | \\"$'\n'
|
|
echo -n " egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
|
|
echo -n " u 1:(\$2/(1024*1024)) w lines title 'available' lw 2 lc rgb '#FF0000'"
|
|
## Add dashed line with 75% of "available" value from host/available
|
|
echo -n ", \\"$'\n'
|
|
echo -n " '< rrdtool fetch \"$rrd_vps_path/$resource/host/available/size.rrd\""
|
|
echo -n " AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
|
|
echo -n " tail -n +2 | \\"$'\n'
|
|
echo -n " egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
|
|
echo -n " u 1:((3 * \$2)/(4*1024*1024)) w lines title '75% available' lw 1 dashtype 2 lc rgb '#FF0000'"
|
|
## Add dashed line with 95% of "available" value from host/available
|
|
echo -n ", \\"$'\n'
|
|
echo -n " '< rrdtool fetch \"$rrd_vps_path/$resource/host/available/size.rrd\""
|
|
echo -n " AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
|
|
echo -n " tail -n +2 | \\"$'\n'
|
|
echo -n " egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
|
|
echo -n " u 1:((9 * \$2)/(10*1024*1024)) w lines title '90% available' lw 2 dashtype 2 lc rgb '#FF0000'"
|
|
echo
|
|
|
|
}
|
|
|
|
|
|
|
|
cmdline.spec.gnu vps-subscribe
|
|
cmdline.spec::cmd:vps-subscribe:run() {
|
|
:
|
|
}
|
|
cmdline.spec.gnu add
|
|
cmdline.spec:vps-subscribe:cmd:add:run() {
|
|
|
|
: :posarg: CHANNEL 'Channel which will be sent to given topic'
|
|
: :posarg: TOPIC 'Ntfy topic to recieve messages of given channel
|
|
(format: "[MYSERVER:]MYTOPICS"
|
|
Examples: "ntfy.0k.io:main,storage,alerts",
|
|
"main{1,3,7}"
|
|
)'
|
|
: :posarg: [VPS...] 'Target host(s) to get stats'
|
|
|
|
printf "%s\0" "$CHANNEL" "$TOPIC" |
|
|
vps_mux subscribe:add "${VPS[@]}"
|
|
|
|
}
|
|
|
|
cmdline.spec.gnu rm
|
|
cmdline.spec:vps-subscribe:cmd:rm:run() {
|
|
|
|
: :posarg: CHANNEL 'Channel which will be sent to given topic'
|
|
: :posarg: TOPIC 'Ntfy topic to recieve messages of given channel
|
|
(format: "[MYSERVER:]MYTOPICS"
|
|
Examples: "ntfy.0k.io:main,storage,alerts",
|
|
"main{1,3,7}"
|
|
)'
|
|
: :posarg: [VPS...] 'Target host(s) to get stats'
|
|
|
|
|
|
printf "%s\0" "$CHANNEL" "$TOPIC" |
|
|
vps_mux subscribe:rm "${VPS[@]}"
|
|
|
|
}
|
|
|
|
|
|
|
|
cmdline::parse "$@"
|