|
|
|
@ -259,7 +259,7 @@ vps_check() { |
|
|
|
fi </dev/null |
|
|
|
compose_content=$(ssh:run "root@$vps" -- cat /opt/apps/myc-deploy/compose.yml </dev/null) || |
|
|
|
{ echo "${DARKRED}no-compose${NORMAL}"; return 1; } |
|
|
|
echo "$compose_content" | grep backup >/dev/null 2>&1 || |
|
|
|
echo "$compose_content" | yq -e ".rsync-backup" >/dev/null 2>&1 || |
|
|
|
{ echo "${DARKRED}no-backup${NORMAL}"; return 1; } |
|
|
|
} |
|
|
|
|
|
|
|
@ -656,6 +656,156 @@ EOF |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
NTFY_TOPIC_FILE="/etc/ntfy/topics.yml" |
|
|
|
NTFY_CONFIG_FILE="/etc/ntfy/ntfy.conf" |
|
|
|
subscribe:ntfy:topic-file-exists() { |
|
|
|
local vps="$1" |
|
|
|
if ! out=$(echo "[ -f \"$NTFY_TOPIC_FILE\" ] && echo ok || true" | \ |
|
|
|
ssh:run "root@$vps" -- bash); then |
|
|
|
err "Unable to check for existence of '$NTFY_TOPIC_FILE'." |
|
|
|
fi |
|
|
|
if [ -z "$out" ]; then |
|
|
|
err "File '$NTFY_TOPIC_FILE' not found on $vps." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
} |
|
|
|
|
|
|
|
subscribe:ntfy:config-file-exists() { |
|
|
|
local vps="$1" |
|
|
|
if ! out=$(echo "[ -f \"$NTFY_CONFIG_FILE\" ] && echo ok || true" | \ |
|
|
|
ssh:run "root@$vps" -- bash); then |
|
|
|
err "Unable to check for existence of '$NTFY_CONFIG_FILE'." |
|
|
|
fi |
|
|
|
if [ -z "$out" ]; then |
|
|
|
err "File '$NTFY_CONFIG_FILE' not found on $vps." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
ntfy:rm() { |
|
|
|
local channel="$1" topic="$2" vps="$3" |
|
|
|
subscribe:ntfy:topic-file-exists "$vps" || return 1 |
|
|
|
if ! out=$(echo "yq -i 'del(.[\"$channel\"][] | select(. == \"$TOPIC\"))' \"$NTFY_TOPIC_FILE\"" | \ |
|
|
|
ssh:run "root@$vps" -- bash); then |
|
|
|
err "Failed to remove channel '$channel' from '$NTFY_TOPIC_FILE'." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
info "Channel '$channel' removed from '$NTFY_TOPIC_FILE' on $vps." |
|
|
|
ssh:run "root@$vps" -- cat "$NTFY_TOPIC_FILE" |
|
|
|
} |
|
|
|
|
|
|
|
ntfy:add() { |
|
|
|
local channel="$1" topic="$2" vps="$3" |
|
|
|
vps_connection_check "$vps" </dev/null || return 1 |
|
|
|
subscribe:ntfy:topic-file-exists "$vps" || return 1 |
|
|
|
if ! out=$(echo "yq '. | has(\"$channel\")' \"$NTFY_TOPIC_FILE\"" | \ |
|
|
|
ssh:run "root@$vps" -- bash); then |
|
|
|
err "Failed to check if channel '$channel' with topic '$topic' is already in '$NTFY_TOPIC_FILE'." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
if [ "$out" != "true" ]; then |
|
|
|
## Channel does not exist |
|
|
|
if ! out=$(echo "yq -i '.[\"$channel\"] = []' \"$NTFY_TOPIC_FILE\"" | \ |
|
|
|
ssh:run "root@$vps" -- bash); then |
|
|
|
err "Failed to create a new channel '$channel' entry in '$NTFY_TOPIC_FILE'." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
else |
|
|
|
## Channel exists |
|
|
|
if ! out=$(echo "yq '.[\"$channel\"] | any_c(. == \"$topic\")' \"$NTFY_TOPIC_FILE\"" | \ |
|
|
|
ssh:run "root@$vps" -- bash); then |
|
|
|
err "Failed to check if channel '$channel' with topic '$topic' is already in '$NTFY_TOPIC_FILE'." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
if [ "$out" == "true" ]; then |
|
|
|
info "Channel '$channel' with topic '$topic' already exists in '$NTFY_TOPIC_FILE'." |
|
|
|
return 0 |
|
|
|
fi |
|
|
|
fi |
|
|
|
if ! out=$(echo "yq -i '.[\"$channel\"] += [\"$topic\"]' \"$NTFY_TOPIC_FILE\"" | \ |
|
|
|
ssh:run "root@$vps" -- bash); then |
|
|
|
err "Failed to add channel '$channel' with topic '$topic' to '$NTFY_TOPIC_FILE'." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
info "Channel '$channel' added with topic '$topic' to '$NTFY_TOPIC_FILE' on $vps." |
|
|
|
} |
|
|
|
|
|
|
|
NTFY_BROKER_SERVER="ntfy.0k.io" |
|
|
|
ntfy:topic-access() { |
|
|
|
local action="$1" topic="$2" vps="$3" |
|
|
|
subscribe:ntfy:config-file-exists "$vps" || return 1 |
|
|
|
|
|
|
|
local user |
|
|
|
user=$(ntfy:get-login "$vps") || return 1 |
|
|
|
case "$action" in |
|
|
|
"write") |
|
|
|
ssh "ntfy@$NTFY_BROKER_SERVER" "topic-access" \ |
|
|
|
"$user" "$topic" "write-only" </dev/null || { |
|
|
|
err "Failed to grant write access to '$user' for topic '$topic'." |
|
|
|
return 1 |
|
|
|
} |
|
|
|
info "Granted write access for '$user' to topic '$topic'." |
|
|
|
;; |
|
|
|
"remove") |
|
|
|
ssh "ntfy@$NTFY_BROKER_SERVER" "topic-access" -r "$user" "$topic" </dev/null || { |
|
|
|
err "Failed to reset access of '$user' for topic '$topic'." |
|
|
|
return 1 |
|
|
|
} |
|
|
|
info "Access for '$user' to topic '$topic' was resetted successfully." |
|
|
|
;; |
|
|
|
*) |
|
|
|
err "Invalid action '$action'." |
|
|
|
return 1 |
|
|
|
;; |
|
|
|
esac |
|
|
|
} |
|
|
|
|
|
|
|
ntfy:get-login() { |
|
|
|
local vps="$1" |
|
|
|
if ! out=$(echo ". \"$NTFY_CONFIG_FILE\" && echo \"\$LOGIN\"" | \ |
|
|
|
ssh:run "root@$vps" -- bash); then |
|
|
|
err "Failed to get ntfy login from '$NTFY_CONFIG_FILE'." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
if [ -z "$out" ]; then |
|
|
|
err "Unexpected empty login retrieved from sourcing '$NTFY_CONFIG_FILE'." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
echo "$out" |
|
|
|
} |
|
|
|
|
|
|
|
subscribe:add() { |
|
|
|
local vps="$1" |
|
|
|
read-0 channel topic || { |
|
|
|
err "Couldn't read CHANNEL and TOPIC arguments." |
|
|
|
return 1 |
|
|
|
} |
|
|
|
vps_connection_check "$vps" </dev/null || return 1 |
|
|
|
ntfy:topic-access "write" "$topic" "$vps" </dev/null || return 1 |
|
|
|
ntfy:add "$channel" "$topic" "$vps" || { |
|
|
|
err "Failed to add channel '$channel' with topic '$topic' to '$NTFY_TOPIC_FILE'." |
|
|
|
echo " Removing topic access." >&2 |
|
|
|
ntfy:topic-access "remove" "$topic" "$vps" </dev/null |
|
|
|
return 1 |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
subscribe:rm() { |
|
|
|
local vps="$1" |
|
|
|
read-0 channel topic || { |
|
|
|
err "Couldn't read CHANNEL and TOPIC arguments." |
|
|
|
return 1 |
|
|
|
} |
|
|
|
vps_connection_check "$vps" </dev/null || return 1 |
|
|
|
|
|
|
|
ntfy:rm "$channel" "$topic" "$vps" || return 1 |
|
|
|
ntfy:topic-access "remove" "$topic" "$vps" </dev/null || { |
|
|
|
err "Failed to remove topic access for '$topic' on '$vps'." |
|
|
|
return 1 |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
vps_backup_recover() { |
|
|
|
local vps="$1" admin server id path rtype force type |
|
|
|
|
|
|
|
@ -757,13 +907,25 @@ vps_install_backup() { |
|
|
|
vps_connection_check "$vps" </dev/null || return 1 |
|
|
|
|
|
|
|
read-0 admin server |
|
|
|
if ! type=$(ssh:run "root@$vps" -- vps get-type); then |
|
|
|
if ! type=$(ssh:run "root@$vps" -- vps get-type </dev/null); then |
|
|
|
err "Could not get type." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
|
|
|
|
if ! out=$(ssh:run "root@$vps" -- vps install backup "$server" 2>&1); then |
|
|
|
err "Command 'vps install backup $server' on $vps failed:" |
|
|
|
backup_opts=() |
|
|
|
local opt |
|
|
|
while read-0 opt; do |
|
|
|
case "$opt" in |
|
|
|
--ignore-domain-check|--ignore-ping-check) |
|
|
|
backup_opts+=("$opt") |
|
|
|
;; |
|
|
|
*) |
|
|
|
err "Unknown option '$opt'." |
|
|
|
return 1 |
|
|
|
;; |
|
|
|
esac |
|
|
|
done |
|
|
|
if ! out=$(ssh:run "root@$vps" -- vps install backup "$server" "${backup_opts[@]}" 2>&1); then |
|
|
|
err "Command 'vps install backup $server ${backup_opts[@]}' on $vps failed:" |
|
|
|
echo "$out" | prefix " ${DARKGRAY}|${NORMAL} " >&2 |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
@ -791,16 +953,16 @@ vps_install_backup() { |
|
|
|
if [ "$type" == "compose" ]; then |
|
|
|
if ! ssh:run "root@$vps" -- \ |
|
|
|
docker exec myc_cron_1 \ |
|
|
|
cat /etc/cron.d/rsync-backup >/dev/null 2>&1; then |
|
|
|
grep rsync-backup /etc/crontabs/root >/dev/null 2>&1; then |
|
|
|
ssh:run "root@$vps" -- compose --debug up || { |
|
|
|
err "Command 'compose --debug up' failed." |
|
|
|
return 1 |
|
|
|
} |
|
|
|
if ! ssh:run "root@$vps" -- \ |
|
|
|
docker exec myc_cron_1 \ |
|
|
|
cat /etc/cron.d/rsync-backup >/dev/null 2>&1; then |
|
|
|
grep rsync-backup /etc/crontabs/root >/dev/null 2>&1; then |
|
|
|
err "Launched 'compose up' successfully but ${YELLOW}cron${NORMAL} container is not setup as expected." |
|
|
|
echo " Was waiting for existence of '/etc/cron.d/rsync-backup' in it." >&2 |
|
|
|
echo " Was waiting for existence of a line mentionning 'rsync-backup' in '/etc/crontabs/root' in it." >&2 |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
fi |
|
|
|
@ -1072,6 +1234,11 @@ cmdline.spec:vps-install:cmd:backup:run() { |
|
|
|
|
|
|
|
: :posarg: BACKUP_TARGET 'Backup target. |
|
|
|
(ie: myadmin@backup.domain.org:10023/256)' |
|
|
|
: :optfla: --ignore-domain-check \ |
|
|
|
"Allow to bypass the domain check in |
|
|
|
compose file (only used in compose |
|
|
|
installation)." |
|
|
|
: :optfla: --ignore-ping-check "Allow to bypass the ping check of host." |
|
|
|
: :posarg: [VPS...] 'Target host(s) to check' |
|
|
|
|
|
|
|
|
|
|
|
@ -1088,7 +1255,15 @@ cmdline.spec:vps-install:cmd:backup:run() { |
|
|
|
|
|
|
|
admin=${BACKUP_TARGET%%@*} |
|
|
|
server=${BACKUP_TARGET#*@} |
|
|
|
p0 "$admin" "$server" | |
|
|
|
opts=() |
|
|
|
|
|
|
|
[ -n "$opt_ignore_ping_check" ] && |
|
|
|
opts+=("--ignore-ping-check") |
|
|
|
|
|
|
|
[ -n "$opt_ignore_domain_check" ] && |
|
|
|
opts+=("--ignore-domain-check") |
|
|
|
|
|
|
|
p0 "$admin" "$server" "${opts[@]}" | |
|
|
|
vps_mux vps_install_backup "${VPS[@]}" |
|
|
|
} |
|
|
|
|
|
|
|
@ -1246,7 +1421,7 @@ cmdline.spec::cmd:vps-stats:run() { |
|
|
|
opts_rrdfetch+=(-e "$end") |
|
|
|
fi |
|
|
|
fi |
|
|
|
local resources=(c.memory c.network load_avg) |
|
|
|
local resources=(c.memory c.network load_avg disk) |
|
|
|
if [ -n "${opt_resource}" ]; then |
|
|
|
resources=(${opt_resource//,/ }) |
|
|
|
fi |
|
|
|
@ -1464,7 +1639,7 @@ graph:def:c.memory() { |
|
|
|
fi |
|
|
|
container="${container//\'/}" |
|
|
|
container="${container//@/\\@}" |
|
|
|
echo -n " ${rrdfetch_cmd} u 1:((\$3 - \$2)/1000000000) w lines title '${container//_/\\_}'" |
|
|
|
echo -n " ${rrdfetch_cmd} u 1:(\$3/(1000*1000*1000)) w lines title '${container//_/\\_}'" |
|
|
|
done |
|
|
|
echo |
|
|
|
} |
|
|
|
@ -1593,5 +1768,105 @@ graph:def:load_avg() { |
|
|
|
echo |
|
|
|
} |
|
|
|
|
|
|
|
graph:def:disk() { |
|
|
|
local vps="$1" i="$2" |
|
|
|
shift 2 |
|
|
|
local opts_rrdfetch=("$@") |
|
|
|
rrd_vps_path="$VAR_DIR/rrd/$vps" |
|
|
|
[ -f "$rrd_vps_path/$resource.rrd" ] || { |
|
|
|
warn "No containers data yet for vps '$vps'... Ignoring" |
|
|
|
return 0 |
|
|
|
} |
|
|
|
gnuplot_line_config=( |
|
|
|
"set term qt $i title \"$vps $resource\" replotonresize noraise" |
|
|
|
"set title '$vps'" |
|
|
|
"set xdata time" |
|
|
|
"set timefmt '%s'" |
|
|
|
"set ylabel '${resource//_/\\_} Usage'" |
|
|
|
"set format y '%s'" |
|
|
|
"set ytics format '%g GiB'" |
|
|
|
"set mouse mouseformat 6" |
|
|
|
"set yrange [0:*] " |
|
|
|
"set border behind" |
|
|
|
) |
|
|
|
printf "%s\n" "${gnuplot_line_config[@]}" |
|
|
|
first=1 |
|
|
|
for value in used:2 size:3; do |
|
|
|
label="${value%:*}" |
|
|
|
col_num="${value#*:}" |
|
|
|
rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/$resource.rrd\"" |
|
|
|
rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n' |
|
|
|
rrdfetch_cmd+=" tail -n +2 | \\"$'\n' |
|
|
|
rrdfetch_cmd+=" egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'" |
|
|
|
rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}") |
|
|
|
rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< } |
|
|
|
first_ts= |
|
|
|
first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ") |
|
|
|
if [ -z "$first_ts" ]; then |
|
|
|
warn "No data for $resource on vps $vps, skipping..." |
|
|
|
continue |
|
|
|
fi |
|
|
|
last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1 | cut -f 1 -d " ") |
|
|
|
if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then |
|
|
|
data_start_ts="$first_ts" |
|
|
|
fi |
|
|
|
if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then |
|
|
|
data_stop_ts="$last_ts" |
|
|
|
fi |
|
|
|
if [ -n "$first" ]; then |
|
|
|
first= |
|
|
|
echo "plot \\" |
|
|
|
else |
|
|
|
echo ", \\" |
|
|
|
fi |
|
|
|
container="${container//\'/}" |
|
|
|
container="${container//@/\\@}" |
|
|
|
echo -n " ${rrdfetch_cmd} u 1:(\$${col_num}/(1024*1024)) w lines title '${label}'" |
|
|
|
done |
|
|
|
echo |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
cmdline.spec.gnu vps-subscribe |
|
|
|
cmdline.spec::cmd:vps-subscribe:run() { |
|
|
|
: |
|
|
|
} |
|
|
|
cmdline.spec.gnu add |
|
|
|
cmdline.spec:vps-subscribe:cmd:add:run() { |
|
|
|
|
|
|
|
: :posarg: CHANNEL 'Channel which will be sent to given topic' |
|
|
|
: :posarg: TOPIC 'Ntfy topic to recieve messages of given channel |
|
|
|
(format: "[MYSERVER:]MYTOPICS" |
|
|
|
Examples: "ntfy.0k.io:main,storage,alerts", |
|
|
|
"main{1,3,7}" |
|
|
|
)' |
|
|
|
: :posarg: [VPS...] 'Target host(s) to get stats' |
|
|
|
|
|
|
|
printf "%s\0" "$CHANNEL" "$TOPIC" | |
|
|
|
vps_mux subscribe:add "${VPS[@]}" |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
cmdline.spec.gnu rm |
|
|
|
cmdline.spec:vps-subscribe:cmd:rm:run() { |
|
|
|
|
|
|
|
: :posarg: CHANNEL 'Channel which will be sent to given topic' |
|
|
|
: :posarg: TOPIC 'Ntfy topic to recieve messages of given channel |
|
|
|
(format: "[MYSERVER:]MYTOPICS" |
|
|
|
Examples: "ntfy.0k.io:main,storage,alerts", |
|
|
|
"main{1,3,7}" |
|
|
|
)' |
|
|
|
: :posarg: [VPS...] 'Target host(s) to get stats' |
|
|
|
|
|
|
|
|
|
|
|
printf "%s\0" "$CHANNEL" "$TOPIC" | |
|
|
|
vps_mux subscribe:rm "${VPS[@]}" |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
cmdline::parse "$@" |
Boris Gallet