new: [vps] add ``check backup`` command

bin/0km

@@ -259,7 +259,7 @@ vps_check() {
     fi </dev/null
     compose_content=$(ssh:run "root@$vps" -- cat /opt/apps/myc-deploy/compose.yml </dev/null) ||
         { echo "${DARKRED}no-compose${NORMAL}"; return 1; }
-    echo "$compose_content" | grep backup >/dev/null 2>&1 ||
+    echo "$compose_content" | yq -e ".rsync-backup" >/dev/null 2>&1 ||
         { echo "${DARKRED}no-backup${NORMAL}"; return 1; }
 }
 
@@ -656,6 +656,156 @@ EOF
 
 }
 
+NTFY_TOPIC_FILE="/etc/ntfy/topics.yml"
+NTFY_CONFIG_FILE="/etc/ntfy/ntfy.conf"
+subscribe:ntfy:topic-file-exists() {
+    local vps="$1"
+    if ! out=$(echo "[ -f \"$NTFY_TOPIC_FILE\" ] && echo ok || true" | \
+                   ssh:run "root@$vps" -- bash); then
+        err "Unable to check for existence of '$NTFY_TOPIC_FILE'."
+    fi
+    if [ -z "$out" ]; then
+        err "File '$NTFY_TOPIC_FILE' not found on $vps."
+        return 1
+    fi
+}
+
+subscribe:ntfy:config-file-exists() {
+    local vps="$1"
+    if ! out=$(echo "[ -f \"$NTFY_CONFIG_FILE\" ] && echo ok || true" | \
+                   ssh:run "root@$vps" -- bash); then
+        err "Unable to check for existence of '$NTFY_CONFIG_FILE'."
+    fi
+    if [ -z "$out" ]; then
+        err "File '$NTFY_CONFIG_FILE' not found on $vps."
+        return 1
+    fi
+}
+
+
+ntfy:rm() {
+    local channel="$1" topic="$2" vps="$3"
+    subscribe:ntfy:topic-file-exists "$vps" || return 1
+    if ! out=$(echo "yq -i 'del(.[\"$channel\"][] | select(. == \"$TOPIC\"))' \"$NTFY_TOPIC_FILE\"" | \
+                   ssh:run "root@$vps" -- bash); then
+        err "Failed to remove channel '$channel' from '$NTFY_TOPIC_FILE'."
+        return 1
+    fi
+    info "Channel '$channel' removed from '$NTFY_TOPIC_FILE' on $vps."
+    ssh:run "root@$vps" -- cat "$NTFY_TOPIC_FILE"
+}
+
+ntfy:add() {
+    local channel="$1" topic="$2" vps="$3"
+    vps_connection_check "$vps" </dev/null || return 1
+    subscribe:ntfy:topic-file-exists "$vps" || return 1
+    if ! out=$(echo "yq '. | has(\"$channel\")' \"$NTFY_TOPIC_FILE\"" | \
+                   ssh:run "root@$vps" -- bash); then
+        err "Failed to check if channel '$channel' with topic '$topic' is already in '$NTFY_TOPIC_FILE'."
+        return 1
+    fi
+    if [ "$out" != "true" ]; then
+        ## Channel does not exist
+        if ! out=$(echo "yq -i '.[\"$channel\"] = []' \"$NTFY_TOPIC_FILE\"" | \
+                       ssh:run "root@$vps" -- bash); then
+            err "Failed to create a new channel '$channel' entry in '$NTFY_TOPIC_FILE'."
+            return 1
+        fi
+    else
+        ## Channel exists
+        if ! out=$(echo "yq '.[\"$channel\"] | any_c(. == \"$topic\")' \"$NTFY_TOPIC_FILE\"" | \
+                       ssh:run "root@$vps" -- bash); then
+            err "Failed to check if channel '$channel' with topic '$topic' is already in '$NTFY_TOPIC_FILE'."
+            return 1
+        fi
+        if [ "$out" == "true" ]; then
+            info "Channel '$channel' with topic '$topic' already exists in '$NTFY_TOPIC_FILE'."
+            return 0
+        fi
+    fi
+    if ! out=$(echo "yq -i '.[\"$channel\"] += [\"$topic\"]' \"$NTFY_TOPIC_FILE\"" | \
+                   ssh:run "root@$vps" -- bash); then
+        err "Failed to add channel '$channel' with topic '$topic' to '$NTFY_TOPIC_FILE'."
+        return 1
+    fi
+    info "Channel '$channel' added with topic '$topic' to '$NTFY_TOPIC_FILE' on $vps."
+}
+
+NTFY_BROKER_SERVER="ntfy.0k.io"
+ntfy:topic-access() {
+    local action="$1" topic="$2" vps="$3"
+    subscribe:ntfy:config-file-exists "$vps" || return 1
+
+    local user
+    user=$(ntfy:get-login "$vps") || return 1
+    case "$action" in
+        "write")
+            ssh "ntfy@$NTFY_BROKER_SERVER" "topic-access" \
+                "$user" "$topic" "write-only" </dev/null || {
+                err "Failed to grant write access to '$user' for topic '$topic'."
+                return 1
+            }
+            info "Granted write access for '$user' to topic '$topic'."
+            ;;
+        "remove")
+            ssh "ntfy@$NTFY_BROKER_SERVER" "topic-access" -r "$user" "$topic" </dev/null || {
+                err "Failed to reset access of '$user' for topic '$topic'."
+                return 1
+            }
+            info "Access for '$user' to topic '$topic' was resetted successfully."
+            ;;
+        *)
+            err "Invalid action '$action'."
+            return 1
+            ;;
+    esac
+}
+
+ntfy:get-login() {
+    local vps="$1"
+    if ! out=$(echo ". \"$NTFY_CONFIG_FILE\" && echo \"\$LOGIN\"" | \
+                   ssh:run "root@$vps" -- bash); then
+        err "Failed to get ntfy login from '$NTFY_CONFIG_FILE'."
+        return 1
+    fi
+    if [ -z "$out" ]; then
+        err "Unexpected empty login retrieved from sourcing '$NTFY_CONFIG_FILE'."
+        return 1
+    fi
+    echo "$out"
+}
+
+subscribe:add() {
+    local vps="$1"
+    read-0 channel topic || {
+        err "Couldn't read CHANNEL and TOPIC arguments."
+        return 1
+    }
+    vps_connection_check "$vps" </dev/null || return 1
+    ntfy:topic-access "write" "$topic" "$vps" </dev/null || return 1
+    ntfy:add "$channel" "$topic" "$vps" || {
+        err "Failed to add channel '$channel' with topic '$topic' to '$NTFY_TOPIC_FILE'."
+        echo "  Removing topic access." >&2
+        ntfy:topic-access "remove" "$topic" "$vps" </dev/null
+        return 1
+    }
+}
+
+subscribe:rm() {
+    local vps="$1"
+    read-0 channel topic || {
+        err "Couldn't read CHANNEL and TOPIC arguments."
+        return 1
+    }
+    vps_connection_check "$vps" </dev/null || return 1
+
+    ntfy:rm "$channel" "$topic" "$vps" || return 1
+    ntfy:topic-access "remove" "$topic" "$vps" </dev/null || {
+        err "Failed to remove topic access for '$topic' on '$vps'."
+        return 1
+    }
+}
+
 vps_backup_recover() {
     local vps="$1" admin server id path rtype force type
 
@@ -1594,4 +1744,45 @@ graph:def:load_avg() {
 }
 
 
+
+
+cmdline.spec.gnu vps-subscribe
+cmdline.spec::cmd:vps-subscribe:run() {
+    :
+}
+cmdline.spec.gnu add
+cmdline.spec:vps-subscribe:cmd:add:run() {
+
+    : :posarg: CHANNEL         'Channel which will be sent to given topic'
+    : :posarg: TOPIC           'Ntfy topic to recieve messages of given channel
+                               (format: "[MYSERVER:]MYTOPICS"
+                                 Examples: "ntfy.0k.io:main,storage,alerts",
+                                 "main{1,3,7}"
+                               )'
+    : :posarg: [VPS...]        'Target host(s) to get stats'
+
+    printf "%s\0" "$CHANNEL" "$TOPIC" |
+        vps_mux subscribe:add  "${VPS[@]}"
+
+}
+
+cmdline.spec.gnu rm
+cmdline.spec:vps-subscribe:cmd:rm:run() {
+
+    : :posarg: CHANNEL         'Channel which will be sent to given topic'
+    : :posarg: TOPIC           'Ntfy topic to recieve messages of given channel
+                               (format: "[MYSERVER:]MYTOPICS"
+                                 Examples: "ntfy.0k.io:main,storage,alerts",
+                                 "main{1,3,7}"
+                               )'
+    : :posarg: [VPS...]        'Target host(s) to get stats'
+
+
+    printf "%s\0" "$CHANNEL" "$TOPIC" |
+        vps_mux subscribe:rm "${VPS[@]}"
+
+}
+
+
+
 cmdline::parse "$@"

bin/vps

@@ -2615,4 +2615,70 @@ cmdline.spec:monujo:cmd:set-version:run() {
 }
 
 
+
+cmdline.spec::cmd:check:run() {
+    :
+}
+
+cmdline.spec.gnu check
+cmdline.spec:check:cmd:backup:run() {
+
+    : :optfla: --notify,-n      "Send result through notify"
+    : :optval: --service,-s      "The backup service name (defaults to 'rsync-backup')"
+
+    ## ALERT if backup is set and last backup is older than 24h
+
+    local STATE_FILE="/var/run/myc-manage/backup.state"
+    mkdir -p "${STATE_FILE%/*}"
+
+    service=${opt_service:-rsync-backup}
+    project_name=$(compose:project_name) || exit 1
+
+    ## check if service exists in compose.yml
+    if ! compose:service:exists "$project_name" "$service"; then
+        warn "no service ${DARKYELLOW}$service${NORMAL}. Ignoring."
+        return 0
+    fi
+
+    last_backup_datetime=$(
+        cat /srv/datastore/data/cron/var/log/cron/*rsync-backup_script{_*,}.log | grep "total size is" | sort | tail -n 1 | cut -f -2 -d " ")
+    last_backup_ts=$(date -d "$last_backup_datetime" +%s)
+    max_ts=$(date -d "24 hours ago" +%s)
+
+    state="ok"
+    if [ "$last_backup_ts" -lt "$max_ts" ]; then
+        state="ko"
+    fi
+
+    if [ -z "$opt_notify" ]; then
+        if [ "$state" == "ok" ]; then
+            info "Everything is ${GREEN}ok${NORMAL}. (last backup: ${DARKCYAN}$last_backup_datetime${NORMAL})"
+            return 0
+        fi
+        warn "Last backup older than 1 day. (last backup: ${DARKCYAN}$last_backup_datetime${NORMAL})"
+        return 1
+    fi
+
+    ## notify
+
+    last_state=$(cat "$STATE_FILE" 2>/dev/null) || true
+    if [ "$state" == "$last_state" ]; then
+       [ "$state" == "ko" ] || return 0
+       is_old=$(find "$STATE_FILE" -type f -mtime +2) || return 1
+       [ -n "$is_old" ] || return 0
+    fi
+
+    echo "$state" > "$STATE_FILE"
+
+    message="[$(hostname)]: WARNING no backup done in the last 24h (No backup since $days days and $hours hours)"
+
+    timestamp=$(date +%s)
+    time_difference=$((timestamp - last_backup_ts))
+    days=$((time_difference / 86400))
+    hours=$((time_difference % 86400 / 3600))
+    message="WARNING: no backup done in the last 24h (No backup since $days days and $hours hours)"
+    send -c check.alert -t "ALERT Backup" "$message"
+}
+
+
 cmdline::parse "$@"

etc/cron.d/check-backup

@@ -0,0 +1,4 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+12 * * * *   root   vps check backup -n 2>&1 | logger -t stats