diff --git a/bin/0km b/bin/0km index 3cc5651..0b07a0b 100755 --- a/bin/0km +++ b/bin/0km @@ -259,7 +259,7 @@ vps_check() { fi /dev/null 2>&1 || + echo "$compose_content" | yq -e ".rsync-backup" >/dev/null 2>&1 || { echo "${DARKRED}no-backup${NORMAL}"; return 1; } } @@ -656,6 +656,126 @@ EOF } +NTFY_TOPIC_FILE="/etc/ntfy/topics.yml" +subscribe:ntfy:exists() { + local vps="$1" + if ! out=$(echo "[ -f \"$NTFY_TOPIC_FILE\" ] && echo ok || true" | \ + ssh:run "root@$vps" -- bash); then + err "Unable to check for existence of '$NTFY_TOPIC_FILE'." + fi + if [ -z "$out" ]; then + err "File '$NTFY_TOPIC_FILE' not found on $vps." + return 1 + fi +} + + +ntfy:rm() { + local channel="$1" topic="$2" vps="$3" + subscribe:ntfy:exists "$vps" || return 1 + if ! out=$(echo "yq -i 'del(.[\"$channel\"][] | select(. == \"$TOPIC\"))' \"$NTFY_TOPIC_FILE\"" | \ + ssh:run "root@$vps" -- bash); then + err "Failed to remove channel '$channel' from '$NTFY_TOPIC_FILE'." + return 1 + fi + info "Channel '$channel' removed from '$NTFY_TOPIC_FILE' on $vps." + ssh:run "root@$vps" -- cat "$NTFY_TOPIC_FILE" +} + +ntfy:add() { + local channel="$1" topic="$2" vps="$3" + vps_connection_check "$vps" &2 + exit 1 +fi + +if [ ! -f "/root/.ssh/ntfy-key" ]; then + cat $ntfy_key | gpg -d --batch --yes --passphrase 'uniquepass' > /root/.ssh/ntfy-key || >&2 echo "Error while copying ntfy key to root" +fi + +## Request token to ntfy server and add to config file +ntfy_host="core-01.0k.io" +if ! ssh-keygen -F $ntfy_host -f /root/.ssh/known_hosts >/dev/null; then + ssh-keyscan -H $ntfy_host >> /root/.ssh/known_hosts || >&2 echo "Error while adding ntfy server to known_hosts" +fi + +## if the config file doesn’t exist and LOGIN PASSWORD ARE not in we request them +config_file="/etc/ntfy/ntfy.conf" +mkdir -p "${config_file%/*}" +if [ -f "$config_file" ] || touch $config_file || { + echo "Error: couldn’t create config file $config_file" >&2; + exit 1 +}; then + ## if the config file is not complete we request new credentials + if ! grep -qE '^LOGIN=' "$config_file" || ! grep -qE '^PASSWORD=' "$config_file"; then + + cred=$(ssh -i /root/.ssh/ntfy-key ntfy@core-01.0k.io request-token) || >&2 echo "Error while requesting token to ntfy server" + login_ntfy=$(printf "%s" "${cred%$'\n'*}") + password_ntfy=$(printf "%s" "${cred#$'\n'*}") + + if [ -z "$login_ntfy" ] || [[ "$login_ntfy" == *$'\n'* ]]; then + echo "Error: couldn’t infer credential from ntfy server" >&2; + printf "%s" "$cred" | sed -r 's/^ |/g' >&2; + exit 1 + fi + + if grep -qE '^LOGIN=' "$config_file"; then + sed -i "s/^LOGIN=.*/LOGIN='$login'/" "$config_file" + else + echo "LOGIN='$login'" >> "$config_file" + fi + + if grep -qE '^PASSWORD=' "$config_file"; then + sed -i "s/^PASSWORD=.*/PASSWORD='$password'/" "$config_file" + else + echo "PASSWORD='$password'" >> "$config_file" + fi + else + echo "NTFY Config file is already complete" >&2; + fi +fi ## Marker to probe if this script finished it's job -echo "done" > /var/run/myc-installer.0k.io.state +echo "done" > /var/run/myc-installer.0k.io.state \ No newline at end of file diff --git a/bin/myc-update b/bin/myc-update index 745f385..b83fd34 100755 --- a/bin/myc-update +++ b/bin/myc-update @@ -96,7 +96,7 @@ for d in /etc/cron.{d,daily,hourly,monthly,weekly}; do ln -sfn "/opt/apps/myc-manage\$d/"* "\$d/" && find -L "\$d" -maxdepth 1 -type l -ilname "/opt/apps/myc-manage\$d/"\* -delete done -EOF +EOF Wrap -d "Updating sysctl scripts" <&2 + exit 1 +fi + +if [ ! -f "/root/.ssh/ntfy-key" ]; then + cat $ntfy_key | gpg -d --batch --yes --passphrase 'uniquepass' > /root/.ssh/ntfy-key || >&2 echo "Error while copying ntfy key to root" +fi + +## Request token to ntfy server and add to config file +ntfy_host="core-01.0k.io" +if ! ssh-keygen -F $ntfy_host -f /root/.ssh/known_hosts >/dev/null; then + ssh-keyscan -H $ntfy_host >> /root/.ssh/known_hosts || >&2 echo "Error while adding ntfy server to known_hosts" +fi + +## if the config file doesn’t exist and LOGIN PASSWORD ARE not in we request them +config_file="/etc/ntfy/ntfy.conf" +mkdir -p "${config_file%/*}" +if [ -f "$config_file" ] || touch $config_file || { + echo "Error: couldn’t create config file $config_file" >&2; + exit 1 +}; then + ## if the config file is not complete we request new credentials + if ! grep -qE '^LOGIN=' "$config_file" || ! grep -qE '^PASSWORD=' "$config_file"; then + + cred=$(ssh -i /root/.ssh/ntfy-key ntfy@core-01.0k.io request-token) || >&2 echo "Error while requesting token to ntfy server" + login_ntfy=$(printf "%s" "${cred%$'\n'*}") + password_ntfy=$(printf "%s" "${cred#$'\n'*}") + + if [ -z "$login_ntfy" ] || [[ "$login_ntfy" == *$'\n'* ]]; then + echo "Error: couldn’t infer credential from ntfy server" >&2; + printf "%s" "$cred" | sed -r 's/^ |/g' >&2; + exit 1 + fi + + if grep -qE '^LOGIN=' "$config_file"; then + sed -i "s/^LOGIN=.*/LOGIN='$login'/" "$config_file" + else + echo "LOGIN='$login'" >> "$config_file" + fi + + if grep -qE '^PASSWORD=' "$config_file"; then + sed -i "s/^PASSWORD=.*/PASSWORD='$password'/" "$config_file" + else + echo "PASSWORD='$password'" >> "$config_file" + fi + else + echo "NTFY Config file is already complete" >&2; + fi +fi +EOF for keyfile in {/root,/home/debian}/.ssh/authorized_keys; do [ -e "$keyfile" ] || continue diff --git a/bin/send b/bin/send new file mode 100755 index 0000000..ad86e0a --- /dev/null +++ b/bin/send @@ -0,0 +1,138 @@ +#!/bin/bash + +## Send a notification with NTFY and check if the config file is complete + +if [[ "$UID" == "0" ]]; then + NTFY_CONFIG_DIR="${NTFY_CONFIG_DIR:-/etc/ntfy}" +else + NTFY_CONFIG_DIR="${NTFY_CONFIG_DIR:-~/.config/ntfy}" +fi +NTFY_CONFIG_FILE="$NTFY_CONFIG_DIR/ntfy.conf" + +SERVER="https://ntfy.0k.io/" + +[ -f "$NTFY_CONFIG_DIR/topics.yml" ] || { + echo "Error: no 'topics.yml' file found in $NTFY_CONFIG_DIR" >&2 + echo " Please setup the topics for the notification channels in this file." >&2 + exit 1 +} + +if ! [ -e "$NTFY_CONFIG_FILE" ]; then + mkdir -p "${NTFY_CONFIG_FILE%/*}" + ## default option to change if needed + echo "SERVER=$SERVER" > "$NTFY_CONFIG_FILE" +elif ! grep -q "^SERVER=" "$NTFY_CONFIG_FILE"; then + echo "SERVER=$SERVER" >> "$NTFY_CONFIG_FILE" +fi + +source "$NTFY_CONFIG_FILE" + +for var in SERVER LOGIN PASSWORD; do + if ! [ -v "$var" ]; then + echo "Error: missing $var in $NTFY_CONFIG_FILE" + exit 1 + fi +done + + +exname=${0##*/} +channels=("main") + + +usage="Usage: $exname [-c CHANNEL...] [-t TITLE ] MESSAGE +---------------------------------------------- +--- Send MESSAGE with TITLE to the differents topics defined by a CHANNEL. --- +--- If no CHANNEL is provided, the message will be sent to the default channel. --- +---------------------------------------------- +-c CHANNEL: One or multiple channels. If no CHANNEL is provided, + the message will be sent to the main channel. + You can provide multiple channels with -c channel1 -c channel2 ... + topics are configured in $NTFY_CONFIG_DIR/topics.yml +-t TITLE: If no TITLE is provided, the message will be sent with the hostname as title. +- MESSAGE: The message to send. +" + +while [[ $# -gt 0 ]]; do + arg="$1" + shift + case "$arg" in + -h|--help) + echo "$usage" + exit 0 + ;; + -c|--channel) + [ -n "$1" ] || { + echo "Error: no argument for channel option." >&2 + echo "$usage" >&2 + exit 1 + } + IFS=", " channels+=($1) + shift + ;; + -t|--title) + title="$1" + [ -z "$title" ] || { + echo "Error: no argument for title option." >&2 + echo "$usage" >&2 + exit 1 + } + shift + ;; + *) + [ -z "$message" ] && { message="$arg"; continue; } + echo "Error : Unexpected positional argument '$arg'." >&2 + echo "$usage" >&2 + exit 1 + ;; + esac +done + +[ -n "$message" ] || { + echo "Error: missing message." >&2 + echo "$usage" >&2 + exit 1 +} + +read-0() { + local eof= IFS='' + while [ "$1" ]; do + read -r -d '' -- "$1" || eof=1 + shift + done + [ -z "$eof" ] +} + +curl_opts=( + -s + -u "$LOGIN:$PASSWORD" + -d "$message" +) + +if [ -n "$title" ]; then + curl_opts+=(-H "Title: [$(hostname)] $title") +fi + +declare -A sent_topic=() + +for channel in "${channels[@]}"; do + + channel_quoted=$(printf "%q" "$channel") + while read-0 topic; do + ttopic=$(printf "%s" "$topic" | yq "type") + if [ "$ttopic" != '!!str' ]; then + echo "Error: Unexpected '$ttopic' type for value of channel $channel." >&2 + exit 1 + fi + topic=$(printf "%s" "$topic" | yq -r " \"\" + .") + if ! [[ "$topic" =~ ^[a-zA-Z0-9\$\{\}*\ \,_.-]+$ ]]; then + echo "Error: Invalid topic value '$topic' expression in $channel channel." >&2 + exit 1 + fi + new_topics=($(eval echo "${topic//\*/\\*}")) + for new_topic in "${new_topics[@]}"; do + [ -n "${sent_topic["$new_topic"]}" ] && continue + sent_topic["$new_topic"]=1 + echo curl "${curl_opts[@]}" "$SERVER/${new_topic}" # > /dev/null + done + done < <(yq -0 -r=false -e ".[\"$channel_quoted\"] | .[]" "$NTFY_CONFIG_DIR/topics.yml") +done \ No newline at end of file diff --git a/bin/vps b/bin/vps index d8f9a4a..504f385 100755 --- a/bin/vps +++ b/bin/vps @@ -2615,4 +2615,73 @@ cmdline.spec:monujo:cmd:set-version:run() { } + +cmdline.spec::cmd:check:run() { + : +} + +cmdline.spec.gnu check +cmdline.spec:check:cmd:backup:run() { + + : :optfla: --notify,-n "Send result through notify" + : :optval: --service,-s "The backup service name (defaults to 'rsync-backup')" + + + ## Check on daily bases if backup exist in config and when is the last backup done : + ## ALERT if backup is set and last backup is older than 24h + + local STATE_FILE="/var/run/myc-manage/backup.state" + mkdir -p "${STATE_FILE%/*}" + + service=${opt_service:-rsync-backup} + project_name=$(compose:project_name) || exit 1 + + ## check if service exists in compose.yml + if ! compose:service:exists "$project_name" "$service"; then + warn "no service ${DARKYELLOW}$service${NORMAL}. Ignoring." + return 0 + fi + + last_backup_datetime=$( + cat /srv/datastore/data/cron/var/log/cron/*rsync-backup_script{_*,}.log | grep "total size is" | sort | tail -n 1 | cut -f -2 -d " ") + last_backup_ts=$(date -d "$last_backup_datetime" +%s) + max_ts=$(date -d "24 hours ago" +%s) + + state="ok" + if [ "$last_backup_ts" -lt "$max_ts" ]; then + state="ko" + fi + + if [ -z "$opt_notify" ]; then + if [ "$state" == "ok" ]; then + info "Everything is ${GREEN}ok${NORMAL}. (last backup: ${DARKCYAN}$last_backup_datetime${NORMAL})" + return 0 + fi + warn "Last backup older than 1 day. (last backup: ${DARKCYAN}$last_backup_datetime${NORMAL})" + return 1 + fi + + ## notify + + last_state=$(cat "$STATE_FILE" 2>/dev/null) || true + if [ "$state" == "$last_state" ]; then + [ "$state" == "ko" ] || return 0 + is_old=$(find "$STATE_FILE" -type f -mtime +2) || return 1 + [ -n "$is_old" ] || return 0 + fi + + echo "$state" > "$STATE_FILE" + + message="[$(hostname)]: WARNING no backup done in the last 24h (No backup since $days days and $hours hours)" + + timestamp=$(date +%s) + time_difference=$((timestamp - last_backup_ts)) + days=$((time_difference / 86400)) + hours=$((time_difference % 86400 / 3600)) + message="WARNING: no backup done in the last 24h (No backup since $days days and $hours hours)" + send -t "ALERT Backup" "$message" + +} + + cmdline::parse "$@" diff --git a/etc/cron.d/check-backup b/etc/cron.d/check-backup new file mode 100644 index 0000000..0960869 --- /dev/null +++ b/etc/cron.d/check-backup @@ -0,0 +1,4 @@ +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin + +12 * * * * root vps check backup -n 2>&1 | logger -t stats \ No newline at end of file diff --git a/etc/ssh/ntfy-key b/etc/ssh/ntfy-key new file mode 100644 index 0000000..b971d3d Binary files /dev/null and b/etc/ssh/ntfy-key differ