From b27f776ce6cf5e286e9108fe9655bcfb486058b6 Mon Sep 17 00:00:00 2001 From: Boris Gallet Date: Fri, 22 Mar 2024 10:55:10 +0100 Subject: [PATCH] new: [send] add send cmd, request auth on install and update, add vps check backup function and cron --- bin/0km | 163 +++++++++++++++++++++++++++++++++++++++- bin/myc-install | 55 +++++++++++++- bin/myc-update | 58 +++++++++++++- bin/send | 138 ++++++++++++++++++++++++++++++++++ bin/vps | 69 +++++++++++++++++ etc/cron.d/check-backup | 4 + etc/ssh/ntfy-key | Bin 0 -> 1458 bytes 7 files changed, 484 insertions(+), 3 deletions(-) create mode 100755 bin/send create mode 100644 etc/cron.d/check-backup create mode 100644 etc/ssh/ntfy-key diff --git a/bin/0km b/bin/0km index 3cc5651..0b07a0b 100755 --- a/bin/0km +++ b/bin/0km @@ -259,7 +259,7 @@ vps_check() { fi /dev/null 2>&1 || + echo "$compose_content" | yq -e ".rsync-backup" >/dev/null 2>&1 || { echo "${DARKRED}no-backup${NORMAL}"; return 1; } } @@ -656,6 +656,126 @@ EOF } +NTFY_TOPIC_FILE="/etc/ntfy/topics.yml" +subscribe:ntfy:exists() { + local vps="$1" + if ! out=$(echo "[ -f \"$NTFY_TOPIC_FILE\" ] && echo ok || true" | \ + ssh:run "root@$vps" -- bash); then + err "Unable to check for existence of '$NTFY_TOPIC_FILE'." + fi + if [ -z "$out" ]; then + err "File '$NTFY_TOPIC_FILE' not found on $vps." + return 1 + fi +} + + +ntfy:rm() { + local channel="$1" topic="$2" vps="$3" + subscribe:ntfy:exists "$vps" || return 1 + if ! out=$(echo "yq -i 'del(.[\"$channel\"][] | select(. == \"$TOPIC\"))' \"$NTFY_TOPIC_FILE\"" | \ + ssh:run "root@$vps" -- bash); then + err "Failed to remove channel '$channel' from '$NTFY_TOPIC_FILE'." + return 1 + fi + info "Channel '$channel' removed from '$NTFY_TOPIC_FILE' on $vps." + ssh:run "root@$vps" -- cat "$NTFY_TOPIC_FILE" +} + +ntfy:add() { + local channel="$1" topic="$2" vps="$3" + vps_connection_check "$vps" &2 + exit 1 +fi + +if [ ! -f "/root/.ssh/ntfy-key" ]; then + cat $ntfy_key | gpg -d --batch --yes --passphrase 'uniquepass' > /root/.ssh/ntfy-key || >&2 echo "Error while copying ntfy key to root" +fi + +## Request token to ntfy server and add to config file +ntfy_host="core-01.0k.io" +if ! ssh-keygen -F $ntfy_host -f /root/.ssh/known_hosts >/dev/null; then + ssh-keyscan -H $ntfy_host >> /root/.ssh/known_hosts || >&2 echo "Error while adding ntfy server to known_hosts" +fi + +## if the config file doesn’t exist and LOGIN PASSWORD ARE not in we request them +config_file="/etc/ntfy/ntfy.conf" +mkdir -p "${config_file%/*}" +if [ -f "$config_file" ] || touch $config_file || { + echo "Error: couldn’t create config file $config_file" >&2; + exit 1 +}; then + ## if the config file is not complete we request new credentials + if ! grep -qE '^LOGIN=' "$config_file" || ! grep -qE '^PASSWORD=' "$config_file"; then + + cred=$(ssh -i /root/.ssh/ntfy-key ntfy@core-01.0k.io request-token) || >&2 echo "Error while requesting token to ntfy server" + login_ntfy=$(printf "%s" "${cred%$'\n'*}") + password_ntfy=$(printf "%s" "${cred#$'\n'*}") + + if [ -z "$login_ntfy" ] || [[ "$login_ntfy" == *$'\n'* ]]; then + echo "Error: couldn’t infer credential from ntfy server" >&2; + printf "%s" "$cred" | sed -r 's/^ |/g' >&2; + exit 1 + fi + + if grep -qE '^LOGIN=' "$config_file"; then + sed -i "s/^LOGIN=.*/LOGIN='$login'/" "$config_file" + else + echo "LOGIN='$login'" >> "$config_file" + fi + + if grep -qE '^PASSWORD=' "$config_file"; then + sed -i "s/^PASSWORD=.*/PASSWORD='$password'/" "$config_file" + else + echo "PASSWORD='$password'" >> "$config_file" + fi + else + echo "NTFY Config file is already complete" >&2; + fi +fi ## Marker to probe if this script finished it's job -echo "done" > /var/run/myc-installer.0k.io.state +echo "done" > /var/run/myc-installer.0k.io.state \ No newline at end of file diff --git a/bin/myc-update b/bin/myc-update index 745f385..b83fd34 100755 --- a/bin/myc-update +++ b/bin/myc-update @@ -96,7 +96,7 @@ for d in /etc/cron.{d,daily,hourly,monthly,weekly}; do ln -sfn "/opt/apps/myc-manage\$d/"* "\$d/" && find -L "\$d" -maxdepth 1 -type l -ilname "/opt/apps/myc-manage\$d/"\* -delete done -EOF +EOF Wrap -d "Updating sysctl scripts" <&2 + exit 1 +fi + +if [ ! -f "/root/.ssh/ntfy-key" ]; then + cat $ntfy_key | gpg -d --batch --yes --passphrase 'uniquepass' > /root/.ssh/ntfy-key || >&2 echo "Error while copying ntfy key to root" +fi + +## Request token to ntfy server and add to config file +ntfy_host="core-01.0k.io" +if ! ssh-keygen -F $ntfy_host -f /root/.ssh/known_hosts >/dev/null; then + ssh-keyscan -H $ntfy_host >> /root/.ssh/known_hosts || >&2 echo "Error while adding ntfy server to known_hosts" +fi + +## if the config file doesn’t exist and LOGIN PASSWORD ARE not in we request them +config_file="/etc/ntfy/ntfy.conf" +mkdir -p "${config_file%/*}" +if [ -f "$config_file" ] || touch $config_file || { + echo "Error: couldn’t create config file $config_file" >&2; + exit 1 +}; then + ## if the config file is not complete we request new credentials + if ! grep -qE '^LOGIN=' "$config_file" || ! grep -qE '^PASSWORD=' "$config_file"; then + + cred=$(ssh -i /root/.ssh/ntfy-key ntfy@core-01.0k.io request-token) || >&2 echo "Error while requesting token to ntfy server" + login_ntfy=$(printf "%s" "${cred%$'\n'*}") + password_ntfy=$(printf "%s" "${cred#$'\n'*}") + + if [ -z "$login_ntfy" ] || [[ "$login_ntfy" == *$'\n'* ]]; then + echo "Error: couldn’t infer credential from ntfy server" >&2; + printf "%s" "$cred" | sed -r 's/^ |/g' >&2; + exit 1 + fi + + if grep -qE '^LOGIN=' "$config_file"; then + sed -i "s/^LOGIN=.*/LOGIN='$login'/" "$config_file" + else + echo "LOGIN='$login'" >> "$config_file" + fi + + if grep -qE '^PASSWORD=' "$config_file"; then + sed -i "s/^PASSWORD=.*/PASSWORD='$password'/" "$config_file" + else + echo "PASSWORD='$password'" >> "$config_file" + fi + else + echo "NTFY Config file is already complete" >&2; + fi +fi +EOF for keyfile in {/root,/home/debian}/.ssh/authorized_keys; do [ -e "$keyfile" ] || continue diff --git a/bin/send b/bin/send new file mode 100755 index 0000000..ad86e0a --- /dev/null +++ b/bin/send @@ -0,0 +1,138 @@ +#!/bin/bash + +## Send a notification with NTFY and check if the config file is complete + +if [[ "$UID" == "0" ]]; then + NTFY_CONFIG_DIR="${NTFY_CONFIG_DIR:-/etc/ntfy}" +else + NTFY_CONFIG_DIR="${NTFY_CONFIG_DIR:-~/.config/ntfy}" +fi +NTFY_CONFIG_FILE="$NTFY_CONFIG_DIR/ntfy.conf" + +SERVER="https://ntfy.0k.io/" + +[ -f "$NTFY_CONFIG_DIR/topics.yml" ] || { + echo "Error: no 'topics.yml' file found in $NTFY_CONFIG_DIR" >&2 + echo " Please setup the topics for the notification channels in this file." >&2 + exit 1 +} + +if ! [ -e "$NTFY_CONFIG_FILE" ]; then + mkdir -p "${NTFY_CONFIG_FILE%/*}" + ## default option to change if needed + echo "SERVER=$SERVER" > "$NTFY_CONFIG_FILE" +elif ! grep -q "^SERVER=" "$NTFY_CONFIG_FILE"; then + echo "SERVER=$SERVER" >> "$NTFY_CONFIG_FILE" +fi + +source "$NTFY_CONFIG_FILE" + +for var in SERVER LOGIN PASSWORD; do + if ! [ -v "$var" ]; then + echo "Error: missing $var in $NTFY_CONFIG_FILE" + exit 1 + fi +done + + +exname=${0##*/} +channels=("main") + + +usage="Usage: $exname [-c CHANNEL...] [-t TITLE ] MESSAGE +---------------------------------------------- +--- Send MESSAGE with TITLE to the differents topics defined by a CHANNEL. --- +--- If no CHANNEL is provided, the message will be sent to the default channel. --- +---------------------------------------------- +-c CHANNEL: One or multiple channels. If no CHANNEL is provided, + the message will be sent to the main channel. + You can provide multiple channels with -c channel1 -c channel2 ... + topics are configured in $NTFY_CONFIG_DIR/topics.yml +-t TITLE: If no TITLE is provided, the message will be sent with the hostname as title. +- MESSAGE: The message to send. +" + +while [[ $# -gt 0 ]]; do + arg="$1" + shift + case "$arg" in + -h|--help) + echo "$usage" + exit 0 + ;; + -c|--channel) + [ -n "$1" ] || { + echo "Error: no argument for channel option." >&2 + echo "$usage" >&2 + exit 1 + } + IFS=", " channels+=($1) + shift + ;; + -t|--title) + title="$1" + [ -z "$title" ] || { + echo "Error: no argument for title option." >&2 + echo "$usage" >&2 + exit 1 + } + shift + ;; + *) + [ -z "$message" ] && { message="$arg"; continue; } + echo "Error : Unexpected positional argument '$arg'." >&2 + echo "$usage" >&2 + exit 1 + ;; + esac +done + +[ -n "$message" ] || { + echo "Error: missing message." >&2 + echo "$usage" >&2 + exit 1 +} + +read-0() { + local eof= IFS='' + while [ "$1" ]; do + read -r -d '' -- "$1" || eof=1 + shift + done + [ -z "$eof" ] +} + +curl_opts=( + -s + -u "$LOGIN:$PASSWORD" + -d "$message" +) + +if [ -n "$title" ]; then + curl_opts+=(-H "Title: [$(hostname)] $title") +fi + +declare -A sent_topic=() + +for channel in "${channels[@]}"; do + + channel_quoted=$(printf "%q" "$channel") + while read-0 topic; do + ttopic=$(printf "%s" "$topic" | yq "type") + if [ "$ttopic" != '!!str' ]; then + echo "Error: Unexpected '$ttopic' type for value of channel $channel." >&2 + exit 1 + fi + topic=$(printf "%s" "$topic" | yq -r " \"\" + .") + if ! [[ "$topic" =~ ^[a-zA-Z0-9\$\{\}*\ \,_.-]+$ ]]; then + echo "Error: Invalid topic value '$topic' expression in $channel channel." >&2 + exit 1 + fi + new_topics=($(eval echo "${topic//\*/\\*}")) + for new_topic in "${new_topics[@]}"; do + [ -n "${sent_topic["$new_topic"]}" ] && continue + sent_topic["$new_topic"]=1 + echo curl "${curl_opts[@]}" "$SERVER/${new_topic}" # > /dev/null + done + done < <(yq -0 -r=false -e ".[\"$channel_quoted\"] | .[]" "$NTFY_CONFIG_DIR/topics.yml") +done \ No newline at end of file diff --git a/bin/vps b/bin/vps index d8f9a4a..504f385 100755 --- a/bin/vps +++ b/bin/vps @@ -2615,4 +2615,73 @@ cmdline.spec:monujo:cmd:set-version:run() { } + +cmdline.spec::cmd:check:run() { + : +} + +cmdline.spec.gnu check +cmdline.spec:check:cmd:backup:run() { + + : :optfla: --notify,-n "Send result through notify" + : :optval: --service,-s "The backup service name (defaults to 'rsync-backup')" + + + ## Check on daily bases if backup exist in config and when is the last backup done : + ## ALERT if backup is set and last backup is older than 24h + + local STATE_FILE="/var/run/myc-manage/backup.state" + mkdir -p "${STATE_FILE%/*}" + + service=${opt_service:-rsync-backup} + project_name=$(compose:project_name) || exit 1 + + ## check if service exists in compose.yml + if ! compose:service:exists "$project_name" "$service"; then + warn "no service ${DARKYELLOW}$service${NORMAL}. Ignoring." + return 0 + fi + + last_backup_datetime=$( + cat /srv/datastore/data/cron/var/log/cron/*rsync-backup_script{_*,}.log | grep "total size is" | sort | tail -n 1 | cut -f -2 -d " ") + last_backup_ts=$(date -d "$last_backup_datetime" +%s) + max_ts=$(date -d "24 hours ago" +%s) + + state="ok" + if [ "$last_backup_ts" -lt "$max_ts" ]; then + state="ko" + fi + + if [ -z "$opt_notify" ]; then + if [ "$state" == "ok" ]; then + info "Everything is ${GREEN}ok${NORMAL}. (last backup: ${DARKCYAN}$last_backup_datetime${NORMAL})" + return 0 + fi + warn "Last backup older than 1 day. (last backup: ${DARKCYAN}$last_backup_datetime${NORMAL})" + return 1 + fi + + ## notify + + last_state=$(cat "$STATE_FILE" 2>/dev/null) || true + if [ "$state" == "$last_state" ]; then + [ "$state" == "ko" ] || return 0 + is_old=$(find "$STATE_FILE" -type f -mtime +2) || return 1 + [ -n "$is_old" ] || return 0 + fi + + echo "$state" > "$STATE_FILE" + + message="[$(hostname)]: WARNING no backup done in the last 24h (No backup since $days days and $hours hours)" + + timestamp=$(date +%s) + time_difference=$((timestamp - last_backup_ts)) + days=$((time_difference / 86400)) + hours=$((time_difference % 86400 / 3600)) + message="WARNING: no backup done in the last 24h (No backup since $days days and $hours hours)" + send -t "ALERT Backup" "$message" + +} + + cmdline::parse "$@" diff --git a/etc/cron.d/check-backup b/etc/cron.d/check-backup new file mode 100644 index 0000000..0960869 --- /dev/null +++ b/etc/cron.d/check-backup @@ -0,0 +1,4 @@ +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin + +12 * * * * root vps check backup -n 2>&1 | logger -t stats \ No newline at end of file diff --git a/etc/ssh/ntfy-key b/etc/ssh/ntfy-key new file mode 100644 index 0000000000000000000000000000000000000000..b971d3dd25b80ac1675c79abf36e1a3174cedafb GIT binary patch literal 1458 zcmV;j1x@;l4Fm}T0^7du>h-j)DF4#x0gDX*#5D)Ew5*OOtRQwr5acd>9IbY7=o&Sk z$(umP-(G~yHzZ^vvzPBT+`m@C1bRocFS?|1&3+F}US|*zaXMR=qmJ!tvvma4w&dRW z7yuyasRc|=`PiYHumZ82AMn5c3!whVlS$+46sfW?R~D6i#Oc?3HNE0R=;xkqp3cBt zIpclUIc}+$2=d0lZAjNk-E+uKb;p;_?(4IrR{IsCcDkN510bF>G5Y(+KUa1Zaj-LF zJSVu?a8S7By-%5u1`-l=G&*!PYfW%}y%a+O@6YHX-y#A8Uu=tW0;28#WY@p&8)KBU zV1rY-kepWf?U4@b2jzexFfEAj6~y{{>k)ff*KC8Ll|Cx-Gr18RLGckqplnz^+9yN> z3pVOLZe1(Rl5XI2+E~oJ7NcC(cvuc`Ly8G_gqG!0v;|Xdql|X%DZ>ascd*pk8{%tp zJ5$@<)dvooLVDu-q4tV#w+E>$jNhMssrXhN*PsNI>vL>c{wB|{Zv63XWAXi2hJ?of zSLk#vU517^W!uNin#3O)l0O;jUDZ{h+N8}ISsTa|N3HzZmzx6t#)m`U8R>;b=+59(%t)pKX* zF5Ub)y7D1MC{{^PrzcnOhMs^(JjQtA25Q=8?+3G(zuq1dt0=j?%<)-_g|9JyDWq(U z>t->BAgWI8{@C6^^lji)-*AdzQ9n@M^!dw2%Rra)h{~R}3sIIk+MjnTOG|umS55e& zy`QQ88p1D*reMx8*Ug!?8X;}PnXDSk-zw%0at;-(zaZ{JTmpWM@5kAmL;}t|&EkGS zU-PFt4ZylUNL#L}L1(|720}5+pg1n^w|2)+sR>Z!GDY6nSA_6z=!;w`Zs6EYc{&Fi zLw2S>?GFZsizi#Gelc;xlHKWHdezLAk$CG4Z3kF#pxnV}(q5rRO{Ro*od^X`&YrlG zmTjYL8+HufL(250^C!FRZLwU1Tmk~{tQa>0?1!I}P!?fxF# zfy2-CgDnhA?u$I`f(=<`v+(Sd^upvqlBjyU$_*TCqgL%3g)mZafr5S$OJ50+0H zc7(uPfOf~gObI{imn|qVX?+=|!Fv#kBXdhAXy_dw&GRQPUVmy&eb?dO zF$K2P7ycajz~4@Q6q{DCZX(K_onvQlo7i2Kc65z9jVGzx_K;g_3Ay>na4^oFZ}!oY z=uw1|2SRs^TfGrB2q|QT!ZbDywnkMEq9b3LbGC%1avW7mdqdxPSgUf$%m&(k76?>2 z>IT9rnNIX$i#?lvci3_F%^o52h?|<^^oVsTx=4iYU)K04SYQTbR8}qD>BUo)P8I`` zA6NvYWtR@6koiJ>t4-<}w|8ujfsSS42(%cxNIDp_*#;saao8FHdFDCJ0(!Uo=yH@R z)~|u4-T4zJth7X(cSMB6a(@cXVHd}{no@3x$o1CfkgJF4M$$K)aN-8rT;8X-4w>zbtnS5#?)0y@w5!%b zUHpW8p*13{l(EeN1OXZk9Vwu`4S?XG?6@qi_~swDfKQ!9-y>GOlzaOUdXX=Vs~BaZ z!O>{OvYOKFbcwmNr2^4v(Cd4uTpOj9viX0G#LklfF6!UZ?(jze)^``h