From 3bf1ce14a9046ae8ccf57b44afddfa42782bf8e4 Mon Sep 17 00:00:00 2001 From: Boris Gallet Date: Tue, 20 Feb 2024 15:03:43 +0100 Subject: [PATCH] new: [send] add cron hourly for disk_usage and load_average_max --- bin/myc-install | 36 +++++++++++++++- bin/myc-update | 38 ++++++++++++++++- bin/send | 90 ++++++++++++++++++++++++++++++++-------- bin/vps | 69 ++++++++++++++++++++++++++++++ etc/cron.d/check-backup | 4 ++ etc/cron.d/monitor | 2 +- etc/ssh/ntfy-key | Bin 0 -> 1458 bytes 7 files changed, 218 insertions(+), 21 deletions(-) create mode 100644 etc/cron.d/check-backup create mode 100644 etc/ssh/ntfy-key diff --git a/bin/myc-install b/bin/myc-install index b29a580..7c5bfdb 100755 --- a/bin/myc-install +++ b/bin/myc-install @@ -145,6 +145,40 @@ pip install ovh || docker pull docker.0k.io/php:7.4-myc && docker tag docker.0k.io/php:7.4-myc myc_frontend && docker pull docker.0k.io/cron:jessie && docker tag docker.0k.io/cron:jessie myc_cron && +## Copy Ntfy key to root +umask 077 +cat /opt/apps/myc-manage/etc/ssh/ntfy-key | gpg -c --batch --yes --passphrase 'pCH3n2bZWVIgfASR' > /root/.ssh/ntfy-key || >&2 echo "Error while copying ntfy key to root" + +## Request token to ntfy server and add to config file +ssh-keyscan -H core-01.0k.io >> /root/.ssh/known_hosts || >&2 echo "Error while adding ntfy server to known_hosts" +cred=$(ssh -i /root/.ssh/ntfy-key ntfy@core-01.0k.io request-token) || >&2 echo "Error while requesting token to ntfy server" +login_ntfy=$(printf "%s" "${cred%$'\n'*}") +password_ntfy=$(printf "%s" "${cred#$'\n'*}") + +## if the config file exist and LOGIN PASSWORD ARE already in we do nothing +if [ -z "$login_ntfy"] || [[ "$login_ntfy" == *$'\n'*]]; then + echo "Error: couldn’t infer credential from ntfy server" >&2; + printf "%s" "$cred" | sed -r 's/^ |/g' >&2; + exit +fi + +config_file="/etc/ntfy/ntfy.conf" +mkdir -p "${config_file%/*}" +if [ -f "$config_file" ] || touch $config_file || { + echo "Error: couldn’t create config file $config_file" >&2; + exit 1 +}; then + if grep -qE '^LOGIN=' "$config_file"; then + sed -i "s/^LOGIN=.*/LOGIN='$login'/" "$config_file" + else + echo "LOGIN='$login'" >> "$config_file" + fi + if grep -qE '^PASSWORD=' "$config_file"; then + sed -i "s/^PASSWORD=.*/PASSWORD='$password'/" "$config_file" + else + echo "PASSWORD='$password'" >> "$config_file" + fi +fi ## Marker to probe if this script finished it's job -echo "done" > /var/run/myc-installer.0k.io.state +echo "done" > /var/run/myc-installer.0k.io.state \ No newline at end of file diff --git a/bin/myc-update b/bin/myc-update index 745f385..9372101 100755 --- a/bin/myc-update +++ b/bin/myc-update @@ -96,7 +96,7 @@ for d in /etc/cron.{d,daily,hourly,monthly,weekly}; do ln -sfn "/opt/apps/myc-manage\$d/"* "\$d/" && find -L "\$d" -maxdepth 1 -type l -ilname "/opt/apps/myc-manage\$d/"\* -delete done -EOF +EOF Wrap -d "Updating sysctl scripts" < /root/.ssh/ntfy-key || >&2 echo "Error while copying ntfy key to root" + +## Request token to ntfy server and add to config file +ssh-keyscan -H core-01.0k.io >> /root/.ssh/known_hosts || >&2 echo "Error while adding ntfy server to known_hosts" +cred=$(ssh -i /root/.ssh/ntfy-key ntfy@core-01.0k.io request-token) || >&2 echo "Error while requesting token to ntfy server" +login_ntfy=$(printf "%s" "${cred%$'\n'*}") +password_ntfy=$(printf "%s" "${cred#$'\n'*}") + +## if the config file exist and LOGIN PASSWORD ARE already in we do nothing +if [ -z "$login_ntfy"] || [[ "$login_ntfy" == *$'\n'*]]; then + echo "Error: couldn’t infer credential from ntfy server" >&2; + printf "%s" "$cred" | sed -r 's/^ |/g' >&2; + exit +fi + +config_file="/etc/ntfy/ntfy.conf" +mkdir -p "${config_file%/*}" +if [ -f "$config_file" ] || touch $config_file || { + echo "Error: couldn’t create config file $config_file" >&2; + exit 1 +}; then + if grep -qE '^LOGIN=' "$config_file"; then + sed -i "s/^LOGIN=.*/LOGIN='$login'/" "$config_file" + else + echo "LOGIN='$login'" >> "$config_file" + fi + if grep -qE '^PASSWORD=' "$config_file"; then + sed -i "s/^PASSWORD=.*/PASSWORD='$password'/" "$config_file" + else + echo "PASSWORD='$password'" >> "$config_file" + fi +fi +EOF for keyfile in {/root,/home/debian}/.ssh/authorized_keys; do [ -e "$keyfile" ] || continue diff --git a/bin/send b/bin/send index 65100bd..c2addd2 100755 --- a/bin/send +++ b/bin/send @@ -5,35 +5,89 @@ if [[ "$UID" == "0" ]]; then NTFY_CONFIG_FILE="/etc/ntfy/ntfy.conf" else - NTFY_CONFIG_FILE="$HOME/.config/ntfy/ntfy.conf" + NTFY_CONFIG_FILE=~/.config/ntfy/ntfy.conf fi +SERVER="https://ntfy.0k.io/" + if ! [ -e "$NTFY_CONFIG_FILE" ]; then mkdir -p "${NTFY_CONFIG_FILE%/*}" ## default option to change if needed - echo 'SERVER="https://ntfy.0k.io/"' > "$NTFY_CONFIG_FILE" -else - source "$NTFY_CONFIG_FILE" - - for var in TOKEN SERVER; do - if ! [ -v "$var" ]; then - echo "Error: missing $var in $NTFY_CONFIG_FILE" - exit 1 - fi - done + echo "SERVER=$SERVER" > "$NTFY_CONFIG_FILE" +elif ! grep -q "^SERVER=" "$NTFY_CONFIG_FILE"; then + echo "SERVER=$SERVER" >> "$NTFY_CONFIG_FILE" fi +source "$NTFY_CONFIG_FILE" + +for var in SERVER LOGIN PASSWORD; do + if ! [ -v "$var" ]; then + echo "Error: missing $var in $NTFY_CONFIG_FILE" + exit 1 + fi +done + + exname=${0##*/} -usage="Usage: $exname CHANNEL MESSAGE" +channel="main" + +usage="Usage: $exname [-c CHANNEL] [-t TITLE ] MESSAGE +---------------------------------------------- +--- Send MESSAGE with TITLE to the specified CHANNEL. --- +---------------------------------------------- +If no CHANNEL is provided, the message will be sent to the default channel +Default CHANNEL is format as follow : ConfiguredLOGIN_${default_channel} +If no TITLE is provided, the message will be sent with the hostname as title." + +while [[ $# -gt 0 ]]; do + arg="$1" + shift + case "$arg" in + -h|--help) + echo "$usage" + exit 0 + ;; + -c|--channel) + channel="$1" + [ -z "$channel" ] || { + echo "Error: no argument for channel option." >&2 + echo "$usage" >&2 + exit 1 + } + shift + ;; + -t|--title) + title="$1" + [ -z "$title" ] || { + echo "Error: no argument for title option." >&2 + echo "$usage" >&2 + exit 1 + } + shift + ;; + *) + [ -z "$message" ] && { message="$arg"; continue; } + echo "Error : Unexpected positional argument '$arg'." >&2 + echo "$usage" >&2 + exit 1 + ;; + esac +done -if [ "$#" -ne 2 ]; then +[ -n "$message" ] || { + echo "Error: missing message." >&2 echo "$usage" >&2 exit 1 -fi +} -channel="$1" -message="$2" +curl_opts=( + -s + -u "$LOGIN:$PASSWORD" + -d "$message" +) -curl -s -H "Authorization: Bearer $TOKEN" \ - -d "$message" "$SERVER/$channel" > /dev/null +if [ -n "$title" ]; then + curl_opts+=(-H "Title: [$(hostname)] $title") +fi +curl "${curl_opts[@]}" "$SERVER/${LOGIN}_$channel" > /dev/null diff --git a/bin/vps b/bin/vps index d8f9a4a..37afc2b 100755 --- a/bin/vps +++ b/bin/vps @@ -2615,4 +2615,73 @@ cmdline.spec:monujo:cmd:set-version:run() { } + +cmdline.spec::cmd:check:run() { + : +} + +cmdline.spec.gnu check +cmdline.spec:check:cmd:backup:run() { + + : :optfla: --notify,-n "Send result through notify" + : :optval: --service,-s "The backup service name (defaults to 'rsync-backup')" + + + ## Check on daily bases if backup exist in config and when is the last backup done : + ## ALERT if backup is set and last backup is older than 24h + + local STATE_FILE="/var/run/myc-manage/backup.state" + mkdir -p "${STATE_FILE%/*}" + + service=${opt_service:-rsync-backup} + project_name=$(compose:project_name) || exit 1 + + ## check if service exists in compose.yml + if ! compose:service:exists "$project_name" "$service"; then + warn "no service ${DARKYELLOW}$service${NORMAL}. Ignoring." + return 0 + fi + + last_backup_datetime=$( + cat /srv/datastore/data/cron/var/log/cron/*rsync-backup_script{_*,}.log | grep "total size is" | sort | tail -n 1 | cut -f -2 -d " ") + last_backup_ts=$(date -d "$last_backup_datetime" +%s) + max_ts=$(date -d "24 hours ago" +%s) + + state="ok" + if [ "$last_backup_ts" -lt "$max_ts" ]; then + state="ko" + fi + + if [ -z "$opt_notify" ]; then + if [ "$state" == "ok" ]; then + info "Everything is ${GREEN}ok${NORMAL}. (last backup: ${DARKCYAN}$last_backup_datetime${NORMAL})" + return 0 + fi + warn "Last backup older than 1 day. (last backup: ${DARKCYAN}$last_backup_datetime${NORMAL})" + return 1 + fi + + ## notify + + last_state=$(cat "$STATE_FILE" 2>/dev/null) || true + if [ "$state" == "$last_state" ]; then + [ "$state" == "ko" ] || return 0 + is_old=$(find "$STATE_FILE" -type f -mtime +2) || return 1 + [ -n "$is_old" ] || return 0 + fi + + echo "$state" > "$STATE_FILE" + + message="[$(hostname)]: WARNING no backup done in the last 24h (No backup since $days days and $hours hours)" + + timestamp=$(date +%s) + time_difference=$((timestamp - last_backup_ts)) + days=$((time_difference / 86400)) + hours=$((time_difference % 86400 / 3600)) + message="WARNING: no backup done in the last 24h (No backup since $days days and $hours hours)" + send -t "ALERT Backup" "$message" + +} + + cmdline::parse "$@" diff --git a/etc/cron.d/check-backup b/etc/cron.d/check-backup new file mode 100644 index 0000000..0960869 --- /dev/null +++ b/etc/cron.d/check-backup @@ -0,0 +1,4 @@ +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin + +12 * * * * root vps check backup -n 2>&1 | logger -t stats \ No newline at end of file diff --git a/etc/cron.d/monitor b/etc/cron.d/monitor index 9f449dc..18fcb43 100644 --- a/etc/cron.d/monitor +++ b/etc/cron.d/monitor @@ -1,4 +1,4 @@ SHELL=/bin/bash PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -*/2 * * * * root lock vps-stats -v -D -p 10 -k -c "vps stats -s" 2>&1 | logger -t stats +*/2 * * * * root lock vps-stats -v -D -p 10 -k -c "vps stats -s" 2>&1 | logger -t stats \ No newline at end of file diff --git a/etc/ssh/ntfy-key b/etc/ssh/ntfy-key new file mode 100644 index 0000000000000000000000000000000000000000..b971d3dd25b80ac1675c79abf36e1a3174cedafb GIT binary patch literal 1458 zcmV;j1x@;l4Fm}T0^7du>h-j)DF4#x0gDX*#5D)Ew5*OOtRQwr5acd>9IbY7=o&Sk z$(umP-(G~yHzZ^vvzPBT+`m@C1bRocFS?|1&3+F}US|*zaXMR=qmJ!tvvma4w&dRW z7yuyasRc|=`PiYHumZ82AMn5c3!whVlS$+46sfW?R~D6i#Oc?3HNE0R=;xkqp3cBt zIpclUIc}+$2=d0lZAjNk-E+uKb;p;_?(4IrR{IsCcDkN510bF>G5Y(+KUa1Zaj-LF zJSVu?a8S7By-%5u1`-l=G&*!PYfW%}y%a+O@6YHX-y#A8Uu=tW0;28#WY@p&8)KBU zV1rY-kepWf?U4@b2jzexFfEAj6~y{{>k)ff*KC8Ll|Cx-Gr18RLGckqplnz^+9yN> z3pVOLZe1(Rl5XI2+E~oJ7NcC(cvuc`Ly8G_gqG!0v;|Xdql|X%DZ>ascd*pk8{%tp zJ5$@<)dvooLVDu-q4tV#w+E>$jNhMssrXhN*PsNI>vL>c{wB|{Zv63XWAXi2hJ?of zSLk#vU517^W!uNin#3O)l0O;jUDZ{h+N8}ISsTa|N3HzZmzx6t#)m`U8R>;b=+59(%t)pKX* zF5Ub)y7D1MC{{^PrzcnOhMs^(JjQtA25Q=8?+3G(zuq1dt0=j?%<)-_g|9JyDWq(U z>t->BAgWI8{@C6^^lji)-*AdzQ9n@M^!dw2%Rra)h{~R}3sIIk+MjnTOG|umS55e& zy`QQ88p1D*reMx8*Ug!?8X;}PnXDSk-zw%0at;-(zaZ{JTmpWM@5kAmL;}t|&EkGS zU-PFt4ZylUNL#L}L1(|720}5+pg1n^w|2)+sR>Z!GDY6nSA_6z=!;w`Zs6EYc{&Fi zLw2S>?GFZsizi#Gelc;xlHKWHdezLAk$CG4Z3kF#pxnV}(q5rRO{Ro*od^X`&YrlG zmTjYL8+HufL(250^C!FRZLwU1Tmk~{tQa>0?1!I}P!?fxF# zfy2-CgDnhA?u$I`f(=<`v+(Sd^upvqlBjyU$_*TCqgL%3g)mZafr5S$OJ50+0H zc7(uPfOf~gObI{imn|qVX?+=|!Fv#kBXdhAXy_dw&GRQPUVmy&eb?dO zF$K2P7ycajz~4@Q6q{DCZX(K_onvQlo7i2Kc65z9jVGzx_K;g_3Ay>na4^oFZ}!oY z=uw1|2SRs^TfGrB2q|QT!ZbDywnkMEq9b3LbGC%1avW7mdqdxPSgUf$%m&(k76?>2 z>IT9rnNIX$i#?lvci3_F%^o52h?|<^^oVsTx=4iYU)K04SYQTbR8}qD>BUo)P8I`` zA6NvYWtR@6koiJ>t4-<}w|8ujfsSS42(%cxNIDp_*#;saao8FHdFDCJ0(!Uo=yH@R z)~|u4-T4zJth7X(cSMB6a(@cXVHd}{no@3x$o1CfkgJF4M$$K)aN-8rT;8X-4w>zbtnS5#?)0y@w5!%b zUHpW8p*13{l(EeN1OXZk9Vwu`4S?XG?6@qi_~swDfKQ!9-y>GOlzaOUdXX=Vs~BaZ z!O>{OvYOKFbcwmNr2^4v(Cd4uTpOj9viX0G#LklfF6!UZ?(jze)^``h