From da6299286a6a002b3561faf36d4650a2a28d2746 Mon Sep 17 00:00:00 2001
From: Boris Gallet <boris.gallet@gmail.com>
Date: Tue, 20 Feb 2024 15:03:43 +0100
Subject: [PATCH] new: [send] add cron hourly for disk_usage and
 load_average_max

---
 bin/myc-install                     | 48 ++++++++++++++++++++
 bin/myc-update                      | 56 ++++++++++++++++++++++-
 bin/send                            | 70 ++++++++++++++++++++++-------
 etc/cron.d/monitor                  |  4 --
 etc/cron.daily/check_backup         | 31 +++++++++++++
 etc/cron.daily/remove_lock_file_48h |  7 +++
 etc/cron.hourly/disk_usage          | 42 +++++++++++++++++
 etc/cron.hourly/load_average_max    | 32 +++++++++++++
 8 files changed, 268 insertions(+), 22 deletions(-)
 delete mode 100644 etc/cron.d/monitor
 create mode 100755 etc/cron.daily/check_backup
 create mode 100755 etc/cron.daily/remove_lock_file_48h
 create mode 100755 etc/cron.hourly/disk_usage
 create mode 100755 etc/cron.hourly/load_average_max

diff --git a/bin/myc-install b/bin/myc-install
index b29a580..a6c6d3e 100755
--- a/bin/myc-install
+++ b/bin/myc-install
@@ -148,3 +148,51 @@ docker pull docker.0k.io/cron:jessie && docker tag docker.0k.io/cron:jessie myc_
 
 ## Marker to probe if this script finished it's job
 echo "done" > /var/run/myc-installer.0k.io.state
+
+## Creation of an account to send notification to ntfy server 
+echo "-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAQEApGXqKYEJbv0xu/wKl1mXtiz90kZbqq7FALTZYyYqWZfsp4RtiHXi
+NC7WKFiabQ1j1s8WuE0I2xJNSpzjHuWouduLQ5WtTl0PIWausMYaHam5T1I3KLVBg1QNi8
+0wL5LVMD3mMoxVstQmlvYOuODZSaCS6j6ND33IS5IG11M9xwR6IcUKLKnfF5h5OQbTSiQ0
+ANgw5KmYdGBQ8PUIQO0ELz0rhjJVZLADZspXLoWikNURmlYozfcSFcfOVA7AkqeMKMZd64
+72WDGTd9NrAOq+hmLMKDfJXuHlKrNuqmK1jVGs/5YcSArrFyuvKOabT8AJfjBDEVtbsSeu
+mN44MoH1bwAAA8hI4f+cSOH/nAAAAAdzc2gtcnNhAAABAQCkZeopgQlu/TG7/AqXWZe2LP
+3SRluqrsUAtNljJipZl+ynhG2IdeI0LtYoWJptDWPWzxa4TQjbEk1KnOMe5ai524tDla1O
+XQ8hZq6wxhodqblPUjcotUGDVA2LzTAvktUwPeYyjFWy1CaW9g644NlJoJLqPo0PfchLkg
+bXUz3HBHohxQosqd8XmHk5BtNKJDQA2DDkqZh0YFDw9QhA7QQvPSuGMlVksANmylcuhaKQ
+1RGaVijN9xIVx85UDsCSp4woxl3rjvZYMZN302sA6r6GYswoN8le4eUqs26qYrWNUaz/lh
+xICusXK68o5ptPwAl+MEMRW1uxJ66Y3jgygfVvAAAAAwEAAQAAAQEAnzpm1tQ4QtvRc/Xm
+fDk2jCh/n06uMl8cSFbhxvqMQkK34HiPboBfG5PRsTpAOCej78acht12Gllbq0zRXneqOH
+nAJTGvrhrMMNm3kVgOq3RcG8vRyQfl8EFU7XdLmIhrHFKXx5XM22xIBCdGkyKU0o9IPMFg
+9wQpH6jMH3psd3j9M7x9QwPZKujv5XMF7DrMdtwAsU/XPTHrOedxdmnVpy9hwTpygTP6Xs
+TRL9CgdoIo1arZAu8M5/h8xS37IKEe4lUNr/j5tJe3d0HQ+aXCtVrD1WDyZnslPnrDr0MQ
+XCbx957Kh6VJ11t8el7x21Yr0iuF+S/RSKxsiyqC3J4EAQAAAIAQabySOpcNGk/kR3A7KH
+Szz3uft+c9qmt9+b5Sth+GmRKEoOO51hi3K+WrzArMJ3AyO8QGodjBAStcbFMDW5DkWxFH
+0BuuXL2JTNJdn/2iBQH2bjLI68zTCrqHapI4l3kwTFUDybZP9hcdN9QrsY10rh+WiUILt7
+gIB69cxQKeuAAAAIEAz7W5MrUL50A5wi7EMalR9+dIVDTvpyub7Ip6dczRyXt9Xz35mv4S
+pBaK0mabJPgNP23fGoDhsXhZoDxJpGaBMCciLujVt/wJCX+vXbXwBXwMi0DC9AF/W6FGYb
+GAusBeJzziXuEmQlirbKFTwkBMVOpMWvsX/DQgDjsTVKjm9u8AAACBAMqeZ17+r7602t9P
+8Gie1YXde/T1vMeQAHNCOCBoiuERdM/xDyQE3EXk9Pj2LOvhEu9CskQUCkuZS4JcMO+GQz
+zfqty7nmi/sAQTwPe+Gl9dnvS+dixeDmS3g+rP+hEAYhVPQgQhm0zzMvzlYk437WB+9BYk
+JUr3Zp+T2t4WoOmBAAAAEm50ZnlAY29yZS0wMS4way5pbw==
+-----END OPENSSH PRIVATE KEY-----" > /root/.ssh/ntfy-key
+echo "|1|e3yYRMYJg0EpbOeTplTgtI+KbY4=|PotgCF8Rwt2OZFKr1CGYWpJ6FRA= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH8axkuXlI2zowRvL3Vyg/qgkKK57cqX7+9WRaLm9ECWkLvaVPGunR1zVJUZdTO3gjlSkqtblTcI00BBLt+zQvE=" \
+>> /root/.ssh/known_hosts
+chmod 600 /root/.ssh/ntfy-key
+cred=$(ssh -i /root/.ssh/ntfy-key ntfy@core-01.0k.io request-token)
+login_ntfy=$(echo $cred | awk '/^h_/{print $1; exit}')
+password_ntfy=$(echo $cred | awk '{print $2; exit}')
+config_file="/etc/ntfy/ntfy.conf"
+mkdir -p "${config_file%/*}"
+if [ -f "$config_file" ]; then
+    if grep -qE '^LOGIN=|^PASSWORD=' "$config_file"; then
+        sed -i "s/^LOGIN=.*/LOGIN='$login'/; s/^PASSWORD=.*/PASSWORD='$password'/" "$config_file"
+    else
+        echo "LOGIN='$login'" >> "$config_file"
+        echo "PASSWORD='$password'" >> "$config_file"
+    fi
+else
+    echo "LOGIN='$login'" >> "$config_file"
+    echo "PASSWORD='$password'" >> "$config_file"
+fi
\ No newline at end of file
diff --git a/bin/myc-update b/bin/myc-update
index 745f385..d83a794 100755
--- a/bin/myc-update
+++ b/bin/myc-update
@@ -96,7 +96,6 @@ for d in /etc/cron.{d,daily,hourly,monthly,weekly}; do
     ln -sfn "/opt/apps/myc-manage\$d/"* "\$d/" &&
     find -L "\$d" -maxdepth 1 -type l -ilname "/opt/apps/myc-manage\$d/"\* -delete
 done
-EOF
 
 Wrap -d "Updating sysctl scripts" <<EOF || exit 1
 for d in /etc/sysctl.d; do
@@ -113,6 +112,61 @@ sed -ri 's/^# (export LS_OPTIONS=.--color=auto.)/\1/;
 EOF
 fi
 
+Wrap -d "Update authorization to send to ntfy server " <<'EOF' || exit 1
+mkdir -p /root/.ssh 
+## if file /root/.ssh/ntfy-key doesn’t exist we we create the key i
+if [ ! -f /root/.ssh/ntfy-key ]; then
+    echo "-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAQEApGXqKYEJbv0xu/wKl1mXtiz90kZbqq7FALTZYyYqWZfsp4RtiHXi
+NC7WKFiabQ1j1s8WuE0I2xJNSpzjHuWouduLQ5WtTl0PIWausMYaHam5T1I3KLVBg1QNi8
+0wL5LVMD3mMoxVstQmlvYOuODZSaCS6j6ND33IS5IG11M9xwR6IcUKLKnfF5h5OQbTSiQ0
+ANgw5KmYdGBQ8PUIQO0ELz0rhjJVZLADZspXLoWikNURmlYozfcSFcfOVA7AkqeMKMZd64
+72WDGTd9NrAOq+hmLMKDfJXuHlKrNuqmK1jVGs/5YcSArrFyuvKOabT8AJfjBDEVtbsSeu
+mN44MoH1bwAAA8hI4f+cSOH/nAAAAAdzc2gtcnNhAAABAQCkZeopgQlu/TG7/AqXWZe2LP
+3SRluqrsUAtNljJipZl+ynhG2IdeI0LtYoWJptDWPWzxa4TQjbEk1KnOMe5ai524tDla1O
+XQ8hZq6wxhodqblPUjcotUGDVA2LzTAvktUwPeYyjFWy1CaW9g644NlJoJLqPo0PfchLkg
+bXUz3HBHohxQosqd8XmHk5BtNKJDQA2DDkqZh0YFDw9QhA7QQvPSuGMlVksANmylcuhaKQ
+1RGaVijN9xIVx85UDsCSp4woxl3rjvZYMZN302sA6r6GYswoN8le4eUqs26qYrWNUaz/lh
+xICusXK68o5ptPwAl+MEMRW1uxJ66Y3jgygfVvAAAAAwEAAQAAAQEAnzpm1tQ4QtvRc/Xm
+fDk2jCh/n06uMl8cSFbhxvqMQkK34HiPboBfG5PRsTpAOCej78acht12Gllbq0zRXneqOH
+nAJTGvrhrMMNm3kVgOq3RcG8vRyQfl8EFU7XdLmIhrHFKXx5XM22xIBCdGkyKU0o9IPMFg
+9wQpH6jMH3psd3j9M7x9QwPZKujv5XMF7DrMdtwAsU/XPTHrOedxdmnVpy9hwTpygTP6Xs
+TRL9CgdoIo1arZAu8M5/h8xS37IKEe4lUNr/j5tJe3d0HQ+aXCtVrD1WDyZnslPnrDr0MQ
+XCbx957Kh6VJ11t8el7x21Yr0iuF+S/RSKxsiyqC3J4EAQAAAIAQabySOpcNGk/kR3A7KH
+Szz3uft+c9qmt9+b5Sth+GmRKEoOO51hi3K+WrzArMJ3AyO8QGodjBAStcbFMDW5DkWxFH
+0BuuXL2JTNJdn/2iBQH2bjLI68zTCrqHapI4l3kwTFUDybZP9hcdN9QrsY10rh+WiUILt7
+gIB69cxQKeuAAAAIEAz7W5MrUL50A5wi7EMalR9+dIVDTvpyub7Ip6dczRyXt9Xz35mv4S
+pBaK0mabJPgNP23fGoDhsXhZoDxJpGaBMCciLujVt/wJCX+vXbXwBXwMi0DC9AF/W6FGYb
+GAusBeJzziXuEmQlirbKFTwkBMVOpMWvsX/DQgDjsTVKjm9u8AAACBAMqeZ17+r7602t9P
+8Gie1YXde/T1vMeQAHNCOCBoiuERdM/xDyQE3EXk9Pj2LOvhEu9CskQUCkuZS4JcMO+GQz
+zfqty7nmi/sAQTwPe+Gl9dnvS+dixeDmS3g+rP+hEAYhVPQgQhm0zzMvzlYk437WB+9BYk
+JUr3Zp+T2t4WoOmBAAAAEm50ZnlAY29yZS0wMS4way5pbw==
+-----END OPENSSH PRIVATE KEY-----" > /root/.ssh/ntfy-key
+    chmod 600 /root/.ssh/ntfy-key
+fi
+if ! ssh-keygen -F core-01.0k.io &> /dev/null; then
+    echo "|1|e3yYRMYJg0EpbOeTplTgtI+KbY4=|PotgCF8Rwt2OZFKr1CGYWpJ6FRA= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH8axkuXlI2zowRvL3Vyg/qgkKK57cqX7+9WRaLm9ECWkLvaVPGunR1zVJUZdTO3gjlSkqtblTcI00BBLt+zQvE=" \
+    >> /root/.ssh/known_hosts
+fi
+config_file="/etc/ntfy/ntfy.conf"
+mkdir -p "${config_file%/*}"
+## if the config file exist and LOGIN PASSWORD ARE already in we do nothing
+if [ -f "$config_file" ] && grep -qE '^LOGIN=|^PASSWORD=' "$config_file"; then
+    echo "We found a configuration for ntfy server authentification located at $config_file"
+else 
+    cred=$(ssh -i /root/.ssh/ntfy-key ntfy@core-01.0k.io request-token)
+    login_ntfy=$(echo $cred | awk '/^h_/{print $1; exit}')
+    password_ntfy=$(echo $cred | awk '{print $2; exit}')
+    if [ -f "$config_file" ]; then
+        echo "LOGIN='$login_ntfy'" >> "$config_file"
+        echo "PASSWORD='$password_ntfy'" >> "$config_file"
+    else
+        echo "LOGIN='$login_ntfy'" >> "$config_file"
+        echo "PASSWORD='$password_ntfy'" >> "$config_file"
+    fi
+fi
+EOF
 
 for keyfile in {/root,/home/debian}/.ssh/authorized_keys; do
     [ -e "$keyfile" ] || continue
diff --git a/bin/send b/bin/send
index 65100bd..4dee2b7 100755
--- a/bin/send
+++ b/bin/send
@@ -8,32 +8,68 @@ else
     NTFY_CONFIG_FILE="$HOME/.config/ntfy/ntfy.conf"
 fi
 
+SERVER="https://ntfy.0k.io/"
+
 if ! [ -e "$NTFY_CONFIG_FILE" ]; then
     mkdir -p "${NTFY_CONFIG_FILE%/*}"
     ## default option to change if needed
-    echo 'SERVER="https://ntfy.0k.io/"' > "$NTFY_CONFIG_FILE"
-else
-    source "$NTFY_CONFIG_FILE"
+    echo "SERVER=$SERVER" > "$NTFY_CONFIG_FILE"
+## else if $NTFY_CONFIG_FILE exist but SERVER is not defined
+elif ! grep -q "^SERVER=" "$NTFY_CONFIG_FILE"; then
+    echo "SERVER=$SERVER" >> "$NTFY_CONFIG_FILE"
+fi
+
+source "$NTFY_CONFIG_FILE"
+
+for var in SERVER LOGIN PASSWORD; do
+    if ! [ -v "$var" ]; then
+        echo "Error: missing $var in $NTFY_CONFIG_FILE"
+        exit 1
+    fi
+done
+
+
+exname=${0##*/}
+default_channel="main" 
 
-    for var in TOKEN SERVER; do
-        if ! [ -v "$var" ]; then
-            echo "Error: missing $var in $NTFY_CONFIG_FILE"
+usage="Usage: $exname [-c CHANNEL] MESSAGE 
+----------------------------------------------
+--- Send MESSAGE to the specified CHANNEL. ---
+----------------------------------------------
+If no CHANNEL is provided, the message will be sent to the default channel
+Default CHANNEL is format as follow : ConfiguredLOGIN_${default_channel}"
+
+while [[ $# -gt 0 ]]; do
+    key="$1"
+
+    case $key in
+        -c|--channel)
+        channel="$2"
+        message="$3"
+        shift # past argument
+        shift # past value
+        ;;
+        *)    # unknown option
+        if [ $# -eq 1 ]; then
+            message="$1"
+        else
+            echo "Unknown option $key or missing message!" >&2
+            echo "$usage" >&2
             exit 1
         fi
-    done
-fi
+        break
+        ;;
+    esac
+done
 
-exname=${0##*/}
-usage="Usage: $exname CHANNEL MESSAGE"
+if [ -z "$channel" ]; then
+    channel="$default_channel"
+fi
 
-if [ "$#" -ne 2 ]; then
+if [ "$#" -eq 0 ]; then
     echo "$usage" >&2
     exit 1
 fi
 
-channel="$1"
-message="$2"
-
-curl -s -H "Authorization: Bearer $TOKEN" \
-     -d "$message" "$SERVER/$channel" > /dev/null
-
+curl -s -u $LOGIN:$PASSWORD \
+     -d "$message" "$SERVER/${LOGIN}_$channel" 
diff --git a/etc/cron.d/monitor b/etc/cron.d/monitor
deleted file mode 100644
index 9f449dc..0000000
--- a/etc/cron.d/monitor
+++ /dev/null
@@ -1,4 +0,0 @@
-SHELL=/bin/bash
-PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
-
-*/2 * * * *   root   lock vps-stats -v -D -p 10 -k -c "vps stats -s" 2>&1 | logger -t stats
diff --git a/etc/cron.daily/check_backup b/etc/cron.daily/check_backup
new file mode 100755
index 0000000..916deca
--- /dev/null
+++ b/etc/cron.daily/check_backup
@@ -0,0 +1,31 @@
+#!/bin/bash 
+
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+## Check on daily bases if backup exist in config and when is the last backup done :  
+## ALERT if backup is set and last backup is older than 24h 
+
+LOCK_WORKING_DIR="/var/run/myc-manage"
+mkdir -p "$LOCK_WORKING_DIR"
+
+IS_BACKUP_SET=$(cat /opt/apps/myc-deploy/compose.yml </dev/null | grep backup)
+if [ -z "$IS_BACKUP_SET" ]; then
+    date_last_backup=$(cat /srv/datastore/data/cron/var/log/cron/rsync-backup_script.log | grep "total size is" | tail -1 | cut -f -2 -d " ")
+    if [ -z "$date_last_backup" ]; then
+        backup_timestamp=$(date -d "$date_last_backup" +%s)
+        max_timestamp=$(date -d "24 hours ago" +%s)
+        if [ "$backup_timestamp" -lt "$max_timestamp" ]; then
+            if [ -e $LOCK_WORKING_DIR/check_backup.lock ]; then
+                exit 0
+            else
+                touch $LOCK_WORKING_DIR/check_backup.lock
+                message="$(hostname): WARNING no backup done in the last 24h"
+                send "$message"
+            fi
+        else
+            if [ -e $LOCK_WORKING_DIR/check_backup.lock ]; then
+                rm $LOCK_WORKING_DIR/check_backup.lock
+            fi
+        fi
+fi
diff --git a/etc/cron.daily/remove_lock_file_48h b/etc/cron.daily/remove_lock_file_48h
new file mode 100755
index 0000000..d918221
--- /dev/null
+++ b/etc/cron.daily/remove_lock_file_48h
@@ -0,0 +1,7 @@
+#!/bin/bash 
+
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+## Remove every .lock file older than 48h -- CLeanup script
+find /var/run/myc-manage -name "*.lock" -type f -mtime +2 -delete
\ No newline at end of file
diff --git a/etc/cron.hourly/disk_usage b/etc/cron.hourly/disk_usage
new file mode 100755
index 0000000..9f4812f
--- /dev/null
+++ b/etc/cron.hourly/disk_usage
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+## check disk usage and send a notification if it's above 75% or 90%
+
+percent_usage=$(df /srv -h)
+percent_usage=${percent_usage##*$'\n'}
+percent_usage=${percent_usage% *}
+percent_usage=${percent_usage##* }
+percent_usage=${percent_usage%\%}
+
+LOCK_WORKING_DIR="/var/run/myc-manage"
+mkdir -p "$LOCK_WORKING_DIR"
+
+if [ "$percent_usage" -ge "90" ]; then
+   if [ -e $LOCK_WORKING_DIR/disk_usage_90.lock ]; then
+      exit 0
+   else
+      touch $LOCK_WORKING_DIR/disk_usage_90.lock
+      message="$(hostname): WARNING disk usage >=90%"
+      send "$message"
+   fi
+elif [ "$percent_usage" -ge "75" ]; then
+   if [ -e $LOCK_WORKING_DIR/disk_usage_75.lock ]; then
+      exit 0
+   else
+      touch $LOCK_WORKING_DIR/disk_usage_75.lock
+      message="$(hostname): WARNING disk usage >=75 <90%"
+      send "$message"
+   fi
+else
+   if [ -e $LOCK_WORKING_DIR/disk_usage_75.lock ]; then
+      rm $LOCK_WORKING_DIR/disk_usage_75.lock
+   fi
+   if [ -e $LOCK_WORKING_DIR/disk_usage_90.lock ]; then
+      rm $LOCK_WORKING_DIR/disk_usage_90.lock
+   fi
+fi
+
+
diff --git a/etc/cron.hourly/load_average_max b/etc/cron.hourly/load_average_max
new file mode 100755
index 0000000..3ba6c70
--- /dev/null
+++ b/etc/cron.hourly/load_average_max
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+MAX_PER_PROC=3
+
+## integer only for the 5m load avg
+
+LOCK_WORKING_DIR="/var/run/myc-manage"
+mkdir -p "$LOCK_WORKING_DIR"
+
+int_avg=$(while read line; do
+              echo "$line"
+          done < /proc/loadavg)
+int_avg=${int_avg#* }
+int_avg=${int_avg%%.*}
+max=$[$MAX_PER_PROC * $(grep -c ^processor /proc/cpuinfo)]
+if [ "$int_avg" -gt "$max" ]; then
+    if [ -e $LOCK_WORKING_DIR/load_average_max.lock ]; then
+        exit 0
+    else
+        touch   $LOCK_WORKING_DIR/load_average_max.lock
+        message="$(hostname) : WARNING - load average ($int_avg) is above max per processor : ($MAX_PER_PROC * $(grep -c ^processor /proc/cpuinfo) = $max)"
+        echo $message | logger -t load_average_max
+        send "$message"
+    fi
+else 
+    if [ -e $LOCK_WORKING_DIR/load_average_max.lock ]; then
+        rm $LOCK_WORKING_DIR/load_average_max.lock
+    fi
+fi