From 81d4dca51c06dbfdb81cea0019e0f8fc9f12802b Mon Sep 17 00:00:00 2001
From: Boris Gallet <boris.gallet@gmail.com>
Date: Tue, 20 Feb 2024 15:03:43 +0100
Subject: [PATCH] new: [send] add cron hourly for disk_usage and
 load_average_max

---
 bin/myc-update                   |  1 -
 etc/cron.d/alerting              |  3 +++
 etc/cron.hourly/disk_usage       | 38 ++++++++++++++++++++++++++++++++
 etc/cron.hourly/load_average_max | 26 ++++++++++++++++++++++
 4 files changed, 67 insertions(+), 1 deletion(-)
 create mode 100644 etc/cron.d/alerting
 create mode 100755 etc/cron.hourly/disk_usage
 create mode 100755 etc/cron.hourly/load_average_max

diff --git a/bin/myc-update b/bin/myc-update
index 745f385..6b1a0e1 100755
--- a/bin/myc-update
+++ b/bin/myc-update
@@ -96,7 +96,6 @@ for d in /etc/cron.{d,daily,hourly,monthly,weekly}; do
     ln -sfn "/opt/apps/myc-manage\$d/"* "\$d/" &&
     find -L "\$d" -maxdepth 1 -type l -ilname "/opt/apps/myc-manage\$d/"\* -delete
 done
-EOF
 
 Wrap -d "Updating sysctl scripts" <<EOF || exit 1
 for d in /etc/sysctl.d; do
diff --git a/etc/cron.d/alerting b/etc/cron.d/alerting
new file mode 100644
index 0000000..c347245
--- /dev/null
+++ b/etc/cron.d/alerting
@@ -0,0 +1,3 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
diff --git a/etc/cron.hourly/disk_usage b/etc/cron.hourly/disk_usage
new file mode 100755
index 0000000..9f26710
--- /dev/null
+++ b/etc/cron.hourly/disk_usage
@@ -0,0 +1,38 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+## check disk usage and send a notification if it's above 75% or 90%
+
+percent_usage=$(df /srv -h)
+percent_usage=${percent_usage##*$'\n'}
+percent_usage=${percent_usage% *}
+percent_usage=${percent_usage##* }
+percent_usage=${percent_usage%\%}
+
+LOCK_WORKING_DIR="/var/run/myc-manage"
+mkdir -p "$LOCK_WORKING_DIR"
+
+if [ "$percent_usage" -ge "90" ]; then
+   if [ -e $LOCK_WORKING_DIR/disk_usage_90.lock ]; then
+      exit 0
+   else
+      touch $LOCK_WORKING_DIR/disk_usage_90.lock
+      message="$(hostname): WARNING disk usage >=90%"
+      send "disk_usage" "$message"
+   fi
+elif [ "$percent_usage" -ge "75" ]; then
+   if [ -e $LOCK_WORKING_DIR/disk_usage_75.lock ]; then
+      exit 0
+   else
+      touch $LOCK_WORKING_DIR/disk_usage_75.lock
+      message="$(hostname): WARNING disk usage >=75 <90%"
+      send "disk_usage" "$message"
+   fi
+else
+   if [ -e $LOCK_WORKING_DIR/disk_usage_75.lock ]; then
+      rm $LOCK_WORKING_DIR/disk_usage_75.lock
+   fi
+   if [ -e $LOCK_WORKING_DIR/disk_usage_90.lock ]; then
+      rm $LOCK_WORKING_DIR/disk_usage_90.lock
+   fi
+fi
diff --git a/etc/cron.hourly/load_average_max b/etc/cron.hourly/load_average_max
new file mode 100755
index 0000000..1e65da8
--- /dev/null
+++ b/etc/cron.hourly/load_average_max
@@ -0,0 +1,26 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+MAX_PER_PROC=3
+
+## integer only for the 5m load avg
+
+LOCK_WORKING_DIR="/var/run/myc-manage"
+mkdir -p "$LOCK_WORKING_DIR"
+
+int_avg=$(while read line; do
+              echo "$line"
+          done < /proc/loadavg)
+int_avg=${int_avg#* }
+int_avg=${int_avg%%.*}
+max=$[$MAX_PER_PROC * $(grep -c ^processor /proc/cpuinfo)]
+if [ "$int_avg" -gt "$max" ]; then
+    if [ -e $LOCK_WORKING_DIR/load_average_max.lock ]; then
+        exit 0
+    else
+        touch   $LOCK_WORKING_DIR/load_average_max.lock
+        message="$(hostname) : WARNING - load average ($int_avg) is above max per processor : ($MAX_PER_PROC * $(grep -c ^processor /proc/cpuinfo) = $max)"
+        echo $message | logger -t load_average_max
+        send "load_average_max" "$message"
+    fi
+fi