From bcbf089935847702d5006c234f8fe81c7dcc1306 Mon Sep 17 00:00:00 2001
From: Boris Gallet <boris.gallet@gmail.com>
Date: Tue, 20 Feb 2024 15:03:43 +0100
Subject: [PATCH] new: [send] add cron hourly for disk_usage and
 load_average_max

---
 bin/myc-update                   |  1 +
 etc/cron.d/alerting              |  3 +++
 etc/cron.hourly/disk_usage       | 38 ++++++++++++++++++++++++++++++++
 etc/cron.hourly/load_average_max | 26 ++++++++++++++++++++++
 4 files changed, 68 insertions(+)
 create mode 100644 etc/cron.d/alerting
 create mode 100755 etc/cron.hourly/disk_usage
 create mode 100755 etc/cron.hourly/load_average_max

diff --git a/bin/myc-update b/bin/myc-update
index 12437cd..84b6e0e 100755
--- a/bin/myc-update
+++ b/bin/myc-update
@@ -62,6 +62,7 @@ EOF
 
 Wrap -d "Updating cron scripts" <<EOF || exit 1
 ln -sfn /opt/apps/myc-manage/etc/cron.d/* /etc/cron.d/
+ln -sfn /opt/apps/myc-manage/etc/cron.hourly/* /etc/cron.hourly/
 find -L /etc/cron.d -maxdepth 1 -type l -ilname /opt/apps/myc-manage/etc/cron.d/\* -delete
 EOF
 
diff --git a/etc/cron.d/alerting b/etc/cron.d/alerting
new file mode 100644
index 0000000..c347245
--- /dev/null
+++ b/etc/cron.d/alerting
@@ -0,0 +1,3 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
diff --git a/etc/cron.hourly/disk_usage b/etc/cron.hourly/disk_usage
new file mode 100755
index 0000000..9f26710
--- /dev/null
+++ b/etc/cron.hourly/disk_usage
@@ -0,0 +1,38 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+## check disk usage and send a notification if it's above 75% or 90%
+
+percent_usage=$(df /srv -h)
+percent_usage=${percent_usage##*$'\n'}
+percent_usage=${percent_usage% *}
+percent_usage=${percent_usage##* }
+percent_usage=${percent_usage%\%}
+
+LOCK_WORKING_DIR="/var/run/myc-manage"
+mkdir -p "$LOCK_WORKING_DIR"
+
+if [ "$percent_usage" -ge "90" ]; then
+   if [ -e $LOCK_WORKING_DIR/disk_usage_90.lock ]; then
+      exit 0
+   else
+      touch $LOCK_WORKING_DIR/disk_usage_90.lock
+      message="$(hostname): WARNING disk usage >=90%"
+      send "disk_usage" "$message"
+   fi
+elif [ "$percent_usage" -ge "75" ]; then
+   if [ -e $LOCK_WORKING_DIR/disk_usage_75.lock ]; then
+      exit 0
+   else
+      touch $LOCK_WORKING_DIR/disk_usage_75.lock
+      message="$(hostname): WARNING disk usage >=75 <90%"
+      send "disk_usage" "$message"
+   fi
+else
+   if [ -e $LOCK_WORKING_DIR/disk_usage_75.lock ]; then
+      rm $LOCK_WORKING_DIR/disk_usage_75.lock
+   fi
+   if [ -e $LOCK_WORKING_DIR/disk_usage_90.lock ]; then
+      rm $LOCK_WORKING_DIR/disk_usage_90.lock
+   fi
+fi
diff --git a/etc/cron.hourly/load_average_max b/etc/cron.hourly/load_average_max
new file mode 100755
index 0000000..1e65da8
--- /dev/null
+++ b/etc/cron.hourly/load_average_max
@@ -0,0 +1,26 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+MAX_PER_PROC=3
+
+## integer only for the 5m load avg
+
+LOCK_WORKING_DIR="/var/run/myc-manage"
+mkdir -p "$LOCK_WORKING_DIR"
+
+int_avg=$(while read line; do
+              echo "$line"
+          done < /proc/loadavg)
+int_avg=${int_avg#* }
+int_avg=${int_avg%%.*}
+max=$[$MAX_PER_PROC * $(grep -c ^processor /proc/cpuinfo)]
+if [ "$int_avg" -gt "$max" ]; then
+    if [ -e $LOCK_WORKING_DIR/load_average_max.lock ]; then
+        exit 0
+    else
+        touch   $LOCK_WORKING_DIR/load_average_max.lock
+        message="$(hostname) : WARNING - load average ($int_avg) is above max per processor : ($MAX_PER_PROC * $(grep -c ^processor /proc/cpuinfo) = $max)"
+        echo $message | logger -t load_average_max
+        send "load_average_max" "$message"
+    fi
+fi