new: [send] add cron hourly for disk_usage and load_average_max

10 months ago · 2169bbd043
6 changed files with 227 additions and 18 deletions
--- a/bin/myc-install
+++ b/bin/myc-install
@ -148,3 +148,51 @@ docker pull docker.0k.io/cron:jessie && docker tag docker.0k.io/cron:jessie myc_
 ## Marker to probe if this script finished it's job
 echo "done" > /var/run/myc-installer.0k.io.state
 ## Creation of an account to send notification to ntfy server 
 echo "-----BEGIN OPENSSH PRIVATE KEY-----
 b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
 NhAAAAAwEAAQAAAQEApGXqKYEJbv0xu/wKl1mXtiz90kZbqq7FALTZYyYqWZfsp4RtiHXi
 NC7WKFiabQ1j1s8WuE0I2xJNSpzjHuWouduLQ5WtTl0PIWausMYaHam5T1I3KLVBg1QNi8
 0wL5LVMD3mMoxVstQmlvYOuODZSaCS6j6ND33IS5IG11M9xwR6IcUKLKnfF5h5OQbTSiQ0
 ANgw5KmYdGBQ8PUIQO0ELz0rhjJVZLADZspXLoWikNURmlYozfcSFcfOVA7AkqeMKMZd64
 72WDGTd9NrAOq+hmLMKDfJXuHlKrNuqmK1jVGs/5YcSArrFyuvKOabT8AJfjBDEVtbsSeu
 mN44MoH1bwAAA8hI4f+cSOH/nAAAAAdzc2gtcnNhAAABAQCkZeopgQlu/TG7/AqXWZe2LP
 3SRluqrsUAtNljJipZl+ynhG2IdeI0LtYoWJptDWPWzxa4TQjbEk1KnOMe5ai524tDla1O
 XQ8hZq6wxhodqblPUjcotUGDVA2LzTAvktUwPeYyjFWy1CaW9g644NlJoJLqPo0PfchLkg
 bXUz3HBHohxQosqd8XmHk5BtNKJDQA2DDkqZh0YFDw9QhA7QQvPSuGMlVksANmylcuhaKQ
 1RGaVijN9xIVx85UDsCSp4woxl3rjvZYMZN302sA6r6GYswoN8le4eUqs26qYrWNUaz/lh
 xICusXK68o5ptPwAl+MEMRW1uxJ66Y3jgygfVvAAAAAwEAAQAAAQEAnzpm1tQ4QtvRc/Xm
 fDk2jCh/n06uMl8cSFbhxvqMQkK34HiPboBfG5PRsTpAOCej78acht12Gllbq0zRXneqOH
 nAJTGvrhrMMNm3kVgOq3RcG8vRyQfl8EFU7XdLmIhrHFKXx5XM22xIBCdGkyKU0o9IPMFg
 9wQpH6jMH3psd3j9M7x9QwPZKujv5XMF7DrMdtwAsU/XPTHrOedxdmnVpy9hwTpygTP6Xs
 TRL9CgdoIo1arZAu8M5/h8xS37IKEe4lUNr/j5tJe3d0HQ+aXCtVrD1WDyZnslPnrDr0MQ
 XCbx957Kh6VJ11t8el7x21Yr0iuF+S/RSKxsiyqC3J4EAQAAAIAQabySOpcNGk/kR3A7KH
 Szz3uft+c9qmt9+b5Sth+GmRKEoOO51hi3K+WrzArMJ3AyO8QGodjBAStcbFMDW5DkWxFH
 0BuuXL2JTNJdn/2iBQH2bjLI68zTCrqHapI4l3kwTFUDybZP9hcdN9QrsY10rh+WiUILt7
 gIB69cxQKeuAAAAIEAz7W5MrUL50A5wi7EMalR9+dIVDTvpyub7Ip6dczRyXt9Xz35mv4S
 pBaK0mabJPgNP23fGoDhsXhZoDxJpGaBMCciLujVt/wJCX+vXbXwBXwMi0DC9AF/W6FGYb
 GAusBeJzziXuEmQlirbKFTwkBMVOpMWvsX/DQgDjsTVKjm9u8AAACBAMqeZ17+r7602t9P
 8Gie1YXde/T1vMeQAHNCOCBoiuERdM/xDyQE3EXk9Pj2LOvhEu9CskQUCkuZS4JcMO+GQz
 zfqty7nmi/sAQTwPe+Gl9dnvS+dixeDmS3g+rP+hEAYhVPQgQhm0zzMvzlYk437WB+9BYk
 JUr3Zp+T2t4WoOmBAAAAEm50ZnlAY29yZS0wMS4way5pbw==
 -----END OPENSSH PRIVATE KEY-----" > /root/.ssh/ntfy-key
 echo "|1|e3yYRMYJg0EpbOeTplTgtI+KbY4=|PotgCF8Rwt2OZFKr1CGYWpJ6FRA= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH8axkuXlI2zowRvL3Vyg/qgkKK57cqX7+9WRaLm9ECWkLvaVPGunR1zVJUZdTO3gjlSkqtblTcI00BBLt+zQvE=" \
 >> /root/.ssh/known_hosts
 chmod 600 /root/.ssh/ntfy-key
 cred=$(ssh -i /root/.ssh/ntfy-key ntfy@core-01.0k.io request-token)
 login_ntfy=$(echo $cred | awk '/^h_/{print $1; exit}')
 password_ntfy=$(echo $cred | awk '{print $2; exit}')
 config_file="/etc/ntfy/ntfy.conf"
 mkdir -p "${config_file%/*}"
 if [ -f "$config_file" ]; then
    if grep -qE '^LOGIN=|^PASSWORD=' "$config_file"; then
        sed -i "s/^LOGIN=.*/LOGIN='$login'/; s/^PASSWORD=.*/PASSWORD='$password'/" "$config_file"
    else
        echo "LOGIN='$login'" >> "$config_file"
        echo "PASSWORD='$password'" >> "$config_file"
    fi
 else
    echo "LOGIN='$login'" >> "$config_file"
    echo "PASSWORD='$password'" >> "$config_file"
 fi
--- a/bin/myc-update
+++ b/bin/myc-update
@ -96,7 +96,6 @@ for d in /etc/cron.{d,daily,hourly,monthly,weekly}; do
    ln -sfn "/opt/apps/myc-manage\$d/"* "\$d/" &&
    find -L "\$d" -maxdepth 1 -type l -ilname "/opt/apps/myc-manage\$d/"\* -delete
 done
 EOF
 Wrap -d "Updating sysctl scripts" <<EOF || exit 1
 for d in /etc/sysctl.d; do
@ -113,6 +112,61 @@ sed -ri 's/^# (export LS_OPTIONS=.--color=auto.)/\1/;
 EOF
 fi
 Wrap -d "Update authorization to send to ntfy server " <<'EOF' || exit 1
 mkdir -p /root/.ssh 
 ## if file /root/.ssh/ntfy-key doesn’t exist we we create the key i
 if [ ! -f /root/.ssh/ntfy-key ]; then
    echo "-----BEGIN OPENSSH PRIVATE KEY-----
 b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
 NhAAAAAwEAAQAAAQEApGXqKYEJbv0xu/wKl1mXtiz90kZbqq7FALTZYyYqWZfsp4RtiHXi
 NC7WKFiabQ1j1s8WuE0I2xJNSpzjHuWouduLQ5WtTl0PIWausMYaHam5T1I3KLVBg1QNi8
 0wL5LVMD3mMoxVstQmlvYOuODZSaCS6j6ND33IS5IG11M9xwR6IcUKLKnfF5h5OQbTSiQ0
 ANgw5KmYdGBQ8PUIQO0ELz0rhjJVZLADZspXLoWikNURmlYozfcSFcfOVA7AkqeMKMZd64
 72WDGTd9NrAOq+hmLMKDfJXuHlKrNuqmK1jVGs/5YcSArrFyuvKOabT8AJfjBDEVtbsSeu
 mN44MoH1bwAAA8hI4f+cSOH/nAAAAAdzc2gtcnNhAAABAQCkZeopgQlu/TG7/AqXWZe2LP
 3SRluqrsUAtNljJipZl+ynhG2IdeI0LtYoWJptDWPWzxa4TQjbEk1KnOMe5ai524tDla1O
 XQ8hZq6wxhodqblPUjcotUGDVA2LzTAvktUwPeYyjFWy1CaW9g644NlJoJLqPo0PfchLkg
 bXUz3HBHohxQosqd8XmHk5BtNKJDQA2DDkqZh0YFDw9QhA7QQvPSuGMlVksANmylcuhaKQ
 1RGaVijN9xIVx85UDsCSp4woxl3rjvZYMZN302sA6r6GYswoN8le4eUqs26qYrWNUaz/lh
 xICusXK68o5ptPwAl+MEMRW1uxJ66Y3jgygfVvAAAAAwEAAQAAAQEAnzpm1tQ4QtvRc/Xm
 fDk2jCh/n06uMl8cSFbhxvqMQkK34HiPboBfG5PRsTpAOCej78acht12Gllbq0zRXneqOH
 nAJTGvrhrMMNm3kVgOq3RcG8vRyQfl8EFU7XdLmIhrHFKXx5XM22xIBCdGkyKU0o9IPMFg
 9wQpH6jMH3psd3j9M7x9QwPZKujv5XMF7DrMdtwAsU/XPTHrOedxdmnVpy9hwTpygTP6Xs
 TRL9CgdoIo1arZAu8M5/h8xS37IKEe4lUNr/j5tJe3d0HQ+aXCtVrD1WDyZnslPnrDr0MQ
 XCbx957Kh6VJ11t8el7x21Yr0iuF+S/RSKxsiyqC3J4EAQAAAIAQabySOpcNGk/kR3A7KH
 Szz3uft+c9qmt9+b5Sth+GmRKEoOO51hi3K+WrzArMJ3AyO8QGodjBAStcbFMDW5DkWxFH
 0BuuXL2JTNJdn/2iBQH2bjLI68zTCrqHapI4l3kwTFUDybZP9hcdN9QrsY10rh+WiUILt7
 gIB69cxQKeuAAAAIEAz7W5MrUL50A5wi7EMalR9+dIVDTvpyub7Ip6dczRyXt9Xz35mv4S
 pBaK0mabJPgNP23fGoDhsXhZoDxJpGaBMCciLujVt/wJCX+vXbXwBXwMi0DC9AF/W6FGYb
 GAusBeJzziXuEmQlirbKFTwkBMVOpMWvsX/DQgDjsTVKjm9u8AAACBAMqeZ17+r7602t9P
 8Gie1YXde/T1vMeQAHNCOCBoiuERdM/xDyQE3EXk9Pj2LOvhEu9CskQUCkuZS4JcMO+GQz
 zfqty7nmi/sAQTwPe+Gl9dnvS+dixeDmS3g+rP+hEAYhVPQgQhm0zzMvzlYk437WB+9BYk
 JUr3Zp+T2t4WoOmBAAAAEm50ZnlAY29yZS0wMS4way5pbw==
 -----END OPENSSH PRIVATE KEY-----" > /root/.ssh/ntfy-key
    chmod 600 /root/.ssh/ntfy-key
 fi
 if ! ssh-keygen -F core-01.0k.io &> /dev/null; then
    echo "|1|e3yYRMYJg0EpbOeTplTgtI+KbY4=|PotgCF8Rwt2OZFKr1CGYWpJ6FRA= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH8axkuXlI2zowRvL3Vyg/qgkKK57cqX7+9WRaLm9ECWkLvaVPGunR1zVJUZdTO3gjlSkqtblTcI00BBLt+zQvE=" \
    >> /root/.ssh/known_hosts
 fi
 config_file="/etc/ntfy/ntfy.conf"
 mkdir -p "${config_file%/*}"
 ## if the config file exist and LOGIN PASSWORD ARE already in we do nothing
 if [ -f "$config_file" ] && grep -qE '^LOGIN=|^PASSWORD=' "$config_file"; then
    echo "We found a configuration for ntfy server authentification located at $config_file"
 else 
    cred=$(ssh -i /root/.ssh/ntfy-key ntfy@core-01.0k.io request-token)
    login_ntfy=$(echo $cred | awk '/^h_/{print $1; exit}')
    password_ntfy=$(echo $cred | awk '{print $2; exit}')
    if [ -f "$config_file" ]; then
        echo "LOGIN='$login_ntfy'" >> "$config_file"
        echo "PASSWORD='$password_ntfy'" >> "$config_file"
    else
        echo "LOGIN='$login_ntfy'" >> "$config_file"
        echo "PASSWORD='$password_ntfy'" >> "$config_file"
    fi
 fi
 EOF
 for keyfile in {/root,/home/debian}/.ssh/authorized_keys; do
    [ -e "$keyfile" ] || continue
--- a/bin/send
+++ b/bin/send
@ -8,32 +8,68 @@ else
    NTFY_CONFIG_FILE="$HOME/.config/ntfy/ntfy.conf"
 fi
 SERVER="https://ntfy.0k.io/"
 if ! [ -e "$NTFY_CONFIG_FILE" ]; then
    mkdir -p "${NTFY_CONFIG_FILE%/*}"
    ## default option to change if needed
    echo 'SERVER="https://ntfy.0k.io/"' > "$NTFY_CONFIG_FILE"
 else
    source "$NTFY_CONFIG_FILE"
    echo "SERVER=$SERVER" > "$NTFY_CONFIG_FILE"
 ## else if $NTFY_CONFIG_FILE exist but SERVER is not defined
 elif ! grep -q "^SERVER=" "$NTFY_CONFIG_FILE"; then
    echo "SERVER=$SERVER" >> "$NTFY_CONFIG_FILE"
 fi
 source "$NTFY_CONFIG_FILE"
 for var in SERVER LOGIN PASSWORD; do
    if ! [ -v "$var" ]; then
        echo "Error: missing $var in $NTFY_CONFIG_FILE"
        exit 1
    fi
 done
 exname=${0##*/}
 default_channel="main" 
    for var in TOKEN SERVER; do
        if ! [ -v "$var" ]; then
            echo "Error: missing $var in $NTFY_CONFIG_FILE"
 usage="Usage: $exname [-c CHANNEL] MESSAGE 
 ----------------------------------------------
 --- Send MESSAGE to the specified CHANNEL. ---
 ----------------------------------------------
 If no CHANNEL is provided, the message will be sent to the default channel
 Default CHANNEL is format as follow : ConfiguredLOGIN_${default_channel}"
 while [[ $# -gt 0 ]]; do
    key="$1"
    case $key in
        -c|--channel)
        channel="$2"
        message="$3"
        shift # past argument
        shift # past value
        ;;
        *)    # unknown option
        if [ $# -eq 1 ]; then
            message="$1"
        else
            echo "Unknown option $key or missing message!" >&2
            echo "$usage" >&2
            exit 1
        fi
    done
 fi
        break
        ;;
    esac
 done
 exname=${0##*/}
 usage="Usage: $exname CHANNEL MESSAGE"
 if [ -z "$channel" ]; then
    channel="$default_channel"
 fi
 if [ "$#" -ne 2 ]; then
 if [ "$#" -eq 0 ]; then
    echo "$usage" >&2
    exit 1
 fi
 channel="$1"
 message="$2"
 curl -s -H "Authorization: Bearer $TOKEN" \
     -d "$message" "$SERVER/$channel" > /dev/null
 curl -s -u $LOGIN:$PASSWORD \
     -d "$message" "$SERVER/${LOGIN}_$channel" 
--- a/etc/cron.d/alerting
+++ b/etc/cron.d/alerting
@ -0,0 +1,3 @@
 SHELL=/bin/bash
 PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
--- a/etc/cron.hourly/disk_usage
+++ b/etc/cron.hourly/disk_usage
@ -0,0 +1,40 @@
 #!/bin/bash
 SHELL=/bin/bash
 PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
 ## check disk usage and send a notification if it's above 75% or 90%
 percent_usage=$(df /srv -h)
 percent_usage=${percent_usage##*$'\n'}
 percent_usage=${percent_usage% *}
 percent_usage=${percent_usage##* }
 percent_usage=${percent_usage%\%}
 LOCK_WORKING_DIR="/var/run/myc-manage"
 mkdir -p "$LOCK_WORKING_DIR"
 if [ "$percent_usage" -ge "90" ]; then
   if [ -e $LOCK_WORKING_DIR/disk_usage_90.lock ]; then
      exit 0
   else
      touch $LOCK_WORKING_DIR/disk_usage_90.lock
      message="$(hostname): WARNING disk usage >=90%"
      send "disk_usage" "$message"
   fi
 elif [ "$percent_usage" -ge "75" ]; then
   if [ -e $LOCK_WORKING_DIR/disk_usage_75.lock ]; then
      exit 0
   else
      touch $LOCK_WORKING_DIR/disk_usage_75.lock
      message="$(hostname): WARNING disk usage >=75 <90%"
      send "disk_usage" "$message"
   fi
 else
   if [ -e $LOCK_WORKING_DIR/disk_usage_75.lock ]; then
      rm $LOCK_WORKING_DIR/disk_usage_75.lock
   fi
   if [ -e $LOCK_WORKING_DIR/disk_usage_90.lock ]; then
      rm $LOCK_WORKING_DIR/disk_usage_90.lock
   fi
 fi
--- a/etc/cron.hourly/load_average_max
+++ b/etc/cron.hourly/load_average_max
@ -0,0 +1,28 @@
 #!/bin/bash
 SHELL=/bin/bash
 PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
 MAX_PER_PROC=3
 ## integer only for the 5m load avg
 LOCK_WORKING_DIR="/var/run/myc-manage"
 mkdir -p "$LOCK_WORKING_DIR"
 int_avg=$(while read line; do
              echo "$line"
          done < /proc/loadavg)
 int_avg=${int_avg#* }
 int_avg=${int_avg%%.*}
 max=$[$MAX_PER_PROC * $(grep -c ^processor /proc/cpuinfo)]
 if [ "$int_avg" -gt "$max" ]; then
    if [ -e $LOCK_WORKING_DIR/load_average_max.lock ]; then
        exit 0
    else
        touch   $LOCK_WORKING_DIR/load_average_max.lock
        message="$(hostname) : WARNING - load average ($int_avg) is above max per processor : ($MAX_PER_PROC * $(grep -c ^processor /proc/cpuinfo) = $max)"
        echo $message | logger -t load_average_max
        send "load_average_max" "$message"
    fi
 fi