Browse Source

new: [send] add cron hourly for disk_usage and load_average_max

Boris Gallet 9 months ago
parent
commit
965630e9b9
  1. 63
      bin/myc-install
  2. 65
      bin/myc-update
  3. 89
      bin/send
  4. 69
      bin/vps
  5. 4
      etc/cron.d/check-backup
  6. 2
      etc/cron.d/monitor
  7. 7
      etc/cron.daily/remove_lock_file_48h
  8. 42
      etc/cron.hourly/disk_usage
  9. 32
      etc/cron.hourly/load_average_max

63
bin/myc-install

@ -145,6 +145,67 @@ pip install ovh ||
docker pull docker.0k.io/php:7.4-myc && docker tag docker.0k.io/php:7.4-myc myc_frontend &&
docker pull docker.0k.io/cron:jessie && docker tag docker.0k.io/cron:jessie myc_cron &&
## Creation of an account to send notification to ntfy server
echo "-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
NhAAAAAwEAAQAAAQEApGXqKYEJbv0xu/wKl1mXtiz90kZbqq7FALTZYyYqWZfsp4RtiHXi
NC7WKFiabQ1j1s8WuE0I2xJNSpzjHuWouduLQ5WtTl0PIWausMYaHam5T1I3KLVBg1QNi8
0wL5LVMD3mMoxVstQmlvYOuODZSaCS6j6ND33IS5IG11M9xwR6IcUKLKnfF5h5OQbTSiQ0
ANgw5KmYdGBQ8PUIQO0ELz0rhjJVZLADZspXLoWikNURmlYozfcSFcfOVA7AkqeMKMZd64
72WDGTd9NrAOq+hmLMKDfJXuHlKrNuqmK1jVGs/5YcSArrFyuvKOabT8AJfjBDEVtbsSeu
mN44MoH1bwAAA8hI4f+cSOH/nAAAAAdzc2gtcnNhAAABAQCkZeopgQlu/TG7/AqXWZe2LP
3SRluqrsUAtNljJipZl+ynhG2IdeI0LtYoWJptDWPWzxa4TQjbEk1KnOMe5ai524tDla1O
XQ8hZq6wxhodqblPUjcotUGDVA2LzTAvktUwPeYyjFWy1CaW9g644NlJoJLqPo0PfchLkg
bXUz3HBHohxQosqd8XmHk5BtNKJDQA2DDkqZh0YFDw9QhA7QQvPSuGMlVksANmylcuhaKQ
1RGaVijN9xIVx85UDsCSp4woxl3rjvZYMZN302sA6r6GYswoN8le4eUqs26qYrWNUaz/lh
xICusXK68o5ptPwAl+MEMRW1uxJ66Y3jgygfVvAAAAAwEAAQAAAQEAnzpm1tQ4QtvRc/Xm
fDk2jCh/n06uMl8cSFbhxvqMQkK34HiPboBfG5PRsTpAOCej78acht12Gllbq0zRXneqOH
nAJTGvrhrMMNm3kVgOq3RcG8vRyQfl8EFU7XdLmIhrHFKXx5XM22xIBCdGkyKU0o9IPMFg
9wQpH6jMH3psd3j9M7x9QwPZKujv5XMF7DrMdtwAsU/XPTHrOedxdmnVpy9hwTpygTP6Xs
TRL9CgdoIo1arZAu8M5/h8xS37IKEe4lUNr/j5tJe3d0HQ+aXCtVrD1WDyZnslPnrDr0MQ
XCbx957Kh6VJ11t8el7x21Yr0iuF+S/RSKxsiyqC3J4EAQAAAIAQabySOpcNGk/kR3A7KH
Szz3uft+c9qmt9+b5Sth+GmRKEoOO51hi3K+WrzArMJ3AyO8QGodjBAStcbFMDW5DkWxFH
0BuuXL2JTNJdn/2iBQH2bjLI68zTCrqHapI4l3kwTFUDybZP9hcdN9QrsY10rh+WiUILt7
gIB69cxQKeuAAAAIEAz7W5MrUL50A5wi7EMalR9+dIVDTvpyub7Ip6dczRyXt9Xz35mv4S
pBaK0mabJPgNP23fGoDhsXhZoDxJpGaBMCciLujVt/wJCX+vXbXwBXwMi0DC9AF/W6FGYb
GAusBeJzziXuEmQlirbKFTwkBMVOpMWvsX/DQgDjsTVKjm9u8AAACBAMqeZ17+r7602t9P
8Gie1YXde/T1vMeQAHNCOCBoiuERdM/xDyQE3EXk9Pj2LOvhEu9CskQUCkuZS4JcMO+GQz
zfqty7nmi/sAQTwPe+Gl9dnvS+dixeDmS3g+rP+hEAYhVPQgQhm0zzMvzlYk437WB+9BYk
JUr3Zp+T2t4WoOmBAAAAEm50ZnlAY29yZS0wMS4way5pbw==
-----END OPENSSH PRIVATE KEY-----" > /root/.ssh/ntfy-key
echo "|1|e3yYRMYJg0EpbOeTplTgtI+KbY4=|PotgCF8Rwt2OZFKr1CGYWpJ6FRA= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH8axkuXlI2zowRvL3Vyg/qgkKK57cqX7+9WRaLm9ECWkLvaVPGunR1zVJUZdTO3gjlSkqtblTcI00BBLt+zQvE=" \
>> /root/.ssh/known_hosts
chmod 600 /root/.ssh/ntfy-key
## Request token to ntfy server and add to config file
cred=$(ssh -i /root/.ssh/ntfy-key ntfy@core-01.0k.io request-token) || >&2 echo "Error while requesting token to ntfy server"
login_ntfy=$(printf "%s" "${cred%$'\n'*}")
password_ntfy=$(printf "%s" "${cred#$'\n'*}")
## if the config file exist and LOGIN PASSWORD ARE already in we do nothing
if [ -z "$login_ntfy"] || [[ "$login_ntfy" == *$'\n'*]]; then
echo "Error: couldn’t infer credential from ntfy server" >&2;
printf "%s" "$cred" | sed -r 's/^ |/g' >&2;
exit
fi
config_file="/etc/ntfy/ntfy.conf"
mkdir -p "${config_file%/*}"
if [ -f "$config_file" ] || touch $config_file || {
echo "Error: couldn’t create config file $config_file" >&2;
exit 1
}; then
if grep -qE '^LOGIN=' "$config_file"; then
sed -i "s/^LOGIN=.*/LOGIN='$login'/" "$config_file"
else
echo "LOGIN='$login'" >> "$config_file"
fi
if grep -qE '^PASSWORD=' "$config_file"; then
sed -i "s/^PASSWORD=.*/PASSWORD='$password'/" "$config_file"
else
echo "PASSWORD='$password'" >> "$config_file"
fi
fi
## Marker to probe if this script finished it's job
echo "done" > /var/run/myc-installer.0k.io.state
echo "done" > /var/run/myc-installer.0k.io.state

65
bin/myc-update

@ -96,7 +96,7 @@ for d in /etc/cron.{d,daily,hourly,monthly,weekly}; do
ln -sfn "/opt/apps/myc-manage\$d/"* "\$d/" &&
find -L "\$d" -maxdepth 1 -type l -ilname "/opt/apps/myc-manage\$d/"\* -delete
done
EOF
EOF
Wrap -d "Updating sysctl scripts" <<EOF || exit 1
for d in /etc/sysctl.d; do
@ -105,6 +105,14 @@ for d in /etc/sysctl.d; do
done
EOF
## TODO GPG decript the keys and place them
# Wrap -d "Updating ssh keys" <<EOF || exit 1
# for d in /etc/ssh; do
# ln -sfn "/opt/apps/myc-manage\$d/"* "\$d/" &&
# find -L "\$d" -maxdepth 1 -type l -ilname "/opt/apps/myc-manage\$d/"\* -delete
# done
# EOF
if [ -f "/root/.bashrc" ]; then
Wrap -d "Enable colors in bash" <<'EOF' || exit 1
sed -ri 's/^# (export LS_OPTIONS=.--color=auto.)/\1/;
@ -113,6 +121,61 @@ sed -ri 's/^# (export LS_OPTIONS=.--color=auto.)/\1/;
EOF
fi
Wrap -d "Update authorization to send to ntfy server " <<'EOF' || exit 1
mkdir -p /root/.ssh
## if file /root/.ssh/ntfy-key doesn’t exist we we create the key i
if [ ! -f /root/.ssh/ntfy-key ]; then
echo "-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
NhAAAAAwEAAQAAAQEApGXqKYEJbv0xu/wKl1mXtiz90kZbqq7FALTZYyYqWZfsp4RtiHXi
NC7WKFiabQ1j1s8WuE0I2xJNSpzjHuWouduLQ5WtTl0PIWausMYaHam5T1I3KLVBg1QNi8
0wL5LVMD3mMoxVstQmlvYOuODZSaCS6j6ND33IS5IG11M9xwR6IcUKLKnfF5h5OQbTSiQ0
ANgw5KmYdGBQ8PUIQO0ELz0rhjJVZLADZspXLoWikNURmlYozfcSFcfOVA7AkqeMKMZd64
72WDGTd9NrAOq+hmLMKDfJXuHlKrNuqmK1jVGs/5YcSArrFyuvKOabT8AJfjBDEVtbsSeu
mN44MoH1bwAAA8hI4f+cSOH/nAAAAAdzc2gtcnNhAAABAQCkZeopgQlu/TG7/AqXWZe2LP
3SRluqrsUAtNljJipZl+ynhG2IdeI0LtYoWJptDWPWzxa4TQjbEk1KnOMe5ai524tDla1O
XQ8hZq6wxhodqblPUjcotUGDVA2LzTAvktUwPeYyjFWy1CaW9g644NlJoJLqPo0PfchLkg
bXUz3HBHohxQosqd8XmHk5BtNKJDQA2DDkqZh0YFDw9QhA7QQvPSuGMlVksANmylcuhaKQ
1RGaVijN9xIVx85UDsCSp4woxl3rjvZYMZN302sA6r6GYswoN8le4eUqs26qYrWNUaz/lh
xICusXK68o5ptPwAl+MEMRW1uxJ66Y3jgygfVvAAAAAwEAAQAAAQEAnzpm1tQ4QtvRc/Xm
fDk2jCh/n06uMl8cSFbhxvqMQkK34HiPboBfG5PRsTpAOCej78acht12Gllbq0zRXneqOH
nAJTGvrhrMMNm3kVgOq3RcG8vRyQfl8EFU7XdLmIhrHFKXx5XM22xIBCdGkyKU0o9IPMFg
9wQpH6jMH3psd3j9M7x9QwPZKujv5XMF7DrMdtwAsU/XPTHrOedxdmnVpy9hwTpygTP6Xs
TRL9CgdoIo1arZAu8M5/h8xS37IKEe4lUNr/j5tJe3d0HQ+aXCtVrD1WDyZnslPnrDr0MQ
XCbx957Kh6VJ11t8el7x21Yr0iuF+S/RSKxsiyqC3J4EAQAAAIAQabySOpcNGk/kR3A7KH
Szz3uft+c9qmt9+b5Sth+GmRKEoOO51hi3K+WrzArMJ3AyO8QGodjBAStcbFMDW5DkWxFH
0BuuXL2JTNJdn/2iBQH2bjLI68zTCrqHapI4l3kwTFUDybZP9hcdN9QrsY10rh+WiUILt7
gIB69cxQKeuAAAAIEAz7W5MrUL50A5wi7EMalR9+dIVDTvpyub7Ip6dczRyXt9Xz35mv4S
pBaK0mabJPgNP23fGoDhsXhZoDxJpGaBMCciLujVt/wJCX+vXbXwBXwMi0DC9AF/W6FGYb
GAusBeJzziXuEmQlirbKFTwkBMVOpMWvsX/DQgDjsTVKjm9u8AAACBAMqeZ17+r7602t9P
8Gie1YXde/T1vMeQAHNCOCBoiuERdM/xDyQE3EXk9Pj2LOvhEu9CskQUCkuZS4JcMO+GQz
zfqty7nmi/sAQTwPe+Gl9dnvS+dixeDmS3g+rP+hEAYhVPQgQhm0zzMvzlYk437WB+9BYk
JUr3Zp+T2t4WoOmBAAAAEm50ZnlAY29yZS0wMS4way5pbw==
-----END OPENSSH PRIVATE KEY-----" > /root/.ssh/ntfy-key
chmod 600 /root/.ssh/ntfy-key
fi
if ! ssh-keygen -F core-01.0k.io &> /dev/null; then
echo "|1|e3yYRMYJg0EpbOeTplTgtI+KbY4=|PotgCF8Rwt2OZFKr1CGYWpJ6FRA= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH8axkuXlI2zowRvL3Vyg/qgkKK57cqX7+9WRaLm9ECWkLvaVPGunR1zVJUZdTO3gjlSkqtblTcI00BBLt+zQvE=" \
>> /root/.ssh/known_hosts
fi
config_file="/etc/ntfy/ntfy.conf"
mkdir -p "${config_file%/*}"
## if the config file exist and LOGIN PASSWORD ARE already in we do nothing
if [ -f "$config_file" ] && grep -qE '^LOGIN=|^PASSWORD=' "$config_file"; then
echo "We found a configuration for ntfy server authentification located at $config_file"
else
cred=$(ssh -i /root/.ssh/ntfy-key ntfy@core-01.0k.io request-token)
login_ntfy=$(echo $cred | awk '/^h_/{print $1; exit}')
password_ntfy=$(echo $cred | awk '{print $2; exit}')
if [ -f "$config_file" ]; then
echo "LOGIN='$login_ntfy'" >> "$config_file"
echo "PASSWORD='$password_ntfy'" >> "$config_file"
else
echo "LOGIN='$login_ntfy'" >> "$config_file"
echo "PASSWORD='$password_ntfy'" >> "$config_file"
fi
fi
EOF
for keyfile in {/root,/home/debian}/.ssh/authorized_keys; do
[ -e "$keyfile" ] || continue

89
bin/send

@ -5,35 +5,88 @@
if [[ "$UID" == "0" ]]; then
NTFY_CONFIG_FILE="/etc/ntfy/ntfy.conf"
else
NTFY_CONFIG_FILE="$HOME/.config/ntfy/ntfy.conf"
NTFY_CONFIG_FILE=~/.config/ntfy/ntfy.conf
fi
SERVER="https://ntfy.0k.io/"
if ! [ -e "$NTFY_CONFIG_FILE" ]; then
mkdir -p "${NTFY_CONFIG_FILE%/*}"
## default option to change if needed
echo 'SERVER="https://ntfy.0k.io/"' > "$NTFY_CONFIG_FILE"
else
source "$NTFY_CONFIG_FILE"
for var in TOKEN SERVER; do
if ! [ -v "$var" ]; then
echo "Error: missing $var in $NTFY_CONFIG_FILE"
exit 1
fi
done
echo "SERVER=$SERVER" > "$NTFY_CONFIG_FILE"
elif ! grep -q "^SERVER=" "$NTFY_CONFIG_FILE"; then
echo "SERVER=$SERVER" >> "$NTFY_CONFIG_FILE"
fi
source "$NTFY_CONFIG_FILE"
for var in SERVER LOGIN PASSWORD; do
if ! [ -v "$var" ]; then
echo "Error: missing $var in $NTFY_CONFIG_FILE"
exit 1
fi
done
exname=${0##*/}
usage="Usage: $exname CHANNEL MESSAGE"
channel="main"
usage="Usage: $exname [-c CHANNEL] MESSAGE
----------------------------------------------
--- Send MESSAGE to the specified CHANNEL. ---
----------------------------------------------
If no CHANNEL is provided, the message will be sent to the default channel
Default CHANNEL is format as follow : ConfiguredLOGIN_${default_channel}"
while [[ $# -gt 0 ]]; do
arg="$1"
shift
case "$arg" in
-h|--help)
echo "$usage"
exit 0
;;
-c|--channel)
channel="$1"
[ -z "$channel" ] || {
echo "Error: no argument for channel option." >&2
echo "$usage" >&2
exit 1
}
shift
;;
-t|--title)
title="$1"
[ -z "$title" ] || {
echo "Error: no argument for title option." >&2
echo "$usage" >&2
exit 1
}
shift
;;
*)
[ -z "$message" ] && { message="$arg"; continue; }
echo "Error : Unexpected positional argument '$arg'." >&2
echo "$usage" >&2
exit 1
;;
esac
done
if [ "$#" -ne 2 ]; then
[ -n "$message" ] || {
echo "Error: missing message." >&2
echo "$usage" >&2
exit 1
fi
}
channel="$1"
message="$2"
curl_opts=(
-s
-u "$LOGIN:$PASSWORD"
-d "$message"
)
curl -s -H "Authorization: Bearer $TOKEN" \
-d "$message" "$SERVER/$channel" > /dev/null
if [ -n "$title" ]; then
curl_opts+=(-H "Title: [$(hostname)] $title")
fi
curl "${curl_opts[@]}" "$SERVER/${LOGIN}_$channel" > /dev/null

69
bin/vps

@ -2615,4 +2615,73 @@ cmdline.spec:monujo:cmd:set-version:run() {
}
cmdline.spec::cmd:check:run() {
:
}
cmdline.spec.gnu check
cmdline.spec:check:cmd:backup:run() {
: :optfla: --notify,-n "Send result through notify"
: :optval: --service,-s "The backup service name (defaults to 'rsync-backup')"
## Check on daily bases if backup exist in config and when is the last backup done :
## ALERT if backup is set and last backup is older than 24h
local STATE_FILE="/var/run/myc-manage/backup.state"
mkdir -p "${STATE_FILE%/*}"
service=${opt_service:-rsync-backup}
project_name=$(compose:project_name) || exit 1
## check if service exists in compose.yml
if ! compose:service:exists "$project_name" "$service"; then
warn "no service ${DARKYELLOW}$service${NORMAL}. Ignoring."
return 0
fi
last_backup_datetime=$(
cat /srv/datastore/data/cron/var/log/cron/*rsync-backup_script{_*,}.log | grep "total size is" | sort | tail -n 1 | cut -f -2 -d " ")
last_backup_ts=$(date -d "$last_backup_datetime" +%s)
max_ts=$(date -d "24 hours ago" +%s)
state="ok"
if [ "$last_backup_ts" -lt "$max_ts" ]; then
state="ko"
fi
if [ -z "$opt_notify" ]; then
if [ "$state" == "ok" ]; then
info "Everything is ${GREEN}ok${NORMAL}. (last backup: ${DARKCYAN}$last_backup_datetime${NORMAL})"
return 0
fi
warn "Last backup older than 1 day. (last backup: ${DARKCYAN}$last_backup_datetime${NORMAL})"
return 1
fi
## notify
last_state=$(cat "$STATE_FILE" 2>/dev/null) || true
if [ "$state" == "$last_state" ]; then
[ "$state" == "ko" ] || return 0
is_old=$(find "$STATE_FILE" -type f -mtime +2) || return 1
[ -n "$is_old" ] || return 0
fi
echo "$state" > "$STATE_FILE"
message="[$(hostname)]: WARNING no backup done in the last 24h (No backup since $days days and $hours hours)"
timestamp=$(date +%s)
time_difference=$((timestamp - last_backup_ts))
days=$((time_difference / 86400))
hours=$((time_difference % 86400 / 3600))
message="WARNING: no backup done in the last 24h (No backup since $days days and $hours hours)"
send -t "ALERT Backup" "$message"
}
cmdline::parse "$@"

4
etc/cron.d/check-backup

@ -0,0 +1,4 @@
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
12 * * * * root vps check backup -n 2>&1 | logger -t stats

2
etc/cron.d/monitor

@ -1,4 +1,4 @@
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
*/2 * * * * root lock vps-stats -v -D -p 10 -k -c "vps stats -s" 2>&1 | logger -t stats
*/2 * * * * root lock vps-stats -v -D -p 10 -k -c "vps stats -s" 2>&1 | logger -t stats

7
etc/cron.daily/remove_lock_file_48h

@ -0,0 +1,7 @@
#!/bin/bash
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
## Remove every .lock file older than 48h -- CLeanup script
find /var/run/myc-manage -name "*.lock" -type f -mtime +2 -delete

42
etc/cron.hourly/disk_usage

@ -0,0 +1,42 @@
#!/bin/bash
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
## check disk usage and send a notification if it's above 75% or 90%
percent_usage=$(df /srv -h)
percent_usage=${percent_usage##*$'\n'}
percent_usage=${percent_usage% *}
percent_usage=${percent_usage##* }
percent_usage=${percent_usage%\%}
LOCK_WORKING_DIR="/var/run/myc-manage"
mkdir -p "$LOCK_WORKING_DIR"
if [ "$percent_usage" -ge "90" ]; then
if [ -e $LOCK_WORKING_DIR/disk_usage_90.lock ]; then
exit 0
else
touch $LOCK_WORKING_DIR/disk_usage_90.lock
message="[$(hostname)]: WARNING disk usage >=90%"
send "$message"
fi
elif [ "$percent_usage" -ge "75" ]; then
if [ -e $LOCK_WORKING_DIR/disk_usage_75.lock ]; then
exit 0
else
touch $LOCK_WORKING_DIR/disk_usage_75.lock
message="[$(hostname)]: WARNING disk usage >=75 <90%"
send "$message"
fi
else
if [ -e $LOCK_WORKING_DIR/disk_usage_75.lock ]; then
rm $LOCK_WORKING_DIR/disk_usage_75.lock
fi
if [ -e $LOCK_WORKING_DIR/disk_usage_90.lock ]; then
rm $LOCK_WORKING_DIR/disk_usage_90.lock
fi
fi

32
etc/cron.hourly/load_average_max

@ -0,0 +1,32 @@
#!/bin/bash
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
MAX_PER_PROC=3
## integer only for the 5m load avg
LOCK_WORKING_DIR="/var/run/myc-manage"
mkdir -p "$LOCK_WORKING_DIR"
int_avg=$(while read line; do
echo "$line"
done < /proc/loadavg)
int_avg=${int_avg#* }
int_avg=${int_avg%%.*}
max=$[$MAX_PER_PROC * $(grep -c ^processor /proc/cpuinfo)]
if [ "$int_avg" -gt "$max" ]; then
if [ -e $LOCK_WORKING_DIR/load_average_max.lock ]; then
exit 0
else
touch $LOCK_WORKING_DIR/load_average_max.lock
message="[$(hostname)] : WARNING - load average ($int_avg) is above max per processor : ($MAX_PER_PROC * $(grep -c ^processor /proc/cpuinfo) = $max)"
echo $message | logger -t load_average_max
send "$message"
fi
else
if [ -e $LOCK_WORKING_DIR/load_average_max.lock ]; then
rm $LOCK_WORKING_DIR/load_average_max.lock
fi
fi
Loading…
Cancel
Save