|
|
|
@ -18,6 +18,8 @@ version=0.1 |
|
|
|
desc='Install backup' |
|
|
|
help="" |
|
|
|
|
|
|
|
version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; } |
|
|
|
|
|
|
|
|
|
|
|
docker:running-container-projects() { |
|
|
|
:cache: scope=session |
|
|
|
@ -600,6 +602,15 @@ compose:service:container_first() { |
|
|
|
export -f compose:service:container_first |
|
|
|
|
|
|
|
|
|
|
|
docker:running_containers() { |
|
|
|
:cache: scope=session |
|
|
|
|
|
|
|
docker ps --format="{{.ID}}" |
|
|
|
} |
|
|
|
decorator._mangle_fn docker:running_containers |
|
|
|
export -f docker:running_containers |
|
|
|
|
|
|
|
|
|
|
|
compose:project:containers() { |
|
|
|
local project="$1" opts |
|
|
|
|
|
|
|
@ -1002,6 +1013,245 @@ EOF |
|
|
|
return $errlvl |
|
|
|
} |
|
|
|
|
|
|
|
docker:api() { |
|
|
|
local endpoint="$1" |
|
|
|
curl -sS --unix-socket /var/run/docker.sock "http://localhost$endpoint" |
|
|
|
} |
|
|
|
|
|
|
|
docker:containers:id() { |
|
|
|
docker:api /containers/json | jq -r ".[] | .Id" |
|
|
|
} |
|
|
|
docker:containers:names() { |
|
|
|
docker:api /containers/json | jq -r '.[] | .Names[0] | ltrimstr("/")' |
|
|
|
} |
|
|
|
|
|
|
|
docker:container:stats() { |
|
|
|
container="$1" |
|
|
|
docker:api "/containers/$container/stats?stream=false" |
|
|
|
} |
|
|
|
|
|
|
|
docker:containers:stats() { |
|
|
|
:cache: scope=session |
|
|
|
|
|
|
|
local jobs='' line container id_names sha names name data service project |
|
|
|
local DC="com.docker.compose" |
|
|
|
local PSF_values=( |
|
|
|
".ID" ".Names" ".Label \"$DC.project\"" ".Label \"$DC.service\"" ".Image" |
|
|
|
) |
|
|
|
local PSF="$(printf "{{%s}} " "${PSF_values[@]}")" |
|
|
|
id_names=$(docker ps -a --format="$PSF") || return 1 |
|
|
|
|
|
|
|
## Create a docker container table from name/sha to service, project, image_name |
|
|
|
declare -A resolve |
|
|
|
while read-0a line; do |
|
|
|
sha=${line%% *}; line=${line#* } |
|
|
|
names=${line%% *}; line=${line#* } |
|
|
|
names=(${names//,/ }) |
|
|
|
for name in "${names[@]}"; do |
|
|
|
resolve["$name"]="$line" |
|
|
|
done |
|
|
|
resolve["$sha"]="$line" |
|
|
|
done < <(printf "%s\n" "$id_names") |
|
|
|
|
|
|
|
declare -A data |
|
|
|
while read-0a line; do |
|
|
|
name=${line%% *}; line=${line#* } |
|
|
|
ts=${line%% *}; line=${line#* } |
|
|
|
|
|
|
|
resolved="${resolve["$name"]}" |
|
|
|
project=${resolved%% *}; resolved=${resolved#* } |
|
|
|
service=${resolved%% *}; resolved=${resolved#* } |
|
|
|
image_name="$resolved" |
|
|
|
if [ -z "$service" ]; then |
|
|
|
project="@" |
|
|
|
service=$(docker inspect "$image_name" | jq -r '.[0].RepoTags[0]') |
|
|
|
service=${service//\//_} |
|
|
|
fi |
|
|
|
if [ -n "${data["$project/$service"]}" ]; then |
|
|
|
previous=(${data["$project/$service"]}) |
|
|
|
previous=(${previous[@]:1}) |
|
|
|
current=($line) |
|
|
|
sum=() |
|
|
|
i=0; max=${#previous[@]} |
|
|
|
while (( i < max )); do |
|
|
|
sum+=($((${previous[$i]} + ${current[$i]}))) |
|
|
|
((i++)) |
|
|
|
done |
|
|
|
data["$project/$service"]="$ts ${sum[*]}" |
|
|
|
else |
|
|
|
data["$project/$service"]="$ts $line" |
|
|
|
fi |
|
|
|
done < <( |
|
|
|
for container in "$@"; do |
|
|
|
( |
|
|
|
docker:container:stats "${container}" | |
|
|
|
jq -r ' |
|
|
|
(.name | ltrimstr("/")) |
|
|
|
+ " " + (.read | sub("\\.[0-9]+Z"; "Z") | fromdate | tostring) |
|
|
|
+ " " + (.memory_stats.usage | tostring) |
|
|
|
+ " " + (.memory_stats.stats.inactive_file | tostring) |
|
|
|
+ " " + ((.memory_stats.usage - .memory_stats.stats.inactive_file) | tostring) |
|
|
|
+ " " + (.memory_stats.limit | tostring) |
|
|
|
+ " " + (.networks.eth0.rx_bytes | tostring) |
|
|
|
+ " " + (.networks.eth0.rx_packets | tostring) |
|
|
|
+ " " + (.networks.eth0.rx_errors | tostring) |
|
|
|
+ " " + (.networks.eth0.rx_dropped | tostring) |
|
|
|
+ " " + (.networks.eth0.tx_bytes | tostring) |
|
|
|
+ " " + (.networks.eth0.tx_packets | tostring) |
|
|
|
+ " " + (.networks.eth0.tx_errors | tostring) |
|
|
|
+ " " + (.networks.eth0.tx_dropped | tostring) |
|
|
|
' |
|
|
|
) & |
|
|
|
jobs=1 |
|
|
|
done |
|
|
|
[ -n "$jobs" ] && wait |
|
|
|
) |
|
|
|
for label in "${!data[@]}"; do |
|
|
|
echo "$label ${data[$label]}" |
|
|
|
done |
|
|
|
} |
|
|
|
decorator._mangle_fn docker:containers:stats |
|
|
|
export -f docker:containers:stats |
|
|
|
|
|
|
|
|
|
|
|
col:normalize:size() { |
|
|
|
local alignment=$1 |
|
|
|
|
|
|
|
awk -v alignment="$alignment" '{ |
|
|
|
# Store the entire line in the lines array. |
|
|
|
lines[NR] = $0; |
|
|
|
|
|
|
|
# Split the line into fields. |
|
|
|
split($0, fields); |
|
|
|
|
|
|
|
# Update max for each field. |
|
|
|
for (i = 1; i <= length(fields); i++) { |
|
|
|
if (length(fields[i]) > max[i]) { |
|
|
|
max[i] = length(fields[i]); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
END { |
|
|
|
# Print lines with fields padded to max. |
|
|
|
for (i = 1; i <= NR; i++) { |
|
|
|
split(lines[i], fields); |
|
|
|
|
|
|
|
line = ""; |
|
|
|
for (j = 1; j <= length(fields); j++) { |
|
|
|
# Get alignment for the current field. |
|
|
|
align = substr(alignment, j, 1); |
|
|
|
if (align != "+") { |
|
|
|
align = "-"; # Default to left alignment if not "+". |
|
|
|
} |
|
|
|
|
|
|
|
line = line sprintf("%" align max[j] "s ", fields[j]); |
|
|
|
} |
|
|
|
print line; |
|
|
|
} |
|
|
|
}' |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
rrd:create() { |
|
|
|
local prefix="$1" |
|
|
|
shift |
|
|
|
local label="$1" step="300" src_def |
|
|
|
shift |
|
|
|
if [ -z "$VAR_DIR" ]; then |
|
|
|
err "Unset \$VAR_DIR, can't create rrd graph" |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
mkdir -p "$VAR_DIR" |
|
|
|
if ! [ -d "$VAR_DIR" ]; then |
|
|
|
err "Invalid \$VAR_DIR: '$VAR_DIR' is not a directory" |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
if ! type -p rrdtool >/dev/null 2>&1; then |
|
|
|
apt-get install rrdtool -y --force-yes </dev/null |
|
|
|
if ! type -p rrdtool 2>/dev/null 2>&1; then |
|
|
|
err "Couldn't find nor install 'rrdtool'." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
fi |
|
|
|
|
|
|
|
local RRD_PATH="$VAR_DIR/rrd" |
|
|
|
|
|
|
|
local RRD_FILE="$RRD_PATH/$prefix/$label.rrd" |
|
|
|
mkdir -p "${RRD_FILE%/*}" |
|
|
|
if [ -f "$RRD_FILE" ]; then |
|
|
|
err "File '$RRD_FILE' already exists, use a different label." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
|
|
|
|
local rrd_ds_opts=() |
|
|
|
for src_def in "$@"; do |
|
|
|
IFS=":" read -r name type min max rra_types <<<"$src_def" |
|
|
|
rra_types=${rra_types:-average,max,min} |
|
|
|
rrd_ds_opts+=("DS:$name:$type:900:$min:$max") |
|
|
|
done |
|
|
|
|
|
|
|
local step=120 |
|
|
|
local times=( ## with steps 120 is 2mn datapoint |
|
|
|
2m:1w |
|
|
|
6m:3w |
|
|
|
30m:12w |
|
|
|
3h:1y |
|
|
|
1d:10y |
|
|
|
1w:2080w |
|
|
|
) |
|
|
|
rrd_rra_opts=() |
|
|
|
for time in "${times[@]}"; do |
|
|
|
rrd_rra_opts+=("RRA:"{AVERAGE,MIN,MAX}":0.5:$time") |
|
|
|
done |
|
|
|
cmd=( |
|
|
|
rrdtool create "$RRD_FILE" \ |
|
|
|
--step "$step" \ |
|
|
|
"${rrd_ds_opts[@]}" \ |
|
|
|
"${rrd_rra_opts[@]}" |
|
|
|
) |
|
|
|
|
|
|
|
"${cmd[@]}" || { |
|
|
|
err "Failed command: ${cmd[@]}" |
|
|
|
return 1 |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
rrd:update() { |
|
|
|
local prefix="$1" |
|
|
|
shift |
|
|
|
while read-0a data; do |
|
|
|
[ -z "$data" ] && continue |
|
|
|
IFS="~" read -ra data <<<"${data// /\~}" |
|
|
|
label="${data[0]}" |
|
|
|
ts="${data[1]}" |
|
|
|
for arg in "$@"; do |
|
|
|
IFS="|" read -r name arg <<<"$arg" |
|
|
|
rrd_label="${label}/${name}" |
|
|
|
rrd_create_opt=() |
|
|
|
rrd_update_opt="$ts" |
|
|
|
for col_def in ${arg//,/ }; do |
|
|
|
col=${col_def%%:*}; create_def=${col_def#*:} |
|
|
|
rrd_update_opt="${rrd_update_opt}:${data[$col]}" |
|
|
|
rrd_create_opt+=("$create_def") |
|
|
|
done |
|
|
|
local RRD_ROOT_PATH="$VAR_DIR/rrd" |
|
|
|
local RRD_PATH="$RRD_ROOT_PATH/${prefix%/}" |
|
|
|
local RRD_FILE="${RRD_PATH%/}/${rrd_label#/}.rrd" |
|
|
|
if ! [ -f "$RRD_FILE" ]; then |
|
|
|
info "Creating new RRD file '${RRD_FILE#$RRD_ROOT_PATH/}'" |
|
|
|
if ! rrd:create "$prefix" "${rrd_label}" "${rrd_create_opt[@]}" </dev/null ; then |
|
|
|
err "Couldn't create new RRD file ${rrd_label} with options: '${rrd_create_opt[*]}'" |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
fi |
|
|
|
rrdtool update "$RRD_FILE" "$rrd_update_opt" || { |
|
|
|
err "update failed with options: '$rrd_update_opt'" |
|
|
|
return 1 |
|
|
|
} |
|
|
|
done |
|
|
|
done |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -1728,4 +1978,232 @@ cmdline.spec::cmd:check-fix:run() { |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
awk:require() { |
|
|
|
local require_at_least="$1" version already_installed |
|
|
|
while true; do |
|
|
|
if ! version=$(awk --version 2>/dev/null); then |
|
|
|
version="" |
|
|
|
else |
|
|
|
version=${version%%,*} |
|
|
|
version=${version##* } |
|
|
|
fi |
|
|
|
if [ -z "$version" ] || version_gt "$require_at_least" "$version"; then |
|
|
|
if [ -z "$already_installed" ]; then |
|
|
|
if [ -z "$version" ]; then |
|
|
|
info "No 'gawk' available, probably using a clone. Installing 'gawk'..." |
|
|
|
else |
|
|
|
info "Found gawk version '$version'. Updating 'gawk'..." |
|
|
|
fi |
|
|
|
apt-get install gawk -y </dev/null || { |
|
|
|
err "Failed to install 'gawk'." |
|
|
|
return 1 |
|
|
|
} |
|
|
|
already_installed=true |
|
|
|
else |
|
|
|
if [ -z "$version" ]; then |
|
|
|
err "No 'gawk' available even after having installed one" |
|
|
|
else |
|
|
|
err "'gawk' version '$version' is lower than required" \ |
|
|
|
"'$require_at_least' even after updating 'gawk'." |
|
|
|
fi |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
continue |
|
|
|
fi |
|
|
|
return 0 |
|
|
|
done |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
cmdline.spec.gnu stats |
|
|
|
cmdline.spec::cmd:stats:run() { |
|
|
|
|
|
|
|
: :optval: --format,-f "Either 'silent', 'raw', or 'pretty', default is pretty." |
|
|
|
: :optfla: --silent,-s "Shorthand for '--format silent'" |
|
|
|
: :optval: --resource,-r 'resource(s) separated with a comma' |
|
|
|
|
|
|
|
|
|
|
|
local project_name service_name containers container check |
|
|
|
|
|
|
|
if [[ -n "${opt_silent}" ]]; then |
|
|
|
if [[ -n "${opt_format}" ]]; then |
|
|
|
err "'--silent' conflict with option '--format'." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
opt_format=s |
|
|
|
fi |
|
|
|
opt_format="${opt_format:-pretty}" |
|
|
|
case "${opt_format}" in |
|
|
|
raw|r) |
|
|
|
opt_format="raw" |
|
|
|
: |
|
|
|
;; |
|
|
|
silent|s) |
|
|
|
opt_format="silent" |
|
|
|
;; |
|
|
|
pretty|p) |
|
|
|
opt_format="pretty" |
|
|
|
awk:require 4.1.4 || return 1 |
|
|
|
;; |
|
|
|
*) |
|
|
|
err "Invalid value '$opt_format' for option --format" |
|
|
|
echo " use either 'raw' (shorthand 'r'), 'silent' (shorthand 's') or pretty (shorthand 'p')." >&2 |
|
|
|
return 1 |
|
|
|
esac |
|
|
|
|
|
|
|
local resources=(c.{memory,network} load_avg) |
|
|
|
if [ -n "${opt_resource}" ]; then |
|
|
|
resources=(${opt_resource//,/ }) |
|
|
|
fi |
|
|
|
|
|
|
|
local not_found=() |
|
|
|
for resource in "${resources[@]}"; do |
|
|
|
if ! fn.exists "stats:$resource"; then |
|
|
|
not_found+=("$resource") |
|
|
|
fi |
|
|
|
done |
|
|
|
|
|
|
|
if [[ "${#not_found[@]}" -gt 0 ]]; then |
|
|
|
not_found_msg=$(printf "%s, " "${not_found[@]}") |
|
|
|
not_found_msg=${not_found_msg%, } |
|
|
|
err "Unsupported resource(s) provided: ${not_found_msg}" |
|
|
|
echo " resource must be one-of:" >&2 |
|
|
|
declare -F | egrep -- '-fx? stats:[a-zA-Z0-9_.]+$' | cut -f 3- -d " " | cut -f 2- -d ":" | prefix " - " >&2 |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
|
|
|
|
:state-dir: |
|
|
|
|
|
|
|
for resource in "${resources[@]}"; do |
|
|
|
[ "$opt_format" == "pretty" ] && echo "${WHITE}$resource${NORMAL}:" |
|
|
|
stats:"$resource" "$opt_format" 2>&1 | prefix " " |
|
|
|
set_errlvl "${PIPESTATUS[0]}" || return 1 |
|
|
|
done |
|
|
|
} |
|
|
|
|
|
|
|
stats:c.memory() { |
|
|
|
local format="$1" |
|
|
|
local out |
|
|
|
container_to_check=($(docker:running_containers)) || exit 1 |
|
|
|
out=$(docker:containers:stats "${container_to_check[@]}") |
|
|
|
printf "%s\n" "$out" | rrd:update "containers" "memory|3:usage:GAUGE:U:U,4:inactive:GAUGE:U:U" || { |
|
|
|
return 1 |
|
|
|
} |
|
|
|
case "${format:-p}" in |
|
|
|
raw|r) |
|
|
|
printf "%s\n" "$out" | cut -f 1-5 -d " " |
|
|
|
;; |
|
|
|
pretty|p) |
|
|
|
awk:require 4.1.4 || return 1 |
|
|
|
{ |
|
|
|
echo "container" "__total____" "buffered____" "resident____" |
|
|
|
printf "%s\n" "$out" | |
|
|
|
awk ' |
|
|
|
{ |
|
|
|
offset = strftime("%z", $2); |
|
|
|
print $1, substr($0, index($0,$3)); |
|
|
|
}' | cut -f 1-4 -d " " | |
|
|
|
numfmt --field 2-4 --to=iec-i --format=%8.1fB | |
|
|
|
sed -r 's/(\.[0-9])([A-Z]?iB)/\1:\2/g' | |
|
|
|
sort |
|
|
|
} | col:normalize:size -+++ | |
|
|
|
sed -r 's/(\.[0-9]):([A-Z]?iB)/\1 \2/g' | |
|
|
|
header:make |
|
|
|
;; |
|
|
|
esac |
|
|
|
} |
|
|
|
|
|
|
|
stats:c.network() { |
|
|
|
local format="$1" |
|
|
|
local out |
|
|
|
container_to_check=($(docker:running_containers)) || exit 1 |
|
|
|
out=$(docker:containers:stats "${container_to_check[@]}") |
|
|
|
cols=( |
|
|
|
{rx,tx}_{bytes,packets,errors,dropped} |
|
|
|
) |
|
|
|
idx=5 ## starting column idx for next fields |
|
|
|
defs=() |
|
|
|
for col in "${cols[@]}"; do |
|
|
|
defs+=("$((idx++)):${col}:COUNTER:U:U") |
|
|
|
done |
|
|
|
OLDIFS="$IFS" |
|
|
|
IFS="," defs="${defs[*]}" |
|
|
|
IFS="$OLDIFS" |
|
|
|
printf "%s\n" "$out" | |
|
|
|
rrd:update "containers" \ |
|
|
|
"network|${defs}" || { |
|
|
|
return 1 |
|
|
|
} |
|
|
|
case "${format:-p}" in |
|
|
|
raw|r) |
|
|
|
printf "%s\n" "$out" | cut -f 1,2,7- -d " " |
|
|
|
;; |
|
|
|
pretty|p) |
|
|
|
awk:require 4.1.4 || return 1 |
|
|
|
{ |
|
|
|
echo "container" "_" "_" "_" "RX" "_" "_" "_" "TX" |
|
|
|
echo "_" "__bytes____" "__packets" "__errors" "__dropped" "__bytes____" "__packets" "__errors" "__dropped" |
|
|
|
printf "%s\n" "$out" | |
|
|
|
awk ' |
|
|
|
{ |
|
|
|
offset = strftime("%z", $2); |
|
|
|
print $1, substr($0, index($0,$7)); |
|
|
|
}' | |
|
|
|
numfmt --field 2,6 --to=iec-i --format=%8.1fB | |
|
|
|
numfmt --field 3,4,5,7,8,9 --to=si --format=%8.1f | |
|
|
|
sed -r 's/(\.[0-9])([A-Z]?(iB|B)?)/\1:\2/g' | |
|
|
|
sort |
|
|
|
} | col:normalize:size -++++++++ | |
|
|
|
sed -r ' |
|
|
|
s/(\.[0-9]):([A-Z]?iB)/\1 \2/g; |
|
|
|
s/(\.[0-9]):([KMGTPE])/\1 \2/g; |
|
|
|
s/ ([0-9]+)\.0:B/\1 /g; |
|
|
|
s/ ([0-9]+)\.0:/\1 /g; |
|
|
|
' | |
|
|
|
header:make 2 |
|
|
|
;; |
|
|
|
esac |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
header:make() { |
|
|
|
local nb_line="${1:-1}" |
|
|
|
local line |
|
|
|
while ((nb_line-- > 0)); do |
|
|
|
read-0a line |
|
|
|
echo "${GRAY}$(printf "%s" "$line" | sed -r 's/_/ /g')${NORMAL}" |
|
|
|
done |
|
|
|
cat |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
stats:load_avg() { |
|
|
|
local format="$1" |
|
|
|
local out |
|
|
|
out=$(host:sys:load_avg) |
|
|
|
printf "%s\n" "$out" | rrd:update "" "load_avg|2:load_avg_1:GAUGE:U:U,3:load_avg_5:GAUGE:U:U,4:load_avg_15:GAUGE:U:U" || { |
|
|
|
return 1 |
|
|
|
} |
|
|
|
case "${format:-p}" in |
|
|
|
raw|r) |
|
|
|
printf "%s\n" "$out" | cut -f 2-5 -d " " |
|
|
|
;; |
|
|
|
pretty|p) |
|
|
|
{ |
|
|
|
echo "___1m" "___5m" "__15m" |
|
|
|
printf "%s\n" "$out" | cut -f 3-5 -d " " |
|
|
|
} | col:normalize:size +++ | header:make |
|
|
|
;; |
|
|
|
esac |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
host:sys:load_avg() { |
|
|
|
local uptime |
|
|
|
uptime="$(uptime)" |
|
|
|
uptime=${uptime##*: } |
|
|
|
uptime=${uptime//,/} |
|
|
|
printf "%s " "" "$(date +%s)" "$uptime" |
|
|
|
} |
|
|
|
|
|
|
|
cmdline::parse "$@" |
