new: [0km] add graphing for disk-2 data

bin/0km

@@ -168,6 +168,14 @@ ssh:open-try() {
     return 1
 }
 
+rscolcat:check-install() {
+    type -p rscolcat >/dev/null || {
+        err "'rscolcat' is not installed. Please install it."
+        echo "  sudo wget https://docker.0k.io/downloads/rscolcat-0.1.0 -O /usr/local/bin/rscolcat && " >&2
+        echo "  sudo chmod +x /usr/local/bin/rscolcat" >&2
+        return 1
+    }
+}
 
 ssh:run() {
     local hostname="$1" ssh_options cmd
@@ -1521,7 +1529,8 @@ cmdline.spec::cmd:vps-stats:run() {
         echo ""
         return 1
     } >&2
-
+    rrd_tmpdir=$(mktemp -d -t rrd.XXXXXX)
+    export rrd_tmpdir
     export GNUTERM=qt
     ## rrdtool fetch will use comma for floating point depending on some locals !
     export LC_ALL=C
@@ -1827,6 +1836,166 @@ graph:def:disk() {
 }
 
 
+graph:def:disk-2() {
+    local vps="$1" i="$2"
+    shift 2
+    local opts_rrdfetch=("$@")
+    rrd_vps_path="$VAR_DIR/rrd/$vps"
+    rscolcat:check-install || return 1
+    folders=(
+        $(
+            cd "$rrd_vps_path/$resource/data";
+            find -maxdepth 2 -mindepth 2 -name "size.rrd" -type f |
+                sed -r 's%^./([^/]+)/[^/]+.rrd$%data/\1%g'
+        )
+    )
+    gnuplot_line_config=(
+        "set term qt $i title \"$vps $resource\" replotonresize noraise"
+        "set title '$vps'"
+        "set xdata time"
+        "set timefmt '%s'"
+        "set ylabel '${resource//_/\\_} Usage'"
+        "set format y '%s'"
+        "set ytics format '%g GiB'"
+        "set mouse mouseformat 6"
+        "set yrange [0:*] "
+        "set border behind"
+        'set key outside top right'
+    )
+    declare -A colors
+    colors[host/other]="#000000"
+    colors[docker/build_cache]="#87CEEB"
+    colors[docker/containers]="#0047AB"
+    colors[docker/images]="#00FFFF"
+    colors[docker/local_volumes]="#6495ED"
+    colors[docker/logs]="#000080"
+    
+    
+    printf "%s\n" "${gnuplot_line_config[@]}"
+    first=1
+    rrdfetch_cmd="'< rscolcat concat"
+    lines_def=()
+    legend_def=()
+    label="${value%:*}"
+    idx=1
+    previous_sum=0
+    values=(
+        host/other
+        docker/{build_cache,local_volumes,logs,containers,images}
+        "${folders[@]}"
+    )
+    declare -A values_colnb
+    for subvalue in "${values[@]}" ; do
+        idx=$((idx + 1))
+        values_colnb["$subvalue"]="$idx"
+        [ -e "$rrd_vps_path/$resource/$subvalue/size.rrd" ] || {
+            warn "No data yet for $subvalue on vps $vps, ignoring..."
+            continue
+        }
+        rrdfetch_cmd+=" \\"$'\n'
+        rrdfetch_cmd+="    <(rrdtool fetch \"$rrd_vps_path/$resource/$subvalue/size.rrd\""
+        rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
+        rrdfetch_cmd+="         tail -n +2 | \\"$'\n'
+        rrdfetch_cmd+="         egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\")"
+
+    done
+    rrdfetch_cmd+="'"
+    rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
+    rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
+    eval "$rrdfetch_cmd_bash" > "$rrd_tmpdir/data.txt" || return 1
+    first_ts=
+    first_ts=$(cat "$rrd_tmpdir/data.txt" | head -n 1 | cut -f 1 -d " ")
+    if [ -z "$first_ts" ]; then
+        err "No data for $resource on vps $vps, skipping..."
+        return 1
+    fi
+    last_ts=$(cat "$rrd_tmpdir/data.txt" | tail -n 1 | cut -f 1 -d " ")
+    last_valued_line=$(cat "$rrd_tmpdir/data.txt" | egrep -v '^[0-9]+( -)+' | tail -n 1) || return 1
+    if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
+        data_start_ts="$first_ts"
+    fi
+    if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
+        data_stop_ts="$last_ts"
+    fi
+    last_values=(${last_valued_line#* })
+    idx=0
+    data_sorted_values=$(
+        for last_value in "${last_values[@]}"; do
+            [[ "${values[$idx]}" != "data/"* ]] && {
+                idx=$((idx + 1))
+                continue
+            }
+            if [ "$last_value" == "-" ]; then
+                last_value=0
+            fi
+            printf "%s %.0f\n" "${values[$idx]}" "$last_value"
+            idx=$((idx + 1))
+        done | sort -k 2 -n | cut -f 1 -d " "
+                      )
+    data_sorted_values=(${data_sorted_values})
+    color_step=$(( 255 / ${#data_sorted_values[@]} ))
+    toggle=10
+    for label in "${data_sorted_values[@]}"; do
+        if [ "$toggle" -eq 10 ]; then
+            toggle=240
+        else
+            toggle=10
+        fi
+        colors[$label]=$(printf "#%02x%02x%02x" "$toggle" $(( 255 - (color_step * idx) )) $(( color_step * idx))      )
+        idx=$((idx + 1))
+    done
+    echo "plot \\"
+    first=1
+    line_def=""
+    ordered_values=(
+        host/other
+        docker/{build_cache,local_volumes,logs,containers,images}
+        "${data_sorted_values[@]}"
+    )
+    idx=0
+    for subvalue in "${ordered_values[@]}" ; do
+        colnb="${values_colnb[$subvalue]}"
+        sum_str="(valid(${colnb}) ? column(${colnb}) : 0)"
+        line_def=""
+        if [ -n "$first" ]; then
+            current_sum+="${sum_str}"
+            first=
+            line_def="u 1:(0):(1/(1/((${current_sum})/(1024*1024)))) w filledcurves title '${subvalue//_/\\_}'"
+        else
+            echo -n ", \\"$'\n'
+            current_sum+=" + $sum_str"
+            line_def="u 1:(1/(1/((${previous_sum})/(1024*1024)))):(1/(1/((${current_sum})/(1024*1024)))) w filledcurves title '${subvalue//_/\\_}'"
+        fi
+        previous_sum="$current_sum"
+        if [[ -n "${colors[$subvalue]}" ]]; then
+            line_def+=" lc rgb '${colors[$subvalue]}'"
+        fi
+        echo -n "  '$rrd_tmpdir/data.txt' ${line_def}"
+    done
+    ## Add line with "available" value from host/available
+    echo -n ", \\"$'\n'
+    echo -n "  '< rrdtool fetch \"$rrd_vps_path/$resource/host/available/size.rrd\""
+    echo -n " AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
+    echo -n "         tail -n +2 | \\"$'\n'
+    echo -n "         egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
+    echo -n " u 1:(\$2/(1024*1024)) w lines title 'available' lw 2 lc rgb '#FF0000'"
+    ## Add dashed line with 75% of "available" value from host/available
+    echo -n ", \\"$'\n'
+    echo -n "  '< rrdtool fetch \"$rrd_vps_path/$resource/host/available/size.rrd\""
+    echo -n " AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
+    echo -n "         tail -n +2 | \\"$'\n'
+    echo -n "         egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
+    echo -n " u 1:((3 * \$2)/(4*1024*1024)) w lines title '75% available' lw 1 dashtype 2 lc rgb '#FF0000'"
+    ## Add dashed line with 95% of "available" value from host/available
+    echo -n ", \\"$'\n'
+    echo -n "  '< rrdtool fetch \"$rrd_vps_path/$resource/host/available/size.rrd\""
+    echo -n " AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
+    echo -n "         tail -n +2 | \\"$'\n'
+    echo -n "         egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
+    echo -n " u 1:((9 * \$2)/(10*1024*1024)) w lines title '90% available' lw 2 dashtype 2 lc rgb '#FF0000'"
+    echo
+
+}