myc-manage/bin/0km

#!/bin/bash


. /etc/shlib

include common
include parse
include cmdline
include config


[[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true

version=0.1
desc='Manage 0k related installs'
help=""


##
## Functions
##

is-port-open() {
    local host="$1" port="$2" timeout=5
    start="$SECONDS"
    debug "Testing if $host's port $2 is open ..."
    while true; do
        timeout 1 bash -c "</dev/tcp/${host}/${port}" >/dev/null 2>&1 && break
        sleep 0.2
        if [ "$((SECONDS - start))" -gt "$timeout" ]; then
            return 1
        fi
    done
    debug "  .. $host's port $2 is open."
    return 0
}

resolve() {
    local ent hostname="$1"
    debug "Resolving $1 ..."
    if ent=$(getent ahosts "$hostname"); then
        ent=$(echo "$ent" | egrep ^"[0-9]+.[0-9]+.[0-9]+.[0-9]+\s+" | \
                  head -n 1 | awk '{ print $1 }')
        debug "  .. resolved $1 to $ent."
        echo "$ent"
    else
        debug "  .. couldn't resolve $1."
        return 1
    fi
}


set_errlvl() { return "${1:-1}"; }

export master_pid=$$
ssh:open() {
    local hostname ssh_cmd ssh_options
    ssh_cmd=(ssh)
    ssh_options=()
    while [ "$#" != 0 ]; do
        case "$1" in
            "--stdin-password")
                ssh_cmd=(sshpass "${ssh_cmd[@]}")
                ;;
            -o)
                ssh_options+=("$1" "$2")
                shift
                ;;
            *)
                [ -z "$hostname" ] && hostname="$1" || {
                        err "Surnumerous positional argument '$1'. Expecting only hostname."
                        return 1
                    }
                ;;
        esac
        shift
    done
    full_cmd=(
        "${ssh_cmd[@]}"
        -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
        -o ControlMaster=auto -o ControlPersist=900  \
        -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
        "${ssh_options[@]}" \
        "$hostname" "$@" -- true)
    "${full_cmd[@]}" >/dev/null 2>&1 || {
        err "Failed: ${full_cmd[*]}"
        return 1
    }
    trap_add EXIT,INT 'ssh:quit "$hostname"'
}

ssh:rsync() {
    local src="$1" dst="$2"
    hostname=${src%%:*}
    hostname=${hostname#*@}
    local rsync_ssh_options=(
        -o ControlPath="/tmp/ssh-control-master-${master_pid}-$hostname"
        -o ControlMaster=auto
        -o ControlPersist=900
        -o ConnectTimeout=10
        -o StrictHostKeyChecking=no
    )
    if ! ssh:run "root@$hostname" -- type -p rsync </dev/null >/dev/null; then
        info "No 'rsync' available on '$hostname'. Requesting installation..."
        ssh:run "root@$hostname" -- apt-get install rsync -y || {
            err "Installation of 'rsync' failed on '$hostname'"
            return 1
        }
    fi

    local cmd=(
        rsync -e "ssh ${rsync_ssh_options[*]}"
        -azvArH --delete --delete-excluded
        --partial --partial-dir .rsync-partial
        "$src" "$dst"
    )
    "${cmd[@]}"
}

ssh:open-try() {
    local opts hostnames
    opts=()
    hostnames=()
    while [ "$#" != 0 ]; do
        case "$1" in
            -o)
                opts+=("$1" "$2")
                shift
                ;;
            *)
                hostnames+=("$1")
                ;;
        esac
        shift
    done
    password=''
    for host in "${hostnames[@]}"; do
        debug "Trying $host with publickey."
        ssh:open -o PreferredAuthentications=publickey \
                 "${opts[@]}" \
                 "$host" >/dev/null 2>&1 && {
            echo "$host"$'\n'"$password"$'\n'
            return 0
        }
        debug "  .. failed connecting to $host with publickey."
    done
    local times=0 password
    while [ "$((++times))" -le 3 ]; do
        read -sp "$HOST's password: " password
        errlvl="$?"
        echo >&2
        if [ "$errlvl" -gt 0 ]; then
            exit 1
        fi
        for host in "${hostnames[@]}"; do
            debug "Trying $host with password ($times/3)"
            echo "$password" | ssh:open -o PreferredAuthentications=password \
                                        --stdin-password \
                                        "${opts[@]}" \
                                        "$host" >/dev/null 2>&1 && {
                echo "$host"$'\n'"$password"$'\n'
                return 0
            }
            debug "  .. failed connecting to $host with password."
        done
        err "login failed. Try again... ($((times+1))/3)"
    done
    return 1
}


ssh:run() {
    local hostname="$1" ssh_options cmd
    shift

    ssh_options=()
    cmd=()
    while [ "$#" != 0 ]; do
        case "$1" in
            "--")
                shift
                cmd+=("$@")
                break
                ;;
            *)
                ssh_options+=("$1")
                ;;
        esac
        shift
    done
    ## XXXvlab: keeping in case we need some debug
    # debug "$DARKCYAN$hostname$NORMAL $WHITE\$$NORMAL" "$@"
    # debug "Running cmd: ${cmd[*]}"
    # for arg in "${cmd[@]}"; do
    #     debug "$arg"
    # done
    {
        {
            ssh -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
                -o ControlMaster=auto -o ControlPersist=900 \
                -o "StrictHostKeyChecking=no" \
                 "${ssh_options[@]}" "$hostname" -- "${cmd[@]}"
        } 3>&1 1>&2 2>&3  ## | sed -r "s/^/$DARKCYAN$hostname$NORMAL $DARKRED\!$NORMAL /g"
        set_errlvl "${PIPESTATUS[0]}"
    } 3>&1 1>&2 2>&3
}

ssh:quit() {
    local hostname="$1"
    shift
    ssh -o ControlPath=/tmp/ssh-control-master-${master_pid} \
        -o ControlMaster=auto -o ControlPersist=900 -O exit \
        "$hostname" 2>/dev/null
}


is_ovh_domain_name() {
    local domain="$1"

    [[ "$domain" == *.ovh.net ]] && return 0
    [[ "$domain" == "ns"*".ip-"*".eu" ]] && return 0
    return 1
}

is_ovh_hostname() {
    local domain="$1"

    [[ "$domain" =~ ^vps-[0-9a-f]*$ ]] && return 0
    [[ "$domain" =~ ^vps[0-9]*$ ]] && return 0
    return 1
}

vps_connection_check() {
    local vps="$1"
    ip=$(resolve "$vps") ||
        { echo "${DARKRED}no-resolve${NORMAL}"; return 1; }

    is-port-open "$ip" "22" </dev/null ||
        { echo "${DARKRED}no-port-22-open${NORMAL}"; return 1; }

    ssh:open -o ConnectTimeout=10 -o PreferredAuthentications=publickey \
             "root@$vps" >/dev/null 2>&1 ||
        { echo "${DARKRED}no-ssh-root-access${NORMAL}"; return 1; }
}

vps_check() {
    local vps="$1"
    vps_connection_check "$vps" </dev/null || return 1
    if size=$(
            echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 5 -d ' ' | cut -f 1 -d %" |
                ssh:run "root@$vps" -- bash); then
        if [ "$size" -gt "90" ]; then
            echo "${DARKRED}above-90%-disk-usage${NORMAL}"
        elif [ "$size" -gt "75" ]; then
            echo "${DARKYELLOW}above-75%-disk-usage${NORMAL}"
        fi
    else
         echo "${DARKRED}no-size${NORMAL}"
    fi </dev/null
    compose_content=$(ssh:run "root@$vps" -- cat /opt/apps/myc-deploy/compose.yml </dev/null) ||
        { echo "${DARKRED}no-compose${NORMAL}"; return 1; }
    echo "$compose_content" | grep backup >/dev/null 2>&1 ||
        { echo "${DARKRED}no-backup${NORMAL}"; return 1; }
}



backup:setup-rsync() {
    local admin="$1" vps="$2" server="$3" id="$4"

    [ -z "${BACKUP_SSH_SERVER}" ] || {
        err "Unexpected error: '\$BACKUP_SSH_SERVER' is already set in '$FUNCNAME'."
        return 1
    }

    BACKUP_SSH_OPTIONS=(-o StrictHostKeyChecking=no)
    if [[ "$server" == *":"* ]]; then
        BACKUP_SSH_OPTIONS+=(-p "${server#*:}")
        BACKUP_SSH_SERVER=${server%%:*}
    else
        BACKUP_SSH_SERVER="$server"
    fi

    if ! private_key=$(ssh "${BACKUP_SSH_OPTIONS[@]}" \
                           "$admin"@"${BACKUP_SSH_SERVER}" request-recovery-key "$id"); then
        err "Couldn't request a recovery key for '$id' with account '$admin'."
        return 1
    fi

    if ! VPS_TMP_DIR=$(echo "mktemp -d" | ssh:run "root@$vps" -- bash); then
        err "Couldn't create a temporary directory on vps"
        return 1
    fi

    cat <<EOF | ssh:run "root@$vps" -- bash || return 1
    touch "$VPS_TMP_DIR/recover_key" &&
    chmod go-rwx "$VPS_TMP_DIR/recover_key" &&
    printf "%s\n" "$private_key" >> "$VPS_TMP_DIR/recover_key"
EOF

    BACKUP_SSH_OPTIONS+=(-i "$VPS_TMP_DIR/recover_key" -l rsync)
    BACKUP_VPS_TARGET="$vps"
    BACKUP_IDENT="$id"

    echo "type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null" |
        ssh:run "root@$vps" -- bash || return 1

}


backup:rsync() {
    local ssh_options

    [ -n "${BACKUP_SSH_SERVER}" ] || {
        err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
        return 1
    }

    rsync_options=()
    while [[ "$1" == "-"* ]]; do
        rsync_options+=("$1")
        shift
    done
    local src="$1" dst="$2"

    cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash
rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
              -azvArH --delete --delete-excluded \
              --partial --partial-dir .rsync-partial \
              --numeric-ids ${rsync_options[*]} \
      "${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "${dst}"
EOF

}


backup:path_exists() {
    local src="$1"

    [ -n "${BACKUP_SSH_SERVER}" ] || {
        err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
        return 1
    }

    cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash >/dev/null 2>&1
rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
              -nazvArH --numeric-ids \
      "${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "/tmp/dummy"
EOF

}

file:vps_backup_recover() {
    local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"

    backup:rsync "${path}" "${vps_path}" || return 1

    if [[ "$path" == *"/" ]]; then
        if [ "$path" == "$vps_path"/ ]; then
            msg_target="Directory '$path'"
        else
            msg_target="Directory '$path' -> '$vps_path'"
        fi
    else
        if [ "$path" == "$vps_path" ]; then
            msg_target="File '$path'"
        else
            msg_target="File '$path' -> '$vps_path'"
        fi
    fi

    info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."

}

mailcow:vps_backup_recover() {
    local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"


    if ! compose_yml_files=$(cat <<EOF | ssh:run "root@$vps" -- bash
urn=com.docker.compose.project
docker ps -f "label=\$urn=mailcowdockerized" \
    --format="{{.Label \"\$urn.working_dir\"}}/{{.Label \"\$urn.config_files\"}}" |
    uniq
EOF
         ); then
        err "Couldn't get list of running projects"
        return 1
    fi

    stopped_containers=
    if [ -n "$compose_yml_files" ]; then
        echo "Found running mailcowdockerized containers" >&2
        if [[ "$compose_yml_files" == *$'\n'* ]]; then
            err "Running containers are confusing, did not find only one mailcowdockerized project."
            return 1
        fi
        if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
            ## For some reason, sometimes $urn.config_files holds an absolute path
            compose_yml_files=/${compose_yml_files#*//}
            if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
                err "Running containers are confusing, they don't point to an existing docker-compose.yml."
                return 1
            fi
        fi
        echo "Containers where launched from '$compose_yml_files'" >&2
        COMPOSE_FILE="$compose_yml_files"
        ENV_FILE="${COMPOSE_FILE%/*}/.env"
        if ! echo "[ -e \"${ENV_FILE}\" ]" | ssh:run "root@$vps" -- bash ; then
            err "Running containers are confusing, docker-compose.yml has no '.env' next to it."
            return 1
        fi
        echo "${WHITE}Bringing mailcowdockerized down${NORMAL}"
        echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
            ssh:run "root@$vps" -- bash
        stopped_containers=1
    fi

    if [[ "$path" == "/"* ]]; then

        ##
        ## Additional intelligence to simple file copy
        ##

        if [[ "$path" == "/var/lib/docker/volumes/mailcowdockerized_*-vol-1/_data"* ]]; then
            volume_name=${path#/var/lib/docker/volumes/}
            volume_name=${volume_name%%/*}
            volume_dir=${path%%"$volume_name"*}

            ## Create volumes if not existent
            if ! ssh:run "root@$vps" -- "
                  [ -d '${volume_dir}' ] ||
                      docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
                  [ -d '${volume_dir}' ]
              "; then
                err "Couldn't find nor create '${volume_dir}'."
                return 1
            fi
        fi

        echo "${WHITE}Sync from backup ${path} to VPS ${vps_path}${NORMAL}" >&2
        backup:rsync "${path}" "${vps_path}" || return 1

        if [[ "$path" == *"/" ]]; then
            if [ "$path" == "$vps_path"/ ]; then
                msg_target="Directory '$path'"
            else
                msg_target="Directory '$path' -> '$vps_path'"
            fi
        else
            if [ "$path" == "$vps_path" ]; then
                msg_target="File '$path'"
            else
                msg_target="File '$path' -> '$vps_path'"
            fi
        fi
    else
        ALL_TARGETS=(mailcow postfix rspamd redis crypt vmail{,-attachments} mysql)

        if [[ -n "$path" ]]; then
            targets=()
            bad_targets=()
            for target in ${path//,/ }; do
                if [[ " ${ALL_TARGETS[*]} " != *" $target "* ]]; then
                    bad_targets+=("$target")
                fi
                targets+=("$target")
            done
            if [ "${#bad_targets[@]}" -gt 0 ]; then
                bad_target_msg=$(printf "%s, " "${bad_targets[@]}")
                err "Unknown components: ${bad_target_msg%, }. These are allowed components:"
                printf " - %s\n"  "${ALL_TARGETS[@]}" >&2
                return 1
            fi
            msg_target="Partial mailcow backup"
        else
            targets=("${ALL_TARGETS[@]}")
            msg_target="Full mailcow backup"
        fi


        for target in "${targets[@]}"; do
            case "$target" in
                postfix|rspamd|redis|crypt|vmail|vmail-attachments)

                    volume_name="mailcowdockerized_${target}-vol-1"
                    volume_dir="/var/lib/docker/volumes/${volume_name}/_data"
                    if ! backup:path_exists "${volume_dir}/"; then
                        warn "No '$volume_name' in backup. This might be expected."
                        continue
                    fi
                    ## Create volumes if not existent
                    if ! ssh:run "root@$vps" -- "
                            [ -d '${volume_dir}' ] ||
                                docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
                            [ -d '${volume_dir}' ]
                        "; then
                        err "Couldn't find nor create '${volume_dir}'."
                        return 1
                    fi

                    echo "${WHITE}Downloading of $volume_name${NORMAL}"
                    backup:rsync "${volume_dir}/" "${volume_dir}" || return 1
                    ;;
                mailcow)
                    ## Mailcow git base
                    COMPOSE_FILE=
                    for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
                        backup:path_exists "${mailcow_dir}/" || continue
                        ## this possibly change last value
                        COMPOSE_FILE="$mailcow_dir/docker-compose.yml"
                        ENV_FILE="$mailcow_dir/.env"
                        echo "${WHITE}Download of $mailcow_dir${NORMAL}"
                        backup:rsync "${mailcow_dir}"/ "${mailcow_dir}" || return 1
                        break
                    done
                    if [ -z "$COMPOSE_FILE" ]; then
                        err "Can't find mailcow base installation path in backup."
                        return 1
                    fi

                    ;;
                mysql)
                    if [ -z "$COMPOSE_FILE" ]; then
                        ## Mailcow git base
                        compose_files=()
                        for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
                            ssh:run "root@$vps" -- "[ -e \"$mailcow_dir/docker-compose.yml\" ]" || continue
                            ## this possibly change last value
                            compose_files+=("$mailcow_dir/docker-compose.yml")
                        done
                        if [ "${#compose_files[@]}" == 0 ]; then
                            err "No compose file found for mailcow installation."
                            return 1
                        elif [ "${#compose_files[@]}" -gt 1 ]; then
                            err "Multiple compose files for mailcow found:"
                            for f in "${compose_files[@]}"; do
                                echo " - $f" >&2
                            done
                            echo "Can't decide which to use for mounting mysql container." >&2
                            return 1
                        fi
                        COMPOSE_FILE="${compose_files[0]}"
                        ENV_FILE="${COMPOSE_FILE%/*}/.env"
                        if ! ssh:run "root@$vps" -- "[ -e \"${COMPOSE_FILE%/*}/.env\" ]"; then
                            err "No env file in '$ENV_FILE' found."
                            return 1
                        fi
                    fi

                    ## Mysql database
                    echo "${WHITE}Downloading last backup of mysql backups${NORMAL}"
                    backup:rsync "/var/backups/mysql/" "/var/backups/mysql" || return 1

                    if ! env_content=$(echo "cat '$ENV_FILE'" | ssh:run "root@$vps" -- bash); then
                        err "Can't access env file: '$ENV_FILE'."
                        return 1
                    fi

                    root_password=$(printf "%s\n" "$env_content" | grep ^DBROOT= | cut -f 2 -d =)

                    echo "${WHITE}Bringing mysql-mailcow up${NORMAL}"
                    if ! image=$(cat <<EOF | ssh:run "root@$vps" -- bash
shyaml get-value services.mysql-mailcow.image < "${COMPOSE_FILE}"
EOF
                         ); then
                        err "Failed to get image name of service 'mysql-mailcow' in 'compose.yml'."
                        return 1
                    fi
                    if [ -z "$(ssh:run "root@$vps" -- docker images -q "$image")" ]; then
                        info "Image '$image' not available, pull it."
                        if ! ssh:run "root@$vps" -- \
                             docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
                             pull mysql-mailcow; then

                            err "Failed to pull image of service 'mysql-mailcow'."
                            return 1
                        fi
                    fi

                    if ! container_id=$(cat <<EOF | ssh:run "root@$vps" -- bash
echo "[client]
password=$root_password" > "$VPS_TMP_DIR/my.cnf"

docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
    run -d \
    -v "$VPS_TMP_DIR/my.cnf:/root/.my.cnf:ro" \
    mysql-mailcow

EOF
                         ); then
                        err "Failed to bring up mysql-mailcow"
                        return 1
                    fi

                    START="$SECONDS"
                    retries=0
                    timeout=600
                    while true; do
                        ((retries++))
                        echo "  waiting for mysql db..." \
                             "(retry $retries, $(($SECONDS - $START))s elapsed, timeout is ${timeout}s)" >&2
                        cat <<EOF | ssh:run "root@$vps" -- bash && break
    echo "SELECT 1;" | docker exec -i "$container_id" mysql >/dev/null 2>&1
EOF
                        if (($SECONDS - $START > $timeout)); then
                            err "Failed to connect to mysql-mailcow."
                            echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
                                ssh:run "root@$vps" -- bash
                            return 1
                        fi
                        sleep 0.4
                    done

                    DBUSER=$(printf "%s\n" "$env_content" | grep ^DBUSER= | cut -f 2 -d =)
                    DBPASS=$(printf "%s\n" "$env_content" | grep ^DBPASS= | cut -f 2 -d =)

                    echo "${WHITE}Uploading mysql dump${NORMAL}"
                    cat <<EOF | ssh:run "root@$vps" -- bash

echo "
  DROP DATABASE IF EXISTS mailcow;
  CREATE DATABASE mailcow;
  GRANT ALL PRIVILEGES ON mailcow.* TO '$DBUSER'@'%' IDENTIFIED BY '$DBPASS';
" | docker exec -i "$container_id" mysql

zcat /var/backups/mysql/mailcow/*.gz | docker exec -i "$container_id" mysql mailcow

EOF
                    if [ "$?" != 0 ]; then
                        err "Failed to load mysql dump."
                        echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
                            ssh:run "root@$vps" -- bash
                        return 1
                    fi

                    echo "${WHITE}Bringing mysql-mailcow down${NORMAL}"
                    echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
                        ssh:run "root@$vps" -- bash
                    ;;
                *)
                    err "Unknown component '$target'. Bailing out."
                    return 1
            esac
        done

        ssh:run "root@$vps" -- "rm -rf '$VPS_TMP_DIR'"
    fi

    if [ -n "$stopped_containers" ]; then
        echo "${WHITE}Starting mailcow${NORMAL}" >&2
        echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" up -d" |
            ssh:run "root@$vps" -- bash
    fi
    info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."

}

vps_backup_recover() {
    local vps="$1" admin server id path rtype force type

    read-0 admin server id path rtype force

    if [[ "$vps" == *":"* ]]; then
        vps_path=${vps#*:}
        vps=${vps%%:*}
    else
        vps_path=
    fi

    vps_connection_check "$vps" </dev/null || {
        err "Failed to access '$vps'."
        return 1
    }

    if type=$(ssh:run "root@$vps" -- vps get-type); then
        info "VPS $vps seems to be of ${WHITE}$type${NORMAL} type"
    else
        warn "Couldn't find type of vps '$vps' (command 'vps get-type' failed on vps)."
    fi

    if [ -z "$path" ]; then
        if [ -n "$vps_path" ]; then
            err "You can't provide a VPS with path as destination if you don't provide a path in backup source."
            return 1
        fi
        info "No path provided in backup, so we assume you want ${WHITE}full recovery${NORMAL}."
        if [ "$rtype" != "$type" ]; then
            if [ -n "$force" ]; then
                warn "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type."
            else
                err "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type. (use \`\`-f\`\` to force)"
                return 1
            fi
        fi
    else
        if [ "$path" == "/" ]; then
            if [ -z "$vps_path" ]; then
                err "Recovery of '/' (full backup files) requires that you provide a vps path also."
                return 1
            fi
            if [ "$vps_path" == "/" ]; then
                err "Recovery of '/' (full backup files) requires that you provide" \
                    "a vps path different from '/' also."
                return 1
            fi
        fi
    fi

    ## Sets VPS and internal global variable to allow rsync to work
    ## from vps to backup server.
    backup:setup-rsync "$admin" "$vps" "$server" "$id" || return 1

    if [[ "$path" == "/"* ]]; then
        if ! backup:path_exists "${path}"; then
            err "File or directory '$path' not found in backup."
            return 1
        fi
        if [ -z "$vps_path" ]; then
            if [[ "$path" != *"/" ]] && backup:path_exists "${path}"/ ; then
                path="$path/"
            fi
            vps_path=${path%/}
            vps_path=${vps_path:-/}
        fi
    fi


    case "$rtype-$type" in
        mailcow-*)
            ## Supports having $path and $vps_path set or unset, with additional behavior
            mailcow:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
            ;;
        *-*)
            if [[ "$path" == "/"* ]]; then
            ## For now, will require having $path and $vps_path set, no additional behaviors
                file:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
            else
                if [ -n "$path" ]; then
                    err "Partial component recover of ${rtype:-unknown} backup type on" \
                        "${type:-unknown} type VPS is not yet implemented."
                    return 1
                else
                    err "Full recover of ${rtype:-unknown} backup type on" \
                        "${type:-unknown} type VPS is not yet implemented."
                    return 1
                fi
            fi
            ;;
    esac

}


vps_install_backup() {
    local vps="$1" admin server
    vps_connection_check "$vps" </dev/null || return 1

    read-0 admin server
    if ! type=$(ssh:run "root@$vps" -- vps get-type </dev/null); then
        err "Could not get type."
        return 1
    fi
    backup_opts=()
    local opt
    while read-0 opt; do
        case "$opt" in
            --ignore-domain-check|--ignore-ping-check)
                backup_opts+=("$opt")
                ;;
            *)
                err "Unknown option '$opt'."
                return 1
                ;;
        esac
    done
    if ! out=$(ssh:run "root@$vps" -- vps install backup "$server" "${backup_opts[@]}" 2>&1); then
        err "Command 'vps install backup $server ${backup_opts[@]}' on $vps failed:"
        echo "$out" | prefix "  ${DARKGRAY}|${NORMAL} " >&2
        return 1
    fi

    ## Format of output:
    ##
    ## II Entry for service rsync-backup is already present in '/opt/apps/myc-deploy/compose.yml'.
    ## II You can run this following command from an host having admin access to core-07.0k.io:
    ##    (Or send it to a backup admin of core-07.0k.io)
    ## ssh -p 10023 myadmin@core-07.0k.io ssh-key add 'ssh-rsa AAAAAAAD...QCJ\
    ## 8HH6pVgEpu1twyxpr9xTt7eh..WaJdgPoxmiEwGfjMMNGxs39ggOTKUuSFSmOv8TiA1fzY\
    ## s85hF...dKP1qByJU1k= compose@odoo.sikle.fr'

    key="ssh-rsa ${out##*\'ssh-rsa }"  ## remove everything before last "'ssh-rsa"
    key="${key%\'*}"                   ## remove everything after last '

    if ! [[ "$key" =~ ^"ssh-rsa "[a-zA-Z0-9/+=]+" "[a-zA-Z0-9._-]+"@"[a-zA-Z0-9._-]+$ ]]; then
        err "Unexpected output from 'vps install backup $server'. Can't find key."
        echo "$out" | prefix " ${GRAY}|$NORMAL " >&2
        echo " Extracted key:" >&2
        echo "$key" | prefix " ${GRAY}|$NORMAL " >&2
        return 1
    fi

    if [ "$type" == "compose" ]; then
        if ! ssh:run "root@$vps" -- \
             docker exec myc_cron_1 \
                 cat /etc/cron.d/rsync-backup >/dev/null 2>&1; then
            ssh:run "root@$vps" -- compose --debug up || {
                err "Command 'compose --debug up' failed."
                return 1
            }
            if ! ssh:run "root@$vps" -- \
                 docker exec myc_cron_1 \
                 cat /etc/cron.d/rsync-backup >/dev/null 2>&1; then
                err "Launched 'compose up' successfully but ${YELLOW}cron${NORMAL} container is not setup as expected."
                echo "  Was waiting for existence of '/etc/cron.d/rsync-backup' in it." >&2
                return 1
            fi
        fi
    fi

    dest="$server"
    dest="${dest%/*}"
    ssh_options=()
    if [[ "$dest" == *":"* ]]; then
        port="${dest##*:}"
        dest="${dest%%:*}"
        ssh_options=(-p "$port")
    else
        port=""
        dest="${dest%%:*}"
    fi

    cmd=(ssh "${ssh_options[@]}" "$admin"@"$dest" ssh-key add "$key")
    echo "${WHITE}Launching:${NORMAL} ${cmd[@]}"

    "${cmd[@]}" || {
        err "Failed add key to backup server '$dest'."
        return 1
    }
    echo "${WHITE}Launching backup${NORMAL} from '$vps'"

    ssh:run "root@$vps" -- vps backup || {
        err "First backup failed to run."
        return 1
    }

    echo "Backup is ${GREEN}up and running${NORMAL}."
}


vps_update() {
    local vps="$1"
    vps_connection_check "$vps" || return 1
    ssh:run "root@$vps" -- myc-update </dev/null
}


vps_bash() {
    local vps="$1"
    vps_connection_check "$vps" </dev/null || return 1
    ssh:run "root@$vps" -- bash
}

vps_mux() {
    local fn="$1" vps_done VPS max_size vps
    shift
    VPS=($(printf "%s\n" "$@" | sort))
    max_size=0
    declare -A vps_done;
    new_vps=()
    for name in "${VPS[@]}"; do
        [ -n "${vps_done[$name]}" ] && {
            warn "duplicate vps '$name' provided. Ignoring."
            continue
        }
        vps_done["$name"]=1
        new_vps+=("$name")
        size_name="${#name}"
        [ "$max_size" -lt "${size_name}" ] &&
            max_size="$size_name"
    done
    settmpdir "_0KM_TMP_DIR"
    cat > "$_0KM_TMP_DIR/code"
    for vps in "${new_vps[@]}"; do
        label=$(printf "%-${max_size}s" "$vps")
        (
            {
                {
                     "$fn" "$vps" < "$_0KM_TMP_DIR/code"
                } 3>&1 1>&2 2>&3 | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKRED\!$NORMAL /g"
                set_errlvl "${PIPESTATUS[0]}"
            } 3>&1 1>&2 2>&3  | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKGRAY\|$NORMAL /g"
            set_errlvl "${PIPESTATUS[0]}"
        ) &
    done
    wait
}


[ "$SOURCED" ] && return 0

##
## Command line processing
##


cmdline.spec.gnu
cmdline.spec.reporting

cmdline.spec.gnu vps-setup

cmdline.spec::cmd:vps-setup:run() {

    : :posarg: HOST        'Target host to check/fix ssh-access'

    : :optfla: --force,-f  "Will force domain name change, even if
                            current hostname was not recognized as
                            an ovh domain name. "

    depends sshpass shyaml

    KEY_PATH="ssh-access.public-keys"
    local keys=$(config get-value -y "ssh-access.public-keys") || true
    if [ -z "$keys" ]; then
        err "No ssh publickeys configured in config file."
        echo "  Looking for keys in ${WHITE}${KEY_PATH}${NORMAL}" \
             "in config file." >&2
        config:exists --message 2>&1 | prefix "  "
        if [ "${PIPESTATUS[0]}" == "0" ]; then
            echo "  Config file found in $(config:filename)"
        fi
        return 1
    fi
    local tkey=$(e "$keys" | shyaml get-type)
    if [ "$tkey" != "sequence" ]; then
        err "Value type of ${WHITE}${KEY_PATH}${NORMAL} unexpected (is $tkey, expecting sequence)."
        echo "  Check content of $(config:filename), and make sure to use a sequence." >&2
        return 1
    fi

    local IP NAME keys host_pass_connected
    if ! IP=$(resolve "$HOST"); then
        err "'$HOST' name unresolvable."
        exit 1
    fi

    NAME="$HOST"
    if [ "$IP" != "$HOST" ]; then
        NAME="$HOST ($IP)"
    fi

    if ! is-port-open "$IP" "22"; then
        err "$NAME unreachable or port 22 closed."
        exit 1
    fi
    debug "Host $IP's port 22 is open."
    if ! host_pass_connected=$(ssh:open-try \
                                   {root,debian}@"$HOST"); then
        err "Could not connect to {root,debian}@$HOST with publickey nor password."
        exit 1
    fi

    read-0a host password <<<"$host_pass_connected"

    sudo_if_necessary=
    if [ "$password" -o "${host%%@*}" != "root" ]; then
        if ! ssh:run "$host" -- sudo -n true >/dev/null 2>&1; then
            err "Couldn't do a password-less sudo from $host."
            echo "  This is not yet supported."
            exit 1
        else
            sudo_if_necessary=sudo
        fi
    fi

    SUPPORTED_KEY_TYPE=(ssh-rsa ssh-ed25519)
    Section Checking access
    while read-0 key; do
        prefix="${key%% *}"
        if ! [[ " ${SUPPORTED_KEY_TYPE[*]} " == *" $prefix "* ]]; then
            err "Unsupported key:"$'\n'"$key"
            echo "  Please use only key of the following type:" >&2
            printf "    - %s\n" "${SUPPORTED_KEY_TYPE[@]}" >&2
            return 1
        fi
        label="${key##* }"
        Elt "considering adding key ${DARKYELLOW}$label${NORMAL}"
        dest="/root/.ssh/authorized_keys"
        if ssh:run "$host" -- $sudo_if_necessary grep "\"$key\"" "$dest" >/dev/null 2>&1 </dev/null; then
            print_info "already present"
            print_status noop
            Feed
        else
            if echo "$key" | ssh:run "$host" -- $sudo_if_necessary tee -a "$dest" >/dev/null; then
                print_info added
            else
                echo
                Feedback failure
                return 1
            fi
            Feedback success
        fi
    done < <(e "$keys" | shyaml get-values-0)

    Section Checking ovh hostname file
    Elt "Checking /etc/ovh-hostname"
    if ! ssh:run "$host" -- $sudo_if_necessary [ -e "/etc/ovh-hostname" ]; then
        print_info "creating"
        ssh:run "$host" -- $sudo_if_necessary cp /etc/hostname /etc/ovh-hostname
        ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
        Elt "Checking /etc/ovh-hostname: $ovhname"
        Feedback || return 1
    else
        ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
        Elt "Checking /etc/ovh-hostname: $ovhname"
        print_info "already present"
        print_status noop
        Feed
    fi

    if ! is_ovh_domain_name "$HOST" && ! str_is_ipv4 "$HOST"; then
            Section Checking hostname
            Elt "Checking /etc/hostname..."
            old_etc_hostname="$(ssh:run "$host" -- $sudo_if_necessary cat /etc/hostname)"
            if [ "$old_etc_hostname" != "$HOST" ]; then
                Elt "/etc/hostname is '$old_etc_hostname'"
                if is_ovh_hostname "$old_etc_hostname" || [ -n "$opt_force" ]; then
                    Elt "Hostname '$old_etc_hostname' --> '$HOST'"
                    print_info "creating"
                    echo "$HOST" | ssh:run "$host" -- $sudo_if_necessary tee /etc/hostname >/dev/null &&
                        ssh:run "$host" -- $sudo_if_necessary hostname "$HOST"
                    Feedback || return 1
                else
                    Elt "Hostname '$old_etc_hostname' isn't an ovh domain"
                    print_info "no change"
                    print_status noop
                    Feed
                    warn "Domain name was not changed because it was already set"
                    echo "  (use \`-f\` or \`--force\`) to force domain name change to $HOST." >&2
                fi

            else
                print_info "already set"
                print_status noop
                Feed
            fi
            Elt "Checking consistency between /etc/hostname and \`hostname\`..."
            etc_hostname="$(ssh:run "$host" -- $sudo_if_necessary cat /etc/hostname)"
            transient_hostname="$(ssh:run "$host" -- $sudo_if_necessary hostname)"
            if [ "$etc_hostname" != "$transient_hostname" ]; then
                print_info "change"
                ssh:run "$host" -- $sudo_if_necessary hostname "$etc_hostname"
                Feedback || return 1
            else
                print_info "consistent"
                print_status noop
                Feed
            fi
    else
        info "Not changing domain as '$HOST' doesn't seem to be final domain."
    fi

}


cmdline.spec.gnu vps-check
cmdline.spec::cmd:vps-check:run() {

    : :posarg: [VPS...]        'Target host(s) to check'


    echo "" |
        vps_mux vps_check "${VPS[@]}"
}


cmdline.spec.gnu vps-install
cmdline.spec::cmd:vps-install:run() {
    :
}

cmdline.spec.gnu backup
cmdline.spec:vps-install:cmd:backup:run() {

    : :posarg: BACKUP_TARGET   'Backup target.
                                (ie: myadmin@backup.domain.org:10023/256)'
    : :optfla: --ignore-domain-check \
                                   "Allow to bypass the domain check in
                                    compose file (only used in compose
                                    installation)."
    : :optfla: --ignore-ping-check "Allow to bypass the ping check of host."
    : :posarg: [VPS...]        'Target host(s) to check'


    if [ "${#VPS[@]}" == 0 ]; then
        warn "VPS list provided in command line is empty. Nothing will be done."
        return 0
    fi

    if ! [[ "$BACKUP_TARGET" == *"@"* ]]; then
        err "Missing admin account identifier in backup target."
        echo "  Have you forgottent to specify an account, ie 'myadmin@<MYBACKUP_SERVER>' ?)"
        return 1
    fi

    admin=${BACKUP_TARGET%%@*}
    server=${BACKUP_TARGET#*@}
    opts=()

    [ -n "$opt_ignore_ping_check" ] &&
        opts+=("--ignore-ping-check")

    [ -n "$opt_ignore_domain_check" ] &&
        opts+=("--ignore-domain-check")

    p0 "$admin" "$server" "${opts[@]}" |
        vps_mux vps_install_backup "${VPS[@]}"
}



cmdline.spec.gnu vps-backup
cmdline.spec::cmd:vps-backup:run() {
    :
}


cmdline.spec.gnu ls
cmdline.spec:vps-backup:cmd:ls:run() {
    : :posarg: BACKUP_ID       'Backup id.
                                (ie: myadmin@backup.domain.org:10023)'

    if ! [[ "$BACKUP_ID" == *"@"* ]]; then
        err "Missing admin account identifier in backup id."
        echo "  Have you forgottent to specify an admin account ?" \
             "ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
        return 1
    fi

    id=${BACKUP_ID##*#}
    BACKUP_TARGET=${BACKUP_ID%#*}
    admin=${BACKUP_TARGET%%@*}
    server=${BACKUP_TARGET#*@}

    ## XXXvlab: in this first implementation we expect to have access
    ## to the server main ssh port 22, so we won't use the provided port.
    ssh_options=()
    if [[ "$server" == *":"* ]]; then
        ssh_options+=(-p "${server#*:}")
        server=${server%%:*}
    fi

    ssh "${ssh_options[@]}" "$admin"@"$server" ssh-key ls
}


cmdline.spec.gnu recover
cmdline.spec:vps-backup:cmd:recover:run() {

    : :posarg: BACKUP_ID       'Backup id.
                                (ie: myadmin@backup.domain.org:10023#mx.myvps.org
                                     myadmin@ark-01.org#myid:/a/path
                                     admin@ark-02.io#myid:myqsl,mailcow)'
    : :posarg: VPS_PATH        'Target host(s) to check.
                                (ie: myvps.com
                                     myvps.com:/a/path)'

    : :optval: --date,-D       '"last", or label of version to recover. (Default: "last").'
    : :optfla: --force,-f       'Will allow you to bypass some checks.'


    if ! [[ "$BACKUP_ID" == *"@"* ]]; then
        err "Missing admin account identifier in backup id."
        echo "  Have you forgottent to specify an admin account ?" \
             "ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
        return 1
    fi
    if ! [[ "$BACKUP_ID" == *"@"*"#"* ]]; then
        err "Missing backup label identifier in backup id."
        echo "  Have you forgottent to specify a backup label identifier ?" \
             "ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
        return 1
    fi

    id_path=${BACKUP_ID#*#}
    if [[ "$id_path" == *":"* ]]; then
        id=${id_path%%:*}
        path=${id_path#*:}
    else
        id="$id_path"
        path=
    fi
    BACKUP_TARGET=${BACKUP_ID%#*}
    admin=${BACKUP_TARGET%%@*}
    server=${BACKUP_TARGET#*@}

    ssh_options=()
    if [[ "$server" == *":"* ]]; then
        ssh_options+=(-p "${server#*:}")
        ssh_server=${server%%:*}
    fi

    BACKUP_PATH="/srv/datastore/data/rsync-backup-target/var/mirror"

    if ! content=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key ls 2>/dev/null); then
        err "Access denied to '$admin@${server}'."
        return 1
    fi

    idents=$(echo "$content" | sed -r "s/"$'\e'"\[[0-9]+(;[0-9]+)*m//g" | cut -f 2 -d " ")
    if ! [[ $'\n'"$idents"$'\n' == *$'\n'"$id"$'\n'* ]]; then
        err "Given backup id '$id' not found in $admin@${server}'s idents."
        return 1
    fi

    rtype=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key get-type "$id" ) &&
        info "Backup archive matches ${WHITE}${rtype}${NORMAL} type"

    p0 "$admin" "$server" "$id" "$path" "$rtype" "$opt_force" |
        vps_backup_recover "${VPS_PATH}"
}




cmdline.spec.gnu vps-update
cmdline.spec::cmd:vps-update:run() {

    : :posarg: [VPS...]        'Target host to check'

    echo "" |
        vps_mux vps_update "${VPS[@]}"
}


cmdline.spec.gnu vps-mux
cmdline.spec::cmd:vps-mux:run() {

    : :posarg: [VPS...]        'Target host(s) to check'

    cat | vps_mux vps_bash "${VPS[@]}"
}


cmdline.spec.gnu vps-space
cmdline.spec::cmd:vps-space:run() {

    : :posarg: [VPS...]        'Target host(s) to check'

    echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 2-5 -d ' '" |
        vps_mux vps_bash "${VPS[@]}"
}


cmdline.spec.gnu vps-stats
cmdline.spec::cmd:vps-stats:run() {

    : :posarg: [VPS...]        'Target host(s) to get stats'

    : :optfla: --follow,-f     'Refresh graph every 2m'
    : :optval: --timespan,-t   'timespan START[..END]'
    : :optval: --resource,-r   'resource(s) separated with a comma'
    : :optval: --interval,-i   'refersh interval (default: 60s)'

    local opts_rrdfetch=( -a )
    if [ -n "${opt_timespan}" ]; then
        start=${opt_timespan%\.\.*}
        opts_rrdfetch+=(-s "$start")
        if [ "$start" != "${opt_timespan}" ]; then
            end=${opt_timespan#*..}
            opts_rrdfetch+=(-e "$end")
        fi
    fi
    local resources=(c.memory c.network load_avg)
    if [ -n "${opt_resource}" ]; then
        resources=(${opt_resource//,/ })
    fi

    local not_found=()
    for resource in "${resources[@]}"; do
        if ! fn.exists "graph:def:$resource"; then
            not_found+=("$resource")
        fi
    done

    if [[ "${#not_found[@]}" -gt 0 ]]; then
        not_found_msg=$(printf "%s, " "${not_found[@]}")
        not_found_msg=${not_found_msg%, }
        err "Unsupported resource(s) provided: ${not_found_msg}"
        echo "  resource must be one-of:" >&2
        declare -F | egrep 'graph:def:[a-zA-Z_.]+$' | cut -f 3- -d " " | cut -f 3- -d ":" | prefix "  - " >&2
        return 1
    fi

    if [ "${#VPS[@]}" == 0 ]; then
        err "You must provide a VPS list as positional arguments"
        return 1
    fi

    include cache
    if [ -z "$VAR_DIR" ]; then
        err "Unset \$VAR_DIR, can't downlowd rrd graph"
        return 1
    fi
    mkdir -p "$VAR_DIR/rrd"
    if ! [ -d "$VAR_DIR/rrd" ]; then
        err "Invalid \$VAR_DIR: '$VAR_DIR/rrd' is not a directory"
        return 1
    fi

    (
        for vps in "${VPS[@]}"; do
            (
                {
                    {
                        ssh:open "root@$vps" 2>/dev/null || {
                            err "Can't open connection $vps."
                            return 1
                        }
                        while true; do
                            if ssh:run "root@$vps" -- "[ -d '/var/lib/vps/rrd' ]"; then
                                echo "${WHITE}Collecting stats${NORMAL}..."
                                {
                                    {
                                        ssh:rsync "root@$vps:/var/lib/vps/rrd/" "${VAR_DIR}/rrd/${vps}"
                                    } 3>&1 1>&2 2>&3 | prefix " ${DARKRED}\!${NORMAL} "
                                    set_errlvl "${PIPESTATUS[0]}"
                                } 3>&1 1>&2 2>&3 | prefix " ${GRAY}|${NORMAL} "
                                echo " ${GRAY}..${NORMAL} ${DARKGREEN}done${NORMAL} collecting stats"
                            else
                                warn "No stats found. Did you run 'myc-update' on the vps ?."
                            fi
                            [ -z "$opt_follow" ] && break
                            echo "${WHITE}Sleeping ${DARKYELLOW}${opt_interval:-60}${NORMAL}s..."
                            sleep "${opt_interval:-60}"
                            echo "  ${GRAY}..${NORMAL} ${DARKGREEN}done${NORMAL} sleeping"
                        done
                    } 3>&1 1>&2 2>&3 | prefix " ${DARKRED}\!${GRAY} collect(${DARKCYAN}$vps${GRAY})${NORMAL} "
                    set_errlvl "${PIPESTATUS[0]}"
                } 3>&1 1>&2 2>&3 | prefix " ${GRAY}| collect(${DARKCYAN}$vps${GRAY})${NORMAL} " >&2
            ) &
        done
        wait
    ) &
    collect_pid="$!"
    if [ -z "$opt_follow" ]; then
        echo "${WHITE}Fetching last stats${NORMAL}${GRAY}..${NORMAL}" >&2
        wait
        echo " ${GRAY}..${DARKGREEN} done${NORMAL} fetching stats" >&2
    else
        collect_end_msg="  ${GRAY}..${NORMAL} ${DARKGREEN}stop${NORMAL} collecting daemon (pid: ${DARKYELLOW}$collect_pid${NORMAL})"
        trap_add EXIT \
                 "printf '%s\n' \"$collect_end_msg\" && kill $collect_pid"
        echo "${WHITE}Start collecting daemon${NORMAL} (pid: ${DARKYELLOW}$collect_pid${NORMAL}) ${GRAY}..${NORMAL}" >&2
    fi
    ( depends gnuplot ) || {
        echo ""
        echo "  Gnuplot is required to display graphs..." \
             "You might want to try to install ${WHITE}gnuplot${NORMAL} with:"
        echo ""
        echo "    apt install gnuplot"
        echo ""
        return 1
    } >&2
    ( depends rrdtool ) || {
        echo ""
        echo "  Rrdtool is required to display graphs..." \
             "You might want to try to install ${WHITE}rrdtool${NORMAL} with:"
        echo ""
        echo "    apt install rrdtool"
        echo ""
        return 1
    } >&2

    export GNUTERM=qt
    ## rrdtool fetch will use comma for floating point depending on some locals !
    export LC_ALL=C
    exec {PFD}> >(exec gnuplot 2>/dev/null)
    gnuplot_pid="$!"
    if [ -z "$opt_follow" ]; then
        echo "${WHITE}Draw gnuplot graph${GRAY}..${NORMAL}" >&2
    else
        gnuplot_end_msg="  ${GRAY}..${NORMAL} ${DARKGREEN}stop${NORMAL} gnuplot process (pid: $gnuplot_pid)"
        trap_add EXIT \
                 "printf '%s\n' \"$gnuplot_end_msg\" && kill $gnuplot_pid"
        echo "${WHITE}Start gnuplot process${NORMAL} (pid: $gnuplot_pid) ${GRAY}..${NORMAL}" >&2
    fi
    echo "set term qt noraise replotonresize" >&$PFD
    while true; do
        {
            i=0
            data_start_ts=
            data_stop_ts=
            for resource in "${resources[@]}"; do
                for vps in "${VPS[@]}"; do
                    ((i++))
                    {
                        {
                            rrd_vps_path="$VAR_DIR/rrd/$vps"
                            if [ -d "$rrd_vps_path" ]; then
                                graph:def:"${resource}" "$vps" "$i" "${opts_rrdfetch[@]}"
                            else
                                warn "No data yet, ignoring..."
                            fi
                        } 3>&1 1>&2 2>&3 | prefix " ${DARKRED}\!${GRAY} graph(${DARKCYAN}$vps${GRAY}:${WHITE}$resource${NORMAL})${NORMAL} "
                        set_errlvl "${PIPESTATUS[0]}"
                    } 3>&1 1>&2 2>&3 || continue 1
                done
            done
        } >&$PFD
        if [ -z "$opt_follow" ]; then
            echo " ${GRAY}..${DARKGREEN} done${NORMAL} gnuplot graphing" >&2
            break
        else
            {
            echo "${WHITE}Sleeping ${DARKYELLOW}${opt_interval:-60}${NORMAL}s..."
            sleep "${opt_interval:-60}"
            echo "  ${GRAY}..${NORMAL} ${DARKGREEN}done${NORMAL} sleeping"
            } | prefix "  ${GRAY}| gnuplot:${NORMAL} " >&2
        fi
    done

    if [ -n "$opt_follow" ]; then
        echo "Waiting for child process to finish.." >&2
        wait
        echo " ..done" >&2
    else
        echo "pause mouse close" >&$PFD
    fi
}


graph:def:c.memory() {
    local vps="$1" i="$2"
    shift 2
    local opts_rrdfetch=("$@")
    local resource="memory"
    rrd_vps_path="$VAR_DIR/rrd/$vps"
    [ -d "$rrd_vps_path/containers" ] || {
        warn "No containers data yet for vps '$vps'... Ignoring"
        return 0
    }
    containers=(
        $(
            cd "$rrd_vps_path/containers";
            find -maxdepth 3 -mindepth 3 -name "${resource}.rrd" -type f |
                sed -r 's%^./([^/]+/[^/]+)/[^/]+.rrd$%\1%g'
        )
    )
    gnuplot_line_config=(
        "set term qt $i title \"$vps $resource\" replotonresize noraise"
        "set title '$vps'"
        "set xdata time"
        "set timefmt '%s'"
        "set ylabel '$resource Usage'"
        "set format y '%s'"
        "set ytics format ' %g'"
        "set mouse mouseformat 6"
        "set yrange [0:*] "
        "set border behind"
    )
    printf "%s\n" "${gnuplot_line_config[@]}"
    first=1
    for container in "${containers[@]}"; do
        rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/containers/$container/$resource.rrd\""
        rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
        rrdfetch_cmd+="         tail -n +2 | \\"$'\n'
        rrdfetch_cmd+="         egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
        rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
        rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
        first_ts=
        first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ")
        if [ -z "$first_ts" ]; then
            warn "No data for $container on vps $vps, skipping..."
            continue
        fi
        last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1  | cut -f 1 -d " ")
        if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
            data_start_ts="$first_ts"
        fi
        if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
            data_stop_ts="$last_ts"
        fi
        if [ -n "$first" ]; then
            first=
            echo "plot \\"
        else
            echo ", \\"
        fi
        container="${container//\'/}"
        container="${container//@/\\@}"
        echo -n "  ${rrdfetch_cmd} u 1:((\$3 - \$2)/1000000000) w lines title '${container//_/\\_}'"
    done
    echo
}


graph:def:c.network() {
    local vps="$1" i="$2"
    shift 2
    local opts_rrdfetch=("$@")
    local resource="network"
    rrd_vps_path="$VAR_DIR/rrd/$vps"
    [ -d "$rrd_vps_path/containers" ] || {
        warn "No containers data yet for vps '$vps'... Ignoring"
        return 0
    }
    containers=(
        $(
            cd "$rrd_vps_path/containers";
            find -maxdepth 3 -mindepth 3 -name "${resource}.rrd" -type f |
                sed -r 's%^./([^/]+/[^/]+)/[^/]+.rrd$%\1%g'
        )
    )
    gnuplot_line_config=(
        "set term qt $i title \"$vps $resource\" replotonresize noraise"
        "set title '$vps'"
        "set xdata time"
        "set timefmt '%s'"
        "set ylabel '$resource Usage'"
        "set format y '%s'"
        "set ytics format ' %.2f MiB/s'"
        "set mouse mouseformat 6"
        "set yrange [0:*] "
        "set border behind"
    )
    printf "%s\n" "${gnuplot_line_config[@]}"
    first=1
    for container in "${containers[@]}"; do
        rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/containers/$container/$resource.rrd\""
        rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
        rrdfetch_cmd+="         tail -n +2 | \\"$'\n'
        rrdfetch_cmd+="         egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
        rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
        rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
        first_ts=
        first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ")
        if [ -z "$first_ts" ]; then
            warn "No data for $container on vps $vps, skipping..."
            continue
        fi
        last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1  | cut -f 1 -d " ")
        if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
            data_start_ts="$first_ts"
        fi
        if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
            data_stop_ts="$last_ts"
        fi
        if [ -n "$first" ]; then
            first=
            echo "plot \\"
        else
            echo ", \\"
        fi
        container="${container//\'/}"
        container="${container//@/\\@}"
        echo -n "  ${rrdfetch_cmd} u 1:((\$3 / 1024) / 1024) w lines title '${container//_/\\_}'"
    done
    echo
}


graph:def:load_avg() {
    local vps="$1" i="$2"
    shift 2
    local opts_rrdfetch=("$@")
    rrd_vps_path="$VAR_DIR/rrd/$vps"
    [ -f "$rrd_vps_path/$resource.rrd" ] || {
        warn "No containers data yet for vps '$vps'... Ignoring"
        return 0
    }
    gnuplot_line_config=(
        "set term qt $i title \"$vps $resource\" replotonresize noraise"
        "set title '$vps'"
        "set xdata time"
        "set timefmt '%s'"
        "set ylabel '${resource//_/\\_} Usage'"
        "set format y '%s'"
        "set ytics format '%g'"
        "set mouse mouseformat 6"
        "set yrange [0:*] "
        "set border behind"
    )
    printf "%s\n" "${gnuplot_line_config[@]}"
    first=1
    for value in 1m:2 5m:3 15m:4; do
        label="${value%:*}"
        col_num="${value#*:}"
        rrdfetch_cmd="'< rrdtool fetch \"$rrd_vps_path/$resource.rrd\""
        rrdfetch_cmd+=" AVERAGE ${opts_rrdfetch[*]} | \\"$'\n'
        rrdfetch_cmd+="         tail -n +2 | \\"$'\n'
        rrdfetch_cmd+="         egrep -v \"^$\" | sed -r \"s/ -?nan/ -/g;s/^([0-9]+): /\\1 /g\"'"
        rrdfetch_cmd_bash=$(eval echo "${rrdfetch_cmd}")
        rrdfetch_cmd_bash=${rrdfetch_cmd_bash#< }
        first_ts=
        first_ts=$(eval "$rrdfetch_cmd_bash" | head -n 1 | cut -f 1 -d " ")
        if [ -z "$first_ts" ]; then
            warn "No data for $resource on vps $vps, skipping..."
            continue
        fi
        last_ts=$(eval "$rrdfetch_cmd_bash" | tail -n 1  | cut -f 1 -d " ")
        if [[ -z "$data_start_ts" ]] || [[ "$data_start_ts" > "$first_ts" ]]; then
            data_start_ts="$first_ts"
        fi
        if [[ -z "$data_stop_ts" ]] || [[ "$data_stop_ts" < "$last_ts" ]]; then
            data_stop_ts="$last_ts"
        fi
        if [ -n "$first" ]; then
            first=
            echo "plot \\"
        else
            echo ", \\"
        fi
        container="${container//\'/}"
        container="${container//@/\\@}"
        echo -n "  ${rrdfetch_cmd} u 1:$col_num w lines title '${label}'"
    done
    echo
}


cmdline::parse "$@"