myc-manage/bin/0km

#!/bin/bash


. /etc/shlib

include common
include parse
include cmdline
include config


[[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true

version=0.1
desc='Manage 0k related installs'
help=""


##
## Functions
##

is-port-open() {
    local host="$1" port="$2" timeout=5
    start="$SECONDS"
    debug "Testing if $host's port $2 is open ..."
    while true; do
        timeout 1 bash -c "</dev/tcp/${host}/${port}" >/dev/null 2>&1 && break
        sleep 0.2
        if [ "$((SECONDS - start))" -gt "$timeout" ]; then
            return 1
        fi
    done
}

resolve() {
    local ent hostname="$1"
    debug "Resolving $1 ..."
    if ent=$(getent ahosts "$hostname"); then
        ent=$(echo "$ent" | egrep ^"[0-9]+.[0-9]+.[0-9]+.[0-9]+\s+" | \
                  head -n 1 | awk '{ print $1 }')
        debug "  .. resolved $1 to $ent."
        echo "$ent"
    else
        debug "  .. couldn't resolve $1."
        return 1
    fi
}


set_errlvl() { return "${1:-1}"; }

export master_pid=$$
ssh:open() {
    local hostname ssh_cmd ssh_options
    ssh_cmd=(ssh)
    ssh_options=()
    while [ "$#" != 0 ]; do
        case "$1" in
            "--stdin-password")
                ssh_cmd=(sshpass "${ssh_cmd[@]}")
                ;;
            -o)
                ssh_options+=("$1" "$2")
                shift
                ;;
            *)
                [ -z "$hostname" ] && hostname="$1" || {
                        err "Surnumerous positional argument '$1'. Expecting only hostname."
                        return 1
                    }
                ;;
        esac
        shift
    done
    full_cmd=(
        "${ssh_cmd[@]}"
        -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
        -o ControlMaster=auto -o ControlPersist=900  \
        -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
        "${ssh_options[@]}" \
        "$hostname" "$@" -- true)
    "${full_cmd[@]}" </dev/null >/dev/null 2>&1 || {
        err "Failed: ${full_cmd[*]}"
        return 1
    }
    trap_add EXIT,INT 'ssh:quit "$hostname"'
}


ssh:open-try() {
    local opts hostnames
    opts=()
    hostnames=()
    while [ "$#" != 0 ]; do
        case "$1" in
            -o)
                opts+=("$1" "$2")
                shift
                ;;
            *)
                hostnames+=("$1")
                ;;
        esac
        shift
    done
    password=''
    for host in "${hostnames[@]}"; do
        debug "Trying $host with publickey."
        ssh:open -o PreferredAuthentications=publickey \
                 "${opts[@]}" \
                 "$host" >/dev/null 2>&1 && {
            echo "$host"$'\n'"$password"$'\n'
            return 0
        }
        debug "  .. failed connecting to $host with publickey."
    done
    local times=0 password
    while [ "$((++times))" -le 3 ]; do
        read -sp "$HOST's password: " password
        errlvl="$?"
        echo >&2
        if [ "$errlvl" -gt 0 ]; then
            exit 1
        fi
        for host in "${hostnames[@]}"; do
            debug "Trying $host with password ($times/3)"
            echo "$password" | ssh:open -o PreferredAuthentications=password \
                                        --stdin-password \
                                        "${opts[@]}" \
                                        "$host" >/dev/null 2>&1 && {
                echo "$host"$'\n'"$password"$'\n'
                return 0
            }
            debug "  .. failed connecting to $host with password."
        done
        err "login failed. Try again... ($((times+1))/3)"
    done
    return 1
}


ssh:run() {
    local hostname="$1" ssh_options cmd
    shift

    ssh_options=()
    cmd=()
    while [ "$#" != 0 ]; do
        case "$1" in
            "--")
                shift
                cmd+=("$@")
                break
                ;;
            *)
                ssh_options+=("$1")
                ;;
        esac
        shift
    done
    #echo "$DARKCYAN$hostname$NORMAL $WHITE\$$NORMAL" "$@"
    debug "Running cmd: ${cmd[@]}"
    for arg in "${cmd[@]}"; do
        debug "$arg"
    done
    {
        {
            ssh -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
                -o ControlMaster=auto -o ControlPersist=900 \
                -o "StrictHostKeyChecking=no" \
                 "${ssh_options[@]}" "$hostname" -- "${cmd[@]}"
        } 3>&1 1>&2 2>&3  ## | sed -r "s/^/$DARKCYAN$hostname$NORMAL $DARKRED\!$NORMAL /g"
        set_errlvl "${PIPESTATUS[0]}"
    } 3>&1 1>&2 2>&3
}

ssh:quit() {
    local hostname="$1"
    shift
    ssh -o ControlPath=/tmp/ssh-control-master-${master_pid} \
        -o ControlMaster=auto -o ControlPersist=900 -O exit \
        "$hostname" 2>/dev/null
}


is_ovh_domain_name() {
    local domain="$1"

    [[ "$domain" == *.ovh.net ]] && return 0
    [[ "$domain" == "ns"*".ip-"*".eu" ]] && return 0
    return 1
}

is_ovh_hostname() {
    local domain="$1"

    [[ "$domain" =~ ^vps-[0-9a-f]*$ ]] && return 0
    [[ "$domain" =~ ^vps[0-9]*$ ]] && return 0
    return 1
}

vps_connection_check() {
    local vps="$1"
    ip=$(resolve "$vps") ||
        { echo "${DARKRED}no-resolve${NORMAL}"; return 1; }

    is-port-open "$ip" "22" </dev/null ||
        { echo "${DARKRED}no-port-22-open${NORMAL}"; return 1; }

    ssh:open -o ConnectTimeout=10 -o PreferredAuthentications=publickey \
             "root@$vps" >/dev/null 2>&1 ||
        { echo "${DARKRED}no-ssh-root-access${NORMAL}"; return 1; }
}

vps_check() {
    local vps="$1"
    vps_connection_check "$vps" </dev/null || return 1
    if size=$(
            echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 5 -d ' ' | cut -f 1 -d %" |
                ssh:run "root@$vps" -- bash); then
        if [ "$size" -gt "90" ]; then
            echo "${DARKRED}above-90%-disk-usage${NORMAL}"
        elif [ "$size" -gt "75" ]; then
            echo "${DARKYELLOW}above-75%-disk-usage${NORMAL}"
        fi
    else
         echo "${DARKRED}no-size${NORMAL}"
    fi </dev/null
    compose_content=$(ssh:run "root@$vps" -- cat /opt/apps/myc-deploy/compose.yml </dev/null) ||
        { echo "${DARKRED}no-compose${NORMAL}"; return 1; }
    echo "$compose_content" | grep backup >/dev/null 2>&1 ||
        { echo "${DARKRED}no-backup${NORMAL}"; return 1; }
}


vps:rsync() {
    local vps="$1" id="$2" src="$3" dst="$4"
    if [[ "$src" != *":"* ]]; then
        err "Third argument '$src' should be a remote (include the server name as prefix)."
        return 1
    fi
    server=${src%%:*}
    src=${src#*:}
    cat <<EOF | ssh:run "root@$vps" -- bash
rsync -e "ssh ${ssh_options[*]}" \
              -azvArH --delete --delete-excluded \
              --partial --partial-dir .rsync-partial \
              --numeric-ids \
      "$server":/var/mirror/"${id}${src}" "${dst}"
EOF

}


mailcow:vps_backup_recover() {
    local admin="$1" server="$2" id="$3" vps="$4"

    ## Request recovery key
    ssh_options=()
    if [[ "$server" == *":"* ]]; then
        ssh_options+=(-p "${server#*:}" -o StrictHostKeyChecking=no)
        server=${server%%:*}
    fi
    if ! private_key=$(ssh "${ssh_options[@]}" "$admin"@"$server" request-recovery-key "$id"); then
        err "Couldn't request a recovery key for '$id' with account '$admin'."
        return 1
    fi

    echo "type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null" |
        ssh:run "root@$vps" -- bash

    if ! VPS_TMP_DIR=$(echo "mktemp -d" | ssh:run "root@$vps" -- bash); then
        err "Couldn't create a temporary directory on vps"
        return 1
    fi

    cat <<EOF | ssh:run "root@$vps" -- bash || return 1
    touch "$VPS_TMP_DIR/recover_key" &&
    chmod go-rwx "$VPS_TMP_DIR/recover_key" &&
    printf "%s\n" "$private_key" >> "$VPS_TMP_DIR/recover_key"
EOF

    ssh_options+=(-i "$VPS_TMP_DIR/recover_key" -l rsync)

    if ! compose_yml_files=$(cat <<EOF | ssh:run "root@$vps" -- bash
urn=com.docker.compose.project
docker ps -f "label=\$urn=mailcowdockerized" \
    --format='{{.Label "\$urn.working_dir"}}/{{.Label "\$urn.config_files"}}' |
    uniq
EOF
         ); then
        err "Couldn't get list of running projects"
        return 1
    fi

    stopped_containers=
    if [ -n "$compose_yml_files" ]; then
        echo "Found running mailcowdockerized containers" >&2
        if [[ "$compose_yml_files" == *$'\n'* ]]; then
            err "Running containers are confusing, did not find only one mailcowdockerized project."
            return 1
        fi
        if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
            err "Running containers are confusing, they don't point to an existing docker-compose.yml."
            return 1
        fi
        echo "Containers where launched from '$compose_yml_files'" >&2
        COMPOSE_FILE="$compose_yml_files"
        ENV_FILE="${COMPOSE_FILE%/*}/.env"
        if ! echo "[ -e \"${ENV_FILE}\" ]" | ssh:run "root@$vps" -- bash ; then
            err "Running containers are confusing, docker-compose.yml has no '.env' next to it."
            return 1
        fi
        echo "${WHITE}Bringing mailcowdockerized down${NORMAL}"
        echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
            ssh:run "root@$vps" -- bash
        stopped_containers=1
    fi


    for vol in postfix rspamd redis crypt vmail{,-attachments}; do
        volume_name="mailcowdockerized_${vol}-vol-1"
        volume_dir="/var/lib/docker/volumes/${volume_name}/_data"
        if ! ssh:run "root@$server" -- "[ -d '$BACKUP_PATH/${id}${volume_dir}' ]"; then
            continue
        fi

        ## Create volumes if not existent
        if ! ssh:run "root@$vps" -- "
                  [ -d '${volume_dir}' ] ||
                      docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
                  [ -d '${volume_dir}' ]
              "; then
            err "Couldn't find nor create '${volume_dir}'."
            return 1
        fi

        echo "${WHITE}Downloading of $volume_name${NORMAL}"
        vps:rsync "$vps" "$id" "$server":"${volume_dir}/" "${volume_dir}" || return 1
    done


    ## Mailcow git base
    for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
        if ! ssh:run "root@$server" -- "[ -d '$BACKUP_PATH/${id}${mailcow_dir}' ]"; then
            continue
        else
            ## this possibly change last value
            COMPOSE_FILE="$mailcow_dir/docker-compose.yml"
            ENV_FILE="$mailcow_dir/.env"
            echo "${WHITE}Download of $mailcow_dir${NORMAL}"
            vps:rsync "$vps" "$id" "$server":"${mailcow_dir}"/ "${mailcow_dir}" || return 1
            break
        fi
    done


    ## Mysql database
    echo "${WHITE}Downloading last backup of mysql backups${NORMAL}"
    vps:rsync "$vps" "$id" "$server":"/var/backups/mysql/" "/var/backups/mysql" || return 1

    if ! env_content=$(echo "cat '$ENV_FILE'" | ssh:run "root@$vps" -- bash); then
        err "Can't access env file: '$ENV_FILE'."
        return 1
    fi

    root_password=$(printf "%s\n" "$env_content" | grep ^DBROOT= | cut -f 2 -d =)

    echo "${WHITE}Bringing mysql-mailcow up${NORMAL}"
    if ! container_id=$(cat <<EOF | ssh:run "root@$vps" -- bash
echo "[client]
password=$root_password" > "$VPS_TMP_DIR/my.cnf"

docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
    run -d \
    -v "$VPS_TMP_DIR/my.cnf:/root/.my.cnf:ro" \
    mysql-mailcow

EOF
         ); then
        err "Failed to bring up mysql-mailcow"
        return 1
    fi

    START="$SECONDS"
    while true; do
        echo "  trying to connect..." >&2
        cat <<EOF | ssh:run "root@$vps" -- bash && break
    echo "SELECT 1;" | docker exec -i "$container_id" mysql >/dev/null 2>&1
EOF
        if (($SECONDS - $START > 10)); then
            err "Failed to connect to mysql-mailcow."
            echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
                ssh:run "root@$vps" -- bash
            return 1
        fi
        sleep 0.3
    done

    DBUSER=$(printf "%s\n" "$env_content" | grep ^DBUSER= | cut -f 2 -d =)
    DBPASS=$(printf "%s\n" "$env_content" | grep ^DBPASS= | cut -f 2 -d =)

    echo "${WHITE}Uploading mysql dump${NORMAL}"
    cat <<EOF | ssh:run "root@$vps" -- bash

echo "
  DROP DATABASE IF EXISTS mailcow;
  CREATE DATABASE mailcow;
  GRANT ALL PRIVILEGES ON mailcow.* TO '$DBUSER'@'%' IDENTIFIED BY '$DBPASS';
" | docker exec -i "$container_id" mysql

zcat /var/backups/mysql/mailcow/*.gz | docker exec -i "$container_id" mysql mailcow

EOF
    if [ "$?" != 0 ]; then
        err "Failed to load mysql dump."
        echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
            ssh:run "root@$vps" -- bash
        return 1
    fi

    echo "${WHITE}Bringing mysql-mailcow down${NORMAL}"
    echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
        ssh:run "root@$vps" -- bash
    ssh:run "root@$vps" -- "rm -rf '$VPS_TMP_DIR'"

    if [ -n "$stopped_containers" ]; then
        echo "${WHITE}Starting mailcow${NORMAL}" >&2
        echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" up -d" |
            ssh:run "root@$vps" -- bash
    fi
    info "Mailcow was ${GREEN}successfully${NORMAL} restored."

}

vps_backup_recover() {
    local vps="$1" admin server id rtype force
    vps_connection_check "$vps" </dev/null || return 1

    read-0 admin server id rtype force

    if ! type=$(ssh:run "root@$vps" -- vps get-type); then
        err "Could not get type."
        return 1
    fi

    if [ "$rtype" != "$type" ]; then
        if [ -n "$force" ]; then
            warn "Backup found is of ${rtype:-unknown} type, while vps is of $type type."
        else
            err "Backup found is of ${rtype:-unknown} type, while vps is of $type type. (use \`\`-f\`\` to force)"
            return 1
        fi
    fi

    case "$rtype" in
        mailcow)
            mailcow:vps_backup_recover "$admin" "$server" "$id" "$vps"
            ;;
        *)
            err "Recover on $type type VPS is not yet implemented."
            return 1
            ;;
    esac

}


vps_install_backup() {
    local vps="$1" admin server
    vps_connection_check "$vps" </dev/null || return 1

    read-0 admin server
    if ! type=$(ssh:run "root@$vps" -- vps get-type); then
        err "Could not get type."
        return 1
    fi

    if ! out=$(ssh:run "root@$vps" -- vps install backup "$server" 2>&1); then
        err "Command 'vps install backup $server' failed."
        return 1
    fi

    already_present=
    if e "$out" | grep "^II Entry for service .* is already present" >/dev/null 2>&1; then
        already_present=1
        info "Backup entry is already present in 'compose.yml' of '$vps'"
    fi
    out="${out%$'\n'}"
    out="${out#*$'\n'}"
    key="${out%\'*}"
    key="${key##*\'}"

    if ! [[ "$key" =~ ^"ssh-rsa "[a-zA-Z0-9/+]+" "[a-zA-Z0-9._-]+"@"[a-zA-Z0-9._-]+$ ]]; then
        err "Unexpected output from 'vps install backup $server'. Can't find key."
        echo "$out" | prefix " ${GRAY}|$NORMAL " >&2
        echo " Extracted key:" >&2
        echo "$key" | prefix " ${GRAY}|$NORMAL " >&2
        return 1
    fi

    if [ "$type" == "compose" ] && [ -z "$already_present" ]; then
        ssh:run "root@$vps" -- compose --debug up || {
            err "Command 'compose --debug up' failed."
            return 1
        }
    fi

    dest="$server"
    dest="${dest%/*}"
    ssh_options=()
    if [[ "$dest" == *":"* ]]; then
        port="${dest##*:}"
        dest="${dest%%:*}"
        ssh_options=(-p "$port")
    else
        port=""
        dest="${dest%%:*}"
    fi

    cmd=(ssh "${ssh_options[@]}" "$admin"@"$dest" ssh-key add "$key")
    echo "${WHITE}Launching:${NORMAL} ${cmd[@]}"

    "${cmd[@]}" || {
        err "Failed add key to backup server '$dest'."
        return 1
    }
    echo "${WHITE}Launching backup${NORMAL} from '$vps'"

    ssh:run "root@$vps" -- vps backup || {
        err "First backup failed to run."
        return 1
    }

    echo "Backup is ${GREEN}up and running${NORMAL}."
}


vps_udpate() {
    local vps="$1"
    vps_connection_check "$vps" || return 1
    ssh:run "root@$vps" -- myc-update </dev/null
}


vps_bash() {
    local vps="$1"
    vps_connection_check "$vps" </dev/null || return 1
    ssh:run "root@$vps" -- bash
}

vps_mux() {
    local fn="$1" vps_done VPS max_size vps
    shift
    VPS=($(printf "%s\n" "$@" | sort))
    max_size=0
    declare -A vps_done;
    new_vps=()
    for name in "${VPS[@]}"; do
        [ -n "${vps_done[$name]}" ] && {
            warn "duplicate vps '$name' provided. Ignoring."
            continue
        }
        vps_done["$name"]=1
        new_vps+=("$name")
        size_name="${#name}"
        [ "$max_size" -lt "${size_name}" ] &&
            max_size="$size_name"
    done
    settmpdir "_0KM_TMP_DIR"
    cat > "$_0KM_TMP_DIR/code"
    for vps in "${new_vps[@]}"; do
        label=$(printf "%-${max_size}s" "$vps")
        (
            {
                {
                     "$fn" "$vps" < "$_0KM_TMP_DIR/code"
                } 3>&1 1>&2 2>&3 | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKRED\!$NORMAL /g"
                set_errlvl "${PIPESTATUS[0]}"
            } 3>&1 1>&2 2>&3  | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKGRAY\|$NORMAL /g"
            set_errlvl "${PIPESTATUS[0]}"
        ) &
    done
    wait
}


[ "$SOURCED" ] && return 0

##
## Command line processing
##


cmdline.spec.gnu
cmdline.spec.reporting

cmdline.spec.gnu vps-setup

cmdline.spec::cmd:vps-setup:run() {

    : :posarg: HOST        'Target host to check/fix ssh-access'

    depends sshpass shyaml

    KEY_PATH="ssh-access.public-keys"
    local keys=$(config get-value -y "ssh-access.public-keys") || true
    if [ -z "$keys" ]; then
        err "No ssh publickeys configured in config file."
        echo "  Looking for keys in ${WHITE}${KEY_PATH}${NORMAL}" \
             "in config file." >&2
        config:exists --message 2>&1 | prefix "  "
        if [ "${PIPESTATUS[0]}" == "0" ]; then
            echo "  Config file found in $(config:filename)"
        fi
        return 1
    fi
    local tkey=$(e "$keys" | shyaml get-type)
    if [ "$tkey" != "sequence" ]; then
        err "Value type of ${WHITE}${KEY_PATH}${NORMAL} unexpected (is $tkey, expecting sequence)."
        echo "  Check content of $(config:filename), and make sure to use a sequence." >&2
        return 1
    fi

    local IP NAME keys host_pass_connected
    if ! IP=$(resolve "$HOST"); then
        err "'$HOST' name unresolvable."
        exit 1
    fi

    NAME="$HOST"
    if [ "$IP" != "$HOST" ]; then
        NAME="$HOST ($IP)"
    fi

    if ! is-port-open "$IP" "22"; then
        err "$NAME unreachable or port 22 closed."
        exit 1
    fi
    debug "Host $IP's port 22 is open."
    if ! host_pass_connected=$(ssh:open-try \
                                   {root,debian}@"$HOST"); then
        err "Could not connect to {root,debian}@$HOST with publickey nor password."
        exit 1
    fi

    read-0a host password <<<"$host_pass_connected"

    sudo_if_necessary=
    if [ "$password" -o "${host%%@*}" != "root" ]; then
        if ! ssh:run "$host" -- sudo -n true >/dev/null 2>&1; then
            err "Couldn't do a password-less sudo from $host."
            echo "  This is not yet supported."
            exit 1
        else
            sudo_if_necessary=sudo
        fi
    fi

    Section Checking access
    while read-0 key; do
        prefix="${key%% *}"
        if [ "$prefix" != "ssh-rsa" ]; then
            err "Unsupported key:"$'\n'"$key"
            return 1
        fi
        label="${key##* }"
        Elt "considering adding key ${DARKYELLOW}$label${NORMAL}"
        dest="/root/.ssh/authorized_keys"
        if ssh:run "$host" -- $sudo_if_necessary grep "\"$key\"" "$dest" >/dev/null 2>&1 </dev/null; then
            print_info "already present"
            print_status noop
            Feed
        else
            if echo "$key" | ssh:run "$host" -- $sudo_if_necessary tee -a "$dest" >/dev/null; then
                print_info added
            else
                echo
                Feedback failure
                return 1
            fi
            Feedback success
        fi
    done < <(e "$keys" | shyaml get-values-0)

    Section Checking ovh hostname file
    Elt "Checking /etc/ovh-hostname"
    if ! ssh:run "$host" -- $sudo_if_necessary [ -e "/etc/ovh-hostname" ]; then
        print_info "creating"
        ssh:run "$host" -- $sudo_if_necessary cp /etc/hostname /etc/ovh-hostname
        ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
        Elt "Checking /etc/ovh-hostname: $ovhname"
        Feedback || return 1
    else
        ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
        Elt "Checking /etc/ovh-hostname: $ovhname"
        print_info "already present"
        print_status noop
        Feed
    fi

    if ! is_ovh_domain_name "$HOST" && ! str_is_ipv4 "$HOST"; then
            Section Checking hostname
            Elt "Checking /etc/hostname..."
            if [ "$old" != "$HOST" ]; then
                old="$(ssh:run "$host" -- $sudo_if_necessary cat /etc/hostname)"
                Elt "Hostname is '$old'"
                if is_ovh_hostname "$old"; then
                    Elt "Hostname '$old' --> '$HOST'"
                    print_info "creating"
                    echo "$HOST" | ssh:run "$host" -- $sudo_if_necessary tee /etc/hostname >/dev/null &&
                        ssh:run "$host" -- $sudo_if_necessary hostname "$HOST"
                    Feedback || return 1
                else
                    print_info "not changing"
                    print_status noop
                    Feed
                fi

            else
                print_info "already set"
                print_status noop
                Feed
            fi
    else
        info "Not changing domain as '$HOST' doesn't seem to be final domain."
    fi

}


cmdline.spec.gnu vps-check
cmdline.spec::cmd:vps-check:run() {

    : :posarg: [VPS...]        'Target host(s) to check'


    echo "" |
        vps_mux vps_check "${VPS[@]}"
}


cmdline.spec.gnu vps-install
cmdline.spec::cmd:vps-install:run() {
    :
}

cmdline.spec.gnu backup
cmdline.spec:vps-install:cmd:backup:run() {

    : :posarg: BACKUP_TARGET   'Backup target.
                                (ie: myadmin@backup.domain.org:10023/256)'
    : :posarg: [VPS...]        'Target host(s) to check'


    if [ "${#VPS[@]}" == 0 ]; then
        warn "VPS list provided in command line is empty. Nothing will be done."
        return 0
    fi

    if ! [[ "$BACKUP_TARGET" == *"@"* ]]; then
        err "Missing admin account identifier in backup target."
        echo "  Have you forgottent to specify an account, ie 'myadmin@<MYBACKUP_SERVER>' ?)"
        return 1
    fi

    admin=${BACKUP_TARGET%%@*}
    server=${BACKUP_TARGET#*@}
    p0 "$admin" "$server" |
        vps_mux vps_install_backup "${VPS[@]}"
}



cmdline.spec.gnu vps-backup
cmdline.spec::cmd:vps-backup:run() {
    :
}


cmdline.spec.gnu ls
cmdline.spec:vps-backup:cmd:ls:run() {
    : :posarg: BACKUP_ID       'Backup id.
                                (ie: myadmin@backup.domain.org:10023)'

    if ! [[ "$BACKUP_ID" == *"@"* ]]; then
        err "Missing admin account identifier in backup id."
        echo "  Have you forgottent to specify an admin account ?" \
             "ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
        return 1
    fi

    id=${BACKUP_ID##*#}
    BACKUP_TARGET=${BACKUP_ID%#*}
    admin=${BACKUP_TARGET%%@*}
    server=${BACKUP_TARGET#*@}

    ## XXXvlab: in this first implementation we expect to have access
    ## to the server main ssh port 22, so we won't use the provided port.
    ssh_options=()
    if [[ "$server" == *":"* ]]; then
        ssh_options+=(-p "${server#*:}")
        server=${server%%:*}
    fi

    ssh "${ssh_options[@]}" "$admin"@"$server" ssh-key ls
}


cmdline.spec.gnu recover
cmdline.spec:vps-backup:cmd:recover:run() {

    : :posarg: BACKUP_ID       'Backup id.
                                (ie: myadmin@backup.domain.org:10023#mx.myvps.org)'
    : :posarg: [VPS...]        'Target host(s) to check'

    : :optval: --date,-D       '"last", or label of version to recover. (Default: "last").'
    : :optfla: --force,-f       'Will allow you to bypass some checks.'


    if [ "${#VPS[@]}" == 0 ]; then
        warn "VPS list provided in command line is empty. Nothing will be done."
        return 0
    fi

    if ! [[ "$BACKUP_ID" == *"@"* ]]; then
        err "Missing admin account identifier in backup id."
        echo "  Have you forgottent to specify an admin account ?" \
             "ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
        return 1
    fi
    if ! [[ "$BACKUP_ID" == *"@"*"#"* ]]; then
        err "Missing backup label identifier in backup id."
        echo "  Have you forgottent to specify a backup label identifier ?" \
             "ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
        return 1
    fi

    id=${BACKUP_ID##*#}
    BACKUP_TARGET=${BACKUP_ID%#*}
    admin=${BACKUP_TARGET%%@*}
    server=${BACKUP_TARGET#*@}

    ## XXXvlab: in this first implementation we expect to have access
    ## to the server main ssh port 22, so we won't use the provided port.
    # ssh_options=()
    if [[ "$server" == *":"* ]]; then
        # ssh_options+=(-p "${server#*:}")
        ssh_server=${server%%:*}
    fi

    BACKUP_PATH="/srv/datastore/data/rsync-backup-target/var/mirror"

    if ! rtype=$(echo "
                  if [ -d '$BACKUP_PATH/$id/var/lib/docker/volumes/mailcowdockerized_crypt-vol-1' ]; then
                      echo mailcow
                  elif [ -d '$BACKUP_PATH/$id/compose.yml' ]; then
                      echo compose
                  fi
                  true
                " | ssh:run "root@${ssh_server}" -- bash ); then
        err "Could not get backup type."
        return 1
    fi

    if [ -z "$rtype" ]; then
        err "Unknown type of backup on '${server}#${id}'."
        return 1
    fi

    p0 "$admin" "$server" "$id" "$rtype" "$opt_force" |
        vps_mux vps_backup_recover "${VPS[@]}"
}




cmdline.spec.gnu vps-update
cmdline.spec::cmd:vps-update:run() {

    : :posarg: [VPS...]        'Target host to check'

    echo "" |
        vps_mux vps_update "${VPS[@]}"
}


cmdline.spec.gnu vps-mux
cmdline.spec::cmd:vps-mux:run() {

    : :posarg: [VPS...]        'Target host(s) to check'

    cat | vps_mux vps_bash "${VPS[@]}"
}


cmdline.spec.gnu vps-space
cmdline.spec::cmd:vps-space:run() {

    : :posarg: [VPS...]        'Target host(s) to check'

    echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 2-5 -d ' '" |
        vps_mux vps_bash "${VPS[@]}"
}




cmdline::parse "$@"