myc-manage/bin/0km

#!/bin/bash


. /etc/shlib

include common
include parse
include cmdline
include config


[[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true

version=0.1
desc='Manage 0k related installs'
help=""


##
## Functions
##

is-port-open() {
    local host="$1" port="$2" timeout=5
    start="$SECONDS"
    debug "Testing if $host's port $2 is open ..."
    while true; do
        timeout 1 bash -c "</dev/tcp/${host}/${port}" >/dev/null 2>&1 && break
        sleep 0.2
        if [ "$((SECONDS - start))" -gt "$timeout" ]; then
            return 1
        fi
    done
}

resolve() {
    local ent hostname="$1"
    debug "Resolving $1 ..."
    if ent=$(getent ahosts "$hostname"); then
        ent=$(echo "$ent" | egrep ^"[0-9]+.[0-9]+.[0-9]+.[0-9]+\s+" | \
                  head -n 1 | awk '{ print $1 }')
        debug "  .. resolved $1 to $ent."
        echo "$ent"
    else
        debug "  .. couldn't resolve $1."
        return 1
    fi
}


set_errlvl() { return "${1:-1}"; }

export master_pid=$$
ssh:open() {
    local hostname ssh_cmd ssh_options
    ssh_cmd=(ssh)
    ssh_options=()
    while [ "$#" != 0 ]; do
        case "$1" in
            "--stdin-password")
                ssh_cmd=(sshpass "${ssh_cmd[@]}")
                ;;
            -o)
                ssh_options+=("$1" "$2")
                shift
                ;;
            *)
                [ -z "$hostname" ] && hostname="$1" || {
                        err "Surnumerous positional argument '$1'. Expecting only hostname."
                        return 1
                    }
                ;;
        esac
        shift
    done
    full_cmd=(
        "${ssh_cmd[@]}"
        -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
        -o ControlMaster=auto -o ControlPersist=900  \
        -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
        "${ssh_options[@]}" \
        "$hostname" "$@" -- true)
    "${full_cmd[@]}" </dev/null >/dev/null 2>&1 || {
        err "Failed: ${full_cmd[*]}"
        return 1
    }
    trap_add EXIT,INT 'ssh:quit "$hostname"'
}


ssh:open-try() {
    local opts hostnames
    opts=()
    hostnames=()
    while [ "$#" != 0 ]; do
        case "$1" in
            -o)
                opts+=("$1" "$2")
                shift
                ;;
            *)
                hostnames+=("$1")
                ;;
        esac
        shift
    done
    password=''
    for host in "${hostnames[@]}"; do
        debug "Trying $host with publickey."
        ssh:open -o PreferredAuthentications=publickey \
                 "${opts[@]}" \
                 "$host" >/dev/null 2>&1 && {
            echo "$host"$'\n'"$password"$'\n'
            return 0
        }
        debug "  .. failed connecting to $host with publickey."
    done
    local times=0 password
    while [ "$((++times))" -le 3 ]; do
        read -sp "$HOST's password: " password
        errlvl="$?"
        echo >&2
        if [ "$errlvl" -gt 0 ]; then
            exit 1
        fi
        for host in "${hostnames[@]}"; do
            debug "Trying $host with password ($times/3)"
            echo "$password" | ssh:open -o PreferredAuthentications=password \
                                        --stdin-password \
                                        "${opts[@]}" \
                                        "$host" >/dev/null 2>&1 && {
                echo "$host"$'\n'"$password"$'\n'
                return 0
            }
            debug "  .. failed connecting to $host with password."
        done
        err "login failed. Try again... ($((times+1))/3)"
    done
    return 1
}


ssh:run() {
    local hostname="$1" ssh_options cmd
    shift

    ssh_options=()
    cmd=()
    while [ "$#" != 0 ]; do
        case "$1" in
            "--")
                shift
                cmd+=("$@")
                break
                ;;
            *)
                ssh_options+=("$1")
                ;;
        esac
        shift
    done
    #echo "$DARKCYAN$hostname$NORMAL $WHITE\$$NORMAL" "$@"
    debug "Running cmd: ${cmd[@]}"
    for arg in "${cmd[@]}"; do
        debug "$arg"
    done
    {
        {
            ssh -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
                -o ControlMaster=auto -o ControlPersist=900 \
                -o "StrictHostKeyChecking=no" \
                "$hostname" "${ssh_options[@]}" -- "${cmd[@]}"
        } 3>&1 1>&2 2>&3  ## | sed -r "s/^/$DARKCYAN$hostname$NORMAL $DARKRED\!$NORMAL /g"
        set_errlvl "${PIPESTATUS[0]}"
    } 3>&1 1>&2 2>&3
}

ssh:quit() {
    local hostname="$1"
    shift
    ssh -o ControlPath=/tmp/ssh-control-master-${master_pid} \
        -o ControlMaster=auto -o ControlPersist=900 -O exit \
        "$hostname" 2>/dev/null
}


is_ovh_domain_name() {
    local domain="$1"

    [[ "$domain" == *.ovh.net ]] && return 0
    [[ "$domain" == "ns"*".ip-"*".eu" ]] && return 0
    return 1
}

is_ovh_hostname() {
    local domain="$1"

    [[ "$domain" =~ ^vps-[0-9a-f]*$ ]] && return 0
    [[ "$domain" =~ ^vps[0-9]*$ ]] && return 0
    return 1
}

vps_connection_check() {
    local vps="$1"
    ip=$(resolve "$vps") ||
        { echo "${DARKRED}no-resolve${NORMAL}"; return 1; }

    is-port-open "$ip" "22" </dev/null ||
        { echo "${DARKRED}no-port-22-open${NORMAL}"; return 1; }

    ssh:open -o ConnectTimeout=2 -o PreferredAuthentications=publickey \
             "root@$vps" >/dev/null 2>&1 ||
        { echo "${DARKRED}no-ssh-root-access${NORMAL}"; return 1; }
}

vps_check() {
    local vps="$1"
    vps_connection_check "$vps" </dev/null || return 1
    if size=$(
            echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 5 -d ' ' | cut -f 1 -d %" |
                ssh:run "root@$vps" -- bash); then
        if [ "$size" -gt "90" ]; then
            echo "${DARKRED}above-90%-disk-usage${NORMAL}"
        elif [ "$size" -gt "75" ]; then
            echo "${DARKYELLOW}above-75%-disk-usage${NORMAL}"
        fi
    else
         echo "${DARKRED}no-size${NORMAL}"
    fi </dev/null
    compose_content=$(ssh:run "root@$vps" -- cat /opt/apps/myc-deploy/compose.yml </dev/null) ||
        { echo "${DARKRED}no-compose${NORMAL}"; return 1; }
    echo "$compose_content" | grep backup >/dev/null 2>&1 ||
        { echo "${DARKRED}no-backup${NORMAL}"; return 1; }
}


vps_install_backup() {
    local vps="$1" admin server
    vps_connection_check "$vps" </dev/null || return 1

    read-0 admin server
    if ! type=$(ssh:run "root@$vps" -- vps get-type); then
        err "Could not get type."
        return 1
    fi

    if ! out=$(ssh:run "root@$vps" -- vps install backup "$server" 2>&1); then
        err "Command 'vps install backup $server' failed."
        return 1
    fi

    already_present=
    if e "$out" | grep "^II Entry for service .* is already present" >/dev/null 2>&1; then
        already_present=1
        info "Backup entry is already present in 'compose.yml' of '$vps'"
    fi
    out="${out%$'\n'}"
    out="${out#*$'\n'}"
    key="${out%\'*}"
    key="${key##*\'}"

    if ! [[ "$key" =~ ^"ssh-rsa "[a-zA-Z0-9/+]+" "[a-zA-Z0-9._-]+"@"[a-zA-Z0-9._-]+$ ]]; then
        err "Unexpected output from 'vps install backup $server'. Can't find key."
        echo "$out" | prefix " ${GRAY}|$NORMAL " >&2
        echo " Extracted key:" >&2
        echo "$key" | prefix " ${GRAY}|$NORMAL " >&2
        return 1
    fi

    if [ "$type" == "compose" ] && [ -z "$already_present" ]; then
        ssh:run "root@$vps" -- compose --debug up || {
            err "Command 'compose --debug up' failed."
            return 1
        }
    fi

    dest="$server"
    dest="${dest%/*}"
    ssh_options=()
    if [[ "$dest" == *":"* ]]; then
        port="${dest##*:}"
        dest="${dest%%:*}"
        ssh_options=(-p "$port")
    else
        port=""
        dest="${dest%%:*}"
    fi

    cmd=(ssh "${ssh_options[@]}" "$admin"@"$dest" ssh-key add "$key")
    echo "${WHITE}Launching:${NORMAL} ${cmd[@]}"

    "${cmd[@]}" || {
        err "Failed add key to backup server '$dest'."
        return 1
    }
    echo "${WHITE}Launching backup${NORMAL} from '$vps'"

    ssh:run "root@$vps" -- vps backup || {
        err "First backup failed to run."
        return 1
    }

    echo "Backup is ${GREEN}up and running${NORMAL}."
}


vps_udpate() {
    local vps="$1"
    vps_connection_check "$vps" || return 1
    ssh:run "root@$vps" -- myc-update </dev/null
}


vps_bash() {
    local vps="$1"
    vps_connection_check "$vps" </dev/null || return 1
    ssh:run "root@$vps" -- bash
}

vps_mux() {
    local fn="$1" vps_done VPS max_size vps
    shift
    VPS=($(printf "%s\n" "$@" | sort))
    max_size=0
    declare -A vps_done;
    new_vps=()
    for name in "${VPS[@]}"; do
        [ -n "${vps_done[$name]}" ] && {
            warn "duplicate vps '$name' provided. Ignoring."
            continue
        }
        vps_done["$name"]=1
        new_vps+=("$name")
        size_name="${#name}"
        [ "$max_size" -lt "${size_name}" ] &&
            max_size="$size_name"
    done
    settmpdir "_0KM_TMP_DIR"
    cat > "$_0KM_TMP_DIR/code"
    for vps in "${new_vps[@]}"; do
        label=$(printf "%-${max_size}s" "$vps")
        (
            {
                {
                     "$fn" "$vps" < "$_0KM_TMP_DIR/code"
                } 3>&1 1>&2 2>&3 | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKRED\!$NORMAL /g"
                set_errlvl "${PIPESTATUS[0]}"
            } 3>&1 1>&2 2>&3  | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKGRAY\|$NORMAL /g"
            set_errlvl "${PIPESTATUS[0]}"
        ) &
    done
    wait
}


[ "$SOURCED" ] && return 0

##
## Command line processing
##


cmdline.spec.gnu
cmdline.spec.reporting

cmdline.spec.gnu vps-setup

cmdline.spec::cmd:vps-setup:run() {

    : :posarg: HOST        'Target host to check/fix ssh-access'

    depends sshpass shyaml

    KEY_PATH="ssh-access.public-keys"
    local keys=$(config get-value -y "ssh-access.public-keys") || true
    if [ -z "$keys" ]; then
        err "No ssh publickeys configured in config file."
        echo "  Looking for keys in ${WHITE}${KEY_PATH}${NORMAL}" \
             "in config file." >&2
        config:exists --message 2>&1 | prefix "  "
        if [ "${PIPESTATUS[0]}" == "0" ]; then
            echo "  Config file found in $(config:filename)"
        fi
        return 1
    fi
    local tkey=$(e "$keys" | shyaml get-type)
    if [ "$tkey" != "sequence" ]; then
        err "Value type of ${WHITE}${KEY_PATH}${NORMAL} unexpected (is $tkey, expecting sequence)."
        echo "  Check content of $(config:filename), and make sure to use a sequence." >&2
        return 1
    fi

    local IP NAME keys host_pass_connected
    if ! IP=$(resolve "$HOST"); then
        err "'$HOST' name unresolvable."
        exit 1
    fi

    NAME="$HOST"
    if [ "$IP" != "$HOST" ]; then
        NAME="$HOST ($IP)"
    fi

    if ! is-port-open "$IP" "22"; then
        err "$NAME unreachable or port 22 closed."
        exit 1
    fi
    debug "Host $IP's port 22 is open."
    if ! host_pass_connected=$(ssh:open-try \
                                   {root,debian}@"$HOST"); then
        err "Could not connect to {root,debian}@$HOST with publickey nor password."
        exit 1
    fi

    read-0a host password <<<"$host_pass_connected"

    sudo_if_necessary=
    if [ "$password" -o "${host%%@*}" != "root" ]; then
        if ! ssh:run "$host" -- sudo -n true >/dev/null 2>&1; then
            err "Couldn't do a password-less sudo from $host."
            echo "  This is not yet supported."
            exit 1
        else
            sudo_if_necessary=sudo
        fi
    fi

    Section Checking access
    while read-0 key; do
        prefix="${key%% *}"
        if [ "$prefix" != "ssh-rsa" ]; then
            err "Unsupported key:"$'\n'"$key"
            return 1
        fi
        label="${key##* }"
        Elt "considering adding key ${DARKYELLOW}$label${NORMAL}"
        dest="/root/.ssh/authorized_keys"
        if ssh:run "$host" -- $sudo_if_necessary grep "\"$key\"" "$dest" >/dev/null 2>&1 </dev/null; then
            print_info "already present"
            print_status noop
            Feed
        else
            if echo "$key" | ssh:run "$host" -- $sudo_if_necessary tee -a "$dest" >/dev/null; then
                print_info added
            else
                echo
                Feedback failure
                return 1
            fi
            Feedback success
        fi
    done < <(e "$keys" | shyaml get-values-0)

    Section Checking ovh hostname file
    Elt "Checking /etc/ovh-hostname"
    if ! ssh:run "$host" -- $sudo_if_necessary [ -e "/etc/ovh-hostname" ]; then
        print_info "creating"
        ssh:run "$host" -- $sudo_if_necessary cp /etc/hostname /etc/ovh-hostname
        ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
        Elt "Checking /etc/ovh-hostname: $ovhname"
        Feedback || return 1
    else
        ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
        Elt "Checking /etc/ovh-hostname: $ovhname"
        print_info "already present"
        print_status noop
        Feed
    fi

    if ! is_ovh_domain_name "$HOST" && ! str_is_ipv4 "$HOST"; then
            Section Checking hostname
            Elt "Checking /etc/hostname..."
            if [ "$old" != "$HOST" ]; then
                old="$(ssh:run "$host" -- $sudo_if_necessary cat /etc/hostname)"
                Elt "Hostname is '$old'"
                if is_ovh_hostname "$old"; then
                    Elt "Hostname '$old' --> '$HOST'"
                    print_info "creating"
                    echo "$HOST" | ssh:run "$host" -- $sudo_if_necessary tee /etc/hostname >/dev/null &&
                        ssh:run "$host" -- $sudo_if_necessary hostname "$HOST"
                    Feedback || return 1
                else
                    print_info "not changing"
                    print_status noop
                    Feed
                fi

            else
                print_info "already set"
                print_status noop
                Feed
            fi
    else
        info "Not changing domain as '$HOST' doesn't seem to be final domain."
    fi

}


cmdline.spec.gnu vps-check
cmdline.spec::cmd:vps-check:run() {

    : :posarg: [VPS...]        'Target host(s) to check'


    echo "" |
        vps_mux vps_check "${VPS[@]}"
}


cmdline.spec.gnu vps-install
cmdline.spec::cmd:vps-install:run() {
    :
}

cmdline.spec.gnu backup
cmdline.spec:vps-install:cmd:backup:run() {

    : :posarg: BACKUP_TARGET   'Backup target.
                                (ie: myadmin@backup.domain.org:10023/256)'
    : :posarg: [VPS...]        'Target host(s) to check'


    if [ "${#VPS[@]}" == 0 ]; then
        warn "VPS list provided in command line is empty. Nothing will be done."
        return 0
    fi

    if ! [[ "$BACKUP_TARGET" == *"@"* ]]; then
        err "Missing admin account identifier in backup target."
        echo "  Have you forgottent to specify an account, ie 'myadmin@<MYBACKUP_SERVER>' ?)"
        return 1
    fi

    admin=${BACKUP_TARGET%%@*}
    server=${BACKUP_TARGET#*@}
    p0 "$admin" "$server" |
        vps_mux vps_install_backup "${VPS[@]}"
}



cmdline.spec.gnu vps-update
cmdline.spec::cmd:vps-update:run() {

    : :posarg: [VPS...]        'Target host to check'

    echo "" |
        vps_mux vps_update "${VPS[@]}"
}


cmdline.spec.gnu vps-mux
cmdline.spec::cmd:vps-mux:run() {

    : :posarg: [VPS...]        'Target host(s) to check'

    cat | vps_mux vps_bash "${VPS[@]}"
}


cmdline.spec.gnu vps-space
cmdline.spec::cmd:vps-space:run() {

    : :posarg: [VPS...]        'Target host(s) to check'

    echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 2-5 -d ' '" |
        vps_mux vps_bash "${VPS[@]}"
}




cmdline::parse "$@"