#!/bin/bash . /etc/shlib include common include parse include cmdline include config [[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true version=0.1 desc='Manage 0k related installs' help="" ## ## Functions ## is-port-open() { local host="$1" port="$2" timeout=5 start="$SECONDS" debug "Testing if $host's port $2 is open ..." while true; do timeout 1 bash -c "/dev/null 2>&1 && break sleep 0.2 if [ "$((SECONDS - start))" -gt "$timeout" ]; then return 1 fi done } resolve() { local ent hostname="$1" debug "Resolving $1 ..." if ent=$(getent ahosts "$hostname"); then ent=$(echo "$ent" | egrep ^"[0-9]+.[0-9]+.[0-9]+.[0-9]+\s+" | \ head -n 1 | awk '{ print $1 }') debug " .. resolved $1 to $ent." echo "$ent" else debug " .. couldn't resolve $1." return 1 fi } set_errlvl() { return "${1:-1}"; } export master_pid=$$ ssh:open() { local hostname ssh_cmd ssh_options ssh_cmd=(ssh) ssh_options=() while [ "$#" != 0 ]; do case "$1" in "--stdin-password") ssh_cmd=(sshpass "${ssh_cmd[@]}") ;; -o) ssh_options+=("$1" "$2") shift ;; *) [ -z "$hostname" ] && hostname="$1" || { err "Surnumerous positional argument '$1'. Expecting only hostname." return 1 } ;; esac shift done full_cmd=( "${ssh_cmd[@]}" -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \ -o ControlMaster=auto -o ControlPersist=900 \ -o ConnectTimeout=5 -o StrictHostKeyChecking=no \ "${ssh_options[@]}" \ "$hostname" "$@" -- true) "${full_cmd[@]}" /dev/null 2>&1 || { err "Failed: ${full_cmd[*]}" return 1 } trap_add EXIT,INT 'ssh:quit "$hostname"' } ssh:open-try() { local opts hostnames opts=() hostnames=() while [ "$#" != 0 ]; do case "$1" in -o) opts+=("$1" "$2") shift ;; *) hostnames+=("$1") ;; esac shift done password='' for host in "${hostnames[@]}"; do debug "Trying $host with publickey." ssh:open -o PreferredAuthentications=publickey \ "${opts[@]}" \ "$host" >/dev/null 2>&1 && { echo "$host"$'\n'"$password"$'\n' return 0 } debug " .. failed connecting to $host with publickey." done local times=0 password while [ "$((++times))" -le 3 ]; do read -sp "$HOST's password: " password errlvl="$?" echo >&2 if [ "$errlvl" -gt 0 ]; then exit 1 fi for host in "${hostnames[@]}"; do debug "Trying $host with password ($times/3)" echo "$password" | ssh:open -o PreferredAuthentications=password \ --stdin-password \ "${opts[@]}" \ "$host" >/dev/null 2>&1 && { echo "$host"$'\n'"$password"$'\n' return 0 } debug " .. failed connecting to $host with password." done err "login failed. Try again... ($((times+1))/3)" done return 1 } ssh:run() { local hostname="$1" ssh_options cmd shift ssh_options=() cmd=() while [ "$#" != 0 ]; do case "$1" in "--") shift cmd+=("$@") break ;; *) ssh_options+=("$1") ;; esac shift done #echo "$DARKCYAN$hostname$NORMAL $WHITE\$$NORMAL" "$@" debug "Running cmd: ${cmd[@]}" for arg in "${cmd[@]}"; do debug "$arg" done { { ssh -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \ -o ControlMaster=auto -o ControlPersist=900 \ -o "StrictHostKeyChecking=no" \ "${ssh_options[@]}" "$hostname" -- "${cmd[@]}" } 3>&1 1>&2 2>&3 ## | sed -r "s/^/$DARKCYAN$hostname$NORMAL $DARKRED\!$NORMAL /g" set_errlvl "${PIPESTATUS[0]}" } 3>&1 1>&2 2>&3 } ssh:quit() { local hostname="$1" shift ssh -o ControlPath=/tmp/ssh-control-master-${master_pid} \ -o ControlMaster=auto -o ControlPersist=900 -O exit \ "$hostname" 2>/dev/null } is_ovh_domain_name() { local domain="$1" [[ "$domain" == *.ovh.net ]] && return 0 [[ "$domain" == "ns"*".ip-"*".eu" ]] && return 0 return 1 } is_ovh_hostname() { local domain="$1" [[ "$domain" =~ ^vps-[0-9a-f]*$ ]] && return 0 [[ "$domain" =~ ^vps[0-9]*$ ]] && return 0 return 1 } vps_connection_check() { local vps="$1" ip=$(resolve "$vps") || { echo "${DARKRED}no-resolve${NORMAL}"; return 1; } is-port-open "$ip" "22" /dev/null 2>&1 || { echo "${DARKRED}no-ssh-root-access${NORMAL}"; return 1; } } vps_check() { local vps="$1" vps_connection_check "$vps" /dev/null 2>&1 || { echo "${DARKRED}no-backup${NORMAL}"; return 1; } } vps:rsync() { local vps="$1" id="$2" src="$3" dst="$4" if [[ "$src" != *":"* ]]; then err "Third argument '$src' should be a remote (include the server name as prefix)." return 1 fi server=${src%%:*} src=${src#*:} cat </dev/null 2>&1 || apt-get install -y rsync > "$VPS_TMP_DIR/recover_key" EOF ssh_options+=(-i "$VPS_TMP_DIR/recover_key" -l rsync) if ! compose_yml_files=$(cat <&2 if [[ "$compose_yml_files" == *$'\n'* ]]; then err "Running containers are confusing, did not find only one mailcowdockerized project." return 1 fi if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then err "Running containers are confusing, they don't point to an existing docker-compose.yml." return 1 fi echo "Containers where launched from '$compose_yml_files'" >&2 COMPOSE_FILE="$compose_yml_files" ENV_FILE="${COMPOSE_FILE%/*}/.env" if ! echo "[ -e \"${ENV_FILE}\" ]" | ssh:run "root@$vps" -- bash ; then err "Running containers are confusing, docker-compose.yml has no '.env' next to it." return 1 fi echo "${WHITE}Bringing mailcowdockerized down${NORMAL}" echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" | ssh:run "root@$vps" -- bash stopped_containers=1 fi for vol in postfix rspamd redis crypt vmail{,-attachments}; do volume_name="mailcowdockerized_${vol}-vol-1" volume_dir="/var/lib/docker/volumes/${volume_name}/_data" if ! ssh:run "root@$server" -- "[ -d '$BACKUP_PATH/${id}${volume_dir}' ]"; then continue fi ## Create volumes if not existent if ! ssh:run "root@$vps" -- " [ -d '${volume_dir}' ] || docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9 [ -d '${volume_dir}' ] "; then err "Couldn't find nor create '${volume_dir}'." return 1 fi echo "${WHITE}Downloading of $volume_name${NORMAL}" vps:rsync "$vps" "$id" "$server":"${volume_dir}/" "${volume_dir}" || return 1 done ## Mailcow git base for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do if ! ssh:run "root@$server" -- "[ -d '$BACKUP_PATH/${id}${mailcow_dir}' ]"; then continue else ## this possibly change last value COMPOSE_FILE="$mailcow_dir/docker-compose.yml" ENV_FILE="$mailcow_dir/.env" echo "${WHITE}Download of $mailcow_dir${NORMAL}" vps:rsync "$vps" "$id" "$server":"${mailcow_dir}"/ "${mailcow_dir}" || return 1 break fi done ## Mysql database echo "${WHITE}Downloading last backup of mysql backups${NORMAL}" vps:rsync "$vps" "$id" "$server":"/var/backups/mysql/" "/var/backups/mysql" || return 1 if ! env_content=$(echo "cat '$ENV_FILE'" | ssh:run "root@$vps" -- bash); then err "Can't access env file: '$ENV_FILE'." return 1 fi root_password=$(printf "%s\n" "$env_content" | grep ^DBROOT= | cut -f 2 -d =) echo "${WHITE}Bringing mysql-mailcow up${NORMAL}" if ! container_id=$(cat < "$VPS_TMP_DIR/my.cnf" docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \ run -d \ -v "$VPS_TMP_DIR/my.cnf:/root/.my.cnf:ro" \ mysql-mailcow EOF ); then err "Failed to bring up mysql-mailcow" return 1 fi START="$SECONDS" while true; do echo " trying to connect..." >&2 cat </dev/null 2>&1 EOF if (($SECONDS - $START > 10)); then err "Failed to connect to mysql-mailcow." echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" | ssh:run "root@$vps" -- bash return 1 fi sleep 0.3 done DBUSER=$(printf "%s\n" "$env_content" | grep ^DBUSER= | cut -f 2 -d =) DBPASS=$(printf "%s\n" "$env_content" | grep ^DBPASS= | cut -f 2 -d =) echo "${WHITE}Uploading mysql dump${NORMAL}" cat <&2 echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" up -d" | ssh:run "root@$vps" -- bash fi info "Mailcow was ${GREEN}successfully${NORMAL} restored." } vps_backup_recover() { local vps="$1" admin server id rtype force vps_connection_check "$vps" &1); then err "Command 'vps install backup $server' failed." return 1 fi already_present= if e "$out" | grep "^II Entry for service .* is already present" >/dev/null 2>&1; then already_present=1 info "Backup entry is already present in 'compose.yml' of '$vps'" fi out="${out%$'\n'}" out="${out#*$'\n'}" key="${out%\'*}" key="${key##*\'}" if ! [[ "$key" =~ ^"ssh-rsa "[a-zA-Z0-9/+]+" "[a-zA-Z0-9._-]+"@"[a-zA-Z0-9._-]+$ ]]; then err "Unexpected output from 'vps install backup $server'. Can't find key." echo "$out" | prefix " ${GRAY}|$NORMAL " >&2 echo " Extracted key:" >&2 echo "$key" | prefix " ${GRAY}|$NORMAL " >&2 return 1 fi if [ "$type" == "compose" ] && [ -z "$already_present" ]; then ssh:run "root@$vps" -- compose --debug up || { err "Command 'compose --debug up' failed." return 1 } fi dest="$server" dest="${dest%/*}" ssh_options=() if [[ "$dest" == *":"* ]]; then port="${dest##*:}" dest="${dest%%:*}" ssh_options=(-p "$port") else port="" dest="${dest%%:*}" fi cmd=(ssh "${ssh_options[@]}" "$admin"@"$dest" ssh-key add "$key") echo "${WHITE}Launching:${NORMAL} ${cmd[@]}" "${cmd[@]}" || { err "Failed add key to backup server '$dest'." return 1 } echo "${WHITE}Launching backup${NORMAL} from '$vps'" ssh:run "root@$vps" -- vps backup || { err "First backup failed to run." return 1 } echo "Backup is ${GREEN}up and running${NORMAL}." } vps_udpate() { local vps="$1" vps_connection_check "$vps" || return 1 ssh:run "root@$vps" -- myc-update "$_0KM_TMP_DIR/code" for vps in "${new_vps[@]}"; do label=$(printf "%-${max_size}s" "$vps") ( { { "$fn" "$vps" < "$_0KM_TMP_DIR/code" } 3>&1 1>&2 2>&3 | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKRED\!$NORMAL /g" set_errlvl "${PIPESTATUS[0]}" } 3>&1 1>&2 2>&3 | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKGRAY\|$NORMAL /g" set_errlvl "${PIPESTATUS[0]}" ) & done wait } [ "$SOURCED" ] && return 0 ## ## Command line processing ## cmdline.spec.gnu cmdline.spec.reporting cmdline.spec.gnu vps-setup cmdline.spec::cmd:vps-setup:run() { : :posarg: HOST 'Target host to check/fix ssh-access' depends sshpass shyaml KEY_PATH="ssh-access.public-keys" local keys=$(config get-value -y "ssh-access.public-keys") || true if [ -z "$keys" ]; then err "No ssh publickeys configured in config file." echo " Looking for keys in ${WHITE}${KEY_PATH}${NORMAL}" \ "in config file." >&2 config:exists --message 2>&1 | prefix " " if [ "${PIPESTATUS[0]}" == "0" ]; then echo " Config file found in $(config:filename)" fi return 1 fi local tkey=$(e "$keys" | shyaml get-type) if [ "$tkey" != "sequence" ]; then err "Value type of ${WHITE}${KEY_PATH}${NORMAL} unexpected (is $tkey, expecting sequence)." echo " Check content of $(config:filename), and make sure to use a sequence." >&2 return 1 fi local IP NAME keys host_pass_connected if ! IP=$(resolve "$HOST"); then err "'$HOST' name unresolvable." exit 1 fi NAME="$HOST" if [ "$IP" != "$HOST" ]; then NAME="$HOST ($IP)" fi if ! is-port-open "$IP" "22"; then err "$NAME unreachable or port 22 closed." exit 1 fi debug "Host $IP's port 22 is open." if ! host_pass_connected=$(ssh:open-try \ {root,debian}@"$HOST"); then err "Could not connect to {root,debian}@$HOST with publickey nor password." exit 1 fi read-0a host password <<<"$host_pass_connected" sudo_if_necessary= if [ "$password" -o "${host%%@*}" != "root" ]; then if ! ssh:run "$host" -- sudo -n true >/dev/null 2>&1; then err "Couldn't do a password-less sudo from $host." echo " This is not yet supported." exit 1 else sudo_if_necessary=sudo fi fi Section Checking access while read-0 key; do prefix="${key%% *}" if [ "$prefix" != "ssh-rsa" ]; then err "Unsupported key:"$'\n'"$key" return 1 fi label="${key##* }" Elt "considering adding key ${DARKYELLOW}$label${NORMAL}" dest="/root/.ssh/authorized_keys" if ssh:run "$host" -- $sudo_if_necessary grep "\"$key\"" "$dest" >/dev/null 2>&1 /dev/null; then print_info added else echo Feedback failure return 1 fi Feedback success fi done < <(e "$keys" | shyaml get-values-0) Section Checking ovh hostname file Elt "Checking /etc/ovh-hostname" if ! ssh:run "$host" -- $sudo_if_necessary [ -e "/etc/ovh-hostname" ]; then print_info "creating" ssh:run "$host" -- $sudo_if_necessary cp /etc/hostname /etc/ovh-hostname ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname) Elt "Checking /etc/ovh-hostname: $ovhname" Feedback || return 1 else ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname) Elt "Checking /etc/ovh-hostname: $ovhname" print_info "already present" print_status noop Feed fi if ! is_ovh_domain_name "$HOST" && ! str_is_ipv4 "$HOST"; then Section Checking hostname Elt "Checking /etc/hostname..." if [ "$old" != "$HOST" ]; then old="$(ssh:run "$host" -- $sudo_if_necessary cat /etc/hostname)" Elt "Hostname is '$old'" if is_ovh_hostname "$old"; then Elt "Hostname '$old' --> '$HOST'" print_info "creating" echo "$HOST" | ssh:run "$host" -- $sudo_if_necessary tee /etc/hostname >/dev/null && ssh:run "$host" -- $sudo_if_necessary hostname "$HOST" Feedback || return 1 else print_info "not changing" print_status noop Feed fi else print_info "already set" print_status noop Feed fi else info "Not changing domain as '$HOST' doesn't seem to be final domain." fi } cmdline.spec.gnu vps-check cmdline.spec::cmd:vps-check:run() { : :posarg: [VPS...] 'Target host(s) to check' echo "" | vps_mux vps_check "${VPS[@]}" } cmdline.spec.gnu vps-install cmdline.spec::cmd:vps-install:run() { : } cmdline.spec.gnu backup cmdline.spec:vps-install:cmd:backup:run() { : :posarg: BACKUP_TARGET 'Backup target. (ie: myadmin@backup.domain.org:10023/256)' : :posarg: [VPS...] 'Target host(s) to check' if [ "${#VPS[@]}" == 0 ]; then warn "VPS list provided in command line is empty. Nothing will be done." return 0 fi if ! [[ "$BACKUP_TARGET" == *"@"* ]]; then err "Missing admin account identifier in backup target." echo " Have you forgottent to specify an account, ie 'myadmin@' ?)" return 1 fi admin=${BACKUP_TARGET%%@*} server=${BACKUP_TARGET#*@} p0 "$admin" "$server" | vps_mux vps_install_backup "${VPS[@]}" } cmdline.spec.gnu vps-backup cmdline.spec::cmd:vps-backup:run() { : } cmdline.spec.gnu ls cmdline.spec:vps-backup:cmd:ls:run() { : :posarg: BACKUP_ID 'Backup id. (ie: myadmin@backup.domain.org:10023)' if ! [[ "$BACKUP_ID" == *"@"* ]]; then err "Missing admin account identifier in backup id." echo " Have you forgottent to specify an admin account ?" \ "ie 'myadmin@#' ?)" return 1 fi id=${BACKUP_ID##*#} BACKUP_TARGET=${BACKUP_ID%#*} admin=${BACKUP_TARGET%%@*} server=${BACKUP_TARGET#*@} ## XXXvlab: in this first implementation we expect to have access ## to the server main ssh port 22, so we won't use the provided port. ssh_options=() if [[ "$server" == *":"* ]]; then ssh_options+=(-p "${server#*:}") server=${server%%:*} fi ssh "${ssh_options[@]}" "$admin"@"$server" ssh-key ls } cmdline.spec.gnu recover cmdline.spec:vps-backup:cmd:recover:run() { : :posarg: BACKUP_ID 'Backup id. (ie: myadmin@backup.domain.org:10023#mx.myvps.org)' : :posarg: [VPS...] 'Target host(s) to check' : :optval: --date,-D '"last", or label of version to recover. (Default: "last").' : :optfla: --force,-f 'Will allow you to bypass some checks.' if [ "${#VPS[@]}" == 0 ]; then warn "VPS list provided in command line is empty. Nothing will be done." return 0 fi if ! [[ "$BACKUP_ID" == *"@"* ]]; then err "Missing admin account identifier in backup id." echo " Have you forgottent to specify an admin account ?" \ "ie 'myadmin@#' ?)" return 1 fi if ! [[ "$BACKUP_ID" == *"@"*"#"* ]]; then err "Missing backup label identifier in backup id." echo " Have you forgottent to specify a backup label identifier ?" \ "ie 'myadmin@#' ?)" return 1 fi id=${BACKUP_ID##*#} BACKUP_TARGET=${BACKUP_ID%#*} admin=${BACKUP_TARGET%%@*} server=${BACKUP_TARGET#*@} ## XXXvlab: in this first implementation we expect to have access ## to the server main ssh port 22, so we won't use the provided port. # ssh_options=() if [[ "$server" == *":"* ]]; then # ssh_options+=(-p "${server#*:}") ssh_server=${server%%:*} fi BACKUP_PATH="/srv/datastore/data/rsync-backup-target/var/mirror" if ! rtype=$(echo " if [ -d '$BACKUP_PATH/$id/var/lib/docker/volumes/mailcowdockerized_crypt-vol-1' ]; then echo mailcow elif [ -d '$BACKUP_PATH/$id/compose.yml' ]; then echo compose fi true " | ssh:run "root@${ssh_server}" -- bash ); then err "Could not get backup type." return 1 fi if [ -z "$rtype" ]; then err "Unknown type of backup on '${server}#${id}'." return 1 fi p0 "$admin" "$server" "$id" "$rtype" "$opt_force" | vps_mux vps_backup_recover "${VPS[@]}" } cmdline.spec.gnu vps-update cmdline.spec::cmd:vps-update:run() { : :posarg: [VPS...] 'Target host to check' echo "" | vps_mux vps_update "${VPS[@]}" } cmdline.spec.gnu vps-mux cmdline.spec::cmd:vps-mux:run() { : :posarg: [VPS...] 'Target host(s) to check' cat | vps_mux vps_bash "${VPS[@]}" } cmdline.spec.gnu vps-space cmdline.spec::cmd:vps-space:run() { : :posarg: [VPS...] 'Target host(s) to check' echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 2-5 -d ' '" | vps_mux vps_bash "${VPS[@]}" } cmdline::parse "$@"