new: [recover] add action to restore a full mailcow VPS

Signed-off-by: Valentin Lab <valentin.lab@kalysto.org>

README.org

@@ -864,6 +864,15 @@ odoo:
 
 ** Récupération de donnée
 
+*** Depuis le VPS backuppé
+
+Les VPS backuppés peuvent avoir besoin de récupérer les données
+archivées. Pour le moment, comme il n'y pas d'accès aux versions
+précédentes des backups, l'intérêt de cette fonctionnalité reste
+limité.
+
+**** Par répertoire
+
 Via la commande =vps=, l'action =recover-target= permet de recouvrir
 les données du service d'archivage (si celui-ci à été correctement
 installé avec les commandes spécifiée dans la [[*rsync-backup][section =rsync-backup=]]).
@@ -885,6 +894,21 @@ destination peut-être entièrement modifié : cette commande effacera et
 modifiera le contenu du répertoire de destination.
 
 
+*** Depuis un hôte d'administration
+
+**** Récupération d'un VPS complet
+
+Depuis un hôte d'adminstration, et via la command =0km=, nous
+pouvons re-déployer un backup existant sur un nouveau VPS.
+
+#+begin_quote
+0km vps-backup recover myadmin@core-06.0k.io:10023#mail.mybackupedvps.com mynewvps.com
+#+end_quote
+
+Attention, cela supprimera toute installation =mailcow= précédente
+(donnée comprise) sur le VPS de destination.
+
+
 ** Troubleshooting
 
 S'il semble qu'il y ait un soucis, tu peux visualiser le =docker-compose.yml= qui est

bin/0km

@@ -169,7 +169,7 @@ ssh:run() {
             ssh -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
                 -o ControlMaster=auto -o ControlPersist=900 \
                 -o "StrictHostKeyChecking=no" \
-                "$hostname" "${ssh_options[@]}" -- "${cmd[@]}"
+                 "${ssh_options[@]}" "$hostname" -- "${cmd[@]}"
         } 3>&1 1>&2 2>&3  ## | sed -r "s/^/$DARKCYAN$hostname$NORMAL $DARKRED\!$NORMAL /g"
         set_errlvl "${PIPESTATUS[0]}"
     } 3>&1 1>&2 2>&3
@@ -208,7 +208,7 @@ vps_connection_check() {
     is-port-open "$ip" "22" </dev/null ||
         { echo "${DARKRED}no-port-22-open${NORMAL}"; return 1; }
 
-    ssh:open -o ConnectTimeout=2 -o PreferredAuthentications=publickey \
+    ssh:open -o ConnectTimeout=10 -o PreferredAuthentications=publickey \
              "root@$vps" >/dev/null 2>&1 ||
         { echo "${DARKRED}no-ssh-root-access${NORMAL}"; return 1; }
 }
@@ -234,6 +234,239 @@ vps_check() {
 }
 
 
+vps:rsync() {
+    local vps="$1" id="$2" src="$3" dst="$4"
+    if [[ "$src" != *":"* ]]; then
+        err "Third argument '$src' should be a remote (include the server name as prefix)."
+        return 1
+    fi
+    server=${src%%:*}
+    src=${src#*:}
+    cat <<EOF | ssh:run "root@$vps" -- bash
+rsync -e "ssh ${ssh_options[*]}" \
+              -azvArH --delete --delete-excluded \
+              --partial --partial-dir .rsync-partial \
+              --numeric-ids \
+      "$server":/var/mirror/"${id}${src}" "${dst}"
+EOF
+
+}
+
+
+mailcow:vps_backup_recover() {
+    local admin="$1" server="$2" id="$3" vps="$4"
+
+    ## Request recovery key
+    ssh_options=()
+    if [[ "$server" == *":"* ]]; then
+        ssh_options+=(-p "${server#*:}" -o StrictHostKeyChecking=no)
+        server=${server%%:*}
+    fi
+    if ! private_key=$(ssh "${ssh_options[@]}" "$admin"@"$server" request-recovery-key "$id"); then
+        err "Couldn't request a recovery key for '$id' with account '$admin'."
+        return 1
+    fi
+
+    echo "type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null" |
+        ssh:run "root@$vps" -- bash
+
+    if ! VPS_TMP_DIR=$(echo "mktemp -d" | ssh:run "root@$vps" -- bash); then
+        err "Couldn't create a temporary directory on vps"
+        return 1
+    fi
+
+    cat <<EOF | ssh:run "root@$vps" -- bash || return 1
+    touch "$VPS_TMP_DIR/recover_key" &&
+    chmod go-rwx "$VPS_TMP_DIR/recover_key" &&
+    printf "%s\n" "$private_key" >> "$VPS_TMP_DIR/recover_key"
+EOF
+
+    ssh_options+=(-i "$VPS_TMP_DIR/recover_key" -l rsync)
+
+    if ! compose_yml_files=$(cat <<EOF | ssh:run "root@$vps" -- bash
+urn=com.docker.compose.project
+docker ps -f "label=\$urn=mailcowdockerized" \
+    --format='{{.Label "\$urn.working_dir"}}/{{.Label "\$urn.config_files"}}' |
+    uniq
+EOF
+         ); then
+        err "Couldn't get list of running projects"
+        return 1
+    fi
+
+    stopped_containers=
+    if [ -n "$compose_yml_files" ]; then
+        echo "Found running mailcowdockerized containers" >&2
+        if [[ "$compose_yml_files" == *$'\n'* ]]; then
+            err "Running containers are confusing, did not find only one mailcowdockerized project."
+            return 1
+        fi
+        if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
+            err "Running containers are confusing, they don't point to an existing docker-compose.yml."
+            return 1
+        fi
+        echo "Containers where launched from '$compose_yml_files'" >&2
+        COMPOSE_FILE="$compose_yml_files"
+        ENV_FILE="${COMPOSE_FILE%/*}/.env"
+        if ! echo "[ -e \"${ENV_FILE}\" ]" | ssh:run "root@$vps" -- bash ; then
+            err "Running containers are confusing, docker-compose.yml has no '.env' next to it."
+            return 1
+        fi
+        echo "${WHITE}Bringing mailcowdockerized down${NORMAL}"
+        echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
+            ssh:run "root@$vps" -- bash
+        stopped_containers=1
+    fi
+
+
+    for vol in postfix rspamd redis crypt vmail{,-attachments}; do
+        volume_name="mailcowdockerized_${vol}-vol-1"
+        volume_dir="/var/lib/docker/volumes/${volume_name}/_data"
+        if ! ssh:run "root@$server" -- "[ -d '$BACKUP_PATH/${id}${volume_dir}' ]"; then
+            continue
+        fi
+
+        ## Create volumes if not existent
+        if ! ssh:run "root@$vps" -- "
+                  [ -d '${volume_dir}' ] ||
+                      docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
+                  [ -d '${volume_dir}' ]
+              "; then
+            err "Couldn't find nor create '${volume_dir}'."
+            return 1
+        fi
+
+        echo "${WHITE}Downloading of $volume_name${NORMAL}"
+        vps:rsync "$vps" "$id" "$server":"${volume_dir}/" "${volume_dir}" || return 1
+    done
+
+
+    ## Mailcow git base
+    for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
+        if ! ssh:run "root@$server" -- "[ -d '$BACKUP_PATH/${id}${mailcow_dir}' ]"; then
+            continue
+        else
+            ## this possibly change last value
+            COMPOSE_FILE="$mailcow_dir/docker-compose.yml"
+            ENV_FILE="$mailcow_dir/.env"
+            echo "${WHITE}Download of $mailcow_dir${NORMAL}"
+            vps:rsync "$vps" "$id" "$server":"${mailcow_dir}"/ "${mailcow_dir}" || return 1
+            break
+        fi
+    done
+
+
+    ## Mysql database
+    echo "${WHITE}Downloading last backup of mysql backups${NORMAL}"
+    vps:rsync "$vps" "$id" "$server":"/var/backups/mysql/" "/var/backups/mysql" || return 1
+
+    if ! env_content=$(echo "cat '$ENV_FILE'" | ssh:run "root@$vps" -- bash); then
+        err "Can't access env file: '$ENV_FILE'."
+        return 1
+    fi
+
+    root_password=$(printf "%s\n" "$env_content" | grep ^DBROOT= | cut -f 2 -d =)
+
+    echo "${WHITE}Bringing mysql-mailcow up${NORMAL}"
+    if ! container_id=$(cat <<EOF | ssh:run "root@$vps" -- bash
+echo "[client]
+password=$root_password" > "$VPS_TMP_DIR/my.cnf"
+
+docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
+    run -d \
+    -v "$VPS_TMP_DIR/my.cnf:/root/.my.cnf:ro" \
+    mysql-mailcow
+
+EOF
+         ); then
+        err "Failed to bring up mysql-mailcow"
+        return 1
+    fi
+
+    START="$SECONDS"
+    while true; do
+        echo "  trying to connect..." >&2
+        cat <<EOF | ssh:run "root@$vps" -- bash && break
+    echo "SELECT 1;" | docker exec -i "$container_id" mysql >/dev/null 2>&1
+EOF
+        if (($SECONDS - $START > 10)); then
+            err "Failed to connect to mysql-mailcow."
+            echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
+                ssh:run "root@$vps" -- bash
+            return 1
+        fi
+        sleep 0.3
+    done
+
+    DBUSER=$(printf "%s\n" "$env_content" | grep ^DBUSER= | cut -f 2 -d =)
+    DBPASS=$(printf "%s\n" "$env_content" | grep ^DBPASS= | cut -f 2 -d =)
+
+    echo "${WHITE}Uploading mysql dump${NORMAL}"
+    cat <<EOF | ssh:run "root@$vps" -- bash
+
+echo "
+  DROP DATABASE IF EXISTS mailcow;
+  CREATE DATABASE mailcow;
+  GRANT ALL PRIVILEGES ON mailcow.* TO '$DBUSER'@'%' IDENTIFIED BY '$DBPASS';
+" | docker exec -i "$container_id" mysql
+
+zcat /var/backups/mysql/mailcow/*.gz | docker exec -i "$container_id" mysql mailcow
+
+EOF
+    if [ "$?" != 0 ]; then
+        err "Failed to load mysql dump."
+        echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
+            ssh:run "root@$vps" -- bash
+        return 1
+    fi
+
+    echo "${WHITE}Bringing mysql-mailcow down${NORMAL}"
+    echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
+        ssh:run "root@$vps" -- bash
+    ssh:run "root@$vps" -- "rm -rf '$VPS_TMP_DIR'"
+
+    if [ -n "$stopped_containers" ]; then
+        echo "${WHITE}Starting mailcow${NORMAL}" >&2
+        echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" up -d" |
+            ssh:run "root@$vps" -- bash
+    fi
+    info "Mailcow was ${GREEN}successfully${NORMAL} restored."
+
+}
+
+vps_backup_recover() {
+    local vps="$1" admin server id rtype force
+    vps_connection_check "$vps" </dev/null || return 1
+
+    read-0 admin server id rtype force
+
+    if ! type=$(ssh:run "root@$vps" -- vps get-type); then
+        err "Could not get type."
+        return 1
+    fi
+
+    if [ "$rtype" != "$type" ]; then
+        if [ -n "$force" ]; then
+            warn "Backup found is of ${rtype:-unknown} type, while vps is of $type type."
+        else
+            err "Backup found is of ${rtype:-unknown} type, while vps is of $type type. (use \`\`-f\`\` to force)"
+            return 1
+        fi
+    fi
+
+    case "$rtype" in
+        mailcow)
+            mailcow:vps_backup_recover "$admin" "$server" "$id" "$vps"
+            ;;
+        *)
+            err "Recover on $type type VPS is not yet implemented."
+            return 1
+            ;;
+    esac
+
+}
+
+
 vps_install_backup() {
     local vps="$1" admin server
     vps_connection_check "$vps" </dev/null || return 1
@@ -540,6 +773,109 @@ cmdline.spec:vps-install:cmd:backup:run() {
 
 
 
+cmdline.spec.gnu vps-backup
+cmdline.spec::cmd:vps-backup:run() {
+    :
+}
+
+
+cmdline.spec.gnu ls
+cmdline.spec:vps-backup:cmd:ls:run() {
+    : :posarg: BACKUP_ID       'Backup id.
+                                (ie: myadmin@backup.domain.org:10023)'
+
+    if ! [[ "$BACKUP_ID" == *"@"* ]]; then
+        err "Missing admin account identifier in backup id."
+        echo "  Have you forgottent to specify an admin account ?" \
+             "ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
+        return 1
+    fi
+
+    id=${BACKUP_ID##*#}
+    BACKUP_TARGET=${BACKUP_ID%#*}
+    admin=${BACKUP_TARGET%%@*}
+    server=${BACKUP_TARGET#*@}
+
+    ## XXXvlab: in this first implementation we expect to have access
+    ## to the server main ssh port 22, so we won't use the provided port.
+    ssh_options=()
+    if [[ "$server" == *":"* ]]; then
+        ssh_options+=(-p "${server#*:}")
+        server=${server%%:*}
+    fi
+
+    ssh "${ssh_options[@]}" "$admin"@"$server" ssh-key ls
+}
+
+
+cmdline.spec.gnu recover
+cmdline.spec:vps-backup:cmd:recover:run() {
+
+    : :posarg: BACKUP_ID       'Backup id.
+                                (ie: myadmin@backup.domain.org:10023#mx.myvps.org)'
+    : :posarg: [VPS...]        'Target host(s) to check'
+
+    : :optval: --date,-D       '"last", or label of version to recover. (Default: "last").'
+    : :optfla: --force,-f       'Will allow you to bypass some checks.'
+
+
+    if [ "${#VPS[@]}" == 0 ]; then
+        warn "VPS list provided in command line is empty. Nothing will be done."
+        return 0
+    fi
+
+    if ! [[ "$BACKUP_ID" == *"@"* ]]; then
+        err "Missing admin account identifier in backup id."
+        echo "  Have you forgottent to specify an admin account ?" \
+             "ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
+        return 1
+    fi
+    if ! [[ "$BACKUP_ID" == *"@"*"#"* ]]; then
+        err "Missing backup label identifier in backup id."
+        echo "  Have you forgottent to specify a backup label identifier ?" \
+             "ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
+        return 1
+    fi
+
+    id=${BACKUP_ID##*#}
+    BACKUP_TARGET=${BACKUP_ID%#*}
+    admin=${BACKUP_TARGET%%@*}
+    server=${BACKUP_TARGET#*@}
+
+    ## XXXvlab: in this first implementation we expect to have access
+    ## to the server main ssh port 22, so we won't use the provided port.
+    # ssh_options=()
+    if [[ "$server" == *":"* ]]; then
+        # ssh_options+=(-p "${server#*:}")
+        ssh_server=${server%%:*}
+    fi
+
+    BACKUP_PATH="/srv/datastore/data/rsync-backup-target/var/mirror"
+
+    if ! rtype=$(echo "
+                  if [ -d '$BACKUP_PATH/$id/var/lib/docker/volumes/mailcowdockerized_crypt-vol-1' ]; then
+                      echo mailcow
+                  elif [ -d '$BACKUP_PATH/$id/compose.yml' ]; then
+                      echo compose
+                  fi
+                  true
+                " | ssh:run "root@${ssh_server}" -- bash ); then
+        err "Could not get backup type."
+        return 1
+    fi
+
+    if [ -z "$rtype" ]; then
+        err "Unknown type of backup on '${server}#${id}'."
+        return 1
+    fi
+
+    p0 "$admin" "$server" "$id" "$rtype" "$opt_force" |
+        vps_mux vps_backup_recover "${VPS[@]}"
+}
+
+
+
+
 cmdline.spec.gnu vps-update
 cmdline.spec::cmd:vps-update:run() {