new: [0km] add support for file/path based partial recovery

Signed-off-by: Valentin Lab <valentin.lab@kalysto.org>

README.org

@@ -908,6 +908,22 @@ pouvons re-déployer un backup existant sur un nouveau VPS.
 Attention, cela supprimera toute installation =mailcow= précédente
 (donnée comprise) sur le VPS de destination.
 
+**** Récupération partielle
+
+***** Récupération d'un répertoire ou fichier précis
+
+Depuis un hôte d'adminstration, et via la command =0km=, nous pouvons
+récupérer un répertoire ou un fichier précis d'un backup existant sur
+un nouveau VPS.
+
+C'est la même commande que pour la récupération complète, on rajoute à
+la source un chemin et possible aussi à la destination.
+
+#+begin_quote
+0km vps-backup recover myadmin@core-06.0k.io:10023#mail.mybackupedvps.com:/mon/chemin mynewvps.com
+0km vps-backup recover myadmin@core-06.0k.io:10023#mail.mybackupedvps.com:/mon/chemin mynewvps.com:/ma/dest
+#+end_quote
+
 
 ** Troubleshooting
 

bin/0km

@@ -234,47 +234,29 @@ vps_check() {
 }
 
 
-vps:rsync() {
-    rsync_options=()
-    while [[ "$1" == "-"* ]]; do
-        rsync_options+=("$1")
-        shift
-    done
-    local vps="$1" id="$2" src="$3" dst="$4"
-    if [[ "$src" != *":"* ]]; then
-        err "Third argument '$src' should be a remote (include the server name as prefix)."
-        return 1
-    fi
-    server=${src%%:*}
-    src=${src#*:}
-    cat <<EOF | ssh:run "root@$vps" -- bash
-rsync -e "ssh ${ssh_options[*]}" \
-              -azvArH --delete --delete-excluded \
-              --partial --partial-dir .rsync-partial \
-              --numeric-ids ${rsync_options[*]} \
-      "$server":/var/mirror/"${id}${src}" "${dst}"
-EOF
 
-}
+backup:setup-rsync() {
+    local admin="$1" vps="$2" server="$3" id="$4"
 
+    [ -z "${BACKUP_SSH_SERVER}" ] || {
+        err "Unexpected error: '\$BACKUP_SSH_SERVER' is already set in '$FUNCNAME'."
+        return 1
+    }
 
-mailcow:vps_backup_recover() {
-    local admin="$1" server="$2" id="$3" vps="$4"
-
-    ## Request recovery key
-    ssh_options=()
+    BACKUP_SSH_OPTIONS=(-o StrictHostKeyChecking=no)
     if [[ "$server" == *":"* ]]; then
-        ssh_options+=(-p "${server#*:}" -o StrictHostKeyChecking=no)
-        server=${server%%:*}
+        BACKUP_SSH_OPTIONS+=(-p "${server#*:}")
+        BACKUP_SSH_SERVER=${server%%:*}
+    else
+        BACKUP_SSH_SERVER="$server"
     fi
-    if ! private_key=$(ssh "${ssh_options[@]}" "$admin"@"$server" request-recovery-key "$id"); then
+
+    if ! private_key=$(ssh "${BACKUP_SSH_OPTIONS[@]}" \
+                           "$admin"@"${BACKUP_SSH_SERVER}" request-recovery-key "$id"); then
         err "Couldn't request a recovery key for '$id' with account '$admin'."
         return 1
     fi
 
-    echo "type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null" |
-        ssh:run "root@$vps" -- bash
-
     if ! VPS_TMP_DIR=$(echo "mktemp -d" | ssh:run "root@$vps" -- bash); then
         err "Couldn't create a temporary directory on vps"
         return 1
@@ -286,12 +268,89 @@ mailcow:vps_backup_recover() {
     printf "%s\n" "$private_key" >> "$VPS_TMP_DIR/recover_key"
 EOF
 
-    ssh_options+=(-i "$VPS_TMP_DIR/recover_key" -l rsync)
+    BACKUP_SSH_OPTIONS+=(-i "$VPS_TMP_DIR/recover_key" -l rsync)
+    BACKUP_VPS_TARGET="$vps"
+    BACKUP_IDENT="$id"
+
+    echo "type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null" |
+        ssh:run "root@$vps" -- bash || return 1
+
+}
+
+
+backup:rsync() {
+    local ssh_options
+
+    [ -n "${BACKUP_SSH_SERVER}" ] || {
+        err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
+        return 1
+    }
+
+    rsync_options=()
+    while [[ "$1" == "-"* ]]; do
+        rsync_options+=("$1")
+        shift
+    done
+    local src="$1" dst="$2"
+
+    cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash
+rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
+              -azvArH --delete --delete-excluded \
+              --partial --partial-dir .rsync-partial \
+              --numeric-ids ${rsync_options[*]} \
+      "${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "${dst}"
+EOF
+
+}
+
+
+backup:path_exists() {
+    local src="$1"
+
+    [ -n "${BACKUP_SSH_SERVER}" ] || {
+        err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
+        return 1
+    }
+
+    cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash >/dev/null 2>&1
+rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
+              -nazvArH --numeric-ids \
+      "${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "/tmp/dummy"
+EOF
+
+}
+
+file:vps_backup_recover() {
+    local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"
+
+    backup:rsync "${path}" "${vps_path}" || return 1
+
+    if [[ "$path" == *"/" ]]; then
+        if [ "$path" == "$vps_path"/ ]; then
+            msg_target="Directory '$path'"
+        else
+            msg_target="Directory '$path' -> '$vps_path'"
+        fi
+    else
+        if [ "$path" == "$vps_path" ]; then
+            msg_target="File '$path'"
+        else
+            msg_target="File '$path' -> '$vps_path'"
+        fi
+    fi
+
+    info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."
+
+}
+
+mailcow:vps_backup_recover() {
+    local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"
+
 
     if ! compose_yml_files=$(cat <<EOF | ssh:run "root@$vps" -- bash
 urn=com.docker.compose.project
 docker ps -f "label=\$urn=mailcowdockerized" \
-    --format='{{.Label "\$urn.working_dir"}}/{{.Label "\$urn.config_files"}}' |
+    --format="{{.Label \"\$urn.working_dir\"}}/{{.Label \"\$urn.config_files\"}}" |
     uniq
 EOF
          ); then
@@ -306,10 +365,14 @@ EOF
             err "Running containers are confusing, did not find only one mailcowdockerized project."
             return 1
         fi
+        if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
+            ## For some reason, sometimes $urn.config_files holds an absolute path
+            compose_yml_files=/${compose_yml_files#*//}
             if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
                 err "Running containers are confusing, they don't point to an existing docker-compose.yml."
                 return 1
             fi
+        fi
         echo "Containers where launched from '$compose_yml_files'" >&2
         COMPOSE_FILE="$compose_yml_files"
         ENV_FILE="${COMPOSE_FILE%/*}/.env"
@@ -323,15 +386,53 @@ EOF
         stopped_containers=1
     fi
 
+    if [[ -n "$path" ]]; then
+
+        ##
+        ## Additional intelligence to simple file copy
+        ##
+
+        if [[ "$path" == "/var/lib/docker/volumes/mailcowdockerized_*-vol-1/_data"* ]]; then
+            volume_name=${path#/var/lib/docker/volumes/}
+            volume_name=${volume_name%%/*}
+            volume_dir=${path%%"$volume_name"*}
+
+            ## Create volumes if not existent
+            if ! ssh:run "root@$vps" -- "
+                  [ -d '${volume_dir}' ] ||
+                      docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
+                  [ -d '${volume_dir}' ]
+              "; then
+                err "Couldn't find nor create '${volume_dir}'."
+                return 1
+            fi
+        fi
+
+        echo "${WHITE}Sync from backup ${path} to VPS ${vps_path}${NORMAL}" >&2
+        backup:rsync "${path}" "${vps_path}" || return 1
+
+        if [[ "$path" == *"/" ]]; then
+            if [ "$path" == "$vps_path"/ ]; then
+                msg_target="Directory '$path'"
+            else
+                msg_target="Directory '$path' -> '$vps_path'"
+            fi
+        else
+            if [ "$path" == "$vps_path" ]; then
+                msg_target="File '$path'"
+            else
+                msg_target="File '$path' -> '$vps_path'"
+            fi
+        fi
+    else
 
         for vol in postfix rspamd redis crypt vmail{,-attachments}; do
             volume_name="mailcowdockerized_${vol}-vol-1"
             volume_dir="/var/lib/docker/volumes/${volume_name}/_data"
-        if ! vps:rsync -nd --no-r "$vps" "$id" "$server":"${volume_dir}/" "/tmp/dummy" >/dev/null 2>&1; then
+            if ! backup:path_exists "${volume_dir}/"; then
                 warn "No '$volume_name' in backup. This might be expected."
                 continue
             fi
-
             ## Create volumes if not existent
             if ! ssh:run "root@$vps" -- "
                   [ -d '${volume_dir}' ] ||
@@ -343,28 +444,29 @@ EOF
             fi
 
             echo "${WHITE}Downloading of $volume_name${NORMAL}"
-        vps:rsync "$vps" "$id" "$server":"${volume_dir}/" "${volume_dir}" || return 1
+            backup:rsync "${volume_dir}/" "${volume_dir}" || return 1
         done
 
-
         ## Mailcow git base
+        COMPOSE_FILE=
         for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
-        if ! ssh:run "root@$server" -- "[ -d '$BACKUP_PATH/${id}${mailcow_dir}' ]"; then
-            continue
-        else
+            backup:path_exists "${mailcow_dir}/" || continue
             ## this possibly change last value
             COMPOSE_FILE="$mailcow_dir/docker-compose.yml"
             ENV_FILE="$mailcow_dir/.env"
             echo "${WHITE}Download of $mailcow_dir${NORMAL}"
-            vps:rsync "$vps" "$id" "$server":"${mailcow_dir}"/ "${mailcow_dir}" || return 1
+            backup:rsync "${mailcow_dir}"/ "${mailcow_dir}" || return 1
             break
-        fi
         done
 
+        if [ -z "$COMPOSE_FILE" ]; then
+            err "Can't find mailcow base installation path in backup."
+            return 1
+        fi
 
         ## Mysql database
         echo "${WHITE}Downloading last backup of mysql backups${NORMAL}"
-    vps:rsync "$vps" "$id" "$server":"/var/backups/mysql/" "/var/backups/mysql" || return 1
+        backup:rsync "/var/backups/mysql/" "/var/backups/mysql" || return 1
 
         if ! env_content=$(echo "cat '$ENV_FILE'" | ssh:run "root@$vps" -- bash); then
             err "Can't access env file: '$ENV_FILE'."
@@ -374,6 +476,24 @@ EOF
         root_password=$(printf "%s\n" "$env_content" | grep ^DBROOT= | cut -f 2 -d =)
 
         echo "${WHITE}Bringing mysql-mailcow up${NORMAL}"
+        if ! image=$(cat <<EOF | ssh:run "root@$vps" -- bash
+shyaml get-value services.mysql-mailcow.image < "${COMPOSE_FILE}"
+EOF
+             ); then
+            err "Failed to get image name of service 'mysql-mailcow' in 'compose.yml'."
+            return 1
+        fi
+        if [ -z "$(ssh:run "root@$vps" -- docker images -q "$image")" ]; then
+            info "Image '$image' not available, pull it."
+            if ! ssh:run "root@$vps" -- \
+                 docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
+                 pull mysql-mailcow; then
+
+                err "Failed to pull image of service 'mysql-mailcow'."
+                return 1
+            fi
+        fi
+
         if ! container_id=$(cat <<EOF | ssh:run "root@$vps" -- bash
 echo "[client]
 password=$root_password" > "$VPS_TMP_DIR/my.cnf"
@@ -390,18 +510,22 @@ EOF
         fi
 
         START="$SECONDS"
+        retries=0
+        timeout=600
         while true; do
-        echo "  trying to connect..." >&2
+            ((retries++))
+            echo "  waiting for mysql db..." \
+                 "(retry $retries, $(($SECONDS - $START))s elapsed, timeout is ${timeout}s)" >&2
             cat <<EOF | ssh:run "root@$vps" -- bash && break
     echo "SELECT 1;" | docker exec -i "$container_id" mysql >/dev/null 2>&1
 EOF
-        if (($SECONDS - $START > 10)); then
+            if (($SECONDS - $START > $timeout)); then
                 err "Failed to connect to mysql-mailcow."
                 echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
                     ssh:run "root@$vps" -- bash
                 return 1
             fi
-        sleep 0.3
+            sleep 0.4
         done
 
         DBUSER=$(printf "%s\n" "$env_content" | grep ^DBUSER= | cut -f 2 -d =)
@@ -430,43 +554,100 @@ EOF
         echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
             ssh:run "root@$vps" -- bash
         ssh:run "root@$vps" -- "rm -rf '$VPS_TMP_DIR'"
+        msg_target="Full mailcow backup"
+    fi
 
     if [ -n "$stopped_containers" ]; then
         echo "${WHITE}Starting mailcow${NORMAL}" >&2
         echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" up -d" |
             ssh:run "root@$vps" -- bash
     fi
-    info "Mailcow was ${GREEN}successfully${NORMAL} restored."
+    info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."
 
 }
 
 vps_backup_recover() {
-    local vps="$1" admin server id rtype force
-    vps_connection_check "$vps" </dev/null || return 1
+    local vps="$1" admin server id path rtype force type
 
-    read-0 admin server id rtype force
+    read-0 admin server id path rtype force
 
-    if ! type=$(ssh:run "root@$vps" -- vps get-type); then
-        err "Could not get type."
-        return 1
+    if [[ "$vps" == *":"* ]]; then
+        vps_path=${vps#*:}
+        vps=${vps%%:*}
+    else
+        vps_path=
     fi
 
+    vps_connection_check "$vps" </dev/null || {
+        err "Failed to access '$vps'."
+        return 1
+    }
+
+    type=$(ssh:run "root@$vps" -- vps get-type) && {
+        info "VPS $vps seems to be of ${WHITE}$type${NORMAL} type"
+    }
+
+    if [ -z "$path" ]; then
+        if [ -n "$vps_path" ]; then
+            err "You can't provide a VPS with path as destination if you don't provide a path in backup source."
+            return 1
+        fi
+        info "No path provided in backup, so we assume you want ${WHITE}full recovery${NORMAL}."
         if [ "$rtype" != "$type" ]; then
             if [ -n "$force" ]; then
-            warn "Backup found is of ${rtype:-unknown} type, while vps is of $type type."
+                warn "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type."
             else
-            err "Backup found is of ${rtype:-unknown} type, while vps is of $type type. (use \`\`-f\`\` to force)"
+                err "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type. (use \`\`-f\`\` to force)"
                 return 1
             fi
         fi
+    else
+        if [ "$path" == "/" ]; then
+            if [ -z "$vps_path" ]; then
+                err "Recovery of '/' (full backup files) requires that you provide a vps path also."
+                return 1
+            fi
+            if [ "$vps_path" == "/" ]; then
+                err "Recovery of '/' (full backup files) requires that you provide" \
+                    "a vps path different from '/' also."
+                return 1
+            fi
+        fi
+    fi
+
+    ## Sets VPS and internal global variable to allow rsync to work
+    ## from vps to backup server.
+    backup:setup-rsync "$admin" "$vps" "$server" "$id" || return 1
 
-    case "$rtype" in
-        mailcow)
-            mailcow:vps_backup_recover "$admin" "$server" "$id" "$vps"
+    if [ -n "$path" ]; then
+        if ! backup:path_exists "${path}"; then
+            err "File or directory '$path' not found in backup."
+            return 1
+        fi
+        if [ -z "$vps_path" ]; then
+            if [[ "$path" != *"/" ]] && backup:path_exists "${path}"/ ; then
+                path="$path/"
+            fi
+            vps_path=${path%/}
+            vps_path=${vps_path:-/}
+        fi
+    fi
+
+
+    case "$rtype-$type" in
+        mailcow-*)
+            ## Supports having $path and $vps_path set or unset, with additional behavior
+            mailcow:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
             ;;
-        *)
-            err "Recover on $type type VPS is not yet implemented."
+        *-*)
+            if [ -n "$path" ]; then
+            ## For now, will require having $path and $vps_path set, no additional behaviors
+                file:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
+            else
+                err "Full recover of ${rtype:-unknown} backup type on" \
+                    "${type:-unknown} type VPS is not yet implemented."
                 return 1
+            fi
             ;;
     esac
 
@@ -818,18 +999,16 @@ cmdline.spec.gnu recover
 cmdline.spec:vps-backup:cmd:recover:run() {
 
     : :posarg: BACKUP_ID       'Backup id.
-                                (ie: myadmin@backup.domain.org:10023#mx.myvps.org)'
-    : :posarg: [VPS...]        'Target host(s) to check'
+                                (ie: myadmin@backup.domain.org:10023#mx.myvps.org
+                                     myadmin@ark-01.org#myid:/a/path)'
+    : :posarg: VPS_PATH        'Target host(s) to check.
+                                (ie: myvps.com
+                                     myvps.com:/a/path)'
 
     : :optval: --date,-D       '"last", or label of version to recover. (Default: "last").'
     : :optfla: --force,-f       'Will allow you to bypass some checks.'
 
 
-    if [ "${#VPS[@]}" == 0 ]; then
-        warn "VPS list provided in command line is empty. Nothing will be done."
-        return 0
-    fi
-
     if ! [[ "$BACKUP_ID" == *"@"* ]]; then
         err "Missing admin account identifier in backup id."
         echo "  Have you forgottent to specify an admin account ?" \
@@ -843,7 +1022,14 @@ cmdline.spec:vps-backup:cmd:recover:run() {
         return 1
     fi
 
-    id=${BACKUP_ID##*#}
+    id_path=${BACKUP_ID#*#}
+    if [[ "$id_path" == *":"* ]]; then
+        id=${id_path%%:*}
+        path=${id_path#*:}
+    else
+        id="$id_path"
+        path=
+    fi
     BACKUP_TARGET=${BACKUP_ID%#*}
     admin=${BACKUP_TARGET%%@*}
     server=${BACKUP_TARGET#*@}
@@ -856,20 +1042,22 @@ cmdline.spec:vps-backup:cmd:recover:run() {
 
     BACKUP_PATH="/srv/datastore/data/rsync-backup-target/var/mirror"
 
-    if ! rtype=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key get-type "$id" ); then
-        err "Could not get backup type."
-        echo "  Do you have admin access to $admin@$server ?" >&2
-        echo "  Are you sure '$id' backup identifier belongs to '$admin' admin on '$server' ?" >&2
+    if ! content=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key ls 2>/dev/null); then
+        err "Access denied to '$admin@${server}'."
         return 1
     fi
 
-    if [ -z "$rtype" ]; then
-        err "Unknown type of backup on '${server}#${id}'."
+    idents=$(echo "$content" | sed -r "s/"$'\e'"\[[0-9]+(;[0-9]+)*m//g" | cut -f 2 -d " ")
+    if ! [[ $'\n'"$idents"$'\n' == *$'\n'"$id"$'\n'* ]]; then
+        err "Given backup id '$id' not found in $admin@${server}'s idents."
         return 1
     fi
 
-    p0 "$admin" "$server" "$id" "$rtype" "$opt_force" |
-        vps_mux vps_backup_recover "${VPS[@]}"
+    rtype=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key get-type "$id" ) &&
+        info "Backup archive matches ${WHITE}${rtype}${NORMAL} type"
+
+    p0 "$admin" "$server" "$id" "$path" "$rtype" "$opt_force" |
+        vps_backup_recover "${VPS_PATH}"
 }