From 07e13bb31366ec18564d4fa44e07e064b2e42df0 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Fri, 14 May 2021 15:15:56 +0200 Subject: [PATCH] new: [0km] add support for file/path based partial recovery Signed-off-by: Valentin Lab --- README.org | 16 ++ bin/0km | 444 ++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 332 insertions(+), 128 deletions(-) diff --git a/README.org b/README.org index 5447798..0e7b365 100644 --- a/README.org +++ b/README.org @@ -908,6 +908,22 @@ pouvons re-déployer un backup existant sur un nouveau VPS. Attention, cela supprimera toute installation =mailcow= précédente (donnée comprise) sur le VPS de destination. +**** Récupération partielle + +***** Récupération d'un répertoire ou fichier précis + +Depuis un hôte d'adminstration, et via la command =0km=, nous pouvons +récupérer un répertoire ou un fichier précis d'un backup existant sur +un nouveau VPS. + +C'est la même commande que pour la récupération complète, on rajoute à +la source un chemin et possible aussi à la destination. + +#+begin_quote +0km vps-backup recover myadmin@core-06.0k.io:10023#mail.mybackupedvps.com:/mon/chemin mynewvps.com +0km vps-backup recover myadmin@core-06.0k.io:10023#mail.mybackupedvps.com:/mon/chemin mynewvps.com:/ma/dest +#+end_quote + ** Troubleshooting diff --git a/bin/0km b/bin/0km index 5b7f82e..07dfdd9 100755 --- a/bin/0km +++ b/bin/0km @@ -234,47 +234,29 @@ vps_check() { } -vps:rsync() { - rsync_options=() - while [[ "$1" == "-"* ]]; do - rsync_options+=("$1") - shift - done - local vps="$1" id="$2" src="$3" dst="$4" - if [[ "$src" != *":"* ]]; then - err "Third argument '$src' should be a remote (include the server name as prefix)." - return 1 - fi - server=${src%%:*} - src=${src#*:} - cat </dev/null 2>&1 || apt-get install -y rsync > "$VPS_TMP_DIR/recover_key" EOF - ssh_options+=(-i "$VPS_TMP_DIR/recover_key" -l rsync) + BACKUP_SSH_OPTIONS+=(-i "$VPS_TMP_DIR/recover_key" -l rsync) + BACKUP_VPS_TARGET="$vps" + BACKUP_IDENT="$id" + + echo "type -p rsync >/dev/null 2>&1 || apt-get install -y rsync /dev/null 2>&1 +rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \ + -nazvArH --numeric-ids \ + "${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "/tmp/dummy" +EOF + +} + +file:vps_backup_recover() { + local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6" + + backup:rsync "${path}" "${vps_path}" || return 1 + + if [[ "$path" == *"/" ]]; then + if [ "$path" == "$vps_path"/ ]; then + msg_target="Directory '$path'" + else + msg_target="Directory '$path' -> '$vps_path'" + fi + else + if [ "$path" == "$vps_path" ]; then + msg_target="File '$path'" + else + msg_target="File '$path' -> '$vps_path'" + fi + fi + + info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps." + +} + +mailcow:vps_backup_recover() { + local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6" + if ! compose_yml_files=$(cat <&2 COMPOSE_FILE="$compose_yml_files" @@ -323,58 +386,115 @@ EOF stopped_containers=1 fi + if [[ -n "$path" ]]; then - for vol in postfix rspamd redis crypt vmail{,-attachments}; do - volume_name="mailcowdockerized_${vol}-vol-1" - volume_dir="/var/lib/docker/volumes/${volume_name}/_data" - if ! vps:rsync -nd --no-r "$vps" "$id" "$server":"${volume_dir}/" "/tmp/dummy" >/dev/null 2>&1; then - warn "No '$volume_name' in backup. This might be expected." - continue - fi + ## + ## Additional intelligence to simple file copy + ## + + if [[ "$path" == "/var/lib/docker/volumes/mailcowdockerized_*-vol-1/_data"* ]]; then + volume_name=${path#/var/lib/docker/volumes/} + volume_name=${volume_name%%/*} + volume_dir=${path%%"$volume_name"*} - ## Create volumes if not existent - if ! ssh:run "root@$vps" -- " + ## Create volumes if not existent + if ! ssh:run "root@$vps" -- " [ -d '${volume_dir}' ] || docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9 [ -d '${volume_dir}' ] "; then - err "Couldn't find nor create '${volume_dir}'." - return 1 + err "Couldn't find nor create '${volume_dir}'." + return 1 + fi fi - echo "${WHITE}Downloading of $volume_name${NORMAL}" - vps:rsync "$vps" "$id" "$server":"${volume_dir}/" "${volume_dir}" || return 1 - done - + echo "${WHITE}Sync from backup ${path} to VPS ${vps_path}${NORMAL}" >&2 + backup:rsync "${path}" "${vps_path}" || return 1 - ## Mailcow git base - for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do - if ! ssh:run "root@$server" -- "[ -d '$BACKUP_PATH/${id}${mailcow_dir}' ]"; then - continue + if [[ "$path" == *"/" ]]; then + if [ "$path" == "$vps_path"/ ]; then + msg_target="Directory '$path'" + else + msg_target="Directory '$path' -> '$vps_path'" + fi else + if [ "$path" == "$vps_path" ]; then + msg_target="File '$path'" + else + msg_target="File '$path' -> '$vps_path'" + fi + fi + else + + for vol in postfix rspamd redis crypt vmail{,-attachments}; do + volume_name="mailcowdockerized_${vol}-vol-1" + volume_dir="/var/lib/docker/volumes/${volume_name}/_data" + if ! backup:path_exists "${volume_dir}/"; then + warn "No '$volume_name' in backup. This might be expected." + continue + fi + ## Create volumes if not existent + if ! ssh:run "root@$vps" -- " + [ -d '${volume_dir}' ] || + docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9 + [ -d '${volume_dir}' ] + "; then + err "Couldn't find nor create '${volume_dir}'." + return 1 + fi + + echo "${WHITE}Downloading of $volume_name${NORMAL}" + backup:rsync "${volume_dir}/" "${volume_dir}" || return 1 + done + + ## Mailcow git base + COMPOSE_FILE= + for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do + backup:path_exists "${mailcow_dir}/" || continue ## this possibly change last value COMPOSE_FILE="$mailcow_dir/docker-compose.yml" ENV_FILE="$mailcow_dir/.env" echo "${WHITE}Download of $mailcow_dir${NORMAL}" - vps:rsync "$vps" "$id" "$server":"${mailcow_dir}"/ "${mailcow_dir}" || return 1 + backup:rsync "${mailcow_dir}"/ "${mailcow_dir}" || return 1 break + done + + if [ -z "$COMPOSE_FILE" ]; then + err "Can't find mailcow base installation path in backup." + return 1 fi - done + ## Mysql database + echo "${WHITE}Downloading last backup of mysql backups${NORMAL}" + backup:rsync "/var/backups/mysql/" "/var/backups/mysql" || return 1 - ## Mysql database - echo "${WHITE}Downloading last backup of mysql backups${NORMAL}" - vps:rsync "$vps" "$id" "$server":"/var/backups/mysql/" "/var/backups/mysql" || return 1 + if ! env_content=$(echo "cat '$ENV_FILE'" | ssh:run "root@$vps" -- bash); then + err "Can't access env file: '$ENV_FILE'." + return 1 + fi - if ! env_content=$(echo "cat '$ENV_FILE'" | ssh:run "root@$vps" -- bash); then - err "Can't access env file: '$ENV_FILE'." - return 1 - fi + root_password=$(printf "%s\n" "$env_content" | grep ^DBROOT= | cut -f 2 -d =) + + echo "${WHITE}Bringing mysql-mailcow up${NORMAL}" + if ! image=$(cat < "$VPS_TMP_DIR/my.cnf" @@ -384,31 +504,35 @@ docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \ mysql-mailcow EOF - ); then - err "Failed to bring up mysql-mailcow" - return 1 - fi + ); then + err "Failed to bring up mysql-mailcow" + return 1 + fi - START="$SECONDS" - while true; do - echo " trying to connect..." >&2 - cat <&2 + cat </dev/null 2>&1 EOF - if (($SECONDS - $START > 10)); then - err "Failed to connect to mysql-mailcow." - echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" | - ssh:run "root@$vps" -- bash - return 1 - fi - sleep 0.3 - done + if (($SECONDS - $START > $timeout)); then + err "Failed to connect to mysql-mailcow." + echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" | + ssh:run "root@$vps" -- bash + return 1 + fi + sleep 0.4 + done - DBUSER=$(printf "%s\n" "$env_content" | grep ^DBUSER= | cut -f 2 -d =) - DBPASS=$(printf "%s\n" "$env_content" | grep ^DBPASS= | cut -f 2 -d =) + DBUSER=$(printf "%s\n" "$env_content" | grep ^DBUSER= | cut -f 2 -d =) + DBPASS=$(printf "%s\n" "$env_content" | grep ^DBPASS= | cut -f 2 -d =) - echo "${WHITE}Uploading mysql dump${NORMAL}" - cat <&2 echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" up -d" | ssh:run "root@$vps" -- bash fi - info "Mailcow was ${GREEN}successfully${NORMAL} restored." + info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps." } vps_backup_recover() { - local vps="$1" admin server id rtype force - vps_connection_check "$vps" &2 - echo " Are you sure '$id' backup identifier belongs to '$admin' admin on '$server' ?" >&2 + if ! content=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key ls 2>/dev/null); then + err "Access denied to '$admin@${server}'." return 1 fi - if [ -z "$rtype" ]; then - err "Unknown type of backup on '${server}#${id}'." + idents=$(echo "$content" | sed -r "s/"$'\e'"\[[0-9]+(;[0-9]+)*m//g" | cut -f 2 -d " ") + if ! [[ $'\n'"$idents"$'\n' == *$'\n'"$id"$'\n'* ]]; then + err "Given backup id '$id' not found in $admin@${server}'s idents." return 1 fi - p0 "$admin" "$server" "$id" "$rtype" "$opt_force" | - vps_mux vps_backup_recover "${VPS[@]}" + rtype=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key get-type "$id" ) && + info "Backup archive matches ${WHITE}${rtype}${NORMAL} type" + + p0 "$admin" "$server" "$id" "$path" "$rtype" "$opt_force" | + vps_backup_recover "${VPS_PATH}" }