From c6f4ef4f92805c92e6db442c02d3d26bb88aacad Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Mon, 3 May 2021 11:24:29 +0200 Subject: [PATCH] new: [vps] add ``recover-target`` action to recover files/directory from backup Signed-off-by: Valentin Lab --- README.org | 23 +++++ bin/vps | 294 ++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 293 insertions(+), 24 deletions(-) diff --git a/README.org b/README.org index 846aa1a..376d84d 100644 --- a/README.org +++ b/README.org @@ -859,6 +859,29 @@ odoo: #+END_SRC +** Récupération de donnée + +Via la commande =vps=, l'action =recover-target= permet de recouvrir +les données du service d'archivage (si celui-ci à été correctement +installé avec les commandes spécifiée dans la [[*rsync-backup][section =rsync-backup=]]). + +Récupération d'un répertoire: + +#+begin_src sh +vps recover-target "cron/" /tmp/cron +#+end_src + +Cette commande va récupérer le contenu archivé dans "cron/" pour le mettre +sur l'hôte courant dans "/tmp/cron". + +Il est possible de spécifier l'option =--dry-run= (ou =-n=) pour ne +rien modifier et voir quels sont les actions qui seront menées. + +Attention à l'usage de cette commande, en effet le répertoire de +destination peut-être entièrement modifié : cette commande effacera et +modifiera le contenu du répertoire de destination. + + ** Troubleshooting S'il semble qu'il y ait un soucis, tu peux visualiser le =docker-compose.yml= qui est diff --git a/bin/vps b/bin/vps index 42994e1..734f7de 100755 --- a/bin/vps +++ b/bin/vps @@ -9,6 +9,7 @@ include cmdline include config include cache include fn +include docker [[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true @@ -106,6 +107,9 @@ type:is-compose() { vps:get-type() { + + :cache: scope=session + local fn for fn in $(declare -F | cut -f 3 -d " " | egrep "^type:is-"); do "$fn" && { @@ -115,6 +119,7 @@ vps:get-type() { done return 1 } +decorator._mangle_fn vps:get-type mirror-dir:sources() { @@ -307,7 +312,7 @@ compose:install-backup() { fi fi - ping_check "$DOMAIN" || return 1 + ping_check "$host" || return 1 if [ -e "/root/.ssh/rsync_rsa" ]; then warn "deleting private key in /root/.ssh/rsync_rsa, has we are not using it anymore." @@ -368,6 +373,235 @@ EOF } +backup-action() { + local action="$1" + shift + vps_type=$(vps:get-type) || { + err "Failed to get type of installation." + return 1 + } + if ! fn.exists "${vps_type}:${action}"; then + err "type '${vps_type}' has no ${vps_type}:${action} implemented yet." + return 1 + fi + "${vps_type}:${action}" "$@" +} + + +compose:get_default_backup_host_ident() { + local service_name="$1" ## Optional + local compose_file service_cfg cfg target + + compose_file=$(compose:get-compose-yml) + service_name="${service_name:-rsync-backup}" + if ! service_cfg=$(cat "$compose_file" | + shyaml get-value -y "$service_name" 2>/dev/null); then + err "No service named '$service_name' found in 'compose.yml'." + return 1 + fi + cfg=$(e "$service_cfg" | shyaml get-value -y options) || { + err "No ${WHITE}options${NORMAL} in ${DARKYELLOW}$service_name${NORMAL}'s" \ + "entry in '$compose_file'." + return 1 + } + if ! target=$(e "$cfg" | shyaml get-value target); then + err "No ${WHITE}options.target${NORMAL} in ${DARKYELLOW}$service_name${NORMAL}'s" \ + "entry in '$compose_file'." + fi + if ! target=$(e "$cfg" | shyaml get-value target); then + err "No ${WHITE}options.target${NORMAL} in ${DARKYELLOW}$service_name${NORMAL}'s" \ + "entry in '$compose_file'." + fi + if ! ident=$(e "$cfg" | shyaml get-value ident); then + err "No ${WHITE}options.ident${NORMAL} in ${DARKYELLOW}$service_name${NORMAL}'s" \ + "entry in '$compose_file'." + fi + echo "$target $ident" +} + + +mailcow:get_default_backup_host_ident() { + local content cron_line ident found dest cmd_line + if ! [ -e "/etc/cron.d/mirror-dir" ]; then + err "No '/etc/cron.d/mirror-dir' found." + return 1 + fi + content=$(cat /etc/cron.d/mirror-dir) || { + err "Can't read '/etc/cron.d/mirror-dir'." + return 1 + } + if ! cron_line=$(e "$content" | grep "mirror-dir backup"); then + err "Can't find 'mirror-dir backup' line in '/etc/cron.d/mirror-dir'." + return 1 + fi + + cron_line=${cron_line%|*} + cmd_line=(${cron_line#*root}) + + found= + dest= + for arg in "${cmd_line[@]}"; do + [ -n "$found" ] && { + dest="$arg" + break + } + [ "$arg" == "-d" ] && { + found=1 + } + done + + if ! [[ "$dest" =~ ^[\'\"a-zA-Z0-9:/.-]+$ ]]; then + err "Can't find valid destination in 'mirror-dir backup' arguments from '/etc/cron.d/mirror-dir'." + return 1 + fi + + if [[ "$dest" == \"*\" ]] || [[ "$dest" == \'*\' ]]; then + ## unquoting, the eval should be safe because of previous check + dest=$(eval e "$dest") + fi + if [ -z "$dest" ]; then + err "Can't find destination in 'mirror-dir backup' arguments from '/etc/cron.d/mirror-dir'." + return 1 + fi + + ## looking for ident + + found= + ident= + for arg in "${cmd_line[@]}"; do + [ -n "$found" ] && { + ident="$arg" + break + } + [ "$arg" == "-h" ] && { + found=1 + } + done + + if ! [[ "$ident" =~ ^[\'\"a-zA-Z0-9.-]+$ ]]; then + err "Can't find valid identifier in 'mirror-dir backup' arguments from '/etc/cron.d/mirror-dir'." + return 1 + fi + + if [[ "$ident" == \"*\" ]] || [[ "$ident" == \'*\' ]]; then + ## unquoting, the eval should be safe because of previous check + ident=$(eval e "$ident") + fi + if [ -z "$ident" ]; then + err "Can't find destination in 'mirror-dir backup' arguments from '/etc/cron.d/mirror-dir'." + return 1 + fi + + echo "$dest $ident" +} + + +compose:get_cron_docker_cmd() { + local cron_line cmd_line docker_cmd + if ! cron_line=$(docker exec myc_cron_1 cat /etc/cron.d/rsync-backup | grep "\* \* \*"); then + err "Can't find cron_line in cron container." + echo " Have you forgotten to run 'compose up' ?" >&2 + return 1 + fi + + cron_line=${cron_line%|*} + cron_line=${cron_line%"2>&1"*} + + cmd_line="${cron_line#*root}" + eval "args=($cmd_line)" + + ## should be last argument + + docker_cmd=$(echo ${args[@]: -1}) + + if ! [[ "$docker_cmd" == "docker run --rm -e "* ]]; then + echo "docker command found should start with 'docker run'." >&2 + echo "Here's command:" >&2 + echo " $docker_cmd" >&2 + return 1 + fi + + e "$docker_cmd" +} + + +compose:recover-target() { + local backup_host="$1" ident="$2" src="$3" dst="$4" service_name="${5:-rsync-backup}" + + docker_image="myc_${service_name}" + if ! docker_has_image "$docker_image"; then + compose build "${service_name}" || { + err "Couldn't find nor build image for service '$service_name'." + return 1 + } + fi + + dst="${dst%/}" ## remove final slash + + ssh_options=(-o StrictHostKeyChecking=no) + if [[ "$backup_host" == *":"* ]]; then + port="${backup_host##*:}" + backup_host="${backup_host%%:*}" + ssh_options+=(-p "$port") + else + port="" + backup_host="${backup_host%%:*}" + fi + + rsync_opts=( + -e "ssh ${ssh_options[*]} -i /var/lib/rsync/.ssh/id_rsa -l rsync" + -azvArH --delete --delete-excluded + --partial --partial-dir .rsync-partial + --numeric-ids + ) + if [ "$DRY_RUN" ]; then + rsync_opts+=("-n") + fi + cmd=( + docker run --rm --entrypoint rsync \ + -v "/srv/datastore/config/${service_name}/var/lib/rsync":/var/lib/rsync \ + -v "${dst%/*}":/mnt/dest \ + "$docker_image" \ + "${rsync_opts[@]}" "$backup_host":"/var/mirror/$ident/$src" "/mnt/dest/${dst##*/}" + ) + echo "${WHITE}Launching: ${NORMAL} ${cmd[@]}" + "${cmd[@]}" +} + + +mailcow:recover-target() { + local backup_host="$1" ident="$2" src="$3" dst="$4" + + dst="${dst%/}" ## remove final slash + + ssh_options=(-o StrictHostKeyChecking=no) + if [[ "$backup_host" == *":"* ]]; then + port="${backup_host##*:}" + backup_host="${backup_host%%:*}" + ssh_options+=(-p "$port") + else + port="" + backup_host="${backup_host%%:*}" + fi + + rsync_opts=( + -e "ssh ${ssh_options[*]} -i /var/lib/rsync/.ssh/id_rsa -l rsync" + -azvArH --delete --delete-excluded + --partial --partial-dir .rsync-partial + --numeric-ids + ) + if [ "$DRY_RUN" ]; then + rsync_opts+=("-n") + fi + cmd=( + rsync "${rsync_opts[@]}" "$backup_host":"/var/mirror/$ident/$src" "${dst}" + ) + echo "${WHITE}Launching: ${NORMAL} ${cmd[@]}" + "${cmd[@]}" +} + + + [ "$SOURCED" ] && return 0 ## @@ -553,29 +787,7 @@ set_errlvl() { return "${1:-1}"; } cmdline.spec:backup:cmd:compose:run() { local cron_line args - if ! cron_line=$(docker exec myc_cron_1 cat /etc/cron.d/rsync-backup | grep "\* \* \*"); then - err "Can't find cron_line in cron container." - echo " Have you forgotten to run 'compose up' ?" >&2 - exit 1 - fi - - cron_line=${cron_line%|*} - cron_line=${cron_line%"2>&1"*} - - cmd_line="${cron_line#*root}" - - eval "args=($cmd_line)" - - ## should be last argument - - docker_cmd=$(echo ${args[@]: -1}) - - if ! [[ "$docker_cmd" == "docker run --rm -e "* ]]; then - echo "docker command found should start with 'docker run'." >&2 - echo "Here's command:" >&2 - echo " $docker_cmd" >&2 - exit 1 - fi + docker_cmd=$(compose:get_cron_docker_cmd) || return 1 echo "${WHITE}Launching:${NORMAL} docker exec -i myc_cron_1 $docker_cmd" @@ -595,4 +807,38 @@ cmdline.spec:backup:cmd:compose:run() { } + +cmdline.spec.gnu recover-target +cmdline.spec::cmd:recover-target:run() { + + : :posarg: BACKUP_DIR 'Source directory on backup side' + : :posarg: HOST_DIR 'Target directory on host side' + + : :optval: --backup-host,-B "The backup host" + + : :optfla: --dry-run,-n "Don't do anything, instead tell what it + would do." + + + ## if no backup host take the one by default + backup_host="$opt_backup_host" + if [ -z "$backup_host" ]; then + backup_host_ident=$(backup-action get_default_backup_host_ident) || return 1 + read -r backup_host ident <<<"$backup_host_ident" + fi + + if [[ "$BACKUP_DIR" == /* ]]; then + err "BACKUP_DIR must be a relative path from the root of your backup." + return 1 + fi + + REAL_HOST_DIR=$(realpath "$HOST_DIR") || { + err "Can't find HOST_DIR '$HOST_DIR'." + return 1 + } + export DRY_RUN="${opt_dry_run}" + backup-action recover-target "$backup_host" "$ident" "$BACKUP_DIR" "$REAL_HOST_DIR" +} + + cmdline::parse "$@"