myc-manage/bin/vps

#!/bin/bash


. /etc/shlib

include common
include parse
include cmdline
include config
include cache
include fn
include docker


[[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true

version=0.1
desc='Install backup'
help=""


docker:running-container-projects() {
    :cache: scope=session

    docker ps --format '{{.Label "com.docker.compose.project"}}' | sort | uniq
}
decorator._mangle_fn docker:running-container-projects


ssh:mk-private-key() {
    local host="$1" service_name="$2"
    (
        settmpdir VPS_TMPDIR
        ssh-keygen -t rsa -N "" -f "$VPS_TMPDIR/rsync_rsa" -C "$service_name@$host" >/dev/null
        cat "$VPS_TMPDIR/rsync_rsa"
    )
}


mailcow:has-images-running() {
    local images
    images=$(docker ps --format '{{.Image}}' | sort | uniq)
    [[ $'\n'"$images" == *$'\n'"mailcow/"* ]]
}


mailcow:has-container-project-mentionning-mailcow() {
    local projects
    projects=$(docker:running-container-projects) || return 1
    [[ $'\n'"$projects"$'\n' == *mailcow* ]]
}


mailcow:has-running-containers() {
    mailcow:has-images-running ||
        mailcow:has-container-project-mentionning-mailcow
}


mailcow:get-root() {
    :cache: scope=session

    local dir

    for dir in {/opt{,/apps},/root}/mailcow-dockerized; do
        [ -d "$dir" ] || continue
        [ -r "$dir/mailcow.conf" ] || continue
        echo "$dir"
        return 0
    done
    return 1
}
decorator._mangle_fn mailcow:get-root


compose:get-compose-yml() {
    :cache: scope=session

    local path
    [ -e "/etc/compose/local.conf" ] && . "/etc/compose/local.conf"

    path=${DEFAULT_COMPOSE_FILE:-/etc/compose/compose.yml}

    [ -e "$path" ] || return 1
    echo "$path"
}
decorator._mangle_fn compose:get-compose-yml


compose:has-container-project-myc() {
    local projects
    projects=$(docker:running-container-projects) || return 1
    [[ $'\n'"$projects"$'\n' == *$'\n'"myc"$'\n'* ]]
}


type:is-mailcow() {
    mailcow:get-root >/dev/null ||
    mailcow:has-running-containers
}


type:is-compose() {
    compose:get-compose-yml >/dev/null &&
    compose:has-container-project-myc
}


vps:get-type() {

    :cache: scope=session

    local fn
    for fn in $(declare -F | cut -f 3 -d " " | egrep "^type:is-"); do
        "$fn" && {
            echo "${fn#type:is-}"
            return 0
           }
    done
    return 1
}
decorator._mangle_fn vps:get-type


mirror-dir:sources() {
    :cache: scope=session

    if ! shyaml get-values default.sources < /etc/mirror-dir/config.yml; then
        err "Couldn't query 'default.sources' in '/etc/mirror-dir/config.yml'."
        return 1
    fi
}
decorator._mangle_fn mirror-dir:sources


mirror-dir:check-add() {
    local elt="$1" sources
    sources=$(mirror-dir:sources) || return 1
    if [[ $'\n'"$sources"$'\n' ==  *$'\n'"$elt"$'\n'* ]]; then
        info "Volume $elt already in sources"
    else
        Elt "Adding directory $elt"
        sed -i "/sources:/a\  - \"${elt}\"" \
            /etc/mirror-dir/config.yml
        Feedback || return 1
    fi
}


mirror-dir:check-add-vol() {
    local elt="$1"
    mirror-dir:check-add "/var/lib/docker/volumes/*_${elt}-*/_data"
}

## The first colon is to prevent auto-export of function from shlib
: ; bash-bug-5() { { cat; } < <(e) >/dev/null; ! cat "$1"; } && bash-bug-5 <(e) 2>/dev/null &&
    export BASH_BUG_5=1 && unset -f bash_bug_5


wrap() {
    local label="$1" code="$2"
    shift 2
    export VERBOSE=1
    interpreter=/bin/bash
    if [ -n "$BASH_BUG_5" ]; then
        (
            settmpdir tmpdir
            fname=${label##*/}
            e "$code" > "$tmpdir/$fname" &&
                chmod +x "$tmpdir/$fname" &&
                Wrap -vsd "$label" -- "$interpreter" "$tmpdir/$fname" "$@"
        )
    else
        Wrap -vsd "$label" -- "$interpreter" <(e "$code") "$@"
    fi
}


ping_check() {
    #global ignore_ping_check
    local host="$1"

    ip=$(getent ahosts "$host" | egrep "^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+" |
             head -n 1 | cut -f 1 -d " ") || return 1

    my_ip=$(curl -s myip.kal.fr)
    if [ "$ip" != "$my_ip" ]; then
        if [ -n "$ignore_ping_check" ]; then
            warn "IP of '$host' ($ip) doesn't match mine ($my_ip). Ignoring due to ``--ignore-ping-check`` option."
        else
            err "IP of '$host' ($ip) doesn't match mine ($my_ip). Use ``--ignore-ping-check`` to ignore check."
            return 1
        fi
    fi
}


mailcow:install-backup() {

    local BACKUP_SERVER="$1" ignore_ping_check="$2" mailcow_root DOMAIN

    ## find installation
    mailcow_root=$(mailcow:get-root) || {
        err "Couldn't find a valid mailcow root directory."
        return 1
    }

    ## check ok

    DOMAIN=$(cat "$mailcow_root/.env" | grep ^MAILCOW_HOSTNAME= | cut -f 2 -d =) || {
        err "Couldn't find MAILCOW_HOSTNAME in file \"$mailcow_root/.env\"."
        return 1
    }

    ping_check "$DOMAIN" || return 1

    MYSQL_ROOT_PASSWORD=$(cat "$mailcow_root/.env" | grep ^DBROOT= | cut -f 2 -d =) || {
        err "Couldn't find DBROOT in file \"$mailcow_root/.env\"."
        return 1
    }

    MYSQL_CONTAINER=${MYSQL_CONTAINER:-mailcowdockerized_mysql-mailcow_1}
    container_id=$(docker ps -f name="$MYSQL_CONTAINER" --format "{{.ID}}")
    if [ -z "$container_id" ]; then
        err "Couldn't find docker container named '$MYSQL_CONTAINER'."
        return 1
    fi


    export MYSQL_ROOT_PASSWORD
    export MYSQL_CONTAINER
    export BACKUP_SERVER
    export DOMAIN

    wrap "Install rsync-backup on host" "
        cd /srv/charm-store/rsync-backup
        bash ./hooks/install.d/60-install.sh
" || return 1

    wrap "Mysql dump install" "
        cd /srv/charm-store/mariadb
        bash ./hooks/install.d/60-backup.sh
" || return 1

    ## Using https://github.com/mailcow/mailcow-dockerized/blob/master/helper-scripts/backup_and_restore.sh
    for elt in "vmail{,-attachments-vol}" crypt redis rspamd postfix; do
        mirror-dir:check-add-vol "$elt" || return 1
    done

    mirror-dir:check-add "$mailcow_root" || return 1
    mirror-dir:check-add "/var/backups/mysql" || return 1
    mirror-dir:check-add "/etc" || return 1


    dest="$BACKUP_SERVER"
    dest="${dest%/*}"
    ssh_options=()
    if [[ "$dest" == *":"* ]]; then
        port="${dest##*:}"
        dest="${dest%%:*}"
        ssh_options=(-p "$port")
    else
        port=""
        dest="${dest%%:*}"
    fi

    info "You can run this following command from an host having admin access to $dest:"
    echo "   (Or send it to a backup admin of $dest)" >&2
    echo "ssh ${ssh_options[@]} myadmin@$dest ssh-key add '$(cat /var/lib/rsync/.ssh/id_rsa.pub)'"
}

compose:has_domain() {
    local compose_file="$1" host="$2" name conf relation relation_value domain server_aliases

    while read-0 name conf ; do
	    name=$(e "$name" | shyaml get-value)
        if [[ "$name" =~ ^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+ ]]; then
	        [ "$host" == "$name" ] && return 0
	    fi

	    rel=$(e "$conf" | shyaml -y get-value relations 2>/dev/null) || continue
        for relation in web-proxy publish-dir; do
	        relation_value=$(e "$rel" | shyaml -y get-value "$relation" 2>/dev/null) || continue
	        while read-0 label conf_relation; do
		        domain=$(e "$conf_relation" | shyaml get-value "domain" 2>/dev/null) && {
	                [ "$host" == "$domain" ] && return 0
	            }
		        server_aliases=$(e "$conf_relation" | shyaml get-values "server-aliases" 2>/dev/null) && {
	                [[ $'\n'"$server_aliases" == *$'\n'"$host"$'\n'* ]] && return 0
         	    }
	        done < <(e "$relation_value" | shyaml -y key-values-0)
        done
    done < <(shyaml -y key-values-0 < "$compose_file")
    return 1
}


compose:install-backup() {

    local BACKUP_SERVER="$1" service_name="$2" compose_file="$3" ignore_ping_check="$4" ignore_domain_check="$5"

    ## XXXvlab: far from perfect as it mimics and depends internal
    ## logic of current default way to get a domain in compose-core
    host=$(hostname)

    if ! compose:has_domain "$compose_file" "$host"; then
        if [ -n "$ignore_domain_check" ]; then
            warn "domain of '$host' not found in compose file '$compose_file'. Ignoring due to ``--ignore-domain-check`` option."
        else
            err "domain of '$host' not found in compose file '$compose_file'. Use ``--ignore-domain-check`` to ignore check."
            return 1
        fi
    fi

    ping_check "$host" || return 1

    if [ -e "/root/.ssh/rsync_rsa" ]; then
        warn "deleting private key in /root/.ssh/rsync_rsa, has we are not using it anymore."
        rm -fv /root/.ssh/rsync_rsa
    fi
    if [ -e "/root/.ssh/rsync_rsa.pub" ]; then
        warn "deleting public key in /root/.ssh/rsync_rsa.pub, has we are not using it anymore."
        rm -fv /root/.ssh/rsync_rsa.pub
    fi

    if service_cfg=$(cat "$compose_file" |
                         shyaml get-value -y "$service_name" 2>/dev/null); then
        info "Entry for service ${DARKYELLOW}$service_name${NORMAL}" \
             "is already present in '$compose_file'."
        cfg=$(e "$service_cfg" | shyaml get-value -y options) || {
            err "No ${WHITE}options${NORMAL} in ${DARKYELLOW}$service_name${NORMAL}'s" \
                "entry in '$compose_file'."
            return 1
        }
        private_key=$(e "$cfg" | shyaml get-value private-key) || return 1
        target=$(e "$cfg" | shyaml get-value target) || return 1
        if [ "$target" != "$BACKUP_SERVER" ]; then
            err "Existing backup target '$target' is different" \
                "from specified '$BACKUP_SERVER'"
            return 1
        fi
    else
        private_key=$(ssh:mk-private-key "$host" "$service_name")

        cat <<EOF >> "$compose_file"

$service_name:
  options:
    ident: $host
    target: $BACKUP_SERVER
    private-key: |
$(e "$private_key" | sed -r 's/^/      /g')
EOF
    fi


    dest="$BACKUP_SERVER"
    dest="${dest%/*}"
    ssh_options=()
    if [[ "$dest" == *":"* ]]; then
        port="${dest##*:}"
        dest="${dest%%:*}"
        ssh_options=(-p "$port")
    else
        port=""
        dest="${dest%%:*}"
    fi

    info "You can run this following command from an host having admin access to $dest:"
    echo "   (Or send it to a backup admin of $dest)" >&2
    public_key=$(ssh-keygen -y -f <(e "$private_key"$'\n'))
    echo "ssh ${ssh_options[@]} myadmin@$dest ssh-key add '$public_key ${service_name}@$host'"
}


backup-action() {
    local action="$1"
    shift
    vps_type=$(vps:get-type) || {
        err "Failed to get type of installation."
        return 1
    }
    if ! fn.exists "${vps_type}:${action}"; then
        err "type '${vps_type}' has no ${vps_type}:${action} implemented yet."
        return 1
    fi
    "${vps_type}:${action}" "$@"
}


compose:get_default_backup_host_ident() {
    local service_name="$1"  ## Optional
    local compose_file service_cfg cfg target

    compose_file=$(compose:get-compose-yml)
    service_name="${service_name:-rsync-backup}"
    if ! service_cfg=$(cat "$compose_file" |
                           shyaml get-value -y "$service_name" 2>/dev/null); then
        err "No service named '$service_name' found in 'compose.yml'."
        return 1
    fi
    cfg=$(e "$service_cfg" | shyaml get-value -y options) || {
        err "No ${WHITE}options${NORMAL} in ${DARKYELLOW}$service_name${NORMAL}'s" \
            "entry in '$compose_file'."
        return 1
    }
    if ! target=$(e "$cfg" | shyaml get-value target); then
        err "No ${WHITE}options.target${NORMAL} in ${DARKYELLOW}$service_name${NORMAL}'s" \
            "entry in '$compose_file'."
    fi
    if ! target=$(e "$cfg" | shyaml get-value target); then
        err "No ${WHITE}options.target${NORMAL} in ${DARKYELLOW}$service_name${NORMAL}'s" \
            "entry in '$compose_file'."
    fi
    if ! ident=$(e "$cfg" | shyaml get-value ident); then
        err "No ${WHITE}options.ident${NORMAL} in ${DARKYELLOW}$service_name${NORMAL}'s" \
            "entry in '$compose_file'."
    fi
    echo "$target $ident"
}


mailcow:get_default_backup_host_ident() {
    local content cron_line ident found dest cmd_line
    if ! [ -e "/etc/cron.d/mirror-dir" ]; then
        err "No '/etc/cron.d/mirror-dir' found."
        return 1
    fi
    content=$(cat /etc/cron.d/mirror-dir) || {
        err "Can't read '/etc/cron.d/mirror-dir'."
        return 1
    }
    if ! cron_line=$(e "$content" | grep "mirror-dir backup"); then
        err "Can't find 'mirror-dir backup' line in  '/etc/cron.d/mirror-dir'."
        return 1
    fi

    cron_line=${cron_line%|*}
    cmd_line=(${cron_line#*root})

    found=
    dest=
    for arg in "${cmd_line[@]}"; do
        [ -n "$found" ] && {
            dest="$arg"
            break
        }
        [ "$arg" == "-d" ] && {
            found=1
        }
    done

    if ! [[ "$dest" =~ ^[\'\"a-zA-Z0-9:/.-]+$ ]]; then
        err "Can't find valid destination in 'mirror-dir backup' arguments from '/etc/cron.d/mirror-dir'."
        return 1
    fi

    if [[ "$dest" == \"*\" ]] || [[ "$dest" == \'*\' ]]; then
        ## unquoting, the eval should be safe because of previous check
        dest=$(eval e "$dest")
    fi
    if [ -z "$dest" ]; then
        err "Can't find destination in 'mirror-dir backup' arguments from '/etc/cron.d/mirror-dir'."
        return 1
    fi

    ## looking for ident

    found=
    ident=
    for arg in "${cmd_line[@]}"; do
        [ -n "$found" ] && {
            ident="$arg"
            break
        }
        [ "$arg" == "-h" ] && {
            found=1
        }
    done

    if ! [[ "$ident" =~ ^[\'\"a-zA-Z0-9.-]+$ ]]; then
        err "Can't find valid identifier in 'mirror-dir backup' arguments from '/etc/cron.d/mirror-dir'."
        return 1
    fi

    if [[ "$ident" == \"*\" ]] || [[ "$ident" == \'*\' ]]; then
        ## unquoting, the eval should be safe because of previous check
        ident=$(eval e "$ident")
    fi
    if [ -z "$ident" ]; then
        err "Can't find destination in 'mirror-dir backup' arguments from '/etc/cron.d/mirror-dir'."
        return 1
    fi

    echo "$dest $ident"
}


compose:get_cron_docker_cmd() {
    local cron_line cmd_line docker_cmd
    if ! cron_line=$(docker exec myc_cron_1 cat /etc/cron.d/rsync-backup | grep "\* \* \*"); then
        err "Can't find cron_line in cron container."
        echo "  Have you forgotten to run 'compose up' ?" >&2
        return 1
    fi

    cron_line=${cron_line%|*}
    cron_line=${cron_line%"2>&1"*}

    cmd_line="${cron_line#*root}"
    eval "args=($cmd_line)"

    ## should be last argument

    docker_cmd=$(echo ${args[@]: -1})

    if ! [[ "$docker_cmd" == "docker run --rm -e "* ]]; then
        echo "docker command found should start with 'docker run'." >&2
        echo "Here's command:" >&2
        echo "  $docker_cmd" >&2
        return 1
    fi

    e "$docker_cmd"
}


compose:recover-target() {
    local backup_host="$1" ident="$2" src="$3" dst="$4" service_name="${5:-rsync-backup}"

    docker_image="myc_${service_name}"
    if ! docker_has_image "$docker_image"; then
        compose build "${service_name}" || {
            err "Couldn't find nor build image for service '$service_name'."
            return 1
        }
    fi

    dst="${dst%/}"  ## remove final slash

    ssh_options=(-o StrictHostKeyChecking=no)
    if [[ "$backup_host" == *":"* ]]; then
        port="${backup_host##*:}"
        backup_host="${backup_host%%:*}"
        ssh_options+=(-p "$port")
    else
        port=""
        backup_host="${backup_host%%:*}"
    fi

    rsync_opts=(
        -e "ssh ${ssh_options[*]} -i /var/lib/rsync/.ssh/id_rsa -l rsync"
        -azvArH --delete --delete-excluded
        --partial --partial-dir .rsync-partial
        --numeric-ids
    )
    if [ "$DRY_RUN" ]; then
        rsync_opts+=("-n")
    fi
    cmd=(
    docker run --rm --entrypoint rsync \
           -v "/srv/datastore/config/${service_name}/var/lib/rsync":/var/lib/rsync \
           -v "${dst%/*}":/mnt/dest \
           "$docker_image" \
           "${rsync_opts[@]}" "$backup_host":"/var/mirror/$ident/$src" "/mnt/dest/${dst##*/}"
    )
    echo "${WHITE}Launching: ${NORMAL} ${cmd[@]}"
    "${cmd[@]}"
}


mailcow:recover-target() {
    local backup_host="$1" ident="$2" src="$3" dst="$4"

    dst="${dst%/}"  ## remove final slash

    ssh_options=(-o StrictHostKeyChecking=no)
    if [[ "$backup_host" == *":"* ]]; then
        port="${backup_host##*:}"
        backup_host="${backup_host%%:*}"
        ssh_options+=(-p "$port")
    else
        port=""
        backup_host="${backup_host%%:*}"
    fi

    rsync_opts=(
        -e "ssh ${ssh_options[*]} -i /var/lib/rsync/.ssh/id_rsa -l rsync"
        -azvArH --delete --delete-excluded
        --partial --partial-dir .rsync-partial
        --numeric-ids
    )
    if [ "$DRY_RUN" ]; then
        rsync_opts+=("-n")
    fi
    cmd=(
        rsync "${rsync_opts[@]}" "$backup_host":"/var/mirror/$ident/$src" "${dst}"
    )
    echo "${WHITE}Launching: ${NORMAL} ${cmd[@]}"
    "${cmd[@]}"
}



[ "$SOURCED" ] && return 0

##
## Command line processing
##


cmdline.spec.gnu
cmdline.spec.reporting

cmdline.spec.gnu install


cmdline.spec::cmd:install:run() {
    :
}


cmdline.spec.gnu get-type
cmdline.spec::cmd:get-type:run() {
    vps:get-type
}


cmdline.spec:install:cmd:backup:run() {

    : :posarg: BACKUP_SERVER        'Target backup server'

    : :optfla: --ignore-domain-check \
                                   "Allow to bypass the domain check in
                                    compose file (only used in compose
                                    installation)."
    : :optfla: --ignore-ping-check "Allow to bypass the ping check of host."

    local vps_type

    vps_type=$(vps:get-type) || {
        err "Failed to get type of installation."
        return 1
    }
    if ! fn.exists "${vps_type}:install-backup"; then
        err "type '${vps_type}' has no backup installation implemented yet."
        return 1
    fi

    opts=()

    [ "$opt_ignore_ping_check" ] &&
        opts+=("--ignore-ping-check")

    if [ "$vps_type" == "compose" ]; then
        [ "$opt_ignore_domain_check" ] &&
            opts+=("--ignore-domain-check")
    fi

    "cmdline.spec:install:cmd:${vps_type}-backup:run" "${opts[@]}" "$BACKUP_SERVER"
}


DEFAULT_BACKUP_SERVICE_NAME=rsync-backup
cmdline.spec.gnu compose-backup
cmdline.spec:install:cmd:compose-backup:run() {

    : :posarg: BACKUP_SERVER        'Target backup server'

    : :optval: --service-name,-s   "YAML service name in compose
                                    file to check for existence of key.
                                    Defaults to '$DEFAULT_BACKUP_SERVICE_NAME'"
    : :optval: --compose-file,-f   "Compose file location. Defaults to
                                    the value of '\$DEFAULT_COMPOSE_FILE'"

    : :optfla: --ignore-domain-check \
                                   "Allow to bypass the domain check in
                                    compose file."
    : :optfla: --ignore-ping-check "Allow to bypass the ping check of host."


    local service_name compose_file


    [ -e "/etc/compose/local.conf" ] && source /etc/compose/local.conf

    compose_file=${opt_compose_file:-$DEFAULT_COMPOSE_FILE}
    service_name=${opt_service_name:-$DEFAULT_BACKUP_SERVICE_NAME}

    if ! [ -e "$compose_file" ]; then
        err "Compose file not found in '$compose_file'."
        return 1
    fi

    compose:install-backup "$BACKUP_SERVER" "$service_name" "$compose_file" \
	    "$opt_ignore_ping_check" "$opt_ignore_domain_check"

}


cmdline.spec:install:cmd:mailcow-backup:run() {

    : :posarg: BACKUP_SERVER        'Target backup server'

    : :optfla: --ignore-ping-check "Allow to bypass the ping check of host."

    "mailcow:install-backup" "$BACKUP_SERVER" "$opt_ignore_ping_check"
}



cmdline.spec.gnu backup
cmdline.spec::cmd:backup:run() {

    local vps_type
    vps_type=$(vps:get-type) || {
        err "Failed to get type of installation."
        return 1
    }
    if ! fn.exists "cmdline.spec:backup:cmd:${vps_type}:run"; then
        err "type '${vps_type}' has no backup process implemented yet."
        return 1
    fi

    "cmdline.spec:backup:cmd:${vps_type}:run"
}


cmdline.spec:backup:cmd:mailcow:run() {

    local cmd_line cron_line cmd

    for f in  mysql-backup mirror-dir; do
        [ -e "/etc/cron.d/$f" ] || {
            err "Can't find '/etc/cron.d/$f'."
            echo "  Have you forgotten to run 'vps install backup BACKUP_HOST' ?" >&2
            return 1
        }

        if ! cron_line=$(cat "/etc/cron.d/$f" |
                         grep -v "^#" | grep "\* \* \*"); then
            err "Can't find cron_line in '/etc/cron.d/$f'." \
                "Have you modified it ?"
            return 1
        fi

        cron_line=${cron_line%|*}
        cmd_line=(${cron_line#*root})

        if [ "$f" == "mirror-dir" ]; then
            cmd=()
            for arg in "${cmd_line[@]}"; do
                [ "$arg" != "-q" ] && cmd+=("$arg")
            done
        else
            cmd=("${cmd_line[@]}")
        fi

        code="${cmd[*]}"
        echo "${WHITE}Launching:${NORMAL} ${code}"
        {
            {
                (
                    ## Some commands are using colors that are already
                    ## set by this current program and will trickle
                    ## down unwantedly
                    ansi_color no
                    eval "${code}"
                ) | sed -r "s/^/ ${GRAY}|${NORMAL} /g"
                set_errlvl "${PIPESTATUS[0]}"
            } 3>&1 1>&2 2>&3 | sed -r "s/^/ $DARKRED\!$NORMAL /g"
            set_errlvl "${PIPESTATUS[0]}"
        } 3>&1 1>&2 2>&3

        if [ "$?" != "0" ]; then
            err "Failed."
            return 1
        fi
    done
    info "Mysql backup and subsequent mirror-dir ${DARKGREEN}succeeded${NORMAL}."
}


set_errlvl() { return "${1:-1}"; }


cmdline.spec:backup:cmd:compose:run() {

    local cron_line args
    docker_cmd=$(compose:get_cron_docker_cmd) || return 1

    echo "${WHITE}Launching:${NORMAL} docker exec -i myc_cron_1 $docker_cmd"

    {
        {
            eval "docker exec -i myc_cron_1 $docker_cmd" | sed -r "s/^/ ${GRAY}|${NORMAL} /g"
            set_errlvl "${PIPESTATUS[0]}"
        } 3>&1 1>&2 2>&3 | sed -r "s/^/ $DARKRED\!$NORMAL /g"
        set_errlvl "${PIPESTATUS[0]}"
    } 3>&1 1>&2 2>&3

    if [ "$?" != "0" ]; then
        err "Failed."
        return 1
    fi
    info "mirror-dir ${DARKGREEN}succeeded${NORMAL}."

}


cmdline.spec.gnu recover-target
cmdline.spec::cmd:recover-target:run() {

    : :posarg: BACKUP_DIR           'Source directory on backup side'
    : :posarg: HOST_DIR             'Target directory on host side'

    : :optval: --backup-host,-B     "The backup host"

    : :optfla: --dry-run,-n         "Don't do anything, instead tell what it
                                     would do."


    ## if no backup host take the one by default
    backup_host="$opt_backup_host"
    if [ -z "$backup_host" ]; then
        backup_host_ident=$(backup-action get_default_backup_host_ident) || return 1
        read -r backup_host ident <<<"$backup_host_ident"
    fi

    if [[ "$BACKUP_DIR" == /* ]]; then
        err "BACKUP_DIR must be a relative path from the root of your backup."
        return 1
    fi

    REAL_HOST_DIR=$(realpath "$HOST_DIR") || {
        err "Can't find HOST_DIR '$HOST_DIR'."
        return 1
    }
    export DRY_RUN="${opt_dry_run}"
    backup-action recover-target "$backup_host" "$ident" "$BACKUP_DIR" "$REAL_HOST_DIR"
}


cmdline::parse "$@"