Myceliandre
/
myc-manage
3
2
Fork 1
Code Issues 1 Pull Requests 2 Releases Wiki Activity
Browse Source

new: [0km] add support for file/path based partial recovery 

Signed-off-by: Valentin Lab <valentin.lab@kalysto.org>
rc1
Valentin Lab 3 years ago
parent
9668d9d326
commit
07e13bb313
2 changed files with 332 additions and 128 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 16
      README.org
  2. 444
      bin/0km

16
README.org
View File

@ -908,6 +908,22 @@ pouvons re-déployer un backup existant sur un nouveau VPS.
Attention, cela supprimera toute installation =mailcow= précédente
(donnée comprise) sur le VPS de destination.
**** Récupération partielle
***** Récupération d'un répertoire ou fichier précis
Depuis un hôte d'adminstration, et via la command =0km=, nous pouvons
récupérer un répertoire ou un fichier précis d'un backup existant sur
un nouveau VPS.
C'est la même commande que pour la récupération complète, on rajoute à
la source un chemin et possible aussi à la destination.
#+begin_quote
0km vps-backup recover myadmin@core-06.0k.io:10023#mail.mybackupedvps.com:/mon/chemin mynewvps.com
0km vps-backup recover myadmin@core-06.0k.io:10023#mail.mybackupedvps.com:/mon/chemin mynewvps.com:/ma/dest
#+end_quote
** Troubleshooting

444
bin/0km
View File

@ -234,47 +234,29 @@ vps_check() {
}
vps:rsync() {
rsync_options=()
while [[ "$1" == "-"* ]]; do
rsync_options+=("$1")
shift
done
local vps="$1" id="$2" src="$3" dst="$4"
if [[ "$src" != *":"* ]]; then
err "Third argument '$src' should be a remote (include the server name as prefix)."
return 1
fi
server=${src%%:*}
src=${src#*:}
cat <<EOF | ssh:run "root@$vps" -- bash
rsync -e "ssh ${ssh_options[*]}" \
-azvArH --delete --delete-excluded \
--partial --partial-dir .rsync-partial \
--numeric-ids ${rsync_options[*]} \
"$server":/var/mirror/"${id}${src}" "${dst}"
EOF
}
backup:setup-rsync() {
local admin="$1" vps="$2" server="$3" id="$4"
mailcow:vps_backup_recover() {
local admin="$1" server="$2" id="$3" vps="$4"
[ -z "${BACKUP_SSH_SERVER}" ] || {
err "Unexpected error: '\$BACKUP_SSH_SERVER' is already set in '$FUNCNAME'."
return 1
}
## Request recovery key
ssh_options=()
BACKUP_SSH_OPTIONS=(-o StrictHostKeyChecking=no)
if [[ "$server" == *":"* ]]; then
ssh_options+=(-p "${server#*:}" -o StrictHostKeyChecking=no)
server=${server%%:*}
BACKUP_SSH_OPTIONS+=(-p "${server#*:}")
BACKUP_SSH_SERVER=${server%%:*}
else
BACKUP_SSH_SERVER="$server"
fi
if ! private_key=$(ssh "${ssh_options[@]}" "$admin"@"$server" request-recovery-key "$id"); then
if ! private_key=$(ssh "${BACKUP_SSH_OPTIONS[@]}" \
"$admin"@"${BACKUP_SSH_SERVER}" request-recovery-key "$id"); then
err "Couldn't request a recovery key for '$id' with account '$admin'."
return 1
fi
echo "type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null" |
ssh:run "root@$vps" -- bash
if ! VPS_TMP_DIR=$(echo "mktemp -d" | ssh:run "root@$vps" -- bash); then
err "Couldn't create a temporary directory on vps"
return 1
@ -286,12 +268,89 @@ mailcow:vps_backup_recover() {
printf "%s\n" "$private_key" >> "$VPS_TMP_DIR/recover_key"
EOF
ssh_options+=(-i "$VPS_TMP_DIR/recover_key" -l rsync)
BACKUP_SSH_OPTIONS+=(-i "$VPS_TMP_DIR/recover_key" -l rsync)
BACKUP_VPS_TARGET="$vps"
BACKUP_IDENT="$id"
echo "type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null" |
ssh:run "root@$vps" -- bash || return 1
}
backup:rsync() {
local ssh_options
[ -n "${BACKUP_SSH_SERVER}" ] || {
err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
return 1
}
rsync_options=()
while [[ "$1" == "-"* ]]; do
rsync_options+=("$1")
shift
done
local src="$1" dst="$2"
cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash
rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
-azvArH --delete --delete-excluded \
--partial --partial-dir .rsync-partial \
--numeric-ids ${rsync_options[*]} \
"${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "${dst}"
EOF
}
backup:path_exists() {
local src="$1"
[ -n "${BACKUP_SSH_SERVER}" ] || {
err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
return 1
}
cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash >/dev/null 2>&1
rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
-nazvArH --numeric-ids \
"${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "/tmp/dummy"
EOF
}
file:vps_backup_recover() {
local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"
backup:rsync "${path}" "${vps_path}" || return 1
if [[ "$path" == *"/" ]]; then
if [ "$path" == "$vps_path"/ ]; then
msg_target="Directory '$path'"
else
msg_target="Directory '$path' -> '$vps_path'"
fi
else
if [ "$path" == "$vps_path" ]; then
msg_target="File '$path'"
else
msg_target="File '$path' -> '$vps_path'"
fi
fi
info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."
}
mailcow:vps_backup_recover() {
local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"
if ! compose_yml_files=$(cat <<EOF | ssh:run "root@$vps" -- bash
urn=com.docker.compose.project
docker ps -f "label=\$urn=mailcowdockerized" \
--format='{{.Label "\$urn.working_dir"}}/{{.Label "\$urn.config_files"}}' |
--format="{{.Label \"\$urn.working_dir\"}}/{{.Label \"\$urn.config_files\"}}" |
uniq
EOF
); then
@ -307,8 +366,12 @@ EOF
return 1
fi
if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
err "Running containers are confusing, they don't point to an existing docker-compose.yml."
return 1
## For some reason, sometimes $urn.config_files holds an absolute path
compose_yml_files=/${compose_yml_files#*//}
if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
err "Running containers are confusing, they don't point to an existing docker-compose.yml."
return 1
fi
fi
echo "Containers where launched from '$compose_yml_files'" >&2
COMPOSE_FILE="$compose_yml_files"
@ -323,58 +386,115 @@ EOF
stopped_containers=1
fi
if [[ -n "$path" ]]; then
for vol in postfix rspamd redis crypt vmail{,-attachments}; do
volume_name="mailcowdockerized_${vol}-vol-1"
volume_dir="/var/lib/docker/volumes/${volume_name}/_data"
if ! vps:rsync -nd --no-r "$vps" "$id" "$server":"${volume_dir}/" "/tmp/dummy" >/dev/null 2>&1; then
warn "No '$volume_name' in backup. This might be expected."
continue
fi
##
## Additional intelligence to simple file copy
##
if [[ "$path" == "/var/lib/docker/volumes/mailcowdockerized_*-vol-1/_data"* ]]; then
volume_name=${path#/var/lib/docker/volumes/}
volume_name=${volume_name%%/*}
volume_dir=${path%%"$volume_name"*}
## Create volumes if not existent
if ! ssh:run "root@$vps" -- "
## Create volumes if not existent
if ! ssh:run "root@$vps" -- "
[ -d '${volume_dir}' ] ||
docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
[ -d '${volume_dir}' ]
"; then
err "Couldn't find nor create '${volume_dir}'."
return 1
err "Couldn't find nor create '${volume_dir}'."
return 1
fi
fi
echo "${WHITE}Downloading of $volume_name${NORMAL}"
vps:rsync "$vps" "$id" "$server":"${volume_dir}/" "${volume_dir}" || return 1
done
echo "${WHITE}Sync from backup ${path} to VPS ${vps_path}${NORMAL}" >&2
backup:rsync "${path}" "${vps_path}" || return 1
## Mailcow git base
for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
if ! ssh:run "root@$server" -- "[ -d '$BACKUP_PATH/${id}${mailcow_dir}' ]"; then
continue
if [[ "$path" == *"/" ]]; then
if [ "$path" == "$vps_path"/ ]; then
msg_target="Directory '$path'"
else
msg_target="Directory '$path' -> '$vps_path'"
fi
else
if [ "$path" == "$vps_path" ]; then
msg_target="File '$path'"
else
msg_target="File '$path' -> '$vps_path'"
fi
fi
else
for vol in postfix rspamd redis crypt vmail{,-attachments}; do
volume_name="mailcowdockerized_${vol}-vol-1"
volume_dir="/var/lib/docker/volumes/${volume_name}/_data"
if ! backup:path_exists "${volume_dir}/"; then
warn "No '$volume_name' in backup. This might be expected."
continue
fi
## Create volumes if not existent
if ! ssh:run "root@$vps" -- "
[ -d '${volume_dir}' ] ||
docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
[ -d '${volume_dir}' ]
"; then
err "Couldn't find nor create '${volume_dir}'."
return 1
fi
echo "${WHITE}Downloading of $volume_name${NORMAL}"
backup:rsync "${volume_dir}/" "${volume_dir}" || return 1
done
## Mailcow git base
COMPOSE_FILE=
for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
backup:path_exists "${mailcow_dir}/" || continue
## this possibly change last value
COMPOSE_FILE="$mailcow_dir/docker-compose.yml"
ENV_FILE="$mailcow_dir/.env"
echo "${WHITE}Download of $mailcow_dir${NORMAL}"
vps:rsync "$vps" "$id" "$server":"${mailcow_dir}"/ "${mailcow_dir}" || return 1
backup:rsync "${mailcow_dir}"/ "${mailcow_dir}" || return 1
break
done
if [ -z "$COMPOSE_FILE" ]; then
err "Can't find mailcow base installation path in backup."
return 1
fi
done
## Mysql database
echo "${WHITE}Downloading last backup of mysql backups${NORMAL}"
backup:rsync "/var/backups/mysql/" "/var/backups/mysql" || return 1
## Mysql database
echo "${WHITE}Downloading last backup of mysql backups${NORMAL}"
vps:rsync "$vps" "$id" "$server":"/var/backups/mysql/" "/var/backups/mysql" || return 1
if ! env_content=$(echo "cat '$ENV_FILE'" | ssh:run "root@$vps" -- bash); then
err "Can't access env file: '$ENV_FILE'."
return 1
fi
if ! env_content=$(echo "cat '$ENV_FILE'" | ssh:run "root@$vps" -- bash); then
err "Can't access env file: '$ENV_FILE'."
return 1
fi
root_password=$(printf "%s\n" "$env_content" | grep ^DBROOT= | cut -f 2 -d =)
echo "${WHITE}Bringing mysql-mailcow up${NORMAL}"
if ! image=$(cat <<EOF | ssh:run "root@$vps" -- bash
shyaml get-value services.mysql-mailcow.image < "${COMPOSE_FILE}"
EOF
); then
err "Failed to get image name of service 'mysql-mailcow' in 'compose.yml'."
return 1
fi
if [ -z "$(ssh:run "root@$vps" -- docker images -q "$image")" ]; then
info "Image '$image' not available, pull it."
if ! ssh:run "root@$vps" -- \
docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
pull mysql-mailcow; then
root_password=$(printf "%s\n" "$env_content" | grep ^DBROOT= | cut -f 2 -d =)
err "Failed to pull image of service 'mysql-mailcow'."
return 1
fi
fi
echo "${WHITE}Bringing mysql-mailcow up${NORMAL}"
if ! container_id=$(cat <<EOF | ssh:run "root@$vps" -- bash
if ! container_id=$(cat <<EOF | ssh:run "root@$vps" -- bash
echo "[client]
password=$root_password" > "$VPS_TMP_DIR/my.cnf"
@ -384,31 +504,35 @@ docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
mysql-mailcow
EOF
); then
err "Failed to bring up mysql-mailcow"
return 1
fi
); then
err "Failed to bring up mysql-mailcow"
return 1
fi
START="$SECONDS"
while true; do
echo " trying to connect..." >&2
cat <<EOF | ssh:run "root@$vps" -- bash && break
START="$SECONDS"
retries=0
timeout=600
while true; do
((retries++))
echo " waiting for mysql db..." \
"(retry $retries, $(($SECONDS - $START))s elapsed, timeout is ${timeout}s)" >&2
cat <<EOF | ssh:run "root@$vps" -- bash && break
echo "SELECT 1;" | docker exec -i "$container_id" mysql >/dev/null 2>&1
EOF
if (($SECONDS - $START > 10)); then
err "Failed to connect to mysql-mailcow."
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
ssh:run "root@$vps" -- bash
return 1
fi
sleep 0.3
done
if (($SECONDS - $START > $timeout)); then
err "Failed to connect to mysql-mailcow."
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
ssh:run "root@$vps" -- bash
return 1
fi
sleep 0.4
done
DBUSER=$(printf "%s\n" "$env_content" | grep ^DBUSER= | cut -f 2 -d =)
DBPASS=$(printf "%s\n" "$env_content" | grep ^DBPASS= | cut -f 2 -d =)
DBUSER=$(printf "%s\n" "$env_content" | grep ^DBUSER= | cut -f 2 -d =)
DBPASS=$(printf "%s\n" "$env_content" | grep ^DBPASS= | cut -f 2 -d =)
echo "${WHITE}Uploading mysql dump${NORMAL}"
cat <<EOF | ssh:run "root@$vps" -- bash
echo "${WHITE}Uploading mysql dump${NORMAL}"
cat <<EOF | ssh:run "root@$vps" -- bash
echo "
DROP DATABASE IF EXISTS mailcow;
@ -419,54 +543,111 @@ echo "
zcat /var/backups/mysql/mailcow/*.gz | docker exec -i "$container_id" mysql mailcow
EOF
if [ "$?" != 0 ]; then
err "Failed to load mysql dump."
if [ "$?" != 0 ]; then
err "Failed to load mysql dump."
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
ssh:run "root@$vps" -- bash
return 1
fi
echo "${WHITE}Bringing mysql-mailcow down${NORMAL}"
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
ssh:run "root@$vps" -- bash
return 1
ssh:run "root@$vps" -- "rm -rf '$VPS_TMP_DIR'"
msg_target="Full mailcow backup"
fi
echo "${WHITE}Bringing mysql-mailcow down${NORMAL}"
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
ssh:run "root@$vps" -- bash
ssh:run "root@$vps" -- "rm -rf '$VPS_TMP_DIR'"
if [ -n "$stopped_containers" ]; then
echo "${WHITE}Starting mailcow${NORMAL}" >&2
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" up -d" |
ssh:run "root@$vps" -- bash
fi
info "Mailcow was ${GREEN}successfully${NORMAL} restored."
info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."
}
vps_backup_recover() {
local vps="$1" admin server id rtype force
vps_connection_check "$vps" </dev/null || return 1
local vps="$1" admin server id path rtype force type
read-0 admin server id rtype force
read-0 admin server id path rtype force
if ! type=$(ssh:run "root@$vps" -- vps get-type); then
err "Could not get type."
if [[ "$vps" == *":"* ]]; then
vps_path=${vps#*:}
vps=${vps%%:*}
else
vps_path=
fi
vps_connection_check "$vps" </dev/null || {
err "Failed to access '$vps'."
return 1
}
type=$(ssh:run "root@$vps" -- vps get-type) && {
info "VPS $vps seems to be of ${WHITE}$type${NORMAL} type"
}
if [ -z "$path" ]; then
if [ -n "$vps_path" ]; then
err "You can't provide a VPS with path as destination if you don't provide a path in backup source."
return 1
fi
info "No path provided in backup, so we assume you want ${WHITE}full recovery${NORMAL}."
if [ "$rtype" != "$type" ]; then
if [ -n "$force" ]; then
warn "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type."
else
err "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type. (use \`\`-f\`\` to force)"
return 1
fi
fi
else
if [ "$path" == "/" ]; then
if [ -z "$vps_path" ]; then
err "Recovery of '/' (full backup files) requires that you provide a vps path also."
return 1
fi
if [ "$vps_path" == "/" ]; then
err "Recovery of '/' (full backup files) requires that you provide" \
"a vps path different from '/' also."
return 1
fi
fi
fi
if [ "$rtype" != "$type" ]; then
if [ -n "$force" ]; then
warn "Backup found is of ${rtype:-unknown} type, while vps is of $type type."
else
err "Backup found is of ${rtype:-unknown} type, while vps is of $type type. (use \`\`-f\`\` to force)"
## Sets VPS and internal global variable to allow rsync to work
## from vps to backup server.
backup:setup-rsync "$admin" "$vps" "$server" "$id" || return 1
if [ -n "$path" ]; then
if ! backup:path_exists "${path}"; then
err "File or directory '$path' not found in backup."
return 1
fi
if [ -z "$vps_path" ]; then
if [[ "$path" != *"/" ]] && backup:path_exists "${path}"/ ; then
path="$path/"
fi
vps_path=${path%/}
vps_path=${vps_path:-/}
fi
fi
case "$rtype" in
mailcow)
mailcow:vps_backup_recover "$admin" "$server" "$id" "$vps"
case "$rtype-$type" in
mailcow-*)
## Supports having $path and $vps_path set or unset, with additional behavior
mailcow:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
;;
*)
err "Recover on $type type VPS is not yet implemented."
return 1
*-*)
if [ -n "$path" ]; then
## For now, will require having $path and $vps_path set, no additional behaviors
file:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
else
err "Full recover of ${rtype:-unknown} backup type on" \
"${type:-unknown} type VPS is not yet implemented."
return 1
fi
;;
esac
@ -818,18 +999,16 @@ cmdline.spec.gnu recover
cmdline.spec:vps-backup:cmd:recover:run() {
: :posarg: BACKUP_ID 'Backup id.
(ie: myadmin@backup.domain.org:10023#mx.myvps.org)'
: :posarg: [VPS...] 'Target host(s) to check'
(ie: myadmin@backup.domain.org:10023#mx.myvps.org
myadmin@ark-01.org#myid:/a/path)'
: :posarg: VPS_PATH 'Target host(s) to check.
(ie: myvps.com
myvps.com:/a/path)'
: :optval: --date,-D '"last", or label of version to recover. (Default: "last").'
: :optfla: --force,-f 'Will allow you to bypass some checks.'
if [ "${#VPS[@]}" == 0 ]; then
warn "VPS list provided in command line is empty. Nothing will be done."
return 0
fi
if ! [[ "$BACKUP_ID" == *"@"* ]]; then
err "Missing admin account identifier in backup id."
echo " Have you forgottent to specify an admin account ?" \
@ -843,7 +1022,14 @@ cmdline.spec:vps-backup:cmd:recover:run() {
return 1
fi
id=${BACKUP_ID##*#}
id_path=${BACKUP_ID#*#}
if [[ "$id_path" == *":"* ]]; then
id=${id_path%%:*}
path=${id_path#*:}
else
id="$id_path"
path=
fi
BACKUP_TARGET=${BACKUP_ID%#*}
admin=${BACKUP_TARGET%%@*}
server=${BACKUP_TARGET#*@}
@ -856,20 +1042,22 @@ cmdline.spec:vps-backup:cmd:recover:run() {
BACKUP_PATH="/srv/datastore/data/rsync-backup-target/var/mirror"
if ! rtype=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key get-type "$id" ); then
err "Could not get backup type."
echo " Do you have admin access to $admin@$server ?" >&2
echo " Are you sure '$id' backup identifier belongs to '$admin' admin on '$server' ?" >&2
if ! content=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key ls 2>/dev/null); then
err "Access denied to '$admin@${server}'."
return 1
fi
if [ -z "$rtype" ]; then
err "Unknown type of backup on '${server}#${id}'."
idents=$(echo "$content" | sed -r "s/"$'\e'"\[[0-9]+(;[0-9]+)*m//g" | cut -f 2 -d " ")
if ! [[ $'\n'"$idents"$'\n' == *$'\n'"$id"$'\n'* ]]; then
err "Given backup id '$id' not found in $admin@${server}'s idents."
return 1
fi
p0 "$admin" "$server" "$id" "$rtype" "$opt_force" |
vps_mux vps_backup_recover "${VPS[@]}"
rtype=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key get-type "$id" ) &&
info "Backup archive matches ${WHITE}${rtype}${NORMAL} type"
p0 "$admin" "$server" "$id" "$path" "$rtype" "$opt_force" |
vps_backup_recover "${VPS_PATH}"
}

Write Preview
Loading…
Cancel
Save