|
|
|
@ -169,7 +169,7 @@ ssh:run() { |
|
|
|
ssh -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \ |
|
|
|
-o ControlMaster=auto -o ControlPersist=900 \ |
|
|
|
-o "StrictHostKeyChecking=no" \ |
|
|
|
"$hostname" "${ssh_options[@]}" -- "${cmd[@]}" |
|
|
|
"${ssh_options[@]}" "$hostname" -- "${cmd[@]}" |
|
|
|
} 3>&1 1>&2 2>&3 ## | sed -r "s/^/$DARKCYAN$hostname$NORMAL $DARKRED\!$NORMAL /g" |
|
|
|
set_errlvl "${PIPESTATUS[0]}" |
|
|
|
} 3>&1 1>&2 2>&3 |
|
|
|
@ -208,7 +208,7 @@ vps_connection_check() { |
|
|
|
is-port-open "$ip" "22" </dev/null || |
|
|
|
{ echo "${DARKRED}no-port-22-open${NORMAL}"; return 1; } |
|
|
|
|
|
|
|
ssh:open -o ConnectTimeout=2 -o PreferredAuthentications=publickey \ |
|
|
|
ssh:open -o ConnectTimeout=10 -o PreferredAuthentications=publickey \ |
|
|
|
"root@$vps" >/dev/null 2>&1 || |
|
|
|
{ echo "${DARKRED}no-ssh-root-access${NORMAL}"; return 1; } |
|
|
|
} |
|
|
|
@ -234,6 +234,239 @@ vps_check() { |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
vps:rsync() { |
|
|
|
local vps="$1" id="$2" src="$3" dst="$4" |
|
|
|
if [[ "$src" != *":"* ]]; then |
|
|
|
err "Third argument '$src' should be a remote (include the server name as prefix)." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
server=${src%%:*} |
|
|
|
src=${src#*:} |
|
|
|
cat <<EOF | ssh:run "root@$vps" -- bash |
|
|
|
rsync -e "ssh ${ssh_options[*]}" \ |
|
|
|
-azvArH --delete --delete-excluded \ |
|
|
|
--partial --partial-dir .rsync-partial \ |
|
|
|
--numeric-ids \ |
|
|
|
"$server":/var/mirror/"${id}${src}" "${dst}" |
|
|
|
EOF |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
mailcow:vps_backup_recover() { |
|
|
|
local admin="$1" server="$2" id="$3" vps="$4" |
|
|
|
|
|
|
|
## Request recovery key |
|
|
|
ssh_options=() |
|
|
|
if [[ "$server" == *":"* ]]; then |
|
|
|
ssh_options+=(-p "${server#*:}" -o StrictHostKeyChecking=no) |
|
|
|
server=${server%%:*} |
|
|
|
fi |
|
|
|
if ! private_key=$(ssh "${ssh_options[@]}" "$admin"@"$server" request-recovery-key "$id"); then |
|
|
|
err "Couldn't request a recovery key for '$id' with account '$admin'." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
|
|
|
|
echo "type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null" | |
|
|
|
ssh:run "root@$vps" -- bash |
|
|
|
|
|
|
|
if ! VPS_TMP_DIR=$(echo "mktemp -d" | ssh:run "root@$vps" -- bash); then |
|
|
|
err "Couldn't create a temporary directory on vps" |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
|
|
|
|
cat <<EOF | ssh:run "root@$vps" -- bash || return 1 |
|
|
|
touch "$VPS_TMP_DIR/recover_key" && |
|
|
|
chmod go-rwx "$VPS_TMP_DIR/recover_key" && |
|
|
|
printf "%s\n" "$private_key" >> "$VPS_TMP_DIR/recover_key" |
|
|
|
EOF |
|
|
|
|
|
|
|
ssh_options+=(-i "$VPS_TMP_DIR/recover_key" -l rsync) |
|
|
|
|
|
|
|
if ! compose_yml_files=$(cat <<EOF | ssh:run "root@$vps" -- bash |
|
|
|
urn=com.docker.compose.project |
|
|
|
docker ps -f "label=\$urn=mailcowdockerized" \ |
|
|
|
--format='{{.Label "\$urn.working_dir"}}/{{.Label "\$urn.config_files"}}' | |
|
|
|
uniq |
|
|
|
EOF |
|
|
|
); then |
|
|
|
err "Couldn't get list of running projects" |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
|
|
|
|
stopped_containers= |
|
|
|
if [ -n "$compose_yml_files" ]; then |
|
|
|
echo "Found running mailcowdockerized containers" >&2 |
|
|
|
if [[ "$compose_yml_files" == *$'\n'* ]]; then |
|
|
|
err "Running containers are confusing, did not find only one mailcowdockerized project." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then |
|
|
|
err "Running containers are confusing, they don't point to an existing docker-compose.yml." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
echo "Containers where launched from '$compose_yml_files'" >&2 |
|
|
|
COMPOSE_FILE="$compose_yml_files" |
|
|
|
ENV_FILE="${COMPOSE_FILE%/*}/.env" |
|
|
|
if ! echo "[ -e \"${ENV_FILE}\" ]" | ssh:run "root@$vps" -- bash ; then |
|
|
|
err "Running containers are confusing, docker-compose.yml has no '.env' next to it." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
echo "${WHITE}Bringing mailcowdockerized down${NORMAL}" |
|
|
|
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" | |
|
|
|
ssh:run "root@$vps" -- bash |
|
|
|
stopped_containers=1 |
|
|
|
fi |
|
|
|
|
|
|
|
|
|
|
|
for vol in postfix rspamd redis crypt vmail{,-attachments}; do |
|
|
|
volume_name="mailcowdockerized_${vol}-vol-1" |
|
|
|
volume_dir="/var/lib/docker/volumes/${volume_name}/_data" |
|
|
|
if ! ssh:run "root@$server" -- "[ -d '$BACKUP_PATH/${id}${volume_dir}' ]"; then |
|
|
|
continue |
|
|
|
fi |
|
|
|
|
|
|
|
## Create volumes if not existent |
|
|
|
if ! ssh:run "root@$vps" -- " |
|
|
|
[ -d '${volume_dir}' ] || |
|
|
|
docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9 |
|
|
|
[ -d '${volume_dir}' ] |
|
|
|
"; then |
|
|
|
err "Couldn't find nor create '${volume_dir}'." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
|
|
|
|
echo "${WHITE}Downloading of $volume_name${NORMAL}" |
|
|
|
vps:rsync "$vps" "$id" "$server":"${volume_dir}/" "${volume_dir}" || return 1 |
|
|
|
done |
|
|
|
|
|
|
|
|
|
|
|
## Mailcow git base |
|
|
|
for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do |
|
|
|
if ! ssh:run "root@$server" -- "[ -d '$BACKUP_PATH/${id}${mailcow_dir}' ]"; then |
|
|
|
continue |
|
|
|
else |
|
|
|
## this possibly change last value |
|
|
|
COMPOSE_FILE="$mailcow_dir/docker-compose.yml" |
|
|
|
ENV_FILE="$mailcow_dir/.env" |
|
|
|
echo "${WHITE}Download of $mailcow_dir${NORMAL}" |
|
|
|
vps:rsync "$vps" "$id" "$server":"${mailcow_dir}"/ "${mailcow_dir}" || return 1 |
|
|
|
break |
|
|
|
fi |
|
|
|
done |
|
|
|
|
|
|
|
|
|
|
|
## Mysql database |
|
|
|
echo "${WHITE}Downloading last backup of mysql backups${NORMAL}" |
|
|
|
vps:rsync "$vps" "$id" "$server":"/var/backups/mysql/" "/var/backups/mysql" || return 1 |
|
|
|
|
|
|
|
if ! env_content=$(echo "cat '$ENV_FILE'" | ssh:run "root@$vps" -- bash); then |
|
|
|
err "Can't access env file: '$ENV_FILE'." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
|
|
|
|
root_password=$(printf "%s\n" "$env_content" | grep ^DBROOT= | cut -f 2 -d =) |
|
|
|
|
|
|
|
echo "${WHITE}Bringing mysql-mailcow up${NORMAL}" |
|
|
|
if ! container_id=$(cat <<EOF | ssh:run "root@$vps" -- bash |
|
|
|
echo "[client] |
|
|
|
password=$root_password" > "$VPS_TMP_DIR/my.cnf" |
|
|
|
|
|
|
|
docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \ |
|
|
|
run -d \ |
|
|
|
-v "$VPS_TMP_DIR/my.cnf:/root/.my.cnf:ro" \ |
|
|
|
mysql-mailcow |
|
|
|
|
|
|
|
EOF |
|
|
|
); then |
|
|
|
err "Failed to bring up mysql-mailcow" |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
|
|
|
|
START="$SECONDS" |
|
|
|
while true; do |
|
|
|
echo " trying to connect..." >&2 |
|
|
|
cat <<EOF | ssh:run "root@$vps" -- bash && break |
|
|
|
echo "SELECT 1;" | docker exec -i "$container_id" mysql >/dev/null 2>&1 |
|
|
|
EOF |
|
|
|
if (($SECONDS - $START > 10)); then |
|
|
|
err "Failed to connect to mysql-mailcow." |
|
|
|
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" | |
|
|
|
ssh:run "root@$vps" -- bash |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
sleep 0.3 |
|
|
|
done |
|
|
|
|
|
|
|
DBUSER=$(printf "%s\n" "$env_content" | grep ^DBUSER= | cut -f 2 -d =) |
|
|
|
DBPASS=$(printf "%s\n" "$env_content" | grep ^DBPASS= | cut -f 2 -d =) |
|
|
|
|
|
|
|
echo "${WHITE}Uploading mysql dump${NORMAL}" |
|
|
|
cat <<EOF | ssh:run "root@$vps" -- bash |
|
|
|
|
|
|
|
echo " |
|
|
|
DROP DATABASE IF EXISTS mailcow; |
|
|
|
CREATE DATABASE mailcow; |
|
|
|
GRANT ALL PRIVILEGES ON mailcow.* TO '$DBUSER'@'%' IDENTIFIED BY '$DBPASS'; |
|
|
|
" | docker exec -i "$container_id" mysql |
|
|
|
|
|
|
|
zcat /var/backups/mysql/mailcow/*.gz | docker exec -i "$container_id" mysql mailcow |
|
|
|
|
|
|
|
EOF |
|
|
|
if [ "$?" != 0 ]; then |
|
|
|
err "Failed to load mysql dump." |
|
|
|
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" | |
|
|
|
ssh:run "root@$vps" -- bash |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
|
|
|
|
echo "${WHITE}Bringing mysql-mailcow down${NORMAL}" |
|
|
|
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" | |
|
|
|
ssh:run "root@$vps" -- bash |
|
|
|
ssh:run "root@$vps" -- "rm -rf '$VPS_TMP_DIR'" |
|
|
|
|
|
|
|
if [ -n "$stopped_containers" ]; then |
|
|
|
echo "${WHITE}Starting mailcow${NORMAL}" >&2 |
|
|
|
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" up -d" | |
|
|
|
ssh:run "root@$vps" -- bash |
|
|
|
fi |
|
|
|
info "Mailcow was ${GREEN}successfully${NORMAL} restored." |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
vps_backup_recover() { |
|
|
|
local vps="$1" admin server id rtype force |
|
|
|
vps_connection_check "$vps" </dev/null || return 1 |
|
|
|
|
|
|
|
read-0 admin server id rtype force |
|
|
|
|
|
|
|
if ! type=$(ssh:run "root@$vps" -- vps get-type); then |
|
|
|
err "Could not get type." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
|
|
|
|
if [ "$rtype" != "$type" ]; then |
|
|
|
if [ -n "$force" ]; then |
|
|
|
warn "Backup found is of ${rtype:-unknown} type, while vps is of $type type." |
|
|
|
else |
|
|
|
err "Backup found is of ${rtype:-unknown} type, while vps is of $type type. (use \`\`-f\`\` to force)" |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
fi |
|
|
|
|
|
|
|
case "$rtype" in |
|
|
|
mailcow) |
|
|
|
mailcow:vps_backup_recover "$admin" "$server" "$id" "$vps" |
|
|
|
;; |
|
|
|
*) |
|
|
|
err "Recover on $type type VPS is not yet implemented." |
|
|
|
return 1 |
|
|
|
;; |
|
|
|
esac |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
vps_install_backup() { |
|
|
|
local vps="$1" admin server |
|
|
|
vps_connection_check "$vps" </dev/null || return 1 |
|
|
|
@ -540,6 +773,109 @@ cmdline.spec:vps-install:cmd:backup:run() { |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
cmdline.spec.gnu vps-backup |
|
|
|
cmdline.spec::cmd:vps-backup:run() { |
|
|
|
: |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
cmdline.spec.gnu ls |
|
|
|
cmdline.spec:vps-backup:cmd:ls:run() { |
|
|
|
: :posarg: BACKUP_ID 'Backup id. |
|
|
|
(ie: myadmin@backup.domain.org:10023)' |
|
|
|
|
|
|
|
if ! [[ "$BACKUP_ID" == *"@"* ]]; then |
|
|
|
err "Missing admin account identifier in backup id." |
|
|
|
echo " Have you forgottent to specify an admin account ?" \ |
|
|
|
"ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)" |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
|
|
|
|
id=${BACKUP_ID##*#} |
|
|
|
BACKUP_TARGET=${BACKUP_ID%#*} |
|
|
|
admin=${BACKUP_TARGET%%@*} |
|
|
|
server=${BACKUP_TARGET#*@} |
|
|
|
|
|
|
|
## XXXvlab: in this first implementation we expect to have access |
|
|
|
## to the server main ssh port 22, so we won't use the provided port. |
|
|
|
ssh_options=() |
|
|
|
if [[ "$server" == *":"* ]]; then |
|
|
|
ssh_options+=(-p "${server#*:}") |
|
|
|
server=${server%%:*} |
|
|
|
fi |
|
|
|
|
|
|
|
ssh "${ssh_options[@]}" "$admin"@"$server" ssh-key ls |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
cmdline.spec.gnu recover |
|
|
|
cmdline.spec:vps-backup:cmd:recover:run() { |
|
|
|
|
|
|
|
: :posarg: BACKUP_ID 'Backup id. |
|
|
|
(ie: myadmin@backup.domain.org:10023#mx.myvps.org)' |
|
|
|
: :posarg: [VPS...] 'Target host(s) to check' |
|
|
|
|
|
|
|
: :optval: --date,-D '"last", or label of version to recover. (Default: "last").' |
|
|
|
: :optfla: --force,-f 'Will allow you to bypass some checks.' |
|
|
|
|
|
|
|
|
|
|
|
if [ "${#VPS[@]}" == 0 ]; then |
|
|
|
warn "VPS list provided in command line is empty. Nothing will be done." |
|
|
|
return 0 |
|
|
|
fi |
|
|
|
|
|
|
|
if ! [[ "$BACKUP_ID" == *"@"* ]]; then |
|
|
|
err "Missing admin account identifier in backup id." |
|
|
|
echo " Have you forgottent to specify an admin account ?" \ |
|
|
|
"ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)" |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
if ! [[ "$BACKUP_ID" == *"@"*"#"* ]]; then |
|
|
|
err "Missing backup label identifier in backup id." |
|
|
|
echo " Have you forgottent to specify a backup label identifier ?" \ |
|
|
|
"ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)" |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
|
|
|
|
id=${BACKUP_ID##*#} |
|
|
|
BACKUP_TARGET=${BACKUP_ID%#*} |
|
|
|
admin=${BACKUP_TARGET%%@*} |
|
|
|
server=${BACKUP_TARGET#*@} |
|
|
|
|
|
|
|
## XXXvlab: in this first implementation we expect to have access |
|
|
|
## to the server main ssh port 22, so we won't use the provided port. |
|
|
|
# ssh_options=() |
|
|
|
if [[ "$server" == *":"* ]]; then |
|
|
|
# ssh_options+=(-p "${server#*:}") |
|
|
|
ssh_server=${server%%:*} |
|
|
|
fi |
|
|
|
|
|
|
|
BACKUP_PATH="/srv/datastore/data/rsync-backup-target/var/mirror" |
|
|
|
|
|
|
|
if ! rtype=$(echo " |
|
|
|
if [ -d '$BACKUP_PATH/$id/var/lib/docker/volumes/mailcowdockerized_crypt-vol-1' ]; then |
|
|
|
echo mailcow |
|
|
|
elif [ -d '$BACKUP_PATH/$id/compose.yml' ]; then |
|
|
|
echo compose |
|
|
|
fi |
|
|
|
true |
|
|
|
" | ssh:run "root@${ssh_server}" -- bash ); then |
|
|
|
err "Could not get backup type." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
|
|
|
|
if [ -z "$rtype" ]; then |
|
|
|
err "Unknown type of backup on '${server}#${id}'." |
|
|
|
return 1 |
|
|
|
fi |
|
|
|
|
|
|
|
p0 "$admin" "$server" "$id" "$rtype" "$opt_force" | |
|
|
|
vps_mux vps_backup_recover "${VPS[@]}" |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
cmdline.spec.gnu vps-update |
|
|
|
cmdline.spec::cmd:vps-update:run() { |
|
|
|
|
|
|
|
|
