You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1170 lines
37 KiB
1170 lines
37 KiB
#!/bin/bash
|
|
|
|
|
|
. /etc/shlib
|
|
|
|
include common
|
|
include parse
|
|
include cmdline
|
|
include config
|
|
|
|
|
|
[[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true
|
|
|
|
version=0.1
|
|
desc='Manage 0k related installs'
|
|
help=""
|
|
|
|
|
|
##
|
|
## Functions
|
|
##
|
|
|
|
is-port-open() {
|
|
local host="$1" port="$2" timeout=5
|
|
start="$SECONDS"
|
|
debug "Testing if $host's port $2 is open ..."
|
|
while true; do
|
|
timeout 1 bash -c "</dev/tcp/${host}/${port}" >/dev/null 2>&1 && break
|
|
sleep 0.2
|
|
if [ "$((SECONDS - start))" -gt "$timeout" ]; then
|
|
return 1
|
|
fi
|
|
done
|
|
}
|
|
|
|
resolve() {
|
|
local ent hostname="$1"
|
|
debug "Resolving $1 ..."
|
|
if ent=$(getent ahosts "$hostname"); then
|
|
ent=$(echo "$ent" | egrep ^"[0-9]+.[0-9]+.[0-9]+.[0-9]+\s+" | \
|
|
head -n 1 | awk '{ print $1 }')
|
|
debug " .. resolved $1 to $ent."
|
|
echo "$ent"
|
|
else
|
|
debug " .. couldn't resolve $1."
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
|
|
set_errlvl() { return "${1:-1}"; }
|
|
|
|
export master_pid=$$
|
|
ssh:open() {
|
|
local hostname ssh_cmd ssh_options
|
|
ssh_cmd=(ssh)
|
|
ssh_options=()
|
|
while [ "$#" != 0 ]; do
|
|
case "$1" in
|
|
"--stdin-password")
|
|
ssh_cmd=(sshpass "${ssh_cmd[@]}")
|
|
;;
|
|
-o)
|
|
ssh_options+=("$1" "$2")
|
|
shift
|
|
;;
|
|
*)
|
|
[ -z "$hostname" ] && hostname="$1" || {
|
|
err "Surnumerous positional argument '$1'. Expecting only hostname."
|
|
return 1
|
|
}
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
full_cmd=(
|
|
"${ssh_cmd[@]}"
|
|
-o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
|
|
-o ControlMaster=auto -o ControlPersist=900 \
|
|
-o ConnectTimeout=5 -o StrictHostKeyChecking=no \
|
|
"${ssh_options[@]}" \
|
|
"$hostname" "$@" -- true)
|
|
"${full_cmd[@]}" </dev/null >/dev/null 2>&1 || {
|
|
err "Failed: ${full_cmd[*]}"
|
|
return 1
|
|
}
|
|
trap_add EXIT,INT 'ssh:quit "$hostname"'
|
|
}
|
|
|
|
|
|
ssh:open-try() {
|
|
local opts hostnames
|
|
opts=()
|
|
hostnames=()
|
|
while [ "$#" != 0 ]; do
|
|
case "$1" in
|
|
-o)
|
|
opts+=("$1" "$2")
|
|
shift
|
|
;;
|
|
*)
|
|
hostnames+=("$1")
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
password=''
|
|
for host in "${hostnames[@]}"; do
|
|
debug "Trying $host with publickey."
|
|
ssh:open -o PreferredAuthentications=publickey \
|
|
"${opts[@]}" \
|
|
"$host" >/dev/null 2>&1 && {
|
|
echo "$host"$'\n'"$password"$'\n'
|
|
return 0
|
|
}
|
|
debug " .. failed connecting to $host with publickey."
|
|
done
|
|
local times=0 password
|
|
while [ "$((++times))" -le 3 ]; do
|
|
read -sp "$HOST's password: " password
|
|
errlvl="$?"
|
|
echo >&2
|
|
if [ "$errlvl" -gt 0 ]; then
|
|
exit 1
|
|
fi
|
|
for host in "${hostnames[@]}"; do
|
|
debug "Trying $host with password ($times/3)"
|
|
echo "$password" | ssh:open -o PreferredAuthentications=password \
|
|
--stdin-password \
|
|
"${opts[@]}" \
|
|
"$host" >/dev/null 2>&1 && {
|
|
echo "$host"$'\n'"$password"$'\n'
|
|
return 0
|
|
}
|
|
debug " .. failed connecting to $host with password."
|
|
done
|
|
err "login failed. Try again... ($((times+1))/3)"
|
|
done
|
|
return 1
|
|
}
|
|
|
|
|
|
ssh:run() {
|
|
local hostname="$1" ssh_options cmd
|
|
shift
|
|
|
|
ssh_options=()
|
|
cmd=()
|
|
while [ "$#" != 0 ]; do
|
|
case "$1" in
|
|
"--")
|
|
shift
|
|
cmd+=("$@")
|
|
break
|
|
;;
|
|
*)
|
|
ssh_options+=("$1")
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
#echo "$DARKCYAN$hostname$NORMAL $WHITE\$$NORMAL" "$@"
|
|
debug "Running cmd: ${cmd[@]}"
|
|
for arg in "${cmd[@]}"; do
|
|
debug "$arg"
|
|
done
|
|
{
|
|
{
|
|
ssh -o ControlPath=/tmp/ssh-control-master-${master_pid}-$hostname \
|
|
-o ControlMaster=auto -o ControlPersist=900 \
|
|
-o "StrictHostKeyChecking=no" \
|
|
"${ssh_options[@]}" "$hostname" -- "${cmd[@]}"
|
|
} 3>&1 1>&2 2>&3 ## | sed -r "s/^/$DARKCYAN$hostname$NORMAL $DARKRED\!$NORMAL /g"
|
|
set_errlvl "${PIPESTATUS[0]}"
|
|
} 3>&1 1>&2 2>&3
|
|
}
|
|
|
|
ssh:quit() {
|
|
local hostname="$1"
|
|
shift
|
|
ssh -o ControlPath=/tmp/ssh-control-master-${master_pid} \
|
|
-o ControlMaster=auto -o ControlPersist=900 -O exit \
|
|
"$hostname" 2>/dev/null
|
|
}
|
|
|
|
|
|
is_ovh_domain_name() {
|
|
local domain="$1"
|
|
|
|
[[ "$domain" == *.ovh.net ]] && return 0
|
|
[[ "$domain" == "ns"*".ip-"*".eu" ]] && return 0
|
|
return 1
|
|
}
|
|
|
|
is_ovh_hostname() {
|
|
local domain="$1"
|
|
|
|
[[ "$domain" =~ ^vps-[0-9a-f]*$ ]] && return 0
|
|
[[ "$domain" =~ ^vps[0-9]*$ ]] && return 0
|
|
return 1
|
|
}
|
|
|
|
vps_connection_check() {
|
|
local vps="$1"
|
|
ip=$(resolve "$vps") ||
|
|
{ echo "${DARKRED}no-resolve${NORMAL}"; return 1; }
|
|
|
|
is-port-open "$ip" "22" </dev/null ||
|
|
{ echo "${DARKRED}no-port-22-open${NORMAL}"; return 1; }
|
|
|
|
ssh:open -o ConnectTimeout=10 -o PreferredAuthentications=publickey \
|
|
"root@$vps" >/dev/null 2>&1 ||
|
|
{ echo "${DARKRED}no-ssh-root-access${NORMAL}"; return 1; }
|
|
}
|
|
|
|
vps_check() {
|
|
local vps="$1"
|
|
vps_connection_check "$vps" </dev/null || return 1
|
|
if size=$(
|
|
echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 5 -d ' ' | cut -f 1 -d %" |
|
|
ssh:run "root@$vps" -- bash); then
|
|
if [ "$size" -gt "90" ]; then
|
|
echo "${DARKRED}above-90%-disk-usage${NORMAL}"
|
|
elif [ "$size" -gt "75" ]; then
|
|
echo "${DARKYELLOW}above-75%-disk-usage${NORMAL}"
|
|
fi
|
|
else
|
|
echo "${DARKRED}no-size${NORMAL}"
|
|
fi </dev/null
|
|
compose_content=$(ssh:run "root@$vps" -- cat /opt/apps/myc-deploy/compose.yml </dev/null) ||
|
|
{ echo "${DARKRED}no-compose${NORMAL}"; return 1; }
|
|
echo "$compose_content" | grep backup >/dev/null 2>&1 ||
|
|
{ echo "${DARKRED}no-backup${NORMAL}"; return 1; }
|
|
}
|
|
|
|
|
|
|
|
backup:setup-rsync() {
|
|
local admin="$1" vps="$2" server="$3" id="$4"
|
|
|
|
[ -z "${BACKUP_SSH_SERVER}" ] || {
|
|
err "Unexpected error: '\$BACKUP_SSH_SERVER' is already set in '$FUNCNAME'."
|
|
return 1
|
|
}
|
|
|
|
BACKUP_SSH_OPTIONS=(-o StrictHostKeyChecking=no)
|
|
if [[ "$server" == *":"* ]]; then
|
|
BACKUP_SSH_OPTIONS+=(-p "${server#*:}")
|
|
BACKUP_SSH_SERVER=${server%%:*}
|
|
else
|
|
BACKUP_SSH_SERVER="$server"
|
|
fi
|
|
|
|
if ! private_key=$(ssh "${BACKUP_SSH_OPTIONS[@]}" \
|
|
"$admin"@"${BACKUP_SSH_SERVER}" request-recovery-key "$id"); then
|
|
err "Couldn't request a recovery key for '$id' with account '$admin'."
|
|
return 1
|
|
fi
|
|
|
|
if ! VPS_TMP_DIR=$(echo "mktemp -d" | ssh:run "root@$vps" -- bash); then
|
|
err "Couldn't create a temporary directory on vps"
|
|
return 1
|
|
fi
|
|
|
|
cat <<EOF | ssh:run "root@$vps" -- bash || return 1
|
|
touch "$VPS_TMP_DIR/recover_key" &&
|
|
chmod go-rwx "$VPS_TMP_DIR/recover_key" &&
|
|
printf "%s\n" "$private_key" >> "$VPS_TMP_DIR/recover_key"
|
|
EOF
|
|
|
|
BACKUP_SSH_OPTIONS+=(-i "$VPS_TMP_DIR/recover_key" -l rsync)
|
|
BACKUP_VPS_TARGET="$vps"
|
|
BACKUP_IDENT="$id"
|
|
|
|
echo "type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null" |
|
|
ssh:run "root@$vps" -- bash || return 1
|
|
|
|
}
|
|
|
|
|
|
backup:rsync() {
|
|
local ssh_options
|
|
|
|
[ -n "${BACKUP_SSH_SERVER}" ] || {
|
|
err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
|
|
return 1
|
|
}
|
|
|
|
rsync_options=()
|
|
while [[ "$1" == "-"* ]]; do
|
|
rsync_options+=("$1")
|
|
shift
|
|
done
|
|
local src="$1" dst="$2"
|
|
|
|
cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash
|
|
rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
|
|
-azvArH --delete --delete-excluded \
|
|
--partial --partial-dir .rsync-partial \
|
|
--numeric-ids ${rsync_options[*]} \
|
|
"${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "${dst}"
|
|
EOF
|
|
|
|
}
|
|
|
|
|
|
backup:path_exists() {
|
|
local src="$1"
|
|
|
|
[ -n "${BACKUP_SSH_SERVER}" ] || {
|
|
err "Unexpected error: '\$BACKUP_SSH_SERVER' is not set in 'rsync_exists'."
|
|
return 1
|
|
}
|
|
|
|
cat <<EOF | ssh:run "root@${BACKUP_VPS_TARGET}" -- bash >/dev/null 2>&1
|
|
rsync -e "ssh ${BACKUP_SSH_OPTIONS[*]}" \
|
|
-nazvArH --numeric-ids \
|
|
"${BACKUP_SSH_SERVER}":/var/mirror/"${BACKUP_IDENT}${src}" "/tmp/dummy"
|
|
EOF
|
|
|
|
}
|
|
|
|
file:vps_backup_recover() {
|
|
local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"
|
|
|
|
backup:rsync "${path}" "${vps_path}" || return 1
|
|
|
|
if [[ "$path" == *"/" ]]; then
|
|
if [ "$path" == "$vps_path"/ ]; then
|
|
msg_target="Directory '$path'"
|
|
else
|
|
msg_target="Directory '$path' -> '$vps_path'"
|
|
fi
|
|
else
|
|
if [ "$path" == "$vps_path" ]; then
|
|
msg_target="File '$path'"
|
|
else
|
|
msg_target="File '$path' -> '$vps_path'"
|
|
fi
|
|
fi
|
|
|
|
info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."
|
|
|
|
}
|
|
|
|
mailcow:vps_backup_recover() {
|
|
local admin="$1" server="$2" id="$3" path="$4" vps="$5" vps_path="$6"
|
|
|
|
|
|
if ! compose_yml_files=$(cat <<EOF | ssh:run "root@$vps" -- bash
|
|
urn=com.docker.compose.project
|
|
docker ps -f "label=\$urn=mailcowdockerized" \
|
|
--format="{{.Label \"\$urn.working_dir\"}}/{{.Label \"\$urn.config_files\"}}" |
|
|
uniq
|
|
EOF
|
|
); then
|
|
err "Couldn't get list of running projects"
|
|
return 1
|
|
fi
|
|
|
|
stopped_containers=
|
|
if [ -n "$compose_yml_files" ]; then
|
|
echo "Found running mailcowdockerized containers" >&2
|
|
if [[ "$compose_yml_files" == *$'\n'* ]]; then
|
|
err "Running containers are confusing, did not find only one mailcowdockerized project."
|
|
return 1
|
|
fi
|
|
if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
|
|
## For some reason, sometimes $urn.config_files holds an absolute path
|
|
compose_yml_files=/${compose_yml_files#*//}
|
|
if ! echo "[ -e \"$compose_yml_files\" ]" | ssh:run "root@$vps" -- bash ; then
|
|
err "Running containers are confusing, they don't point to an existing docker-compose.yml."
|
|
return 1
|
|
fi
|
|
fi
|
|
echo "Containers where launched from '$compose_yml_files'" >&2
|
|
COMPOSE_FILE="$compose_yml_files"
|
|
ENV_FILE="${COMPOSE_FILE%/*}/.env"
|
|
if ! echo "[ -e \"${ENV_FILE}\" ]" | ssh:run "root@$vps" -- bash ; then
|
|
err "Running containers are confusing, docker-compose.yml has no '.env' next to it."
|
|
return 1
|
|
fi
|
|
echo "${WHITE}Bringing mailcowdockerized down${NORMAL}"
|
|
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
|
|
ssh:run "root@$vps" -- bash
|
|
stopped_containers=1
|
|
fi
|
|
|
|
if [[ "$path" == "/"* ]]; then
|
|
|
|
##
|
|
## Additional intelligence to simple file copy
|
|
##
|
|
|
|
if [[ "$path" == "/var/lib/docker/volumes/mailcowdockerized_*-vol-1/_data"* ]]; then
|
|
volume_name=${path#/var/lib/docker/volumes/}
|
|
volume_name=${volume_name%%/*}
|
|
volume_dir=${path%%"$volume_name"*}
|
|
|
|
## Create volumes if not existent
|
|
if ! ssh:run "root@$vps" -- "
|
|
[ -d '${volume_dir}' ] ||
|
|
docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
|
|
[ -d '${volume_dir}' ]
|
|
"; then
|
|
err "Couldn't find nor create '${volume_dir}'."
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
echo "${WHITE}Sync from backup ${path} to VPS ${vps_path}${NORMAL}" >&2
|
|
backup:rsync "${path}" "${vps_path}" || return 1
|
|
|
|
if [[ "$path" == *"/" ]]; then
|
|
if [ "$path" == "$vps_path"/ ]; then
|
|
msg_target="Directory '$path'"
|
|
else
|
|
msg_target="Directory '$path' -> '$vps_path'"
|
|
fi
|
|
else
|
|
if [ "$path" == "$vps_path" ]; then
|
|
msg_target="File '$path'"
|
|
else
|
|
msg_target="File '$path' -> '$vps_path'"
|
|
fi
|
|
fi
|
|
else
|
|
ALL_TARGETS=(mailcow postfix rspamd redis crypt vmail{,-attachments} mysql)
|
|
|
|
if [[ -n "$path" ]]; then
|
|
targets=()
|
|
bad_targets=()
|
|
for target in ${path//,/ }; do
|
|
if [[ " ${ALL_TARGETS[*]} " != *" $target "* ]]; then
|
|
bad_targets+=("$target")
|
|
fi
|
|
targets+=("$target")
|
|
done
|
|
if [ "${#bad_targets[@]}" -gt 0 ]; then
|
|
bad_target_msg=$(printf "%s, " "${bad_targets[@]}")
|
|
err "Unknown components: ${bad_target_msg%, }. These are allowed components:"
|
|
printf " - %s\n" "${ALL_TARGETS[@]}" >&2
|
|
return 1
|
|
fi
|
|
msg_target="Partial mailcow backup"
|
|
else
|
|
targets=("${ALL_TARGETS[@]}")
|
|
msg_target="Full mailcow backup"
|
|
fi
|
|
|
|
|
|
for target in "${targets[@]}"; do
|
|
case "$target" in
|
|
postfix|rspamd|redis|crypt|vmail|vmail-attachments)
|
|
|
|
volume_name="mailcowdockerized_${target}-vol-1"
|
|
volume_dir="/var/lib/docker/volumes/${volume_name}/_data"
|
|
if ! backup:path_exists "${volume_dir}/"; then
|
|
warn "No '$volume_name' in backup. This might be expected."
|
|
continue
|
|
fi
|
|
## Create volumes if not existent
|
|
if ! ssh:run "root@$vps" -- "
|
|
[ -d '${volume_dir}' ] ||
|
|
docker run --rm -v ${volume_name}:/tmp/dummy docker.0k.io/alpine:3.9
|
|
[ -d '${volume_dir}' ]
|
|
"; then
|
|
err "Couldn't find nor create '${volume_dir}'."
|
|
return 1
|
|
fi
|
|
|
|
echo "${WHITE}Downloading of $volume_name${NORMAL}"
|
|
backup:rsync "${volume_dir}/" "${volume_dir}" || return 1
|
|
;;
|
|
mailcow)
|
|
## Mailcow git base
|
|
COMPOSE_FILE=
|
|
for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
|
|
backup:path_exists "${mailcow_dir}/" || continue
|
|
## this possibly change last value
|
|
COMPOSE_FILE="$mailcow_dir/docker-compose.yml"
|
|
ENV_FILE="$mailcow_dir/.env"
|
|
echo "${WHITE}Download of $mailcow_dir${NORMAL}"
|
|
backup:rsync "${mailcow_dir}"/ "${mailcow_dir}" || return 1
|
|
break
|
|
done
|
|
if [ -z "$COMPOSE_FILE" ]; then
|
|
err "Can't find mailcow base installation path in backup."
|
|
return 1
|
|
fi
|
|
|
|
;;
|
|
mysql)
|
|
if [ -z "$COMPOSE_FILE" ]; then
|
|
## Mailcow git base
|
|
compose_files=()
|
|
for mailcow_dir in /opt/{apps/,}mailcow-dockerized; do
|
|
ssh:run "root@$vps" -- "[ -e \"$mailcow_dir/docker-compose.yml\" ]" || continue
|
|
## this possibly change last value
|
|
compose_files+=("$mailcow_dir/docker-compose.yml")
|
|
done
|
|
if [ "${#compose_files[@]}" == 0 ]; then
|
|
err "No compose file found for mailcow installation."
|
|
return 1
|
|
elif [ "${#compose_files[@]}" -gt 1 ]; then
|
|
err "Multiple compose files for mailcow found:"
|
|
for f in "${compose_files[@]}"; do
|
|
echo " - $f" >&2
|
|
done
|
|
echo "Can't decide which to use for mounting mysql container." >&2
|
|
return 1
|
|
fi
|
|
COMPOSE_FILE="${compose_files[0]}"
|
|
ENV_FILE="${COMPOSE_FILE%/*}/.env"
|
|
if ! ssh:run "root@$vps" -- "[ -e \"${COMPOSE_FILE%/*}/.env\" ]"; then
|
|
err "No env file in '$ENV_FILE' found."
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
## Mysql database
|
|
echo "${WHITE}Downloading last backup of mysql backups${NORMAL}"
|
|
backup:rsync "/var/backups/mysql/" "/var/backups/mysql" || return 1
|
|
|
|
if ! env_content=$(echo "cat '$ENV_FILE'" | ssh:run "root@$vps" -- bash); then
|
|
err "Can't access env file: '$ENV_FILE'."
|
|
return 1
|
|
fi
|
|
|
|
root_password=$(printf "%s\n" "$env_content" | grep ^DBROOT= | cut -f 2 -d =)
|
|
|
|
echo "${WHITE}Bringing mysql-mailcow up${NORMAL}"
|
|
if ! image=$(cat <<EOF | ssh:run "root@$vps" -- bash
|
|
shyaml get-value services.mysql-mailcow.image < "${COMPOSE_FILE}"
|
|
EOF
|
|
); then
|
|
err "Failed to get image name of service 'mysql-mailcow' in 'compose.yml'."
|
|
return 1
|
|
fi
|
|
if [ -z "$(ssh:run "root@$vps" -- docker images -q "$image")" ]; then
|
|
info "Image '$image' not available, pull it."
|
|
if ! ssh:run "root@$vps" -- \
|
|
docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
|
|
pull mysql-mailcow; then
|
|
|
|
err "Failed to pull image of service 'mysql-mailcow'."
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
if ! container_id=$(cat <<EOF | ssh:run "root@$vps" -- bash
|
|
echo "[client]
|
|
password=$root_password" > "$VPS_TMP_DIR/my.cnf"
|
|
|
|
docker-compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
|
|
run -d \
|
|
-v "$VPS_TMP_DIR/my.cnf:/root/.my.cnf:ro" \
|
|
mysql-mailcow
|
|
|
|
EOF
|
|
); then
|
|
err "Failed to bring up mysql-mailcow"
|
|
return 1
|
|
fi
|
|
|
|
START="$SECONDS"
|
|
retries=0
|
|
timeout=600
|
|
while true; do
|
|
((retries++))
|
|
echo " waiting for mysql db..." \
|
|
"(retry $retries, $(($SECONDS - $START))s elapsed, timeout is ${timeout}s)" >&2
|
|
cat <<EOF | ssh:run "root@$vps" -- bash && break
|
|
echo "SELECT 1;" | docker exec -i "$container_id" mysql >/dev/null 2>&1
|
|
EOF
|
|
if (($SECONDS - $START > $timeout)); then
|
|
err "Failed to connect to mysql-mailcow."
|
|
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
|
|
ssh:run "root@$vps" -- bash
|
|
return 1
|
|
fi
|
|
sleep 0.4
|
|
done
|
|
|
|
DBUSER=$(printf "%s\n" "$env_content" | grep ^DBUSER= | cut -f 2 -d =)
|
|
DBPASS=$(printf "%s\n" "$env_content" | grep ^DBPASS= | cut -f 2 -d =)
|
|
|
|
echo "${WHITE}Uploading mysql dump${NORMAL}"
|
|
cat <<EOF | ssh:run "root@$vps" -- bash
|
|
|
|
echo "
|
|
DROP DATABASE IF EXISTS mailcow;
|
|
CREATE DATABASE mailcow;
|
|
GRANT ALL PRIVILEGES ON mailcow.* TO '$DBUSER'@'%' IDENTIFIED BY '$DBPASS';
|
|
" | docker exec -i "$container_id" mysql
|
|
|
|
zcat /var/backups/mysql/mailcow/*.gz | docker exec -i "$container_id" mysql mailcow
|
|
|
|
EOF
|
|
if [ "$?" != 0 ]; then
|
|
err "Failed to load mysql dump."
|
|
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
|
|
ssh:run "root@$vps" -- bash
|
|
return 1
|
|
fi
|
|
|
|
echo "${WHITE}Bringing mysql-mailcow down${NORMAL}"
|
|
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" down" |
|
|
ssh:run "root@$vps" -- bash
|
|
;;
|
|
*)
|
|
err "Unknown component '$target'. Bailing out."
|
|
return 1
|
|
esac
|
|
done
|
|
|
|
ssh:run "root@$vps" -- "rm -rf '$VPS_TMP_DIR'"
|
|
fi
|
|
|
|
if [ -n "$stopped_containers" ]; then
|
|
echo "${WHITE}Starting mailcow${NORMAL}" >&2
|
|
echo "docker-compose -f \"${COMPOSE_FILE}\" --env-file \"${ENV_FILE}\" up -d" |
|
|
ssh:run "root@$vps" -- bash
|
|
fi
|
|
info "$msg_target was ${GREEN}successfully${NORMAL} restored on $vps."
|
|
|
|
}
|
|
|
|
vps_backup_recover() {
|
|
local vps="$1" admin server id path rtype force type
|
|
|
|
read-0 admin server id path rtype force
|
|
|
|
if [[ "$vps" == *":"* ]]; then
|
|
vps_path=${vps#*:}
|
|
vps=${vps%%:*}
|
|
else
|
|
vps_path=
|
|
fi
|
|
|
|
vps_connection_check "$vps" </dev/null || {
|
|
err "Failed to access '$vps'."
|
|
return 1
|
|
}
|
|
|
|
type=$(ssh:run "root@$vps" -- vps get-type) && {
|
|
info "VPS $vps seems to be of ${WHITE}$type${NORMAL} type"
|
|
}
|
|
|
|
if [ -z "$path" ]; then
|
|
if [ -n "$vps_path" ]; then
|
|
err "You can't provide a VPS with path as destination if you don't provide a path in backup source."
|
|
return 1
|
|
fi
|
|
info "No path provided in backup, so we assume you want ${WHITE}full recovery${NORMAL}."
|
|
if [ "$rtype" != "$type" ]; then
|
|
if [ -n "$force" ]; then
|
|
warn "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type."
|
|
else
|
|
err "Backup found is of ${rtype:-unknown} type, while vps is of ${type:-unknown} type. (use \`\`-f\`\` to force)"
|
|
return 1
|
|
fi
|
|
fi
|
|
else
|
|
if [ "$path" == "/" ]; then
|
|
if [ -z "$vps_path" ]; then
|
|
err "Recovery of '/' (full backup files) requires that you provide a vps path also."
|
|
return 1
|
|
fi
|
|
if [ "$vps_path" == "/" ]; then
|
|
err "Recovery of '/' (full backup files) requires that you provide" \
|
|
"a vps path different from '/' also."
|
|
return 1
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
## Sets VPS and internal global variable to allow rsync to work
|
|
## from vps to backup server.
|
|
backup:setup-rsync "$admin" "$vps" "$server" "$id" || return 1
|
|
|
|
if [[ "$path" == "/"* ]]; then
|
|
if ! backup:path_exists "${path}"; then
|
|
err "File or directory '$path' not found in backup."
|
|
return 1
|
|
fi
|
|
if [ -z "$vps_path" ]; then
|
|
if [[ "$path" != *"/" ]] && backup:path_exists "${path}"/ ; then
|
|
path="$path/"
|
|
fi
|
|
vps_path=${path%/}
|
|
vps_path=${vps_path:-/}
|
|
fi
|
|
fi
|
|
|
|
|
|
case "$rtype-$type" in
|
|
mailcow-*)
|
|
## Supports having $path and $vps_path set or unset, with additional behavior
|
|
mailcow:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
|
|
;;
|
|
*-*)
|
|
if [[ "$path" == "/"* ]]; then
|
|
## For now, will require having $path and $vps_path set, no additional behaviors
|
|
file:vps_backup_recover "$admin" "$server" "$id" "$path" "$vps" "$vps_path"
|
|
else
|
|
if [ -n "$path" ]; then
|
|
err "Partial component recover of ${rtype:-unknown} backup type on" \
|
|
"${type:-unknown} type VPS is not yet implemented."
|
|
return 1
|
|
else
|
|
err "Full recover of ${rtype:-unknown} backup type on" \
|
|
"${type:-unknown} type VPS is not yet implemented."
|
|
return 1
|
|
fi
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
}
|
|
|
|
|
|
vps_install_backup() {
|
|
local vps="$1" admin server
|
|
vps_connection_check "$vps" </dev/null || return 1
|
|
|
|
read-0 admin server
|
|
if ! type=$(ssh:run "root@$vps" -- vps get-type); then
|
|
err "Could not get type."
|
|
return 1
|
|
fi
|
|
|
|
if ! out=$(ssh:run "root@$vps" -- vps install backup "$server" 2>&1); then
|
|
err "Command 'vps install backup $server' failed."
|
|
return 1
|
|
fi
|
|
|
|
out="${out%$'\n'}"
|
|
out="${out#*$'\n'}"
|
|
key="${out%\'*}"
|
|
key="${key##*\'}"
|
|
|
|
if ! [[ "$key" =~ ^"ssh-rsa "[a-zA-Z0-9/+]+" "[a-zA-Z0-9._-]+"@"[a-zA-Z0-9._-]+$ ]]; then
|
|
err "Unexpected output from 'vps install backup $server'. Can't find key."
|
|
echo "$out" | prefix " ${GRAY}|$NORMAL " >&2
|
|
echo " Extracted key:" >&2
|
|
echo "$key" | prefix " ${GRAY}|$NORMAL " >&2
|
|
return 1
|
|
fi
|
|
|
|
if [ "$type" == "compose" ]; then
|
|
if ! ssh:run "root@$vps" -- \
|
|
docker exec myc_cron_1 \
|
|
cat /etc/cron.d/rsync-backup >/dev/null 2>&1; then
|
|
ssh:run "root@$vps" -- compose --debug up || {
|
|
err "Command 'compose --debug up' failed."
|
|
return 1
|
|
}
|
|
if ! ssh:run "root@$vps" -- \
|
|
docker exec myc_cron_1 \
|
|
cat /etc/cron.d/rsync-backup >/dev/null 2>&1; then
|
|
err "Launched 'compose up' successfully but ${YELLOW}cron${NORMAL} container is not setup as expected."
|
|
echo " Was waiting for existence of '/etc/cron.d/rsync-backup' in it." >&2
|
|
return 1
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
dest="$server"
|
|
dest="${dest%/*}"
|
|
ssh_options=()
|
|
if [[ "$dest" == *":"* ]]; then
|
|
port="${dest##*:}"
|
|
dest="${dest%%:*}"
|
|
ssh_options=(-p "$port")
|
|
else
|
|
port=""
|
|
dest="${dest%%:*}"
|
|
fi
|
|
|
|
cmd=(ssh "${ssh_options[@]}" "$admin"@"$dest" ssh-key add "$key")
|
|
echo "${WHITE}Launching:${NORMAL} ${cmd[@]}"
|
|
|
|
"${cmd[@]}" || {
|
|
err "Failed add key to backup server '$dest'."
|
|
return 1
|
|
}
|
|
echo "${WHITE}Launching backup${NORMAL} from '$vps'"
|
|
|
|
ssh:run "root@$vps" -- vps backup || {
|
|
err "First backup failed to run."
|
|
return 1
|
|
}
|
|
|
|
echo "Backup is ${GREEN}up and running${NORMAL}."
|
|
}
|
|
|
|
|
|
vps_udpate() {
|
|
local vps="$1"
|
|
vps_connection_check "$vps" || return 1
|
|
ssh:run "root@$vps" -- myc-update </dev/null
|
|
}
|
|
|
|
|
|
vps_bash() {
|
|
local vps="$1"
|
|
vps_connection_check "$vps" </dev/null || return 1
|
|
ssh:run "root@$vps" -- bash
|
|
}
|
|
|
|
vps_mux() {
|
|
local fn="$1" vps_done VPS max_size vps
|
|
shift
|
|
VPS=($(printf "%s\n" "$@" | sort))
|
|
max_size=0
|
|
declare -A vps_done;
|
|
new_vps=()
|
|
for name in "${VPS[@]}"; do
|
|
[ -n "${vps_done[$name]}" ] && {
|
|
warn "duplicate vps '$name' provided. Ignoring."
|
|
continue
|
|
}
|
|
vps_done["$name"]=1
|
|
new_vps+=("$name")
|
|
size_name="${#name}"
|
|
[ "$max_size" -lt "${size_name}" ] &&
|
|
max_size="$size_name"
|
|
done
|
|
settmpdir "_0KM_TMP_DIR"
|
|
cat > "$_0KM_TMP_DIR/code"
|
|
for vps in "${new_vps[@]}"; do
|
|
label=$(printf "%-${max_size}s" "$vps")
|
|
(
|
|
{
|
|
{
|
|
"$fn" "$vps" < "$_0KM_TMP_DIR/code"
|
|
} 3>&1 1>&2 2>&3 | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKRED\!$NORMAL /g"
|
|
set_errlvl "${PIPESTATUS[0]}"
|
|
} 3>&1 1>&2 2>&3 | sed -r "s/^/$DARKCYAN$label$NORMAL $DARKGRAY\|$NORMAL /g"
|
|
set_errlvl "${PIPESTATUS[0]}"
|
|
) &
|
|
done
|
|
wait
|
|
}
|
|
|
|
|
|
[ "$SOURCED" ] && return 0
|
|
|
|
##
|
|
## Command line processing
|
|
##
|
|
|
|
|
|
cmdline.spec.gnu
|
|
cmdline.spec.reporting
|
|
|
|
cmdline.spec.gnu vps-setup
|
|
|
|
cmdline.spec::cmd:vps-setup:run() {
|
|
|
|
: :posarg: HOST 'Target host to check/fix ssh-access'
|
|
|
|
depends sshpass shyaml
|
|
|
|
KEY_PATH="ssh-access.public-keys"
|
|
local keys=$(config get-value -y "ssh-access.public-keys") || true
|
|
if [ -z "$keys" ]; then
|
|
err "No ssh publickeys configured in config file."
|
|
echo " Looking for keys in ${WHITE}${KEY_PATH}${NORMAL}" \
|
|
"in config file." >&2
|
|
config:exists --message 2>&1 | prefix " "
|
|
if [ "${PIPESTATUS[0]}" == "0" ]; then
|
|
echo " Config file found in $(config:filename)"
|
|
fi
|
|
return 1
|
|
fi
|
|
local tkey=$(e "$keys" | shyaml get-type)
|
|
if [ "$tkey" != "sequence" ]; then
|
|
err "Value type of ${WHITE}${KEY_PATH}${NORMAL} unexpected (is $tkey, expecting sequence)."
|
|
echo " Check content of $(config:filename), and make sure to use a sequence." >&2
|
|
return 1
|
|
fi
|
|
|
|
local IP NAME keys host_pass_connected
|
|
if ! IP=$(resolve "$HOST"); then
|
|
err "'$HOST' name unresolvable."
|
|
exit 1
|
|
fi
|
|
|
|
NAME="$HOST"
|
|
if [ "$IP" != "$HOST" ]; then
|
|
NAME="$HOST ($IP)"
|
|
fi
|
|
|
|
if ! is-port-open "$IP" "22"; then
|
|
err "$NAME unreachable or port 22 closed."
|
|
exit 1
|
|
fi
|
|
debug "Host $IP's port 22 is open."
|
|
if ! host_pass_connected=$(ssh:open-try \
|
|
{root,debian}@"$HOST"); then
|
|
err "Could not connect to {root,debian}@$HOST with publickey nor password."
|
|
exit 1
|
|
fi
|
|
|
|
read-0a host password <<<"$host_pass_connected"
|
|
|
|
sudo_if_necessary=
|
|
if [ "$password" -o "${host%%@*}" != "root" ]; then
|
|
if ! ssh:run "$host" -- sudo -n true >/dev/null 2>&1; then
|
|
err "Couldn't do a password-less sudo from $host."
|
|
echo " This is not yet supported."
|
|
exit 1
|
|
else
|
|
sudo_if_necessary=sudo
|
|
fi
|
|
fi
|
|
|
|
Section Checking access
|
|
while read-0 key; do
|
|
prefix="${key%% *}"
|
|
if [ "$prefix" != "ssh-rsa" ]; then
|
|
err "Unsupported key:"$'\n'"$key"
|
|
return 1
|
|
fi
|
|
label="${key##* }"
|
|
Elt "considering adding key ${DARKYELLOW}$label${NORMAL}"
|
|
dest="/root/.ssh/authorized_keys"
|
|
if ssh:run "$host" -- $sudo_if_necessary grep "\"$key\"" "$dest" >/dev/null 2>&1 </dev/null; then
|
|
print_info "already present"
|
|
print_status noop
|
|
Feed
|
|
else
|
|
if echo "$key" | ssh:run "$host" -- $sudo_if_necessary tee -a "$dest" >/dev/null; then
|
|
print_info added
|
|
else
|
|
echo
|
|
Feedback failure
|
|
return 1
|
|
fi
|
|
Feedback success
|
|
fi
|
|
done < <(e "$keys" | shyaml get-values-0)
|
|
|
|
Section Checking ovh hostname file
|
|
Elt "Checking /etc/ovh-hostname"
|
|
if ! ssh:run "$host" -- $sudo_if_necessary [ -e "/etc/ovh-hostname" ]; then
|
|
print_info "creating"
|
|
ssh:run "$host" -- $sudo_if_necessary cp /etc/hostname /etc/ovh-hostname
|
|
ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
|
|
Elt "Checking /etc/ovh-hostname: $ovhname"
|
|
Feedback || return 1
|
|
else
|
|
ovhname=$(ssh:run "$host" -- $sudo_if_necessary cat /etc/ovh-hostname)
|
|
Elt "Checking /etc/ovh-hostname: $ovhname"
|
|
print_info "already present"
|
|
print_status noop
|
|
Feed
|
|
fi
|
|
|
|
if ! is_ovh_domain_name "$HOST" && ! str_is_ipv4 "$HOST"; then
|
|
Section Checking hostname
|
|
Elt "Checking /etc/hostname..."
|
|
if [ "$old" != "$HOST" ]; then
|
|
old="$(ssh:run "$host" -- $sudo_if_necessary cat /etc/hostname)"
|
|
Elt "Hostname is '$old'"
|
|
if is_ovh_hostname "$old"; then
|
|
Elt "Hostname '$old' --> '$HOST'"
|
|
print_info "creating"
|
|
echo "$HOST" | ssh:run "$host" -- $sudo_if_necessary tee /etc/hostname >/dev/null &&
|
|
ssh:run "$host" -- $sudo_if_necessary hostname "$HOST"
|
|
Feedback || return 1
|
|
else
|
|
print_info "not changing"
|
|
print_status noop
|
|
Feed
|
|
fi
|
|
|
|
else
|
|
print_info "already set"
|
|
print_status noop
|
|
Feed
|
|
fi
|
|
else
|
|
info "Not changing domain as '$HOST' doesn't seem to be final domain."
|
|
fi
|
|
|
|
}
|
|
|
|
|
|
cmdline.spec.gnu vps-check
|
|
cmdline.spec::cmd:vps-check:run() {
|
|
|
|
: :posarg: [VPS...] 'Target host(s) to check'
|
|
|
|
|
|
echo "" |
|
|
vps_mux vps_check "${VPS[@]}"
|
|
}
|
|
|
|
|
|
cmdline.spec.gnu vps-install
|
|
cmdline.spec::cmd:vps-install:run() {
|
|
:
|
|
}
|
|
|
|
cmdline.spec.gnu backup
|
|
cmdline.spec:vps-install:cmd:backup:run() {
|
|
|
|
: :posarg: BACKUP_TARGET 'Backup target.
|
|
(ie: myadmin@backup.domain.org:10023/256)'
|
|
: :posarg: [VPS...] 'Target host(s) to check'
|
|
|
|
|
|
if [ "${#VPS[@]}" == 0 ]; then
|
|
warn "VPS list provided in command line is empty. Nothing will be done."
|
|
return 0
|
|
fi
|
|
|
|
if ! [[ "$BACKUP_TARGET" == *"@"* ]]; then
|
|
err "Missing admin account identifier in backup target."
|
|
echo " Have you forgottent to specify an account, ie 'myadmin@<MYBACKUP_SERVER>' ?)"
|
|
return 1
|
|
fi
|
|
|
|
admin=${BACKUP_TARGET%%@*}
|
|
server=${BACKUP_TARGET#*@}
|
|
p0 "$admin" "$server" |
|
|
vps_mux vps_install_backup "${VPS[@]}"
|
|
}
|
|
|
|
|
|
|
|
cmdline.spec.gnu vps-backup
|
|
cmdline.spec::cmd:vps-backup:run() {
|
|
:
|
|
}
|
|
|
|
|
|
cmdline.spec.gnu ls
|
|
cmdline.spec:vps-backup:cmd:ls:run() {
|
|
: :posarg: BACKUP_ID 'Backup id.
|
|
(ie: myadmin@backup.domain.org:10023)'
|
|
|
|
if ! [[ "$BACKUP_ID" == *"@"* ]]; then
|
|
err "Missing admin account identifier in backup id."
|
|
echo " Have you forgottent to specify an admin account ?" \
|
|
"ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
|
|
return 1
|
|
fi
|
|
|
|
id=${BACKUP_ID##*#}
|
|
BACKUP_TARGET=${BACKUP_ID%#*}
|
|
admin=${BACKUP_TARGET%%@*}
|
|
server=${BACKUP_TARGET#*@}
|
|
|
|
## XXXvlab: in this first implementation we expect to have access
|
|
## to the server main ssh port 22, so we won't use the provided port.
|
|
ssh_options=()
|
|
if [[ "$server" == *":"* ]]; then
|
|
ssh_options+=(-p "${server#*:}")
|
|
server=${server%%:*}
|
|
fi
|
|
|
|
ssh "${ssh_options[@]}" "$admin"@"$server" ssh-key ls
|
|
}
|
|
|
|
|
|
cmdline.spec.gnu recover
|
|
cmdline.spec:vps-backup:cmd:recover:run() {
|
|
|
|
: :posarg: BACKUP_ID 'Backup id.
|
|
(ie: myadmin@backup.domain.org:10023#mx.myvps.org
|
|
myadmin@ark-01.org#myid:/a/path
|
|
admin@ark-02.io#myid:myqsl,mailcow)'
|
|
: :posarg: VPS_PATH 'Target host(s) to check.
|
|
(ie: myvps.com
|
|
myvps.com:/a/path)'
|
|
|
|
: :optval: --date,-D '"last", or label of version to recover. (Default: "last").'
|
|
: :optfla: --force,-f 'Will allow you to bypass some checks.'
|
|
|
|
|
|
if ! [[ "$BACKUP_ID" == *"@"* ]]; then
|
|
err "Missing admin account identifier in backup id."
|
|
echo " Have you forgottent to specify an admin account ?" \
|
|
"ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
|
|
return 1
|
|
fi
|
|
if ! [[ "$BACKUP_ID" == *"@"*"#"* ]]; then
|
|
err "Missing backup label identifier in backup id."
|
|
echo " Have you forgottent to specify a backup label identifier ?" \
|
|
"ie 'myadmin@<MYBACKUP_SERVER>#<BACKUP_IDENT>' ?)"
|
|
return 1
|
|
fi
|
|
|
|
id_path=${BACKUP_ID#*#}
|
|
if [[ "$id_path" == *":"* ]]; then
|
|
id=${id_path%%:*}
|
|
path=${id_path#*:}
|
|
else
|
|
id="$id_path"
|
|
path=
|
|
fi
|
|
BACKUP_TARGET=${BACKUP_ID%#*}
|
|
admin=${BACKUP_TARGET%%@*}
|
|
server=${BACKUP_TARGET#*@}
|
|
|
|
ssh_options=()
|
|
if [[ "$server" == *":"* ]]; then
|
|
ssh_options+=(-p "${server#*:}")
|
|
ssh_server=${server%%:*}
|
|
fi
|
|
|
|
BACKUP_PATH="/srv/datastore/data/rsync-backup-target/var/mirror"
|
|
|
|
if ! content=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key ls 2>/dev/null); then
|
|
err "Access denied to '$admin@${server}'."
|
|
return 1
|
|
fi
|
|
|
|
idents=$(echo "$content" | sed -r "s/"$'\e'"\[[0-9]+(;[0-9]+)*m//g" | cut -f 2 -d " ")
|
|
if ! [[ $'\n'"$idents"$'\n' == *$'\n'"$id"$'\n'* ]]; then
|
|
err "Given backup id '$id' not found in $admin@${server}'s idents."
|
|
return 1
|
|
fi
|
|
|
|
rtype=$(ssh:run "$admin"@"${ssh_server}" "${ssh_options[@]}" -- ssh-key get-type "$id" ) &&
|
|
info "Backup archive matches ${WHITE}${rtype}${NORMAL} type"
|
|
|
|
p0 "$admin" "$server" "$id" "$path" "$rtype" "$opt_force" |
|
|
vps_backup_recover "${VPS_PATH}"
|
|
}
|
|
|
|
|
|
|
|
|
|
cmdline.spec.gnu vps-update
|
|
cmdline.spec::cmd:vps-update:run() {
|
|
|
|
: :posarg: [VPS...] 'Target host to check'
|
|
|
|
echo "" |
|
|
vps_mux vps_update "${VPS[@]}"
|
|
}
|
|
|
|
|
|
cmdline.spec.gnu vps-mux
|
|
cmdline.spec::cmd:vps-mux:run() {
|
|
|
|
: :posarg: [VPS...] 'Target host(s) to check'
|
|
|
|
cat | vps_mux vps_bash "${VPS[@]}"
|
|
}
|
|
|
|
|
|
cmdline.spec.gnu vps-space
|
|
cmdline.spec::cmd:vps-space:run() {
|
|
|
|
: :posarg: [VPS...] 'Target host(s) to check'
|
|
|
|
echo "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 2-5 -d ' '" |
|
|
vps_mux vps_bash "${VPS[@]}"
|
|
}
|
|
|
|
|
|
|
|
|
|
cmdline::parse "$@"
|