From 76b5a506a733afb84dbb20723ef115fde2b86ea2 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Wed, 7 Apr 2021 12:15:12 +0200 Subject: [PATCH] new: [vps] add mailcow installation Signed-off-by: Valentin Lab --- bin/mailcow-backup-install | 94 ---------- bin/vps | 364 +++++++++++++++++++++++++++++++++---- 2 files changed, 326 insertions(+), 132 deletions(-) delete mode 100755 bin/mailcow-backup-install diff --git a/bin/mailcow-backup-install b/bin/mailcow-backup-install deleted file mode 100755 index aac28da..0000000 --- a/bin/mailcow-backup-install +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/bash - -. /etc/shlib - -include common -include pretty - - -is-valid-mailcow-root-dir() { - local dir="$1" - ( - [ -e "$dir" ] && - cd "$dir" && - [ -e ".git" ] - ) -} - -## find installation -MAILCOW_ROOT=${MAILCOW_ROOT:-/opt/mailcow-dockerized} -MYSQL_CONTAINER=${MYSQL_CONTAINER:-mailcowdockerized_mysql-mailcow_1} - - -## check ok - -is-valid-mailcow-root-dir "$MAILCOW_ROOT" || { - err "Directory '$MAILCOW_ROOT' is not a valid mailcow root installation directory." - echo " You might want to setup \$MAILCOW_ROOT to the proper" \ - "location of your mailcow root install" >&2 - exit 1 -} - - - -BACKUP_SERVER=${BACKUP_SERVER:-core-06.0k.io:10023} -DOMAIN=$(cat "$MAILCOW_ROOT/.env" | grep ^MAILCOW_HOSTNAME= | cut -f 2 -d =) || { - err "Couldn't find MAILCOW_ROOT in file \"$MAILCOW_ROOT/.env\"." - exit 1 -} - - -## get MYSQL_ROOT_PASSWORD - -MYSQL_ROOT_PASSWORD=$(cat "$MAILCOW_ROOT/.env" | grep ^DBROOT= | cut -f 2 -d =) || { - err "Couldn't find DBROOT in file \"$MAILCOW_ROOT/.env\"." - exit 1 -} - - -container_id=$(docker ps -f name="$MYSQL_CONTAINER" --format "{{.ID}}") -if [ -z "$container_id" ]; then - err "Couldn't find docker container named '$MYSQL_CONTAINER'." - exit 1 -fi - - -export MYSQL_ROOT_PASSWORD -export MYSQL_CONTAINER -export BACKUP_SERVER -export DOMAIN - - -( - cd /srv/charm-store/rsync-backup - bash ./hooks/install.d/60-install.sh -) || { - echo "rsync-backup failed to install." - exit 1 -} - -( - cd /srv/charm-store/mysql - bash ./hooks/install.d/60-backup.sh -) || { - echo "mysql dump failed to install." - exit 1 -} - -if ! sources=$(shyaml get-values default.sources < /etc/mirror-dir/config.yml); then - echo "Couldn't query 'default.sources' in '/etc/mirror-dir/config.yml'." >&2 - exit 1 -fi - -if ! echo "$sources" | grep "^/var/lib/docker/volumes/\*_vmail{,-attachments-vol}-\*/_data$" 2>/dev/null; then - sed -i '/sources:/a\ - "/var/lib/docker/volumes/*_vmail{,-attachments-vol}-*/_data"' \ - /etc/mirror-dir/config.yml -fi - - -dest="$BACKUP_SERVER" -dest="${dest%/*}" -dest="${dest%%:*}" - -echo "Contacting '$dest' host, to add key in authorized key:" -ssh "root@${dest}" -- compose-add-rsync-key "\"$DOMAIN\"" "\"$(cat /var/lib/rsync/.ssh/id_rsa.pub)\"" diff --git a/bin/vps b/bin/vps index 2d7ab94..ffcc8c2 100755 --- a/bin/vps +++ b/bin/vps @@ -7,6 +7,8 @@ include common include parse include cmdline include config +include cache +include fn [[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true @@ -16,50 +18,228 @@ desc='Install backup' help="" +docker:running-container-projects() { + :cache: scope=session + docker ps --format '{{.Label "com.docker.compose.project"}}' | sort | uniq +} +decorator._mangle_fn docker:running-container-projects -[ "$SOURCED" ] && return 0 -## -## Command line processing -## +ssh:mk-private-key() { + local host="$1" service_name="$2" + ( + settmpdir VPS_TMPDIR + ssh-keygen -t rsa -N "" -f "$VPS_TMPDIR/rsync_rsa" -C "$service_name@$host" >/dev/null + cat "$VPS_TMPDIR/rsync_rsa" + ) +} -cmdline.spec.gnu -cmdline.spec.reporting +mailcow:has-images-running() { + local images + images=$(docker ps --format '{{.Image}}' | sort | uniq) + [[ $'\n'"$images" == *$'\n'"mailcow/"* ]] +} -cmdline.spec.gnu install -cmdline.spec.gnu backup +mailcow:has-container-project-mentionning-mailcow() { + local projects + projects=$(docker:running-container-projects) || return 1 + [[ $'\n'"$projects"$'\n' == *mailcow* ]] +} -cmdline.spec::cmd:install:run() { - : +mailcow:has-running-containers() { + mailcow:has-images-running || + mailcow:has-container-project-mentionning-mailcow } +mailcow:get-root() { + :cache: scope=session -cmdline.spec:install:cmd:backup:run() { + local dir - : :posarg: BACKUP_SERVER 'Target backup server' + for dir in {/opt{,/apps},/root}/mailcow-dockerized; do + [ -d "$dir" ] || continue + [ -r "$dir/mailcow.conf" ] || continue + echo "$dir" + return 0 + done + return 1 +} +decorator._mangle_fn mailcow:get-root - : :optval: --service-name,-s "YAML service name in compose - file to check for existence of key. - Defaults to 'rsync-backup'" - : :optval: --compose-file,-f "Compose file location. Defaults to - the value of '\$DEFAULT_COMPOSE_FILE'" - local service_name compose_file +compose:get-compose-yml() { + :cache: scope=session - [ -e "/etc/compose/local.conf" ] && source /etc/compose/local.conf + local path + [ -e "/etc/compose/local.conf" ] && . "/etc/compose/local.conf" - compose_file=${opt_compose_file:-$DEFAULT_COMPOSE_FILE} - service_name=${opt_service_name:-rsync-backup} + path=${DEFAULT_COMPOSE_FILE:-/etc/compose/compose.yml} - if ! [ -e "$compose_file" ]; then - err "Compose file not found in '$compose_file'." + [ -e "$path" ] || return 1 + echo "$path" +} +decorator._mangle_fn compose:get-compose-yml + + +compose:has-container-project-myc() { + local projects + projects=$(docker:running-container-projects) || return 1 + [[ $'\n'"$projects"$'\n' == *$'\n'"myc"$'\n'* ]] +} + + +type:is-mailcow() { + mailcow:get-root >/dev/null || + mailcow:has-running-containers +} + + +type:is-compose() { + compose:get-compose-yml >/dev/null && + compose:has-container-project-myc +} + + +vps:get-type() { + local fn + for fn in $(declare -F | cut -f 3 -d " " | egrep "^type:is-"); do + "$fn" && { + echo "${fn#type:is-}" + return 0 + } + done + return 1 +} + + +mirror-dir:sources() { + :cache: scope=session + + if ! shyaml get-values default.sources < /etc/mirror-dir/config.yml; then + err "Couldn't query 'default.sources' in '/etc/mirror-dir/config.yml'." return 1 fi +} +decorator._mangle_fn mirror-dir:sources + + +mirror-dir:check-add() { + local elt="$1" sources + sources=$(mirror-dir:sources) || return 1 + if [[ $'\n'"$sources"$'\n' == *$'\n'"$elt"$'\n'* ]]; then + info "Volume $elt already in sources" + else + Elt "Adding directory $elt" + sed -i "/sources:/a\ - \"${elt}\"" \ + /etc/mirror-dir/config.yml + Feedback || return 1 + fi +} + + +mirror-dir:check-add-vol() { + local elt="$1" + mirror-dir:check-add "/var/lib/docker/volumes/*_${elt}-*/_data" +} + +## The first colon is to prevent auto-export of function from shlib +: ; bash-bug-5() { { cat; } < <(e) >/dev/null; ! cat "$1"; } && bash-bug-5 <(e) 2>/dev/null && + export BASH_BUG_5=1 && unset -f bash_bug_5 + + +wrap() { + local label="$1" code="$2" + shift 2 + export VERBOSE=1 + interpreter=/bin/bash + if [ -n "$BASH_BUG_5" ]; then + ( + settmpdir tmpdir + fname=${label##*/} + e "$code" > "$tmpdir/$fname" && + chmod +x "$tmpdir/$fname" && + Wrap -vsd "$label" -- "$interpreter" "$tmpdir/$fname" "$@" + ) + else + Wrap -vsd "$label" -- "$interpreter" <(e "$code") "$@" + fi +} + + +mailcow:install-backup() { + + local BACKUP_SERVER="$1" mailcow_root DOMAIN + + ## find installation + mailcow_root=$(mailcow:get-root) || { + err "Couldn't find a valid mailcow root directory." + return 1 + } + + + ## check ok + + DOMAIN=$(cat "$mailcow_root/.env" | grep ^MAILCOW_HOSTNAME= | cut -f 2 -d =) || { + err "Couldn't find MAILCOW_HOSTNAME in file \"$mailcow_root/.env\"." + return 1 + } + + + MYSQL_ROOT_PASSWORD=$(cat "$mailcow_root/.env" | grep ^DBROOT= | cut -f 2 -d =) || { + err "Couldn't find DBROOT in file \"$mailcow_root/.env\"." + return 1 + } + + MYSQL_CONTAINER=${MYSQL_CONTAINER:-mailcowdockerized_mysql-mailcow_1} + container_id=$(docker ps -f name="$MYSQL_CONTAINER" --format "{{.ID}}") + if [ -z "$container_id" ]; then + err "Couldn't find docker container named '$MYSQL_CONTAINER'." + return 1 + fi + + + export MYSQL_ROOT_PASSWORD + export MYSQL_CONTAINER + export BACKUP_SERVER + export DOMAIN + + wrap "Install rsync-backup on host" " + cd /srv/charm-store/rsync-backup + bash ./hooks/install.d/60-install.sh +" || return 1 + + wrap "Mysql dump install" " + cd /srv/charm-store/mariadb + bash ./hooks/install.d/60-backup.sh +" || return 1 + + ## Using https://github.com/mailcow/mailcow-dockerized/blob/master/helper-scripts/backup_and_restore.sh + for elt in "vmail{,-attachments-vol}" crypt redis rspamd postfix; do + mirror-dir:check-add-vol "$elt" || return 1 + done + + mirror-dir:check-add "$mailcow_root" || return 1 + mirror-dir:check-add "/var/backups/mysql" || return 1 + mirror-dir:check-add "/etc" || return 1 + + + dest="$BACKUP_SERVER" + dest="${dest%/*}" + dest="${dest%%:*}" + + info "You should add key on '$dest' host:" + echo compose-add-rsync-key -R "\"$DOMAIN\"" "\"$(cat /var/lib/rsync/.ssh/id_rsa.pub)\"" + +} + +compose:install-backup() { + + local BACKUP_SERVER="$1" service_name="$2" compose_file="$3" force="$4" ## XXXvlab: far from perfect as it mimics and depends internal ## logic of current default way to get a domain in compose-core @@ -70,39 +250,147 @@ cmdline.spec:install:cmd:backup:run() { return 1 fi - ip=$(getent ahosts "$host" | egrep "^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+" | head -n 1 | cut -f 1 -d " ") || return 1 + ip=$(getent ahosts "$host" | egrep "^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+" | + head -n 1 | cut -f 1 -d " ") || return 1 + my_ip=$(curl -s myip.kal.fr) if [ "$ip" != "$my_ip" ]; then - err "IP of '$host' ($ip) doesn't match mine ($my_ip)." - return 1 + if [ -n "$force" ]; then + warn "IP of '$host' ($ip) doesn't match mine ($my_ip). Ignoring due to ``-f`` option." + else + err "IP of '$host' ($ip) doesn't match mine ($my_ip). Use ``-f`` to force." + return 1 + fi fi if [ -e "/root/.ssh/rsync_rsa" ]; then - if ! [ -e "/root/.ssh/rsync_rsa.pub" ]; then - err "Didn't find public key in '/root/.ssh/rsync_rsa.pub'. (Private key is present !)." + warn "deleting private key in /root/.ssh/rsync_rsa, has we are not using it anymore." + rm -fv /root/.ssh/rsync_rsa + fi + if [ -e "/root/.ssh/rsync_rsa.pub" ]; then + warn "deleting public key in /root/.ssh/rsync_rsa.pub, has we are not using it anymore." + rm -fv /root/.ssh/rsync_rsa.pub + fi + + if service_cfg=$(cat "$compose_file" | + shyaml get-value -y "$service_name" 2>/dev/null); then + info "Entry for service ${DARKYELLOW}$service_name${NORMAL}" \ + "is already present in '$compose_file'." + cfg=$(e "$service_cfg" | shyaml get-value -y options) || { + err "No ${WHITE}options${NORMAL} in ${DARKYELLOW}$service_name${NORMAL}'s" \ + "entry in '$compose_file'." + return 1 + } + private_key=$(e "$cfg" | shyaml get-value private-key) + target=$(e "$cfg" | shyaml get-value target) + if [ "$target" != "$BACKUP_SERVER" ]; then + err "Existing backup target '$target' is different" \ + "from specified '$BACKUP_SERVER'" return 1 fi else - Wrap -d "Creating rsync key pair" -- \ - ssh-keygen -t rsa -N \"\" -f /root/.ssh/rsync_rsa -C "rsync@$host" - fi + private_key=$(ssh:mk-private-key "$host" "$service_name") - if egrep "^$service_name:" "$compose_file" >/dev/null; then - err "There's already a backup service named '$service_name'" - return 1 - fi - - cat <> "$compose_file" + cat <> "$compose_file" $service_name: options: ident: $host target: $BACKUP_SERVER private-key: | -$(cat /root/.ssh/rsync_rsa | sed -r 's/^/ /g') +$(e "$private_key" | sed -r 's/^/ /g') EOF + fi + + info "You can run this following command on $BACKUP_SERVER:" + public_key=$(ssh-keygen -y -f <(e "$private_key")) + echo "compose-add-rsync-key -R '$host' '$public_key ${service_name}@$host'" + +} + + +[ "$SOURCED" ] && return 0 + +## +## Command line processing +## + + +cmdline.spec.gnu +cmdline.spec.reporting + +cmdline.spec.gnu install +cmdline.spec.gnu backup + + +cmdline.spec::cmd:install:run() { + + : +} + + +cmdline.spec.gnu get-type +cmdline.spec::cmd:get-type:run() { + vps:get-type +} + +cmdline.spec:install:cmd:backup:run() { + + : :posarg: BACKUP_SERVER 'Target backup server' + + local vps_type + + vps_type=$(vps:get-type) || { + err "Failed to get type of installation." + return 1 + } + if ! fn.exists "${vps_type}:install-backup"; then + err "type '${vps_type}' has no backup installation implemented yet." + return 1 + fi + + "cmdline.spec:install:cmd:$vps_type-backup:run" "$BACKUP_SERVER" +} + + +DEFAULT_BACKUP_SERVICE_NAME=rsync-backup +cmdline.spec:install:cmd:compose-backup:run() { + + : :posarg: BACKUP_SERVER 'Target backup server' + + : :optval: --service-name,-s "YAML service name in compose + file to check for existence of key. + Defaults to '$DEFAULT_BACKUP_SERVICE_NAME'" + : :optval: --compose-file,-f "Compose file location. Defaults to + the value of '\$DEFAULT_COMPOSE_FILE'" + + : :optval: --force,-F "Compose file location. Defaults to + the value of '\$DEFAULT_COMPOSE_FILE'" + + local service_name compose_file + + + [ -e "/etc/compose/local.conf" ] && source /etc/compose/local.conf + + compose_file=${opt_compose_file:-$DEFAULT_COMPOSE_FILE} + service_name=${opt_service_name:-$DEFAULT_BACKUP_SERVICE_NAME} + + if ! [ -e "$compose_file" ]; then + err "Compose file not found in '$compose_file'." + return 1 + fi + + compose:install-backup "$BACKUP_SERVER" "$service_name" "$compose_file" "$opt_force" + +} + + +cmdline.spec:install:cmd:mailcow-backup:run() { + + : :posarg: BACKUP_SERVER 'Target backup server' + "mailcow:install-backup" "$BACKUP_SERVER" }