new: [mailcow-backup-install] new script

Signed-off-by: Valentin Lab <valentin.lab@kalysto.org>
4 years ago · e5ab06a200
2 changed files with 122 additions and 165 deletions
--- a/README.org
+++ b/README.org
@ -615,137 +615,28 @@ tail -f /srv/datastore/data/cron/var/log/cron/letsencrypt-renew_script.log -n 20

 *** mysql

-**** docker sans compose
-
-***** backup régulier depuis l'hôte
-
-#+begin_src sh
-MYSQL_ROOT_PASSWORD=xxx
-MYSQL_CONTAINER=mailcowdockerized_mysql-mailcow_1
-
-
-apt-get install -y mysql-client </dev/null
-
-cat <<EOF > ~/.my.cnf
-[client]
-password=${MYSQL_ROOT_PASSWORD}
-EOF
-chmod 600 ~/.my.cnf
-
-cat <<'EOF' > /usr/local/sbin/mysql-backup
-#!/bin/bash
-
-. /etc/shlib
-
-include common
-include pretty
-
-usage="$exname [--host HOST] [DATABASE...]"
-
-
-DBS=()
-host=
-while [ "$1" ]; do
-    case "$1" in
-        "--help"|"-h")
-            print_usage
-            exit 0
-        ;;
-        "--host")
-            host="$2"
-            shift
-        ;;
-        *)
-            DBS+=("$1")
-        ;;
-    esac
-    shift
-done
-
-
-mysql_opts=()
-if [ "$host" ]; then
-    mysql_opts+=(-h "$host")
-fi
-
-m() {
-    mysql "${mysql_opts[@]}" -Bs "$@"
-}
-
-md() {
-    mysqldump "${mysql_opts[@]}" "$@"
-}
+**** sur installation mailcow

-mysql_databases() {
-     echo "SHOW DATABASES" | m
-}
-
-mysql_tables() {
-    local db="$1"
-     echo "SHOW TABLES" | m "$db"
-}
-
-
-if [ "${#DBS[@]}" == 0 ]; then
-    DBS=($(mysql_databases)) || exit 1
-fi
-
-mkdir -p /var/backups/mysql
-
-for db in "${DBS[@]}"; do
-    if [[ "$db" == "information_schema" || "$db" == "performance_schema" || "$db" == "mysql" ]]; then
-        continue
-    fi
-    echo "Dumping database $db..." >&2
-    # omitting all the rotation logic
-    dst=/"var/backups/mysql/$db"
-    [ -d "$dst.old" ] && rm -rf "$dst.old"
-    [ -d "$dst" ] && mv "$dst" "$dst.old"
-    mkdir -p "$dst.inprogress"
-    (( start = SECONDS ))
-    md "$db" --routines --no-data --add-drop-database --database "$db" | gzip --rsyncable > "$dst.inprogress/schema.sql.gz"
-    tables=$(mysql_tables "$db")
-    for table in $tables; do
-        backup_file="$dst.inprogress/${table}.sql.gz"
-        echo "  Dumping $table into ${backup_file}"
-        md "$db" "$table" | gzip --rsyncable > "$backup_file" || break
-    done
-    mv "$dst.inprogress" "$dst"
-    [ -d "$dst.old" ] && rm -rf "$dst.old"
-    (( elapsed = SECONDS - start ))
-    echo "  ..dumped $db to $dst ($(du -sh "$dst" | cut -f 1) in ${elapsed}s)" >&2
-done
-EOF
-chmod +x /usr/local/sbin/mysql-backup
+Le script =mailcow-backup-install= fourni via =myc-manage= s'occupe
+de mettre en place le dump de =mysql=, et le système pour envoyer les
+backup régulièrement via rsync.

-cat <<EOF > /etc/cron.d/mysql-backup
-SHELL=/bin/bash
-PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
-
-0 * * * *   root   /usr/local/sbin/mysql-backup --host \$(docker-ip "$MYSQL_CONTAINER" 2>/dev/null | sed -r 's/ +/ /g' | cut -f 3 -d " ") | logger -t mysql-backup
-
-EOF
+**** docker sans compose

-mkdir -p /etc/mirror-dir &&
-cat <<EOF >> /etc/mirror-dir/mirror-dir.conf
-/var/backups/mysql:
-  exclude:
-    - "/*.inprogress"
-EOF
+#+begin_src sh
+export MYSQL_ROOT_PASSWORD=xxx
+export MYSQL_CONTAINER=mailcowdockerized_mysql-mailcow_1

+/srv/charm-store/mysql/hooks/install.d/60-backup.sh
 #+end_src


-
 *** rsync-backup

 **** Installation du backup via compose

-A faire depuis le serveur ayant des services à backupper.
-
-Il y a une partie à copier-coller sur le serveur de backup.
-
-Ne pas oublier de relancer le serveur de backup pour le prendre en compte.
+A faire depuis le serveur ayant des services géré par =compose= à
+backupper.

 #+begin_src sh
 DOMAIN=xxx.fr
@ -762,68 +653,42 @@ rsync-backup:
 $(cat /root/.ssh/rsync_rsa | sed -r 's/^/      /g')
 EOF

-echo "copier-coller ceci dans le '/etc/compose/compose.yml' de ${BACKUP_SERVER%:*}:"
-echo
-cat <<EOF
-      $DOMAIN: "$(
-    cat /root/.ssh/rsync_rsa.pub |
-        fold -w 67 | sed -r 's/$/\\/g;2,$ s/^/ /g;$,$ s/\\$//g')"
+dest="$BACKUP_SERVER"
+dest="${dest%/*}"
+dest="${dest%%:*}"
+echo "Contacting '$dest' host, to add key in authorized key:"
+ssh "root@${dest}" -- compose-add-rsync-key "\"$DOMAIN\"" "\"$(cat /root/.ssh/rsync_rsa.pub)\""

-EOF

 compose build mongo ## force the build of the new mongo image with regular dumps
 #+end_src

-**** Installation du backup sur un host debian
+**** Installation du backup sur une installe mailcow

-Cela fonctionnera sur tout host ayant une base debian.
+Le script suivant va s'occuper de tout, il doit être lancé
+depuis l'hôte faisant tourner l'installation =mailcow=.

 #+begin_src sh
-DOMAIN=xxx.fr
-BACKUP_SERVER=core-06.0k.io:10023
-## les répertoires à sauvegarder
-DESTS=(
-  /var/backups
-  /etc
-)
-
-## installation de rsync
-apt-get install -y rsync </dev/null
-
-## Création de l'utilisateur rsync
-mkdir -p /var/lib/rsync && \
-groupadd -r rsync && \
-useradd -r rsync -d /var/lib/rsync -g rsync && \
-chown rsync:rsync /var/lib/rsync
-
-## Création de la clé ssh pour rsync
-su -c 'ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa -q -C "rsync@'"$DOMAIN" - rsync
+mailcow-backup-install
+#+end_src

-echo "copier-coller ceci dans le '/etc/compose/compose.yml' de ${BACKUP_SERVER%:*}:"
-echo
-cat <<EOF
-      $DOMAIN: "$(
-    cat /var/lib/rsync/.ssh/rsync_rsa.pub |
-        fold -w 67 | sed -r 's/$/\\/g;2,$ s/^/ /g;$,$ s/\\$//g')"
+**** Installation du backup sur un host debian

-EOF
+Cela fonctionnera sur tout host ayant une base debian.

-cat <<EOF > /etc/cron.d/mirror-dir
-SHELL=/bin/bash
-PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+#+begin_src sh
+export DOMAIN=mail.myceliandre.fr
+export BACKUP_SERVER=core-06.0k.io:10023

-$((RANDOM % 60)) * * * *   root   mirror-dir -d $BACKUP_SERVER -u rsync ${DESTS[@]} 2>&1 | logger -t mirror-dir
-EOF
+/srv/charm-store/rsync-backup/hooks/install.d/60-install.sh

 #+end_src

-les logs sont dans =/var/log/mirror.log=.
-
 Note, il est possible de spécifier des exclusions pour chaque
 répértoire mirroré de telle façon:

 #+begin_src sh
-cat <<EOF > /etc/mirror-dir/mirror-dir.conf
+cat <<EOF > /etc/mirror-dir/config.yml
 /home:
    exclude:
        - /*/.cache/
@ -843,8 +708,6 @@ EOF
 #+end_src


-
-
 * Interventions avancées

 Description des process avancés d'intervention sur une installation existante.
--- a/bin/mailcow-backup-install
+++ b/bin/mailcow-backup-install
@ -0,0 +1,94 @@
+#!/bin/bash
+
+. /etc/shlib
+
+include common
+include pretty
+
+
+is-valid-mailcow-root-dir() {
+    local dir="$1"
+    (
+        [ -e "$dir" ] &&
+        cd "$dir" &&
+        [ -e ".git" ]
+    )
+}
+
+## find installation
+MAILCOW_ROOT=${MAILCOW_ROOT:-/opt/mailcow-dockerized}
+MYSQL_CONTAINER=${MYSQL_CONTAINER:-mailcowdockerized_mysql-mailcow_1}
+
+
+## check ok
+
+is-valid-mailcow-root-dir "$MAILCOW_ROOT" || {
+    err "Directory '$MAILCOW_ROOT' is not a valid mailcow root installation directory."
+    echo "  You might want to setup \$MAILCOW_ROOT to the proper" \
+         "location of your mailcow root install" >&2
+    exit 1
+}
+
+
+
+BACKUP_SERVER=${BACKUP_SERVER:-core-06.0k.io:10023}
+DOMAIN=$(cat "$MAILCOW_ROOT/.env" | grep ^MAILCOW_HOSTNAME= | cut -f 2 -d =) || {
+    err "Couldn't find MAILCOW_ROOT in file \"$MAILCOW_ROOT/.env\"."
+    exit 1
+}
+
+
+## get MYSQL_ROOT_PASSWORD
+
+MYSQL_ROOT_PASSWORD=$(cat "$MAILCOW_ROOT/.env" | grep ^DBROOT= | cut -f 2 -d =) || {
+    err "Couldn't find DBROOT in file \"$MAILCOW_ROOT/.env\"."
+    exit 1
+}
+
+
+container_id=$(docker ps -f name="$MYSQL_CONTAINER" --format "{{.ID}}")
+if [ -z "$container_id" ]; then
+    err "Couldn't find docker container named '$MYSQL_CONTAINER'."
+    exit 1
+fi
+
+
+export MYSQL_ROOT_PASSWORD
+export MYSQL_CONTAINER
+export BACKUP_SERVER
+export DOMAIN
+
+
+(
+    cd /srv/charm-store/rsync-backup
+    bash ./hooks/install.d/60-install.sh
+) || {
+    echo "rsync-backup failed to install."
+    exit 1
+}
+
+(
+    cd /srv/charm-store/mysql
+    bash ./hooks/install.d/60-backup.sh
+) || {
+    echo "mysql dump failed to install."
+    exit 1
+}
+
+if ! sources=$(shyaml get-values default.sources < /etc/mirror-dir/config.yml); then
+    echo "Couldn't query 'default.sources' in '/etc/mirror-dir/config.yml'." >&2
+    exit 1
+fi
+
+if ! echo "$sources" | grep "^/var/lib/docker/volumes/\*_vmail{,-attachments-vol}-\*/_data$" 2>/dev/null; then
+    sed -i '/sources:/a\  - "/var/lib/docker/volumes/*_vmail{,-attachments-vol}-*/_data"' \
+        /etc/mirror-dir/config.yml
+fi
+
+
+dest="$BACKUP_SERVER"
+dest="${dest%/*}"
+dest="${dest%%:*}"
+
+echo "Contacting '$dest' host, to add key in authorized key:"
+ssh "root@${dest}" -- compose-add-rsync-key "\"$DOMAIN\"" "\"$(cat /var/lib/rsync/.ssh/id_rsa.pub)\""