new: [rsync-backup-target] add recover key mechanism

This allows to get a one-time credential to have read access on
specified backup slot.

Signed-off-by: Valentin Lab <valentin.lab@kalysto.org>

rsync-backup-target/README.org

@@ -55,6 +55,24 @@ $ ssh myadmin@$RSYNC_BACKUP_TARGET ssh-key ls
 $
 #+end_example
 
+** Requesting a recover only key
+
+As an admin, by requesting a recover-only key on an ident that you
+own, you are allowed to read (and only read) the content of the given
+ident. This will allow you to give the credentials to any new host to
+have a direct read access so-as to deploy the backup on a new host.
+
+#+begin_example
+$ ssh myadmin@$RSYNC_BACKUP_TARGET ssh-key request-recovery-key myident > /tmp/private_key
+$ chmod 500 /tmp/private_key
+$ rsync -e "ssh -p 22 -i /tmp/private_key -l rsync" \
+      -azvArH --delete --delete-excluded \
+      --partial --partial-dir .rsync-partial \
+      --numeric-ids $RSYNC_BACKUP_TARGET:/var/mirror/myident/etc/ /tmp/etc
+#+end_example
+
+This key will expire after 15 mn of the last recovery.
+
 
 * Troubleshooting
 

rsync-backup-target/build/Dockerfile

@@ -3,7 +3,8 @@ FROM alpine:3.9
 MAINTAINER Valentin Lab <valentin.lab@kalysto.org>
 
 ## coreutils is for ``date`` support of ``--rfc-3339=seconds`` argument.
-RUN apk add rsync sudo bash openssh-server coreutils
+## findutils is for ``find`` support of ``--newermt`` argument.
+RUN apk add rsync sudo bash openssh-server coreutils findutils
 RUN ssh-keygen -A
 
 ## New user/group rsync/rsync with home dir in /var/lib/rsync

rsync-backup-target/build/entrypoint.sh

@@ -16,7 +16,7 @@ if ! egrep '^[^:]+:x:101:101:' /etc/passwd; then
     ## Then it is a first run of this container, users
     ## need to be created. Notice that container will be
     ## re-created anew if user config was changed.
-    for user_dir in /etc/rsync/keys/admin/*; do
+    for user_dir in /etc/rsync/keys/admin/* /etc/rsync/keys/recover; do
         [ -d "$user_dir" ] || continue
         user="${user_dir##*/}"
         [ "$user" != "rsync" ] || continue

rsync-backup-target/build/src/etc/sudoers.d/recover

@@ -0,0 +1,7 @@
+## allow admin users to request a recovery key, this is really not
+## sufficient, but the real check is done on the
+## ``ssh-admin-cmd-validate`` side.
+
+%rsync ALL=(root) NOPASSWD: /usr/local/sbin/request-recovery-key *
+%rsync ALL=(root) NOPASSWD: /bin/touch /etc/rsync/keys/recover/*
+%rsync ALL=(root) NOPASSWD: /usr/local/sbin/ssh-update-keys

rsync-backup-target/build/src/etc/sudoers.d/rsync

@@ -4,3 +4,4 @@
 rsync ALL=(root) NOPASSWD: /usr/bin/rsync --server * . /var/mirror/*
 
 %rsync ALL=(root) NOPASSWD: /usr/local/sbin/ssh-key *
+%rsync ALL=(root) NOPASSWD: /usr/local/sbin/ssh-update-keys

rsync-backup-target/build/src/usr/local/sbin/request-recovery-key

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+RSYNC_KEY_PATH=/etc/rsync/keys
+RECOVER_KEY_PATH=${RSYNC_KEY_PATH}/recover
+
+
+ANSI_ESC=$'\e['
+
+NORMAL="${ANSI_ESC}0m"
+
+GRAY="${ANSI_ESC}1;30m"
+RED="${ANSI_ESC}1;31m"
+GREEN="${ANSI_ESC}1;32m"
+YELLOW="${ANSI_ESC}1;33m"
+BLUE="${ANSI_ESC}1;34m"
+PINK="${ANSI_ESC}1;35m"
+CYAN="${ANSI_ESC}1;36m"
+WHITE="${ANSI_ESC}1;37m"
+
+DARKGRAY="${ANSI_ESC}0;30m"
+DARKRED="${ANSI_ESC}0;31m"
+DARKGREEN="${ANSI_ESC}0;32m"
+DARKYELLOW="${ANSI_ESC}0;33m"
+DARKBLUE="${ANSI_ESC}0;34m"
+DARKPINK="${ANSI_ESC}0;35m"
+DARKCYAN="${ANSI_ESC}0;36m"
+DARKWHITE="${ANSI_ESC}0;37m"
+
+
+ssh:mk-private-key() {
+    local comment="$1"
+    (
+        tmpdir=$(mktemp -d)
+        chmod go-rwx "$tmpdir"
+        ssh-keygen -t rsa -N "" -f "$tmpdir/rsync_rsa" -C "$service_name@$host" >/dev/null
+        cat "$tmpdir/rsync_rsa"
+        rm -rf "$tmpdir"
+    )
+}
+
+
+md5() {
+    local md5
+    md5=$(cat | md5sum)
+    echo "${md5%% *}"
+}
+
+
+request-recovery-key() {
+    local label="$1" ident="$2" key public_key
+
+    ## Admin should have claimed the ident with at least one backup key
+    if ! [ -e "${RSYNC_KEY_PATH}/backup/$label/$ident.pub" ]; then
+        echo "Error: Current admin '$label' has no ident '$ident' claimed." >&2
+        return 1
+    fi
+
+    ## Find new label
+    while true; do
+        key=$(ssh:mk-private-key "recover@$ident")
+        md5=$(printf "%s" "$key" | md5)
+        [ -e "${RECOVER_KEY_PATH}/$md5" ] || break
+    done
+
+    mkdir -p "${RECOVER_KEY_PATH}"
+    public_key=$(ssh-keygen -y -f <(printf "%s\n" "$key"))
+    printf "%s %s\n" "$public_key" "recover@$ident" > "${RECOVER_KEY_PATH}/$md5.pub"
+    touch "${RECOVER_KEY_PATH}/$md5"
+    chmod go-rwx "${RECOVER_KEY_PATH}/$md5"
+    printf "%s\n" "$key" | tee -a "${RECOVER_KEY_PATH}/$md5"
+
+    /usr/local/sbin/ssh-update-keys
+}
+
+
+request-recovery-key "$@"

rsync-backup-target/build/src/usr/local/sbin/ssh-admin-cmd-validate

@@ -71,6 +71,15 @@ elif [[ "$SSH_ORIGINAL_COMMAND" =~ ^"ssh-key rm "[a-zA-Z0-9._-]+$ ]]; then
 
     # echo "Would accept: $SSH_ORIGINAL_COMMAND" >&2
 	exec sudo /usr/local/sbin/ssh-key rm "$label" "${ssh_args[@]:2}"
+elif [[ "$SSH_ORIGINAL_COMMAND" =~ ^"request-recovery-key "[a-zA-Z0-9._-]+$ ]]; then
+	log "ACCEPTED: $SSH_ORIGINAL_COMMAND"
+
+    ## Interpret \ to allow passing spaces (want to avoid possible issue with \n)
+    #read -a ssh_args <<< "${SSH_ORIGINAL_COMMAND}"
+    ssh_args=(${SSH_ORIGINAL_COMMAND})
+
+    # echo "Would accept: $SSH_ORIGINAL_COMMAND" >&2
+	exec sudo /usr/local/sbin/request-recovery-key "$label" "${ssh_args[@]:1}"
 else
 
     log "NOT MATCHING ANY ALLOWED COMMAND"

rsync-backup-target/build/src/usr/local/sbin/ssh-cmd-validate

@@ -19,7 +19,7 @@ log() {
            >> "$LOG"
 }
 
-log "NEW CONNECTION"
+log "NEW BACKUP CONNECTION"
 
 if [ -z "$1" ] || ! [[ "$1" =~ ^[a-zA-Z0-9._-]+$ ]]; then
     log "INVALID SETUP, ARG IS: '$1'"
@@ -37,6 +37,7 @@ reject() {
     exit 1
 }
 
+sudo /usr/local/sbin/ssh-update-keys
 
 if [[ "$SSH_ORIGINAL_COMMAND" =~ [\&\(\{\;\<\>\`\$\}] ]]; then
     log "BAD CHARS DETECTED"

rsync-backup-target/build/src/usr/local/sbin/ssh-recover-cmd-validate

@@ -0,0 +1,97 @@
+#!/bin/bash
+
+## Note that the shebang is not used, but it's the login shell that
+## will execute this command.
+
+RSYNC_KEY_PATH=/etc/rsync/keys
+RECOVER_KEY_PATH=${RSYNC_KEY_PATH}/recover
+
+exname=$(basename "$0")
+
+mkdir -p /var/log/rsync
+
+LOG="/var/log/rsync/$exname.log"
+
+
+ssh_connection=(${SSH_CONNECTION})
+SSH_SOURCE_IP="${ssh_connection[0]}:${ssh_connection[1]}"
+
+log() {
+    printf "%s [%s] %s - %s\n" \
+           "$(date --rfc-3339=seconds)" "$$" "$SSH_SOURCE_IP" "$*" \
+           >> "$LOG"
+}
+
+log "NEW RECOVER CONNECTION"
+
+if [ -z "$1" ] || ! [[ "$1" =~ ^[a-z0-9]+$ ]]; then
+    log "INVALID SETUP, ARG 1 SHOULD BE MD5 AND IS: '$1'"
+    echo "Your command has been rejected. Contact administrator."
+    exit 1
+fi
+
+md5="$1"
+log "RECOVER KEY $md5"
+
+if [ -z "$2" ] || ! [[ "$2" =~ ^[a-zA-Z0-9._-]+$ ]]; then
+    log "INVALID SETUP, IDENT IS: '$1'"
+    echo "Your command has been rejected. Contact administrator."
+    exit 1
+fi
+
+ident="$2"
+log "IDENTIFIED AS $ident"
+
+reject() {
+    log "REJECTED: $SSH_ORIGINAL_COMMAND"
+    # echo "ORIG: $SSH_ORIGINAL_COMMAND" >&2
+    echo "Your command has been rejected and reported to sys admin." >&2
+    exit 1
+}
+
+
+if [[ "$SSH_ORIGINAL_COMMAND" =~ [\&\(\{\;\<\>\`\$\}] ]]; then
+    log "BAD CHARS DETECTED"
+    # echo "Bad chars: $SSH_ORIGINAL_COMMAND" >&2
+    reject
+fi
+
+if [[ "$SSH_ORIGINAL_COMMAND" =~ ^"rsync --server --sender -"[vnloHgDtpArRzCeiLsfx\.]+(" --"[a-z=%-]+|" --partial-dir .rsync-partial")*" . /var/mirror/$ident"(|/.*)$ ]]; then
+
+    ## Interpret \ to allow passing spaces (want to avoid possible issue with \n)
+    #read -a ssh_args <<< "${SSH_ORIGINAL_COMMAND}"
+    ssh_args=(${SSH_ORIGINAL_COMMAND})
+
+    last_arg="${ssh_args[@]: -1:1}"
+    if ! new_path=$(realpath "$last_arg" 2>/dev/null); then
+        log "FINAL PATH INVALID"
+        reject
+    fi
+
+    if [[ "$new_path" != "$last_arg" ]] &&
+           [[ "$new_path" != "/var/mirror/$ident/"* ]] &&
+           [[ "$new_path" != "/var/mirror/$ident" ]]; then
+        log "FINAL PATH SUSPICIOUS"
+        reject
+    fi
+
+    sudo /usr/local/sbin/ssh-update-keys
+    if ! [ -e "${RECOVER_KEY_PATH}/$md5" ]; then
+        log "RECOVERY KEY $md5 JUST EXPIRED"
+        reject
+    fi
+
+	log "ACCEPTED RECOVER COMMAND: $SSH_ORIGINAL_COMMAND"
+	sudo "${ssh_args[@]}"
+    errlvl="$?"
+
+    for key_file in "${RECOVER_KEY_PATH}/$md5"{,.pub}; do
+        [ -e "$key_file" ] || continue
+        sudo touch "$key_file"  ## Update modified time to keep key longer
+    done
+
+    exit "$errlvl"
+else
+    log "REFUSED COMMAND AS IT DOESN'T MATCH ANY EXPECTED COMMAND"
+	reject
+fi

rsync-backup-target/build/src/usr/local/sbin/ssh-update-keys

@@ -1,36 +1,68 @@
 #!/bin/bash
 
+## Keep in mind possible race conditions as this script will be called
+## from different place to update the access tokens.
+
 
 ##
-## code
+## Code
 ##
 
-KEYS=/etc/rsync/keys
+RSYNC_KEY_PATH=/etc/rsync/keys
 RSYNC_HOME=/var/lib/rsync
+BACKUP_KEY_PATH=${RSYNC_KEY_PATH}/backup
+RECOVER_KEY_PATH=${RSYNC_KEY_PATH}/recover
+
+
+mkdir -p "$RSYNC_HOME/.ssh" "$RECOVER_KEY_PATH"
+
+## delete old recovery keys
+find "${RECOVER_KEY_PATH}" \
+    -maxdepth 1 -not -newermt "-15 minutes" \
+    -type f -delete
 
-mkdir -p "$RSYNC_HOME/.ssh"
 
 ##
 ## New
 ##
 
-touch "$RSYNC_HOME"/.ssh/authorized_keys.new
+pid=$$
+new="$RSYNC_HOME"/.ssh/authorized_keys.tmp."$pid"
+touch "$new"
 
-for f in "$KEYS"/backup/*/*.pub; do
+for f in "$BACKUP_KEY_PATH"/*/*.pub "$RECOVER_KEY_PATH"/*.pub; do
     [ -e "$f" ] || continue
     content=$(cat "$f")
+    if [[ "$content" == *" "*" "*@* ]]; then
+        ident="${content##*@}"
+    else
         ident="${f##*/}"
         ident="${ident%.pub}"
+    fi
     if ! [[ "$ident" =~ ^[a-zA-Z0-9._-]+$ ]]; then
         echo "bad: '$ident'" >&2
         continue
     fi
-    echo "command=\"/usr/local/sbin/ssh-cmd-validate \\\"$ident\\\"\",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty $content"
-done >> "$RSYNC_HOME"/.ssh/authorized_keys.new
+    if [[ "$f" == "${RECOVER_KEY_PATH}"/*.pub ]]; then
+        basename=${f##*/}
+        basename=${basename%.pub}
+        cmd="/usr/local/sbin/ssh-recover-cmd-validate $basename"
+    else
+        cmd=/usr/local/sbin/ssh-cmd-validate
+    fi
+    echo "command=\"$cmd \\\"$ident\\\"\",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty $content"
+done >> "$new"
 
-[ -e ""$RSYNC_HOME"/.ssh/authorized_keys" ] &&
+[ -e "$RSYNC_HOME"/.ssh/authorized_keys ] &&
   mv "$RSYNC_HOME"/.ssh/authorized_keys{,.old}
-mv "$RSYNC_HOME"/.ssh/authorized_keys{.new,}
 
-chown rsync:rsync -R "$RSYNC_HOME"/.ssh -R
+## XXXvlab: Atomic operation. It's the last call to this instruction
+## that will prevail. There are some very special cases where some
+## added key would not be added as expected: for instance an older
+## call to ``ssh-update-key``, if made before a specific public key
+## file was added to directory, could take a longer time to reach this
+## next instruction than a more recent call (that would be after
+## the specific public key was added).
+mv "$new" "$RSYNC_HOME"/.ssh/authorized_keys
 
+chown rsync:rsync "$RSYNC_HOME"/.ssh -R