new: [cron] new charm

cron/build/Dockerfile

@@ -0,0 +1,20 @@
+FROM debian:jessie
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes cron moreutils && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+## XXXvlab: these should be added by logrotate only
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes netcat-openbsd logrotate && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY ./src/usr/bin/lock /usr/bin/lock
+COPY ./src/usr/bin/docker-send-signal /usr/bin/docker-send-signal
+COPY ./src/usr/bin/docker-17.06.2-ce /usr/bin/docker
+
+COPY ./entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT [ "/entrypoint.sh" ]

cron/build/README

@@ -0,0 +1,16 @@
+
+
+Warning, this charm will require access to ``/var/run/docker.sock``,
+and this IS EQUIVALENT to root access to host.
+
+Warning, must use ``/etc/cron`` and not ``/etc/cron.d``.
+
+
+docker was downloaded with:
+
+wget https://get.docker.com/builds/Linux/x86_64/docker-1.9.1
+
+
+It changed, check:
+
+  https://download.docker.com/linux/static/stable/x86_64/

cron/build/entrypoint.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+##
+## /var/log might be plugged into an empty volume for saving logs, so we
+## must make sure that /var/log/exim4 exists and has correct permissions.
+
+mkdir -p /var/log/exim4
+chmod -R u+rw /var/log/exim4
+chown -R Debian-exim /var/log/exim4
+
+
+echo "Propagating docker shell environment variables to CRON scripts."
+
+rm -f /etc/cron.d/*
+cp -a /etc/cron/* /etc/cron.d/
+
+for f in /etc/crontab /etc/cron.d/*; do
+    [ -e "$f" ] || continue
+    mv "$f" /tmp/tempfile
+    {
+        declare -xp | egrep '_PORT_[0-9]+_'  | sed -r 's/^declare -x //g'
+        echo "TZ=$TZ"
+        echo
+        cat /tmp/tempfile
+    } > "$f"
+    rm /tmp/tempfile
+done
+
+
+echo "Launching cron."
+## Give back PID 1, so that cron process receives signals.
+exec /usr/sbin/cron -f

cron/build/src/usr/bin/README

@@ -0,0 +1,2 @@
+WARNING, lock shell script is a copy from ``kal-scripts``. Please
+do not do any modification to it without sending it back to ``kal-scripts``.

cron/build/src/usr/bin/docker-send-signal

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+exname=$(basename "$0")
+usage="$exname [-h|--help] CONTAINER SIGNAL"
+
+container=
+signal=
+while [ "$1" ]; do
+    case "$1" in
+        "--help"|"-h")
+            echo "$usage" >&2
+            exit 0
+        ;;
+        *)
+            [ -z "$container" ] && { container=$1 ; shift ; continue ; }
+            [ -z "$signal" ] && { signal=$1 ; shift ; continue ; }
+            echo "Unexpected argument '$1'." >&2
+            exit 1
+        ;;
+    esac
+    shift
+done
+
+if [ -z "$container" ]; then
+    echo "You must provide a container name/id as first argument." >&2
+    echo "$usage" >&2
+    exit 1
+fi
+
+if [ -z "$signal" ]; then
+    echo "You must provide a signal to send to $container aargument." >&2
+    echo "$usage" >&2
+    exit 1
+fi
+
+container_id="$(docker inspect --format="{{ .Id }}" "$container")"
+
+echo -e "POST /containers/$container_id/kill?signal=$signal HTTP/1.0\r\n" | nc -U /var/run/docker.sock

cron/build/src/usr/bin/lock

@@ -0,0 +1,363 @@
+#!/bin/bash
+
+##
+## TODO
+## - don't sleep 1 but wait in flock for 1 second
+## - every waiting proc should write at least their PID and priority,
+##   to leave alive PID with higher priority the precedence. (and probably
+##   a check to the last probing time, and invalidate it if it is higher than 10s
+##   for example.)
+## - could add the time they waited in the waiting list, and last probe.
+## - should execute "$@", if user needs '-c' it can run ``bash -c ""``
+
+exname="$(basename "$0")"
+usage="$exname LOCKLABELS [-k] [FLOCK_OPTIONS] -- [CMD...]"
+
+verb() { [ -z "$verbose" ] || echo "$@" >&2 ; }
+err() { echo "$@" >&2; }
+die() { echo "$@" >&2; exit 1; }
+
+md5_compat() { md5sum | cut -c -32; true; }
+
+LOCKLABELS=
+flock_opts=()
+command=()
+nonblock=
+errcode=1
+timeout=
+cmd=
+priority=1
+remove_duplicate=
+while [ "$1" ]; do
+    case "$1" in
+        -h|--help)
+            echo "$help"
+            exit 0
+            ;;
+        -V|--version)
+            echo "$version"
+            exit 0
+            ;;
+        -c)
+            cmd="$2"
+            shift
+            ;;
+        -p|--priority)
+            priority=$2
+            shift
+            ;;
+        -D)
+            remove_duplicate=true
+            ;;
+        -k)
+            kill=yes
+            ;;
+        -n|--nb|--nonblock)
+            nonblock=true
+            ;;
+        -w|--wait|--timeout)
+            timeout=$2   ## will manage this
+            shift
+            ;;
+        -E|--conflict-exit-code)
+            errcode=$2   ## will manage this
+            shift
+            ;;
+        -v|--verbose)
+            verbose=true   ## will manage this
+            ;;
+        -n|--nb|--nonblock)
+            nonblock=true   ## will manage this
+            ;;
+        --)
+            [ "$cmd" ] && die "'--' and '-c' are mutualy exclusive"
+            shift
+            command+=("$@")
+            break 2
+            ;;
+        *)
+            [ -z "$LOCKLABELS" ] && { LOCKLABELS=$1 ; shift ; continue ; }
+            flock_opts+=("$1")
+            ;;
+    esac
+    shift
+done
+
+if [ -z "$LOCKLABELS" ]; then
+    err "You must provide a lock file as first argument."
+    err "$usage"
+    exit 1
+fi
+
+if [ "$remove_duplicate" ]; then
+    md5code=$(
+        if [ "$cmd" ]; then
+            echo bash -c "$cmd"
+        else
+            echo "${command[@]}"
+        fi | md5_compat)
+fi
+
+
+function is_int () {  [[ "$1" =~ ^-?[0-9]+$ ]] ; }
+
+is_pid_alive() {
+    local pid="$1"
+    ps --pid "$pid" >/dev/null 2>&1
+}
+
+
+is_pgid_alive() {
+    local pgid="$1"
+    [ "$(ps -e -o pgid,pid= | egrep "^ *$pgid ")" ]
+}
+
+
+pgid_from_pid() {
+    local pid="$1"
+    pgid=$(ps -o pgid= "$pid" 2>/dev/null | egrep -o "[0-9]+")
+    if ! is_int "$pgid"; then
+        err "Could not retrieve a valid PGID from PID '$pid' (returned '$pgid')."
+        return 1
+    fi
+    echo "$pgid"
+}
+
+
+ensure_kill() {
+    local pid="$1" timeout=5 start=$SECONDS kill_count=0 pgid
+    pgid=$(pgid_from_pid "$pid")
+    while is_pid_alive "$pid"; do
+          if is_pgid_alive "$pgid"; then
+              if [ "$kill_count" -gt 4 ]; then
+                  err "FATAL: duplicate command, GPID=$pgid has resisted kill procedure. Aborting."
+                  return 1
+              elif [ "$kill_count" -gt 2 ]; then
+                  err "duplicate command, PGID wouldn't close itself, force kill PGID: kill -9 -- -$pgid"
+                  kill -9 -- "$pgid"
+                  sleep 1
+              else
+                  err "duplicate command, Sending SIGKILL to PGID: kill -- -$pgid"
+                  kill -- -"$pgid"
+                  sleep 1
+              fi
+              ((kill_count++))
+          fi
+          if [ "$((SECONDS - start))" -gt "$timeout" ]; then
+              err "timeout reached. $pid"
+              return 1
+          fi
+    done
+    return 0
+}
+
+
+acquire_pid_file() {
+    local label=$1
+    lockfile="/var/lock/lockcmd-$label.lock"
+    mkdir -p /var/run/lockcmd
+    pidfile="/var/run/lockcmd/$label.pid"
+    export pidfile
+    (
+    verb() { [ -z "$verbose" ] || echo "$exname($label) $pid> $@" >&2 ; }
+    err() { echo "$exname($label) $pid> $@" >&2; }
+
+    start=$SECONDS
+    kill_count=0
+    pgid_not_alive_count=0
+    while true; do
+        ## ask for lock on $lockfile (fd 200)
+        if ! flock -n -x 200; then
+            verb "Couldn't acquire primary lock... (elapsed $((SECONDS - start)))"
+        else
+            verb "Acquired lock '$label' on pidfile, inspecting pidfile."
+            if ! [ -e "$pidfile" ]; then
+                verb "No pidfile, inscribing my PID"
+                echo -e "$pid $priority" > "$pidfile"
+                exit 0
+            fi
+
+            if ! content=$(cat "$pidfile" 2>/dev/null); then
+                err "Can't read $pidfile"
+                exit 1
+            fi
+            read opid opriority < <(echo "$content" | head -n 1)
+            opriority=${opriority:-1}
+            verb "Previous PID is $opid, with priority $opriority"
+            if ! is_pid_alive "$opid"; then
+                err "Ignoring stale PID $opid"
+                echo -e "$pid $priority" > "$pidfile"
+                exit 0
+            else
+                if [ "$remove_duplicate" ]; then  ## Add my pid and md5 if not already there.
+                    same_cmd_pids=$(
+                        echo "$content" | tail -n +1 | \
+                            egrep "^[0-9]+ $md5code$" 2>/dev/null | \
+                            cut -f 1 -d " ")
+                    same_pids=()
+                    found_myself=
+                    for spid in $same_cmd_pids; do
+                        if [ "$spid" == "$pid" ]; then
+                            found_myself=true
+                            continue
+                        fi
+                        same_pids+=("$spid")
+                    done
+                    [ "$found_myself" ] || echo "$pid $md5code" >> "$pidfile"
+                fi
+                flock -u 200  ## reopen the lock to give a chance to the other process to remove the pidfile.
+                if [ "$remove_duplicate" ]; then  ## Add my pid and md5 if not already there.
+                    for spid in "${same_pids[@]}"; do
+                        if ! ensure_kill "$spid"; then
+                            err "Couldn't kill previous duplicate command."
+                            exit 1
+                        fi
+                    done
+                fi
+                pgid=$(pgid_from_pid "$opid")
+                verb "PGID of previous PID is $pgid"
+                if is_pgid_alive "$pgid"; then
+                    verb "Previous PGID is still alive"
+                    if [ "$kill" ] && [ "$priority" -ge "$opriority" ]; then
+                        if [ "$kill_count" -gt 4 ]; then
+                            err "$pid>FATAL: GPID=$pgid has resisted kill procedure. Aborting."
+                            exit 1
+                        elif [ "$kill_count" -gt 2 ]; then
+                            err "PGID wouldn't close itself, force kill PGID: kill -9 -- -$pgid" >&2
+                            kill -9 -- "$pgid"
+                            sleep 1
+                        else
+                            err "Sending SIGKILL to PGID: kill -- -$pgid" >&2
+                            kill -- -"$pgid"
+                            sleep 1
+                        fi
+                        ((kill_count++))
+                    else
+                        if [ "$nonblock" ]; then
+                            verb "Nonblock options forces exit."
+                            exit 1
+                        else
+                            verb "Couldn't acquire Lock... (elapsed $((SECONDS - start)))"
+                        fi
+                    fi
+                else
+                    if [ "$pgid_not_alive_count" -gt 4 ]; then
+                        verb "$pid>A lock exists for label $label, but PGID:$pgid in it isn't alive while child $pid is ?!?."
+                        err "$pid>Can't force seizing the lock." >&2
+                        exit 1
+                    fi
+                    ((pgid_not_alive_count++))
+                fi
+            fi
+        fi
+
+        if [ "$timeout" ] && [ "$timeout" -lt "$((SECONDS - start))" ]; then
+            err "Timeout reached (${timeout}s) while waiting for lock on $label"
+            exit "$errcode"
+        fi
+        sleep 1
+    done
+    ) 200> "$lockfile"
+}
+
+remove_pid_file() {
+    local label=$1
+    lockfile="/var/lock/lockcmd-$label.lock"
+    mkdir -p /var/run/lockcmd
+    pidfile="/var/run/lockcmd/$label.pid"
+
+(
+    verb() { [ -z "$verbose" ] || echo "$exname($label) $pid> $@" >&2 ; }
+    err() { echo "$exname($label) $pid> $@" >&2; }
+    verb "Asking lock to delete $pidfile."
+    timeout=5
+    start=$SECONDS
+    while true; do
+        ## ask for lock on $lockfile (fd 200)
+        if ! flock -n -x 200; then
+            verb "Couldn't acquire primary lock... (elapsed $((SECONDS - start)))"
+        else
+            verb "Acquired lock '$label' on pidfile."
+            if ! [ -e "$pidfile" ]; then
+                verb "No more pidfile, somebody deleted for us ?1?"
+                exit 1
+            fi
+            if ! content=$(cat "$pidfile" 2>/dev/null); then
+                err "Can't read $pidfile"
+                exit 1
+            fi
+            read opid opriority < <(echo "$content" | head -n 1)
+            opriority=${opriority:-1}
+            if [ "$opid" == "$pid" ]; then
+                verb "Deleted pidfile. Releasing lock."
+                rm -f "$pidfile"
+                exit 0
+            else
+                verb "Removing duplicates in pidfile. Releasing lock."
+                [ "$remove_duplicate" ] && sed -ri "/^$pid $md5code$/d" "$pidfile"
+                exit 0
+            fi
+        fi
+        if [ "$timeout" ] && [ "$timeout" -lt "$((SECONDS - start))" ]; then
+            err "Timeout reached (${timeout}s) while waiting for lock on $label"
+            exit "$errcode"
+        fi
+        sleep 1
+    done
+) 200> "$lockfile"
+
+}
+
+
+## appends a command to the signal handler functions
+#
+# example: trap_add EXIT,INT close_ssh "$ip"
+trap_add() {
+    local sigs="$1" sig cmd old_cmd
+    shift || {
+        echo "${FUNCNAME} usage error" >&2
+        return 1
+    }
+    cmd="$@"
+    while IFS="," read -d "," sig; do
+        prev_cmd="$(trap -p "$sig")"
+        if [ "$prev_cmd" ]; then
+            new_cmd="${prev_cmd#trap -- \'}"
+            new_cmd="${new_cmd%\' "$sig"};$cmd"
+        else
+            new_cmd="$cmd"
+        fi
+        trap -- "$new_cmd" "$sig" || {
+            echo "unable to add command '$@' to trap $sig" >&2 ;
+            return 1
+        }
+    done < <(echo "$sigs,")
+}
+
+remove_all_pid_file() {
+    while read -d "," label; do
+        {
+            remove_pid_file "$label" || err "Could not delete $label"
+        } &
+    done < <(echo "$LOCKLABELS,")
+    wait
+}
+
+##
+## Code
+##
+
+pid="$$"
+
+trap_add EXIT "remove_all_pid_file"
+while read -d "," label; do
+    acquire_pid_file "$label" || exit "$errcode" &
+done < <(echo "$LOCKLABELS,")
+wait
+if [ "$cmd" ]; then
+    bash -c "$cmd"
+else
+    "${command[@]}"
+fi
+errlvl="$?"
+exit "$?"

cron/hooks/init

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+## Init is run on host
+## For now it is run every time the script is launched, but
+## it should be launched only once after build.
+
+## Accessible variables are:
+## - SERVICE_NAME        Name of current service
+## - DOCKER_BASE_IMAGE   Base image from which this service might be built if any
+## - SERVICE_DATASTORE           Location on host of the DATASTORE of this service
+## - SERVICE_CONFIGSTORE         Location on host of the CONFIGSTORE of this service
+
+timezone=$(cat /etc/timezone)
+init-config-add "
+$CHARM_NAME:
+  volumes:
+    - /etc/timezone:/etc/timezone:ro
+  environment:
+    TZ: $timezone
+"
+
+info "Timezone is set to $timezone."

cron/hooks/log_rotate-relation-joined

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+## Should be executable N time in a row with same result.
+
+. lib/common
+
+set -e
+
+LOGS=/var/log/cron
+
+## XXXvlab: hum it seems apache logging is run as root, so well...
+# logs_creds=$(cached_cmd_on_base_image apache "stat -c '%u %g' '$LOGS'") || {
+#     debug "Failed to query for www-data gid in ${DARKYELLOW}apache${NORMAL} base image."
+#     return 1
+# }
+
+## XXXvlab: a lot of this intelligence should be moved away into ``logrotate`` charm
+DST="$CONFIGSTORE/$TARGET_SERVICE_NAME/etc/logrotate.d/$SERVICE_NAME"
+file_put "$DST" <<EOF
+/var/log/docker/$SERVICE_NAME/*_script.log {
+    weekly
+    missingok
+    dateext
+    dateyesterday
+    dateformat _%Y-%m-%d
+    extension .log
+    rotate 52
+    compress
+    delaycompress
+    notifempty
+    create 640 root root
+    sharedscripts
+}
+EOF
+
+config-add "\
+$MASTER_TARGET_CHARM_NAME:
+  volumes:
+    - $DST:/etc/logrotate.d/docker-${SERVICE_NAME}:ro
+    - $SERVICE_DATASTORE$LOGS:/var/log/docker/$SERVICE_NAME:rw
+"
+config-add "\
+$MASTER_BASE_CHARM_NAME:
+  volumes:
+    - $SERVICE_DATASTORE$LOGS:$LOGS:rw
+"

cron/metadata.yml

@@ -0,0 +1,15 @@
+description: Cron daemon
+config-resources:
+  - /etc/cron
+  - /etc/cron.daily
+  - /etc/cron.weekly
+  - /etc/cron.hourly
+  - /etc/cron.monthly
+  - /usr/local/bin
+data-resources:
+  - /var/log/cron
+host-resources:
+  - /var/run/docker.sock
+provides:
+  schedule-command:
+    tech-dep: False