first import

bin/compose

@@ -0,0 +1,635 @@
+#!/bin/bash
+
+. /etc/shlib
+
+
+include pretty
+
+depends shyaml docker
+
+if [ -r /etc/default/charm ]; then
+    . /etc/default/charm
+fi
+
+if [ -r /etc/default/$exname ]; then
+    . /etc/default/$exname
+fi
+
+usage="$exname CHARM"'
+
+Deploy and manage a swarm of containers to provide services based on
+a ``compose.yml`` definition and charms from a ``charm-store``.
+'
+
+export DEFAULT_COMPOSE_FILE
+
+##
+## Functions
+##
+
+export APACHE_CONFIG_LOCATION=$CONFIGSTORE/apache/etc/apache2/sites-enabled
+
+apache_ssl_proxy_config () {
+    local DOMAIN=$1 TARGET=$2
+
+    cat <<EOF
+<IfModule mod_ssl.c>
+
+    <VirtualHost *:443>
+        ServerAdmin ${ADMIN_MAIL:-contact@$DOMAIN}
+        ServerName ${DOMAIN}
+
+        ServerSignature Off
+        CustomLog /var/log/apache2/s-${DOMAIN}_access.log combined
+        ErrorLog /var/log/apache2/s-${DOMAIN}_error.log
+        ErrorLog syslog:local2
+
+        <IfModule mod_proxy.c>
+            ProxyRequests Off
+            <Proxy *>
+                Order deny,allow
+                Allow from all
+            </Proxy>
+            ProxyVia On
+            ProxyPass / http://$TARGET/ retry=0
+            <Location / >
+            ProxyPassReverse /
+            </Location>
+        </IfModule>
+
+        ## Forbid any cache, this is only usefull on dev server.
+        #Header set Cache-Control "no-cache"
+        #Header set Access-Control-Allow-Origin "*"
+        #Header set Access-Control-Allow-Methods "POST, GET, OPTIONS"
+        #Header set Access-Control-Allow-Headers "origin, content-type, accept"
+
+        RequestHeader set "X-Forwarded-Proto" "https"
+
+        ## Fix IE problem (httpapache proxy dav error 408/409)
+        SetEnv proxy-nokeepalive 1
+        #ServerSignature On
+        SSLProxyEngine On
+        SSLEngine On
+
+        ## Full stance
+        SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem
+        SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key
+
+        SSLVerifyClient None
+
+    </VirtualHost>
+
+</IfModule>
+EOF
+
+}
+export -f apache_ssl_proxy_config
+
+apache_ssl_config() {
+    local DOMAIN=$1
+
+    cat <<EOF
+<IfModule mod_ssl.c>
+
+    <VirtualHost *:443>
+        ServerAdmin ${ADMIN_MAIL:-contact@$DOMAIN}
+        ServerName ${DOMAIN}
+
+        ServerSignature Off
+        CustomLog /var/log/apache2/s-${DOMAIN}_access.log combined
+        ErrorLog /var/log/apache2/s-${DOMAIN}_error.log
+        ErrorLog syslog:local2
+
+        DocumentRoot /var/www/${DOMAIN}
+
+        <Directory />
+                Options FollowSymLinks
+                AllowOverride None
+        </Directory>
+
+        <Directory /var/www/${DOMAIN}>
+                Options Indexes FollowSymLinks MultiViews
+                AllowOverride all
+                Order allow,deny
+                allow from all
+        </Directory>
+
+        SSLEngine On
+
+        ## Full stance
+        SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem
+        SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key
+        SSLVerifyClient None
+
+    </VirtualHost>
+
+</IfModule>
+EOF
+
+}
+export -f apache_ssl_config
+
+apache_ssl_add () {
+    local DOMAIN=$1
+    [ -e "$APACHE_CONFIG_LOCATION/$DOMAIN.conf" ] && return 0
+    mkdir -p "$APACHE_CONFIG_LOCATION"
+    apache_ssl_config $DOMAIN > $APACHE_CONFIG_LOCATION/$DOMAIN.conf
+    echo "Added $DOMAIN apache config." >&2
+
+}
+export -f apache_ssl_add
+apache_ssl_proxy_add () {
+    local DOMAIN=$1 TARGET=$2
+    [ -e "$APACHE_CONFIG_LOCATION/$DOMAIN.conf" ] && return 0
+
+    mkdir -p "$APACHE_CONFIG_LOCATION"
+    apache_ssl_proxy_config $DOMAIN $TARGET > $APACHE_CONFIG_LOCATION/$DOMAIN.conf
+    echo "Added $DOMAIN as a proxy to $TARGET." >&2
+}
+export -f apache_ssl_proxy_add
+
+gen_password() {
+    python -c 'import random; \
+        xx = "azertyuiopqsdfghjklmwxcvbn1234567890AZERTYUIOPQSDFGHJKLMWXCVBN+_-"; \
+        print "".join([xx[random.randint(0, len(xx)-1)] for x in range(0, 14)])'
+}
+export -f gen_password
+
+
+file_put() {
+    local TARGET="$1"
+    mkdir -p "$(dirname "$TARGET")" &&
+    cat - > "$TARGET"
+}
+export -f file_put
+
+
+apache_data_dir() {
+    local DOMAIN=$1 DATA_COMMA_SEPARATED=$2
+
+    export APACHE_DOCKER_IMAGE=$(service_base_docker_image apache)
+
+DOCKER_SITE_PATH=/var/www/$DOMAIN
+BASE=$DATASTORE/apache
+DST=$BASE/$DOCKER_SITE_PATH
+    DATA=()
+    while IFS="," read -ra ADDR; do
+        for dir in "${ADDR[@]}"; do
+            mkdir -p "$DST/$dir"
+            DATA+=($dir)
+        done
+    done <<< "$DATA_COMMA_SEPARATED"
+
+    if [ -z "$APACHE_DOCKER_GID" ] &&
+           ! grep "^export APACHE_DOCKER_GID=" /etc/compose.local.conf >/dev/null 2>&1; then
+        echo "Adding APACHE_DOCKER_GID to '/etc/compose.local.conf'."
+
+        export APACHE_DOCKER_GID=$(docker run "$APACHE_DOCKER_IMAGE" id -g www-data)
+
+            cat <<EOF >> /etc/compose.local.conf
+export APACHE_DOCKER_GID=$APACHE_DOCKER_GID
+EOF
+
+    fi
+
+    dirs=()
+    for d in "${DATA[@]}"; do
+        dirs+=("$DST/$d")
+    done
+
+    chgrp www-data "${dirs[@]}" -R && chmod 775 "${dirs[@]}" -R
+}
+export -f apache_data_dir
+
+
+export _DOCKER_COMPOSE_DEF=""
+get_compose_def() {
+    local local_compose
+
+    if [ "$_DOCKER_COMPOSE_DEF" ]; then
+        echo "$_DOCKER_COMPOSE_DEF"
+        return 0
+    fi
+
+    ##
+    ## Adding sub services configurations
+    ##
+
+    additional_services=
+    if [ -z "$*" ]; then
+        info "No service provided, using \$DEFAULT_SERVICES variable. Target services: $DEFAULT_SERVICES"
+        additional_services=$DEFAULT_SERVICES
+    fi
+    declare -A loaded
+    for target_service in "$@" $additional_services; do
+
+        services=$(get_ordered_service_dependencies "$target_service") || return 1
+        for service in $services; do
+
+            if [ "${loaded[$service]}" ]; then
+                continue
+            fi
+            loaded[$service]=1
+            export _DOCKER_COMPOSE_DEF="$_DOCKER_COMPOSE_DEF
+$service:
+$(get_service_def "$service" | sed -r 's/^/  /g')"
+
+        done
+    done
+    echo "$_DOCKER_COMPOSE_DEF"
+}
+export -f get_compose_def
+
+get_service_def() {
+    local service="$1"
+
+    if [ -z  "$service" ]; then
+        echo "Please specify a service." >&2
+        return 1
+    fi
+
+    if [ -d "$CHARM_STORE/$service" ]; then
+        compose_file="$CHARM_STORE/$service/compose.yml"
+        local_compose=""
+        if [ -e "$compose_file" ]; then
+            debug "Found compose.yml in $service directory. Including in 'docker-compose.conf'."
+            local_compose="$(cat "$compose_file")"
+        fi
+        metadata_file="$CHARM_STORE/$service/metadata.yml"
+        if [ -e "$metadata_file" ]; then
+            debug "Found metadata.yml in $service directory. Including in 'docker-compose.conf'."
+            docker_compose_entry=$(get_docker_compose_entry_from_metadata "$service" < "$metadata_file") || return 1
+            local_compose="$local_compose
+$docker_compose_entry"
+        fi
+        echo "$local_compose"
+        return 0
+    fi
+
+    err "service '$DARKYELLOW$service$NORMAL' not found." >&2
+    return 1
+}
+export -f get_service_def
+
+service_base_docker_image() {
+    local service="$1"
+    service_def="$(get_service_def "$service")" || return 1
+
+    service_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null)
+    if [ "$?" != 0 ]; then
+        service_build=$(echo "$service_def" | shyaml get-value build)
+        if [ "$?" != 0 ]; then
+            echo "Service '$service' has no 'image' nor 'build' parameter." >&2
+            return 1
+        fi
+        service_dockerfile="$COMPOSE_YML_PATH/$service_build/Dockerfile"
+        if ! [ -e "$service_dockerfile" ]; then
+            echo "No Dockerfile found in '$service_dockerfile' location." >&2
+            return 1
+        fi
+
+        grep '^FROM' "$service_dockerfile" | xargs echo | cut -f 2 -d " "
+    else
+        echo "$service_image"
+    fi
+
+}
+export -f service_base_docker_image
+
+
+read-0() {
+    local eof
+    eof=
+    while [ "$1" ]; do
+        IFS='' read -r -d '' "$1" || eof=true
+        shift
+    done
+    test "$eof" != true
+}
+export -f read-0
+
+array_values_to_stdin() {
+    local e
+    if [ "$#" -ne "1" ]; then
+        print_syntax_warning "$FUNCNAME: need one argument."
+        return 1
+    fi
+    var="$1"
+    eval "for e in \"\${$var[@]}\"; do echo -en \"\$e\\0\"; done"
+}
+
+array_keys_to_stdin() {
+    local e
+    if [ "$#" -ne "1" ]; then
+        print_syntax_warning "$FUNCNAME: need one argument."
+        return 1
+    fi
+    var="$1"
+    eval "for e in \"\${!$var[@]}\"; do echo -en \"\$e\\0\"; done"
+}
+
+array_kv_to_stdin() {
+    local e
+    if [ "$#" -ne "1" ]; then
+        print_syntax_warning "$FUNCNAME: need one argument."
+        return 1
+    fi
+    var="$1"
+    eval "for e in \"\${!$var[@]}\"; do echo -n \"\$e\"; echo -en '\0'; echo -n \"\${$var[\$e]}\"; echo -en '\0'; done"
+}
+
+
+array_pop() {
+    local narr="$1" nres="$2"
+    for key in $(eval "echo \${!$narr[@]}"); do
+        eval "$nres=\${$narr[\"\$key\"]}"
+        eval "unset $narr[\"\$key\"]"
+        return 0
+    done
+}
+export -f array_pop
+
+array_member() {
+    local src elt
+    src="$1"
+    elt="$2"
+    while read-0 key; do
+        if [ "$(eval "echo -n \"\${$src[\$key]}\"")" == "$elt" ]; then
+            return 0
+        fi
+    done < <(array_keys_to_stdin "$src")
+    return 1
+}
+export -f array_member
+
+
+get_service_deps() {
+    local service="$1"
+    service_def=$(get_service_def "$service") || return 1
+    echo "$service_def" | shyaml get-values links 2>/dev/null
+    return 0
+}
+export -f get_service_deps
+
+## a service is not always a container.
+## XXXvlab: a service name should not be a container name neither... see this later.
+# get_container_name() {
+#     local service="$1"
+#     get_service_def "$service" | shyaml get-values links 2>/dev/null
+#     if [ "$(get_md_service_def "$service" | shyaml get-value subordinate 2>/dev/null)" != "true" ]; then
+#         echo "$service"
+#         return 0
+#     fi
+
+# }
+
+_rec_get_depth() {
+    local elt=$1
+
+    if [ "${depths[$elt]}" ]; then
+        return 0
+    fi
+    deps=$(get_service_deps "$elt") || return 1
+    if [ -z "$deps" ]; then
+        depths[$elt]=0
+    fi
+
+    max=0
+    for dep in $deps; do
+        _rec_get_depth "$dep" || return 1
+        if (( "${depths[$dep]}" > "$max" )); then
+            max="${depths[$dep]}"
+        fi
+    done
+    depths[$elt]=$((max + 1))
+}
+
+get_ordered_service_dependencies() {
+    local services=("$@")
+
+    if [ -z "${services[*]}" ]; then
+        print_syntax_error "$FUNCNAME: no arguments"
+    fi
+
+    declare -A depths
+    visited=()
+    heads=("${services[@]}")
+    while [ "${#heads[@]}" != 0 ]; do
+        array_pop heads head
+        visited+=("$head")
+        _rec_get_depth "$head" || return 1
+    done
+
+    i=0
+    while [ "${#depths[@]}" != 0 ]; do
+        for key in "${!depths[@]}"; do
+            value="${depths[$key]}"
+            if [ "$value" == "$i" ]; then
+                echo "$key"
+                unset depths[$key]
+            fi
+        done
+        i=$((i + 1))
+    done
+}
+
+run_service_hook () {
+    local service="$1" action="$2"
+
+    services=$(get_ordered_service_dependencies "$service") || return 1
+    ## init in order
+    for service in $services; do
+        TARGET_SCRIPT="$COMPOSE_YML_PATH/service/$service/hooks/$2"
+        [ -e "$TARGET_SCRIPT" ] && {
+            [ "$verbose" ] && echo "Init $service"
+            SERVICE_NAME=$service \
+                        DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") \
+                        SERVICE_DATASTORE="$DATASTORE/$service" \
+                        SERVICE_CONFIGSTORE="$CONFIGSTORE/$service" \
+                        "$TARGET_SCRIPT"
+        }
+    done
+}
+
+run_service_action () {
+    local service="$1" action="$2"
+    shift shift
+    run_service_hook "$service" init
+
+    services=$(get_ordered_service_dependencies "$service") || return 1
+    for service in $services; do
+        TARGET_SCRIPT="$COMPOSE_YML_PATH/service/$service/actions/$2"
+        if [ -e "$TARGET_SCRIPT" ]; then
+            [ "$verbose" ] && echo "Init $service"
+            SERVICE_NAME=$service \
+                        CONTAINER_NAME=$(get_container_name "$service") \
+                        DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") \
+                        SERVICE_DATASTORE="$DATASTORE/$service" \
+                        SERVICE_CONFIGSTORE="$CONFIGSTORE/$service" \
+                        echo "$TARGET_SCRIPT" "$@"
+        else
+            echo "Service '$service' does not have any action '$action' defined." >&2
+            return 1
+        fi
+    done
+}
+
+get_docker_compose_entry_from_metadata() {
+    local service="$1"
+    metadata="$(cat -)"
+
+    export DATASTORE CONFIGSTORE
+    ## resources to volumes
+    volumes=$(
+        for resource_type in data config; do
+            while read-0 resource; do
+                eval "echo \"  - \$${resource_type^^}STORE/\$service\$resource:\$resource:rw\""
+            done < <(echo "$metadata" | shyaml get-values-0 "${resource_type}-resources" 2>/dev/null)
+        done
+        while read-0 resource; do
+            if [[ "$resource" ==  *:* ]]; then
+                echo "  - $resource:rw"
+            else
+                echo "  - $resource:$resource:rw"
+            fi
+        done < <(echo "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null)
+           )
+    if [ "$volumes" ]; then
+        echo "volumes:"
+        echo "$volumes"
+    fi
+
+    ## resources to volumes
+    image=$(echo "$metadata" | shyaml get-values "docker-image" 2>/dev/null)
+    if [ "$image" ]; then
+        echo "image: $image"
+    else
+        if ! [ -d "$CHARM_STORE/$service/build" ]; then
+            die "No 'docker-image' value set in 'metadata.yml'  nor 'build/' directory found in charm $DARKYELLOW$service$NORMAL."
+        fi
+        echo "build: $service/build"
+    fi
+}
+export -f get_docker_compose_entry_from_metadata
+
+launch_docker_compose() {
+    debug "Creating temporary docker-compose directory in '$tmpdir'."
+    tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX)
+    function finish {
+        debug "Removing temporary docker-compose directory in '$tmpdir'."
+        rm -rf "$tmpdir"
+    }
+    trap finish EXIT
+    get_compose_def > "$tmpdir/docker-compose.yml" || return 1
+    ## XXXvlab: could be more specific and only link the needed charms
+    ln -sf "$CHARM_STORE/"* "$tmpdir/"
+    cd "$tmpdir" && docker-compose "$@"
+}
+
+
+
+##
+## Argument parsing
+##
+
+fullargs=()
+opts=()
+posargs=()
+no_hooks=
+no_init=
+while [ "$#" != 0 ]; do
+    case "$1" in
+        # --help|-h)
+        #     print_help
+        #     exit 0
+        #     ;;
+        --verbose|-v)
+            fullargs+=("$1")
+            export VERBOSE=true
+            ;;
+        --no-hooks)
+            export no_hooks=true
+            ;;
+        --no-init)
+            export no_init=true
+            ;;
+        --debug)
+            export DEBUG=true
+            export VERBOSE=true
+            ;;
+        --)
+            fullargs+=("$1")
+            shift
+            opts=("${opts[@]}" "$@")
+            break 2
+            ;;
+        -*)
+            fullargs+=("$1")
+            opts=("${opts[@]}" "$1" "$2")
+            shift
+            ;;
+        *)
+            fullargs+=("$1")
+            posargs=("${posargs[@]}" "$1")
+            ;;
+    esac
+    shift
+done
+
+
+##
+## Actual code
+##
+
+export CHARM_STORE=${CHARM_STORE:-/srv/charm-store}
+export DOCKER_DATASTORE=${DOCKER_DATASTORE:-/srv/docker-datastore}
+
+
+## XXXvlab: should provide YML config opportunities in possible parent dirs ?
+##  userdir ? and global /etc/compose.yml ?
+. /etc/compose.conf
+. /etc/compose.local.conf
+
+if ! [ -d "$CHARM_STORE" ]; then
+    err "Charm store path $YELLOW$CHARM_STORE$NORMAL does not exists. "
+    err "Please check your $YELLOW\$CHARM_STORE$NORMAL variable value."
+    exit 1
+fi
+
+
+if [ -z "$(cd "$CHARM_STORE"; ls)" ]; then
+    err "no available charms in charm store $YELLOW$CHARM_STORE$NORMAL. Either:"
+    err "  - check $YELLOW\$CHARM_STORE$NORMAL variable value"
+    err "  - download charms in $CHARM_STORE"
+    print_error "Charm store is empty. Cannot continue."
+fi
+
+
+action="${posargs[0]}"
+case "$action" in
+    load|save)
+        service="${posargs[1]}"
+        run_service_action "$service" "$action" "${opts[@]}" "${posargs[@]:2}"
+        ;;
+    up)
+        service="${posargs[1]}"
+
+        ## init in order
+        [ "$no_init" ] || run_service_hook "$service" init
+
+        ## XXXvlab: to be removed when all relation and service stuff is resolved
+        if [ -z "$no_hooks" ]; then
+            for script in "$CHARM_STORE/"*/hooks.d/*.sh; do
+                [ -e "$script" ] || continue
+                [ -x "$script" ] || { echo "compose: script $script is not executable." >&2; exit 1; }
+                (
+                    cd "$(dirname "$script/..")";
+                    "$script" "$@"
+                ) || { echo "compose: hook $script failed. Stopping." >&2; exit 1; }
+            done
+        fi
+
+        launch_docker_compose "${fullargs[@]}"
+        ;;
+    *) launch_docker_compose "${fullargs[@]}";;
+esac

sample/compose.sample.yml

@@ -0,0 +1,75 @@
+##
+## Links deployable services
+##
+
+## Syntax:
+
+## <SERVICE>:
+##   charm: <CHARM>         # optional: defaults to charm of same name
+##   link:
+##     <RELATION>: <CHARM>  # or:
+##     <RELATION>:
+##       <SERVICE>:
+##         <YML RELATION'S CONFIGURATION>
+##
+
+## launching up a SERVICE, will spawn required SERVICES for given linked RELATIONS.
+## if SERVICE has no definition here AND has a charm with no requirements, it'll be
+## spawned on itself as if a unwritten empty SERVICE definition existed:
+##
+##   <SERVICE>:
+##
+
+
+enquetes:
+  charm: enquetes
+  link:
+    publish-dir:
+      apache:
+        source: git:$MAIN_GIT/enquete
+        branch: master
+        location: /opt/apps/limesurvey
+        data_dirs:
+          - tmp
+          - upload
+        domain: enquetes.$MAIN_DOMAIN
+    mysql-database:
+      mysql:
+        user: lmenquete
+        dbname: lmenquete
+
+fo:
+  charm: formanoo_nfo
+  link:
+    publish-dir:
+      apache:
+        source: git:$MAIN_GIT/formanoo_nfo
+        branch: master
+        location: /opt/apps/formanoo_nfo
+        data_dirs:
+          - assets/eclasseurs
+          - assets/mpdf/tmp
+          - api/v1/mpdf/tmp
+          - users
+        domain: www.$MAIN_DOMAIN
+    postgres-database:
+      postgres:
+        user: formanoo_nfo
+        dbname: formanoo_nfo
+
+bo:
+  charm: odoo
+  link:
+    pg-database:
+      postgres:
+        user: odoo
+        dbname: extranet
+    web-proxy: apache
+    birt-reports: birt
+
+monitor:
+  charm: facette
+  link:
+    web-proxy:
+      domain: monitor.${MAIN_DOMAIN}
+