commit d2867007b9ab9d4da9c34904a1c6b2a90c6e134a Author: Valentin Lab Date: Tue Dec 29 18:39:08 2015 +0700 first import diff --git a/bin/compose b/bin/compose new file mode 100755 index 0000000..72163c7 --- /dev/null +++ b/bin/compose @@ -0,0 +1,635 @@ +#!/bin/bash + +. /etc/shlib + + +include pretty + +depends shyaml docker + +if [ -r /etc/default/charm ]; then + . /etc/default/charm +fi + +if [ -r /etc/default/$exname ]; then + . /etc/default/$exname +fi + +usage="$exname CHARM"' + +Deploy and manage a swarm of containers to provide services based on +a ``compose.yml`` definition and charms from a ``charm-store``. +' + +export DEFAULT_COMPOSE_FILE + +## +## Functions +## + +export APACHE_CONFIG_LOCATION=$CONFIGSTORE/apache/etc/apache2/sites-enabled + +apache_ssl_proxy_config () { + local DOMAIN=$1 TARGET=$2 + + cat < + + + ServerAdmin ${ADMIN_MAIL:-contact@$DOMAIN} + ServerName ${DOMAIN} + + ServerSignature Off + CustomLog /var/log/apache2/s-${DOMAIN}_access.log combined + ErrorLog /var/log/apache2/s-${DOMAIN}_error.log + ErrorLog syslog:local2 + + + ProxyRequests Off + + Order deny,allow + Allow from all + + ProxyVia On + ProxyPass / http://$TARGET/ retry=0 + + ProxyPassReverse / + + + + ## Forbid any cache, this is only usefull on dev server. + #Header set Cache-Control "no-cache" + #Header set Access-Control-Allow-Origin "*" + #Header set Access-Control-Allow-Methods "POST, GET, OPTIONS" + #Header set Access-Control-Allow-Headers "origin, content-type, accept" + + RequestHeader set "X-Forwarded-Proto" "https" + + ## Fix IE problem (httpapache proxy dav error 408/409) + SetEnv proxy-nokeepalive 1 + #ServerSignature On + SSLProxyEngine On + SSLEngine On + + ## Full stance + SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem + SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key + + SSLVerifyClient None + + + + +EOF + +} +export -f apache_ssl_proxy_config + +apache_ssl_config() { + local DOMAIN=$1 + + cat < + + + ServerAdmin ${ADMIN_MAIL:-contact@$DOMAIN} + ServerName ${DOMAIN} + + ServerSignature Off + CustomLog /var/log/apache2/s-${DOMAIN}_access.log combined + ErrorLog /var/log/apache2/s-${DOMAIN}_error.log + ErrorLog syslog:local2 + + DocumentRoot /var/www/${DOMAIN} + + + Options FollowSymLinks + AllowOverride None + + + + Options Indexes FollowSymLinks MultiViews + AllowOverride all + Order allow,deny + allow from all + + + SSLEngine On + + ## Full stance + SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem + SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key + SSLVerifyClient None + + + + +EOF + +} +export -f apache_ssl_config + +apache_ssl_add () { + local DOMAIN=$1 + [ -e "$APACHE_CONFIG_LOCATION/$DOMAIN.conf" ] && return 0 + mkdir -p "$APACHE_CONFIG_LOCATION" + apache_ssl_config $DOMAIN > $APACHE_CONFIG_LOCATION/$DOMAIN.conf + echo "Added $DOMAIN apache config." >&2 + +} +export -f apache_ssl_add +apache_ssl_proxy_add () { + local DOMAIN=$1 TARGET=$2 + [ -e "$APACHE_CONFIG_LOCATION/$DOMAIN.conf" ] && return 0 + + mkdir -p "$APACHE_CONFIG_LOCATION" + apache_ssl_proxy_config $DOMAIN $TARGET > $APACHE_CONFIG_LOCATION/$DOMAIN.conf + echo "Added $DOMAIN as a proxy to $TARGET." >&2 +} +export -f apache_ssl_proxy_add + +gen_password() { + python -c 'import random; \ + xx = "azertyuiopqsdfghjklmwxcvbn1234567890AZERTYUIOPQSDFGHJKLMWXCVBN+_-"; \ + print "".join([xx[random.randint(0, len(xx)-1)] for x in range(0, 14)])' +} +export -f gen_password + + +file_put() { + local TARGET="$1" + mkdir -p "$(dirname "$TARGET")" && + cat - > "$TARGET" +} +export -f file_put + + +apache_data_dir() { + local DOMAIN=$1 DATA_COMMA_SEPARATED=$2 + + export APACHE_DOCKER_IMAGE=$(service_base_docker_image apache) + +DOCKER_SITE_PATH=/var/www/$DOMAIN +BASE=$DATASTORE/apache +DST=$BASE/$DOCKER_SITE_PATH + DATA=() + while IFS="," read -ra ADDR; do + for dir in "${ADDR[@]}"; do + mkdir -p "$DST/$dir" + DATA+=($dir) + done + done <<< "$DATA_COMMA_SEPARATED" + + if [ -z "$APACHE_DOCKER_GID" ] && + ! grep "^export APACHE_DOCKER_GID=" /etc/compose.local.conf >/dev/null 2>&1; then + echo "Adding APACHE_DOCKER_GID to '/etc/compose.local.conf'." + + export APACHE_DOCKER_GID=$(docker run "$APACHE_DOCKER_IMAGE" id -g www-data) + + cat <> /etc/compose.local.conf +export APACHE_DOCKER_GID=$APACHE_DOCKER_GID +EOF + + fi + + dirs=() + for d in "${DATA[@]}"; do + dirs+=("$DST/$d") + done + + chgrp www-data "${dirs[@]}" -R && chmod 775 "${dirs[@]}" -R +} +export -f apache_data_dir + + +export _DOCKER_COMPOSE_DEF="" +get_compose_def() { + local local_compose + + if [ "$_DOCKER_COMPOSE_DEF" ]; then + echo "$_DOCKER_COMPOSE_DEF" + return 0 + fi + + ## + ## Adding sub services configurations + ## + + additional_services= + if [ -z "$*" ]; then + info "No service provided, using \$DEFAULT_SERVICES variable. Target services: $DEFAULT_SERVICES" + additional_services=$DEFAULT_SERVICES + fi + declare -A loaded + for target_service in "$@" $additional_services; do + + services=$(get_ordered_service_dependencies "$target_service") || return 1 + for service in $services; do + + if [ "${loaded[$service]}" ]; then + continue + fi + loaded[$service]=1 + export _DOCKER_COMPOSE_DEF="$_DOCKER_COMPOSE_DEF +$service: +$(get_service_def "$service" | sed -r 's/^/ /g')" + + done + done + echo "$_DOCKER_COMPOSE_DEF" +} +export -f get_compose_def + +get_service_def() { + local service="$1" + + if [ -z "$service" ]; then + echo "Please specify a service." >&2 + return 1 + fi + + if [ -d "$CHARM_STORE/$service" ]; then + compose_file="$CHARM_STORE/$service/compose.yml" + local_compose="" + if [ -e "$compose_file" ]; then + debug "Found compose.yml in $service directory. Including in 'docker-compose.conf'." + local_compose="$(cat "$compose_file")" + fi + metadata_file="$CHARM_STORE/$service/metadata.yml" + if [ -e "$metadata_file" ]; then + debug "Found metadata.yml in $service directory. Including in 'docker-compose.conf'." + docker_compose_entry=$(get_docker_compose_entry_from_metadata "$service" < "$metadata_file") || return 1 + local_compose="$local_compose +$docker_compose_entry" + fi + echo "$local_compose" + return 0 + fi + + err "service '$DARKYELLOW$service$NORMAL' not found." >&2 + return 1 +} +export -f get_service_def + +service_base_docker_image() { + local service="$1" + service_def="$(get_service_def "$service")" || return 1 + + service_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null) + if [ "$?" != 0 ]; then + service_build=$(echo "$service_def" | shyaml get-value build) + if [ "$?" != 0 ]; then + echo "Service '$service' has no 'image' nor 'build' parameter." >&2 + return 1 + fi + service_dockerfile="$COMPOSE_YML_PATH/$service_build/Dockerfile" + if ! [ -e "$service_dockerfile" ]; then + echo "No Dockerfile found in '$service_dockerfile' location." >&2 + return 1 + fi + + grep '^FROM' "$service_dockerfile" | xargs echo | cut -f 2 -d " " + else + echo "$service_image" + fi + +} +export -f service_base_docker_image + + +read-0() { + local eof + eof= + while [ "$1" ]; do + IFS='' read -r -d '' "$1" || eof=true + shift + done + test "$eof" != true +} +export -f read-0 + +array_values_to_stdin() { + local e + if [ "$#" -ne "1" ]; then + print_syntax_warning "$FUNCNAME: need one argument." + return 1 + fi + var="$1" + eval "for e in \"\${$var[@]}\"; do echo -en \"\$e\\0\"; done" +} + +array_keys_to_stdin() { + local e + if [ "$#" -ne "1" ]; then + print_syntax_warning "$FUNCNAME: need one argument." + return 1 + fi + var="$1" + eval "for e in \"\${!$var[@]}\"; do echo -en \"\$e\\0\"; done" +} + +array_kv_to_stdin() { + local e + if [ "$#" -ne "1" ]; then + print_syntax_warning "$FUNCNAME: need one argument." + return 1 + fi + var="$1" + eval "for e in \"\${!$var[@]}\"; do echo -n \"\$e\"; echo -en '\0'; echo -n \"\${$var[\$e]}\"; echo -en '\0'; done" +} + + +array_pop() { + local narr="$1" nres="$2" + for key in $(eval "echo \${!$narr[@]}"); do + eval "$nres=\${$narr[\"\$key\"]}" + eval "unset $narr[\"\$key\"]" + return 0 + done +} +export -f array_pop + +array_member() { + local src elt + src="$1" + elt="$2" + while read-0 key; do + if [ "$(eval "echo -n \"\${$src[\$key]}\"")" == "$elt" ]; then + return 0 + fi + done < <(array_keys_to_stdin "$src") + return 1 +} +export -f array_member + + +get_service_deps() { + local service="$1" + service_def=$(get_service_def "$service") || return 1 + echo "$service_def" | shyaml get-values links 2>/dev/null + return 0 +} +export -f get_service_deps + +## a service is not always a container. +## XXXvlab: a service name should not be a container name neither... see this later. +# get_container_name() { +# local service="$1" +# get_service_def "$service" | shyaml get-values links 2>/dev/null +# if [ "$(get_md_service_def "$service" | shyaml get-value subordinate 2>/dev/null)" != "true" ]; then +# echo "$service" +# return 0 +# fi + +# } + +_rec_get_depth() { + local elt=$1 + + if [ "${depths[$elt]}" ]; then + return 0 + fi + deps=$(get_service_deps "$elt") || return 1 + if [ -z "$deps" ]; then + depths[$elt]=0 + fi + + max=0 + for dep in $deps; do + _rec_get_depth "$dep" || return 1 + if (( "${depths[$dep]}" > "$max" )); then + max="${depths[$dep]}" + fi + done + depths[$elt]=$((max + 1)) +} + +get_ordered_service_dependencies() { + local services=("$@") + + if [ -z "${services[*]}" ]; then + print_syntax_error "$FUNCNAME: no arguments" + fi + + declare -A depths + visited=() + heads=("${services[@]}") + while [ "${#heads[@]}" != 0 ]; do + array_pop heads head + visited+=("$head") + _rec_get_depth "$head" || return 1 + done + + i=0 + while [ "${#depths[@]}" != 0 ]; do + for key in "${!depths[@]}"; do + value="${depths[$key]}" + if [ "$value" == "$i" ]; then + echo "$key" + unset depths[$key] + fi + done + i=$((i + 1)) + done +} + +run_service_hook () { + local service="$1" action="$2" + + services=$(get_ordered_service_dependencies "$service") || return 1 + ## init in order + for service in $services; do + TARGET_SCRIPT="$COMPOSE_YML_PATH/service/$service/hooks/$2" + [ -e "$TARGET_SCRIPT" ] && { + [ "$verbose" ] && echo "Init $service" + SERVICE_NAME=$service \ + DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") \ + SERVICE_DATASTORE="$DATASTORE/$service" \ + SERVICE_CONFIGSTORE="$CONFIGSTORE/$service" \ + "$TARGET_SCRIPT" + } + done +} + +run_service_action () { + local service="$1" action="$2" + shift shift + run_service_hook "$service" init + + services=$(get_ordered_service_dependencies "$service") || return 1 + for service in $services; do + TARGET_SCRIPT="$COMPOSE_YML_PATH/service/$service/actions/$2" + if [ -e "$TARGET_SCRIPT" ]; then + [ "$verbose" ] && echo "Init $service" + SERVICE_NAME=$service \ + CONTAINER_NAME=$(get_container_name "$service") \ + DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") \ + SERVICE_DATASTORE="$DATASTORE/$service" \ + SERVICE_CONFIGSTORE="$CONFIGSTORE/$service" \ + echo "$TARGET_SCRIPT" "$@" + else + echo "Service '$service' does not have any action '$action' defined." >&2 + return 1 + fi + done +} + +get_docker_compose_entry_from_metadata() { + local service="$1" + metadata="$(cat -)" + + export DATASTORE CONFIGSTORE + ## resources to volumes + volumes=$( + for resource_type in data config; do + while read-0 resource; do + eval "echo \" - \$${resource_type^^}STORE/\$service\$resource:\$resource:rw\"" + done < <(echo "$metadata" | shyaml get-values-0 "${resource_type}-resources" 2>/dev/null) + done + while read-0 resource; do + if [[ "$resource" == *:* ]]; then + echo " - $resource:rw" + else + echo " - $resource:$resource:rw" + fi + done < <(echo "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null) + ) + if [ "$volumes" ]; then + echo "volumes:" + echo "$volumes" + fi + + ## resources to volumes + image=$(echo "$metadata" | shyaml get-values "docker-image" 2>/dev/null) + if [ "$image" ]; then + echo "image: $image" + else + if ! [ -d "$CHARM_STORE/$service/build" ]; then + die "No 'docker-image' value set in 'metadata.yml' nor 'build/' directory found in charm $DARKYELLOW$service$NORMAL." + fi + echo "build: $service/build" + fi +} +export -f get_docker_compose_entry_from_metadata + +launch_docker_compose() { + debug "Creating temporary docker-compose directory in '$tmpdir'." + tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX) + function finish { + debug "Removing temporary docker-compose directory in '$tmpdir'." + rm -rf "$tmpdir" + } + trap finish EXIT + get_compose_def > "$tmpdir/docker-compose.yml" || return 1 + ## XXXvlab: could be more specific and only link the needed charms + ln -sf "$CHARM_STORE/"* "$tmpdir/" + cd "$tmpdir" && docker-compose "$@" +} + + + +## +## Argument parsing +## + +fullargs=() +opts=() +posargs=() +no_hooks= +no_init= +while [ "$#" != 0 ]; do + case "$1" in + # --help|-h) + # print_help + # exit 0 + # ;; + --verbose|-v) + fullargs+=("$1") + export VERBOSE=true + ;; + --no-hooks) + export no_hooks=true + ;; + --no-init) + export no_init=true + ;; + --debug) + export DEBUG=true + export VERBOSE=true + ;; + --) + fullargs+=("$1") + shift + opts=("${opts[@]}" "$@") + break 2 + ;; + -*) + fullargs+=("$1") + opts=("${opts[@]}" "$1" "$2") + shift + ;; + *) + fullargs+=("$1") + posargs=("${posargs[@]}" "$1") + ;; + esac + shift +done + + +## +## Actual code +## + +export CHARM_STORE=${CHARM_STORE:-/srv/charm-store} +export DOCKER_DATASTORE=${DOCKER_DATASTORE:-/srv/docker-datastore} + + +## XXXvlab: should provide YML config opportunities in possible parent dirs ? +## userdir ? and global /etc/compose.yml ? +. /etc/compose.conf +. /etc/compose.local.conf + +if ! [ -d "$CHARM_STORE" ]; then + err "Charm store path $YELLOW$CHARM_STORE$NORMAL does not exists. " + err "Please check your $YELLOW\$CHARM_STORE$NORMAL variable value." + exit 1 +fi + + +if [ -z "$(cd "$CHARM_STORE"; ls)" ]; then + err "no available charms in charm store $YELLOW$CHARM_STORE$NORMAL. Either:" + err " - check $YELLOW\$CHARM_STORE$NORMAL variable value" + err " - download charms in $CHARM_STORE" + print_error "Charm store is empty. Cannot continue." +fi + + +action="${posargs[0]}" +case "$action" in + load|save) + service="${posargs[1]}" + run_service_action "$service" "$action" "${opts[@]}" "${posargs[@]:2}" + ;; + up) + service="${posargs[1]}" + + ## init in order + [ "$no_init" ] || run_service_hook "$service" init + + ## XXXvlab: to be removed when all relation and service stuff is resolved + if [ -z "$no_hooks" ]; then + for script in "$CHARM_STORE/"*/hooks.d/*.sh; do + [ -e "$script" ] || continue + [ -x "$script" ] || { echo "compose: script $script is not executable." >&2; exit 1; } + ( + cd "$(dirname "$script/..")"; + "$script" "$@" + ) || { echo "compose: hook $script failed. Stopping." >&2; exit 1; } + done + fi + + launch_docker_compose "${fullargs[@]}" + ;; + *) launch_docker_compose "${fullargs[@]}";; +esac diff --git a/sample/compose.sample.yml b/sample/compose.sample.yml new file mode 100644 index 0000000..1670032 --- /dev/null +++ b/sample/compose.sample.yml @@ -0,0 +1,75 @@ +## +## Links deployable services +## + +## Syntax: + +## : +## charm: # optional: defaults to charm of same name +## link: +## : # or: +## : +## : +## +## + +## launching up a SERVICE, will spawn required SERVICES for given linked RELATIONS. +## if SERVICE has no definition here AND has a charm with no requirements, it'll be +## spawned on itself as if a unwritten empty SERVICE definition existed: +## +## : +## + + +enquetes: + charm: enquetes + link: + publish-dir: + apache: + source: git:$MAIN_GIT/enquete + branch: master + location: /opt/apps/limesurvey + data_dirs: + - tmp + - upload + domain: enquetes.$MAIN_DOMAIN + mysql-database: + mysql: + user: lmenquete + dbname: lmenquete + +fo: + charm: formanoo_nfo + link: + publish-dir: + apache: + source: git:$MAIN_GIT/formanoo_nfo + branch: master + location: /opt/apps/formanoo_nfo + data_dirs: + - assets/eclasseurs + - assets/mpdf/tmp + - api/v1/mpdf/tmp + - users + domain: www.$MAIN_DOMAIN + postgres-database: + postgres: + user: formanoo_nfo + dbname: formanoo_nfo + +bo: + charm: odoo + link: + pg-database: + postgres: + user: odoo + dbname: extranet + web-proxy: apache + birt-reports: birt + +monitor: + charm: facette + link: + web-proxy: + domain: monitor.${MAIN_DOMAIN} +