From 7c48bbb6770da360f89bbc126841a55641b63db9 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Wed, 30 Dec 2015 13:54:44 +0700 Subject: [PATCH] fix: merging new ``compose.yml`` to get the relations informations. --- bin/compose | 1522 +++++++++++++++++++++++++++++++++++++++++---------- test/test | 1098 +++++++++++++++++++++++++++++++++++++ 2 files changed, 2318 insertions(+), 302 deletions(-) create mode 100755 test/test diff --git a/bin/compose b/bin/compose index 72163c7..7f64b5f 100755 --- a/bin/compose +++ b/bin/compose @@ -1,19 +1,19 @@ #!/bin/bash -. /etc/shlib +#:- +. /etc/shlib +#:- include pretty +include parse depends shyaml docker -if [ -r /etc/default/charm ]; then - . /etc/default/charm -fi +[[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true -if [ -r /etc/default/$exname ]; then - . /etc/default/$exname -fi + +export VARDIR=/var/lib/compose usage="$exname CHARM"' @@ -24,129 +24,103 @@ a ``compose.yml`` definition and charms from a ``charm-store``. export DEFAULT_COMPOSE_FILE ## -## Functions +## Merge YAML files ## -export APACHE_CONFIG_LOCATION=$CONFIGSTORE/apache/etc/apache2/sites-enabled - -apache_ssl_proxy_config () { - local DOMAIN=$1 TARGET=$2 - - cat < - - - ServerAdmin ${ADMIN_MAIL:-contact@$DOMAIN} - ServerName ${DOMAIN} - - ServerSignature Off - CustomLog /var/log/apache2/s-${DOMAIN}_access.log combined - ErrorLog /var/log/apache2/s-${DOMAIN}_error.log - ErrorLog syslog:local2 - - - ProxyRequests Off - - Order deny,allow - Allow from all - - ProxyVia On - ProxyPass / http://$TARGET/ retry=0 - - ProxyPassReverse / - - - - ## Forbid any cache, this is only usefull on dev server. - #Header set Cache-Control "no-cache" - #Header set Access-Control-Allow-Origin "*" - #Header set Access-Control-Allow-Methods "POST, GET, OPTIONS" - #Header set Access-Control-Allow-Headers "origin, content-type, accept" - - RequestHeader set "X-Forwarded-Proto" "https" - - ## Fix IE problem (httpapache proxy dav error 408/409) - SetEnv proxy-nokeepalive 1 - #ServerSignature On - SSLProxyEngine On - SSLEngine On - - ## Full stance - SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem - SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key - - SSLVerifyClient None - - - - +_merge_yaml_common_code=" +import sys +import yaml + +def fc(filename): + with open(filename) as f: + return f.read() + +def merge(*args): + # sys.stderr.write('%r\n' % (args, )) + args = [arg for arg in args if arg is not None] + if len(args) == 0: + return None + if len(args) == 1: + return args[0] + if all(isinstance(arg, (int, basestring, bool)) for arg in args): + return args[-1] + elif all(isinstance(arg, list) for arg in args): + res = [] + for arg in args: + res.extend(arg) + return res + elif all(isinstance(arg, dict) for arg in args): + keys = set() + for arg in args: + keys |= set(arg.keys()) + dct = {} + for key in keys: + sub_args = [] + for arg in args: + if key in arg: + sub_args.append(arg) + try: + dct[key] = merge(*(a[key] for a in sub_args)) + except NotImplementedError as e: + raise NotImplementedError( + e.args[0], + '%s.%s' % (key, e.args[1]) if e.args[1] else key, + e.args[2]) + if dct[key] is None: + del dct[key] + return dct + else: + raise NotImplementedError( + 'Unsupported types: %s' + % (', '.join(list(set(arg.__class__.__name__ for arg in args)))), '', args) + return None + +def merge_cli(*args): + try: + c = merge(*args) + except NotImplementedError as e: + sys.stderr.write('%s. Conflicting key is %r. Values are:\n%s\n' % (e.args[0], e.args[1], e.args[2])) + exit(1) + if c is not None: + print '%s' % yaml.dump(c, default_flow_style=False) + +" + + +merge_yaml() { + + if ! [ -r "$state_tmpdir/merge_yaml.py" ]; then + cat < "$state_tmpdir/merge_yaml.py" + +$_merge_yaml_common_code + +merge_cli(*(yaml.load(fc(f)) for f in sys.argv[1:])) EOF + fi + python "$state_tmpdir/merge_yaml.py" "$@" } -export -f apache_ssl_proxy_config - -apache_ssl_config() { - local DOMAIN=$1 - - cat < +export -f merge_yaml - - ServerAdmin ${ADMIN_MAIL:-contact@$DOMAIN} - ServerName ${DOMAIN} +merge_yaml_str() { + local entries="$@" - ServerSignature Off - CustomLog /var/log/apache2/s-${DOMAIN}_access.log combined - ErrorLog /var/log/apache2/s-${DOMAIN}_error.log - ErrorLog syslog:local2 + if ! [ -r "$state_tmpdir/merge_yaml_str.py" ]; then + cat < "$state_tmpdir/merge_yaml_str.py" - DocumentRoot /var/www/${DOMAIN} +$_merge_yaml_common_code - - Options FollowSymLinks - AllowOverride None - - - - Options Indexes FollowSymLinks MultiViews - AllowOverride all - Order allow,deny - allow from all - - - SSLEngine On - - ## Full stance - SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem - SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key - SSLVerifyClient None - - - - +merge_cli(*(yaml.load(f) for f in sys.argv[1:])) EOF + fi + python "$state_tmpdir/merge_yaml_str.py" "$@" } -export -f apache_ssl_config - -apache_ssl_add () { - local DOMAIN=$1 - [ -e "$APACHE_CONFIG_LOCATION/$DOMAIN.conf" ] && return 0 - mkdir -p "$APACHE_CONFIG_LOCATION" - apache_ssl_config $DOMAIN > $APACHE_CONFIG_LOCATION/$DOMAIN.conf - echo "Added $DOMAIN apache config." >&2 - -} -export -f apache_ssl_add -apache_ssl_proxy_add () { - local DOMAIN=$1 TARGET=$2 - [ -e "$APACHE_CONFIG_LOCATION/$DOMAIN.conf" ] && return 0 +export -f merge_yaml_str - mkdir -p "$APACHE_CONFIG_LOCATION" - apache_ssl_proxy_config $DOMAIN $TARGET > $APACHE_CONFIG_LOCATION/$DOMAIN.conf - echo "Added $DOMAIN as a proxy to $TARGET." >&2 -} -export -f apache_ssl_proxy_add +## +## Functions +## gen_password() { python -c 'import random; \ @@ -164,50 +138,84 @@ file_put() { export -f file_put -apache_data_dir() { - local DOMAIN=$1 DATA_COMMA_SEPARATED=$2 +_get_docker_compose_links() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" links charm charm_part + if [ -z "$service" ]; then + print_syntax_error "$FUNCNAME: Please specify a service as first argument." + return 1 + fi - export APACHE_DOCKER_IMAGE=$(service_base_docker_image apache) + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi -DOCKER_SITE_PATH=/var/www/$DOMAIN -BASE=$DATASTORE/apache -DST=$BASE/$DOCKER_SITE_PATH - DATA=() - while IFS="," read -ra ADDR; do - for dir in "${ADDR[@]}"; do - mkdir -p "$DST/$dir" - DATA+=($dir) - done - done <<< "$DATA_COMMA_SEPARATED" + master_charm=$(_get_master_charm_for_service "$service") || return 1 - if [ -z "$APACHE_DOCKER_GID" ] && - ! grep "^export APACHE_DOCKER_GID=" /etc/compose.local.conf >/dev/null 2>&1; then - echo "Adding APACHE_DOCKER_GID to '/etc/compose.local.conf'." + deps=() + while read-0 relation_name target_service relation_config reverse; do + [ "$master_charm" == "$target_service" ] && continue + if [ "$reverse" ]; then + deps+=("$(echo -en "$target_service:\n links:\n - $master_charm")") + else + deps+=("$(echo -en "$master_charm:\n links:\n - $target_service")") + fi + done < <(get_compose_relations "$service") || return 1 - export APACHE_DOCKER_GID=$(docker run "$APACHE_DOCKER_IMAGE" id -g www-data) + merge_yaml_str "${deps[@]}" | tee "$cache_file" +} - cat <> /etc/compose.local.conf -export APACHE_DOCKER_GID=$APACHE_DOCKER_GID -EOF +## +## By Reading the metadata.yml, we create a docker-compose.yml mixin. +## Some metadata.yml (of subordinates) will indeed modify other +## services than themselves. +_get_docker_compose_service_mixin() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" links charm charm_part + if [ -z "$service" ]; then + print_syntax_error "$FUNCNAME: Please specify a service as first argument." + return 1 + fi + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 fi - dirs=() - for d in "${DATA[@]}"; do - dirs+=("$DST/$d") - done + master_charm=$(_get_master_charm_for_service "$service") || return 1 - chgrp www-data "${dirs[@]}" -R && chmod 775 "${dirs[@]}" -R -} -export -f apache_data_dir + ## The compose part + + links_yaml=$(_get_docker_compose_links "$service") || return 1 + + ## the charm part + #debug "Get charm name from service name $DARKYELLOW$service$NORMAL." + charm=$(get_service_charm "$service") || return 1 + charm_part=$(get_docker_compose_mixin_from_metadata "$charm") || return 1 -export _DOCKER_COMPOSE_DEF="" -get_compose_def() { - local local_compose + ## Merge results + + if [ "$charm_part" ]; then + charm_yaml="$(echo -en "${master_charm}:\n$(echo "$charm_part" | prefix " ")")" + merge_yaml_str "$links_yaml" "$charm_yaml" + else + echo "$links_yaml" + fi > "$cache_file" + cat "$cache_file" +} +export -f _get_docker_compose_service_mixin - if [ "$_DOCKER_COMPOSE_DEF" ]; then - echo "$_DOCKER_COMPOSE_DEF" +## +## Get full `docker-compose.yml` format for all listed services (and +## their deps) +## +get_docker_compose() { + local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" entries services + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" return 0 fi @@ -215,76 +223,115 @@ get_compose_def() { ## Adding sub services configurations ## - additional_services= - if [ -z "$*" ]; then - info "No service provided, using \$DEFAULT_SERVICES variable. Target services: $DEFAULT_SERVICES" - additional_services=$DEFAULT_SERVICES - fi - declare -A loaded - for target_service in "$@" $additional_services; do + declare -A entries + for target_service in "$@"; do services=$(get_ordered_service_dependencies "$target_service") || return 1 for service in $services; do - if [ "${loaded[$service]}" ]; then + if [ "${entries[$service]}" ]; then + ## Prevent double inclusion of same service if this + ## service is deps of two or more of your + ## requirements. continue fi - loaded[$service]=1 - export _DOCKER_COMPOSE_DEF="$_DOCKER_COMPOSE_DEF -$service: -$(get_service_def "$service" | sed -r 's/^/ /g')" + ## mark the service as "loaded" as well as it's containers + ## if this is a subordinate service + entries[$service]=$(_get_docker_compose_service_mixin "$service") || return 1 done done - echo "$_DOCKER_COMPOSE_DEF" + + merge_yaml_str "${entries[@]}" > "$cache_file" + export _current_docker_compose="$(cat "$cache_file")" + echo "$_current_docker_compose" } -export -f get_compose_def +export -f get_docker_compose -get_service_def() { - local service="$1" - if [ -z "$service" ]; then - echo "Please specify a service." >&2 - return 1 +## XXXvlab: a lot to be done to cache the results +get_compose_service_def () { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 fi + [ -z "$service" ] && print_syntax_error "Missing service as first argument." + + service_def_base= if [ -d "$CHARM_STORE/$service" ]; then - compose_file="$CHARM_STORE/$service/compose.yml" - local_compose="" - if [ -e "$compose_file" ]; then - debug "Found compose.yml in $service directory. Including in 'docker-compose.conf'." - local_compose="$(cat "$compose_file")" - fi - metadata_file="$CHARM_STORE/$service/metadata.yml" - if [ -e "$metadata_file" ]; then - debug "Found metadata.yml in $service directory. Including in 'docker-compose.conf'." - docker_compose_entry=$(get_docker_compose_entry_from_metadata "$service" < "$metadata_file") || return 1 - local_compose="$local_compose -$docker_compose_entry" - fi - echo "$local_compose" - return 0 + service_def_base="charm: $service" + fi + value= + if [ -r "$COMPOSE_YML_FILE" ]; then + value=$(shyaml get-value "$service" 2>/dev/null < "$COMPOSE_YML_FILE") fi - err "service '$DARKYELLOW$service$NORMAL' not found." >&2 - return 1 + if [ -z "$service_def_base" -a -z "$value" ]; then + err "Invalid service $DARKYELLOW$service$NORMAL: no definition in" \ + "compose file nor charm with same name." + return 1 + fi + + merge_yaml <(echo "$service_def_base") <(echo "$value") > "$cache_file" + cat "$cache_file" +} +export -f get_compose_service_def + + +## XXXvlab: MUST CACHE +get_service_charm () { + local service="$1" + if [ -z "$service" ]; then + print_syntax_error "$FUNCNAME: Please specify a service as first argument." + return 1 + fi + service_def=$(get_compose_service_def "$service") || return 1 + charm=$(echo "$service_def" | shyaml get-value charm 2>/dev/null) + if [ -z "$charm" ]; then + err "Missing charm in service $DARKYELLOW$service$NORMAL definition." + return 1 + fi + echo "$charm" +} +export -f get_service_charm + +## built above the docker-compose abstraction, so it relies on the +## full docker-compose.yml to be already built. +get_service_def() { + local service="$1" def + if [ -z "$_current_docker_compose" ]; then + print_syntax_error "$FUNCNAME is meant to be called after"\ + "\$_current_docker_compose has been calculated." + fi + + def=$(echo "$_current_docker_compose" | shyaml get-value "$service" 2>/dev/null) + if [ -z "$def" ]; then + err "No definition for service $DARKYELLOW$service$NORMAL." + return 1 + fi + echo "$def" } export -f get_service_def + +## Return the base docker image name of a service service_base_docker_image() { local service="$1" service_def="$(get_service_def "$service")" || return 1 - service_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null) if [ "$?" != 0 ]; then - service_build=$(echo "$service_def" | shyaml get-value build) + service_build=$(echo "$service_def" | shyaml get-value build 2>/dev/null) if [ "$?" != 0 ]; then - echo "Service '$service' has no 'image' nor 'build' parameter." >&2 + err "Service $DARKYELLOW$service$NORMAL has no ${WHITE}image${NORMAL} nor ${WHITE}build${NORMAL} parameter." + echo "$service_def" >&2 return 1 fi - service_dockerfile="$COMPOSE_YML_PATH/$service_build/Dockerfile" + service_dockerfile="$CHARM_STORE/${service_build}/Dockerfile" if ! [ -e "$service_dockerfile" ]; then - echo "No Dockerfile found in '$service_dockerfile' location." >&2 + err "No Dockerfile found in '$service_dockerfile' location." return 1 fi @@ -296,17 +343,90 @@ service_base_docker_image() { } export -f service_base_docker_image +## XXXvlab: provided in shlib/common +# read-0() { +# local eof +# eof= +# while [ "$1" ]; do +# IFS='' read -r -d '' "$1" || eof=true +# shift +# done +# test "$eof" != true +# } +# export -f read-0 + +fetch_file() { + local src="$1" -read-0() { - local eof - eof= - while [ "$1" ]; do - IFS='' read -r -d '' "$1" || eof=true - shift - done - test "$eof" != true + case "$src" in + *"://"*) + err "Unsupported target scheme." + return 1 + ;; + *) + ## Try direct + if ! [ -r "$src" ]; then + err "File '$src' not found/readable." + return 1 + fi + cat "$src" || return 1 + ;; + esac +} +export -f fetch_file + +## receives stdin content to decompress on stdout +## stdout content should be tar format. +uncompress_file() { + local filename="$1" + + ## Warning, the content of the file is already as stdin, the filename + ## is there to hint for correct decompression. + case "$filename" in + *".gz") + gunzip + ;; + *".bz2") + bunzip2 + ;; + *) + cat + ;; + esac + +} +export -f uncompress_file + +get_file() { + local src="$1" + + fetch_file "$src" | uncompress_file "$src" } -export -f read-0 +export -f get_file + + +cmd_on_base_image() { + local charm="$1" + shift + base_image=$(service_base_docker_image "$charm") || return 1 + + docker run -i --entrypoint /bin/bash "$base_image" -c "$*" +} +export -f cmd_on_base_image + +cached_cmd_on_base_image() { + local charm="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" + shift + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + base_image=$(service_base_docker_image "$charm") || return 1 + result=$(cmd_on_base_image "$charm" "$@") || return 1 + echo "$result" | tee "$cache_file" +} +export -f cached_cmd_on_base_image array_values_to_stdin() { local e @@ -363,29 +483,60 @@ array_member() { export -f array_member +get_charm_relation_def () { + local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + relation_def metadata + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + metadata="$(get_charm_metadata "$charm")" || return 1 + relation_def="$(echo "$metadata" | shyaml get-value "provides.${relation_name}" 2>/dev/null)" + echo "$relation_def" | tee "$cache_file" +} +export -f get_charm_relation_def + + +get_charm_reverse_tech_dep_relation() { + local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + relation_def metadata + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + relation_def=$(get_charm_relation_def "$charm" "$relation_name" 2>/dev/null) + value=$(echo "$relation_def" | shyaml get-value 'reverse-tech-dep' 2>/dev/null) + echo "$value" | tee "$cache_file" +} +export -f get_charm_reverse_tech_dep_relation + +## +## Use compose file to get deps, and relation definition in metadata.yml +## for reverse-tech-dep attribute. get_service_deps() { - local service="$1" - service_def=$(get_service_def "$service") || return 1 - echo "$service_def" | shyaml get-values links 2>/dev/null - return 0 + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + ( + set -o pipefail + get_compose_relations "$service" | \ + while read-0 relation_name target_service _relation_config reverse; do + echo "$target_service" + done | tee "$cache_file" + ) || return 1 } export -f get_service_deps -## a service is not always a container. -## XXXvlab: a service name should not be a container name neither... see this later. -# get_container_name() { -# local service="$1" -# get_service_def "$service" | shyaml get-values links 2>/dev/null -# if [ "$(get_md_service_def "$service" | shyaml get-value subordinate 2>/dev/null)" != "true" ]; then -# echo "$service" -# return 0 -# fi - -# } _rec_get_depth() { local elt=$1 + #debug "Asking for $DARKYELLOW$elt$NORMAL dependencies" if [ "${depths[$elt]}" ]; then return 0 fi @@ -405,10 +556,17 @@ _rec_get_depth() { } get_ordered_service_dependencies() { - local services=("$@") + local services=("$@") cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + #debug "Figuring ordered deps of $DARKYELLOW$services$NORMAL" if [ -z "${services[*]}" ]; then print_syntax_error "$FUNCNAME: no arguments" + return 1 fi declare -A depths @@ -430,101 +588,757 @@ get_ordered_service_dependencies() { fi done i=$((i + 1)) - done + done > "$cache_file" + cat "$cache_file" } + run_service_hook () { - local service="$1" action="$2" + local services="$1" action="$2" loaded - services=$(get_ordered_service_dependencies "$service") || return 1 - ## init in order + declare -A loaded for service in $services; do - TARGET_SCRIPT="$COMPOSE_YML_PATH/service/$service/hooks/$2" - [ -e "$TARGET_SCRIPT" ] && { - [ "$verbose" ] && echo "Init $service" - SERVICE_NAME=$service \ - DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") \ - SERVICE_DATASTORE="$DATASTORE/$service" \ - SERVICE_CONFIGSTORE="$CONFIGSTORE/$service" \ + for subservice in $(get_ordered_service_dependencies "$service"); do + if [ "${loaded[$subservice]}" ]; then + ## Prevent double inclusion of same service if this + ## service is deps of two or more of your + ## requirements. + continue + fi + + charm=$(get_service_charm "$subservice") || return 1 + TARGET_SCRIPT="$CHARM_STORE/$charm/hooks/$action" + [ -e "$TARGET_SCRIPT" ] || continue + + Wrap -d "$YELLOW$action$NORMAL hook of charm $DARKYELLOW$charm$NORMAL" </dev/null + if [ "$?" != 0 ]; then + err "The key $WHITE$key$NORMAL was not found in relation's data." + return 1 + fi +} +export -f relation-get + + +relation-base-compose-get () { + local key="$1" + echo "$RELATION_BASE_COMPOSE_DEF" | shyaml get-value "options.$key" 2>/dev/null + if [ "$?" != 0 ]; then + err "The key $WHITE$key$NORMAL was not found in base service compose definition.." + return 1 + fi } +export -f relation-base-compose-get + + +relation-target-compose-get () { + local key="$1" + echo "$RELATION_BASE_COMPOSE_DEF" | shyaml get-value "options.$key" 2>/dev/null + if [ "$?" != 0 ]; then + err "The key $WHITE$key$NORMAL was not found in base service compose definition.." + return 1 + fi +} +export -f relation-base-compose-get + + +relation-set () { + local key="$1" value="$2" + if [ -z "$RELATION_DATA_FILE" ]; then + err "$FUNCNAME: relation does not seems to be correctly setup." + return 1 + fi + + if ! [ -r "$RELATION_DATA_FILE" ]; then + err "$FUNCNAME: can't read relation's data." >&2 + return 1 + fi + + _config_merge "$RELATION_DATA_FILE" <(echo "$key: $value") +} +export -f relation-set + + +_config_merge() { + local config_filename="$1" merge_to_file="$2" + touch "$config_filename" && + merge_yaml "$config_filename" "$merge_to_file" > "$config_filename.tmp" || return 1 + mv "$config_filename.tmp" "$config_filename" +} +export -f _config_merge + + +config-add() { + local metadata="$1" + _config_merge "$RELATION_CONFIG" <(echo "$metadata") +} +export -f config-add + + +logstdout() { + local name="$1" + sed -r 's%^%'"${name}"'> %g' +} +export -f logstdout + + +logstderr() { + local name="$1" + sed -r 's%^(.*)$%'"${RED}${name}>${NORMAL} \1"'%g' +} +export -f logstderr + + +_run_service_relation () { + local relation_name="$1" service="$2" target_service="$3" relation_config="$4" relation_dir services + + charm=$(get_service_charm "$service") || return 1 + target_charm=$(get_service_charm "$target_service") || return 1 + base_script_name=$(echo "$relation_name" | tr "-" "_" )-relation-joined + script_name="hooks/${base_script_name}" + + relation_dir=$(get_relation_data_dir "$charm" "$target_charm" "$relation_name") || return 1 + RELATION_DATA_FILE=$(get_relation_data_file "$charm" "$target_charm" "$relation_name" "$relation_config") || return 1 + RELATION_BASE_COMPOSE_DEF=$(get_compose_service_def "$service") || return 1 + RELATION_TARGET_COMPOSE_DEF=$(get_compose_service_def "$target_service") || return 1 + export BASE_SERVICE_NAME=$service + export TARGET_SERVICE_NAME=$target_service + export BASE_CHARM_NAME=$charm + export TARGET_CHARM_NAME=$target_charm + export RELATION_DATA_FILE RELATION_BASE_COMPOSE_DEF RELATION_TARGET_COMPOSE_DEF + + target_errlvl=0 + if ! [ -e "$CHARM_STORE/$target_charm/$script_name" ]; then + verb "Missing relation script '$script_name' in $DARKYELLOW$target_charm$NORMAL." + else + verb "Building ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \ + "for target charm $DARKYELLOW$target_charm$NORMAL" + export RELATION_CONFIG="$relation_dir/config_provider" + { + ( + cd "$CHARM_STORE/$target_charm" + export SERVICE_NAME=$target_service + export DOCKER_BASE_IMAGE=$(service_base_docker_image "$target_service") + export SERVICE_DATASTORE="$DATASTORE/$target_charm" + export SERVICE_CONFIGSTORE="$CONFIGSTORE/$target_charm" + "$script_name" + echo "$?" > "$relation_dir/target_errlvl" + ) | logstdout "$DARKYELLOW$target_charm$NORMAL/$base_script_name ${GREEN}@${NORMAL}" + } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$target_charm$NORMAL/$base_script_name ${RED}@${NORMAL}" 3>&1 1>&2 2>&3 + target_errlvl="$(cat "$relation_dir/target_errlvl")" + if [ -e "$RELATION_CONFIG" ]; then + debug "Merging some new config info in $DARKYELLOW$target_service$NORMAL" + _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" + rm "$RELATION_CONFIG" + ((target_errlvl |= "$?")) + fi + fi + + if [ "$target_errlvl" == 0 ]; then + errlvl=0 + if [ -e "$CHARM_STORE/$charm/$script_name" ]; then + verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \ + "for charm $DARKYELLOW$charm$NORMAL" + export RELATION_CONFIG="$relation_dir/config_providee" + export RELATION_DATA="$(cat "$RELATION_DATA_FILE")" + { + ( + cd "$CHARM_STORE/$charm" + export SERVICE_NAME=$service + export DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") + export SERVICE_DATASTORE="$DATASTORE/$charm" + export SERVICE_CONFIGSTORE="$CONFIGSTORE/$charm" + "$script_name" + echo "$?" > "$relation_dir/errlvl" + ) | logstdout "$DARKYELLOW$charm$NORMAL/$base_script_name ${GREEN}@${NORMAL}" + } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$charm$NORMAL/$base_script_name ${RED}@$NORMAL" 3>&1 1>&2 2>&3 + errlvl="$(cat "$relation_dir/errlvl")" + if [ -e "$RELATION_CONFIG" ]; then + _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" + rm "$RELATION_CONFIG" + fi + if [ "$errlvl" != 0 ]; then + err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$target_charm$NORMAL failed to run properly." + fi + else + verb "No relation script '$script_name' in charm $DARKYELLOW$charm$NORMAL. Ignoring." + fi + else + err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$target_charm$NORMAL failed to run properly." + fi + + if [ "$target_errlvl" == 0 -a "$errlvl" == 0 ]; then + debug "Relation $DARKBLUE$relation_name$NORMAL is established" \ + "between $DARKYELLOW$service$NORMAL and $DARKYELLOW$target_service$NORMAL." + return 0 + else + return 1 + fi +} +export -f _run_service_relation + + +get_compose_relations() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" + + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + compose_def="$(get_compose_service_def "$service")" || return 1 + ( + set -o pipefail + if [ "$compose_def" ]; then + while read-0 relation_name relation_def; do + case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in + "str") + target_service="$(echo "$relation_def" | shyaml get-value 2>/dev/null)" + reverse="$(get_charm_reverse_tech_dep_relation "$target_service" "$relation_name")" + echo -en "$relation_name\0$target_service\0\0$reverse\0" + ;; + "sequence") + while read-0 target_service; do + reverse="$(get_charm_reverse_tech_dep_relation "$target_service" "$relation_name")" + echo -en "$relation_name\0$target_service\0\0$reverse\0" + done < <(echo "$relation_def" | shyaml get-values-0 2>/dev/null) + ;; + "struct") + while read-0 target_service relation_config; do + reverse="$(get_charm_reverse_tech_dep_relation "$target_service" "$relation_name")" + echo -en "$relation_name\0$target_service\0$relation_config\0$reverse\0" + done < <(echo "$relation_def" | shyaml key-values-0 2>/dev/null) + ;; + esac + done < <(echo "$compose_def" | shyaml key-values-0 relations 2>/dev/null) + fi | tee "$cache_file" + ) + if [ "$?" != 0 ]; then + rm -f "$cache_file" ## no cache + return 1 + fi + return 0 +} + +run_service_relations () { + local services="$1" + for service in $services; do + for subservice in $(get_ordered_service_dependencies "$service"); do + while read-0 relation_name target_service relation_config reverse; do + export relation_config + Wrap -d "Building $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW$target_service$NORMAL" < "$action_errlvl_file" + ) | logstdout "$DARKYELLOW$charm$NORMAL/${DARKCYAN}$action${NORMAL} ${GREEN}@${NORMAL}" + } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$charm$NORMAL/${DARKCYAN}$action${NORMAL} ${RED}@$NORMAL" 3>&1 1>&2 2>&3 + if ! [ -e "$action_errlvl_file" ]; then + err "Action $DARKYELLOW$service$NORMAL:$DARKCYAN$action$NORMAL has failed without having time" \ + "to output an errlvl" + return 1 + fi + return "$(cat "$action_errlvl_file")" +} +export -f _run_service_action_direct + +_run_service_action_relation() { + local service="$1" action="$2" charm target_charm relation_name target_script relation_config _dummy + shift; shift + + read-0 charm target_charm relation_name target_script relation_config || true + + if read-0 _dummy || [ "$_dummy" ]; then + print_syntax_error "$FUNCNAME: too many arguments in action descriptor" + return 1 + fi + + export RELATION_DATA_FILE=$(get_relation_data_file "$charm" "$target_charm" "$relation_name" "$relation_config") + + compose_file=$(get_compose_yml_location) || return 1 + export action_errlvl_file="$state_tmpdir/action-$service-$charm-$action-errlvl" + export state_tmpdir + { + ( + set +e ## Prevents unwanted leaks from parent shell + cd "$CHARM_STORE/$charm" + export METADATA_CONFIG=$(cat "$CHARM_STORE/$charm/metadata.yml") + export SERVICE_NAME=$service + export RELATION_TARGET_CHARM="$target_charm" + export RELATION_CHARM="$charm" + export ACTION_NAME=$action + export CONTAINER_NAME=$(_get_master_charm_for_service "$service") + export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME") + export SERVICE_DATASTORE="$DATASTORE/$service" + export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service" + exname="$exname $ACTION_NAME $SERVICE_NAME" \ + stdbuf -oL -eL "$target_script" "$@" + echo "$?" > "$action_errlvl_file" + ) | logstdout "$DARKYELLOW$charm$NORMAL/${DARKCYAN}$action${NORMAL} ${GREEN}@${NORMAL}" + } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$charm$NORMAL/${DARKCYAN}$action${NORMAL} ${RED}@$NORMAL" 3>&1 1>&2 2>&3 + if ! [ -e "$action_errlvl_file" ]; then + err "Action $DARKYELLOW$service$NORMAL:$DARKCYAN$action$NORMAL has failed without having time" \ + "to output an errlvl" + return 1 + fi + return "$(cat "$action_errlvl_file")" +} +export -f _run_service_action_relation + + +get_relation_data_dir() { + local charm="$1" target_charm="$2" relation_name="$3" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + project=$(get_default_project_name) || return 1 + relation_dir="$VARDIR/relations/$project/$charm-$target_charm/$relation_name" + if ! [ -d "$relation_dir" ]; then + mkdir -p "$relation_dir" || return 1 + chmod go-rwx "$relation_dir" || return 1 ## protecting this directory + fi + echo "$relation_dir" || tee "$cache_file" +} +export -f get_relation_data_dir + + +get_relation_data_file() { + local charm="$1" target_charm="$2" relation_name="$3" relation_config="$4" + + relation_dir=$(get_relation_data_dir "$charm" "$target_charm" "$relation_name") || return 1 + relation_data_file="$relation_dir/data" + if ! [ -e "$relation_data_file" ]; then + echo "$relation_config" > "$relation_data_file" + chmod go-rwx "$relation_data_file" ## protecting this file + fi + echo "$relation_data_file" +} +export -f get_relation_data_file + + +has_service_action () { + local service="$1" action="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + target_script charm target_charm + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + charm=$(get_service_charm "$service") || return 1 + + ## Action directly provided ? + + target_script="$CHARM_STORE/$charm/actions/$action" + if [ -x "$target_script" ]; then + echo -en "direct\0$charm\0$target_script" | tee "$cache_file" + return 0 + fi + + ## Action provided by relation ? + while read-0 relation_name target_service relation_config reverse; do + target_charm=$(get_service_charm "$target_service") || return 1 + target_script="$CHARM_STORE/$target_charm/actions/relations/$relation_name/$action" + if [ -x "$target_script" ]; then + echo -en "relation\0$charm\0$target_charm\0$relation_name\0$target_script\0$relation_config" | tee "$cache_file" + return 0 + fi + done < <(get_compose_relations "$service") + + return 1 +} +export -f has_service_action + run_service_action () { local service="$1" action="$2" - shift shift - run_service_hook "$service" init + shift ; shift - services=$(get_ordered_service_dependencies "$service") || return 1 - for service in $services; do - TARGET_SCRIPT="$COMPOSE_YML_PATH/service/$service/actions/$2" - if [ -e "$TARGET_SCRIPT" ]; then - [ "$verbose" ] && echo "Init $service" - SERVICE_NAME=$service \ - CONTAINER_NAME=$(get_container_name "$service") \ - DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") \ - SERVICE_DATASTORE="$DATASTORE/$service" \ - SERVICE_CONFIGSTORE="$CONFIGSTORE/$service" \ - echo "$TARGET_SCRIPT" "$@" - else - echo "Service '$service' does not have any action '$action' defined." >&2 + { + if ! read-0 action_type; then + info "Service $DARKYELLOW$service$NORMAL does not have any action $DARKCYAN$action$NORMAL defined." + info " Add an executable script to '$CHARM_STORE/$charm/actions/$action' to implement action." return 1 fi - done + + Section "running $DARKYELLOW$service$NORMAL/$DARKCYAN$action$NORMAL ($action_type)"; Feed + "_run_service_action_${action_type}" "$service" "$action" "$@" + + } < <(has_service_action "$service" "$action") } +export -f run_service_action -get_docker_compose_entry_from_metadata() { - local service="$1" - metadata="$(cat -)" - export DATASTORE CONFIGSTORE - ## resources to volumes - volumes=$( - for resource_type in data config; do - while read-0 resource; do - eval "echo \" - \$${resource_type^^}STORE/\$service\$resource:\$resource:rw\"" - done < <(echo "$metadata" | shyaml get-values-0 "${resource_type}-resources" 2>/dev/null) - done - while read-0 resource; do - if [[ "$resource" == *:* ]]; then - echo " - $resource:rw" - else - echo " - $resource:$resource:rw" +get_compose_relation_config() { + local service=$1 relation_config cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + compose_service_def=$(get_compose_service_def "$service") || return 1 + + echo "$compose_service_def" | shyaml get-value "relations" 2>/dev/null | tee "$cache_file" +} +export -f get_compose_relation_config + + +## Return key-values-0 +get_compose_relation_config_for_service() { + local service=$1 relation_name=$2 relation_config + compose_service_relations=$(get_compose_relation_config "$service") || return 1 + if ! relation_config=$( + echo "$compose_service_relations" | + shyaml get-value "${relation_name}" 2>/dev/null); then + err "Couldn't find $DARKYELLOW${service}$NORMAL/${WHITE}${relation_name}$NORMAL" \ + "relation config in compose configuration." + return 1 + fi + if [ -z "$relation_config" ]; then + err "Relation ${WHITE}mysql-database$NORMAL is empty in compose configuration." + return 1 + fi + if ! echo "$relation_config" | shyaml key-values-0 2>/dev/null; then + err "No key/values in ${DARKBLUE}mysql-database$NORMAL of compose config." + return 1 + fi +} +export -f get_compose_relation_config_for_service + + +_get_master_charm_for_service() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + charm metadata requires master_charm target_charm target_service service_def + + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + charm=$(get_service_charm "$service") || return 1 + metadata=$(get_charm_metadata "$charm") || return 1 + + if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" != "True" ]; then + ## just return charm name + echo "$charm" > "$cache_file" + echo "$charm" + return 0 + fi + + ## fetch the container relation + requires="$(echo "$metadata" | shyaml get-value "requires" 2>/dev/null)" + if [ -z "$requires" ]; then + die "Charm $DARKYELLOW$charm$NORMAL is a subordinate but does not have any 'requires' " \ + "section." + fi + master_charm= + while read-0 relation_name relation; do + if [ "$(echo "$relation" | shyaml get-value "scope" 2>/dev/null)" == "container" ]; then + # debug "$DARKYELLOW$service$NORMAL's relation" \ + # "$DARKBLUE${relation_name}$NORMAL is a container." + interface="$(echo "$relation" | shyaml get-value "interface" 2>/dev/null)" + if [ -z "$interface" ]; then + err "No ${WHITE}$interface${NORMAL} set for relation $relation_name." + return 1 fi - done < <(echo "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null) - ) - if [ "$volumes" ]; then - echo "volumes:" - echo "$volumes" + service_def=$(get_compose_service_def "$service") || return 1 + target_service=$(echo "$service_def" | + shyaml keys "relations.${interface}" 2>/dev/null | + head -n 1) + if [ -z "$target_service" ]; then + err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \ + "${DARKYELLOW}$service$NORMAL compose definition." + return 1 + fi + target_charm=$(get_service_charm "$target_service") || return 1 + master_charm="$(_get_master_charm_for_service "$target_service")" + break + fi + done < <(echo "$requires" | shyaml key-values-0 2>/dev/null) + if [ -z "$master_charm" ]; then + die "Charm $DARKYELLOW$charm$NORMAL is a subordinate but does not have any relation with" \ + " ${WHITE}scope${NORMAL} set to 'container'." + fi + echo "$master_charm" > "$cache_file" + echo "$master_charm" + return 0 +} +export -f _get_master_charm_for_service + + +get_charm_metadata() { + local charm="$1" + metadata_file="$CHARM_STORE/$charm/metadata.yml" + if ! [ -e "$metadata_file" ]; then + return 0 ## No metadata file is as if metadata was empty fi - ## resources to volumes - image=$(echo "$metadata" | shyaml get-values "docker-image" 2>/dev/null) + cat "$metadata_file" +} + +## +## The result is a mixin that is not allways a complete valid +## docker-compose entry (thinkinf of subordinates). The result +## will be merge with master charms. +get_docker_compose_mixin_from_metadata() { + local charm="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + metadata_file metadata volumes docker_compose subordinate image + + if [ -e "$cache_file" ]; then + debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + mixin= + metadata="$(get_charm_metadata "$charm")" + if [ "$metadata" ]; then + ## resources to volumes + volumes=$( + for resource_type in data config; do + while read-0 resource; do + eval "echo \" - \$${resource_type^^}STORE/\$charm\$resource:\$resource:rw\"" + done < <(echo "$metadata" | shyaml get-values-0 "${resource_type}-resources" 2>/dev/null) + done + while read-0 resource; do + if [[ "$resource" == *:* ]]; then + echo " - $resource:rw" + else + echo " - $resource:$resource:rw" + fi + done < <(echo "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null) + ) + if [ "$volumes" ]; then + mixin=$(echo -en "volumes:\n$volumes") + fi + + docker_compose=$(echo "$metadata" | shyaml get-value "docker-compose" 2>/dev/null) + if [ "$docker_compose" ]; then + mixin=$(merge_yaml_str "$mixin" "$docker_compose") + fi + + if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" == "True" ]; then + subordinate=true + fi + fi + + image=$(echo "$metadata" | shyaml get-value "docker-image" 2>/dev/null) + image_or_build_statement= if [ "$image" ]; then - echo "image: $image" - else - if ! [ -d "$CHARM_STORE/$service/build" ]; then - die "No 'docker-image' value set in 'metadata.yml' nor 'build/' directory found in charm $DARKYELLOW$service$NORMAL." + if [ "$subordinate" ]; then + err "Subordinate charm can not have a ${WHITE}docker-image${NORMAL} value." + return 1 + fi + image_or_build_statement="image: $image" + elif [ -d "$CHARM_STORE/$charm/build" ]; then + if [ "$subordinate" ]; then + err "Subordinate charm can not have a 'build' sub directory." + return 1 fi - echo "build: $service/build" + image_or_build_statement="build: $charm/build" + fi + if [ "$image_or_build_statement" ]; then + mixin=$(merge_yaml_str "$mixin" "$image_or_build_statement") fi + + echo "$mixin" > "$cache_file" + echo "$mixin" +} +export -f get_docker_compose_mixin_from_metadata + + +_save() { + local name="$1" + cat - | tee -a "$docker_compose_dir/.data/$name" } -export -f get_docker_compose_entry_from_metadata + + +get_default_project_name() { + if [ "$DEFAULT_PROJECT_NAME" ]; then + echo "$DEFAULT_PROJECT_NAME" + return 0 + fi + compose_yml_location="$(get_compose_yml_location)" + if [ "$compose_yml_location" ]; then + if normalized_path=$(readlink -e "$compose_yml_location"); then + echo "$(basename "$(dirname "$normalized_path")")" + return 0 + fi + fi + echo "project" + return 0 +} +export -f get_default_project_name + launch_docker_compose() { - debug "Creating temporary docker-compose directory in '$tmpdir'." - tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX) - function finish { - debug "Removing temporary docker-compose directory in '$tmpdir'." - rm -rf "$tmpdir" - } - trap finish EXIT - get_compose_def > "$tmpdir/docker-compose.yml" || return 1 + + docker_compose_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX) + #debug "Creating temporary docker-compose directory in '$docker_compose_tmpdir'." +# trap_add EXIT "debug \"Removing temporary docker-compose directory in $docker_compose_tmpdir.\";\ +# rm -rf \"$docker_compose_tmpdir\"" + trap_add EXIT "rm -rf \"$docker_compose_tmpdir\"" + project=$(get_default_project_name) + mkdir -p "$docker_compose_tmpdir/$project" + docker_compose_dir="$docker_compose_tmpdir/$project" + + if [ -z "$services" ]; then + services=$(get_default_target_services $services) + fi + get_docker_compose $services > "$docker_compose_dir/docker-compose.yml" || return 1 + if [ -e "$state_tmpdir/to-merge-in-docker-compose.yml" ]; then + # debug "Merging some config data in docker-compose.yml:" + # debug "$(cat $state_tmpdir/to-merge-in-docker-compose.yml)" + _config_merge "$docker_compose_dir/docker-compose.yml" "$state_tmpdir/to-merge-in-docker-compose.yml" || return 1 + fi + ## XXXvlab: could be more specific and only link the needed charms - ln -sf "$CHARM_STORE/"* "$tmpdir/" - cd "$tmpdir" && docker-compose "$@" + ln -sf "$CHARM_STORE/"* "$docker_compose_dir/" + mkdir "$docker_compose_dir/.data" + + { + { + cd "$docker_compose_dir" + debug "${WHITE}docker-compose.yml:$NORMAL" + debug "$(cat "$docker_compose_dir/docker-compose.yml")" + debug "${WHITE}Launching$NORMAL: docker-compose $@" + docker-compose "$@" + echo "$?" > "$docker_compose_dir/.data/errlvl" + } | _save stdout + } 3>&1 1>&2 2>&3 | _save stderr 3>&1 1>&2 2>&3 + if tail -n 1 "$docker_compose_dir/.data/stderr" | egrep "Service .+ failed to build: Error getting container [0-9a-f]+ from driver devicemapper: (open|Error mounting) /dev/mapper/docker-.*: no such file or directory$" >/dev/null 2>&1; then + debug NASTY ERROR DETECTED + fi + + docker_compose_errlvl="$(cat "$docker_compose_dir/.data/errlvl")" + if [ -z "docker_compose_dir" ]; then + err "Something went wrong before you could gather docker-compose errorlevel." + return 1 + fi + return "$docker_compose_errlvl" +} + +get_compose_yml_location() { + + parent=$(while ! [ -e "./compose.yml" ]; do + [ "$PWD" == "/" ] && exit 0 + cd .. + done; echo "$PWD" + ) + if [ "$parent" ]; then + echo "$parent/compose.yml" + return 0 + fi + if [ "$DEFAULT_COMPOSE_FILE" ]; then + if ! [ -e "$DEFAULT_COMPOSE_FILE" ]; then + err "No 'compose.yml' was found in current or parent dirs," \ + "and \$DEFAULT_COMPOSE_FILE points to an unexistent file." + die "Please provide a 'compose.yml' file." + fi + echo "$DEFAULT_COMPOSE_FILE" + return 0 + fi + err "No 'compose.yml' was found in current or parent dirs, and no \$DEFAULT_COMPOSE_FILE was set." + die "Please provide a 'compose.yml' file." + return 1 } +export -f get_compose_yml_location + + +get_default_target_services() { + local services=("$@") + if [ -z "${services[*]}" ]; then + if [ "$DEFAULT_SERVICES" ]; then + info "No service provided, using \$DEFAULT_SERVICES variable. Target services: $DEFAULT_SERVICES" + services="$DEFAULT_SERVICES" + else + err "No service provided." + return 1 + fi + fi + echo "${services[*]}" +} + +get_master_services() { + local loaded master_service + declare -A loaded + for service in "$@"; do + master_service=$(_get_master_charm_for_service "$service") || return 1 + if [ "${loaded[$master_service]}" ]; then + continue + fi + echo "$master_service" + loaded["$master_service"]=1 + done | xargs echo +} + + +_setup_state_dir() { + export state_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX) + #debug "Creating temporary state directory in '$state_tmpdir'." +# trap_add EXIT "debug \"Removing temporary state directory in $state_tmpdir.\";\ +# rm -rf \"$state_tmpdir\"" + trap_add EXIT "rm -rf \"$state_tmpdir\"" +} + + +[ "$SOURCED" ] && return 0 @@ -532,6 +1346,7 @@ launch_docker_compose() { ## Argument parsing ## + fullargs=() opts=() posargs=() @@ -547,6 +1362,22 @@ while [ "$#" != 0 ]; do fullargs+=("$1") export VERBOSE=true ;; + -f) + if ! [ -e "$2" ]; then + die "File $2 doesn't exists" + fi + export DEFAULT_COMPOSE_FILE="$2" + shift + ;; + -p) + fullargs+=("$1" "$2") + opts=("${opts[@]}" "$1" "$2") + export DEFAULT_PROJECT_NAME="$2" + shift + ;; + --no-relations) + export no_relations=true + ;; --no-hooks) export no_hooks=true ;; @@ -564,7 +1395,7 @@ while [ "$#" != 0 ]; do break 2 ;; -*) - fullargs+=("$1") + fullargs+=("$1" "$2") opts=("${opts[@]}" "$1" "$2") shift ;; @@ -577,6 +1408,22 @@ while [ "$#" != 0 ]; do done +if [ -z "$DISABLE_SYSTEM_CONFIG_FILE" ]; then + if [ -r /etc/default/charm ]; then + . /etc/default/charm + fi + + if [ -r "/etc/default/$exname" ]; then + . "/etc/default/$exname" + fi + + ## XXXvlab: should provide YML config opportunities in possible parent dirs ? + ## userdir ? and global /etc/compose.yml ? + . /etc/compose.conf + . /etc/compose.local.conf +fi + + ## ## Actual code ## @@ -584,11 +1431,8 @@ done export CHARM_STORE=${CHARM_STORE:-/srv/charm-store} export DOCKER_DATASTORE=${DOCKER_DATASTORE:-/srv/docker-datastore} - -## XXXvlab: should provide YML config opportunities in possible parent dirs ? -## userdir ? and global /etc/compose.yml ? -. /etc/compose.conf -. /etc/compose.local.conf +export COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1 +debug "Found 'compose.yml' file in '$COMPOSE_YML_FILE'." if ! [ -d "$CHARM_STORE" ]; then err "Charm store path $YELLOW$CHARM_STORE$NORMAL does not exists. " @@ -605,31 +1449,105 @@ if [ -z "$(cd "$CHARM_STORE"; ls)" ]; then fi +_setup_state_dir + +## +## Get services in command line. +## + action="${posargs[0]}" +is_service_action= case "$action" in - load|save) - service="${posargs[1]}" - run_service_action "$service" "$action" "${opts[@]}" "${posargs[@]:2}" + up|build|start|stop|config) + services="$(get_default_target_services "${posargs[@]:1}")" || exit 1 + orig_services="${posargs[@]:1}" + ;; + run) + services="${posargs[1]}" ;; - up) - service="${posargs[1]}" + *) + if is_service_action=$(has_service_action "${posargs[1]}" "$action"); then + debug "Found action $DARKCYAN$action$NORMAL in service $DARKYELLOW${posargs[1]}$NORMAL" + services="${posargs[1]}" + else + services="$(get_default_target_services)" + fi + ;; +esac - ## init in order - [ "$no_init" ] || run_service_hook "$service" init +get_docker_compose $services >/dev/null || exit 1 ## precalculate variable \$_currnet_docker_compose + +## +## Pre-action +## - ## XXXvlab: to be removed when all relation and service stuff is resolved - if [ -z "$no_hooks" ]; then - for script in "$CHARM_STORE/"*/hooks.d/*.sh; do + +full_init= +case "$action" in + up|run) + full_init=true + ;; + *) + if [ "$is_service_action" ]; then + full_init=true + fi + ;; +esac + +if [ "$full_init" ]; then + ## init in order + Section initialisation + if [ -z "$no_init" ]; then + run_service_hook "$services" init || exit 1 + fi + + ## Get relations + if [ -z "$no_relations" ]; then + run_service_relations "$services" || exit 1 + fi + + ## XXXvlab: to be removed when all relation and service stuff is resolved + if [ -z "$no_hooks" ]; then + ordered_services=$(get_ordered_service_dependencies $services) || exit 1 + for service in $ordered_services; do + charm=$(get_service_charm "$service") || exit 1 + for script in "$CHARM_STORE/$charm/hooks.d/"*.sh; do [ -e "$script" ] || continue [ -x "$script" ] || { echo "compose: script $script is not executable." >&2; exit 1; } ( - cd "$(dirname "$script/..")"; + debug "Launching '$script'." + cd "$(dirname "$script)")"; "$script" "$@" ) || { echo "compose: hook $script failed. Stopping." >&2; exit 1; } done - fi + done + fi +fi + + +## +## Docker-compose +## - launch_docker_compose "${fullargs[@]}" +case "$action" in + up|start|stop|build) + master_services=$(get_master_services $services) || exit 1 + launch_docker_compose "$action" "${opts[@]}" $master_services + ;; + run) + master_service=$(get_master_services $services) || exit 1 + launch_docker_compose "$action" "$master_service" "${opts[@]}" "${posargs[@]:2}" + ;; + config) + ## removing the services + launch_docker_compose config "${opts[@]}" + warn "Runtime configuration modification (from relations) are not included here." + ;; + *) + if [ "$is_service_action" ]; then + run_service_action "$services" "$action" "${opts[@]}" "${posargs[@]:2}" + else + launch_docker_compose "${fullargs[@]}" + fi ;; - *) launch_docker_compose "${fullargs[@]}";; esac diff --git a/test/test b/test/test new file mode 100755 index 0000000..af00c45 --- /dev/null +++ b/test/test @@ -0,0 +1,1098 @@ +#!/bin/bash + +#!- Library include +. /etc/shlib +#!- + +include shunit + +depends sed grep git mkdir readlink + +export -f matches +export grep + +tmp=/tmp +tprog="../bin/compose" +tprog=$(readlink -f $tprog) + + +export PATH=".:$PATH" +short_tprog=$(basename "$tprog") + +## +## Convenience function +## + +function init_test() { + test_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX) + cd "$test_tmpdir" +} + + + +function tear_test() { + rm -rf "$test_tmpdir" +} + + +## +## Tests +## + +## +# Checking arguments +function test_calling_sourcing { + + assert_list < $test_tmpdir/testcharm/metadata.yml + +EOF2 + +. "$tprog" + +_setup_state_dir +test -z "\$(get_docker_compose_mixin_from_metadata testcharm)" + +## -- volumes + +export CHARM_STORE=$test_tmpdir +export CONFIGSTORE=/tmp/CONFIG +export DATASTORE=/tmp/DATA +mkdir -p $test_tmpdir/testcharm +cat < $test_tmpdir/testcharm/metadata.yml +data-resources: + - /a +config-resources: + - /b +host-resources: + - /tmp:/tmp +EOF2 + +. "$tprog" + +_setup_state_dir +test "\$(get_docker_compose_mixin_from_metadata testcharm)" == "volumes: + - /tmp/DATA/testcharm/a:/a:rw + - /tmp/CONFIG/testcharm/b:/b:rw + - /tmp:/tmp:rw" + +## -- docker-compose + +export CHARM_STORE=$test_tmpdir +mkdir -p $test_tmpdir/testcharm +cat < $test_tmpdir/testcharm/metadata.yml +docker-compose: + volumes: + - /any:/vol + entrypoint: any +EOF2 + +. "$tprog" + +_setup_state_dir +test "\$(get_docker_compose_mixin_from_metadata testcharm)" == "entrypoint: any +volumes: +- /any:/vol" + +## -- image + +export CHARM_STORE=$test_tmpdir +mkdir -p $test_tmpdir/testcharm +cat < $test_tmpdir/testcharm/metadata.yml +docker-image: toto +EOF2 + +. "$tprog" + +_setup_state_dir +test "\$(get_docker_compose_mixin_from_metadata testcharm)" == "image: toto" + +## -- build + +export CHARM_STORE=$test_tmpdir +mkdir -p $test_tmpdir/testcharm/build +cat < $test_tmpdir/testcharm/metadata.yml +EOF2 + +. "$tprog" + +_setup_state_dir +out="\$(get_docker_compose_mixin_from_metadata testcharm)" || { + echo "Failed" + exit 1 +} +echo "\$out" +test "\$out" == "build: testcharm/build" + + +## -- subordinate with image + +export CHARM_STORE=$test_tmpdir +mkdir -p $test_tmpdir/testcharm +cat < $test_tmpdir/testcharm/metadata.yml +subordinate: true +docker-image: toto +EOF2 + +. "$tprog" + +_setup_state_dir +! get_docker_compose_mixin_from_metadata testcharm + +## -- subordinate with build subdir + +export CHARM_STORE=$test_tmpdir +mkdir -p $test_tmpdir/testcharm/build +cat < $test_tmpdir/testcharm/metadata.yml +subordinate: true +EOF2 + +. "$tprog" + +_setup_state_dir +! get_docker_compose_mixin_from_metadata testcharm + +EOF + tear_test +} + + +function test_merge_yaml { + + init_test + + assert_list < $test_tmpdir/compose.yml +toto: + charm: www + blabla: xxx +EOF2 + +. "$tprog" + +_setup_state_dir +COMPOSE_YML_FILE=$test_tmpdir/compose.yml +test "\$(get_compose_service_def toto)" == "blabla: xxx +charm: www" + +## -- Addition of default charm + +export CHARM_STORE=$test_tmpdir +mkdir -p $test_tmpdir/www + +cat < $test_tmpdir/compose.yml +www: + blabla: xxx +EOF2 + +. "$tprog" + +_setup_state_dir +COMPOSE_YML_FILE=$test_tmpdir/compose.yml +test "\$(get_compose_service_def www)" == "blabla: xxx +charm: www" + +EOF +} + + +## +## +## +function test_get_master_charm_for_service() { + + init_test + + assert_list < $test_tmpdir/www/metadata.yml +EOF2 + +. "$tprog" + +_setup_state_dir +test "\$(_get_master_charm_for_service www)" == "www" + +## -- subordinate + +export CHARM_STORE=$test_tmpdir +mkdir -p $test_tmpdir/{www,mysql} +cat < $test_tmpdir/www/metadata.yml +subordinate: true +requires: + a-label-for-relation: + interface: a-name-relation + scope: container + +EOF2 + +cat < $test_tmpdir/compose.yml +www: + charm: www + relations: + a-name-relation: + mysql: + label: value +EOF2 + +. "$tprog" + +_setup_state_dir +COMPOSE_YML_FILE=$test_tmpdir/compose.yml +test "\$(_get_master_charm_for_service www)" == "mysql" +EOF + +} + + + +## +## +## +function test_get_docker_compose_service_mixin() { + + init_test + + assert_list < $test_tmpdir/www/metadata.yml +data-resources: + - /tmp/a +config-resources: + - /tmp/b +EOF2 + +. "$tprog" + +_setup_state_dir +out=\$(_get_docker_compose_service_mixin www) +test "\$out" == "www: + volumes: + - /www/tmp/a:/tmp/a:rw + - /www/tmp/b:/tmp/b:rw" || { + echo -e "** _get_docker_compose_service_mixin www:\n\$out"; exit 1 +} + + +## -- Simple (compose, but no subordinate) + +export CHARM_STORE=$test_tmpdir +mkdir -p $test_tmpdir/{www,mysql} +cat < $test_tmpdir/www/metadata.yml +data-resources: + - /tmp/a +config-resources: + - /tmp/b +EOF2 + +cat < $test_tmpdir/compose.yml +www: + charm: www + relations: + a-name-relation: + mysql: + label: value +EOF2 + +. "$tprog" + +_setup_state_dir +COMPOSE_YML_FILE=$test_tmpdir/compose.yml +test "\$(_get_docker_compose_service_mixin www)" == "www: + links: + - mysql + volumes: + - /www/tmp/a:/tmp/a:rw + - /www/tmp/b:/tmp/b:rw" + +## -- compose, subordinate + +export CHARM_STORE=$test_tmpdir +mkdir -p $test_tmpdir/{www,mysql} +cat < $test_tmpdir/www/metadata.yml +subordinate: true +data-resources: + - /tmp/a +config-resources: + - /tmp/b +requires: + a-name-relation: + interface: a-name-relation + scope: container +EOF2 + +cat < $test_tmpdir/compose.yml +www: + charm: www + relations: + a-name-relation: + mysql: + label: value +EOF2 + +. "$tprog" + +_setup_state_dir +COMPOSE_YML_FILE=$test_tmpdir/compose.yml +test "\$(_get_docker_compose_service_mixin www)" == "mysql: + volumes: + - /www/tmp/a:/tmp/a:rw + - /www/tmp/b:/tmp/b:rw" +EOF +} + + +function test_get_docker_compose { + + init_test + + assert_list < $test_tmpdir/www/metadata.yml +data-resources: + - /tmp/a +config-resources: + - /tmp/b +EOF2 + +. "$tprog" + +_setup_state_dir +out=\$(get_docker_compose www) +echo "OUT:" +echo "\$out" +test "\$out" == "www: + volumes: + - /www/tmp/a:/tmp/a:rw + - /www/tmp/b:/tmp/b:rw" + + +## -- simple with docker-compose + +export CHARM_STORE=$test_tmpdir +mkdir -p $test_tmpdir/{www,mysql} +cat < $test_tmpdir/www/metadata.yml +data-resources: + - /tmp/a +config-resources: + - /tmp/b +EOF2 + +cat < $test_tmpdir/mysql/metadata.yml +data-resources: + - /tmp/c +config-resources: + - /tmp/d +EOF2 + +cat < $test_tmpdir/compose.yml +web_site: + charm: www + relations: + db-connection: + mysql: + user: toto + dbname: tata +EOF2 + +. "$tprog" + +COMPOSE_YML_FILE=$test_tmpdir/compose.yml + +_setup_state_dir + +out=\$(get_docker_compose www) +test "\$out" == "www: + volumes: + - /www/tmp/a:/tmp/a:rw + - /www/tmp/b:/tmp/b:rw" || { + echo -e "** get_docker_compose www:\n\$out" + exit 1 +} + +out=\$(_get_docker_compose_links web_site) +test "\$out" == "www: + links: + - mysql" || { + echo -e "** _get_docker_compose_links web_site:\n\$out" + exit 1 +} + +out=\$(get_docker_compose web_site) +test "\$out" == "\ +mysql: + volumes: + - /mysql/tmp/c:/tmp/c:rw + - /mysql/tmp/d:/tmp/d:rw +www: + links: + - mysql + volumes: + - /www/tmp/a:/tmp/a:rw + - /www/tmp/b:/tmp/b:rw" || { + echo -e "** get_docker_compose web_site:\n\$out" + exit 1 +} + +## -- subordinate + +export CHARM_STORE=$test_tmpdir +mkdir -p $test_tmpdir/{www,mysql} +cat < $test_tmpdir/www/metadata.yml +subordinate: true +data-resources: + - /tmp/a +config-resources: + - /tmp/b +requires: + my-db-connection: + interface: db-connection + scope: container +EOF2 + +cat < $test_tmpdir/mysql/metadata.yml +data-resources: + - /tmp/c +config-resources: + - /tmp/d +EOF2 + +cat < $test_tmpdir/compose.yml +web_site: + charm: www + relations: + db-connection: + mysql: + user: toto + dbname: tata + +EOF2 + +. "$tprog" + +COMPOSE_YML_FILE=$test_tmpdir/compose.yml + +_setup_state_dir + +# should fail because of missing relations +! get_docker_compose www || exit 1 + +# volumes gets mixed +test "\$(get_docker_compose web_site)" == "mysql: + volumes: + - /www/tmp/a:/tmp/a:rw + - /www/tmp/b:/tmp/b:rw + - /mysql/tmp/c:/tmp/c:rw + - /mysql/tmp/d:/tmp/d:rw" + +## -- subordinate with complex features + +export CHARM_STORE=$test_tmpdir +mkdir -p $test_tmpdir/{www,mysql} +cat < $test_tmpdir/www/metadata.yml +subordinate: true +data-resources: + - /tmp/a +config-resources: + - /tmp/b +requires: + my-db-connection: + interface: db-connection + scope: container +docker-compose: + volumes: + - /special-volume-from-www:/special-volume-from-www +EOF2 + +cat < $test_tmpdir/mysql/metadata.yml +data-resources: + - /tmp/c +config-resources: + - /tmp/d +docker-compose: + entrypoint: custom-entrypoint + volumes: + - /special-volume-from-mysql:/special-volume-from-mysql +EOF2 + +cat < $test_tmpdir/compose.yml +web_site: + charm: www + relations: + db-connection: + mysql: + user: toto + dbname: tata + +EOF2 + +. "$tprog" + +COMPOSE_YML_FILE=$test_tmpdir/compose.yml + +_setup_state_dir + +# should fail because of missing relations +#! get_docker_compose www || exit 1 + +# volumes gets mixed +test "\$(get_docker_compose web_site)" == "mysql: + entrypoint: custom-entrypoint + volumes: + - /www/tmp/a:/tmp/a:rw + - /www/tmp/b:/tmp/b:rw + - /special-volume-from-www:/special-volume-from-www + - /mysql/tmp/c:/tmp/c:rw + - /mysql/tmp/d:/tmp/d:rw + - /special-volume-from-mysql:/special-volume-from-mysql" +EOF + tear_test + +} + + +## XXXvlab: only broken due to Wrap being used for relations + +# function test_run_service_relations { + +# init_test + +# assert_list < $test_tmpdir/www/metadata.yml +# data-resources: +# - /tmp/a +# config-resources: +# - /tmp/b +# EOF2 + +# . "$tprog" + +# _setup_state_dir +# echo "Docker Compose:" +# get_docker_compose www +# echo "Run Service relations:" +# _run_service_relation() { +# echo "\$FUNCNAME: received $*" +# } +# out=\$(run_service_relations www) +# test -z "\$out" + +# ## -- simple with docker-compose + +# export CHARM_STORE=$test_tmpdir +# mkdir -p $test_tmpdir/{www,mysql} +# cat < $test_tmpdir/www/metadata.yml +# data-resources: +# - /tmp/a +# config-resources: +# - /tmp/b +# EOF2 + +# cat < $test_tmpdir/mysql/metadata.yml +# data-resources: +# - /tmp/c +# config-resources: +# - /tmp/d +# EOF2 + +# cat < $test_tmpdir/compose.yml +# web_site: +# charm: www +# relations: +# db-connection: +# mysql: +# user: toto +# dbname: tata + +# EOF2 + +# . "$tprog" + +# COMPOSE_YML_FILE=$test_tmpdir/compose.yml + +# _setup_state_dir + +# _setup_state_dir +# echo "Docker Compose:" +# get_docker_compose web_site +# echo "Run Service relations:" + +# _run_service_relation() { +# echo "\$FUNCNAME \$2 <-- \$1 --> \$3" +# } +# export -f _run_service_relation + +# out=\$(run_service_relations www) +# test -z "\$out" || exit 1 + +# out=\$(run_service_relations web_site) +# echo "OUT: \$out" +# test "\$out" == "_run_service_relation web_site <-- db-connection --> mysql" + + +# ## -- subordinate + +# export CHARM_STORE=$test_tmpdir +# mkdir -p $test_tmpdir/{www,mysql} +# cat < $test_tmpdir/www/metadata.yml +# subordinate: true +# data-resources: +# - /tmp/a +# config-resources: +# - /tmp/b +# requires: +# my-db-connection: +# interface: db-connection +# scope: container +# EOF2 + +# cat < $test_tmpdir/mysql/metadata.yml +# data-resources: +# - /tmp/c +# config-resources: +# - /tmp/d +# EOF2 + +# cat < $test_tmpdir/compose.yml +# web_site: +# charm: www +# relations: +# db-connection: +# mysql: +# user: toto +# dbname: tata + +# EOF2 + +# . "$tprog" + +# COMPOSE_YML_FILE=$test_tmpdir/compose.yml + +# _setup_state_dir +# echo "Docker Compose:" +# get_docker_compose web_site +# echo "Run Service relations:" + +# _run_service_relation() { +# echo "\$FUNCNAME \$2 <-- \$1 --> \$3" +# } +# export -f _run_service_relation + +# out=\$(run_service_relations www) +# test -z "\$out" || exit 1 + +# out=\$(run_service_relations web_site) +# echo "\$out" +# test "\$out" == "_run_service_relation web_site <-- db-connection --> mysql" + + +# EOF +# tear_test + +# } + +function test_get_docker_compose_2() { + init_test + + assert_list < $test_tmpdir/www/metadata.yml +provides: + web-proxy: + reverse-tech-dep: true +EOF2 +touch $test_tmpdir/mysql/metadata.yml + +cat < $test_tmpdir/compose.yml +web_site: + charm: mysql + relations: + web-proxy: + www: + user: toto +EOF2 + +. "$tprog" +COMPOSE_YML_FILE=$test_tmpdir/compose.yml +_setup_state_dir + +out=\$(get_charm_relation_def "www" "web-proxy") || exit 1 +test "\$out" == "reverse-tech-dep: true" || { + echo -e "** get_charm_relation_def:\n\$out"; exit 1 +} + +out=\$(get_charm_reverse_tech_dep_relation "www" "web-proxy") +test "\$out" == "True" || { + echo -e "** get_charm_reverse_tech_dep_relation:\n\$out"; exit 1 +} + +out=\$(_get_docker_compose_links "web_site") +test "\$out" == "www: + links: + - mysql" || { + echo -e "** _get_docker_compose_links:\n\$out"; exit 1 +} + +out=\$(get_docker_compose web_site) +test "\$out" == "www: + links: + - mysql" || { + echo -e "** get_docker_compose:\n\$out"; exit 1 +} + +EOF + + tear_test +} + + +function test_compose_config { + + init_test + + export CHARM_STORE=$test_tmpdir + mkdir -p $test_tmpdir/{www,mysql} + cat < $test_tmpdir/www/metadata.yml +subordinate: true +data-resources: + - /tmp/a +config-resources: + - /tmp/b +requires: + my-db-connection: + interface: db-connection + scope: container +docker-compose: + volumes: + - /special-volume-from-www:/special-volume-from-www +EOF2 + + cat < $test_tmpdir/mysql/metadata.yml +docker-image: + docker.0k.io/mysql +data-resources: + - /tmp/c +config-resources: + - /tmp/d +docker-compose: + entrypoint: custom-entrypoint + volumes: + - /special-volume-from-mysql:/special-volume-from-mysql +EOF2 + + + cat < $test_tmpdir/compose.yml +web_site: + charm: www + relations: + db-connection: + mysql: + user: toto + dbname: tata + +EOF2 + + + assert_list <