From 370458d94a8d228392d98c150c3ff390074f4534 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Mon, 11 Jan 2016 11:57:56 +0700 Subject: [PATCH] save --- bin/compose | 592 ++++++++++++++++++++++++++++++++++++---------------- test/test | 90 +++++++- 2 files changed, 499 insertions(+), 183 deletions(-) diff --git a/bin/compose b/bin/compose index 7f64b5f..92aa56a 100755 --- a/bin/compose +++ b/bin/compose @@ -31,6 +31,30 @@ _merge_yaml_common_code=" import sys import yaml + +try: + # included in standard lib from Python 2.7 + from collections import OrderedDict +except ImportError: + # try importing the backported drop-in replacement + # it's available on PyPI + from ordereddict import OrderedDict + + +## Ensure that there are no collision with legacy OrderedDict +## that could be used for omap for instance. +class MyOrderedDict(OrderedDict): + pass + +yaml.add_representer( + MyOrderedDict, + lambda cls, data: cls.represent_dict(data.items())) + +yaml.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + lambda cls, node: MyOrderedDict(cls.construct_pairs(node))) + + def fc(filename): with open(filename) as f: return f.read() @@ -84,6 +108,7 @@ def merge_cli(*args): if c is not None: print '%s' % yaml.dump(c, default_flow_style=False) + " @@ -102,6 +127,7 @@ EOF } export -f merge_yaml + merge_yaml_str() { local entries="$@" @@ -118,10 +144,41 @@ EOF } export -f merge_yaml_str + +yaml_key_val_str() { + local entries="$@" + + if ! [ -r "$state_tmpdir/yaml_key_val_str.py" ]; then + cat < "$state_tmpdir/yaml_key_val_str.py" + +$_merge_yaml_common_code + +print '%s' % yaml.dump({ + yaml.load(sys.argv[1]): + yaml.load(sys.argv[2])}, default_flow_style=False) + +EOF + fi + + python "$state_tmpdir/yaml_key_val_str.py" "$@" +} +export -f yaml_key_val_str + + ## ## Functions ## +docker_has_image() { + local image="$1" + images=$(docker images -q "$image" 2>/dev/null) || { + err "docker images call has failled unexpectedly." + return 1 + } + [ "$images" ] +} +export -f docker_has_image + gen_password() { python -c 'import random; \ xx = "azertyuiopqsdfghjklmwxcvbn1234567890AZERTYUIOPQSDFGHJKLMWXCVBN+_-"; \ @@ -139,7 +196,8 @@ export -f file_put _get_docker_compose_links() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" links charm charm_part + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + links charm charm_part master_charm if [ -z "$service" ]; then print_syntax_error "$FUNCNAME: Please specify a service as first argument." return 1 @@ -151,15 +209,16 @@ _get_docker_compose_links() { return 0 fi - master_charm=$(_get_master_charm_for_service "$service") || return 1 + master_charm=$(_get_top_master_charm_for_service "$service") || return 1 deps=() while read-0 relation_name target_service relation_config reverse; do - [ "$master_charm" == "$target_service" ] && continue + master_target_charm="$(_get_top_master_charm_for_service "$target_service")" + [ "$master_charm" == "$master_target_charm" ] && continue if [ "$reverse" ]; then - deps+=("$(echo -en "$target_service:\n links:\n - $master_charm")") + deps+=("$(echo -en "$master_target_charm:\n links:\n - $master_charm")") else - deps+=("$(echo -en "$master_charm:\n links:\n - $target_service")") + deps+=("$(echo -en "$master_charm:\n links:\n - $master_target_charm")") fi done < <(get_compose_relations "$service") || return 1 @@ -183,7 +242,7 @@ _get_docker_compose_service_mixin() { return 0 fi - master_charm=$(_get_master_charm_for_service "$service") || return 1 + master_charm=$(_get_top_master_charm_for_service "$service") || return 1 ## The compose part @@ -196,9 +255,8 @@ _get_docker_compose_service_mixin() { charm_part=$(get_docker_compose_mixin_from_metadata "$charm") || return 1 ## Merge results - if [ "$charm_part" ]; then - charm_yaml="$(echo -en "${master_charm}:\n$(echo "$charm_part" | prefix " ")")" + charm_yaml="$(yaml_key_val_str "$master_charm" "$charm_part")" || return 1 merge_yaml_str "$links_yaml" "$charm_yaml" else echo "$links_yaml" @@ -211,8 +269,11 @@ export -f _get_docker_compose_service_mixin ## Get full `docker-compose.yml` format for all listed services (and ## their deps) ## -get_docker_compose() { - local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" entries services + +## @export +## @cache: !system !nofail +stdout +get_docker_compose () { + local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" entries services service if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" @@ -224,9 +285,10 @@ get_docker_compose() { ## declare -A entries - + debug "Compiling 'docker-compose.conf' for $DARKYELLOW$@$NORMAL..." for target_service in "$@"; do services=$(get_ordered_service_dependencies "$target_service") || return 1 + debug "$DARKYELLOW$target_service$NORMAL deps:$DARKYELLOW" $services "$NORMAL" for service in $services; do if [ "${entries[$service]}" ]; then @@ -245,6 +307,9 @@ get_docker_compose() { merge_yaml_str "${entries[@]}" > "$cache_file" export _current_docker_compose="$(cat "$cache_file")" echo "$_current_docker_compose" + debug " ... Compilation of base 'docker-compose.conf' done." || true + # debug " ** ${WHITE}docker-compose.conf${NORMAL}:" + # debug "$_current_docker_compose" } export -f get_docker_compose @@ -281,9 +346,13 @@ get_compose_service_def () { export -f get_compose_service_def -## XXXvlab: MUST CACHE get_service_charm () { - local service="$1" + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi if [ -z "$service" ]; then print_syntax_error "$FUNCNAME: Please specify a service as first argument." return 1 @@ -294,13 +363,13 @@ get_service_charm () { err "Missing charm in service $DARKYELLOW$service$NORMAL definition." return 1 fi - echo "$charm" + echo "$charm" | tee "$cache_file" } export -f get_service_charm ## built above the docker-compose abstraction, so it relies on the ## full docker-compose.yml to be already built. -get_service_def() { +get_service_def () { local service="$1" def if [ -z "$_current_docker_compose" ]; then print_syntax_error "$FUNCNAME is meant to be called after"\ @@ -309,7 +378,7 @@ get_service_def() { def=$(echo "$_current_docker_compose" | shyaml get-value "$service" 2>/dev/null) if [ -z "$def" ]; then - err "No definition for service $DARKYELLOW$service$NORMAL." + err "No definition for service $DARKYELLOW$service$NORMAL in compiled 'docker-compose.conf'." return 1 fi echo "$def" @@ -319,8 +388,21 @@ export -f get_service_def ## Return the base docker image name of a service service_base_docker_image() { - local service="$1" - service_def="$(get_service_def "$service")" || return 1 + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + master_charm charm service_image service_build service_dockerfile + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + master_charm="$(_get_top_master_charm_for_service "$service")" || { + err "Could not compute base charm for service $DARKYELLOW$service$NORMAL." + return 1 + } + service_def="$(get_service_def "$master_charm")" || { + err "Could not get docker-compose service definition for $DARKYELLOW$master_charm$NORMAL." + return 1 + } service_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null) if [ "$?" != 0 ]; then service_build=$(echo "$service_def" | shyaml get-value build 2>/dev/null) @@ -338,7 +420,7 @@ service_base_docker_image() { grep '^FROM' "$service_dockerfile" | xargs echo | cut -f 2 -d " " else echo "$service_image" - fi + fi | tee "$cache_file" } export -f service_base_docker_image @@ -500,7 +582,7 @@ export -f get_charm_relation_def get_charm_reverse_tech_dep_relation() { local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ - relation_def metadata + relation_def metadata value if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" @@ -554,6 +636,7 @@ _rec_get_depth() { done depths[$elt]=$((max + 1)) } +export -f _rec_get_depth get_ordered_service_dependencies() { local services=("$@") cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" @@ -591,7 +674,7 @@ get_ordered_service_dependencies() { done > "$cache_file" cat "$cache_file" } - +export -f get_ordered_service_dependencies run_service_hook () { local services="$1" action="$2" loaded @@ -610,9 +693,11 @@ run_service_hook () { TARGET_SCRIPT="$CHARM_STORE/$charm/hooks/$action" [ -e "$TARGET_SCRIPT" ] || continue + PROJECT_NAME=$(get_default_project_name) || return 1 Wrap -d "$YELLOW$action$NORMAL hook of charm $DARKYELLOW$charm$NORMAL" < "$relation_dir/target_errlvl" ) | logstdout "$DARKYELLOW$target_charm$NORMAL/$base_script_name ${GREEN}@${NORMAL}" } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$target_charm$NORMAL/$base_script_name ${RED}@${NORMAL}" 3>&1 1>&2 2>&3 - target_errlvl="$(cat "$relation_dir/target_errlvl")" + target_errlvl="$(cat "$relation_dir/target_errlvl")" || { + err "Relation script '$script_name' in $DARKYELLOW$target_charm$NORMAL" \ + "failed before outputing an errorlevel." + ((target_errlvl |= "1" )) + } if [ -e "$RELATION_CONFIG" ]; then debug "Merging some new config info in $DARKYELLOW$target_service$NORMAL" - _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" - rm "$RELATION_CONFIG" + _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" && + rm "$RELATION_CONFIG" ((target_errlvl |= "$?")) fi fi @@ -755,26 +859,33 @@ _run_service_relation () { if [ -e "$CHARM_STORE/$charm/$script_name" ]; then verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \ "for charm $DARKYELLOW$charm$NORMAL" - export RELATION_CONFIG="$relation_dir/config_providee" - export RELATION_DATA="$(cat "$RELATION_DATA_FILE")" + RELATION_CONFIG="$relation_dir/config_providee" + RELATION_DATA="$(cat "$RELATION_DATA_FILE")" + DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") || return 1 + export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA { ( cd "$CHARM_STORE/$charm" - export SERVICE_NAME=$service - export DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") - export SERVICE_DATASTORE="$DATASTORE/$charm" - export SERVICE_CONFIGSTORE="$CONFIGSTORE/$charm" + SERVICE_NAME=$service + SERVICE_DATASTORE="$DATASTORE/$charm" + SERVICE_CONFIGSTORE="$CONFIGSTORE/$charm" + export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE "$script_name" echo "$?" > "$relation_dir/errlvl" ) | logstdout "$DARKYELLOW$charm$NORMAL/$base_script_name ${GREEN}@${NORMAL}" } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$charm$NORMAL/$base_script_name ${RED}@$NORMAL" 3>&1 1>&2 2>&3 - errlvl="$(cat "$relation_dir/errlvl")" + errlvl="$(cat "$relation_dir/errlvl")" || { + err "Relation script '$script_name' in $DARKYELLOW$charm$NORMAL" \ + "failed before outputing an errorlevel." + ((errlvl |= "1" )) + } if [ -e "$RELATION_CONFIG" ]; then - _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" - rm "$RELATION_CONFIG" + _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" && + rm "$RELATION_CONFIG" + ((errlvl |= "$?" )) fi if [ "$errlvl" != 0 ]; then - err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$target_charm$NORMAL failed to run properly." + err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$charm$NORMAL failed to run properly." fi else verb "No relation script '$script_name' in charm $DARKYELLOW$charm$NORMAL. Ignoring." @@ -794,8 +905,8 @@ _run_service_relation () { export -f _run_service_relation -get_compose_relations() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" +get_compose_relations () { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" relation_name relation_def if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" @@ -808,7 +919,8 @@ get_compose_relations() { set -o pipefail if [ "$compose_def" ]; then while read-0 relation_name relation_def; do - case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in + ( + case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in "str") target_service="$(echo "$relation_def" | shyaml get-value 2>/dev/null)" reverse="$(get_charm_reverse_tech_dep_relation "$target_service" "$relation_name")" @@ -826,27 +938,36 @@ get_compose_relations() { echo -en "$relation_name\0$target_service\0$relation_config\0$reverse\0" done < <(echo "$relation_def" | shyaml key-values-0 2>/dev/null) ;; - esac + esac + ) > "$cache_file" done < <(echo "$compose_def" | shyaml key-values-0 relations 2>/dev/null) - fi | tee "$cache_file" + fi ) if [ "$?" != 0 ]; then + err "Error while looking for compose relations." rm -f "$cache_file" ## no cache return 1 fi + [ -e "$cache_file" ] && cat "$cache_file" return 0 } +export -f get_compose_relations run_service_relations () { - local services="$1" - for service in $services; do - for subservice in $(get_ordered_service_dependencies "$service"); do + local services="$1" loaded + declare -A loaded + for service in $(get_ordered_service_dependencies $services); do + # debug "Upping dep's relations of ${DARKYELLOW}$service${NORMAL}:" + for subservice in $(get_service_deps "$service") "$service"; do + [ "${loaded[$subservice]}" ] && continue + # debug " Relations of ${DARKYELLOW}$subservice${NORMAL}:" while read-0 relation_name target_service relation_config reverse; do export relation_config - Wrap -d "Building $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW$target_service$NORMAL" < $DARKYELLOW$target_service$NORMAL" </dev/null)" ]; then + new=true + fi + else + new=true + fi + + if [ "$new" ]; then echo "$relation_config" > "$relation_data_file" chmod go-rwx "$relation_data_file" ## protecting this file + echo "$relation_config" | md5_compat > "$relation_data_file.md5_ref" fi echo "$relation_data_file" } @@ -988,7 +1122,6 @@ has_service_action () { echo -en "direct\0$charm\0$target_script" | tee "$cache_file" return 0 fi - ## Action provided by relation ? while read-0 relation_name target_service relation_config reverse; do target_charm=$(get_service_charm "$target_service") || return 1 @@ -999,7 +1132,10 @@ has_service_action () { fi done < <(get_compose_relations "$service") - return 1 + master=$(_get_top_master_charm_for_service "$charm") + [ "$master" == "$charm" ] && return 1 + + has_service_action "$master" "$action" } export -f has_service_action @@ -1038,27 +1174,27 @@ get_compose_relation_config() { export -f get_compose_relation_config -## Return key-values-0 -get_compose_relation_config_for_service() { - local service=$1 relation_name=$2 relation_config - compose_service_relations=$(get_compose_relation_config "$service") || return 1 - if ! relation_config=$( - echo "$compose_service_relations" | - shyaml get-value "${relation_name}" 2>/dev/null); then - err "Couldn't find $DARKYELLOW${service}$NORMAL/${WHITE}${relation_name}$NORMAL" \ - "relation config in compose configuration." - return 1 - fi - if [ -z "$relation_config" ]; then - err "Relation ${WHITE}mysql-database$NORMAL is empty in compose configuration." - return 1 - fi - if ! echo "$relation_config" | shyaml key-values-0 2>/dev/null; then - err "No key/values in ${DARKBLUE}mysql-database$NORMAL of compose config." - return 1 - fi -} -export -f get_compose_relation_config_for_service +# ## Return key-values-0 +# get_compose_relation_config_for_service() { +# local service=$1 relation_name=$2 relation_config +# compose_service_relations=$(get_compose_relation_config "$service") || return 1 +# if ! relation_config=$( +# echo "$compose_service_relations" | +# shyaml get-value "${relation_name}" 2>/dev/null); then +# err "Couldn't find $DARKYELLOW${service}$NORMAL/${WHITE}${relation_name}$NORMAL" \ +# "relation config in compose configuration." +# return 1 +# fi +# if [ -z "$relation_config" ]; then +# err "Relation ${WHITE}mysql-database$NORMAL is empty in compose configuration." +# return 1 +# fi +# if ! echo "$relation_config" | shyaml key-values-0 2>/dev/null; then +# err "No key/values in ${DARKBLUE}mysql-database$NORMAL of compose config." +# return 1 +# fi +# } +# export -f get_compose_relation_config_for_service _get_master_charm_for_service() { @@ -1097,17 +1233,22 @@ _get_master_charm_for_service() { err "No ${WHITE}$interface${NORMAL} set for relation $relation_name." return 1 fi - service_def=$(get_compose_service_def "$service") || return 1 - target_service=$(echo "$service_def" | - shyaml keys "relations.${interface}" 2>/dev/null | - head -n 1) + + ## Action provided by relation ? + + target_service= + while read-0 relation_name candidate_target_service _relation_config _reverse; do + [ "$interface" == "$relation_name" ] && { + target_service="$candidate_target_service" + break + } + done < <(get_compose_relations "$service") if [ -z "$target_service" ]; then err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \ "${DARKYELLOW}$service$NORMAL compose definition." return 1 fi - target_charm=$(get_service_charm "$target_service") || return 1 - master_charm="$(_get_master_charm_for_service "$target_service")" + master_charm=$(get_service_charm "$target_service") || return 1 break fi done < <(echo "$requires" | shyaml key-values-0 2>/dev/null) @@ -1122,8 +1263,30 @@ _get_master_charm_for_service() { export -f _get_master_charm_for_service +_get_top_master_charm_for_service() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + current_service + + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + current_service="$service" + while true; do + master_service=$(_get_master_charm_for_service "$current_service") || return 1 + [ "$master_service" == "$current_service" ] && break + current_service="$master_service" + done + echo "$current_service" | tee "$cache_file" + return 0 +} +export -f _get_top_master_charm_for_service + + get_charm_metadata() { - local charm="$1" + local charm="$1" metadata_file metadata_file="$CHARM_STORE/$charm/metadata.yml" if ! [ -e "$metadata_file" ]; then return 0 ## No metadata file is as if metadata was empty @@ -1131,10 +1294,11 @@ get_charm_metadata() { cat "$metadata_file" } +export -f get_charm_metadata ## -## The result is a mixin that is not allways a complete valid -## docker-compose entry (thinkinf of subordinates). The result +## The result is a mixin that is not always a complete valid +## docker-compose entry (thinking of subordinates). The result ## will be merge with master charms. get_docker_compose_mixin_from_metadata() { local charm="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ @@ -1196,7 +1360,6 @@ get_docker_compose_mixin_from_metadata() { if [ "$image_or_build_statement" ]; then mixin=$(merge_yaml_str "$mixin" "$image_or_build_statement") fi - echo "$mixin" > "$cache_file" echo "$mixin" } @@ -1207,7 +1370,7 @@ _save() { local name="$1" cat - | tee -a "$docker_compose_dir/.data/$name" } - +export -f _save get_default_project_name() { if [ "$DEFAULT_PROJECT_NAME" ]; then @@ -1238,10 +1401,10 @@ launch_docker_compose() { mkdir -p "$docker_compose_tmpdir/$project" docker_compose_dir="$docker_compose_tmpdir/$project" - if [ -z "$services" ]; then - services=$(get_default_target_services $services) + if [ -z "$SERVICE_PACK" ]; then + export SERVICE_PACK=$(get_default_target_services $SERVICE_PACK) fi - get_docker_compose $services > "$docker_compose_dir/docker-compose.yml" || return 1 + get_docker_compose $SERVICE_PACK > "$docker_compose_dir/docker-compose.yml" || return 1 if [ -e "$state_tmpdir/to-merge-in-docker-compose.yml" ]; then # debug "Merging some config data in docker-compose.yml:" # debug "$(cat $state_tmpdir/to-merge-in-docker-compose.yml)" @@ -1255,7 +1418,7 @@ launch_docker_compose() { { { cd "$docker_compose_dir" - debug "${WHITE}docker-compose.yml:$NORMAL" + debug "${WHITE}docker-compose.yml$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL" debug "$(cat "$docker_compose_dir/docker-compose.yml")" debug "${WHITE}Launching$NORMAL: docker-compose $@" docker-compose "$@" @@ -1263,7 +1426,8 @@ launch_docker_compose() { } | _save stdout } 3>&1 1>&2 2>&3 | _save stderr 3>&1 1>&2 2>&3 if tail -n 1 "$docker_compose_dir/.data/stderr" | egrep "Service .+ failed to build: Error getting container [0-9a-f]+ from driver devicemapper: (open|Error mounting) /dev/mapper/docker-.*: no such file or directory$" >/dev/null 2>&1; then - debug NASTY ERROR DETECTED + err "Detected bug https://github.com/docker/docker/issues/4036 ... " + err "Please re-launch your command, or switch from 'devicemapper' driver to 'overlayfs' or 'aufs'." fi docker_compose_errlvl="$(cat "$docker_compose_dir/.data/errlvl")" @@ -1273,6 +1437,7 @@ launch_docker_compose() { fi return "$docker_compose_errlvl" } +export -f launch_docker_compose get_compose_yml_location() { @@ -1305,7 +1470,8 @@ get_default_target_services() { local services=("$@") if [ -z "${services[*]}" ]; then if [ "$DEFAULT_SERVICES" ]; then - info "No service provided, using \$DEFAULT_SERVICES variable. Target services: $DEFAULT_SERVICES" + info "No service provided, using $WHITE\$DEFAULT_SERVICES$NORMAL variable." \ + "Target services: $DARKYELLOW$DEFAULT_SERVICES$NORMAL" services="$DEFAULT_SERVICES" else err "No service provided." @@ -1314,12 +1480,14 @@ get_default_target_services() { fi echo "${services[*]}" } +export -f get_default_target_services + get_master_services() { local loaded master_service declare -A loaded for service in "$@"; do - master_service=$(_get_master_charm_for_service "$service") || return 1 + master_service=$(_get_top_master_charm_for_service "$service") || return 1 if [ "${loaded[$master_service]}" ]; then continue fi @@ -1327,6 +1495,7 @@ get_master_services() { loaded["$master_service"]=1 done | xargs echo } +export -f get_master_services _setup_state_dir() { @@ -1338,74 +1507,39 @@ _setup_state_dir() { } -[ "$SOURCED" ] && return 0 +get_docker_compose_opts_list() { + local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" + if [ -e "$cache_file" ]; then + debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + docker-compose "$@" --help | grep '^Options:' -A 20000 | + tail -n +2 | + egrep "^\s+-" | + sed -r 's/\s+((((-[a-zA-Z]|--[a-zA-Z0-9-]+)( [A-Z=]+|=[^ ]+)?)(, )?)+)\s+.*$/\1/g' | + tee "$cache_file" +} +_MULTIOPTION_REGEX='^((-[a-zA-Z]|--[a-zA-Z0-9-]+)(, )?)+' +_MULTIOPTION_REGEX_LINE_FILTER=$_MULTIOPTION_REGEX'(\s|=)' -## -## Argument parsing -## +get_docker_compose_multi_opts_list() { + opts_list=$(get_docker_compose_opts_list "$@") + echo "$opts_list" | egrep "$_MULTIOPTION_REGEX_LINE_FILTER" | + sed -r "s/^($_MULTIOPTION_REGEX)(\s|=).*$/\1/g" | + tr ',' "\n" | xargs echo +} +get_docker_compose_single_opts_list() { + opts_list=$(get_docker_compose_opts_list "$@") + echo "$opts_list" | egrep -v "$_MULTIOPTION_REGEX_LINE_FILTER" | + tr ',' "\n" | xargs echo +} -fullargs=() -opts=() -posargs=() -no_hooks= -no_init= -while [ "$#" != 0 ]; do - case "$1" in - # --help|-h) - # print_help - # exit 0 - # ;; - --verbose|-v) - fullargs+=("$1") - export VERBOSE=true - ;; - -f) - if ! [ -e "$2" ]; then - die "File $2 doesn't exists" - fi - export DEFAULT_COMPOSE_FILE="$2" - shift - ;; - -p) - fullargs+=("$1" "$2") - opts=("${opts[@]}" "$1" "$2") - export DEFAULT_PROJECT_NAME="$2" - shift - ;; - --no-relations) - export no_relations=true - ;; - --no-hooks) - export no_hooks=true - ;; - --no-init) - export no_init=true - ;; - --debug) - export DEBUG=true - export VERBOSE=true - ;; - --) - fullargs+=("$1") - shift - opts=("${opts[@]}" "$@") - break 2 - ;; - -*) - fullargs+=("$1" "$2") - opts=("${opts[@]}" "$1" "$2") - shift - ;; - *) - fullargs+=("$1") - posargs=("${posargs[@]}" "$1") - ;; - esac - shift -done + +[ "$SOURCED" ] && return 0 if [ -z "$DISABLE_SYSTEM_CONFIG_FILE" ]; then @@ -1423,6 +1557,96 @@ if [ -z "$DISABLE_SYSTEM_CONFIG_FILE" ]; then . /etc/compose.local.conf fi +_setup_state_dir + +## +## Argument parsing +## + +remainder_args=() +compose_opts=() +action_opts=() +no_hooks= +no_init= +action= +stage="main" ## switches from 'main', to 'action', 'remainder' +# DC_MATCH_MULTI= +# DC_MATCH_SINGLE= +while [ "$#" != 0 ]; do + case "$stage" in + "main") + case "$1" in + --help|-h) + no_init=true ; no_hooks=true ; no_relations=true + compose_opts+=("$1") + ;; + --verbose|-v) + export VERBOSE=true + compose_opts+=("$1") + ;; + -f) + [ -e "$2" ] || die "File $2 doesn't exists" + export DEFAULT_COMPOSE_FILE="$2" + shift + ;; + -p) + export DEFAULT_PROJECT_NAME="$2" + shift + ;; + --no-relations) + export no_relations=true + ;; + --no-hooks) + export no_hooks=true + ;; + --no-init) + export no_init=true + ;; + --debug) + export DEBUG=true + export VERBOSE=true + ;; + --*|-*) + compose_opts+=("$1") + ;; + *) + action="$1" + stage="action" + # DC_MATCH_MULTI=$(get_docker_compose_multi_opts_list "$action") || return 1 + # DC_MATCH_SINGLE="$(get_docker_compose_single_opts_list "$action") $(echo "$DC_MATCH_MULTI" | sed -r 's/( |$)/=\* /g')" + ;; + esac + ;; + "action") + case "$1" in + --help|-h) + no_init=true ; no_hooks=true ; no_relations=true + action_opts+=("$1") + ;; + --verbose|-v) + export VERBOSE=true + action_opts+=("$1") + ;; + --*|-*) + action_opts+=("$1") + ;; + *) + action_posargs+=("$1") + stage="remainder" + ;; + esac + ;; + "remainder") + remainder_args+=("$@") + break 3;; + esac + shift +done + +[ "${compose_opts[*]}" ] && debug "Main docker-compose opts: ${compose_opts[*]}" +[ "${action_opts[*]}" ] && debug "Action '$action' opts: ${action_opts[*]}" +[ "${remainder_args[*]}" ] && debug "Remainder args: ${remainder_args[*]}" + ## ## Actual code @@ -1448,34 +1672,37 @@ if [ -z "$(cd "$CHARM_STORE"; ls)" ]; then print_error "Charm store is empty. Cannot continue." fi - -_setup_state_dir - ## ## Get services in command line. ## -action="${posargs[0]}" is_service_action= case "$action" in up|build|start|stop|config) - services="$(get_default_target_services "${posargs[@]:1}")" || exit 1 - orig_services="${posargs[@]:1}" + services="$(get_default_target_services "${action_posargs[@]}")" || exit 1 + orig_services="${action_posargs[@]:1}" ;; run) - services="${posargs[1]}" + services="${action_posargs[0]}" + ;; + "") + services= ;; *) - if is_service_action=$(has_service_action "${posargs[1]}" "$action"); then - debug "Found action $DARKCYAN$action$NORMAL in service $DARKYELLOW${posargs[1]}$NORMAL" - services="${posargs[1]}" + if is_service_action=$(has_service_action "${action_posargs[0]}" "$action"); then + debug "Found action $DARKCYAN$action$NORMAL in service $DARKYELLOW${action_posargs[0]}$NORMAL" + services="${action_posargs[0]}" else - services="$(get_default_target_services)" + services="$(get_default_target_services "${action_posargs[@]}")" fi ;; esac -get_docker_compose $services >/dev/null || exit 1 ## precalculate variable \$_currnet_docker_compose + +get_docker_compose $services >/dev/null || { ## precalculate variable \$_current_docker_compose + err "Fails to compile base 'docker-conmpose.conf'" + exit 1 +} ## ## Pre-action @@ -1487,6 +1714,9 @@ case "$action" in up|run) full_init=true ;; + "") + full_init= + ;; *) if [ "$is_service_action" ]; then full_init=true @@ -1524,6 +1754,7 @@ if [ "$full_init" ]; then fi fi +export SERVICE_PACK="$services" ## ## Docker-compose @@ -1531,23 +1762,28 @@ fi case "$action" in up|start|stop|build) - master_services=$(get_master_services $services) || exit 1 - launch_docker_compose "$action" "${opts[@]}" $master_services + master_services=$(get_master_services $SERVICE_PACK) || exit 1 + launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" $master_services ;; run) - master_service=$(get_master_services $services) || exit 1 - launch_docker_compose "$action" "$master_service" "${opts[@]}" "${posargs[@]:2}" + master_service=$(get_master_services $SERVICE_PACK) || exit 1 + launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "$master_service" "${remainder_args[@]}" ;; + # enter) + # master_service=$(get_master_services $SERVICE_PACK) || exit 1 + # [ "${remainder_args[*]}" ] || remainder_args=("/bin/bash" "-c" "export TERM=xterm; exec bash") + # docker exec -ti "${action_opts[@]}" "$master_service" "${remainder_args[@]}" + # ;; config) ## removing the services - launch_docker_compose config "${opts[@]}" + launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" warn "Runtime configuration modification (from relations) are not included here." ;; *) if [ "$is_service_action" ]; then - run_service_action "$services" "$action" "${opts[@]}" "${posargs[@]:2}" + run_service_action "$SERVICE_PACK" "$action" "${remainder_args[@]}" else - launch_docker_compose "${fullargs[@]}" + launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" fi ;; esac diff --git a/test/test b/test/test index af00c45..85e4bd2 100755 --- a/test/test +++ b/test/test @@ -118,9 +118,13 @@ EOF2 . "$tprog" _setup_state_dir -test "\$(get_docker_compose_mixin_from_metadata testcharm)" == "entrypoint: any -volumes: -- /any:/vol" +out="\$(get_docker_compose_mixin_from_metadata testcharm)" +test "\$out" == "volumes: +- /any:/vol +entrypoint: any" || { + echo -e "** get_docker_compose_mixin_from_metadata testcharm:\n\$out" + exit 1 +} ## -- image @@ -368,6 +372,47 @@ EOF } +function test_yaml_key_val_str { + + init_test + + assert_list < $test_tmpdir/compose.yml +app: + charm: app + relations: + web-proxy: + www: + user: toto + db: mysql +EOF2 + +. "$tprog" +COMPOSE_YML_FILE=$test_tmpdir/compose.yml +_setup_state_dir + +out=\$(_get_docker_compose_links "app") +test "\$out" == "app: + links: + - www + - mysql" || { + echo -e "** _get_docker_compose_links:\n\$out"; exit 1 +} + + ## -- reverse-tech-dep + export CHARM_STORE=$test_tmpdir mkdir -p $test_tmpdir/{www,mysql} cat < $test_tmpdir/www/metadata.yml