diff --git a/bin/compose b/bin/compose index 65a0b98..083c7e4 100755 --- a/bin/compose +++ b/bin/compose @@ -23,6 +23,14 @@ ## This will allow to cache completely without issues function in time. ## - would probably need instrospection in charm custom action to know if these need ## init or relations to be set up. +## - Be clear about when the SERVICE name is used and the CHARM name is used. +## - in case of service contained in another container +## - in normal case +## - in docker-compose, can't use charm name: if we want 2 instances of the same charm +## we are stuck. What will be unique is the name of the service. +## - some relations are configured in compose.yml but should not trigger the loading +## of necessary component (for instance, apache --> log-rotate), if log-rotate is +## not there, this link should considered optional. #:- [ -e /etc/shlib ] && . /etc/shlib || { @@ -75,17 +83,30 @@ ${WHITE}$exname Options${NORMAL}: " -[[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true +## XXXvlab: this doesn't seem to work when 'compose' is called in +## a hook of a charm. +#[[ "${BASH_SOURCE[0]}" == "" ]] && SOURCED=true +$(return >/dev/null 2>&1) && SOURCED=true -export CACHEDIR=/var/cache/compose -export VARDIR=/var/lib/compose +if [ "$UID" == 0 ]; then + CACHEDIR=${CACHEDIR:-/var/cache/compose} + VARDIR=${VARDIR:-/var/lib/compose} +else + [ "$XDG_CONFIG_HOME" ] && CACHEDIR=${CACHEDIR:-$XDG_CONFIG_HOME/compose} + [ "$XDG_DATA_HOME" ] && VARDIR=${VARDIR:-$XDG_DATA_HOME/compose} + CACHEDIR=${CACHEDIR:-$HOME/.cache/compose} + VARDIR=${VARDIR:-$HOME/.local/share/compose} +fi +export VARDIR CACHEDIR + md5_compat() { md5sum | cut -c -32; } quick_cat_file() { quick_cat_stdin < "$1"; } quick_cat_stdin() { local IFS=''; while read -r line; do echo "$line"; done ; } export -f quick_cat_file quick_cat_stdin md5_compat + clean_cache() { local i=0 for f in $(ls -t "$CACHEDIR/"*.cache.* 2>/dev/null | tail -n +500); do @@ -294,6 +315,7 @@ cached_cmd_on_base_image() { } export -f cached_cmd_on_base_image + image_exposed_ports_0() { local image="$1" docker inspect --format='{{range $p, $conf := .Config.ExposedPorts}}{{$p}}{{"\x00"}}{{end}}' "$image" @@ -542,10 +564,11 @@ ensure_db_docker_running () { _set_db_params "$DOCKER_IP" "$DOCKER_NETWORK" return 0 else - err "Db not found. Docker logs follows:" + errlvl="$?" + err "Db not found (errlvl: $errlvl). Tail of docker logs follows:" docker logs --tail=5 "$container_id" 2>&1 | prefix " | " >&2 rm "/tmp/${_DB_NAME}.working" - return 1 + return "$errlvl" fi } export -f ensure_db_docker_running @@ -575,16 +598,16 @@ export -f _dcmd ## Executes code through db -dcmd () { - +dcmd() { + local fun [ "$DB_NAME" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_NAME." [ "$DB_DATADIR" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_DATADIR." - [ "$DB_PASSFILE" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_PASSFILE." + # [ "$DB_PASSFILE" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_PASSFILE." [ "$_PID" ] || print_syntax_error "$FUNCNAME: You must provide \$_PID." - [ "$(type -t is_db_locked)" == "function" ] || print_syntax_error "$FUNCNAME: You must provide function 'is_db_locked'." - [ "$(type -t _set_db_params)" == "function" ] || print_syntax_error "$FUNCNAME: You must provide function '_set_db_params'." - [ "$(type -t ddb)" == "function" ] || print_syntax_error "$FUNCNAME: You must provide function 'ddb'." - + for fun in is_db_locked _set_db_params ddb; do + [ "$(type -t "$fun")" == "function" ] || + print_syntax_error "$FUNCNAME: You must provide function '$fun'." + done ensure_db_docker_running /dev/null 2>&1 && break + sleep 0.2 + if [ "$((SECONDS - start))" -gt "$timeout" ]; then + err "${RED}timeout error${NORMAL}(${timeout}s):"\ + "Could not connect to $host_port." + return 1 + fi + done + return 0 +} +export -f wait_for_tcp_port + + ## Warning: requires a ``ddb`` matching current database to be checked wait_for_docker_ip() { - local name=$1 timeout=15 timeout_count=0 DOCKER_IP= DOCKER_NETWORK= docker_ips= docker_ip= DB_OK= + local name=$1 DOCKER_IP= DOCKER_NETWORK= docker_ips= docker_ip= docker_ip=$(wait_docker_ip "$name" 5) || return 1 IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip" if ! str_is_ipv4 "$DOCKER_IP"; then err "internal 'wait_docker_ip' did not return a valid IP. Returned IP is '$DOCKER_IP'." return 1 fi - timeout_count=0 - DB_OK= _set_db_params "$DOCKER_IP" "$DOCKER_NETWORK" - while [ -z "$DB_OK" ]; do - sleep 1 - echo "SELECT 1;" | ddb >/dev/null && DB_OK=1 - verb "[2/2] Waiting for db service from docker $name... ($[timeout_count + 1]/$timeout)" - ((timeout_count++)) || true - if [ "$timeout_count" == "$timeout" ]; then - err "${RED}timeout error${NORMAL}(${timeout}s):"\ - "Could not connect to db on $DOCKER_IP" \ - "container's IP." - return 1 - fi - done - verb "[2/2] Db is ready !" + while read-0 port; do + IFS="/" read port type <<<"$port" + [ "$type" == "tcp" ] || continue + wait_for_tcp_port "$DOCKER_IP:${port}" || return 17 + verb "Port $DOCKER_IP:${port} checked open." + done < <(image_exposed_ports_0 "$container_id") + + ## Checking direct connection + if ! echo "SELECT 1;" | ddb >/dev/null; then + err "${RED}db connection error${NORMAL}:"\ + "Could not connect to db on $DOCKER_IP" \ + "container's IP. (IP up, TCP ports is(are) open)" + return 18 + fi echo "${DOCKER_NETWORK}:${DOCKER_IP}" return 0 } @@ -711,19 +750,25 @@ get_docker_compose_links() { fi master_service=$(get_top_master_service_for_service "$service") || return 1 - + ## XXXvlab: yuck, this make the assumption that next function is cached, + ## and leverage the fact that the result is stored in a file. All this only + ## to catch a failure of ``get_compose_relations``, that would go out silently. + get_compose_relations "$service" >/dev/null || return 1 ## fetch cache and fail if necessary deps=() while read-0 _relation_name target_service _relation_config tech_dep; do - master_target_service="$(get_top_master_service_for_service "$target_service")" + master_target_service="$(get_top_master_service_for_service "$target_service")" || return 1 [ "$master_service" == "$master_target_service" ] && continue if [ "$tech_dep" == "reversed" ]; then deps+=("$(echo -en "$master_target_service:\n links:\n - $master_service")") elif [ "$tech_dep" == "True" ]; then deps+=("$(echo -en "$master_service:\n links:\n - $master_target_service")") fi - done < <(get_compose_relations "$service") || return 1 - - merge_yaml_str "${deps[@]}" | tee "$cache_file" + ## XXXvlab: an attempt to add depends_on, but this doesn't work well actually + ## as there's a circular dependency issue. We don't really want the full feature + ## of depends_on, but just to add it as targets when doing an 'up' + # deps+=("$(echo -en "$master_service:\n depends_on:\n - $master_target_service")") + done < <(get_compose_relations "$service") + merge_yaml_str "${deps[@]}" | tee "$cache_file" || return 1 if [ "${PIPESTATUS[0]}" != 0 ]; then rm "$cache_file" return 1 @@ -777,7 +822,6 @@ _get_docker_compose_service_mixin() { cat "$cache_file" return 0 fi - master_service=$(get_top_master_service_for_service "$service") || { err "Failed to get top master service for service $DARKYELLOW$service$NORMAL" return 1 @@ -787,17 +831,15 @@ _get_docker_compose_service_mixin() { base_mixin="$master_service: labels: - - \"compose.service=$service\" - - \"compose.master-service=${master_service}\" - - \"compose.project=$(get_default_project_name)\"" + - compose.service=$service + - compose.master-service=${master_service} + - compose.project=$(get_default_project_name)" links_yaml=$(get_docker_compose_links "$service") || return 1 docker_compose_options=$(_get_docker_compose_opts "$service") || return 1 ## the charm part - #debug "Get charm name from service name $DARKYELLOW$service$NORMAL." - charm=$(get_service_charm "$service") || return 1 - charm_part=$(get_docker_compose_mixin_from_metadata "$service" "$charm") || return 1 + charm_part=$(get_docker_compose_mixin_from_metadata "$service") || return 1 ## Merge results if [ "$charm_part" ]; then @@ -822,7 +864,8 @@ export -f _get_docker_compose_service_mixin ## @export ## @cache: !system !nofail +stdout get_docker_compose () { - local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" entries services service start + local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + entries services service start docker_compose_services if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" @@ -892,19 +935,31 @@ _get_compose_service_def_cached () { local service="$1" docker_compose="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" if [ -e "$cache_file" ]; then #debug "$FUNCNAME: STATIC cache hit" - cat "$cache_file" - touch "$cache_file" + cat "$cache_file" && + touch "$cache_file" || return 1 return 0 fi - - service_def_base="charm: $service" value=$(echo "$docker_compose" | shyaml get-value "$service" 2>/dev/null) - merge_yaml <(echo "$service_def_base") <(echo "$value") | tee "$cache_file" - if [ "${PIPESTATUS[0]}" != 0 ]; then - rm "$cache_file" - return 1 + if ! echo "$value" | shyaml get-value "charm" >/dev/null 2>&1; then + if charm.exists "$service"; then + value=$(merge_yaml <(echo "charm: $service") <(echo "$value")) || return 1 + else + err "No ${WHITE}charm${NORMAL} value for service $DARKYELLOW$service$NORMAL" \ + "in compose, nor same name charm found." + return 1 + fi fi - + echo "$value" | tee "$cache_file" || return 1 + # if [ "${PIPESTATUS[0]}" != 0 ]; then + # rm "$cache_file" + # return 1 + # fi + return 0 + # if [ "${PIPESTATUS[0]}" != 0 -o \! -s "$cache_file" ]; then + # rm "$cache_file" + # err "PAS OK $service: $value" + # return 1 + # fi } export -f _get_compose_service_def_cached @@ -914,15 +969,15 @@ get_compose_service_def () { result if [ -e "$cache_file" ]; then #debug "$FUNCNAME: SESSION cache hit" - cat "$cache_file" + cat "$cache_file" || return 1 return 0 fi [ -z "$service" ] && print_syntax_error "Missing service as first argument." - docker_compose=$(cat "$COMPOSE_YML_FILE") || return 1 + docker_compose=$([ "$COMPOSE_YML_FILE" -a -e "$COMPOSE_YML_FILE" ] && cat "$COMPOSE_YML_FILE") result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1 - echo "$result" | tee "$cache_file" + echo "$result" | tee "$cache_file" || return 1 } export -f get_compose_service_def @@ -931,8 +986,8 @@ _get_service_charm_cached () { local service="$1" service_def="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit $1" - cat "$cache_file" - touch "$cache_file" + cat "$cache_file" && + touch "$cache_file" || return 1 return 0 fi charm=$(echo "$service_def" | shyaml get-value charm 2>/dev/null) @@ -940,13 +995,14 @@ _get_service_charm_cached () { err "Missing ${WHITE}charm${NORMAL} value in service $DARKYELLOW$service$NORMAL definition." return 1 fi - echo "$charm" | tee "$cache_file" + echo "$charm" | tee "$cache_file" || return 1 } export -f _get_service_charm_cached get_service_charm () { local service="$1" if [ -z "$service" ]; then + echo ${FUNCNAME[@]} >&2 print_syntax_error "$FUNCNAME: Please specify a service as first argument." return 1 fi @@ -1111,8 +1167,9 @@ get_ordered_service_dependencies() { #debug "Figuring ordered deps of $DARKYELLOW$services$NORMAL" if [ -z "${services[*]}" ]; then - print_syntax_error "$FUNCNAME: no arguments" - return 1 + return 0 + # print_syntax_error "$FUNCNAME: no arguments" + # return 1 fi declare -A depths @@ -1138,11 +1195,12 @@ get_ordered_service_dependencies() { export -f get_ordered_service_dependencies run_service_hook () { - local services="$1" action="$2" loaded + local services="$1" action="$2" subservices loaded declare -A loaded for service in $services; do - for subservice in $(get_ordered_service_dependencies "$service"); do + subservices=$(get_ordered_service_dependencies "$service") || return 1 + for subservice in $subservices; do if [ "${loaded[$subservice]}" ]; then ## Prevent double inclusion of same service if this ## service is deps of two or more of your @@ -1254,11 +1312,12 @@ export -f setup_host_resource setup_host_resources () { - local services="$1" loaded + local services="$1" subservices loaded declare -A loaded for service in $services; do - for subservice in $(get_ordered_service_dependencies "$service"); do + subservices=$(get_ordered_service_dependencies "$service") || return 1 + for subservice in $subservices; do if [ "${loaded[$subservice]}" ]; then ## Prevent double inclusion of same service if this ## service is deps of two or more of your @@ -1469,8 +1528,8 @@ _get_compose_relations_cached () { relation_name relation_def target_service if [ -e "$cache_file" ]; then #debug "$FUNCNAME: STATIC cache hit $1" - cat "$cache_file" - touch "$cache_file" + cat "$cache_file" && + touch "$cache_file" || return 1 return 0 fi @@ -1478,30 +1537,31 @@ _get_compose_relations_cached () { set -o pipefail if [ "$compose_service_def" ]; then while read-0 relation_name relation_def; do + ## XXXvlab: could we use braces here instead of parenthesis ? ( case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in "str") target_service="$(echo "$relation_def" | shyaml get-value 2>/dev/null)" - target_charm=$(get_service_charm "$target_service") + target_charm=$(get_service_charm "$target_service") || return 1 tech_dep="$(get_charm_tech_dep_orientation_for_relation "$target_charm" "$relation_name")" echo -en "$relation_name\0$target_service\0\0$tech_dep\0" ;; "sequence") while read-0 target_service; do - target_charm=$(get_service_charm "$target_service") + target_charm=$(get_service_charm "$target_service") || return 1 tech_dep="$(get_charm_tech_dep_orientation_for_relation "$target_charm" "$relation_name")" echo -en "$relation_name\0$target_service\0\0$tech_dep\0" done < <(echo "$relation_def" | shyaml get-values-0 2>/dev/null) ;; "struct") while read-0 target_service relation_config; do - target_charm=$(get_service_charm "$target_service") + target_charm=$(get_service_charm "$target_service") || return 1 tech_dep="$(get_charm_tech_dep_orientation_for_relation "$target_charm" "$relation_name")" echo -en "$relation_name\0$target_service\0$relation_config\0$tech_dep\0" done < <(echo "$relation_def" | shyaml key-values-0 2>/dev/null) ;; esac - ) > "$cache_file" + ) > "$cache_file" || return 1 done < <(echo "$compose_service_def" | shyaml key-values-0 relations 2>/dev/null) fi ) @@ -1528,7 +1588,6 @@ get_compose_relations () { compose_def="$(get_compose_service_def "$service")" || return 1 _get_compose_relations_cached "$compose_def" > "$cache_file" if [ "$?" != 0 ]; then - err "Error while looking for compose relations." rm -f "$cache_file" ## no cache return 1 fi @@ -1536,15 +1595,27 @@ get_compose_relations () { } export -f get_compose_relations + +get_compose_relation_def() { + local service="$1" relation="$2" relation_name target_service relation_config tech_dep + while read-0 relation_name target_service relation_config tech_dep; do + [ "$relation_name" == "$relation" ] || continue + printf "%s\0%s\0%s\0" "$target_service" "$relation_config" "$tech_dep" + done < <(get_compose_relations "$service") || return 1 +} +export -f get_compose_relation_def + + run_service_relations () { - local services="$1" loaded + local services="$1" loaded subservices PROJECT_NAME=$(get_default_project_name) || return 1 export PROJECT_NAME declare -A loaded - for service in $(get_ordered_service_dependencies $services); do + subservices=$(get_ordered_service_dependencies $services) || return 1 + for service in $subservices; do # debug "Upping dep's relations of ${DARKYELLOW}$service${NORMAL}:" for subservice in $(get_service_deps "$service") "$service"; do @@ -1805,8 +1876,8 @@ _get_master_service_for_service_cached () { charm requires master_charm target_charm target_service service_def found if [ -e "$cache_file" ]; then # debug "$FUNCNAME: STATIC cache hit ($1)" - cat "$cache_file" - touch "$cache_file" + cat "$cache_file" && + touch "$cache_file" || return 1 return 0 fi @@ -1864,17 +1935,16 @@ get_master_service_for_service() { if [ -e "$cache_file" ]; then # debug "$FUNCNAME: SESSION cache hit ($*)" - cat "$cache_file" + cat "$cache_file" || return 1 return 0 fi - charm=$(get_service_charm "$service") || return 1 metadata=$(charm.metadata "$charm" 2>/dev/null) || { metadata="" warn "No charm $DARKPINK$charm$NORMAL found." } result=$(_get_master_service_for_service_cached "$service" "$charm" "$metadata") || return 1 - echo "$result" | tee "$cache_file" + echo "$result" | tee "$cache_file" || return 1 } export -f get_master_service_for_service @@ -1906,16 +1976,16 @@ export -f get_top_master_service_for_service ## docker-compose entry (thinking of subordinates). The result ## will be merge with master charms. _get_docker_compose_mixin_from_metadata_cached() { - local service="$1" charm="$2" metadata="$3" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + local service="$1" charm="$2" metadata="$3" has_build_dir="$4" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ metadata_file metadata volumes docker_compose subordinate image if [ -e "$cache_file" ]; then #debug "$FUNCNAME: STATIC cache hit $1" - cat "$cache_file" - touch "$cache_file" + cat "$cache_file" && + touch "$cache_file" || return 1 return 0 fi - mixin=$(echo -en "labels:\n - \"compose.charm=$charm\"") + mixin=$(echo -en "labels:\n- compose.charm=$charm") if [ "$metadata" ]; then ## resources to volumes volumes=$( @@ -1925,10 +1995,16 @@ _get_docker_compose_mixin_from_metadata_cached() { done < <(echo "$metadata" | shyaml get-values-0 "${resource_type}-resources" 2>/dev/null) done while read-0 resource; do - if [[ "$resource" == *:* ]]; then + if [[ "$resource" == /*:/*:* ]]; then + echo " - $resource" + elif [[ "$resource" == /*:/* ]]; then echo " - $resource:rw" - else + elif [[ "$resource" == /*:* ]]; then + echo " - ${resource%%:*}:$resource" + elif [[ "$resource" =~ ^/[^:]+$ ]]; then echo " - $resource:$resource:rw" + else + die "Invalid host-resource specified in 'metadata.yml'." fi done < <(echo "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null) while read-0 resource; do @@ -1969,7 +2045,7 @@ _get_docker_compose_mixin_from_metadata_cached() { return 1 fi image_or_build_statement="image: $image" - elif [ -d "$(charm.get_dir "$charm")/build" ]; then + elif [ "$has_build_dir" ]; then if [ "$subordinate" ]; then err "Subordinate charm can not have a 'build' sub directory." return 1 @@ -1985,15 +2061,17 @@ export -f _get_docker_compose_mixin_from_metadata_cached get_docker_compose_mixin_from_metadata() { - local service="$1" charm="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" if [ -e "$cache_file" ]; then #debug "$FUNCNAME: SESSION cache hit ($*)" cat "$cache_file" return 0 fi - - metadata="$(charm.metadata "$charm" 2>/dev/null)" - mixin=$(_get_docker_compose_mixin_from_metadata_cached "$service" "$charm" "$metadata") || return 1 + charm=$(get_service_charm "$service") || return 1 + metadata="$(charm.metadata "$charm" 2>/dev/null)" || return 1 + has_build_dir= + [ -d "$(charm.get_dir "$charm")/build" ] && has_build_dir=true + mixin=$(_get_docker_compose_mixin_from_metadata_cached "$service" "$charm" "$metadata" "$has_build_dir") || return 1 echo "$mixin" | tee "$cache_file" } export -f get_docker_compose_mixin_from_metadata @@ -2024,7 +2102,7 @@ export -f get_default_project_name launch_docker_compose() { - local charm + local charm docker_compose_tmpdir docker_compose_dir docker_compose_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX) #debug "Creating temporary docker-compose directory in '$docker_compose_tmpdir'." # trap_add EXIT "debug \"Removing temporary docker-compose directory in $docker_compose_tmpdir.\";\ @@ -2034,9 +2112,9 @@ launch_docker_compose() { mkdir -p "$docker_compose_tmpdir/$project" docker_compose_dir="$docker_compose_tmpdir/$project" - if [ -z "$SERVICE_PACK" ]; then - export SERVICE_PACK=$(get_default_target_services $SERVICE_PACK) - fi + # if [ -z "$SERVICE_PACK" ]; then + # export SERVICE_PACK=$(get_default_target_services $SERVICE_PACK) + # fi get_docker_compose $SERVICE_PACK > "$docker_compose_dir/docker-compose.yml" || return 1 if [ -e "$state_tmpdir/to-merge-in-docker-compose.yml" ]; then # debug "Merging some config data in docker-compose.yml:" @@ -2063,7 +2141,11 @@ launch_docker_compose() { debug "${WHITE}docker-compose.yml$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:" debug "$(cat "$docker_compose_dir/docker-compose.yml" | prefix " $GRAY|$NORMAL ")" debug "${WHITE}Launching$NORMAL: docker-compose $@" - docker-compose "$@" + if [ "$DRY_COMPOSE_RUN" ]; then + echo docker-compose "$@" + else + docker-compose "$@" + fi echo "$?" > "$docker_compose_dir/.data/errlvl" } | _save stdout } 3>&1 1>&2 2>&3 | _save stderr 3>&1 1>&2 2>&3 @@ -2094,7 +2176,8 @@ get_compose_yml_location() { if [ "$DEFAULT_COMPOSE_FILE" ]; then if ! [ -e "$DEFAULT_COMPOSE_FILE" ]; then err "No 'compose.yml' was found in current or parent dirs," \ - "and \$DEFAULT_COMPOSE_FILE points to an unexistent file." + "and \$DEFAULT_COMPOSE_FILE points to an unexistent file." \ + "(${DEFAULT_COMPOSE_FILE})" die "Please provide a 'compose.yml' file." fi echo "$DEFAULT_COMPOSE_FILE" @@ -2135,6 +2218,7 @@ get_master_services() { echo "$master_service" loaded["$master_service"]=1 done | xargs echo + return "${PIPESTATUS[0]}" } export -f get_master_services @@ -2148,18 +2232,50 @@ _setup_state_dir() { } -get_docker_compose_opts_list() { - local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$action"; cat "$(which docker-compose)" | md5_compat)" +get_docker_compose_action_help_msg() { + local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$action"; cat "$(which docker-compose)" | md5_compat)" \ + docker_compose_help_msg if [ -e "$cache_file" ]; then - cat "$cache_file" + cat "$cache_file" && + touch "$cache_file" || return 1 return 0 fi docker_compose_help_msg=$(docker-compose "$action" --help 2>/dev/null) || return 1 + echo "$docker_compose_help_msg" | + tee "$cache_file" || return 1 +} + +get_docker_compose_action_usage() { + local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$action"; cat "$(which docker-compose)" | md5_compat)" \ + docker_compose_help_msg + if [ -e "$cache_file" ]; then + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + docker_compose_help_msg=$(get_docker_compose_action_help_msg "$action") || return 1 + echo "$docker_compose_help_msg" | + grep -m 1 "^Usage:" -A 10000 | + egrep -m 1 "^\$" -B 10000 | + xargs echo | + sed -r 's/^Usage: //g' | + tee "$cache_file" || return 1 +} + +get_docker_compose_opts_list() { + local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$action"; cat "$(which docker-compose)" | md5_compat)" \ + docker_compose_help_msg + if [ -e "$cache_file" ]; then + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + docker_compose_help_msg=$(get_docker_compose_action_help_msg "$action") || return 1 echo "$docker_compose_help_msg" | grep '^Options:' -A 20000 | tail -n +2 | egrep "^\s+-" | sed -r 's/\s+((((-[a-zA-Z]|--[a-zA-Z0-9-]+)( [A-Z=]+|=[^ ]+)?)(, )?)+)\s+.*$/\1/g' | - tee "$cache_file" + tee "$cache_file" || return 1 } _MULTIOPTION_REGEX='^((-[a-zA-Z]|--[a-zA-Z0-9-]+)(, )?)+' @@ -2312,7 +2428,7 @@ digraph g { ratio = auto; EOF for target_service in "$@"; do - services=$(get_ordered_service_dependencies "$target_service") + services=$(get_ordered_service_dependencies "$target_service") || return 1 for service in $services; do [ "${entries[$service]}" ] && continue || entries[$service]=1 if cla_contains "$service" "${services[@]}"; then @@ -2329,6 +2445,11 @@ EOF [ "$SOURCED" ] && return 0 +# if [[ "$UID" != 0 ]]; then +# die "'$exname' requires root permissions (for now). Please run as root." +# fi + + if [ -z "$DISABLE_SYSTEM_CONFIG_FILE" ]; then if [ -r /etc/default/charm ]; then . /etc/default/charm @@ -2362,9 +2483,12 @@ export DOCKER_HOST_NET DOCKER_HOST_IP ## Argument parsing ## +services=() remainder_args=() compose_opts=() action_opts=() +services_args=() +pos_arg_ct=0 no_hooks= no_init= action= @@ -2410,22 +2534,41 @@ while [ "$#" != 0 ]; do export DEBUG=true export VERBOSE=true ;; + --dirs) + echo "CACHEDIR: $CACHEDIR" + echo "VARDIR: $VARDIR" + exit 0 + ;; + --dry-compose-run) + export DRY_COMPOSE_RUN=true + ;; --*|-*) compose_opts+=("$1") ;; *) action="$1" stage="action" - DC_MATCH_MULTI=$(get_docker_compose_multi_opts_list "$action") && - DC_MATCH_SINGLE="$(get_docker_compose_single_opts_list "$action")" - if [ "$DC_MATCH_MULTI" ]; then - DC_MATCH_SINGLE="$DC_MATCH_SINGLE $(echo "$DC_MATCH_MULTI" | sed -r 's/( |$)/=\* /g')" + if DC_USAGE=$(get_docker_compose_action_usage "$action"); then + is_docker_compose_action=true + DC_MATCH_MULTI=$(get_docker_compose_multi_opts_list "$action") && + DC_MATCH_SINGLE="$(get_docker_compose_single_opts_list "$action")" + if [ "$DC_MATCH_MULTI" ]; then + DC_MATCH_SINGLE="$DC_MATCH_SINGLE $(echo "$DC_MATCH_MULTI" | sed -r 's/( |$)/=\* /g')" + fi + pos_args=($(echo "$DC_USAGE" | sed -r 's/\[-[^]]+\] ?//g;s/\[options\] ?//g')) + pos_args=("${pos_args[@]:1}") + # echo "USAGE: $DC_USAGE" + # echo "pos_args: ${pos_args[@]}" + # echo "MULTI: $DC_MATCH_MULTI" + # echo "SINGLE: $DC_MATCH_SINGLE" + # exit 1 + else + stage="remainder" fi - is_docker_compose_action=true ;; esac ;; - "action") + "action") ## Only for docker-compose actions case "$1" in --help|-h) no_init=true ; no_hooks=true ; no_relations=true @@ -2434,10 +2577,10 @@ while [ "$#" != 0 ]; do --*|-*) if [ "$is_docker_compose_action" ]; then if str_matches "$1" $DC_MATCH_MULTI; then - action_opts=("$1" "$2") + action_opts+=("$1" "$2") shift; elif str_matches "$1" $DC_MATCH_SINGLE; then - action_opts=("$1") + action_opts+=("$1") else err "Unknown option '$1'. Please check help." docker-compose "$action" --help >&2 @@ -2446,21 +2589,33 @@ while [ "$#" != 0 ]; do fi ;; *) - action_posargs+=("$1") - stage="remainder" + # echo "LOOP $1 : pos_arg: $pos_arg_ct // ${pos_args[$pos_arg_ct]}" + if [[ "${pos_args[$pos_arg_ct]}" == "[SERVICE...]" ]]; then + services_args+=("$1") + elif [[ "${pos_args[$pos_arg_ct]}" == "SERVICE" ]]; then + services_args=("$1") || exit 1 + stage="remainder" + else + action_posargs+=("$1") + ((pos_arg_ct++)) + fi ;; esac ;; "remainder") remainder_args+=("$@") - break 3;; + break 3 + ;; esac shift done -[ "${compose_opts[*]}" ] && debug "Main docker-compose opts: ${compose_opts[*]}" -[ "${action_opts[*]}" ] && debug "Action $DARKCYAN$action$NORMAL with opts: ${action_opts[*]}" -[ "${remainder_args[*]}" ] && debug "Remainder args: ${remainder_args[*]}" + +[ "${services[*]}" ] && debug " ${DARKWHITE}Services:$NORMAL ${services[*]}" +[ "${compose_opts[*]}" ] && debug " ${DARKWHITE}Main docker-compose opts:$NORMAL ${compose_opts[*]}" +[ "${action_posargs[*]}" ] && debug " ${DARKWHITE}Main docker-compose pos args:$NORMAL ${action_posargs[*]}" +[ "${action_opts[*]}" ] && debug " ${DARKWHITE}Action $DARKCYAN$action$NORMAL with opts:$NORMAL ${action_opts[*]}" +[ "${remainder_args[*]}" ] && debug " ${DARKWHITE}Remainder args:$NORMAL ${remainder_args[*]}" ## @@ -2496,41 +2651,40 @@ charm.sanity_checks || die "Sanity checks about charm-store failed. Please corre ## Get services in command line. ## -is_service_action= -case "$action" in - up|build|start|stop|config|graph) - services="$(get_default_target_services "${action_posargs[@]}" "${remainder_args[@]}")" || exit 1 - orig_services="${action_posargs[@]:1}" - ;; - run) - services="${action_posargs[0]}" - ;; - ""|down) - services= - ;; - *) - if is_service_action=$(has_service_action "${action_posargs[0]}" "$action"); then - { - read-0 action_type - case "$action_type" in - "relation") - read-0 _ target_service _target_charm relation_name - debug "Found action $DARKYELLOW${action_posargs[0]}$NORMAL/$DARKBLUE$relation_name$NORMAL/$DARKCYAN$action$NORMAL (in $DARKYELLOW$target_service$NORMAL)" - ;; - "direct") - debug "Found action $DARKYELLOW${action_posargs[0]}$NORMAL.$DARKCYAN$action$NORMAL" - ;; - esac - } < <(has_service_action "${action_posargs[0]}" "$action") - services="${action_posargs[0]}" - else - services="$(get_default_target_services "${action_posargs[@]}")" - fi - ;; -esac +if [ -z "$is_docker_compose_action" -a "$action" ]; then + + if is_service_action=$(has_service_action "${action_posargs[0]}" "$action"); then + { + read-0 action_type + case "$action_type" in + "relation") + read-0 _ target_service _target_charm relation_name + debug "Found action $DARKYELLOW${action_posargs[0]}$NORMAL/$DARKBLUE$relation_name$NORMAL/$DARKCYAN$action$NORMAL (in $DARKYELLOW$target_service$NORMAL)" + ;; + "direct") + debug "Found action $DARKYELLOW${action_posargs[0]}$NORMAL.$DARKCYAN$action$NORMAL" + ;; + esac + } < <(has_service_action "${action_posargs[0]}" "$action") + services=("${action_posargs[0]}") + else + die "Unknown command: It doesn't match any docker-compose commands nor inner charm actions." + fi +else + if [ "$action" == "up" -a "${#services_args[@]}" == 0 ]; then + services_args=($(shyaml keys <"$COMPOSE_YML_FILE")) + fi + if [ "$action" == "config" ]; then + services_args=("${action_posargs[@]}") + fi + if [ "$is_docker_compose_action" -a "${#services_args[@]}" -gt 0 ]; then + services=($(get_master_services "${services_args[@]}")) || exit 1 + action_posargs+=("${services[@]}") + fi +fi -get_docker_compose $services >/dev/null || { ## precalculate variable \$_current_docker_compose +get_docker_compose "${services_args[@]}" >/dev/null || { ## precalculate variable \$_current_docker_compose err "Fails to compile base 'docker-compose.yml'" exit 1 } @@ -2546,7 +2700,7 @@ case "$action" in full_init=true post_hook=true ;; - ""|down) + ""|down|restart|logs|config) full_init= ;; *) @@ -2572,24 +2726,22 @@ if [ "$full_init" ]; then fi -export SERVICE_PACK="$services" + +export SERVICE_PACK="${services_args[*]}" ## ## Docker-compose ## case "$action" in - up|start|stop|build) - master_services=$(get_master_services $SERVICE_PACK) || exit 1 + up|start|stop|build|run) if [[ "$action" == "up" ]] && ! array_member action_opts -d; then ## force daemon mode for up action_opts=("-d" "${action_opts[@]}") fi - - launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" $master_services + launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}" ;; - run) - master_service=$(get_master_services $SERVICE_PACK) || exit 1 - launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "$master_service" "${remainder_args[@]}" + "") + launch_docker_compose "${compose_opts[@]}" ;; # enter) # master_service=$(get_master_services $SERVICE_PACK) || exit 1 @@ -2601,23 +2753,27 @@ case "$action" in ;; config) ## removing the services + services=($(get_master_services "${action_posargs[@]}")) || exit 1 launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" warn "Runtime configuration modification (from relations) are not included here." ;; down) - remainder_args+=("--remove-orphans") + if ! array_member action_opts --remove-orphans; then ## force daemon mode for up + debug "Adding a default argument of '--remove-orphans'" + action_opts+=("--remove-orphans") + fi launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" ;; *) if [ "$is_service_action" ]; then run_service_action "$SERVICE_PACK" "$action" "${remainder_args[@]}" else - launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" + launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}" fi ;; esac -if [ "$post_hook" ]; then - run_service_hook "$services" post_deploy || exit 1 +if [ "$post_hook" -a "${#services[@]}" != 0 ]; then + run_service_hook "${services[@]}" post_deploy || exit 1 fi diff --git a/test/test b/test/test index 85e4bd2..a7b9fe7 100755 --- a/test/test +++ b/test/test @@ -1,8 +1,5 @@ -#!/bin/bash - -#!- Library include -. /etc/shlib -#!- +#!/usr/bin/env bash-shlib +# -*- mode: shell-script -*- include shunit @@ -19,18 +16,22 @@ tprog=$(readlink -f $tprog) export PATH=".:$PATH" short_tprog=$(basename "$tprog") + ## ## Convenience function ## -function init_test() { +init_test() { test_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX) cd "$test_tmpdir" + export CACHEDIR="$test_tmpdir/.cache" + export VARDIR="$test_tmpdir/.var" + mkdir -p "$CACHEDIR" } -function tear_test() { +tear_test() { rm -rf "$test_tmpdir" } @@ -41,7 +42,7 @@ function tear_test() { ## # Checking arguments -function test_calling_sourcing { +test_calling_sourcing() { assert_list < $test_tmpdir/testcharm/metadata.yml +# XXX new content to invalidate cache EOF2 . "$tprog" _setup_state_dir -out="\$(get_docker_compose_mixin_from_metadata testcharm)" || { - echo "Failed" - exit 1 +out="\$(get_docker_compose_mixin_from_metadata testcharm)" || exit 1 +expected="\ +build: $test_tmpdir/testcharm/build +labels: +- compose.charm=testcharm" + +[ "\$out" == "\$expected" ] || { + echo -e "DIFF:\n\$(diff <(echo "\$out") <(echo "\$expected"))" + exit 1 } -echo "\$out" -test "\$out" == "build: testcharm/build" - ## -- subordinate with image @@ -413,8 +448,7 @@ EOF } - -function test_get_compose_service_def { +test_get_compose_service_def() { init_test @@ -426,11 +460,17 @@ function test_get_compose_service_def { export CHARM_STORE=$test_tmpdir mkdir $test_tmpdir/www +touch $test_tmpdir/www/metadata.yml . "$tprog" _setup_state_dir -test "\$(get_compose_service_def www)" == "charm: www" +out="\$(get_compose_service_def www)" +test "\$out" == "charm: www" || { +echo OUTPUT: +echo "\$out" +false +} ## -- Simple (no docker-compose, no charm dir) @@ -488,13 +528,13 @@ EOF ## ## ## -function test_get_master_charm_for_service() { +function test_get_master_service_for_service() { init_test assert_list < $test_tmpdir/www/metadata.yml subordinate: true requires: @@ -534,7 +575,7 @@ EOF2 _setup_state_dir COMPOSE_YML_FILE=$test_tmpdir/compose.yml -test "\$(_get_master_charm_for_service www)" == "mysql" +test "\$(get_master_service_for_service www)" == "mysql" EOF } @@ -567,11 +608,11 @@ EOF2 . "$tprog" _setup_state_dir -out=\$(_get_docker_compose_service_mixin www) -test "\$out" == "www: - volumes: - - /www/tmp/a:/tmp/a:rw - - /www/tmp/b:/tmp/b:rw" || { + +out=\$(_get_docker_compose_service_mixin www | shyaml get-value www.volumes) +[[ "\$out" == "\ +- /www/tmp/a:/tmp/a:rw +- /www/tmp/b:/tmp/b:rw" ]] || { echo -e "** _get_docker_compose_service_mixin www:\n\$out"; exit 1 } @@ -587,6 +628,8 @@ config-resources: - /tmp/b EOF2 +touch $test_tmpdir/mysql/metadata.yml + cat < $test_tmpdir/compose.yml www: charm: www @@ -600,12 +643,22 @@ EOF2 _setup_state_dir COMPOSE_YML_FILE=$test_tmpdir/compose.yml -test "\$(_get_docker_compose_service_mixin www)" == "www: + +out="\$(_get_docker_compose_service_mixin www)" || exit 1 +[ "\$out" == "www: + labels: + - compose.service=www + - compose.master-service=www + - compose.project=\$(basename "$test_tmpdir") + - compose.charm=www links: - mysql volumes: - /www/tmp/a:/tmp/a:rw - - /www/tmp/b:/tmp/b:rw" + - /www/tmp/b:/tmp/b:rw" ] || { + echo -e "OUT:\n\$out" + exit 1 +} ## -- compose, subordinate @@ -636,10 +689,20 @@ EOF2 _setup_state_dir COMPOSE_YML_FILE=$test_tmpdir/compose.yml -test "\$(_get_docker_compose_service_mixin www)" == "mysql: +out="\$(_get_docker_compose_service_mixin www)" || exit 1 +expected="mysql: + labels: + - compose.service=www + - compose.master-service=mysql + - compose.project=$(basename "$test_tmpdir") + - compose.charm=www volumes: - /www/tmp/a:/tmp/a:rw - /www/tmp/b:/tmp/b:rw" +[ "\$out" == "\$expected" ] || { + echo -e "DIFF:\n\$(diff <(echo "\$out") <(echo "\$expected"))" + exit 1 +} EOF } @@ -669,10 +732,13 @@ _setup_state_dir out=\$(get_docker_compose www) echo "OUT:" echo "\$out" -test "\$out" == "www: - volumes: - - /www/tmp/a:/tmp/a:rw - - /www/tmp/b:/tmp/b:rw" + +out=\$(echo "\$out" | shyaml get-value services.www.volumes) +echo "OUT volumes:" +echo "\$out" +test "\$out" == "\\ +- /www/tmp/a:/tmp/a:rw +- /www/tmp/b:/tmp/b:rw" ## -- simple with docker-compose @@ -709,36 +775,44 @@ COMPOSE_YML_FILE=$test_tmpdir/compose.yml _setup_state_dir -out=\$(get_docker_compose www) -test "\$out" == "www: - volumes: - - /www/tmp/a:/tmp/a:rw - - /www/tmp/b:/tmp/b:rw" || { +out=\$(get_docker_compose www | shyaml get-value services.www.volumes) +test "\$out" == "\\ +- /www/tmp/a:/tmp/a:rw +- /www/tmp/b:/tmp/b:rw" || { echo -e "** get_docker_compose www:\n\$out" exit 1 } -out=\$(_get_docker_compose_links web_site) -test "\$out" == "www: - links: - - mysql" || { - echo -e "** _get_docker_compose_links web_site:\n\$out" +out=\$(get_docker_compose_links web_site | shyaml get-value web_site.links) +test "\$out" == "- mysql" || { + echo -e "** get_docker_compose_links web_site:\n\$out" exit 1 } -out=\$(get_docker_compose web_site) -test "\$out" == "\ +out=\$(get_docker_compose web_site | shyaml get-value services) +expected="\ mysql: + labels: + - compose.service=mysql + - compose.master-service=mysql + - compose.project=$(basename "$test_tmpdir") + - compose.charm=mysql volumes: - /mysql/tmp/c:/tmp/c:rw - /mysql/tmp/d:/tmp/d:rw -www: +web_site: + labels: + - compose.service=web_site + - compose.master-service=web_site + - compose.project=$(basename "$test_tmpdir") + - compose.charm=www links: - mysql volumes: - - /www/tmp/a:/tmp/a:rw - - /www/tmp/b:/tmp/b:rw" || { - echo -e "** get_docker_compose web_site:\n\$out" + - /web_site/tmp/a:/tmp/a:rw + - /web_site/tmp/b:/tmp/b:rw" +test "\$out" == "\$expected" || { + echo -e "** get_docker_compose web_site:\n\$(diff <(echo "\$out") <(echo "\$expected"))" exit 1 } @@ -786,12 +860,16 @@ _setup_state_dir ! get_docker_compose www || exit 1 # volumes gets mixed -test "\$(get_docker_compose web_site)" == "mysql: - volumes: - - /www/tmp/a:/tmp/a:rw - - /www/tmp/b:/tmp/b:rw - - /mysql/tmp/c:/tmp/c:rw - - /mysql/tmp/d:/tmp/d:rw" +out="\$(get_docker_compose web_site | shyaml get-value services.mysql.volumes)" + +test "\$out" == "\ +- /web_site/tmp/a:/tmp/a:rw +- /web_site/tmp/b:/tmp/b:rw +- /mysql/tmp/c:/tmp/c:rw +- /mysql/tmp/d:/tmp/d:rw" || { + echo -e "OUT:\n\$out" + exit 1 +} ## -- subordinate with complex features @@ -844,15 +922,30 @@ _setup_state_dir #! get_docker_compose www || exit 1 # volumes gets mixed -test "\$(get_docker_compose web_site)" == "mysql: - entrypoint: custom-entrypoint - volumes: - - /www/tmp/a:/tmp/a:rw - - /www/tmp/b:/tmp/b:rw - - /special-volume-from-www:/special-volume-from-www - - /mysql/tmp/c:/tmp/c:rw - - /mysql/tmp/d:/tmp/d:rw - - /special-volume-from-mysql:/special-volume-from-mysql" +out="\$(get_docker_compose web_site | shyaml get-value services.mysql)" +expected="\ +entrypoint: custom-entrypoint +labels: +- compose.service=web_site +- compose.charm=www +- compose.service=mysql +- compose.master-service=mysql +- compose.project=$(basename "$test_tmpdir") +- compose.charm=mysql +volumes: +- /web_site/tmp/a:/tmp/a:rw +- /web_site/tmp/b:/tmp/b:rw +- /special-volume-from-www:/special-volume-from-www +- /mysql/tmp/c:/tmp/c:rw +- /mysql/tmp/d:/tmp/d:rw +- /special-volume-from-mysql:/special-volume-from-mysql" + +test "\$out" == "\$expected" || { + echo -e "DIFF:\n\$(diff <(echo "\$out") <(echo "\$expected"))" + exit 1 +} + + EOF tear_test @@ -1032,12 +1125,12 @@ EOF2 COMPOSE_YML_FILE=$test_tmpdir/compose.yml _setup_state_dir -out=\$(_get_docker_compose_links "app") +out=\$(get_docker_compose_links "app") test "\$out" == "app: links: - www - mysql" || { - echo -e "** _get_docker_compose_links:\n\$out"; exit 1 + echo -e "** get_docker_compose_links:\n\$out"; exit 1 } @@ -1048,7 +1141,7 @@ mkdir -p $test_tmpdir/{www,mysql} cat < $test_tmpdir/www/metadata.yml provides: web-proxy: - reverse-tech-dep: true + tech-dep: reversed EOF2 touch $test_tmpdir/mysql/metadata.yml @@ -1066,26 +1159,25 @@ COMPOSE_YML_FILE=$test_tmpdir/compose.yml _setup_state_dir out=\$(get_charm_relation_def "www" "web-proxy") || exit 1 -test "\$out" == "reverse-tech-dep: true" || { +test "\$out" == "tech-dep: reversed" || { echo -e "** get_charm_relation_def:\n\$out"; exit 1 } -out=\$(get_charm_reverse_tech_dep_relation "www" "web-proxy") -test "\$out" == "True" || { - echo -e "** get_charm_reverse_tech_dep_relation:\n\$out"; exit 1 +out=\$(get_charm_tech_dep_orientation_for_relation "www" "web-proxy") +test "\$out" == "reversed" || { + echo -e "** get_charm_tech_dep_orientation_for_relation:\n\$out"; exit 1 } -out=\$(_get_docker_compose_links "web_site") -test "\$out" == "www: +out=\$(get_docker_compose_links "web_site") +expected="www: links: - - mysql" || { - echo -e "** _get_docker_compose_links:\n\$out"; exit 1 + - web_site" +test "\$out" == "\$expected" || { + echo -e "** get_docker_compose_links:\n\$out\nExpected:\n\$expected"; exit 1 } -out=\$(get_docker_compose web_site) -test "\$out" == "www: - links: - - mysql" || { +out=\$(get_docker_compose web_site | shyaml get-value services.www.links) +test "\$out" == "- web_site" || { echo -e "** get_docker_compose:\n\$out"; exit 1 } @@ -1144,23 +1236,23 @@ EOF2 assert_list <