From 5734efecf87de637350d4e9ddc22b42c1cca9f13 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Sun, 21 Oct 2018 01:05:51 +0200 Subject: [PATCH] more big fat changes --- bin/compose | 296 ++++++++++++++++++++++++++++++++++++++-------------- test/test | 215 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 432 insertions(+), 79 deletions(-) diff --git a/bin/compose b/bin/compose index 083c7e4..c1d11b2 100755 --- a/bin/compose +++ b/bin/compose @@ -43,6 +43,7 @@ include pretty include parse include charm include array +include cla depends shyaml docker @@ -73,13 +74,6 @@ In compose message, color coding is enforced as such: $WHITE$exname$NORMAL reads '/etc/compose.conf' for global variables, and '/etc/compose.local.conf' for local host adjustements. -${WHITE}$exname Options${NORMAL}: - -h, --help Print this message and quit - (ignoring any other options) - -V, --version Print current version and quit - (ignoring any other options) - -v, --verbose Be more verbose - -d, --debug Print full debugging information (sets also verbose) " @@ -330,7 +324,7 @@ fn.exists() { declare -F "$1" >/dev/null } -str_matches() { +str_pattern_matches() { local str="$1" shift for pattern in "$@"; do @@ -339,6 +333,16 @@ str_matches() { return 1 } + +str_matches() { + local str="$1" + shift + for pattern in "$@"; do + [[ "$str" == "$pattern" ]] && return 0 + done + return 1 +} + gen_password() { local l=( {a..z} {A..Z} {0..9} ) nl="${#l[@]}" size=${1:-16} while ((size--)); do @@ -1061,7 +1065,7 @@ service_base_docker_image() { return 1 fi - grep '^FROM' "$service_dockerfile" | xargs echo | cut -f 2 -d " " + grep '^FROM' "$service_dockerfile" | xargs printf "%s " | cut -f 2 -d " " else echo "$service_image" fi | tee "$cache_file" @@ -1195,10 +1199,10 @@ get_ordered_service_dependencies() { export -f get_ordered_service_dependencies run_service_hook () { - local services="$1" action="$2" subservices loaded - + local action="$1" service subservice subservices loaded + shift declare -A loaded - for service in $services; do + for service in "$@"; do subservices=$(get_ordered_service_dependencies "$service") || return 1 for subservice in $subservices; do if [ "${loaded[$subservice]}" ]; then @@ -1285,7 +1289,7 @@ export -f host_resource_get_git-sub setup_host_resource () { - local service="$1" service_def location get cfg + local subservice="$1" service_def location get cfg service_def=$(get_compose_service_def "$subservice") || return 1 while read-0 location cfg; do @@ -1312,10 +1316,10 @@ export -f setup_host_resource setup_host_resources () { - local services="$1" subservices loaded + local service subservices subservice loaded declare -A loaded - for service in $services; do + for service in "$@"; do subservices=$(get_ordered_service_dependencies "$service") || return 1 for subservice in $subservices; do if [ "${loaded[$subservice]}" ]; then @@ -1607,14 +1611,14 @@ export -f get_compose_relation_def run_service_relations () { - local services="$1" loaded subservices + local service services loaded subservices subservice PROJECT_NAME=$(get_default_project_name) || return 1 export PROJECT_NAME declare -A loaded - subservices=$(get_ordered_service_dependencies $services) || return 1 + subservices=$(get_ordered_service_dependencies "$@") || return 1 for service in $subservices; do # debug "Upping dep's relations of ${DARKYELLOW}$service${NORMAL}:" @@ -2217,7 +2221,7 @@ get_master_services() { fi echo "$master_service" loaded["$master_service"]=1 - done | xargs echo + done | xargs printf "%s " return "${PIPESTATUS[0]}" } export -f get_master_services @@ -2232,71 +2236,168 @@ _setup_state_dir() { } -get_docker_compose_action_help_msg() { - local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$action"; cat "$(which docker-compose)" | md5_compat)" \ +get_docker_compose_help_msg() { + local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \ docker_compose_help_msg if [ -e "$cache_file" ]; then cat "$cache_file" && touch "$cache_file" || return 1 return 0 fi - docker_compose_help_msg=$(docker-compose "$action" --help 2>/dev/null) || return 1 + docker_compose_help_msg=$(docker-compose $action --help 2>/dev/null) || return 1 echo "$docker_compose_help_msg" | tee "$cache_file" || return 1 } -get_docker_compose_action_usage() { - local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$action"; cat "$(which docker-compose)" | md5_compat)" \ +get_docker_compose_usage() { + local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \ docker_compose_help_msg if [ -e "$cache_file" ]; then cat "$cache_file" && touch "$cache_file" || return 1 return 0 fi - docker_compose_help_msg=$(get_docker_compose_action_help_msg "$action") || return 1 + docker_compose_help_msg=$(get_docker_compose_help_msg $action) || return 1 echo "$docker_compose_help_msg" | grep -m 1 "^Usage:" -A 10000 | egrep -m 1 "^\$" -B 10000 | - xargs echo | + xargs printf "%s " | sed -r 's/^Usage: //g' | tee "$cache_file" || return 1 } -get_docker_compose_opts_list() { - local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$action"; cat "$(which docker-compose)" | md5_compat)" \ + +get_docker_compose_opts_help() { + local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \ docker_compose_help_msg if [ -e "$cache_file" ]; then cat "$cache_file" && touch "$cache_file" || return 1 return 0 fi - docker_compose_help_msg=$(get_docker_compose_action_help_msg "$action") || return 1 - echo "$docker_compose_help_msg" | grep '^Options:' -A 20000 | + docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1 + echo "$docker_compose_opts_help" | + grep '^Options:' -A 20000 | tail -n +2 | + { cat ; echo; } | + egrep -m 1 "^\S*\$" -B 10000 | + head -n -1 | + tee "$cache_file" || return 1 +} + +get_docker_compose_commands_help() { + local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \ + docker_compose_help_msg + if [ -e "$cache_file" ]; then + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1 + echo "$docker_compose_opts_help" | + grep '^Commands:' -A 20000 | + tail -n +2 | + { cat ; echo; } | + egrep -m 1 "^\S*\$" -B 10000 | + head -n -1 | + tee "$cache_file" || return 1 +} + + +get_docker_compose_opts_list() { + local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \ + docker_compose_help_msg + if [ -e "$cache_file" ]; then + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + docker_compose_opts_help=$(get_docker_compose_opts_help $action) || return 1 + echo "$docker_compose_opts_help" | egrep "^\s+-" | sed -r 's/\s+((((-[a-zA-Z]|--[a-zA-Z0-9-]+)( [A-Z=]+|=[^ ]+)?)(, )?)+)\s+.*$/\1/g' | tee "$cache_file" || return 1 } + +options_parser() { + sed -r 's/^(\s+(((-[a-zA-Z]|--[a-zA-Z0-9-]+)([ =]([a-zA-Z_=\"\[]|\])+)?(, | )?)+)\s+)[^ ].*$/\x0\2\x0\0/g' + printf "\0" +} + + +remove_options_in_option_help_msg() { + { + read-0 null + if [ "$null" ]; then + err "options parsing error, should start with an option line." + return 1 + fi + while read-0 opt full_txt;do + multi_opts="$(printf "%s " $opt | multi_opts_filter)" + single_opts="$(printf "%s " $opt | single_opts_filter)" + for to_remove in "$@"; do + str_matches "$to_remove" $multi_opts $single_opts && { + continue 2 + } + done + echo -n "$full_txt" + done + } < <(options_parser) +} + + + _MULTIOPTION_REGEX='^((-[a-zA-Z]|--[a-zA-Z0-9-]+)(, )?)+' _MULTIOPTION_REGEX_LINE_FILTER=$_MULTIOPTION_REGEX'(\s|=)' +multi_opts_filter() { + egrep "$_MULTIOPTION_REGEX_LINE_FILTER" | + sed -r "s/^($_MULTIOPTION_REGEX)(\s|=).*$/\1/g" | + tr ',' "\n" | xargs printf "%s " +} + + +single_opts_filter() { + egrep -v "$_MULTIOPTION_REGEX_LINE_FILTER" | + tr ',' "\n" | xargs printf "%s " +} + + get_docker_compose_multi_opts_list() { local action="$1" opts_list opts_list=$(get_docker_compose_opts_list "$action") || return 1 - echo "$opts_list" | egrep "$_MULTIOPTION_REGEX_LINE_FILTER" | - sed -r "s/^($_MULTIOPTION_REGEX)(\s|=).*$/\1/g" | - tr ',' "\n" | xargs echo + echo "$opts_list" | multi_opts_filter } get_docker_compose_single_opts_list() { local action="$1" opts_list opts_list=$(get_docker_compose_opts_list "$action") || return 1 - echo "$opts_list" | egrep -v "$_MULTIOPTION_REGEX_LINE_FILTER" | - tr ',' "\n" | xargs echo + echo "$opts_list" | single_opts_filter +} + + +display_help() { + print_help + echo "${WHITE}Options${NORMAL}:" + echo " -h, --help Print this message and quit" + echo " (ignoring any other options)" + echo " -V, --version Print current version and quit" + echo " (ignoring any other options)" + echo " --dirs Display data dirs and quit" + echo " (ignoring any other options)" + echo " -v, --verbose Be more verbose" + echo " -d, --debug Print full debugging information (sets also verbose)" + echo " --dry-compose-run If docker-compose will be run, only print out what" + echo " command line will be used." + + get_docker_compose_opts_help | remove_options_in_option_help_msg --version --help --verbose | + filter_docker_compose_help_message + echo + echo "${WHITE}Commands${NORMAL} (thanks to docker-compose):" + get_docker_compose_commands_help | sed -r "s/ ([a-z]+)(\s+)/ ${DARKCYAN}\1${NORMAL}\2/g" } - _graph_service() { local service="$1" base="$1" @@ -2340,9 +2441,10 @@ _graph_service() { } + _graph_node_service() { local service="$1" base="$2" charm="$3" - + cat <&2 + err "Unknown option '$arg'. Please check '${DARKCYAN}$action${NORMAL}' help:" + docker-compose "$action" --help | + filter_docker_compose_help_message >&2 exit 1 fi fi @@ -2591,33 +2723,37 @@ while [ "$#" != 0 ]; do *) # echo "LOOP $1 : pos_arg: $pos_arg_ct // ${pos_args[$pos_arg_ct]}" if [[ "${pos_args[$pos_arg_ct]}" == "[SERVICE...]" ]]; then - services_args+=("$1") + services_args+=("$arg") elif [[ "${pos_args[$pos_arg_ct]}" == "SERVICE" ]]; then - services_args=("$1") || exit 1 + services_args=("$arg") || exit 1 stage="remainder" else - action_posargs+=("$1") + action_posargs+=("$arg") ((pos_arg_ct++)) fi ;; esac ;; "remainder") - remainder_args+=("$@") + remainder_args+=("$arg") + while read-0 arg; do + remainder_args+=("$arg") + done break 3 ;; esac shift -done +done < <(cla.normalize "$@") -[ "${services[*]}" ] && debug " ${DARKWHITE}Services:$NORMAL ${services[*]}" +[ "${services_args[*]}" ] && debug " ${DARKWHITE}Services:$NORMAL ${DARKYELLOW}${services_args[*]}$NORMAL" [ "${compose_opts[*]}" ] && debug " ${DARKWHITE}Main docker-compose opts:$NORMAL ${compose_opts[*]}" [ "${action_posargs[*]}" ] && debug " ${DARKWHITE}Main docker-compose pos args:$NORMAL ${action_posargs[*]}" [ "${action_opts[*]}" ] && debug " ${DARKWHITE}Action $DARKCYAN$action$NORMAL with opts:$NORMAL ${action_opts[*]}" [ "${remainder_args[*]}" ] && debug " ${DARKWHITE}Remainder args:$NORMAL ${remainder_args[*]}" + ## ## Actual code ## @@ -2652,26 +2788,27 @@ charm.sanity_checks || die "Sanity checks about charm-store failed. Please corre ## if [ -z "$is_docker_compose_action" -a "$action" ]; then - - if is_service_action=$(has_service_action "${action_posargs[0]}" "$action"); then + action_service=${remainder_args[0]} + remainder_args=("${remainder_args[@]:1}") + if is_service_action=$(has_service_action "$action_service" "$action"); then { read-0 action_type case "$action_type" in "relation") read-0 _ target_service _target_charm relation_name - debug "Found action $DARKYELLOW${action_posargs[0]}$NORMAL/$DARKBLUE$relation_name$NORMAL/$DARKCYAN$action$NORMAL (in $DARKYELLOW$target_service$NORMAL)" + debug "Found action $DARKYELLOW${action_service}$NORMAL/$DARKBLUE$relation_name$NORMAL/$DARKCYAN$action$NORMAL (in $DARKYELLOW$target_service$NORMAL)" ;; "direct") - debug "Found action $DARKYELLOW${action_posargs[0]}$NORMAL.$DARKCYAN$action$NORMAL" + debug "Found action $DARKYELLOW${action_service}$NORMAL.$DARKCYAN$action$NORMAL" ;; esac - } < <(has_service_action "${action_posargs[0]}" "$action") - services=("${action_posargs[0]}") + } < <(has_service_action "$action_service" "$action") + services_args=("$action_service") else - die "Unknown command: It doesn't match any docker-compose commands nor inner charm actions." + die "Unknown action '${DARKCYAN}$action$NORMAL': It doesn't match any docker-compose commands nor inner charm actions." fi else - if [ "$action" == "up" -a "${#services_args[@]}" == 0 ]; then + if [[ "$action" =~ (up|ps) && "${#services_args[@]}" == 0 ]]; then services_args=($(shyaml keys <"$COMPOSE_YML_FILE")) fi if [ "$action" == "config" ]; then @@ -2700,7 +2837,7 @@ case "$action" in full_init=true post_hook=true ;; - ""|down|restart|logs|config) + ""|down|restart|logs|config|ps) full_init= ;; *) @@ -2710,18 +2847,19 @@ case "$action" in ;; esac + if [ "$full_init" ]; then ## init in order if [ -z "$no_init" ]; then Section setup host resources - setup_host_resources "$services" || exit 1 + setup_host_resources "${services_args[@]}" || exit 1 Section initialisation - run_service_hook "$services" init || exit 1 + run_service_hook init "${services_args[@]}" || exit 1 fi ## Get relations if [ -z "$no_relations" ]; then - run_service_relations "$services" || exit 1 + run_service_relations "${services_args[@]}" || exit 1 fi fi @@ -2736,7 +2874,7 @@ export SERVICE_PACK="${services_args[*]}" case "$action" in up|start|stop|build|run) if [[ "$action" == "up" ]] && ! array_member action_opts -d; then ## force daemon mode for up - action_opts=("-d" "${action_opts[@]}") + action_opts+=("-d" "${action_opts[@]}") fi launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}" ;; @@ -2774,6 +2912,6 @@ case "$action" in esac -if [ "$post_hook" -a "${#services[@]}" != 0 ]; then - run_service_hook "${services[@]}" post_deploy || exit 1 +if [ "$post_hook" -a "${#services_args[@]}" != 0 ]; then + run_service_hook post_deploy "${services_args[@]}" || exit 1 fi diff --git a/test/test b/test/test index a7b9fe7..690e514 100755 --- a/test/test +++ b/test/test @@ -1264,4 +1264,219 @@ EOF tear_test } + +function test_compose_run_args { + + init_test + + export CHARM_STORE=$test_tmpdir + mkdir -p $test_tmpdir/{www,mysql} + cat < $test_tmpdir/www/metadata.yml +EOF2 + + cat < $test_tmpdir/mysql/metadata.yml +EOF2 + + cat < $test_tmpdir/compose.yml +web_site: + charm: www +EOF2 + + export DISABLE_SYSTEM_CONFIG_FILE=true + + assert_list <&1 >/dev/null ) +expected="docker-compose run web_site" + +[ "\$out" == "\$expected" ] || { + echo -e "DIFF:\n\$(diff <(echo "\$out") <(echo "\$expected"))" + exit 1 +} + + +## -- simple single dash arg + +cd "$test_tmpdir" + +out=\$("$tprog" --dry-compose-run run -T web_site 2>&1 >/dev/null ) +expected="docker-compose run -T web_site" + +[ "\$out" == "\$expected" ] || { + echo -e "DIFF:\n\$(diff <(echo "\$out") <(echo "\$expected"))" + exit 1 +} + + +## -- desaggregation of combined single char args + +cd "$test_tmpdir" + +out=\$("$tprog" --dry-compose-run logs -ft web_site 2>&1 >/dev/null) +expected="docker-compose logs -f -t web_site" + +[ "\$out" == "\$expected" ] || { + echo -e "DIFF:\n\$(diff <(echo "\$out") <(echo "\$expected"))" + exit 1 +} + + +## -- desaggregation of combined single char option and valued option char + +cd "$test_tmpdir" + +out=\$("$tprog" --dry-compose-run run -Tv x:y web_site 2>&1 >/dev/null) +expected="docker-compose run -T -v x:y web_site" + +[ "\$out" == "\$expected" ] || { + echo -e "DIFF:\n\$(diff <(echo "\$out") <(echo "\$expected"))" + exit 1 +} + + +## -- simple unexpected single dash arg + +cd "$test_tmpdir" + +out=\$("$tprog" --dry-compose-run run -Z web_site 2>&1) +expected_reg="Unknown option '-Z'" + +[[ "\$out" =~ \$expected_reg ]] || { + echo -e "Can't find '\$expected_reg' in out:\n\$out" + exit 1 +} + +## -- simple unexpected single dash arg after expected one + +cd "$test_tmpdir" + +out=\$("$tprog" --dry-compose-run run -T -Z web_site 2>&1) +expected_reg="Unknown option '-Z'" + +[[ "\$out" =~ \$expected_reg ]] || { + echo -e "Can't find '\$expected_reg' in out:\n\$out" + exit 1 +} + +## -- simple unexpected single dash arg after expected aggregated one + +cd "$test_tmpdir" + +out=\$("$tprog" --dry-compose-run run -TZ web_site 2>&1) +expected_reg="Unknown option '-Z'" + +[[ "\$out" =~ \$expected_reg ]] || { + echo -e "Can't find '\$expected_reg' in out:\n\$out" + exit 1 +} + + +## -- multiple services + +cd "$test_tmpdir" + +out=\$("$tprog" --dry-compose-run logs web_site mysql 2>&1 >/dev/null) +expected="docker-compose logs web_site mysql" + +[ "\$out" == "\$expected" ] || { + echo -e "DIFF:\n\$(diff <(echo "\$out") <(echo "\$expected"))" + exit 1 +} + +## -- single services + +cd "$test_tmpdir" + +out=\$("$tprog" --dry-compose-run run web_site mysql 2>&1 >/dev/null) +expected="docker-compose run web_site mysql" + +[ "\$out" == "\$expected" ] || { + echo -e "DIFF:\n\$(diff <(echo "\$out") <(echo "\$expected"))" + exit 1 +} + + +EOF + + tear_test +} + + +function test_filter_opts { + + src=$(cat <<'EOF' +-d, --detach +--name NAME +--entrypoint CMD +-e KEY=VAL +-l, --label KEY=VAL +-u, --user="" +--no-deps +--rm +-p, --publish=[] +--service-ports +--use-aliases +-v, --volume=[] +-T +-w, --workdir="" +EOF + ) + + export src + + assert_list <