diff --git a/bin/compose-core b/bin/compose-core index ffec7dc..00434a6 100755 --- a/bin/compose-core +++ b/bin/compose-core @@ -82,6 +82,9 @@ $WHITE$exname$NORMAL reads '/etc/compose.conf' for global variables, and " +time_now() { date +%s.%3N; } +time_elapsed() { echo "scale=3; $2 - $1" | bc; } + ## XXXvlab: this doesn't seem to work when 'compose' is called in ## a hook of a charm. #[[ "${BASH_SOURCE[0]}" == "" ]] && SOURCED=true @@ -1454,7 +1457,6 @@ export -f _get_service_charm_cached get_service_charm () { local service="$1" if [ -z "$service" ]; then - echo ${FUNCNAME[@]} >&2 print_syntax_error "$FUNCNAME: Please specify a service as first argument." return 1 fi @@ -2474,10 +2476,15 @@ export -f get_compose_relations get_all_services() { - local cache_file="$state_tmpdir/$FUNCNAME.cache.$GLOBAL_ALL_RELATIONS_HASH" \ + local services compose_yml_services service + if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then + err-d "Can't access global \$GLOBAL_ALL_RELATIONS_HASH" + return 1 + fi + local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$GLOBAL_ALL_RELATIONS_HASH" "$(declare -f "$FUNCNAME")")" \ s rn ts rc td services service if [ -e "$cache_file" ]; then - #debug "$FUNCNAME: SESSION cache hit $1" + #debug "$FUNCNAME: cache hit $1" cat "$cache_file" return 0 fi @@ -2494,8 +2501,17 @@ get_all_services() { services["$service"]=1 echo "$service" done - done < "$GLOBAL_ALL_RELATIONS" > "$cache_file" + done < "$GLOBAL_ALL_RELATIONS" > "$cache_file.wip" + + compose_yml_services=($(compose:yml:root:services)) || return 1 + for service in "${compose_yml_services[@]}"; do + [ "${services[$service]}" ] && continue + services["$service"]=1 + echo "$service" + done >> "$cache_file.wip" + + mv "$cache_file"{.wip,} || return 1 cat "$cache_file" } export -f get_all_services @@ -2559,28 +2575,25 @@ export -f get_service_relation ## (base_service, relation_config) ## get_service_incoming_relations() { - local service="$1" relation="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH")" \ - s rn ts rc td - if [ -z "$GLOBAL_ALL_RELATIONS" ]; then - err-d "Can't access global \$GLOBAL_ALL_RELATIONS" + if [ -z "$SUBSET_ALL_RELATIONS_HASH" ]; then + err-d "Expected \$SUBSET_ALL_RELATIONS_HASH to be set." return 1 fi + local service="$1" relation="$2" \ + cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$SUBSET_ALL_RELATIONS_HASH")" \ + s rn ts rc td if [ -e "$cache_file" ]; then #debug "$FUNCNAME: SESSION cache hit $1" cat "$cache_file" return 0 fi - if [ -z "$GLOBAL_ALL_RELATIONS" ]; then - err "Can't access global \$GLOBAL_ALL_RELATIONS" - return 1 - fi - while read-0 s rn ts rc _td; do [[ "$ts" == "$service" ]] || continue [[ "$rn" == "$relation" ]] || continue relation_data_file=$(get_relation_data_file "$s" "$ts" "$rn" "$rc") || return 1 - printf "%s\0" "$s" "$(cat "$relation_data_file")" - done < "$GLOBAL_ALL_RELATIONS" > "$cache_file" + printf "%s\0" "$s" "$(cat "$relation_data_file")" || return 1 + debug "Found relation $rn from $s to $ts" >&2 + done < "$SUBSET_ALL_RELATIONS" > "$cache_file" cat "$cache_file" } @@ -2921,14 +2934,170 @@ _out_after_value_from_def() { } -get_all_relations () { - local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$(declare -p without_relations)")" \ - services +get_all_compose_yml_service() { + if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then + COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || { + err "Failed to get compose yml hash" + return 1 + } + fi + local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH" + if [ -e "${cache_file}" ]; then + #debug "$FUNCNAME: cache hit: ${cache_file}" + cat "${cache_file}" + return 0 + fi + compose_yml_content=$(get_compose_yml_content) || return 1 + printf "%s" "${compose_yml_content}" | shyaml keys-0 2>/dev/null > "${cache_file}.wip" || { + err "Failed to get keys of compose content." + return 1 + } + mv "${cache_file}"{.wip,} || return 1 + cat "${cache_file}" +} + + +## Outputs all relations array. +_service:all:relations_cached() { + local services service E + services=($(compose:yml:root:services)) || return 1 + get_all_relations "${services[@]}" || return 1 +} + + +## Outputs all relations array. +service:all:relations() { + if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then + COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || { + err "Failed to get compose yml hash." + return 1 + } + fi + local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH" + if [ -e "${cache_file}" ]; then + # debug "$FUNCNAME: SESSION cache hit $1" + cat "${cache_file}" + return 0 + fi + + _service:all:relations_cached > "${cache_file}.wip" || { + err-d "Failed to compute all relations." + return 1 + } + + mv "${cache_file}"{.wip,} || return 1 + cat "${cache_file}" +} + +_service:all:relations_hash_cached() { + if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then + COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || { + err "Failed to get compose yml hash." + return 1 + } + fi + local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH" \ + hash + if [ -e "${cache_file}" ]; then + # debug "$FUNCNAME: SESSION cache hit $1" + cat "${cache_file}" + return 0 + fi + + service:all:relations > "${cache_file}.pre" || { + err-d "Failed to get all relations." + return 1 + } + { + p0 "$(hash_get < "${cache_file}.pre")" || return 1 + cat "${cache_file}.pre" + rm "${cache_file}.pre" + } > "${cache_file}".wip || return 1 + + mv "${cache_file}"{.wip,} || return 1 + cat "${cache_file}" +} + +## Get all relations from all services in the current compose file. +## Sets GLOBAL_ALL_RELATIONS_HASH and returns all relations array. +service:all:set_relations_hash() { + if [ -n "$GLOBAL_ALL_RELATIONS" ]; then + if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then + err "Can't access global \$GLOBAL_ALL_RELATIONS_HASH" + echo " (despite \$GLOBAL_ALL_RELATIONS being set)" >&2 + return 1 + fi + return 0 + fi + ## sets COMPOSE_YML_CONTENT_HASH + _service:all:relations_hash_cached >/dev/null || return 1 + { + read-0 GLOBAL_ALL_RELATIONS_HASH || return 1 + export GLOBAL_ALL_RELATIONS_HASH + ## transfer to statedir + export GLOBAL_ALL_RELATIONS="$state_tmpdir/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH" + cat > "$GLOBAL_ALL_RELATIONS" + } < <(_service:all:relations_hash_cached) + if [ -z "$GLOBAL_ALL_RELATIONS" ]; then + err "Failed to set \$GLOBAL_ALL_RELATIONS." + return 1 + fi + if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then + err "Failed to set \$GLOBAL_ALL_RELATIONS_HASH." + return 1 + fi +} + +get_subset_relations () { + local service all_services services start + if [ -n "$SUBSET_ALL_RELATIONS" ]; then + return 0 + fi + if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then + err-d "Can't access global \$GLOBAL_ALL_RELATIONS_HASH" + return 1 + fi + cache_hash=$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH" "$(declare -f get_subset_relations)") + local cache_file="$CACHEDIR/$FUNCNAME.cache.$cache_hash" + if [ -e "${cache_file}" ]; then + export SUBSET_ALL_RELATIONS="$cache_file" + hash=$(hash_get < "$cache_file") || return 1 + export SUBSET_ALL_RELATIONS_HASH="$hash" + cat "${cache_file}" + return 0 + fi + ## collect all connected services first + all_services=("$@") + while [ "${#all_services[@]}" != 0 ]; do + array_pop all_services service + #debug " Getting relations for $DARKYELLOW$service$NORMAL" + while read-0 s rn ts rc td; do + [[ "$s" == "$service" ]] || continue + #debug " adding relation $DARKBLUE$rn$NORMAL to $DARKYELLOW$ts$NORMAL" + p0 "$service" "$rn" "$ts" "$rc" "$td" + [ -n "${services[$ts]}" ] || all_services+=("$ts") + done < "$GLOBAL_ALL_RELATIONS" + services["$service"]=1 + done > "$cache_file.wip" + mv "$cache_file"{.wip,} || return 1 + export SUBSET_ALL_RELATIONS="$cache_file" + hash=$(hash_get < "$cache_file") || return 1 + export SUBSET_ALL_RELATIONS_HASH="$hash" + cat "$cache_file" +} + +get_all_relations () { + if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then + COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || return 1 + fi if [ -n "$GLOBAL_ALL_RELATIONS" ]; then cat "$GLOBAL_ALL_RELATIONS" || return 1 return 0 fi + local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$COMPOSE_YML_CONTENT_HASH" "$(declare -p without_relations)")" \ + services all_services service services_uses services_provides \ + changed summon required recommended optional if [ -e "${cache_file}" ]; then #debug "$FUNCNAME: SESSION cache hit $1" @@ -3176,7 +3345,7 @@ get_all_relations () { while read-0 p s rn ts rc td; do if [ -z "$p" ] || [ "$p" == "," ]; then relation_done["$s:$rn"]=1 - # printf " .. %-30s %-30s %-30s\n" "--" "$s" "$rn" >&2 + # printf " .. %-30s %-30s %-30s\n" "$s" "$ts" "$rn" >&2 printf "%s\0" "$s" "$rn" "$ts" "$rc" "$td" >> "${cache_file}.wip.final" had_new_relation=1 else @@ -4080,7 +4249,7 @@ get_compose_yml_content() { COMPOSE_YML_FILE=$(get_compose_yml_location) || return 1 fi if [ -e "$COMPOSE_YML_FILE" ]; then - debug "Found $WHITE$exname$NORMAL YAML file in '$COMPOSE_YML_FILE'." + # debug "Found $WHITE$exname$NORMAL YAML file in '$COMPOSE_YML_FILE'." COMPOSE_YML_CONTENT=$(cat "$COMPOSE_YML_FILE") || { err "Could not read '$COMPOSE_YML_FILE'." return 1 @@ -4111,6 +4280,37 @@ get_compose_yml_content() { } export -f get_compose_yml_content +compose:yml:hash() { + local cache_file="$state_tmpdir/$FUNCNAME.cache" + if [ -e "$cache_file" ]; then + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + + compose_yml_content=$(get_compose_yml_content) || return 1 + compose_yml_hash=$(echo "$compose_yml_content" | hash_get) || return 1 + + e "$compose_yml_hash" | tee "$cache_file" || return 1 +} +export -f compose:yml:hash + + +compose:yml:root:services() { + local cache_file="$state_tmpdir/$FUNCNAME.cache" services compose_yml_content + if [ -e "$cache_file" ]; then + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + + compose_yml_content=$(get_compose_yml_content) || return 1 + services=($(e "$compose_yml_content" | shyaml keys)) || return 1 + + e "${services[*]}" | tee "$cache_file" || return 1 +} +export -f compose:yml:root:services + get_default_target_services() { local services=("$@") @@ -4349,9 +4549,12 @@ display_commands_help() { get_docker_charm_action() { local services service charm relation_name target_service relation_config \ - target_charm - services=($(get_compose_yml_content | yq -r 'keys().[]' 2>/dev/null)) || return 1 - NO_CONSTRAINT_CHECK=1 get_all_relations "${services[@]}" >/dev/null || return 1 + target_charm services + ## XXXvlab: this is for get_service_relations + NO_CONSTRAINT_CHECK=True service:all:set_relations_hash || { + err-d "Failed to set relations hash." + return 1 + } services=($(get_all_services)) || return 1 for service in "${services[@]}"; do printf "%s:\n" "$service" @@ -4375,8 +4578,12 @@ export -f get_docker_charm_action get_docker_charm_action_help() { local services service charm relation_name target_service relation_config \ target_charm - services=($(get_compose_yml_content | shyaml keys 2>/dev/null)) - NO_CONSTRAINT_CHECK=1 get_all_relations "${services[@]}" >/dev/null || return 1 + ## XXXvlab: this is for get_service_relations + NO_CONSTRAINT_CHECK=True service:all:set_relations_hash || { + err-d "Failed to set relations hash." + return 1 + } + services=($(get_all_services)) || return 1 for service in "${services[@]}"; do out=$( charm=$(get_service_charm "$service") || return 1 @@ -4667,6 +4874,7 @@ no_init= action= stage="main" ## switches from 'main', to 'action', 'remainder' is_docker_compose_action= +is_docker_compose_action_multi_service= rebuild_relations_to_service=() color= declare -A without_relations @@ -4806,6 +5014,12 @@ while read-0 arg; do fi pos_args=($(echo "$DC_USAGE" | sed -r 's/\[-[^]]+\] ?//g;s/\[options\] ?//g')) pos_args=("${pos_args[@]:1}") + if [[ "${pos_args[0]}" == "[SERVICE...]" ]]; then + is_docker_compose_action_multi_service=1 + elif [[ "${pos_args[0]}" == "SERVICE" ]]; then + is_docker_compose_action_multi_service=0 + fi + # echo "USAGE: $DC_USAGE" # echo "pos_args: ${pos_args[@]}" # echo "MULTI: $DC_MATCH_MULTI" @@ -4916,8 +5130,8 @@ aexport remainder_args COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1 COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1 -export COMPOSE_YML_FILE COMPOSE_YML_CONTENT - +COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || exit 1 +export COMPOSE_YML_FILE COMPOSE_YML_CONTENT COMPOSE_YML_CONTENT_HASH charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct." @@ -4933,10 +5147,9 @@ if [ -z "$is_docker_compose_action" -a "$action" ]; then exit 1 fi - + services_args=($(compose:yml:root:services)) || return 1 ## Required by has_service_action - array_read-0 services_args < <(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys-0 2>/dev/null) - NO_CONSTRAINT_CHECK=1 get_all_relations "${services_args[@]}" >/dev/null || exit 1 + service:all:set_relations_hash remainder_args=("${remainder_args[@]:1}") if has_service_action "$action_service" "$action" >/dev/null; then @@ -4971,7 +5184,7 @@ else case "$action" in ps|up) if [ "${#services_args[@]}" == 0 ]; then - array_read-0 services_args < <(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys-0 2>/dev/null) + services_args=($(compose:yml:root:services)) || return 1 fi ;; config) @@ -4985,11 +5198,48 @@ NO_CONSTRAINT_CHECK=True case "$action" in up) NO_CONSTRAINT_CHECK= + if [ -n "$DEBUG" ]; then + Elt "solve all relations" + start=$(time_now) + fi + service:all:set_relations_hash || exit 1 + + if [ -n "$DEBUG" ]; then + elapsed="$(time_elapsed $start "$(time_now)")" || exit 1 + print_info "$(printf "%.3fs" "$elapsed")" + Feedback + fi + + all_services=($(get_all_services)) || exit 1 + ## check that services_args is a subset of all_services + for service in "${services_args[@]}"; do + [[ " ${all_services[*]} " == *" $service "* ]] || { + err "Service ${DARKYELLOW}$service${NORMAL} is not defined in the current compose file." + echo " Neither is is a dependency of a service in the compose file." >&2 + echo " These are the services directly or indirectly available from current compose file:" >&2 + for service in "${all_services[@]}"; do + echo " - ${DARKYELLOW}$service${NORMAL}" >&2 + done + exit 1 + } + done ;; esac +if [ -n "$is_docker_compose_action_multi_service" ]; then + if [ -n "$DEBUG" ]; then + Elt "get relation subet" + start=$(time_now) + fi + + get_subset_relations "${services_args[@]}" >/dev/null || exit 1 -get_all_relations "${services_args[@]}" >/dev/null || exit 1 + if [ -n "$DEBUG" ]; then + elapsed="$(time_elapsed $start "$(time_now)")" || exit 1 + print_info "$(printf "%.3fs" "$elapsed")" + Feedback + fi +fi if [ "$is_docker_compose_action" -a "${#services_args[@]}" -gt 0 ]; then services=($(get_master_services "${services_args[@]}")) || exit 1 @@ -5006,8 +5256,10 @@ if [ "$is_docker_compose_action" -a "${#services_args[@]}" -gt 0 ]; then seen[$mservice]=1 action_posargs+=("$mservice") done - else + elif [ "$is_docker_compose_action_multi_service" == "1" ]; then action_posargs+=("${services[@]}") + elif [ "$is_docker_compose_action_multi_service" == "0" ]; then + action_posargs+=("${services[0]}") ## only the first service is the legit one fi ## Get rid of subordinates action_posargs=($(get_master_services "${action_posargs[@]}")) || exit 1