From 6f33dfff3efb0968379857caa5de103283e25bd5 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Fri, 1 Nov 2024 21:52:50 +0100 Subject: [PATCH] new: [compose-core] make ``compose status`` resolve all cell in parallel --- bin/compose-core | 524 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 389 insertions(+), 135 deletions(-) diff --git a/bin/compose-core b/bin/compose-core index 4f09fbb..d8e6189 100755 --- a/bin/compose-core +++ b/bin/compose-core @@ -2737,32 +2737,73 @@ export -f service:state charm:upstream-version() { local charm="$1" version cache_file="$state_tmpdir/$FUNCNAME.cache.$1" path if [ -e "$cache_file" ]; then - cat "$cache_file" - return 0 - fi - if ! path=$(charm.has_direct_action "$charm" "upstream-versions"); then - return 0 + { + read-0 errlvl + cat + } <"$cache_file" + return $errlvl fi - version=$("$path" -l 1) || { - err "Failed to get upstream version for ${DARKYELLOW}$charm${NORMAL}." - return 1 - } - if path=$(charm.has_direct_action "$charm" "upstream-version-normalize"); then - version=$("$path" "$version") || { - err "Failed to normalize upstream version for ${DARKYELLOW}$charm${NORMAL}." + ( + if ! mkdir "$cache_file.lock" 2>/dev/null; then + while true; do + sleep 0.1 + [ -d "${cache_file}.lock" ] || break + done + if [ -e "$cache_file" ]; then + { + read-0 errlvl + if [ "$errlvl" == 0 ]; then + cat + else + cat >&2 + fi + } <"$cache_file" + return $errlvl + fi return 1 - } - fi - echo "$version" > "$cache_file" - e "$version" + fi + trap_add EXIT,ERR "rmdir \"${cache_file}\".lock" + if ! path=$(charm.has_direct_action "$charm" "upstream-versions"); then + touch "$cache_file" + return 0 + fi + rm -f "${cache_file}.wip" + touch "${cache_file}.wip" + ( + version=$("$path" -l 1) + errlvl=$? + if [ "$errlvl" != 0 ]; then + err "Action ${WHITE}upstream-versions${NORMAL} failed for ${DARKPINK}$charm${NORMAL}." + return $errlvl + fi + if path=$(charm.has_direct_action "$charm" "upstream-version-normalize"); then + version=$("$path" "$version") + errlvl=$? + if [ "$errlvl" != 0 ]; then + err "Failed to normalize upstream version for ${DARKPINK}$charm${NORMAL}." + return $errlvl + fi + fi + echo "$version" + ) > "${cache_file}.wip" 2>&1 + errlvl=$? + p0 "$errlvl" > "${cache_file}" + if [ "$errlvl" != 0 ]; then + cat "${cache_file}.wip" | tee -a "${cache_file}" >&2 + rm "${cache_file}.wip" + return $errlvl + fi + cat "${cache_file}.wip" | tee -a "${cache_file}" + rm "${cache_file}.wip" + ) } export -f charm:upstream-version service:upstream-version() { local service="$1" version - charm=$(get_service_charm "$service") || return 1 - version=$(charm:upstream-version "$charm") || return 1 + charm=$(get_service_charm "$service") || return $? + version=$(charm:upstream-version "$charm") || return $? e "$version" } export -f service:upstream-version @@ -5704,14 +5745,92 @@ if [ "${PIPESTATUS[0]}" != 0 ]; then fi [ "$action" == "build" ] && exit 0 +state:fields:resolve-parallel() { + local cols rowsservice jobs state_msg out errlvl col + first_job=1 + tick_pid= + concurrent_jobs=0 + MAX_CONCURRENT_JOBS=$((3 + $(nproc))) + rows=() + cols=() + while [ "$#" -gt 0 ]; do + case "$1" in + --) shift; rows=("$@"); break;; + *) cols+=("$1") ;; + esac + shift + done + for col in "${cols[@]}"; do + for service in "${rows[@]}"; do + if [ "$concurrent_jobs" -ge "$MAX_CONCURRENT_JOBS" ]; then + wait -n # -p job_id ## not supported in this version of bash + ## job list is not accurate, but the number of elt is + ((concurrent_jobs--)) + fi + ( + p0 "$service" "$col" "-1" "" ## started + out=$( + case "${col//_/-}" in + root) + if [[ " ${compose_yml_services[*]} " == *" ${service} "* ]]; then + echo "1" + else + echo "0" + fi + ;; + name) e "$service" ;; + charm) get_service_charm "$service" ;; + state) service:state "$service" ;; + type) get_service_type "$service" ;; + upstream-version) service:upstream-version "$service" ;; + *) + if has_service_action "$service" "get-$col" >/dev/null; then + state_msg=$(run_service_action "$service" "get-$col") || exit 1 + if [[ "$state_msg" == *$'\n'* ]]; then + e "${state_msg%%$'\n'*}" + else + e "${state_msg}" + fi + fi + ;; + esac 2>&1 + ) + errlvl="$?" + p0 "$service" "$col" "$errlvl" "$out" + ) & + jobs=("${jobs[@]}" $!) + ((concurrent_jobs++)) + if [ -n "$first_job" ]; then + ## launch tick + ( + while true; do + sleep 0.1 + p0 "" "" "" "" + done + ) & + tick_pid=$! + first_job= + fi + done + done + wait "${jobs[@]}" + if [ -n "$tick_pid" ]; then + kill "$tick_pid" + fi +} +export -f state:fields:resolve-parallel + if [ "$action" == "status" ]; then + if ! [ -t 1 ]; then + state_raw_output=1 + fi + if [[ -n "${state_all_services}" ]] || [[ "${#state_filters[@]}" -gt 0 ]]; then compose_yml_services=($(compose:yml:root:services)) || exit 1 fi if [[ -n "${state_all_services}" ]]; then state_columns=("root" ${state_columns[@]}) fi - state_columns_raw=() for col in "${state_columns[@]}"; do if [[ "$col" =~ ^[+-] ]]; then @@ -5732,143 +5851,278 @@ if [ "$action" == "status" ]; then esac fi done + declare -A state_columns_idx=() + declare -A filter_idx=() + filter_cols=() + non_filter_cols=("${state_columns_raw[@]}") + for filter in "${state_filters[@]}"; do + IFS="=" read -r key value <<<"$filter" + if [[ " ${non_filter_cols[*]} " == *" $key "* ]]; then + ## remove from non_filter_cols + non_filter_cols=(${non_filter_cols[*]/$key}) + fi + state_columns_idx["$col"]="${#filter_cols[@]}" + filter_cols+=("${key}") + done + tot_nb_cols=$(( ${#non_filter_cols[@]} + ${#filter_cols[@]} )) - while read-0-err E "${state_columns_raw[@]}"; do - values=() - for col in "${state_columns_raw[@]}"; do - color= - value="${!col}" - if [ -z "$state_raw_output" ]; then - read -r -- value_trim <<<"${!col}" - case "${col//_/-}" in - root) - case "$value_trim" in - 0) value=" ";; - 1) value="*";; - esac - ;; - name) color=darkyellow;; - charm) color=darkpink;; - state) - case "$value_trim" in - up) color=green;; - down) color=gray;; - deploying) color=yellow;; - *) color=red;; - esac - ;; - type) - case "$value_trim" in - run-once) color=gray;; - stub) color=gray;; - *) color=darkcyan;; - esac - ;; - *) - if [[ "${value_trim}" == "N/A" ]]; then - color=gray - fi - ;; - esac - color="${color^^}" + ## make services_idx + declare -A services_idx=() + idx=0 + for service in "${services_args[@]}"; do + services_idx["$service"]=$((idx++)) + done + ## make state_columns_idx + idx=0 + for col in "${non_filter_cols[@]}"; do + state_columns_idx["$col"]=$((${#filter_cols[@]} + idx++)) + done + + values=() ## all values + new_service_args=("${services_args[@]}") ## will remove service not satisfying filters + while read-0 service col E out; do + if [[ " ${new_service_args[*]} " != *" $service "* ]]; then + continue + fi + col_index="${state_columns_idx[$col]}" + service_index="${services_idx[$service]}" + values[service_index * tot_nb_cols + col_index]="$out" + ## check if all filter are valuated and satisfied + for filter in "${state_filters[@]}"; do + IFS="=" read -r key value <<<"$filter" + col_index="${state_columns_idx[$key]}" + if [ -z "${values[$((service_index * tot_nb_cols + col_index))]}" ]; then + break fi - if [ -n "$color" ]; then - values+=("${!color}$value${NORMAL}") - else - values+=("$value") + if [ "${values[$((service_index * tot_nb_cols + col_index))]}" != "$value" ]; then + new_service_args=(${new_service_args[*]/"$service"}) + break fi done - first=1 - for value in "${values[@]}"; do - if [ -n "$first" ]; then - first= - else - if [ -n "$state_raw_output_nul" ]; then - printf "\0" + done < <(state:fields:resolve-parallel "${filter_cols[@]}" -- "${services_args[@]}") + + + services_args=("${new_service_args[@]}") + if [ "${#services_args[@]}" == 0 ]; then + warn "No services found matching the filters." >&2 + exit 0 + fi + + spinner_chars="⠋⠙⠸⠴⠤⠦⠇" + spinner_idx=0 + SPINNERGRAY=$'\e[38;5;16;48;5;234m' + SPINNERRUNNING=$'\e[38;5;28;48;5;234m' + first_draw=1 + last_draw= + if [ -z "$state_raw_output" ]; then + echo -en "\e[?25l"; stty -echo 2>/dev/null + trap_add EXIT,ERR "echo -en '\e[?25h'; stty echo 2>/dev/null" + fi + errors=() + declare -A errors_hash_idx=() + error_idx=0 + values_valued=0 + values_total=$(( ${#services_args[@]} * ${#state_columns_raw[@]} )) + values_threshold=$(( values_total / 2 )) + while read-0 service col E out; do + if [ -n "$service" ]; then + col_index="${state_columns_idx[$col]}" + service_index="${services_idx[$service]}" + if [[ "$E" -gt 0 ]]; then + error_hash=$(H "$col" "$E" "$out") + matching_error_idx="${errors_hash_idx[$error_hash]}" + if [[ -z "${matching_error_idx}" ]]; then + errors+=("$error_idx:$service:$col:$E:$out") + out="!Err[$((error_idx))]" + errors_hash_idx["$error_hash"]="$error_idx" + error_idx=$((error_idx + 1)) else - printf " " + ## find the error to add the service + error="${errors[$matching_error_idx]}" + error="${error#*:}" + error_service="${error%%:*}" + error_tail="${error#*:}" + errors[matching_error_idx]="$matching_error_idx:$error_service,$service:$error_tail" + out="!Err[$((matching_error_idx))]" fi + elif [[ "$E" == -1 ]]; then + values[service_index * tot_nb_cols + col_index]=$'\t' + continue fi - printf "%s" "$value" - done - if [ -n "$state_raw_output_nul" ]; then - printf "\0" + values[service_index * tot_nb_cols + col_index]="$out" + values_valued=$((values_valued + 1)) + if [[ "$values_valued" == "$values_total" ]]; then + last_draw=1 + else + continue + fi + fi + [ -n "$state_raw_output" ] && continue + [[ $((values_valued)) -lt $((values_threshold)) ]] && continue + + ## Draw table + if [ -n "$first_draw" ]; then + first_draw= + full_table="" else - printf "\n" + ## move up one line per service + full_table=$'\e'"[${#services_args[@]}A" fi - done < <( - set -o pipefail - filter_cols=() - for filter in "${state_filters[@]}"; do - IFS="=" read -r key value <<<"$filter" - ## if not already in state_columns_raw - [[ " ${state_columns_raw[*]} " == *" $key "* ]] || - filter_cols+=("${key//-/_}") - done - for service in "${services_args[@]}"; do - declare -A values=() - for col in "${state_columns_raw[@]}" "${filter_cols[@]}"; do - case "${col//_/-}" in - root) - if [[ " ${compose_yml_services[*]} " == *" ${service} "* ]]; then - value="1" - else - value="0" - fi - ;; - name) value="$service" ;; - charm) - value=$(get_service_charm "$service") || { echo 1; exit 1; } - ;; - state) - value=$(service:state "$service") || { echo 1; exit 1; } - ;; - type) - value=$(get_service_type "$service") || { echo 1; exit 1; } - ;; - upstream-version) - value=$(service:upstream-version "$service") || { echo 1; exit 1; } - value=${value:-N/A} - ;; + spinner_idx=$(( (spinner_idx + 1) % ${#spinner_chars} )) + + while read-0-err E "${state_columns_raw[@]}"; do + line_values=() + for col in "${state_columns_raw[@]}"; do + color= + value="${!col}" + read -r -- value_trim <<<"${!col}" + case "${value_trim}" in + "N/A") color=gray ;; + "!Err"*) color=darkred ;; + "⠿") color=spinnergray ;; *) - if has_service_action "$service" "get-$col" >/dev/null; then - state_msg=$(run_service_action "$service" "get-$col") || { echo 1; exit 1 ; } - if [[ "$state_msg" == *$'\n'* ]]; then - value="${state_msg%%$'\n'*}" - ## XXXvlab: For now, these are not used, but we could - ## display them in additional lines (in same "cell") - msgs="${state_msg#*$'\n'}" - else - value=${state_msg} - fi + if [[ "$spinner_chars" == *"$value_trim"* ]]; then + color=spinnerrunning else - value="N/A" + case "${col//_/-}" in + root) + case "$value_trim" in + 0) value=" ";; + 1) value="*";; + esac + ;; + name) color=darkyellow;; + charm) color=darkpink;; + state) + case "$value_trim" in + up) color=green;; + down) color=gray;; + deploying) color=yellow;; + *) color=red;; + esac + ;; + type) + case "$value_trim" in + run-once) color=gray;; + stub) color=gray;; + *) color=darkcyan;; + esac + ;; + *) + ;; + esac fi ;; esac - values["$col"]="$value" + + color="${color^^}" + if [ -n "$color" ]; then + line_values+=("${!color}$value${NORMAL}") + else + line_values+=("$value") + fi done - for filter in "${state_filters[@]}"; do - IFS="=" read -r key value <<<"$filter" - [[ "${values[$key]}" != "$value" ]] && - continue 2 + first=1 + full_line="" + for value in "${line_values[@]}"; do + if [ -n "$first" ]; then + first= + else + full_line+=" " + fi + full_line+="$value" done + full_table+="$full_line"$'\e[K\n' + done < <( + set -o pipefail + for service in "${services_args[@]}"; do + for col in "${state_columns_raw[@]}"; do + col_index="${state_columns_idx[$col]}" + service_index="${services_idx[$service]}" + value_idx="$((service_index * tot_nb_cols + col_index))" + if ! [[ -v "values[value_idx]" ]]; then + p0 " ⠿ " + continue + fi + value="${values[value_idx]}" + if [[ "$value" == $'\t' ]]; then + p0 " ${spinner_chars:$spinner_idx:1} " + elif [ -z "$value" ]; then + p0 "N/A" + else + p0 "$value" + fi + done + done | { + if [ -z "$state_raw_output" ]; then + col-0:normalize:size "${state_columns_align}" + else + cat + fi + } + echo 0 + ) + printf "%s" "$full_table" + if [ "$E" != 0 ]; then + err "Unexpected failure while drawing table" + exit $E + fi + done < <(state:fields:resolve-parallel "${non_filter_cols[@]}" -- "${services_args[@]}") + + for error in "${errors[@]}"; do + echo "" >&2 + idx=${error%%:*}; error=${error#*:} + service=${error%%:*}; error=${error#*:} + col=${error%%:*}; error=${error#*:} + E=${error%%:*}; error=${error#*:} + service_list_str="" + services=(${service//,/ }) + first=1 + for service in "${services[@]}"; do + if [ -n "$first" ]; then + first= + else + service_list_str+=", " + fi + service_list_str+="${DARKYELLOW}$service${NORMAL}" + done + echo "${RED}Error${DARKRED}[$idx]:${NORMAL} while computing" \ + "${WHITE}$col${NORMAL} for $service_list_str" >&2 + echo "$error" | prefix " ${GRAY}|${NORMAL} " >&2 + echo " ${GRAY}..${NORMAL} ${WHITE}Exited${NORMAL} with errorlevel ${DARKRED}$E${NORMAL}" >&2 + done + if [[ "${#errors[@]}" -gt 0 ]]; then + exit 1 + fi + + if [ -n "$state_raw_output" ]; then + + for service in "${services_args[@]}"; do + first=1 for col in "${state_columns_raw[@]}"; do - p0 "${values[$col]}" + col_index="${state_columns_idx[$col]}" + service_index="${services_idx[$service]}" + value_idx="$((service_index * tot_nb_cols + col_index))" + value="${values[$value_idx]}" + if [ -n "$first" ]; then + first= + else + if [ -n "$state_raw_output_nul" ]; then + printf "\0" + else + printf " " + fi + fi + printf "%s" "$value" done - done | { - if [ -z "$state_raw_output" ]; then - col-0:normalize:size "${state_columns_align}" + if [ -n "$state_raw_output_nul" ]; then + printf "\0" else - cat + printf "\n" fi - } - echo 0 - ) - if [ "$E" != 0 ]; then - echo "E: '$E'" >&2 - exit 1 + done fi + exit 0 fi