From 9480da50c9d1909c09436d52af75971099e5a170 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Wed, 23 Oct 2024 20:40:52 +0200 Subject: [PATCH] new: [compose-core] move cache files to global cache for increased performance --- bin/compose-core | 200 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 143 insertions(+), 57 deletions(-) diff --git a/bin/compose-core b/bin/compose-core index bab302d..fc888ea 100755 --- a/bin/compose-core +++ b/bin/compose-core @@ -761,7 +761,11 @@ export -f get_service_base_image_dir_uid_gid get_service_type() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" + if [ -z "$CHARM_STORE_HASH" ]; then + err-d "Expected \$CHARM_STORE_HASH to be set." + return 1 + fi + local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH" if [ -z "$service" ]; then print_syntax_error "$FUNCNAME: Please specify a service as first argument." return 1 @@ -1282,11 +1286,22 @@ export -f _get_docker_compose_service_mixin ## @export ## @cache: !system !nofail +stdout get_docker_compose () { - local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + if [ -z "$CHARM_STORE_HASH" ]; then + err-d "Expected \$CHARM_STORE_HASH to be set." + return 1 + fi + if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then + err-d "Expected \$COMPOSE_YML_CONTENT_HASH to be set." + return 1 + fi + local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$CHARM_STORE_HASH" "$COMPOSE_YML_CONTENT_HASH")" \ entries services service start docker_compose_services if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" + # debug "$FUNCNAME: cache hit ($*) $cache_file" + touch "$cache_file" || return 1 + cp "$cache_file"{,.wip} || return 1 + export _CURRENT_DOCKER_COMPOSE="$cache_file.wip" + cat "$cache_file" || return 1 return 0 fi @@ -1339,8 +1354,9 @@ get_docker_compose () { merge_yaml_str "$(yaml_key_val_str "services" "$docker_compose_services")" \ "$base_v2" > "$cache_file" || return 1 - export _CURRENT_DOCKER_COMPOSE="$cache_file" - cat "$_CURRENT_DOCKER_COMPOSE" + cp "$cache_file"{,.wip} || return 1 + export _CURRENT_DOCKER_COMPOSE="$cache_file.wip" + cat "$_CURRENT_DOCKER_COMPOSE" || return 1 debug " ..compilation of base 'docker-compose.yml' done $GRAY(in $((SECONDS - start_compilation))s)$NORMAL" || true # debug " ** ${WHITE}docker-compose.yml${NORMAL}:" # debug "$_current_docker_compose" @@ -1387,7 +1403,11 @@ export -f _get_compose_service_def_cached ## XXXvlab: a lot to be done to cache the results get_compose_service_def () { - local service="$1" docker_compose cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + if [ -z "$COMBINED_HASH" ]; then + err-d "Expected \$COMBINED_HASH to be set." + return 1 + fi + local service="$1" docker_compose cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \ result if [ -e "$cache_file" ]; then #debug "$FUNCNAME: SESSION cache hit" @@ -1398,7 +1418,7 @@ get_compose_service_def () { [ -z "$service" ] && print_syntax_error "Missing service as first argument." docker_compose=$(get_compose_yml_content) || return 1 result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1 - charm=$(echo "$result" | shyaml get-value charm 2>/dev/null) || return 1 + charm=$(e "$result" | shyaml get-value charm 2>/dev/null) || return 1 metadata=$(charm.metadata "$charm") || return 1 if default_options=$(printf "%s" "$metadata" | shyaml -y -q get-value default-options); then default_options=$(yaml_key_val_str "options" "$default_options") || return 1 @@ -1632,14 +1652,20 @@ cache:image:produce() { } export -f cache:image:produce +## Will modify current $_CURRENT_DOCKER_COMPOSE file service_ensure_image_ready() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + if [ -z "$COMBINED_HASH" ]; then + err "Expected \$COMBINED_HASH to be set." + return 1 + fi + local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \ master_service service_def service_image service_build service_dockerfile image \ specified_image specified_image_id charm_image_name hash \ service_quoted if [ -e "$cache_file" ]; then #debug "$FUNCNAME: cache hit ($*)" + touch "$cache_file" || return 1 cat "$cache_file" return 0 fi @@ -1898,9 +1924,17 @@ get_ordered_service_dependencies() { } export -f get_ordered_service_dependencies - +## Modify $_CURRENT_DOCKER_COMPOSE file, and fills cache run_service_acquire_images () { local service subservice subservices loaded + _CURRENT_DOCKER_COMPOSE_HASH=$(hash_get < "$_CURRENT_DOCKER_COMPOSE") + local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$_CURRENT_DOCKER_COMPOSE_HASH" "$COMBINED_HASH")" + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + touch "$cache_file" || return 1 + cp "$cache_file" "$_CURRENT_DOCKER_COMPOSE" || return 1 + return 0 + fi declare -A loaded for service in "$@"; do subservices=$(get_ordered_service_dependencies "$service") || return 1 @@ -1921,6 +1955,7 @@ run_service_acquire_images () { loaded[$subservice]=1 done done + cp "$_CURRENT_DOCKER_COMPOSE" "$cache_file" || return 1 return 0 } @@ -2085,6 +2120,10 @@ export -f cfg-get-value relation-get () { + if [ -z "$RELATION_DATA_FILE" ]; then + err-d "$FUNCNAME: var \$RELATION_DATA_FILE is not set." + return 1 + fi cfg-get-value "$1" < "$RELATION_DATA_FILE" } export -f relation-get @@ -2428,7 +2467,11 @@ export -f _get_compose_relations_cached get_compose_relations () { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + if [ -z "$COMBINED_HASH" ]; then + err-d "Expected \$COMBINED_HASH to be set." + return 1 + fi + local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \ compose_def if [ -e "$cache_file" ]; then @@ -2491,7 +2534,11 @@ export -f get_all_services get_service_relations () { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH")" \ + if [ -z "$GLOBAL_ALL_RELATIONS" ]; then + err-d "Can't access global \$GLOBAL_ALL_RELATIONS" + return 1 + fi + local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$GLOBAL_ALL_RELATIONS_HASH" \ s rn ts rc td if [ -e "$cache_file" ]; then #debug "$FUNCNAME: SESSION cache hit $1" @@ -2499,11 +2546,6 @@ get_service_relations () { return 0 fi - if [ -z "$GLOBAL_ALL_RELATIONS" ]; then - err-d "Can't access global \$GLOBAL_ALL_RELATIONS" - return 1 - fi - while read-0 s rn ts rc td; do [[ "$s" == "$service" ]] || continue printf "%s\0" "$rn" "$ts" "$rc" "$td" @@ -2644,6 +2686,34 @@ relation:get() { } export -f relation:get +services:get:upable() { + if [ -z "$CHARM_STORE_HASH" ]; then + err-d "Expected \$CHARM_STORE_HASH to be set." + return 1 + fi + local services_args=("$@") cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$CHARM_STORE_HASH" "$@")" + if [ -e "$cache_file" ]; then + touch "$cache_file" || return 1 + cat "$cache_file" + return 0 + fi + declare -A seen + services=($(get_ordered_service_dependencies "${services_args[@]}")) || exit 1 + for service in "${services[@]}"; do + mservice=$(get_master_service_for_service "$service") || exit 1 + [ "${seen[$mservice]}" ] && continue + type="$(get_service_type "$mservice")" || exit 1 + ## remove run-once + [ "$type" == "run-once" ] && continue + [ "$type" == "stub" ] && continue + seen[$mservice]=1 + echo "$mservice" + done > "$cache_file".wip + mv "$cache_file".wip "$cache_file" + cat "$cache_file" +} +export -f services:get:upable + service:state() { local service="$1" states state project_name=$(get_default_project_name) || return 1 @@ -2819,7 +2889,11 @@ export -f _get_services_provides _get_charm_provides() { - local cache_file="$CACHEDIR/$FUNCNAME.cache.$(charm.store_metadata_hash)" errlvl + if [ -z "$CHARM_STORE_HASH" ]; then + err-d "Expected \$CHARM_STORE_HASH to be set." + return 1 + fi + local cache_file="$CACHEDIR/$FUNCNAME.cache.$CHARM_STORE_HASH" errlvl if [ -e "$cache_file" ]; then #debug "$FUNCNAME: SESSION cache hit" cat "$cache_file" @@ -2958,13 +3032,11 @@ _service:all:relations_cached() { ## Outputs all relations array. service:all:relations() { - if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then - COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || { - err "Failed to get compose yml hash." - return 1 - } + if [ -z "$COMBINED_HASH" ]; then + err-d "Expected \$COMBINED_HASH to be set." + return 1 fi - local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH" + local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMBINED_HASH" if [ -e "${cache_file}" ]; then # debug "$FUNCNAME: SESSION cache hit $1" cat "${cache_file}" @@ -2981,16 +3053,14 @@ service:all:relations() { } _service:all:relations_hash_cached() { - if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then - COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || { - err "Failed to get compose yml hash." - return 1 - } + if [ -z "$COMBINED_HASH" ]; then + err-d "Expected \$COMBINED_HASH to be set." + return 1 fi - local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH" \ + local cache_file="$CACHEDIR/$FUNCNAME.cache.x${COMBINED_HASH}" \ hash if [ -e "${cache_file}" ]; then - # debug "$FUNCNAME: SESSION cache hit $1" + # debug "$FUNCNAME: SESSION cache hit $cache_file" cat "${cache_file}" return 0 fi @@ -3027,7 +3097,7 @@ service:all:set_relations_hash() { read-0 GLOBAL_ALL_RELATIONS_HASH || return 1 export GLOBAL_ALL_RELATIONS_HASH ## transfer to statedir - export GLOBAL_ALL_RELATIONS="$state_tmpdir/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH" + export GLOBAL_ALL_RELATIONS="$CACHEDIR/$FUNCNAME.cache.$COMBINED_HASH" cat > "$GLOBAL_ALL_RELATIONS" } < <(_service:all:relations_hash_cached) if [ -z "$GLOBAL_ALL_RELATIONS" ]; then @@ -3083,14 +3153,15 @@ get_subset_relations () { export -f get_subset_relations get_all_relations () { - if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then - COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || return 1 + if [ -z "$COMBINED_HASH" ]; then + err-d "Expected \$COMBINED_HASH to be set." + return 1 fi if [ -n "$GLOBAL_ALL_RELATIONS" ]; then cat "$GLOBAL_ALL_RELATIONS" || return 1 return 0 fi - local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$COMPOSE_YML_CONTENT_HASH" "$(declare -p without_relations)")" \ + local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$COMBINED_HASH" "$(declare -p without_relations)")" \ services all_services service services_uses services_provides \ changed summon required recommended optional @@ -3615,13 +3686,21 @@ export -f get_relation_data_file has_service_action () { - local service="$1" action="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + if [ -z "$CHARM_STORE_HASH" ]; then + err-d "Can't access global \$CHARM_STORE_HASH" + return 1 + fi + local service="$1" action="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$2.$CHARM_STORE_HASH" \ charm target_charm relation_name target_service relation_config _tech_dep \ path if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" - return 0 + if [ -s "$cache_file" ]; then + cat "$cache_file" + return 0 + else + return 1 + fi fi charm=$(get_service_charm "$service") || return 1 @@ -3643,6 +3722,7 @@ has_service_action () { fi done < <(get_service_relations "$service") + touch "$cache_file" return 1 # master=$(get_top_master_service_for_service "$service") # [ "$master" == "$charm" ] && return 1 @@ -3762,7 +3842,11 @@ export -f _get_master_service_for_service_cached get_master_service_for_service() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + if [ -z "$CHARM_STORE_HASH" ]; then + err-d "Expected \$CHARM_STORE_HASH to be set." + return 1 + fi + local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH" \ charm metadata result if [ -e "$cache_file" ]; then @@ -3782,11 +3866,16 @@ export -f get_master_service_for_service get_top_master_service_for_service() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + if [ -z "$CHARM_STORE_HASH" ]; then + err-d "Expected \$CHARM_STORE_HASH to be set." + return 1 + fi + local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH" \ current_service if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" + touch "$cache_file" || return 1 cat "$cache_file" return 0 fi @@ -4348,6 +4437,12 @@ export -f get_default_target_services get_master_services() { local loaded master_service service + local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" )" + if [ -e "$cache_file" ]; then + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi declare -A loaded for service in "$@"; do master_service=$(get_top_master_service_for_service "$service") || return 1 @@ -4356,8 +4451,10 @@ get_master_services() { fi echo "$master_service" loaded["$master_service"]=1 - done | nspc - return "${PIPESTATUS[0]}" + done > "$cache_file".wip || return 1 + + mv "$cache_file"{.wip,} || return 1 + cat "$cache_file" || return 1 } export -f get_master_services @@ -5264,12 +5361,12 @@ aexport remainder_args ## Actual code ## - COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1 COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1 COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || exit 1 -export COMPOSE_YML_FILE COMPOSE_YML_CONTENT COMPOSE_YML_CONTENT_HASH - +CHARM_STORE_HASH=$(charm.store_metadata_hash) || exit 1 +COMBINED_HASH=$(H "$COMPOSE_YML_CONTENT_HASH" "$CHARM_STORE_HASH") || exit 1 +export COMPOSE_YML_FILE COMPOSE_YML_CONTENT COMPOSE_YML_CONTENT_HASH CHARM_STORE_HASH COMBINED_HASH charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct." ## @@ -5445,18 +5542,7 @@ fi if [ -n "$is_docker_compose_action" ] && [ "${#services_args[@]}" -gt 0 ]; then services=($(get_master_services "${services_args[@]}")) || exit 1 if [ "$action" == "up" ]; then - declare -A seen - services=($(get_ordered_service_dependencies "${services_args[@]}")) || exit 1 - for service in "${services[@]}"; do - mservice=$(get_master_service_for_service "$service") || exit 1 - [ "${seen[$mservice]}" ] && continue - type="$(get_service_type "$mservice")" || exit 1 - ## remove run-once - [ "$type" == "run-once" ] && continue - [ "$type" == "stub" ] && continue - seen[$mservice]=1 - action_posargs+=("$mservice") - done + action_posargs+=($(services:get:upable "${services_args[@]}")) || exit 1 elif [ "$is_docker_compose_action_multi_service" == "1" ]; then action_posargs+=("${services[@]}") elif [ "$is_docker_compose_action_multi_service" == "0" ]; then