new: [compose-core] move cache files to global cache for increased performance

bin/compose-core

@@ -761,7 +761,11 @@ export -f get_service_base_image_dir_uid_gid
 
 
 get_service_type() {
-    local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
+    if [ -z "$CHARM_STORE_HASH" ]; then
+        err-d "Expected \$CHARM_STORE_HASH to be set."
+        return 1
+    fi
+    local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH"
     if [ -z "$service" ]; then
         print_syntax_error "$FUNCNAME: Please specify a service as first argument."
         return 1
@@ -1282,11 +1286,22 @@ export -f _get_docker_compose_service_mixin
 ## @export
 ## @cache: !system !nofail +stdout
 get_docker_compose () {
-    local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
+    if [ -z "$CHARM_STORE_HASH" ]; then
+        err-d "Expected \$CHARM_STORE_HASH to be set."
+        return 1
+    fi
+    if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
+        err-d "Expected \$COMPOSE_YML_CONTENT_HASH to be set."
+        return 1
+    fi
+    local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$CHARM_STORE_HASH" "$COMPOSE_YML_CONTENT_HASH")" \
           entries services service start docker_compose_services
     if [ -e "$cache_file" ]; then
-        # debug "$FUNCNAME: cache hit ($*)"
-        cat "$cache_file"
+        # debug "$FUNCNAME: cache hit ($*) $cache_file"
+        touch "$cache_file" || return 1
+        cp "$cache_file"{,.wip} || return 1
+        export _CURRENT_DOCKER_COMPOSE="$cache_file.wip"
+        cat "$cache_file" || return 1
         return 0
     fi
 
@@ -1339,8 +1354,9 @@ get_docker_compose () {
     merge_yaml_str "$(yaml_key_val_str "services" "$docker_compose_services")" \
                    "$base_v2" > "$cache_file" || return 1
 
-    export _CURRENT_DOCKER_COMPOSE="$cache_file"
-    cat "$_CURRENT_DOCKER_COMPOSE"
+    cp "$cache_file"{,.wip} || return 1
+    export _CURRENT_DOCKER_COMPOSE="$cache_file.wip"
+    cat "$_CURRENT_DOCKER_COMPOSE" || return 1
     debug "  ..compilation of base 'docker-compose.yml' done $GRAY(in $((SECONDS - start_compilation))s)$NORMAL" || true
     # debug " ** ${WHITE}docker-compose.yml${NORMAL}:"
     # debug "$_current_docker_compose"
@@ -1632,14 +1648,20 @@ cache:image:produce() {
 }
 export -f cache:image:produce
 
+## Will modify current $_CURRENT_DOCKER_COMPOSE file
 service_ensure_image_ready() {
-    local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
+    if [ -z "$COMBINED_HASH" ]; then
+        err "Expected \$COMBINED_HASH to be set."
+        return 1
+    fi
+    local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \
           master_service service_def service_image service_build service_dockerfile image \
           specified_image specified_image_id charm_image_name hash \
           service_quoted
 
     if [ -e "$cache_file" ]; then
         #debug "$FUNCNAME: cache hit ($*)"
+        touch "$cache_file" || return 1
         cat "$cache_file"
         return 0
     fi
@@ -1898,9 +1920,17 @@ get_ordered_service_dependencies() {
 }
 export -f get_ordered_service_dependencies
 
-
+## Modify $_CURRENT_DOCKER_COMPOSE file, and fills cache
 run_service_acquire_images () {
     local service subservice subservices loaded
+    _CURRENT_DOCKER_COMPOSE_HASH=$(hash_get < "$_CURRENT_DOCKER_COMPOSE")
+    local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$_CURRENT_DOCKER_COMPOSE_HASH")"
+    if [ -e "$cache_file" ]; then
+        # debug "$FUNCNAME: cache hit ($*)"
+        touch "$cache_file" || return 1
+        cp "$cache_file" "$_CURRENT_DOCKER_COMPOSE" || return 1
+        return 0
+    fi
     declare -A loaded
     for service in "$@"; do
         subservices=$(get_ordered_service_dependencies "$service") || return 1
@@ -1921,6 +1951,7 @@ run_service_acquire_images () {
             loaded[$subservice]=1
         done
     done
+    cp "$_CURRENT_DOCKER_COMPOSE" "$cache_file" || return 1
     return 0
 }
 
@@ -2085,6 +2116,10 @@ export -f cfg-get-value
 
 
 relation-get () {
+    if [ -z "$RELATION_DATA_FILE" ]; then
+        err-d "$FUNCNAME: var \$RELATION_DATA_FILE is not set."
+        return 1
+    fi
     cfg-get-value "$1" < "$RELATION_DATA_FILE"
 }
 export -f relation-get
@@ -2644,6 +2679,34 @@ relation:get() {
 }
 export -f relation:get
 
+services:get:upable() {
+    if [ -z "$CHARM_STORE_HASH" ]; then
+        err-d "Expected \$CHARM_STORE_HASH to be set."
+        return 1
+    fi
+    local services_args=("$@") cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$CHARM_STORE_HASH" "$@")"
+    if [ -e "$cache_file" ]; then
+        touch "$cache_file" || return 1
+        cat "$cache_file"
+        return 0
+    fi
+    declare -A seen
+    services=($(get_ordered_service_dependencies "${services_args[@]}")) || exit 1
+    for service in "${services[@]}"; do
+        mservice=$(get_master_service_for_service "$service") || exit 1
+        [ "${seen[$mservice]}" ] && continue
+        type="$(get_service_type "$mservice")" || exit 1
+        ## remove run-once
+        [ "$type" == "run-once" ] && continue
+        [ "$type" == "stub" ] && continue
+        seen[$mservice]=1
+        echo "$mservice"
+    done > "$cache_file".wip
+    mv "$cache_file".wip "$cache_file"
+    cat "$cache_file"
+}
+export -f services:get:upable
+
 service:state() {
     local service="$1" states state
     project_name=$(get_default_project_name) || return 1
@@ -2819,7 +2882,11 @@ export -f _get_services_provides
 
 
 _get_charm_provides() {
-    local cache_file="$CACHEDIR/$FUNCNAME.cache.$(charm.store_metadata_hash)" errlvl
+    if [ -z "$CHARM_STORE_HASH" ]; then
+        err-d "Expected \$CHARM_STORE_HASH to be set."
+        return 1
+    fi
+    local cache_file="$CACHEDIR/$FUNCNAME.cache.$CHARM_STORE_HASH" errlvl
     if [ -e "$cache_file" ]; then
         #debug "$FUNCNAME: SESSION cache hit"
         cat "$cache_file"
@@ -3615,13 +3682,21 @@ export -f get_relation_data_file
 
 
 has_service_action () {
-    local service="$1" action="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
+    if [ -z "$CHARM_STORE_HASH" ]; then
+        err-d "Can't access global \$CHARM_STORE_HASH"
+        return 1
+    fi
+    local service="$1" action="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$2.$CHARM_STORE_HASH" \
           charm target_charm relation_name target_service relation_config _tech_dep \
           path
     if [ -e "$cache_file" ]; then
         # debug "$FUNCNAME: cache hit ($*)"
-        cat "$cache_file"
-        return 0
+        if [ -s "$cache_file" ]; then
+            cat "$cache_file"
+            return 0
+        else
+            return 1
+        fi
     fi
 
     charm=$(get_service_charm "$service") || return 1
@@ -3643,6 +3718,7 @@ has_service_action () {
         fi
     done < <(get_service_relations "$service")
 
+    touch "$cache_file"
     return 1
     # master=$(get_top_master_service_for_service "$service")
     # [ "$master" == "$charm" ] && return 1
@@ -3762,7 +3838,11 @@ export -f _get_master_service_for_service_cached
 
 
 get_master_service_for_service() {
-    local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
+    if [ -z "$CHARM_STORE_HASH" ]; then
+        err-d "Expected \$CHARM_STORE_HASH to be set."
+        return 1
+    fi
+    local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH" \
           charm metadata result
 
     if [ -e "$cache_file" ]; then
@@ -3782,11 +3862,16 @@ export -f get_master_service_for_service
 
 
 get_top_master_service_for_service() {
-    local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
+    if [ -z "$CHARM_STORE_HASH" ]; then
+        err-d "Expected \$CHARM_STORE_HASH to be set."
+        return 1
+    fi
+    local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH" \
           current_service
 
     if [ -e "$cache_file" ]; then
         # debug "$FUNCNAME: cache hit ($*)"
+        touch "$cache_file" || return 1
         cat "$cache_file"
         return 0
     fi
@@ -4348,6 +4433,12 @@ export -f get_default_target_services
 
 get_master_services() {
     local loaded master_service service
+    local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" )"
+    if [ -e "$cache_file" ]; then
+        cat "$cache_file" &&
+        touch "$cache_file" || return 1
+        return 0
+    fi
     declare -A loaded
     for service in "$@"; do
         master_service=$(get_top_master_service_for_service "$service") || return 1
@@ -4356,8 +4447,10 @@ get_master_services() {
         fi
         echo "$master_service"
         loaded["$master_service"]=1
-    done | nspc
-    return "${PIPESTATUS[0]}"
+    done > "$cache_file".wip || return 1
+
+    mv "$cache_file"{.wip,} || return 1
+    cat "$cache_file" || return 1
 }
 export -f get_master_services
 
@@ -5264,12 +5357,12 @@ aexport remainder_args
 ## Actual code
 ##
 
-
 COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1
 COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1
 COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || exit 1
-export COMPOSE_YML_FILE COMPOSE_YML_CONTENT COMPOSE_YML_CONTENT_HASH
-
+CHARM_STORE_HASH=$(charm.store_metadata_hash) || exit 1
+COMBINED_HASH=$(H "$COMPOSE_YML_CONTENT_HASH" "$CHARM_STORE_HASH") || exit 1
+export COMPOSE_YML_FILE COMPOSE_YML_CONTENT COMPOSE_YML_CONTENT_HASH CHARM_STORE_HASH COMBINED_HASH
 charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct."
 
 ##
@@ -5445,18 +5538,7 @@ fi
 if [ -n "$is_docker_compose_action" ] && [ "${#services_args[@]}" -gt 0 ]; then
     services=($(get_master_services "${services_args[@]}")) || exit 1
     if [ "$action" == "up" ]; then
-        declare -A seen
-        services=($(get_ordered_service_dependencies "${services_args[@]}")) || exit 1
-        for service in "${services[@]}"; do
-            mservice=$(get_master_service_for_service "$service") || exit 1
-            [ "${seen[$mservice]}" ] && continue
-            type="$(get_service_type "$mservice")" || exit 1
-            ## remove run-once
-            [ "$type" == "run-once" ] && continue
-            [ "$type" == "stub" ] && continue
-            seen[$mservice]=1
-            action_posargs+=("$mservice")
-        done
+        action_posargs+=($(services:get:upable "${services_args[@]}")) || exit 1
     elif [ "$is_docker_compose_action_multi_service" == "1" ]; then
         action_posargs+=("${services[@]}")
     elif [ "$is_docker_compose_action_multi_service" == "0" ]; then