0k
/
0k-compose
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

new: [compose-core] move cache files to global cache for increased performance

fix-host-charm-resources
Valentin Lab 2 months ago
parent
662d3f778b
commit
9480da50c9
1 changed files with 143 additions and 57 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 192
      bin/compose-core

192
bin/compose-core
View File

@ -761,7 +761,11 @@ export -f get_service_base_image_dir_uid_gid
get_service_type() { get_service_type() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH"
if [ -z "$service" ]; then if [ -z "$service" ]; then
print_syntax_error "$FUNCNAME: Please specify a service as first argument." print_syntax_error "$FUNCNAME: Please specify a service as first argument."
return 1 return 1
@ -1282,11 +1286,22 @@ export -f _get_docker_compose_service_mixin
## @export ## @export
## @cache: !system !nofail +stdout ## @cache: !system !nofail +stdout
get_docker_compose () { get_docker_compose () {
local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
err-d "Expected \$COMPOSE_YML_CONTENT_HASH to be set."
return 1
fi
local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$CHARM_STORE_HASH" "$COMPOSE_YML_CONTENT_HASH")" \
entries services service start docker_compose_services entries services service start docker_compose_services
if [ -e "$cache_file" ]; then if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
cat "$cache_file"
# debug "$FUNCNAME: cache hit ($*) $cache_file"
touch "$cache_file" || return 1
cp "$cache_file"{,.wip} || return 1
export _CURRENT_DOCKER_COMPOSE="$cache_file.wip"
cat "$cache_file" || return 1
return 0 return 0
fi fi
@ -1339,8 +1354,9 @@ get_docker_compose () {
merge_yaml_str "$(yaml_key_val_str "services" "$docker_compose_services")" \ merge_yaml_str "$(yaml_key_val_str "services" "$docker_compose_services")" \
"$base_v2" > "$cache_file" || return 1 "$base_v2" > "$cache_file" || return 1
export _CURRENT_DOCKER_COMPOSE="$cache_file"
cat "$_CURRENT_DOCKER_COMPOSE"
cp "$cache_file"{,.wip} || return 1
export _CURRENT_DOCKER_COMPOSE="$cache_file.wip"
cat "$_CURRENT_DOCKER_COMPOSE" || return 1
debug " ..compilation of base 'docker-compose.yml' done $GRAY(in $((SECONDS - start_compilation))s)$NORMAL" || true debug " ..compilation of base 'docker-compose.yml' done $GRAY(in $((SECONDS - start_compilation))s)$NORMAL" || true
# debug " ** ${WHITE}docker-compose.yml${NORMAL}:" # debug " ** ${WHITE}docker-compose.yml${NORMAL}:"
# debug "$_current_docker_compose" # debug "$_current_docker_compose"
@ -1387,7 +1403,11 @@ export -f _get_compose_service_def_cached
## XXXvlab: a lot to be done to cache the results ## XXXvlab: a lot to be done to cache the results
get_compose_service_def () { get_compose_service_def () {
local service="$1" docker_compose cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$COMBINED_HASH" ]; then
err-d "Expected \$COMBINED_HASH to be set."
return 1
fi
local service="$1" docker_compose cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \
result result
if [ -e "$cache_file" ]; then if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: SESSION cache hit" #debug "$FUNCNAME: SESSION cache hit"
@ -1398,7 +1418,7 @@ get_compose_service_def () {
[ -z "$service" ] && print_syntax_error "Missing service as first argument." [ -z "$service" ] && print_syntax_error "Missing service as first argument."
docker_compose=$(get_compose_yml_content) || return 1 docker_compose=$(get_compose_yml_content) || return 1
result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1 result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1
charm=$(echo "$result" | shyaml get-value charm 2>/dev/null) || return 1
charm=$(e "$result" | shyaml get-value charm 2>/dev/null) || return 1
metadata=$(charm.metadata "$charm") || return 1 metadata=$(charm.metadata "$charm") || return 1
if default_options=$(printf "%s" "$metadata" | shyaml -y -q get-value default-options); then if default_options=$(printf "%s" "$metadata" | shyaml -y -q get-value default-options); then
default_options=$(yaml_key_val_str "options" "$default_options") || return 1 default_options=$(yaml_key_val_str "options" "$default_options") || return 1
@ -1632,14 +1652,20 @@ cache:image:produce() {
} }
export -f cache:image:produce export -f cache:image:produce
## Will modify current $_CURRENT_DOCKER_COMPOSE file
service_ensure_image_ready() { service_ensure_image_ready() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$COMBINED_HASH" ]; then
err "Expected \$COMBINED_HASH to be set."
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \
master_service service_def service_image service_build service_dockerfile image \ master_service service_def service_image service_build service_dockerfile image \
specified_image specified_image_id charm_image_name hash \ specified_image specified_image_id charm_image_name hash \
service_quoted service_quoted
if [ -e "$cache_file" ]; then if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: cache hit ($*)" #debug "$FUNCNAME: cache hit ($*)"
touch "$cache_file" || return 1
cat "$cache_file" cat "$cache_file"
return 0 return 0
fi fi
@ -1898,9 +1924,17 @@ get_ordered_service_dependencies() {
} }
export -f get_ordered_service_dependencies export -f get_ordered_service_dependencies
## Modify $_CURRENT_DOCKER_COMPOSE file, and fills cache
run_service_acquire_images () { run_service_acquire_images () {
local service subservice subservices loaded local service subservice subservices loaded
_CURRENT_DOCKER_COMPOSE_HASH=$(hash_get < "$_CURRENT_DOCKER_COMPOSE")
local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$_CURRENT_DOCKER_COMPOSE_HASH" "$COMBINED_HASH")"
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
touch "$cache_file" || return 1
cp "$cache_file" "$_CURRENT_DOCKER_COMPOSE" || return 1
return 0
fi
declare -A loaded declare -A loaded
for service in "$@"; do for service in "$@"; do
subservices=$(get_ordered_service_dependencies "$service") || return 1 subservices=$(get_ordered_service_dependencies "$service") || return 1
@ -1921,6 +1955,7 @@ run_service_acquire_images () {
loaded[$subservice]=1 loaded[$subservice]=1
done done
done done
cp "$_CURRENT_DOCKER_COMPOSE" "$cache_file" || return 1
return 0 return 0
} }
@ -2085,6 +2120,10 @@ export -f cfg-get-value
relation-get () { relation-get () {
if [ -z "$RELATION_DATA_FILE" ]; then
err-d "$FUNCNAME: var \$RELATION_DATA_FILE is not set."
return 1
fi
cfg-get-value "$1" < "$RELATION_DATA_FILE" cfg-get-value "$1" < "$RELATION_DATA_FILE"
} }
export -f relation-get export -f relation-get
@ -2428,7 +2467,11 @@ export -f _get_compose_relations_cached
get_compose_relations () { get_compose_relations () {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$COMBINED_HASH" ]; then
err-d "Expected \$COMBINED_HASH to be set."
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \
compose_def compose_def
if [ -e "$cache_file" ]; then if [ -e "$cache_file" ]; then
@ -2491,7 +2534,11 @@ export -f get_all_services
get_service_relations () { get_service_relations () {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH")" \
if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
err-d "Can't access global \$GLOBAL_ALL_RELATIONS"
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$GLOBAL_ALL_RELATIONS_HASH" \
s rn ts rc td s rn ts rc td
if [ -e "$cache_file" ]; then if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: SESSION cache hit $1" #debug "$FUNCNAME: SESSION cache hit $1"
@ -2499,11 +2546,6 @@ get_service_relations () {
return 0 return 0
fi fi
if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
err-d "Can't access global \$GLOBAL_ALL_RELATIONS"
return 1
fi
while read-0 s rn ts rc td; do while read-0 s rn ts rc td; do
[[ "$s" == "$service" ]] || continue [[ "$s" == "$service" ]] || continue
printf "%s\0" "$rn" "$ts" "$rc" "$td" printf "%s\0" "$rn" "$ts" "$rc" "$td"
@ -2644,6 +2686,34 @@ relation:get() {
} }
export -f relation:get export -f relation:get
services:get:upable() {
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
local services_args=("$@") cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$CHARM_STORE_HASH" "$@")"
if [ -e "$cache_file" ]; then
touch "$cache_file" || return 1
cat "$cache_file"
return 0
fi
declare -A seen
services=($(get_ordered_service_dependencies "${services_args[@]}")) || exit 1
for service in "${services[@]}"; do
mservice=$(get_master_service_for_service "$service") || exit 1
[ "${seen[$mservice]}" ] && continue
type="$(get_service_type "$mservice")" || exit 1
## remove run-once
[ "$type" == "run-once" ] && continue
[ "$type" == "stub" ] && continue
seen[$mservice]=1
echo "$mservice"
done > "$cache_file".wip
mv "$cache_file".wip "$cache_file"
cat "$cache_file"
}
export -f services:get:upable
service:state() { service:state() {
local service="$1" states state local service="$1" states state
project_name=$(get_default_project_name) || return 1 project_name=$(get_default_project_name) || return 1
@ -2819,7 +2889,11 @@ export -f _get_services_provides
_get_charm_provides() { _get_charm_provides() {
local cache_file="$CACHEDIR/$FUNCNAME.cache.$(charm.store_metadata_hash)" errlvl
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
local cache_file="$CACHEDIR/$FUNCNAME.cache.$CHARM_STORE_HASH" errlvl
if [ -e "$cache_file" ]; then if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: SESSION cache hit" #debug "$FUNCNAME: SESSION cache hit"
cat "$cache_file" cat "$cache_file"
@ -2958,13 +3032,11 @@ _service:all:relations_cached() {
## Outputs all relations array. ## Outputs all relations array.
service:all:relations() { service:all:relations() {
if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || {
err "Failed to get compose yml hash."
if [ -z "$COMBINED_HASH" ]; then
err-d "Expected \$COMBINED_HASH to be set."
return 1 return 1
}
fi fi
local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH"
local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMBINED_HASH"
if [ -e "${cache_file}" ]; then if [ -e "${cache_file}" ]; then
# debug "$FUNCNAME: SESSION cache hit $1" # debug "$FUNCNAME: SESSION cache hit $1"
cat "${cache_file}" cat "${cache_file}"
@ -2981,16 +3053,14 @@ service:all:relations() {
} }
_service:all:relations_hash_cached() { _service:all:relations_hash_cached() {
if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || {
err "Failed to get compose yml hash."
if [ -z "$COMBINED_HASH" ]; then
err-d "Expected \$COMBINED_HASH to be set."
return 1 return 1
}
fi fi
local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH" \
local cache_file="$CACHEDIR/$FUNCNAME.cache.x${COMBINED_HASH}" \
hash hash
if [ -e "${cache_file}" ]; then if [ -e "${cache_file}" ]; then
# debug "$FUNCNAME: SESSION cache hit $1"
# debug "$FUNCNAME: SESSION cache hit $cache_file"
cat "${cache_file}" cat "${cache_file}"
return 0 return 0
fi fi
@ -3027,7 +3097,7 @@ service:all:set_relations_hash() {
read-0 GLOBAL_ALL_RELATIONS_HASH || return 1 read-0 GLOBAL_ALL_RELATIONS_HASH || return 1
export GLOBAL_ALL_RELATIONS_HASH export GLOBAL_ALL_RELATIONS_HASH
## transfer to statedir ## transfer to statedir
export GLOBAL_ALL_RELATIONS="$state_tmpdir/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH"
export GLOBAL_ALL_RELATIONS="$CACHEDIR/$FUNCNAME.cache.$COMBINED_HASH"
cat > "$GLOBAL_ALL_RELATIONS" cat > "$GLOBAL_ALL_RELATIONS"
} < <(_service:all:relations_hash_cached) } < <(_service:all:relations_hash_cached)
if [ -z "$GLOBAL_ALL_RELATIONS" ]; then if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
@ -3083,14 +3153,15 @@ get_subset_relations () {
export -f get_subset_relations export -f get_subset_relations
get_all_relations () { get_all_relations () {
if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || return 1
if [ -z "$COMBINED_HASH" ]; then
err-d "Expected \$COMBINED_HASH to be set."
return 1
fi fi
if [ -n "$GLOBAL_ALL_RELATIONS" ]; then if [ -n "$GLOBAL_ALL_RELATIONS" ]; then
cat "$GLOBAL_ALL_RELATIONS" || return 1 cat "$GLOBAL_ALL_RELATIONS" || return 1
return 0 return 0
fi fi
local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$COMPOSE_YML_CONTENT_HASH" "$(declare -p without_relations)")" \
local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$COMBINED_HASH" "$(declare -p without_relations)")" \
services all_services service services_uses services_provides \ services all_services service services_uses services_provides \
changed summon required recommended optional changed summon required recommended optional
@ -3615,13 +3686,21 @@ export -f get_relation_data_file
has_service_action () { has_service_action () {
local service="$1" action="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Can't access global \$CHARM_STORE_HASH"
return 1
fi
local service="$1" action="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$2.$CHARM_STORE_HASH" \
charm target_charm relation_name target_service relation_config _tech_dep \ charm target_charm relation_name target_service relation_config _tech_dep \
path path
if [ -e "$cache_file" ]; then if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)" # debug "$FUNCNAME: cache hit ($*)"
if [ -s "$cache_file" ]; then
cat "$cache_file" cat "$cache_file"
return 0 return 0
else
return 1
fi
fi fi
charm=$(get_service_charm "$service") || return 1 charm=$(get_service_charm "$service") || return 1
@ -3643,6 +3722,7 @@ has_service_action () {
fi fi
done < <(get_service_relations "$service") done < <(get_service_relations "$service")
touch "$cache_file"
return 1 return 1
# master=$(get_top_master_service_for_service "$service") # master=$(get_top_master_service_for_service "$service")
# [ "$master" == "$charm" ] && return 1 # [ "$master" == "$charm" ] && return 1
@ -3762,7 +3842,11 @@ export -f _get_master_service_for_service_cached
get_master_service_for_service() { get_master_service_for_service() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH" \
charm metadata result charm metadata result
if [ -e "$cache_file" ]; then if [ -e "$cache_file" ]; then
@ -3782,11 +3866,16 @@ export -f get_master_service_for_service
get_top_master_service_for_service() { get_top_master_service_for_service() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH" \
current_service current_service
if [ -e "$cache_file" ]; then if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)" # debug "$FUNCNAME: cache hit ($*)"
touch "$cache_file" || return 1
cat "$cache_file" cat "$cache_file"
return 0 return 0
fi fi
@ -4348,6 +4437,12 @@ export -f get_default_target_services
get_master_services() { get_master_services() {
local loaded master_service service local loaded master_service service
local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" )"
if [ -e "$cache_file" ]; then
cat "$cache_file" &&
touch "$cache_file" || return 1
return 0
fi
declare -A loaded declare -A loaded
for service in "$@"; do for service in "$@"; do
master_service=$(get_top_master_service_for_service "$service") || return 1 master_service=$(get_top_master_service_for_service "$service") || return 1
@ -4356,8 +4451,10 @@ get_master_services() {
fi fi
echo "$master_service" echo "$master_service"
loaded["$master_service"]=1 loaded["$master_service"]=1
done | nspc
return "${PIPESTATUS[0]}"
done > "$cache_file".wip || return 1
mv "$cache_file"{.wip,} || return 1
cat "$cache_file" || return 1
} }
export -f get_master_services export -f get_master_services
@ -5264,12 +5361,12 @@ aexport remainder_args
## Actual code ## Actual code
## ##
COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1 COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1
COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1 COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1
COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || exit 1 COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || exit 1
export COMPOSE_YML_FILE COMPOSE_YML_CONTENT COMPOSE_YML_CONTENT_HASH
CHARM_STORE_HASH=$(charm.store_metadata_hash) || exit 1
COMBINED_HASH=$(H "$COMPOSE_YML_CONTENT_HASH" "$CHARM_STORE_HASH") || exit 1
export COMPOSE_YML_FILE COMPOSE_YML_CONTENT COMPOSE_YML_CONTENT_HASH CHARM_STORE_HASH COMBINED_HASH
charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct." charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct."
## ##
@ -5445,18 +5542,7 @@ fi
if [ -n "$is_docker_compose_action" ] && [ "${#services_args[@]}" -gt 0 ]; then if [ -n "$is_docker_compose_action" ] && [ "${#services_args[@]}" -gt 0 ]; then
services=($(get_master_services "${services_args[@]}")) || exit 1 services=($(get_master_services "${services_args[@]}")) || exit 1
if [ "$action" == "up" ]; then if [ "$action" == "up" ]; then
declare -A seen
services=($(get_ordered_service_dependencies "${services_args[@]}")) || exit 1
for service in "${services[@]}"; do
mservice=$(get_master_service_for_service "$service") || exit 1
[ "${seen[$mservice]}" ] && continue
type="$(get_service_type "$mservice")" || exit 1
## remove run-once
[ "$type" == "run-once" ] && continue
[ "$type" == "stub" ] && continue
seen[$mservice]=1
action_posargs+=("$mservice")
done
action_posargs+=($(services:get:upable "${services_args[@]}")) || exit 1
elif [ "$is_docker_compose_action_multi_service" == "1" ]; then elif [ "$is_docker_compose_action_multi_service" == "1" ]; then
action_posargs+=("${services[@]}") action_posargs+=("${services[@]}")
elif [ "$is_docker_compose_action_multi_service" == "0" ]; then elif [ "$is_docker_compose_action_multi_service" == "0" ]; then

Write Preview
Loading…
Cancel
Save