0k
/
0k-compose
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

new: [compose-core] move cache files to global cache for increased performance

fix-host-charm-resources
Valentin Lab 2 months ago
parent
662d3f778b
commit
9480da50c9
1 changed files with 143 additions and 57 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 192
      bin/compose-core

192
bin/compose-core
View File

@ -761,7 +761,11 @@ export -f get_service_base_image_dir_uid_gid
get_service_type() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH"
if [ -z "$service" ]; then
print_syntax_error "$FUNCNAME: Please specify a service as first argument."
return 1
@ -1282,11 +1286,22 @@ export -f _get_docker_compose_service_mixin
## @export
## @cache: !system !nofail +stdout
get_docker_compose () {
local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
err-d "Expected \$COMPOSE_YML_CONTENT_HASH to be set."
return 1
fi
local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$CHARM_STORE_HASH" "$COMPOSE_YML_CONTENT_HASH")" \
entries services service start docker_compose_services
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
cat "$cache_file"
# debug "$FUNCNAME: cache hit ($*) $cache_file"
touch "$cache_file" || return 1
cp "$cache_file"{,.wip} || return 1
export _CURRENT_DOCKER_COMPOSE="$cache_file.wip"
cat "$cache_file" || return 1
return 0
fi
@ -1339,8 +1354,9 @@ get_docker_compose () {
merge_yaml_str "$(yaml_key_val_str "services" "$docker_compose_services")" \
"$base_v2" > "$cache_file" || return 1
export _CURRENT_DOCKER_COMPOSE="$cache_file"
cat "$_CURRENT_DOCKER_COMPOSE"
cp "$cache_file"{,.wip} || return 1
export _CURRENT_DOCKER_COMPOSE="$cache_file.wip"
cat "$_CURRENT_DOCKER_COMPOSE" || return 1
debug " ..compilation of base 'docker-compose.yml' done $GRAY(in $((SECONDS - start_compilation))s)$NORMAL" || true
# debug " ** ${WHITE}docker-compose.yml${NORMAL}:"
# debug "$_current_docker_compose"
@ -1387,7 +1403,11 @@ export -f _get_compose_service_def_cached
## XXXvlab: a lot to be done to cache the results
get_compose_service_def () {
local service="$1" docker_compose cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$COMBINED_HASH" ]; then
err-d "Expected \$COMBINED_HASH to be set."
return 1
fi
local service="$1" docker_compose cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \
result
if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: SESSION cache hit"
@ -1398,7 +1418,7 @@ get_compose_service_def () {
[ -z "$service" ] && print_syntax_error "Missing service as first argument."
docker_compose=$(get_compose_yml_content) || return 1
result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1
charm=$(echo "$result" | shyaml get-value charm 2>/dev/null) || return 1
charm=$(e "$result" | shyaml get-value charm 2>/dev/null) || return 1
metadata=$(charm.metadata "$charm") || return 1
if default_options=$(printf "%s" "$metadata" | shyaml -y -q get-value default-options); then
default_options=$(yaml_key_val_str "options" "$default_options") || return 1
@ -1632,14 +1652,20 @@ cache:image:produce() {
}
export -f cache:image:produce
## Will modify current $_CURRENT_DOCKER_COMPOSE file
service_ensure_image_ready() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$COMBINED_HASH" ]; then
err "Expected \$COMBINED_HASH to be set."
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \
master_service service_def service_image service_build service_dockerfile image \
specified_image specified_image_id charm_image_name hash \
service_quoted
if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: cache hit ($*)"
touch "$cache_file" || return 1
cat "$cache_file"
return 0
fi
@ -1898,9 +1924,17 @@ get_ordered_service_dependencies() {
}
export -f get_ordered_service_dependencies
## Modify $_CURRENT_DOCKER_COMPOSE file, and fills cache
run_service_acquire_images () {
local service subservice subservices loaded
_CURRENT_DOCKER_COMPOSE_HASH=$(hash_get < "$_CURRENT_DOCKER_COMPOSE")
local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$_CURRENT_DOCKER_COMPOSE_HASH" "$COMBINED_HASH")"
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
touch "$cache_file" || return 1
cp "$cache_file" "$_CURRENT_DOCKER_COMPOSE" || return 1
return 0
fi
declare -A loaded
for service in "$@"; do
subservices=$(get_ordered_service_dependencies "$service") || return 1
@ -1921,6 +1955,7 @@ run_service_acquire_images () {
loaded[$subservice]=1
done
done
cp "$_CURRENT_DOCKER_COMPOSE" "$cache_file" || return 1
return 0
}
@ -2085,6 +2120,10 @@ export -f cfg-get-value
relation-get () {
if [ -z "$RELATION_DATA_FILE" ]; then
err-d "$FUNCNAME: var \$RELATION_DATA_FILE is not set."
return 1
fi
cfg-get-value "$1" < "$RELATION_DATA_FILE"
}
export -f relation-get
@ -2428,7 +2467,11 @@ export -f _get_compose_relations_cached
get_compose_relations () {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$COMBINED_HASH" ]; then
err-d "Expected \$COMBINED_HASH to be set."
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \
compose_def
if [ -e "$cache_file" ]; then
@ -2491,7 +2534,11 @@ export -f get_all_services
get_service_relations () {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH")" \
if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
err-d "Can't access global \$GLOBAL_ALL_RELATIONS"
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$GLOBAL_ALL_RELATIONS_HASH" \
s rn ts rc td
if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: SESSION cache hit $1"
@ -2499,11 +2546,6 @@ get_service_relations () {
return 0
fi
if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
err-d "Can't access global \$GLOBAL_ALL_RELATIONS"
return 1
fi
while read-0 s rn ts rc td; do
[[ "$s" == "$service" ]] || continue
printf "%s\0" "$rn" "$ts" "$rc" "$td"
@ -2644,6 +2686,34 @@ relation:get() {
}
export -f relation:get
services:get:upable() {
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
local services_args=("$@") cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$CHARM_STORE_HASH" "$@")"
if [ -e "$cache_file" ]; then
touch "$cache_file" || return 1
cat "$cache_file"
return 0
fi
declare -A seen
services=($(get_ordered_service_dependencies "${services_args[@]}")) || exit 1
for service in "${services[@]}"; do
mservice=$(get_master_service_for_service "$service") || exit 1
[ "${seen[$mservice]}" ] && continue
type="$(get_service_type "$mservice")" || exit 1
## remove run-once
[ "$type" == "run-once" ] && continue
[ "$type" == "stub" ] && continue
seen[$mservice]=1
echo "$mservice"
done > "$cache_file".wip
mv "$cache_file".wip "$cache_file"
cat "$cache_file"
}
export -f services:get:upable
service:state() {
local service="$1" states state
project_name=$(get_default_project_name) || return 1
@ -2819,7 +2889,11 @@ export -f _get_services_provides
_get_charm_provides() {
local cache_file="$CACHEDIR/$FUNCNAME.cache.$(charm.store_metadata_hash)" errlvl
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
local cache_file="$CACHEDIR/$FUNCNAME.cache.$CHARM_STORE_HASH" errlvl
if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: SESSION cache hit"
cat "$cache_file"
@ -2958,13 +3032,11 @@ _service:all:relations_cached() {
## Outputs all relations array.
service:all:relations() {
if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || {
err "Failed to get compose yml hash."
if [ -z "$COMBINED_HASH" ]; then
err-d "Expected \$COMBINED_HASH to be set."
return 1
}
fi
local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH"
local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMBINED_HASH"
if [ -e "${cache_file}" ]; then
# debug "$FUNCNAME: SESSION cache hit $1"
cat "${cache_file}"
@ -2981,16 +3053,14 @@ service:all:relations() {
}
_service:all:relations_hash_cached() {
if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || {
err "Failed to get compose yml hash."
if [ -z "$COMBINED_HASH" ]; then
err-d "Expected \$COMBINED_HASH to be set."
return 1
}
fi
local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH" \
local cache_file="$CACHEDIR/$FUNCNAME.cache.x${COMBINED_HASH}" \
hash
if [ -e "${cache_file}" ]; then
# debug "$FUNCNAME: SESSION cache hit $1"
# debug "$FUNCNAME: SESSION cache hit $cache_file"
cat "${cache_file}"
return 0
fi
@ -3027,7 +3097,7 @@ service:all:set_relations_hash() {
read-0 GLOBAL_ALL_RELATIONS_HASH || return 1
export GLOBAL_ALL_RELATIONS_HASH
## transfer to statedir
export GLOBAL_ALL_RELATIONS="$state_tmpdir/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH"
export GLOBAL_ALL_RELATIONS="$CACHEDIR/$FUNCNAME.cache.$COMBINED_HASH"
cat > "$GLOBAL_ALL_RELATIONS"
} < <(_service:all:relations_hash_cached)
if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
@ -3083,14 +3153,15 @@ get_subset_relations () {
export -f get_subset_relations
get_all_relations () {
if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || return 1
if [ -z "$COMBINED_HASH" ]; then
err-d "Expected \$COMBINED_HASH to be set."
return 1
fi
if [ -n "$GLOBAL_ALL_RELATIONS" ]; then
cat "$GLOBAL_ALL_RELATIONS" || return 1
return 0
fi
local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$COMPOSE_YML_CONTENT_HASH" "$(declare -p without_relations)")" \
local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$COMBINED_HASH" "$(declare -p without_relations)")" \
services all_services service services_uses services_provides \
changed summon required recommended optional
@ -3615,13 +3686,21 @@ export -f get_relation_data_file
has_service_action () {
local service="$1" action="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Can't access global \$CHARM_STORE_HASH"
return 1
fi
local service="$1" action="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$2.$CHARM_STORE_HASH" \
charm target_charm relation_name target_service relation_config _tech_dep \
path
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
if [ -s "$cache_file" ]; then
cat "$cache_file"
return 0
else
return 1
fi
fi
charm=$(get_service_charm "$service") || return 1
@ -3643,6 +3722,7 @@ has_service_action () {
fi
done < <(get_service_relations "$service")
touch "$cache_file"
return 1
# master=$(get_top_master_service_for_service "$service")
# [ "$master" == "$charm" ] && return 1
@ -3762,7 +3842,11 @@ export -f _get_master_service_for_service_cached
get_master_service_for_service() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH" \
charm metadata result
if [ -e "$cache_file" ]; then
@ -3782,11 +3866,16 @@ export -f get_master_service_for_service
get_top_master_service_for_service() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH" \
current_service
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
touch "$cache_file" || return 1
cat "$cache_file"
return 0
fi
@ -4348,6 +4437,12 @@ export -f get_default_target_services
get_master_services() {
local loaded master_service service
local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" )"
if [ -e "$cache_file" ]; then
cat "$cache_file" &&
touch "$cache_file" || return 1
return 0
fi
declare -A loaded
for service in "$@"; do
master_service=$(get_top_master_service_for_service "$service") || return 1
@ -4356,8 +4451,10 @@ get_master_services() {
fi
echo "$master_service"
loaded["$master_service"]=1
done | nspc
return "${PIPESTATUS[0]}"
done > "$cache_file".wip || return 1
mv "$cache_file"{.wip,} || return 1
cat "$cache_file" || return 1
}
export -f get_master_services
@ -5264,12 +5361,12 @@ aexport remainder_args
## Actual code
##
COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1
COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1
COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || exit 1
export COMPOSE_YML_FILE COMPOSE_YML_CONTENT COMPOSE_YML_CONTENT_HASH
CHARM_STORE_HASH=$(charm.store_metadata_hash) || exit 1
COMBINED_HASH=$(H "$COMPOSE_YML_CONTENT_HASH" "$CHARM_STORE_HASH") || exit 1
export COMPOSE_YML_FILE COMPOSE_YML_CONTENT COMPOSE_YML_CONTENT_HASH CHARM_STORE_HASH COMBINED_HASH
charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct."
##
@ -5445,18 +5542,7 @@ fi
if [ -n "$is_docker_compose_action" ] && [ "${#services_args[@]}" -gt 0 ]; then
services=($(get_master_services "${services_args[@]}")) || exit 1
if [ "$action" == "up" ]; then
declare -A seen
services=($(get_ordered_service_dependencies "${services_args[@]}")) || exit 1
for service in "${services[@]}"; do
mservice=$(get_master_service_for_service "$service") || exit 1
[ "${seen[$mservice]}" ] && continue
type="$(get_service_type "$mservice")" || exit 1
## remove run-once
[ "$type" == "run-once" ] && continue
[ "$type" == "stub" ] && continue
seen[$mservice]=1
action_posargs+=("$mservice")
done
action_posargs+=($(services:get:upable "${services_args[@]}")) || exit 1
elif [ "$is_docker_compose_action_multi_service" == "1" ]; then
action_posargs+=("${services[@]}")
elif [ "$is_docker_compose_action_multi_service" == "0" ]; then

Write Preview
Loading…
Cancel
Save