0k
/
0k-compose
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

new: [compose-core] move cache files to global cache for increased performance

Valentin Lab 4 weeks ago
parent
662d3f778b
commit
67ce0adc49
1 changed files with 113 additions and 31 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 144
      bin/compose-core

144
bin/compose-core
View File

@ -761,7 +761,11 @@ export -f get_service_base_image_dir_uid_gid
get_service_type() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH"
if [ -z "$service" ]; then
print_syntax_error "$FUNCNAME: Please specify a service as first argument."
return 1
@ -1282,11 +1286,22 @@ export -f _get_docker_compose_service_mixin
## @export
## @cache: !system !nofail +stdout
get_docker_compose () {
local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
err-d "Expected \$COMPOSE_YML_CONTENT_HASH to be set."
return 1
fi
local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$CHARM_STORE_HASH" "$COMPOSE_YML_CONTENT_HASH")" \
entries services service start docker_compose_services
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
cat "$cache_file"
# debug "$FUNCNAME: cache hit ($*) $cache_file"
touch "$cache_file" || return 1
cp "$cache_file"{,.wip} || return 1
export _CURRENT_DOCKER_COMPOSE="$cache_file.wip"
cat "$cache_file" || return 1
return 0
fi
@ -1339,8 +1354,9 @@ get_docker_compose () {
merge_yaml_str "$(yaml_key_val_str "services" "$docker_compose_services")" \
"$base_v2" > "$cache_file" || return 1
export _CURRENT_DOCKER_COMPOSE="$cache_file"
cat "$_CURRENT_DOCKER_COMPOSE"
cp "$cache_file"{,.wip} || return 1
export _CURRENT_DOCKER_COMPOSE="$cache_file.wip"
cat "$_CURRENT_DOCKER_COMPOSE" || return 1
debug " ..compilation of base 'docker-compose.yml' done $GRAY(in $((SECONDS - start_compilation))s)$NORMAL" || true
# debug " ** ${WHITE}docker-compose.yml${NORMAL}:"
# debug "$_current_docker_compose"
@ -1632,14 +1648,20 @@ cache:image:produce() {
}
export -f cache:image:produce
## Will modify current $_CURRENT_DOCKER_COMPOSE file
service_ensure_image_ready() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$COMBINED_HASH" ]; then
err "Expected \$COMBINED_HASH to be set."
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \
master_service service_def service_image service_build service_dockerfile image \
specified_image specified_image_id charm_image_name hash \
service_quoted
if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: cache hit ($*)"
touch "$cache_file" || return 1
cat "$cache_file"
return 0
fi
@ -1898,9 +1920,17 @@ get_ordered_service_dependencies() {
}
export -f get_ordered_service_dependencies
## Modify $_CURRENT_DOCKER_COMPOSE file, and fills cache
run_service_acquire_images () {
local service subservice subservices loaded
_CURRENT_DOCKER_COMPOSE_HASH=$(hash_get < "$_CURRENT_DOCKER_COMPOSE")
local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$_CURRENT_DOCKER_COMPOSE_HASH")"
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
touch "$cache_file" || return 1
cp "$cache_file" "$_CURRENT_DOCKER_COMPOSE" || return 1
return 0
fi
declare -A loaded
for service in "$@"; do
subservices=$(get_ordered_service_dependencies "$service") || return 1
@ -1921,6 +1951,7 @@ run_service_acquire_images () {
loaded[$subservice]=1
done
done
cp "$_CURRENT_DOCKER_COMPOSE" "$cache_file" || return 1
return 0
}
@ -2085,6 +2116,10 @@ export -f cfg-get-value
relation-get () {
if [ -z "$RELATION_DATA_FILE" ]; then
err-d "$FUNCNAME: var \$RELATION_DATA_FILE is not set."
return 1
fi
cfg-get-value "$1" < "$RELATION_DATA_FILE"
}
export -f relation-get
@ -2644,6 +2679,34 @@ relation:get() {
}
export -f relation:get
services:get:upable() {
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
local services_args=("$@") cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$CHARM_STORE_HASH" "$@")"
if [ -e "$cache_file" ]; then
touch "$cache_file" || return 1
cat "$cache_file"
return 0
fi
declare -A seen
services=($(get_ordered_service_dependencies "${services_args[@]}")) || exit 1
for service in "${services[@]}"; do
mservice=$(get_master_service_for_service "$service") || exit 1
[ "${seen[$mservice]}" ] && continue
type="$(get_service_type "$mservice")" || exit 1
## remove run-once
[ "$type" == "run-once" ] && continue
[ "$type" == "stub" ] && continue
seen[$mservice]=1
echo "$mservice"
done > "$cache_file".wip
mv "$cache_file".wip "$cache_file"
cat "$cache_file"
}
export -f services:get:upable
service:state() {
local service="$1" states state
project_name=$(get_default_project_name) || return 1
@ -2819,7 +2882,11 @@ export -f _get_services_provides
_get_charm_provides() {
local cache_file="$CACHEDIR/$FUNCNAME.cache.$(charm.store_metadata_hash)" errlvl
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
local cache_file="$CACHEDIR/$FUNCNAME.cache.$CHARM_STORE_HASH" errlvl
if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: SESSION cache hit"
cat "$cache_file"
@ -3615,13 +3682,21 @@ export -f get_relation_data_file
has_service_action () {
local service="$1" action="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Can't access global \$CHARM_STORE_HASH"
return 1
fi
local service="$1" action="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$2.$CHARM_STORE_HASH" \
charm target_charm relation_name target_service relation_config _tech_dep \
path
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
cat "$cache_file"
return 0
if [ -s "$cache_file" ]; then
cat "$cache_file"
return 0
else
return 1
fi
fi
charm=$(get_service_charm "$service") || return 1
@ -3643,6 +3718,7 @@ has_service_action () {
fi
done < <(get_service_relations "$service")
touch "$cache_file"
return 1
# master=$(get_top_master_service_for_service "$service")
# [ "$master" == "$charm" ] && return 1
@ -3762,7 +3838,11 @@ export -f _get_master_service_for_service_cached
get_master_service_for_service() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH" \
charm metadata result
if [ -e "$cache_file" ]; then
@ -3782,11 +3862,16 @@ export -f get_master_service_for_service
get_top_master_service_for_service() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
if [ -z "$CHARM_STORE_HASH" ]; then
err-d "Expected \$CHARM_STORE_HASH to be set."
return 1
fi
local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH" \
current_service
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
touch "$cache_file" || return 1
cat "$cache_file"
return 0
fi
@ -4348,6 +4433,12 @@ export -f get_default_target_services
get_master_services() {
local loaded master_service service
local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" )"
if [ -e "$cache_file" ]; then
cat "$cache_file" &&
touch "$cache_file" || return 1
return 0
fi
declare -A loaded
for service in "$@"; do
master_service=$(get_top_master_service_for_service "$service") || return 1
@ -4356,8 +4447,10 @@ get_master_services() {
fi
echo "$master_service"
loaded["$master_service"]=1
done | nspc
return "${PIPESTATUS[0]}"
done > "$cache_file".wip || return 1
mv "$cache_file"{.wip,} || return 1
cat "$cache_file" || return 1
}
export -f get_master_services
@ -5264,12 +5357,12 @@ aexport remainder_args
## Actual code
##
COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1
COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1
COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || exit 1
export COMPOSE_YML_FILE COMPOSE_YML_CONTENT COMPOSE_YML_CONTENT_HASH
CHARM_STORE_HASH=$(charm.store_metadata_hash) || exit 1
COMBINED_HASH=$(H "$COMPOSE_YML_CONTENT_HASH" "$CHARM_STORE_HASH") || exit 1
export COMPOSE_YML_FILE COMPOSE_YML_CONTENT COMPOSE_YML_CONTENT_HASH CHARM_STORE_HASH COMBINED_HASH
charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct."
##
@ -5445,18 +5538,7 @@ fi
if [ -n "$is_docker_compose_action" ] && [ "${#services_args[@]}" -gt 0 ]; then
services=($(get_master_services "${services_args[@]}")) || exit 1
if [ "$action" == "up" ]; then
declare -A seen
services=($(get_ordered_service_dependencies "${services_args[@]}")) || exit 1
for service in "${services[@]}"; do
mservice=$(get_master_service_for_service "$service") || exit 1
[ "${seen[$mservice]}" ] && continue
type="$(get_service_type "$mservice")" || exit 1
## remove run-once
[ "$type" == "run-once" ] && continue
[ "$type" == "stub" ] && continue
seen[$mservice]=1
action_posargs+=("$mservice")
done
action_posargs+=($(services:get:upable "${services_args[@]}")) || exit 1
elif [ "$is_docker_compose_action_multi_service" == "1" ]; then
action_posargs+=("${services[@]}")
elif [ "$is_docker_compose_action_multi_service" == "0" ]; then

Write Preview
Loading…
Cancel
Save