0k
/
0k-compose
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

wip

cache-relation
Valentin Lab 2 months ago
parent
ee14f3d953
commit
660074ef1d
1 changed files with 285 additions and 33 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 318
      bin/compose-core

318
bin/compose-core
View File

@ -82,6 +82,9 @@ $WHITE$exname$NORMAL reads '/etc/compose.conf' for global variables, and
" "
time_now() { date +%s.%3N; }
time_elapsed() { echo "scale=3; $2 - $1" | bc; }
## XXXvlab: this doesn't seem to work when 'compose' is called in ## XXXvlab: this doesn't seem to work when 'compose' is called in
## a hook of a charm. ## a hook of a charm.
#[[ "${BASH_SOURCE[0]}" == "" ]] && SOURCED=true #[[ "${BASH_SOURCE[0]}" == "" ]] && SOURCED=true
@ -1454,7 +1457,6 @@ export -f _get_service_charm_cached
get_service_charm () { get_service_charm () {
local service="$1" local service="$1"
if [ -z "$service" ]; then if [ -z "$service" ]; then
echo ${FUNCNAME[@]} >&2
print_syntax_error "$FUNCNAME: Please specify a service as first argument." print_syntax_error "$FUNCNAME: Please specify a service as first argument."
return 1 return 1
fi fi
@ -2474,10 +2476,15 @@ export -f get_compose_relations
get_all_services() { get_all_services() {
local cache_file="$state_tmpdir/$FUNCNAME.cache.$GLOBAL_ALL_RELATIONS_HASH" \
local services compose_yml_services service
if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then
err-d "Can't access global \$GLOBAL_ALL_RELATIONS_HASH"
return 1
fi
local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$GLOBAL_ALL_RELATIONS_HASH" "$(declare -f "$FUNCNAME")")" \
s rn ts rc td services service s rn ts rc td services service
if [ -e "$cache_file" ]; then if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: SESSION cache hit $1"
#debug "$FUNCNAME: cache hit $1"
cat "$cache_file" cat "$cache_file"
return 0 return 0
fi fi
@ -2494,8 +2501,17 @@ get_all_services() {
services["$service"]=1 services["$service"]=1
echo "$service" echo "$service"
done done
done < "$GLOBAL_ALL_RELATIONS" > "$cache_file"
done < "$GLOBAL_ALL_RELATIONS" > "$cache_file.wip"
compose_yml_services=($(compose:yml:root:services)) || return 1
for service in "${compose_yml_services[@]}"; do
[ "${services[$service]}" ] && continue
services["$service"]=1
echo "$service"
done >> "$cache_file.wip"
mv "$cache_file"{.wip,} || return 1
cat "$cache_file" cat "$cache_file"
} }
export -f get_all_services export -f get_all_services
@ -2559,28 +2575,25 @@ export -f get_service_relation
## (base_service, relation_config) ## (base_service, relation_config)
## ##
get_service_incoming_relations() { get_service_incoming_relations() {
local service="$1" relation="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH")" \
s rn ts rc td
if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
err-d "Can't access global \$GLOBAL_ALL_RELATIONS"
if [ -z "$SUBSET_ALL_RELATIONS_HASH" ]; then
err-d "Expected \$SUBSET_ALL_RELATIONS_HASH to be set."
return 1 return 1
fi fi
local service="$1" relation="$2" \
cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$SUBSET_ALL_RELATIONS_HASH")" \
s rn ts rc td
if [ -e "$cache_file" ]; then if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: SESSION cache hit $1" #debug "$FUNCNAME: SESSION cache hit $1"
cat "$cache_file" cat "$cache_file"
return 0 return 0
fi fi
if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
err "Can't access global \$GLOBAL_ALL_RELATIONS"
return 1
fi
while read-0 s rn ts rc _td; do while read-0 s rn ts rc _td; do
[[ "$ts" == "$service" ]] || continue [[ "$ts" == "$service" ]] || continue
[[ "$rn" == "$relation" ]] || continue [[ "$rn" == "$relation" ]] || continue
relation_data_file=$(get_relation_data_file "$s" "$ts" "$rn" "$rc") || return 1 relation_data_file=$(get_relation_data_file "$s" "$ts" "$rn" "$rc") || return 1
printf "%s\0" "$s" "$(cat "$relation_data_file")"
done < "$GLOBAL_ALL_RELATIONS" > "$cache_file"
printf "%s\0" "$s" "$(cat "$relation_data_file")" || return 1
debug "Found relation $rn from $s to $ts" >&2
done < "$SUBSET_ALL_RELATIONS" > "$cache_file"
cat "$cache_file" cat "$cache_file"
} }
@ -2921,14 +2934,170 @@ _out_after_value_from_def() {
} }
get_all_relations () {
local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$(declare -p without_relations)")" \
services
get_all_compose_yml_service() {
if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || {
err "Failed to get compose yml hash"
return 1
}
fi
local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH"
if [ -e "${cache_file}" ]; then
#debug "$FUNCNAME: cache hit: ${cache_file}"
cat "${cache_file}"
return 0
fi
compose_yml_content=$(get_compose_yml_content) || return 1
printf "%s" "${compose_yml_content}" | shyaml keys-0 2>/dev/null > "${cache_file}.wip" || {
err "Failed to get keys of compose content."
return 1
}
mv "${cache_file}"{.wip,} || return 1
cat "${cache_file}"
}
## Outputs all relations array.
_service:all:relations_cached() {
local services service E
services=($(compose:yml:root:services)) || return 1
get_all_relations "${services[@]}" || return 1
}
## Outputs all relations array.
service:all:relations() {
if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || {
err "Failed to get compose yml hash."
return 1
}
fi
local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH"
if [ -e "${cache_file}" ]; then
# debug "$FUNCNAME: SESSION cache hit $1"
cat "${cache_file}"
return 0
fi
_service:all:relations_cached > "${cache_file}.wip" || {
err-d "Failed to compute all relations."
return 1
}
mv "${cache_file}"{.wip,} || return 1
cat "${cache_file}"
}
_service:all:relations_hash_cached() {
if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || {
err "Failed to get compose yml hash."
return 1
}
fi
local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH" \
hash
if [ -e "${cache_file}" ]; then
# debug "$FUNCNAME: SESSION cache hit $1"
cat "${cache_file}"
return 0
fi
service:all:relations > "${cache_file}.pre" || {
err-d "Failed to get all relations."
return 1
}
{
p0 "$(hash_get < "${cache_file}.pre")" || return 1
cat "${cache_file}.pre"
rm "${cache_file}.pre"
} > "${cache_file}".wip || return 1
mv "${cache_file}"{.wip,} || return 1
cat "${cache_file}"
}
## Get all relations from all services in the current compose file.
## Sets GLOBAL_ALL_RELATIONS_HASH and returns all relations array.
service:all:set_relations_hash() {
if [ -n "$GLOBAL_ALL_RELATIONS" ]; then
if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then
err "Can't access global \$GLOBAL_ALL_RELATIONS_HASH"
echo " (despite \$GLOBAL_ALL_RELATIONS being set)" >&2
return 1
fi
return 0
fi
## sets COMPOSE_YML_CONTENT_HASH
_service:all:relations_hash_cached >/dev/null || return 1
{
read-0 GLOBAL_ALL_RELATIONS_HASH || return 1
export GLOBAL_ALL_RELATIONS_HASH
## transfer to statedir
export GLOBAL_ALL_RELATIONS="$state_tmpdir/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH"
cat > "$GLOBAL_ALL_RELATIONS"
} < <(_service:all:relations_hash_cached)
if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
err "Failed to set \$GLOBAL_ALL_RELATIONS."
return 1
fi
if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then
err "Failed to set \$GLOBAL_ALL_RELATIONS_HASH."
return 1
fi
}
get_subset_relations () {
local service all_services services start
if [ -n "$SUBSET_ALL_RELATIONS" ]; then
return 0
fi
if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then
err-d "Can't access global \$GLOBAL_ALL_RELATIONS_HASH"
return 1
fi
cache_hash=$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH" "$(declare -f get_subset_relations)")
local cache_file="$CACHEDIR/$FUNCNAME.cache.$cache_hash"
if [ -e "${cache_file}" ]; then
export SUBSET_ALL_RELATIONS="$cache_file"
hash=$(hash_get < "$cache_file") || return 1
export SUBSET_ALL_RELATIONS_HASH="$hash"
cat "${cache_file}"
return 0
fi
## collect all connected services first
all_services=("$@")
while [ "${#all_services[@]}" != 0 ]; do
array_pop all_services service
#debug " Getting relations for $DARKYELLOW$service$NORMAL"
while read-0 s rn ts rc td; do
[[ "$s" == "$service" ]] || continue
#debug " adding relation $DARKBLUE$rn$NORMAL to $DARKYELLOW$ts$NORMAL"
p0 "$service" "$rn" "$ts" "$rc" "$td"
[ -n "${services[$ts]}" ] || all_services+=("$ts")
done < "$GLOBAL_ALL_RELATIONS"
services["$service"]=1
done > "$cache_file.wip"
mv "$cache_file"{.wip,} || return 1
export SUBSET_ALL_RELATIONS="$cache_file"
hash=$(hash_get < "$cache_file") || return 1
export SUBSET_ALL_RELATIONS_HASH="$hash"
cat "$cache_file"
}
get_all_relations () {
if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || return 1
fi
if [ -n "$GLOBAL_ALL_RELATIONS" ]; then if [ -n "$GLOBAL_ALL_RELATIONS" ]; then
cat "$GLOBAL_ALL_RELATIONS" || return 1 cat "$GLOBAL_ALL_RELATIONS" || return 1
return 0 return 0
fi fi
local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$COMPOSE_YML_CONTENT_HASH" "$(declare -p without_relations)")" \
services all_services service services_uses services_provides \
changed summon required recommended optional
if [ -e "${cache_file}" ]; then if [ -e "${cache_file}" ]; then
#debug "$FUNCNAME: SESSION cache hit $1" #debug "$FUNCNAME: SESSION cache hit $1"
@ -3176,7 +3345,7 @@ get_all_relations () {
while read-0 p s rn ts rc td; do while read-0 p s rn ts rc td; do
if [ -z "$p" ] || [ "$p" == "," ]; then if [ -z "$p" ] || [ "$p" == "," ]; then
relation_done["$s:$rn"]=1 relation_done["$s:$rn"]=1
# printf " .. %-30s %-30s %-30s\n" "--" "$s" "$rn" >&2
# printf " .. %-30s %-30s %-30s\n" "$s" "$ts" "$rn" >&2
printf "%s\0" "$s" "$rn" "$ts" "$rc" "$td" >> "${cache_file}.wip.final" printf "%s\0" "$s" "$rn" "$ts" "$rc" "$td" >> "${cache_file}.wip.final"
had_new_relation=1 had_new_relation=1
else else
@ -4080,7 +4249,7 @@ get_compose_yml_content() {
COMPOSE_YML_FILE=$(get_compose_yml_location) || return 1 COMPOSE_YML_FILE=$(get_compose_yml_location) || return 1
fi fi
if [ -e "$COMPOSE_YML_FILE" ]; then if [ -e "$COMPOSE_YML_FILE" ]; then
debug "Found $WHITE$exname$NORMAL YAML file in '$COMPOSE_YML_FILE'."
# debug "Found $WHITE$exname$NORMAL YAML file in '$COMPOSE_YML_FILE'."
COMPOSE_YML_CONTENT=$(cat "$COMPOSE_YML_FILE") || { COMPOSE_YML_CONTENT=$(cat "$COMPOSE_YML_FILE") || {
err "Could not read '$COMPOSE_YML_FILE'." err "Could not read '$COMPOSE_YML_FILE'."
return 1 return 1
@ -4111,6 +4280,37 @@ get_compose_yml_content() {
} }
export -f get_compose_yml_content export -f get_compose_yml_content
compose:yml:hash() {
local cache_file="$state_tmpdir/$FUNCNAME.cache"
if [ -e "$cache_file" ]; then
cat "$cache_file" &&
touch "$cache_file" || return 1
return 0
fi
compose_yml_content=$(get_compose_yml_content) || return 1
compose_yml_hash=$(echo "$compose_yml_content" | hash_get) || return 1
e "$compose_yml_hash" | tee "$cache_file" || return 1
}
export -f compose:yml:hash
compose:yml:root:services() {
local cache_file="$state_tmpdir/$FUNCNAME.cache" services compose_yml_content
if [ -e "$cache_file" ]; then
cat "$cache_file" &&
touch "$cache_file" || return 1
return 0
fi
compose_yml_content=$(get_compose_yml_content) || return 1
services=($(e "$compose_yml_content" | shyaml keys)) || return 1
e "${services[*]}" | tee "$cache_file" || return 1
}
export -f compose:yml:root:services
get_default_target_services() { get_default_target_services() {
local services=("$@") local services=("$@")
@ -4349,9 +4549,12 @@ display_commands_help() {
get_docker_charm_action() { get_docker_charm_action() {
local services service charm relation_name target_service relation_config \ local services service charm relation_name target_service relation_config \
target_charm
services=($(get_compose_yml_content | yq -r 'keys().[]' 2>/dev/null)) || return 1
NO_CONSTRAINT_CHECK=1 get_all_relations "${services[@]}" >/dev/null || return 1
target_charm services
## XXXvlab: this is for get_service_relations
NO_CONSTRAINT_CHECK=True service:all:set_relations_hash || {
err-d "Failed to set relations hash."
return 1
}
services=($(get_all_services)) || return 1 services=($(get_all_services)) || return 1
for service in "${services[@]}"; do for service in "${services[@]}"; do
printf "%s:\n" "$service" printf "%s:\n" "$service"
@ -4375,8 +4578,12 @@ export -f get_docker_charm_action
get_docker_charm_action_help() { get_docker_charm_action_help() {
local services service charm relation_name target_service relation_config \ local services service charm relation_name target_service relation_config \
target_charm target_charm
services=($(get_compose_yml_content | shyaml keys 2>/dev/null))
NO_CONSTRAINT_CHECK=1 get_all_relations "${services[@]}" >/dev/null || return 1
## XXXvlab: this is for get_service_relations
NO_CONSTRAINT_CHECK=True service:all:set_relations_hash || {
err-d "Failed to set relations hash."
return 1
}
services=($(get_all_services)) || return 1
for service in "${services[@]}"; do for service in "${services[@]}"; do
out=$( out=$(
charm=$(get_service_charm "$service") || return 1 charm=$(get_service_charm "$service") || return 1
@ -4667,6 +4874,7 @@ no_init=
action= action=
stage="main" ## switches from 'main', to 'action', 'remainder' stage="main" ## switches from 'main', to 'action', 'remainder'
is_docker_compose_action= is_docker_compose_action=
is_docker_compose_action_multi_service=
rebuild_relations_to_service=() rebuild_relations_to_service=()
color= color=
declare -A without_relations declare -A without_relations
@ -4806,6 +5014,12 @@ while read-0 arg; do
fi fi
pos_args=($(echo "$DC_USAGE" | sed -r 's/\[-[^]]+\] ?//g;s/\[options\] ?//g')) pos_args=($(echo "$DC_USAGE" | sed -r 's/\[-[^]]+\] ?//g;s/\[options\] ?//g'))
pos_args=("${pos_args[@]:1}") pos_args=("${pos_args[@]:1}")
if [[ "${pos_args[0]}" == "[SERVICE...]" ]]; then
is_docker_compose_action_multi_service=1
elif [[ "${pos_args[0]}" == "SERVICE" ]]; then
is_docker_compose_action_multi_service=0
fi
# echo "USAGE: $DC_USAGE" # echo "USAGE: $DC_USAGE"
# echo "pos_args: ${pos_args[@]}" # echo "pos_args: ${pos_args[@]}"
# echo "MULTI: $DC_MATCH_MULTI" # echo "MULTI: $DC_MATCH_MULTI"
@ -4916,8 +5130,8 @@ aexport remainder_args
COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1 COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1
COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1 COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1
export COMPOSE_YML_FILE COMPOSE_YML_CONTENT
COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || exit 1
export COMPOSE_YML_FILE COMPOSE_YML_CONTENT COMPOSE_YML_CONTENT_HASH
charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct." charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct."
@ -4933,10 +5147,9 @@ if [ -z "$is_docker_compose_action" -a "$action" ]; then
exit 1 exit 1
fi fi
services_args=($(compose:yml:root:services)) || return 1
## Required by has_service_action ## Required by has_service_action
array_read-0 services_args < <(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys-0 2>/dev/null)
NO_CONSTRAINT_CHECK=1 get_all_relations "${services_args[@]}" >/dev/null || exit 1
service:all:set_relations_hash
remainder_args=("${remainder_args[@]:1}") remainder_args=("${remainder_args[@]:1}")
if has_service_action "$action_service" "$action" >/dev/null; then if has_service_action "$action_service" "$action" >/dev/null; then
@ -4971,7 +5184,7 @@ else
case "$action" in case "$action" in
ps|up) ps|up)
if [ "${#services_args[@]}" == 0 ]; then if [ "${#services_args[@]}" == 0 ]; then
array_read-0 services_args < <(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys-0 2>/dev/null)
services_args=($(compose:yml:root:services)) || return 1
fi fi
;; ;;
config) config)
@ -4985,11 +5198,48 @@ NO_CONSTRAINT_CHECK=True
case "$action" in case "$action" in
up) up)
NO_CONSTRAINT_CHECK= NO_CONSTRAINT_CHECK=
if [ -n "$DEBUG" ]; then
Elt "solve all relations"
start=$(time_now)
fi
service:all:set_relations_hash || exit 1
if [ -n "$DEBUG" ]; then
elapsed="$(time_elapsed $start "$(time_now)")" || exit 1
print_info "$(printf "%.3fs" "$elapsed")"
Feedback
fi
all_services=($(get_all_services)) || exit 1
## check that services_args is a subset of all_services
for service in "${services_args[@]}"; do
[[ " ${all_services[*]} " == *" $service "* ]] || {
err "Service ${DARKYELLOW}$service${NORMAL} is not defined in the current compose file."
echo " Neither is is a dependency of a service in the compose file." >&2
echo " These are the services directly or indirectly available from current compose file:" >&2
for service in "${all_services[@]}"; do
echo " - ${DARKYELLOW}$service${NORMAL}" >&2
done
exit 1
}
done
;; ;;
esac esac
if [ -n "$is_docker_compose_action_multi_service" ]; then
if [ -n "$DEBUG" ]; then
Elt "get relation subet"
start=$(time_now)
fi
get_all_relations "${services_args[@]}" >/dev/null || exit 1
get_subset_relations "${services_args[@]}" >/dev/null || exit 1
if [ -n "$DEBUG" ]; then
elapsed="$(time_elapsed $start "$(time_now)")" || exit 1
print_info "$(printf "%.3fs" "$elapsed")"
Feedback
fi
fi
if [ "$is_docker_compose_action" -a "${#services_args[@]}" -gt 0 ]; then if [ "$is_docker_compose_action" -a "${#services_args[@]}" -gt 0 ]; then
services=($(get_master_services "${services_args[@]}")) || exit 1 services=($(get_master_services "${services_args[@]}")) || exit 1
@ -5006,8 +5256,10 @@ if [ "$is_docker_compose_action" -a "${#services_args[@]}" -gt 0 ]; then
seen[$mservice]=1 seen[$mservice]=1
action_posargs+=("$mservice") action_posargs+=("$mservice")
done done
else
elif [ "$is_docker_compose_action_multi_service" == "1" ]; then
action_posargs+=("${services[@]}") action_posargs+=("${services[@]}")
elif [ "$is_docker_compose_action_multi_service" == "0" ]; then
action_posargs+=("${services[0]}") ## only the first service is the legit one
fi fi
## Get rid of subordinates ## Get rid of subordinates
action_posargs=($(get_master_services "${action_posargs[@]}")) || exit 1 action_posargs=($(get_master_services "${action_posargs[@]}")) || exit 1

Write Preview
Loading…
Cancel
Save