0k
/
0k-compose
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

new: static caching system.

raw-remaining-args
Valentin Lab 9 years ago
parent
5dbf63f865
commit
59501a990c
1 changed files with 150 additions and 82 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 232
      bin/compose

232
bin/compose
View File

@ -8,13 +8,20 @@
include pretty
include parse
md5_compat() { md5sum | cut -c -32; }
depends shyaml docker
[[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true
export CACHEDIR=/var/cache/compose
export VARDIR=/var/lib/compose
mkdir -p "$CACHEDIR" || exit 1
trap_add "EXIT" clean_cache
usage="$exname CHARM"'
Deploy and manage a swarm of containers to provide services based on
@ -195,7 +202,7 @@ file_put() {
export -f file_put
_get_docker_compose_links() {
get_docker_compose_links() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
links charm charm_part master_charm
if [ -z "$service" ]; then
@ -209,11 +216,11 @@ _get_docker_compose_links() {
return 0
fi
master_charm=$(_get_top_master_charm_for_service "$service") || return 1
master_charm=$(get_top_master_charm_for_service "$service") || return 1
deps=()
while read-0 relation_name target_service relation_config tech_dep; do
master_target_charm="$(_get_top_master_charm_for_service "$target_service")"
master_target_charm="$(get_top_master_charm_for_service "$target_service")"
[ "$master_charm" == "$master_target_charm" ] && continue
if [ "$tech_dep" == "reversed" ]; then
deps+=("$(echo -en "$master_target_charm:\n links:\n - $master_charm")")
@ -241,7 +248,7 @@ _get_docker_compose_opts() {
fi
compose_def="$(get_compose_service_def "$service")" || return 1
master_charm="$(_get_top_master_charm_for_service "$service")"
master_charm="$(get_top_master_charm_for_service "$service")"
docker_compose_opts=$(echo "$compose_def" | shyaml get-value "docker-compose" 2>/dev/null)
if [ "$docker_compose_opts" ]; then
@ -267,11 +274,11 @@ _get_docker_compose_service_mixin() {
return 0
fi
master_charm=$(_get_top_master_charm_for_service "$service") || return 1
master_charm=$(get_top_master_charm_for_service "$service") || return 1
## The compose part
links_yaml=$(_get_docker_compose_links "$service") || return 1
links_yaml=$(get_docker_compose_links "$service") || return 1
docker_compose_options=$(_get_docker_compose_opts "$service") || return 1
## the charm part
@ -283,7 +290,7 @@ _get_docker_compose_service_mixin() {
## Merge results
if [ "$charm_part" ]; then
charm_yaml="$(yaml_key_val_str "$master_charm" "$charm_part")" || return 1
merge_yaml_str "$links_yaml" "$charm_yaml" "$docker_compose_options"
merge_yaml_str "$links_yaml" "$charm_yaml" "$docker_compose_options" || return 1
else
echo "$links_yaml"
fi | tee "$cache_file"
@ -344,50 +351,49 @@ get_docker_compose () {
export -f get_docker_compose
## XXXvlab: a lot to be done to cache the results
get_compose_service_def () {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
_get_compose_service_def_cached () {
local service="$1" docker_compose="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
#debug "$FUNCNAME: STATIC cache hit"
cat "$cache_file"
touch "$cache_file"
return 0
fi
[ -z "$service" ] && print_syntax_error "Missing service as first argument."
service_def_base="charm: $service"
value=$(echo "$docker_compose" | shyaml get-value "$service" 2>/dev/null)
merge_yaml <(echo "$service_def_base") <(echo "$value") | tee "$cache_file"
}
export -f _get_compose_service_def_cached
service_def_base=
if [ -d "$CHARM_STORE/$service" ]; then
service_def_base="charm: $service"
fi
value=
if [ -r "$COMPOSE_YML_FILE" ]; then
value=$(shyaml get-value "$service" 2>/dev/null < "$COMPOSE_YML_FILE")
fi
if [ -z "$service_def_base" -a -z "$value" ]; then
err "Invalid service $DARKYELLOW$service$NORMAL: no definition in" \
"compose file nor charm with same name."
return 1
## XXXvlab: a lot to be done to cache the results
get_compose_service_def () {
local service="$1" docker_compose cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
result
if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: SESSION cache hit"
cat "$cache_file"
return 0
fi
merge_yaml <(echo "$service_def_base") <(echo "$value") > "$cache_file"
cat "$cache_file"
[ -z "$service" ] && print_syntax_error "Missing service as first argument."
docker_compose=$(cat "$COMPOSE_YML_FILE") || return 1
result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1
echo "$result" | tee "$cache_file"
}
export -f get_compose_service_def
get_service_charm () {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
_get_service_charm_cached () {
local service="$1" service_def="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
# debug "$FUNCNAME: cache hit $1"
cat "$cache_file"
touch "$cache_file"
return 0
fi
if [ -z "$service" ]; then
print_syntax_error "$FUNCNAME: Please specify a service as first argument."
return 1
fi
service_def=$(get_compose_service_def "$service") || return 1
charm=$(echo "$service_def" | shyaml get-value charm 2>/dev/null)
if [ -z "$charm" ]; then
err "Missing charm in service $DARKYELLOW$service$NORMAL definition."
@ -395,6 +401,17 @@ get_service_charm () {
fi
echo "$charm" | tee "$cache_file"
}
export -f _get_service_charm_cached
get_service_charm () {
local service="$1"
if [ -z "$service" ]; then
print_syntax_error "$FUNCNAME: Please specify a service as first argument."
return 1
fi
service_def=$(get_compose_service_def "$service") || return 1
_get_service_charm_cached "$service" "$service_def"
}
export -f get_service_charm
## built above the docker-compose abstraction, so it relies on the
@ -425,7 +442,7 @@ service_base_docker_image() {
cat "$cache_file"
return 0
fi
master_charm="$(_get_top_master_charm_for_service "$service")" || {
master_charm="$(get_top_master_charm_for_service "$service")" || {
err "Could not compute base charm for service $DARKYELLOW$service$NORMAL."
return 1
}
@ -596,7 +613,7 @@ export -f array_member
get_charm_relation_def () {
local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
relation_def metadata
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
@ -611,7 +628,7 @@ export -f get_charm_relation_def
get_charm_tech_dep_orientation_for_relation() {
local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
relation_def metadata value
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
@ -859,8 +876,8 @@ _run_service_relation () {
export BASE_CHARM_NAME=$charm
export TARGET_CHARM_NAME=$target_charm
PROJECT_NAME=$(get_default_project_name) || return 1
MASTER_BASE_CHARM_NAME=$(_get_top_master_charm_for_service "$service") || return 1
MASTER_TARGET_CHARM_NAME=$(_get_top_master_charm_for_service "$target_service") || return 1
MASTER_BASE_CHARM_NAME=$(get_top_master_charm_for_service "$service") || return 1
MASTER_TARGET_CHARM_NAME=$(get_top_master_charm_for_service "$target_service") || return 1
export RELATION_DATA_FILE RELATION_BASE_COMPOSE_DEF RELATION_TARGET_COMPOSE_DEF
export MASTER_BASE_CHARM_NAME MASTER_TARGET_CHARM_NAME PROJECT_NAME
target_errlvl=0
@ -947,19 +964,19 @@ _run_service_relation () {
export -f _run_service_relation
get_compose_relations () {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" relation_name relation_def
_get_compose_relations_cached () {
local compose_service_def="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
relation_name relation_def target_service
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
#debug "$FUNCNAME: STATIC cache hit $1"
cat "$cache_file"
touch "$cache_file"
return 0
fi
compose_def="$(get_compose_service_def "$service")" || return 1
(
set -o pipefail
if [ "$compose_def" ]; then
if [ "$compose_service_def" ]; then
while read-0 relation_name relation_def; do
(
case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in
@ -982,7 +999,7 @@ get_compose_relations () {
;;
esac
) </dev/null >> "$cache_file"
done < <(echo "$compose_def" | shyaml key-values-0 relations 2>/dev/null)
done < <(echo "$compose_service_def" | shyaml key-values-0 relations 2>/dev/null)
fi
)
if [ "$?" != 0 ]; then
@ -993,6 +1010,27 @@ get_compose_relations () {
[ -e "$cache_file" ] && cat "$cache_file"
return 0
}
export -f _get_compose_relations_cached
get_compose_relations () {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
compose_def
if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: SESSION cache hit $1"
cat "$cache_file"
return 0
fi
compose_def="$(get_compose_service_def "$service")" || return 1
_get_compose_relations_cached "$compose_def" > "$cache_file"
if [ "$?" != 0 ]; then
err "Error while looking for compose relations."
rm -f "$cache_file" ## no cache
return 1
fi
cat "$cache_file"
}
export -f get_compose_relations
run_service_relations () {
@ -1038,7 +1076,7 @@ _run_service_action_direct() {
export METADATA_CONFIG=$(cat "$CHARM_STORE/$charm/metadata.yml")
export SERVICE_NAME=$service
export ACTION_NAME=$action
export CONTAINER_NAME=$(_get_top_master_charm_for_service "$service")
export CONTAINER_NAME=$(get_top_master_charm_for_service "$service")
export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME")
export SERVICE_DATASTORE="$DATASTORE/$service"
export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
@ -1081,7 +1119,7 @@ _run_service_action_relation() {
export RELATION_TARGET_CHARM="$target_charm"
export RELATION_CHARM="$charm"
export ACTION_NAME=$action
export CONTAINER_NAME=$(_get_top_master_charm_for_service "$service")
export CONTAINER_NAME=$(get_top_master_charm_for_service "$service")
export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME")
export SERVICE_DATASTORE="$DATASTORE/$service"
export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
@ -1174,7 +1212,7 @@ has_service_action () {
fi
done < <(get_compose_relations "$service")
master=$(_get_top_master_charm_for_service "$charm")
master=$(get_top_master_charm_for_service "$charm")
[ "$master" == "$charm" ] && return 1
has_service_action "$master" "$action"
@ -1239,23 +1277,19 @@ export -f get_compose_relation_config
# export -f get_compose_relation_config_for_service
_get_master_charm_for_service() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
charm metadata requires master_charm target_charm target_service service_def
_get_master_charm_for_service_cached () {
local service="$1" charm="$2" metadata="$3" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
charm requires master_charm target_charm target_service service_def
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
# debug "$FUNCNAME: STATIC cache hit ($1)"
cat "$cache_file"
touch "$cache_file"
return 0
fi
charm=$(get_service_charm "$service") || return 1
metadata=$(get_charm_metadata "$charm") || return 1
if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" != "True" ]; then
## just return charm name
echo "$charm" > "$cache_file"
echo "$charm"
echo "$charm" | tee "$cache_file"
return 0
fi
@ -1298,14 +1332,29 @@ _get_master_charm_for_service() {
die "Charm $DARKYELLOW$charm$NORMAL is a subordinate but does not have any relation with" \
" ${WHITE}scope${NORMAL} set to 'container'."
fi
echo "$master_charm" > "$cache_file"
echo "$master_charm"
return 0
echo "$master_charm" | tee "$cache_file"
}
export -f _get_master_charm_for_service
export -f _get_master_charm_for_service_cached
get_master_charm_for_service() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
charm metadata result
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: SESSION cache hit ($*)"
cat "$cache_file"
return 0
fi
_get_top_master_charm_for_service() {
charm=$(get_service_charm "$service") || return 1
metadata=$(get_charm_metadata "$charm") || return 1
result=$(_get_master_charm_for_service_cached "$service" "$charm" "$metadata") || return 1
echo "$result" | tee "$cache_file"
}
export -f get_master_charm_for_service
get_top_master_charm_for_service() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
current_service
@ -1317,14 +1366,14 @@ _get_top_master_charm_for_service() {
current_service="$service"
while true; do
master_service=$(_get_master_charm_for_service "$current_service") || return 1
master_service=$(get_master_charm_for_service "$current_service") || return 1
[ "$master_service" == "$current_service" ] && break
current_service="$master_service"
done
echo "$current_service" | tee "$cache_file"
return 0
}
export -f _get_top_master_charm_for_service
export -f get_top_master_charm_for_service
get_charm_metadata() {
@ -1342,18 +1391,17 @@ export -f get_charm_metadata
## The result is a mixin that is not always a complete valid
## docker-compose entry (thinking of subordinates). The result
## will be merge with master charms.
get_docker_compose_mixin_from_metadata() {
local charm="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
_get_docker_compose_mixin_from_metadata_cached() {
local charm="$1" metadata="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
metadata_file metadata volumes docker_compose subordinate image
if [ -e "$cache_file" ]; then
debug "$FUNCNAME: cache hit ($*)"
#debug "$FUNCNAME: STATIC cache hit $1"
cat "$cache_file"
touch "$cache_file"
return 0
fi
mixin=
metadata="$(get_charm_metadata "$charm")"
if [ "$metadata" ]; then
## resources to volumes
volumes=$(
@ -1370,11 +1418,6 @@ get_docker_compose_mixin_from_metadata() {
fi
done < <(echo "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null)
while read-0 resource; do
if ! [ -e "$CHARM_STORE/$charm/resources$resource" ]; then
err "No '$resource' resource found in ${BLUE}resources/${NORMAL}" \
"directory of charm $DARKYELLOW$charm$NORMAL."
exit 1
fi
echo " - $CHARM_STORE/$charm/resources$resource:$resource:rw"
done < <(echo "$metadata" | shyaml get-values-0 "charm-resources" 2>/dev/null)
) || return 1
@ -1410,8 +1453,22 @@ get_docker_compose_mixin_from_metadata() {
if [ "$image_or_build_statement" ]; then
mixin=$(merge_yaml_str "$mixin" "$image_or_build_statement")
fi
echo "$mixin" > "$cache_file"
echo "$mixin"
echo "$mixin" | tee "$cache_file"
}
export -f _get_docker_compose_mixin_from_metadata_cached
get_docker_compose_mixin_from_metadata() {
local charm="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: SESSION cache hit ($*)"
cat "$cache_file"
return 0
fi
metadata="$(get_charm_metadata "$charm")" || return 1
mixin=$(_get_docker_compose_mixin_from_metadata_cached "$charm" "$metadata") || return 1
echo "$mixin" | tee "$cache_file"
}
export -f get_docker_compose_mixin_from_metadata
@ -1490,7 +1547,6 @@ launch_docker_compose() {
export -f launch_docker_compose
get_compose_yml_location() {
parent=$(while ! [ -e "./compose.yml" ]; do
[ "$PWD" == "/" ] && exit 0
cd ..
@ -1537,7 +1593,7 @@ get_master_services() {
local loaded master_service
declare -A loaded
for service in "$@"; do
master_service=$(_get_top_master_charm_for_service "$service") || return 1
master_service=$(get_top_master_charm_for_service "$service") || return 1
if [ "${loaded[$master_service]}" ]; then
continue
fi
@ -1559,7 +1615,6 @@ _setup_state_dir() {
get_docker_compose_opts_list() {
local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
if [ -e "$cache_file" ]; then
debug "$FUNCNAME: cache hit ($*)"
cat "$cache_file"
@ -1588,6 +1643,16 @@ get_docker_compose_single_opts_list() {
tr ',' "\n" | xargs echo
}
clean_cache() {
local i=0
for f in $(ls -t "$CACHEDIR/"*.cache.* | tail -n +500); do
((i++))
rm -f "$f"
done
if (( i > 0 )); then
debug "${WHITE}Cleaned cache:${NORMAL} Removed $((i)) elements (current cache size is $(du -sh "$CACHEDIR" | cut -f 1))"
fi
}
[ "$SOURCED" ] && return 0
@ -1750,7 +1815,7 @@ esac
get_docker_compose $services >/dev/null || { ## precalculate variable \$_current_docker_compose
err "Fails to compile base 'docker-conmpose.conf'"
err "Fails to compile base 'docker-compose.conf'"
exit 1
}
@ -1837,3 +1902,6 @@ case "$action" in
fi
;;
esac
Write Preview
Loading…
Cancel
Save