From 59501a990cc9fce63117d34d063f741511700314 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Thu, 28 Jan 2016 15:35:24 +0700 Subject: [PATCH] new: static caching system. --- bin/compose | 232 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 150 insertions(+), 82 deletions(-) diff --git a/bin/compose b/bin/compose index 84ca9ac..d12a07e 100755 --- a/bin/compose +++ b/bin/compose @@ -8,13 +8,20 @@ include pretty include parse +md5_compat() { md5sum | cut -c -32; } + depends shyaml docker [[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true +export CACHEDIR=/var/cache/compose export VARDIR=/var/lib/compose +mkdir -p "$CACHEDIR" || exit 1 + +trap_add "EXIT" clean_cache + usage="$exname CHARM"' Deploy and manage a swarm of containers to provide services based on @@ -195,7 +202,7 @@ file_put() { export -f file_put -_get_docker_compose_links() { +get_docker_compose_links() { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ links charm charm_part master_charm if [ -z "$service" ]; then @@ -209,11 +216,11 @@ _get_docker_compose_links() { return 0 fi - master_charm=$(_get_top_master_charm_for_service "$service") || return 1 + master_charm=$(get_top_master_charm_for_service "$service") || return 1 deps=() while read-0 relation_name target_service relation_config tech_dep; do - master_target_charm="$(_get_top_master_charm_for_service "$target_service")" + master_target_charm="$(get_top_master_charm_for_service "$target_service")" [ "$master_charm" == "$master_target_charm" ] && continue if [ "$tech_dep" == "reversed" ]; then deps+=("$(echo -en "$master_target_charm:\n links:\n - $master_charm")") @@ -241,7 +248,7 @@ _get_docker_compose_opts() { fi compose_def="$(get_compose_service_def "$service")" || return 1 - master_charm="$(_get_top_master_charm_for_service "$service")" + master_charm="$(get_top_master_charm_for_service "$service")" docker_compose_opts=$(echo "$compose_def" | shyaml get-value "docker-compose" 2>/dev/null) if [ "$docker_compose_opts" ]; then @@ -267,11 +274,11 @@ _get_docker_compose_service_mixin() { return 0 fi - master_charm=$(_get_top_master_charm_for_service "$service") || return 1 + master_charm=$(get_top_master_charm_for_service "$service") || return 1 ## The compose part - links_yaml=$(_get_docker_compose_links "$service") || return 1 + links_yaml=$(get_docker_compose_links "$service") || return 1 docker_compose_options=$(_get_docker_compose_opts "$service") || return 1 ## the charm part @@ -283,7 +290,7 @@ _get_docker_compose_service_mixin() { ## Merge results if [ "$charm_part" ]; then charm_yaml="$(yaml_key_val_str "$master_charm" "$charm_part")" || return 1 - merge_yaml_str "$links_yaml" "$charm_yaml" "$docker_compose_options" + merge_yaml_str "$links_yaml" "$charm_yaml" "$docker_compose_options" || return 1 else echo "$links_yaml" fi | tee "$cache_file" @@ -344,50 +351,49 @@ get_docker_compose () { export -f get_docker_compose -## XXXvlab: a lot to be done to cache the results -get_compose_service_def () { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" +_get_compose_service_def_cached () { + local service="$1" docker_compose="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" + #debug "$FUNCNAME: STATIC cache hit" cat "$cache_file" + touch "$cache_file" return 0 fi - [ -z "$service" ] && print_syntax_error "Missing service as first argument." + service_def_base="charm: $service" + value=$(echo "$docker_compose" | shyaml get-value "$service" 2>/dev/null) + merge_yaml <(echo "$service_def_base") <(echo "$value") | tee "$cache_file" +} +export -f _get_compose_service_def_cached - service_def_base= - if [ -d "$CHARM_STORE/$service" ]; then - service_def_base="charm: $service" - fi - value= - if [ -r "$COMPOSE_YML_FILE" ]; then - value=$(shyaml get-value "$service" 2>/dev/null < "$COMPOSE_YML_FILE") - fi - if [ -z "$service_def_base" -a -z "$value" ]; then - err "Invalid service $DARKYELLOW$service$NORMAL: no definition in" \ - "compose file nor charm with same name." - return 1 +## XXXvlab: a lot to be done to cache the results +get_compose_service_def () { + local service="$1" docker_compose cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + result + if [ -e "$cache_file" ]; then + #debug "$FUNCNAME: SESSION cache hit" + cat "$cache_file" + return 0 fi - merge_yaml <(echo "$service_def_base") <(echo "$value") > "$cache_file" - cat "$cache_file" + [ -z "$service" ] && print_syntax_error "Missing service as first argument." + + docker_compose=$(cat "$COMPOSE_YML_FILE") || return 1 + result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1 + echo "$result" | tee "$cache_file" } export -f get_compose_service_def -get_service_charm () { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" +_get_service_charm_cached () { + local service="$1" service_def="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" + # debug "$FUNCNAME: cache hit $1" cat "$cache_file" + touch "$cache_file" return 0 fi - if [ -z "$service" ]; then - print_syntax_error "$FUNCNAME: Please specify a service as first argument." - return 1 - fi - service_def=$(get_compose_service_def "$service") || return 1 charm=$(echo "$service_def" | shyaml get-value charm 2>/dev/null) if [ -z "$charm" ]; then err "Missing charm in service $DARKYELLOW$service$NORMAL definition." @@ -395,6 +401,17 @@ get_service_charm () { fi echo "$charm" | tee "$cache_file" } +export -f _get_service_charm_cached + +get_service_charm () { + local service="$1" + if [ -z "$service" ]; then + print_syntax_error "$FUNCNAME: Please specify a service as first argument." + return 1 + fi + service_def=$(get_compose_service_def "$service") || return 1 + _get_service_charm_cached "$service" "$service_def" +} export -f get_service_charm ## built above the docker-compose abstraction, so it relies on the @@ -425,7 +442,7 @@ service_base_docker_image() { cat "$cache_file" return 0 fi - master_charm="$(_get_top_master_charm_for_service "$service")" || { + master_charm="$(get_top_master_charm_for_service "$service")" || { err "Could not compute base charm for service $DARKYELLOW$service$NORMAL." return 1 } @@ -596,7 +613,7 @@ export -f array_member get_charm_relation_def () { - local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \ relation_def metadata if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" @@ -611,7 +628,7 @@ export -f get_charm_relation_def get_charm_tech_dep_orientation_for_relation() { - local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \ relation_def metadata value if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" @@ -859,8 +876,8 @@ _run_service_relation () { export BASE_CHARM_NAME=$charm export TARGET_CHARM_NAME=$target_charm PROJECT_NAME=$(get_default_project_name) || return 1 - MASTER_BASE_CHARM_NAME=$(_get_top_master_charm_for_service "$service") || return 1 - MASTER_TARGET_CHARM_NAME=$(_get_top_master_charm_for_service "$target_service") || return 1 + MASTER_BASE_CHARM_NAME=$(get_top_master_charm_for_service "$service") || return 1 + MASTER_TARGET_CHARM_NAME=$(get_top_master_charm_for_service "$target_service") || return 1 export RELATION_DATA_FILE RELATION_BASE_COMPOSE_DEF RELATION_TARGET_COMPOSE_DEF export MASTER_BASE_CHARM_NAME MASTER_TARGET_CHARM_NAME PROJECT_NAME target_errlvl=0 @@ -947,19 +964,19 @@ _run_service_relation () { export -f _run_service_relation -get_compose_relations () { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" relation_name relation_def - +_get_compose_relations_cached () { + local compose_service_def="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + relation_name relation_def target_service if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" + #debug "$FUNCNAME: STATIC cache hit $1" cat "$cache_file" + touch "$cache_file" return 0 fi - compose_def="$(get_compose_service_def "$service")" || return 1 ( set -o pipefail - if [ "$compose_def" ]; then + if [ "$compose_service_def" ]; then while read-0 relation_name relation_def; do ( case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in @@ -982,7 +999,7 @@ get_compose_relations () { ;; esac ) > "$cache_file" - done < <(echo "$compose_def" | shyaml key-values-0 relations 2>/dev/null) + done < <(echo "$compose_service_def" | shyaml key-values-0 relations 2>/dev/null) fi ) if [ "$?" != 0 ]; then @@ -993,6 +1010,27 @@ get_compose_relations () { [ -e "$cache_file" ] && cat "$cache_file" return 0 } +export -f _get_compose_relations_cached + +get_compose_relations () { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + compose_def + + if [ -e "$cache_file" ]; then + #debug "$FUNCNAME: SESSION cache hit $1" + cat "$cache_file" + return 0 + fi + + compose_def="$(get_compose_service_def "$service")" || return 1 + _get_compose_relations_cached "$compose_def" > "$cache_file" + if [ "$?" != 0 ]; then + err "Error while looking for compose relations." + rm -f "$cache_file" ## no cache + return 1 + fi + cat "$cache_file" +} export -f get_compose_relations run_service_relations () { @@ -1038,7 +1076,7 @@ _run_service_action_direct() { export METADATA_CONFIG=$(cat "$CHARM_STORE/$charm/metadata.yml") export SERVICE_NAME=$service export ACTION_NAME=$action - export CONTAINER_NAME=$(_get_top_master_charm_for_service "$service") + export CONTAINER_NAME=$(get_top_master_charm_for_service "$service") export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME") export SERVICE_DATASTORE="$DATASTORE/$service" export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service" @@ -1081,7 +1119,7 @@ _run_service_action_relation() { export RELATION_TARGET_CHARM="$target_charm" export RELATION_CHARM="$charm" export ACTION_NAME=$action - export CONTAINER_NAME=$(_get_top_master_charm_for_service "$service") + export CONTAINER_NAME=$(get_top_master_charm_for_service "$service") export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME") export SERVICE_DATASTORE="$DATASTORE/$service" export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service" @@ -1174,7 +1212,7 @@ has_service_action () { fi done < <(get_compose_relations "$service") - master=$(_get_top_master_charm_for_service "$charm") + master=$(get_top_master_charm_for_service "$charm") [ "$master" == "$charm" ] && return 1 has_service_action "$master" "$action" @@ -1239,23 +1277,19 @@ export -f get_compose_relation_config # export -f get_compose_relation_config_for_service -_get_master_charm_for_service() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ - charm metadata requires master_charm target_charm target_service service_def - +_get_master_charm_for_service_cached () { + local service="$1" charm="$2" metadata="$3" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + charm requires master_charm target_charm target_service service_def if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" + # debug "$FUNCNAME: STATIC cache hit ($1)" cat "$cache_file" + touch "$cache_file" return 0 fi - charm=$(get_service_charm "$service") || return 1 - metadata=$(get_charm_metadata "$charm") || return 1 - if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" != "True" ]; then ## just return charm name - echo "$charm" > "$cache_file" - echo "$charm" + echo "$charm" | tee "$cache_file" return 0 fi @@ -1298,14 +1332,29 @@ _get_master_charm_for_service() { die "Charm $DARKYELLOW$charm$NORMAL is a subordinate but does not have any relation with" \ " ${WHITE}scope${NORMAL} set to 'container'." fi - echo "$master_charm" > "$cache_file" - echo "$master_charm" - return 0 + echo "$master_charm" | tee "$cache_file" } -export -f _get_master_charm_for_service +export -f _get_master_charm_for_service_cached + +get_master_charm_for_service() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + charm metadata result + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: SESSION cache hit ($*)" + cat "$cache_file" + return 0 + fi -_get_top_master_charm_for_service() { + charm=$(get_service_charm "$service") || return 1 + metadata=$(get_charm_metadata "$charm") || return 1 + result=$(_get_master_charm_for_service_cached "$service" "$charm" "$metadata") || return 1 + echo "$result" | tee "$cache_file" +} +export -f get_master_charm_for_service + + +get_top_master_charm_for_service() { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ current_service @@ -1317,14 +1366,14 @@ _get_top_master_charm_for_service() { current_service="$service" while true; do - master_service=$(_get_master_charm_for_service "$current_service") || return 1 + master_service=$(get_master_charm_for_service "$current_service") || return 1 [ "$master_service" == "$current_service" ] && break current_service="$master_service" done echo "$current_service" | tee "$cache_file" return 0 } -export -f _get_top_master_charm_for_service +export -f get_top_master_charm_for_service get_charm_metadata() { @@ -1342,18 +1391,17 @@ export -f get_charm_metadata ## The result is a mixin that is not always a complete valid ## docker-compose entry (thinking of subordinates). The result ## will be merge with master charms. -get_docker_compose_mixin_from_metadata() { - local charm="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ +_get_docker_compose_mixin_from_metadata_cached() { + local charm="$1" metadata="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ metadata_file metadata volumes docker_compose subordinate image - if [ -e "$cache_file" ]; then - debug "$FUNCNAME: cache hit ($*)" + #debug "$FUNCNAME: STATIC cache hit $1" cat "$cache_file" + touch "$cache_file" return 0 fi mixin= - metadata="$(get_charm_metadata "$charm")" if [ "$metadata" ]; then ## resources to volumes volumes=$( @@ -1370,11 +1418,6 @@ get_docker_compose_mixin_from_metadata() { fi done < <(echo "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null) while read-0 resource; do - if ! [ -e "$CHARM_STORE/$charm/resources$resource" ]; then - err "No '$resource' resource found in ${BLUE}resources/${NORMAL}" \ - "directory of charm $DARKYELLOW$charm$NORMAL." - exit 1 - fi echo " - $CHARM_STORE/$charm/resources$resource:$resource:rw" done < <(echo "$metadata" | shyaml get-values-0 "charm-resources" 2>/dev/null) ) || return 1 @@ -1410,8 +1453,22 @@ get_docker_compose_mixin_from_metadata() { if [ "$image_or_build_statement" ]; then mixin=$(merge_yaml_str "$mixin" "$image_or_build_statement") fi - echo "$mixin" > "$cache_file" - echo "$mixin" + echo "$mixin" | tee "$cache_file" +} +export -f _get_docker_compose_mixin_from_metadata_cached + + +get_docker_compose_mixin_from_metadata() { + local charm="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" + if [ -e "$cache_file" ]; then + #debug "$FUNCNAME: SESSION cache hit ($*)" + cat "$cache_file" + return 0 + fi + + metadata="$(get_charm_metadata "$charm")" || return 1 + mixin=$(_get_docker_compose_mixin_from_metadata_cached "$charm" "$metadata") || return 1 + echo "$mixin" | tee "$cache_file" } export -f get_docker_compose_mixin_from_metadata @@ -1490,7 +1547,6 @@ launch_docker_compose() { export -f launch_docker_compose get_compose_yml_location() { - parent=$(while ! [ -e "./compose.yml" ]; do [ "$PWD" == "/" ] && exit 0 cd .. @@ -1537,7 +1593,7 @@ get_master_services() { local loaded master_service declare -A loaded for service in "$@"; do - master_service=$(_get_top_master_charm_for_service "$service") || return 1 + master_service=$(get_top_master_charm_for_service "$service") || return 1 if [ "${loaded[$master_service]}" ]; then continue fi @@ -1559,7 +1615,6 @@ _setup_state_dir() { get_docker_compose_opts_list() { local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" - if [ -e "$cache_file" ]; then debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" @@ -1588,6 +1643,16 @@ get_docker_compose_single_opts_list() { tr ',' "\n" | xargs echo } +clean_cache() { + local i=0 + for f in $(ls -t "$CACHEDIR/"*.cache.* | tail -n +500); do + ((i++)) + rm -f "$f" + done + if (( i > 0 )); then + debug "${WHITE}Cleaned cache:${NORMAL} Removed $((i)) elements (current cache size is $(du -sh "$CACHEDIR" | cut -f 1))" + fi +} [ "$SOURCED" ] && return 0 @@ -1750,7 +1815,7 @@ esac get_docker_compose $services >/dev/null || { ## precalculate variable \$_current_docker_compose - err "Fails to compile base 'docker-conmpose.conf'" + err "Fails to compile base 'docker-compose.conf'" exit 1 } @@ -1837,3 +1902,6 @@ case "$action" in fi ;; esac + + +