#!/bin/bash #:- . /etc/shlib #:- include pretty include parse depends shyaml docker [[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true export VARDIR=/var/lib/compose usage="$exname CHARM"' Deploy and manage a swarm of containers to provide services based on a ``compose.yml`` definition and charms from a ``charm-store``. ' export DEFAULT_COMPOSE_FILE ## ## Merge YAML files ## _merge_yaml_common_code=" import sys import yaml def fc(filename): with open(filename) as f: return f.read() def merge(*args): # sys.stderr.write('%r\n' % (args, )) args = [arg for arg in args if arg is not None] if len(args) == 0: return None if len(args) == 1: return args[0] if all(isinstance(arg, (int, basestring, bool)) for arg in args): return args[-1] elif all(isinstance(arg, list) for arg in args): res = [] for arg in args: res.extend(arg) return res elif all(isinstance(arg, dict) for arg in args): keys = set() for arg in args: keys |= set(arg.keys()) dct = {} for key in keys: sub_args = [] for arg in args: if key in arg: sub_args.append(arg) try: dct[key] = merge(*(a[key] for a in sub_args)) except NotImplementedError as e: raise NotImplementedError( e.args[0], '%s.%s' % (key, e.args[1]) if e.args[1] else key, e.args[2]) if dct[key] is None: del dct[key] return dct else: raise NotImplementedError( 'Unsupported types: %s' % (', '.join(list(set(arg.__class__.__name__ for arg in args)))), '', args) return None def merge_cli(*args): try: c = merge(*args) except NotImplementedError as e: sys.stderr.write('%s. Conflicting key is %r. Values are:\n%s\n' % (e.args[0], e.args[1], e.args[2])) exit(1) if c is not None: print '%s' % yaml.dump(c, default_flow_style=False) " merge_yaml() { if ! [ -r "$state_tmpdir/merge_yaml.py" ]; then cat < "$state_tmpdir/merge_yaml.py" $_merge_yaml_common_code merge_cli(*(yaml.load(fc(f)) for f in sys.argv[1:])) EOF fi python "$state_tmpdir/merge_yaml.py" "$@" } export -f merge_yaml merge_yaml_str() { local entries="$@" if ! [ -r "$state_tmpdir/merge_yaml_str.py" ]; then cat < "$state_tmpdir/merge_yaml_str.py" $_merge_yaml_common_code merge_cli(*(yaml.load(f) for f in sys.argv[1:])) EOF fi python "$state_tmpdir/merge_yaml_str.py" "$@" } export -f merge_yaml_str ## ## Functions ## gen_password() { python -c 'import random; \ xx = "azertyuiopqsdfghjklmwxcvbn1234567890AZERTYUIOPQSDFGHJKLMWXCVBN+_-"; \ print "".join([xx[random.randint(0, len(xx)-1)] for x in range(0, 14)])' } export -f gen_password file_put() { local TARGET="$1" mkdir -p "$(dirname "$TARGET")" && cat - > "$TARGET" } export -f file_put _get_docker_compose_links() { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" links charm charm_part if [ -z "$service" ]; then print_syntax_error "$FUNCNAME: Please specify a service as first argument." return 1 fi if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi master_charm=$(_get_master_charm_for_service "$service") || return 1 deps=() while read-0 relation_name target_service relation_config reverse; do [ "$master_charm" == "$target_service" ] && continue if [ "$reverse" ]; then deps+=("$(echo -en "$target_service:\n links:\n - $master_charm")") else deps+=("$(echo -en "$master_charm:\n links:\n - $target_service")") fi done < <(get_compose_relations "$service") || return 1 merge_yaml_str "${deps[@]}" | tee "$cache_file" } ## ## By Reading the metadata.yml, we create a docker-compose.yml mixin. ## Some metadata.yml (of subordinates) will indeed modify other ## services than themselves. _get_docker_compose_service_mixin() { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" links charm charm_part if [ -z "$service" ]; then print_syntax_error "$FUNCNAME: Please specify a service as first argument." return 1 fi if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi master_charm=$(_get_master_charm_for_service "$service") || return 1 ## The compose part links_yaml=$(_get_docker_compose_links "$service") || return 1 ## the charm part #debug "Get charm name from service name $DARKYELLOW$service$NORMAL." charm=$(get_service_charm "$service") || return 1 charm_part=$(get_docker_compose_mixin_from_metadata "$charm") || return 1 ## Merge results if [ "$charm_part" ]; then charm_yaml="$(echo -en "${master_charm}:\n$(echo "$charm_part" | prefix " ")")" merge_yaml_str "$links_yaml" "$charm_yaml" else echo "$links_yaml" fi > "$cache_file" cat "$cache_file" } export -f _get_docker_compose_service_mixin ## ## Get full `docker-compose.yml` format for all listed services (and ## their deps) ## get_docker_compose() { local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" entries services if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi ## ## Adding sub services configurations ## declare -A entries for target_service in "$@"; do services=$(get_ordered_service_dependencies "$target_service") || return 1 for service in $services; do if [ "${entries[$service]}" ]; then ## Prevent double inclusion of same service if this ## service is deps of two or more of your ## requirements. continue fi ## mark the service as "loaded" as well as it's containers ## if this is a subordinate service entries[$service]=$(_get_docker_compose_service_mixin "$service") || return 1 done done merge_yaml_str "${entries[@]}" > "$cache_file" export _current_docker_compose="$(cat "$cache_file")" echo "$_current_docker_compose" } export -f get_docker_compose ## XXXvlab: a lot to be done to cache the results get_compose_service_def () { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi [ -z "$service" ] && print_syntax_error "Missing service as first argument." service_def_base= if [ -d "$CHARM_STORE/$service" ]; then service_def_base="charm: $service" fi value= if [ -r "$COMPOSE_YML_FILE" ]; then value=$(shyaml get-value "$service" 2>/dev/null < "$COMPOSE_YML_FILE") fi if [ -z "$service_def_base" -a -z "$value" ]; then err "Invalid service $DARKYELLOW$service$NORMAL: no definition in" \ "compose file nor charm with same name." return 1 fi merge_yaml <(echo "$service_def_base") <(echo "$value") > "$cache_file" cat "$cache_file" } export -f get_compose_service_def ## XXXvlab: MUST CACHE get_service_charm () { local service="$1" if [ -z "$service" ]; then print_syntax_error "$FUNCNAME: Please specify a service as first argument." return 1 fi service_def=$(get_compose_service_def "$service") || return 1 charm=$(echo "$service_def" | shyaml get-value charm 2>/dev/null) if [ -z "$charm" ]; then err "Missing charm in service $DARKYELLOW$service$NORMAL definition." return 1 fi echo "$charm" } export -f get_service_charm ## built above the docker-compose abstraction, so it relies on the ## full docker-compose.yml to be already built. get_service_def() { local service="$1" def if [ -z "$_current_docker_compose" ]; then print_syntax_error "$FUNCNAME is meant to be called after"\ "\$_current_docker_compose has been calculated." fi def=$(echo "$_current_docker_compose" | shyaml get-value "$service" 2>/dev/null) if [ -z "$def" ]; then err "No definition for service $DARKYELLOW$service$NORMAL." return 1 fi echo "$def" } export -f get_service_def ## Return the base docker image name of a service service_base_docker_image() { local service="$1" service_def="$(get_service_def "$service")" || return 1 service_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null) if [ "$?" != 0 ]; then service_build=$(echo "$service_def" | shyaml get-value build 2>/dev/null) if [ "$?" != 0 ]; then err "Service $DARKYELLOW$service$NORMAL has no ${WHITE}image${NORMAL} nor ${WHITE}build${NORMAL} parameter." echo "$service_def" >&2 return 1 fi service_dockerfile="$CHARM_STORE/${service_build}/Dockerfile" if ! [ -e "$service_dockerfile" ]; then err "No Dockerfile found in '$service_dockerfile' location." return 1 fi grep '^FROM' "$service_dockerfile" | xargs echo | cut -f 2 -d " " else echo "$service_image" fi } export -f service_base_docker_image ## XXXvlab: provided in shlib/common # read-0() { # local eof # eof= # while [ "$1" ]; do # IFS='' read -r -d '' "$1" || eof=true # shift # done # test "$eof" != true # } # export -f read-0 fetch_file() { local src="$1" case "$src" in *"://"*) err "Unsupported target scheme." return 1 ;; *) ## Try direct if ! [ -r "$src" ]; then err "File '$src' not found/readable." return 1 fi cat "$src" || return 1 ;; esac } export -f fetch_file ## receives stdin content to decompress on stdout ## stdout content should be tar format. uncompress_file() { local filename="$1" ## Warning, the content of the file is already as stdin, the filename ## is there to hint for correct decompression. case "$filename" in *".gz") gunzip ;; *".bz2") bunzip2 ;; *) cat ;; esac } export -f uncompress_file get_file() { local src="$1" fetch_file "$src" | uncompress_file "$src" } export -f get_file cmd_on_base_image() { local charm="$1" shift base_image=$(service_base_docker_image "$charm") || return 1 docker run -i --entrypoint /bin/bash "$base_image" -c "$*" } export -f cmd_on_base_image cached_cmd_on_base_image() { local charm="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" shift if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi base_image=$(service_base_docker_image "$charm") || return 1 result=$(cmd_on_base_image "$charm" "$@") || return 1 echo "$result" | tee "$cache_file" } export -f cached_cmd_on_base_image array_values_to_stdin() { local e if [ "$#" -ne "1" ]; then print_syntax_warning "$FUNCNAME: need one argument." return 1 fi var="$1" eval "for e in \"\${$var[@]}\"; do echo -en \"\$e\\0\"; done" } array_keys_to_stdin() { local e if [ "$#" -ne "1" ]; then print_syntax_warning "$FUNCNAME: need one argument." return 1 fi var="$1" eval "for e in \"\${!$var[@]}\"; do echo -en \"\$e\\0\"; done" } array_kv_to_stdin() { local e if [ "$#" -ne "1" ]; then print_syntax_warning "$FUNCNAME: need one argument." return 1 fi var="$1" eval "for e in \"\${!$var[@]}\"; do echo -n \"\$e\"; echo -en '\0'; echo -n \"\${$var[\$e]}\"; echo -en '\0'; done" } array_pop() { local narr="$1" nres="$2" for key in $(eval "echo \${!$narr[@]}"); do eval "$nres=\${$narr[\"\$key\"]}" eval "unset $narr[\"\$key\"]" return 0 done } export -f array_pop array_member() { local src elt src="$1" elt="$2" while read-0 key; do if [ "$(eval "echo -n \"\${$src[\$key]}\"")" == "$elt" ]; then return 0 fi done < <(array_keys_to_stdin "$src") return 1 } export -f array_member get_charm_relation_def () { local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ relation_def metadata if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi metadata="$(get_charm_metadata "$charm")" || return 1 relation_def="$(echo "$metadata" | shyaml get-value "provides.${relation_name}" 2>/dev/null)" echo "$relation_def" | tee "$cache_file" } export -f get_charm_relation_def get_charm_reverse_tech_dep_relation() { local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ relation_def metadata if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi relation_def=$(get_charm_relation_def "$charm" "$relation_name" 2>/dev/null) value=$(echo "$relation_def" | shyaml get-value 'reverse-tech-dep' 2>/dev/null) echo "$value" | tee "$cache_file" } export -f get_charm_reverse_tech_dep_relation ## ## Use compose file to get deps, and relation definition in metadata.yml ## for reverse-tech-dep attribute. get_service_deps() { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi ( set -o pipefail get_compose_relations "$service" | \ while read-0 relation_name target_service _relation_config reverse; do echo "$target_service" done | tee "$cache_file" ) || return 1 } export -f get_service_deps _rec_get_depth() { local elt=$1 #debug "Asking for $DARKYELLOW$elt$NORMAL dependencies" if [ "${depths[$elt]}" ]; then return 0 fi deps=$(get_service_deps "$elt") || return 1 if [ -z "$deps" ]; then depths[$elt]=0 fi max=0 for dep in $deps; do _rec_get_depth "$dep" || return 1 if (( "${depths[$dep]}" > "$max" )); then max="${depths[$dep]}" fi done depths[$elt]=$((max + 1)) } get_ordered_service_dependencies() { local services=("$@") cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi #debug "Figuring ordered deps of $DARKYELLOW$services$NORMAL" if [ -z "${services[*]}" ]; then print_syntax_error "$FUNCNAME: no arguments" return 1 fi declare -A depths visited=() heads=("${services[@]}") while [ "${#heads[@]}" != 0 ]; do array_pop heads head visited+=("$head") _rec_get_depth "$head" || return 1 done i=0 while [ "${#depths[@]}" != 0 ]; do for key in "${!depths[@]}"; do value="${depths[$key]}" if [ "$value" == "$i" ]; then echo "$key" unset depths[$key] fi done i=$((i + 1)) done > "$cache_file" cat "$cache_file" } run_service_hook () { local services="$1" action="$2" loaded declare -A loaded for service in $services; do for subservice in $(get_ordered_service_dependencies "$service"); do if [ "${loaded[$subservice]}" ]; then ## Prevent double inclusion of same service if this ## service is deps of two or more of your ## requirements. continue fi charm=$(get_service_charm "$subservice") || return 1 TARGET_SCRIPT="$CHARM_STORE/$charm/hooks/$action" [ -e "$TARGET_SCRIPT" ] || continue Wrap -d "$YELLOW$action$NORMAL hook of charm $DARKYELLOW$charm$NORMAL" </dev/null if [ "$?" != 0 ]; then err "The key $WHITE$key$NORMAL was not found in relation's data." return 1 fi } export -f relation-get relation-base-compose-get () { local key="$1" echo "$RELATION_BASE_COMPOSE_DEF" | shyaml get-value "options.$key" 2>/dev/null if [ "$?" != 0 ]; then err "The key $WHITE$key$NORMAL was not found in base service compose definition.." return 1 fi } export -f relation-base-compose-get relation-target-compose-get () { local key="$1" echo "$RELATION_BASE_COMPOSE_DEF" | shyaml get-value "options.$key" 2>/dev/null if [ "$?" != 0 ]; then err "The key $WHITE$key$NORMAL was not found in base service compose definition.." return 1 fi } export -f relation-base-compose-get relation-set () { local key="$1" value="$2" if [ -z "$RELATION_DATA_FILE" ]; then err "$FUNCNAME: relation does not seems to be correctly setup." return 1 fi if ! [ -r "$RELATION_DATA_FILE" ]; then err "$FUNCNAME: can't read relation's data." >&2 return 1 fi _config_merge "$RELATION_DATA_FILE" <(echo "$key: $value") } export -f relation-set _config_merge() { local config_filename="$1" merge_to_file="$2" touch "$config_filename" && merge_yaml "$config_filename" "$merge_to_file" > "$config_filename.tmp" || return 1 mv "$config_filename.tmp" "$config_filename" } export -f _config_merge config-add() { local metadata="$1" _config_merge "$RELATION_CONFIG" <(echo "$metadata") } export -f config-add logstdout() { local name="$1" sed -r 's%^%'"${name}"'> %g' } export -f logstdout logstderr() { local name="$1" sed -r 's%^(.*)$%'"${RED}${name}>${NORMAL} \1"'%g' } export -f logstderr _run_service_relation () { local relation_name="$1" service="$2" target_service="$3" relation_config="$4" relation_dir services charm=$(get_service_charm "$service") || return 1 target_charm=$(get_service_charm "$target_service") || return 1 base_script_name=$(echo "$relation_name" | tr "-" "_" )-relation-joined script_name="hooks/${base_script_name}" relation_dir=$(get_relation_data_dir "$charm" "$target_charm" "$relation_name") || return 1 RELATION_DATA_FILE=$(get_relation_data_file "$charm" "$target_charm" "$relation_name" "$relation_config") || return 1 RELATION_BASE_COMPOSE_DEF=$(get_compose_service_def "$service") || return 1 RELATION_TARGET_COMPOSE_DEF=$(get_compose_service_def "$target_service") || return 1 export BASE_SERVICE_NAME=$service export TARGET_SERVICE_NAME=$target_service export BASE_CHARM_NAME=$charm export TARGET_CHARM_NAME=$target_charm export RELATION_DATA_FILE RELATION_BASE_COMPOSE_DEF RELATION_TARGET_COMPOSE_DEF target_errlvl=0 if ! [ -e "$CHARM_STORE/$target_charm/$script_name" ]; then verb "Missing relation script '$script_name' in $DARKYELLOW$target_charm$NORMAL." else verb "Building ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \ "for target charm $DARKYELLOW$target_charm$NORMAL" export RELATION_CONFIG="$relation_dir/config_provider" { ( cd "$CHARM_STORE/$target_charm" export SERVICE_NAME=$target_service export DOCKER_BASE_IMAGE=$(service_base_docker_image "$target_service") export SERVICE_DATASTORE="$DATASTORE/$target_charm" export SERVICE_CONFIGSTORE="$CONFIGSTORE/$target_charm" "$script_name" echo "$?" > "$relation_dir/target_errlvl" ) | logstdout "$DARKYELLOW$target_charm$NORMAL/$base_script_name ${GREEN}@${NORMAL}" } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$target_charm$NORMAL/$base_script_name ${RED}@${NORMAL}" 3>&1 1>&2 2>&3 target_errlvl="$(cat "$relation_dir/target_errlvl")" if [ -e "$RELATION_CONFIG" ]; then debug "Merging some new config info in $DARKYELLOW$target_service$NORMAL" _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" rm "$RELATION_CONFIG" ((target_errlvl |= "$?")) fi fi if [ "$target_errlvl" == 0 ]; then errlvl=0 if [ -e "$CHARM_STORE/$charm/$script_name" ]; then verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \ "for charm $DARKYELLOW$charm$NORMAL" export RELATION_CONFIG="$relation_dir/config_providee" export RELATION_DATA="$(cat "$RELATION_DATA_FILE")" { ( cd "$CHARM_STORE/$charm" export SERVICE_NAME=$service export DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") export SERVICE_DATASTORE="$DATASTORE/$charm" export SERVICE_CONFIGSTORE="$CONFIGSTORE/$charm" "$script_name" echo "$?" > "$relation_dir/errlvl" ) | logstdout "$DARKYELLOW$charm$NORMAL/$base_script_name ${GREEN}@${NORMAL}" } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$charm$NORMAL/$base_script_name ${RED}@$NORMAL" 3>&1 1>&2 2>&3 errlvl="$(cat "$relation_dir/errlvl")" if [ -e "$RELATION_CONFIG" ]; then _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" rm "$RELATION_CONFIG" fi if [ "$errlvl" != 0 ]; then err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$target_charm$NORMAL failed to run properly." fi else verb "No relation script '$script_name' in charm $DARKYELLOW$charm$NORMAL. Ignoring." fi else err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$target_charm$NORMAL failed to run properly." fi if [ "$target_errlvl" == 0 -a "$errlvl" == 0 ]; then debug "Relation $DARKBLUE$relation_name$NORMAL is established" \ "between $DARKYELLOW$service$NORMAL and $DARKYELLOW$target_service$NORMAL." return 0 else return 1 fi } export -f _run_service_relation get_compose_relations() { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi compose_def="$(get_compose_service_def "$service")" || return 1 ( set -o pipefail if [ "$compose_def" ]; then while read-0 relation_name relation_def; do case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in "str") target_service="$(echo "$relation_def" | shyaml get-value 2>/dev/null)" reverse="$(get_charm_reverse_tech_dep_relation "$target_service" "$relation_name")" echo -en "$relation_name\0$target_service\0\0$reverse\0" ;; "sequence") while read-0 target_service; do reverse="$(get_charm_reverse_tech_dep_relation "$target_service" "$relation_name")" echo -en "$relation_name\0$target_service\0\0$reverse\0" done < <(echo "$relation_def" | shyaml get-values-0 2>/dev/null) ;; "struct") while read-0 target_service relation_config; do reverse="$(get_charm_reverse_tech_dep_relation "$target_service" "$relation_name")" echo -en "$relation_name\0$target_service\0$relation_config\0$reverse\0" done < <(echo "$relation_def" | shyaml key-values-0 2>/dev/null) ;; esac done < <(echo "$compose_def" | shyaml key-values-0 relations 2>/dev/null) fi | tee "$cache_file" ) if [ "$?" != 0 ]; then rm -f "$cache_file" ## no cache return 1 fi return 0 } run_service_relations () { local services="$1" for service in $services; do for subservice in $(get_ordered_service_dependencies "$service"); do while read-0 relation_name target_service relation_config reverse; do export relation_config Wrap -d "Building $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW$target_service$NORMAL" < "$action_errlvl_file" ) | logstdout "$DARKYELLOW$charm$NORMAL/${DARKCYAN}$action${NORMAL} ${GREEN}@${NORMAL}" } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$charm$NORMAL/${DARKCYAN}$action${NORMAL} ${RED}@$NORMAL" 3>&1 1>&2 2>&3 if ! [ -e "$action_errlvl_file" ]; then err "Action $DARKYELLOW$service$NORMAL:$DARKCYAN$action$NORMAL has failed without having time" \ "to output an errlvl" return 1 fi return "$(cat "$action_errlvl_file")" } export -f _run_service_action_direct _run_service_action_relation() { local service="$1" action="$2" charm target_charm relation_name target_script relation_config _dummy shift; shift read-0 charm target_charm relation_name target_script relation_config || true if read-0 _dummy || [ "$_dummy" ]; then print_syntax_error "$FUNCNAME: too many arguments in action descriptor" return 1 fi export RELATION_DATA_FILE=$(get_relation_data_file "$charm" "$target_charm" "$relation_name" "$relation_config") compose_file=$(get_compose_yml_location) || return 1 export action_errlvl_file="$state_tmpdir/action-$service-$charm-$action-errlvl" export state_tmpdir { ( set +e ## Prevents unwanted leaks from parent shell cd "$CHARM_STORE/$charm" export METADATA_CONFIG=$(cat "$CHARM_STORE/$charm/metadata.yml") export SERVICE_NAME=$service export RELATION_TARGET_CHARM="$target_charm" export RELATION_CHARM="$charm" export ACTION_NAME=$action export CONTAINER_NAME=$(_get_master_charm_for_service "$service") export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME") export SERVICE_DATASTORE="$DATASTORE/$service" export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service" exname="$exname $ACTION_NAME $SERVICE_NAME" \ stdbuf -oL -eL "$target_script" "$@" echo "$?" > "$action_errlvl_file" ) | logstdout "$DARKYELLOW$charm$NORMAL/${DARKCYAN}$action${NORMAL} ${GREEN}@${NORMAL}" } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$charm$NORMAL/${DARKCYAN}$action${NORMAL} ${RED}@$NORMAL" 3>&1 1>&2 2>&3 if ! [ -e "$action_errlvl_file" ]; then err "Action $DARKYELLOW$service$NORMAL:$DARKCYAN$action$NORMAL has failed without having time" \ "to output an errlvl" return 1 fi return "$(cat "$action_errlvl_file")" } export -f _run_service_action_relation get_relation_data_dir() { local charm="$1" target_charm="$2" relation_name="$3" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi project=$(get_default_project_name) || return 1 relation_dir="$VARDIR/relations/$project/$charm-$target_charm/$relation_name" if ! [ -d "$relation_dir" ]; then mkdir -p "$relation_dir" || return 1 chmod go-rwx "$relation_dir" || return 1 ## protecting this directory fi echo "$relation_dir" || tee "$cache_file" } export -f get_relation_data_dir get_relation_data_file() { local charm="$1" target_charm="$2" relation_name="$3" relation_config="$4" relation_dir=$(get_relation_data_dir "$charm" "$target_charm" "$relation_name") || return 1 relation_data_file="$relation_dir/data" if ! [ -e "$relation_data_file" ]; then echo "$relation_config" > "$relation_data_file" chmod go-rwx "$relation_data_file" ## protecting this file fi echo "$relation_data_file" } export -f get_relation_data_file has_service_action () { local service="$1" action="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ target_script charm target_charm if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi charm=$(get_service_charm "$service") || return 1 ## Action directly provided ? target_script="$CHARM_STORE/$charm/actions/$action" if [ -x "$target_script" ]; then echo -en "direct\0$charm\0$target_script" | tee "$cache_file" return 0 fi ## Action provided by relation ? while read-0 relation_name target_service relation_config reverse; do target_charm=$(get_service_charm "$target_service") || return 1 target_script="$CHARM_STORE/$target_charm/actions/relations/$relation_name/$action" if [ -x "$target_script" ]; then echo -en "relation\0$charm\0$target_charm\0$relation_name\0$target_script\0$relation_config" | tee "$cache_file" return 0 fi done < <(get_compose_relations "$service") return 1 } export -f has_service_action run_service_action () { local service="$1" action="$2" shift ; shift { if ! read-0 action_type; then info "Service $DARKYELLOW$service$NORMAL does not have any action $DARKCYAN$action$NORMAL defined." info " Add an executable script to '$CHARM_STORE/$charm/actions/$action' to implement action." return 1 fi Section "running $DARKYELLOW$service$NORMAL/$DARKCYAN$action$NORMAL ($action_type)"; Feed "_run_service_action_${action_type}" "$service" "$action" "$@" } < <(has_service_action "$service" "$action") } export -f run_service_action get_compose_relation_config() { local service=$1 relation_config cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi compose_service_def=$(get_compose_service_def "$service") || return 1 echo "$compose_service_def" | shyaml get-value "relations" 2>/dev/null | tee "$cache_file" } export -f get_compose_relation_config ## Return key-values-0 get_compose_relation_config_for_service() { local service=$1 relation_name=$2 relation_config compose_service_relations=$(get_compose_relation_config "$service") || return 1 if ! relation_config=$( echo "$compose_service_relations" | shyaml get-value "${relation_name}" 2>/dev/null); then err "Couldn't find $DARKYELLOW${service}$NORMAL/${WHITE}${relation_name}$NORMAL" \ "relation config in compose configuration." return 1 fi if [ -z "$relation_config" ]; then err "Relation ${WHITE}mysql-database$NORMAL is empty in compose configuration." return 1 fi if ! echo "$relation_config" | shyaml key-values-0 2>/dev/null; then err "No key/values in ${DARKBLUE}mysql-database$NORMAL of compose config." return 1 fi } export -f get_compose_relation_config_for_service _get_master_charm_for_service() { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ charm metadata requires master_charm target_charm target_service service_def if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi charm=$(get_service_charm "$service") || return 1 metadata=$(get_charm_metadata "$charm") || return 1 if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" != "True" ]; then ## just return charm name echo "$charm" > "$cache_file" echo "$charm" return 0 fi ## fetch the container relation requires="$(echo "$metadata" | shyaml get-value "requires" 2>/dev/null)" if [ -z "$requires" ]; then die "Charm $DARKYELLOW$charm$NORMAL is a subordinate but does not have any 'requires' " \ "section." fi master_charm= while read-0 relation_name relation; do if [ "$(echo "$relation" | shyaml get-value "scope" 2>/dev/null)" == "container" ]; then # debug "$DARKYELLOW$service$NORMAL's relation" \ # "$DARKBLUE${relation_name}$NORMAL is a container." interface="$(echo "$relation" | shyaml get-value "interface" 2>/dev/null)" if [ -z "$interface" ]; then err "No ${WHITE}$interface${NORMAL} set for relation $relation_name." return 1 fi service_def=$(get_compose_service_def "$service") || return 1 target_service=$(echo "$service_def" | shyaml keys "relations.${interface}" 2>/dev/null | head -n 1) if [ -z "$target_service" ]; then err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \ "${DARKYELLOW}$service$NORMAL compose definition." return 1 fi target_charm=$(get_service_charm "$target_service") || return 1 master_charm="$(_get_master_charm_for_service "$target_service")" break fi done < <(echo "$requires" | shyaml key-values-0 2>/dev/null) if [ -z "$master_charm" ]; then die "Charm $DARKYELLOW$charm$NORMAL is a subordinate but does not have any relation with" \ " ${WHITE}scope${NORMAL} set to 'container'." fi echo "$master_charm" > "$cache_file" echo "$master_charm" return 0 } export -f _get_master_charm_for_service get_charm_metadata() { local charm="$1" metadata_file="$CHARM_STORE/$charm/metadata.yml" if ! [ -e "$metadata_file" ]; then return 0 ## No metadata file is as if metadata was empty fi cat "$metadata_file" } ## ## The result is a mixin that is not allways a complete valid ## docker-compose entry (thinkinf of subordinates). The result ## will be merge with master charms. get_docker_compose_mixin_from_metadata() { local charm="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ metadata_file metadata volumes docker_compose subordinate image if [ -e "$cache_file" ]; then debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi mixin= metadata="$(get_charm_metadata "$charm")" if [ "$metadata" ]; then ## resources to volumes volumes=$( for resource_type in data config; do while read-0 resource; do eval "echo \" - \$${resource_type^^}STORE/\$charm\$resource:\$resource:rw\"" done < <(echo "$metadata" | shyaml get-values-0 "${resource_type}-resources" 2>/dev/null) done while read-0 resource; do if [[ "$resource" == *:* ]]; then echo " - $resource:rw" else echo " - $resource:$resource:rw" fi done < <(echo "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null) ) if [ "$volumes" ]; then mixin=$(echo -en "volumes:\n$volumes") fi docker_compose=$(echo "$metadata" | shyaml get-value "docker-compose" 2>/dev/null) if [ "$docker_compose" ]; then mixin=$(merge_yaml_str "$mixin" "$docker_compose") fi if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" == "True" ]; then subordinate=true fi fi image=$(echo "$metadata" | shyaml get-value "docker-image" 2>/dev/null) image_or_build_statement= if [ "$image" ]; then if [ "$subordinate" ]; then err "Subordinate charm can not have a ${WHITE}docker-image${NORMAL} value." return 1 fi image_or_build_statement="image: $image" elif [ -d "$CHARM_STORE/$charm/build" ]; then if [ "$subordinate" ]; then err "Subordinate charm can not have a 'build' sub directory." return 1 fi image_or_build_statement="build: $charm/build" fi if [ "$image_or_build_statement" ]; then mixin=$(merge_yaml_str "$mixin" "$image_or_build_statement") fi echo "$mixin" > "$cache_file" echo "$mixin" } export -f get_docker_compose_mixin_from_metadata _save() { local name="$1" cat - | tee -a "$docker_compose_dir/.data/$name" } get_default_project_name() { if [ "$DEFAULT_PROJECT_NAME" ]; then echo "$DEFAULT_PROJECT_NAME" return 0 fi compose_yml_location="$(get_compose_yml_location)" if [ "$compose_yml_location" ]; then if normalized_path=$(readlink -e "$compose_yml_location"); then echo "$(basename "$(dirname "$normalized_path")")" return 0 fi fi echo "project" return 0 } export -f get_default_project_name launch_docker_compose() { docker_compose_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX) #debug "Creating temporary docker-compose directory in '$docker_compose_tmpdir'." # trap_add EXIT "debug \"Removing temporary docker-compose directory in $docker_compose_tmpdir.\";\ # rm -rf \"$docker_compose_tmpdir\"" trap_add EXIT "rm -rf \"$docker_compose_tmpdir\"" project=$(get_default_project_name) mkdir -p "$docker_compose_tmpdir/$project" docker_compose_dir="$docker_compose_tmpdir/$project" if [ -z "$services" ]; then services=$(get_default_target_services $services) fi get_docker_compose $services > "$docker_compose_dir/docker-compose.yml" || return 1 if [ -e "$state_tmpdir/to-merge-in-docker-compose.yml" ]; then # debug "Merging some config data in docker-compose.yml:" # debug "$(cat $state_tmpdir/to-merge-in-docker-compose.yml)" _config_merge "$docker_compose_dir/docker-compose.yml" "$state_tmpdir/to-merge-in-docker-compose.yml" || return 1 fi ## XXXvlab: could be more specific and only link the needed charms ln -sf "$CHARM_STORE/"* "$docker_compose_dir/" mkdir "$docker_compose_dir/.data" { { cd "$docker_compose_dir" debug "${WHITE}docker-compose.yml:$NORMAL" debug "$(cat "$docker_compose_dir/docker-compose.yml")" debug "${WHITE}Launching$NORMAL: docker-compose $@" docker-compose "$@" echo "$?" > "$docker_compose_dir/.data/errlvl" } | _save stdout } 3>&1 1>&2 2>&3 | _save stderr 3>&1 1>&2 2>&3 if tail -n 1 "$docker_compose_dir/.data/stderr" | egrep "Service .+ failed to build: Error getting container [0-9a-f]+ from driver devicemapper: (open|Error mounting) /dev/mapper/docker-.*: no such file or directory$" >/dev/null 2>&1; then debug NASTY ERROR DETECTED fi docker_compose_errlvl="$(cat "$docker_compose_dir/.data/errlvl")" if [ -z "docker_compose_dir" ]; then err "Something went wrong before you could gather docker-compose errorlevel." return 1 fi return "$docker_compose_errlvl" } get_compose_yml_location() { parent=$(while ! [ -e "./compose.yml" ]; do [ "$PWD" == "/" ] && exit 0 cd .. done; echo "$PWD" ) if [ "$parent" ]; then echo "$parent/compose.yml" return 0 fi if [ "$DEFAULT_COMPOSE_FILE" ]; then if ! [ -e "$DEFAULT_COMPOSE_FILE" ]; then err "No 'compose.yml' was found in current or parent dirs," \ "and \$DEFAULT_COMPOSE_FILE points to an unexistent file." die "Please provide a 'compose.yml' file." fi echo "$DEFAULT_COMPOSE_FILE" return 0 fi err "No 'compose.yml' was found in current or parent dirs, and no \$DEFAULT_COMPOSE_FILE was set." die "Please provide a 'compose.yml' file." return 1 } export -f get_compose_yml_location get_default_target_services() { local services=("$@") if [ -z "${services[*]}" ]; then if [ "$DEFAULT_SERVICES" ]; then info "No service provided, using \$DEFAULT_SERVICES variable. Target services: $DEFAULT_SERVICES" services="$DEFAULT_SERVICES" else err "No service provided." return 1 fi fi echo "${services[*]}" } get_master_services() { local loaded master_service declare -A loaded for service in "$@"; do master_service=$(_get_master_charm_for_service "$service") || return 1 if [ "${loaded[$master_service]}" ]; then continue fi echo "$master_service" loaded["$master_service"]=1 done | xargs echo } _setup_state_dir() { export state_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX) #debug "Creating temporary state directory in '$state_tmpdir'." # trap_add EXIT "debug \"Removing temporary state directory in $state_tmpdir.\";\ # rm -rf \"$state_tmpdir\"" trap_add EXIT "rm -rf \"$state_tmpdir\"" } [ "$SOURCED" ] && return 0 ## ## Argument parsing ## fullargs=() opts=() posargs=() no_hooks= no_init= while [ "$#" != 0 ]; do case "$1" in # --help|-h) # print_help # exit 0 # ;; --verbose|-v) fullargs+=("$1") export VERBOSE=true ;; -f) if ! [ -e "$2" ]; then die "File $2 doesn't exists" fi export DEFAULT_COMPOSE_FILE="$2" shift ;; -p) fullargs+=("$1" "$2") opts=("${opts[@]}" "$1" "$2") export DEFAULT_PROJECT_NAME="$2" shift ;; --no-relations) export no_relations=true ;; --no-hooks) export no_hooks=true ;; --no-init) export no_init=true ;; --debug) export DEBUG=true export VERBOSE=true ;; --) fullargs+=("$1") shift opts=("${opts[@]}" "$@") break 2 ;; -*) fullargs+=("$1" "$2") opts=("${opts[@]}" "$1" "$2") shift ;; *) fullargs+=("$1") posargs=("${posargs[@]}" "$1") ;; esac shift done if [ -z "$DISABLE_SYSTEM_CONFIG_FILE" ]; then if [ -r /etc/default/charm ]; then . /etc/default/charm fi if [ -r "/etc/default/$exname" ]; then . "/etc/default/$exname" fi ## XXXvlab: should provide YML config opportunities in possible parent dirs ? ## userdir ? and global /etc/compose.yml ? . /etc/compose.conf . /etc/compose.local.conf fi ## ## Actual code ## export CHARM_STORE=${CHARM_STORE:-/srv/charm-store} export DOCKER_DATASTORE=${DOCKER_DATASTORE:-/srv/docker-datastore} export COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1 debug "Found 'compose.yml' file in '$COMPOSE_YML_FILE'." if ! [ -d "$CHARM_STORE" ]; then err "Charm store path $YELLOW$CHARM_STORE$NORMAL does not exists. " err "Please check your $YELLOW\$CHARM_STORE$NORMAL variable value." exit 1 fi if [ -z "$(cd "$CHARM_STORE"; ls)" ]; then err "no available charms in charm store $YELLOW$CHARM_STORE$NORMAL. Either:" err " - check $YELLOW\$CHARM_STORE$NORMAL variable value" err " - download charms in $CHARM_STORE" print_error "Charm store is empty. Cannot continue." fi _setup_state_dir ## ## Get services in command line. ## action="${posargs[0]}" is_service_action= case "$action" in up|build|start|stop|config) services="$(get_default_target_services "${posargs[@]:1}")" || exit 1 orig_services="${posargs[@]:1}" ;; run) services="${posargs[1]}" ;; *) if is_service_action=$(has_service_action "${posargs[1]}" "$action"); then debug "Found action $DARKCYAN$action$NORMAL in service $DARKYELLOW${posargs[1]}$NORMAL" services="${posargs[1]}" else services="$(get_default_target_services)" fi ;; esac get_docker_compose $services >/dev/null || exit 1 ## precalculate variable \$_currnet_docker_compose ## ## Pre-action ## full_init= case "$action" in up|run) full_init=true ;; *) if [ "$is_service_action" ]; then full_init=true fi ;; esac if [ "$full_init" ]; then ## init in order Section initialisation if [ -z "$no_init" ]; then run_service_hook "$services" init || exit 1 fi ## Get relations if [ -z "$no_relations" ]; then run_service_relations "$services" || exit 1 fi ## XXXvlab: to be removed when all relation and service stuff is resolved if [ -z "$no_hooks" ]; then ordered_services=$(get_ordered_service_dependencies $services) || exit 1 for service in $ordered_services; do charm=$(get_service_charm "$service") || exit 1 for script in "$CHARM_STORE/$charm/hooks.d/"*.sh; do [ -e "$script" ] || continue [ -x "$script" ] || { echo "compose: script $script is not executable." >&2; exit 1; } ( debug "Launching '$script'." cd "$(dirname "$script)")"; "$script" "$@" ) || { echo "compose: hook $script failed. Stopping." >&2; exit 1; } done done fi fi ## ## Docker-compose ## case "$action" in up|start|stop|build) master_services=$(get_master_services $services) || exit 1 launch_docker_compose "$action" "${opts[@]}" $master_services ;; run) master_service=$(get_master_services $services) || exit 1 launch_docker_compose "$action" "$master_service" "${opts[@]}" "${posargs[@]:2}" ;; config) ## removing the services launch_docker_compose config "${opts[@]}" warn "Runtime configuration modification (from relations) are not included here." ;; *) if [ "$is_service_action" ]; then run_service_action "$services" "$action" "${opts[@]}" "${posargs[@]:2}" else launch_docker_compose "${fullargs[@]}" fi ;; esac