#!/bin/bash ## ## TODO: ## - subordinate container should really be able to modify base image of their master ## - this could be done through docker-update ## - I'm not happy with the current build using 'build/' directory, this should be ## changed to: ## - always have a base image (specified in metadata), and always have hooks/install ## executed and merge in image (like docker-build-charm). ## - container base image is ALWAYS the image of the master container... this brings ## questions about a double way to express inheritage (through relations as it is ## implemented now, or through this base-image ?) ## - the name of the scripts for relation (aka relation_name-relation-joined) is bad as ## reading the name in a hooks/ dir, there are no way to know if we are the target or ## the base of the relation. ## - we could leverage a 'relations/' dir on the root of the charm, with both: ## 'relations/provide/relation_name' and 'relations/receive/relation_name' ## - a very bad point with the actual naming is that we can't have a providing AND ## receiving a relation with same name. ## - The cache system should keep md5 of docker-compose and other things between runs ## - The cache system should use underlying function that have only arguments inputs. ## This will allow to cache completely without issues function in time. ## - would probably need instrospection in charm custom action to know if these need ## init or relations to be set up. #:- [ -e /etc/shlib ] && . /etc/shlib || { echo "Unsatisfied dependency. Please install 'kal-shlib-core'." exit 1 } #:- include pretty include parse include charm include array depends shyaml docker version=0.1 usage="$exname [COMPOSE_OPTS] [ACTION [ACTION_OPTS]]" help="\ $WHITE$exname$NORMAL jobs is to run various shell scripts to build a running orchestrated and configured docker containers. These shell scripts will have the opportunity to build a 'docker-compose.yml'. Once init script and relations scripts are executed, $WHITE$exname$NORMAL delegate the launching to ${WHITE}docker-compose${NORMAL} by providing it the final 'docker-compose.yml'. $WHITE$exname$NORMAL also leverage charms to offer some additional custom actions per charm, which are simply other scripts that can be run without launching ${WHITE}docker-compose${NORMAL}. In compose message, color coding is enforced as such: - ${DARKCYAN}action$NORMAL, - ${DARKBLUE}relation$NORMAL, - ${DARKYELLOW}charm${NORMAL}/${DARKYELLOW}service${NORMAL}, - ${WHITE}option-name${NORMAL}/${WHITE}command-name${NORMAL}/${WHITE}Section-Title${NORMAL} $WHITE$exname$NORMAL reads '/etc/compose.conf' for global variables, and '/etc/compose.local.conf' for local host adjustements. ${WHITE}$exname Options${NORMAL}: -h, --help Print this message and quit (ignoring any other options) -V, --version Print current version and quit (ignoring any other options) -v, --verbose Be more verbose -d, --debug Print full debugging information (sets also verbose) " [[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true export CACHEDIR=/var/cache/compose export VARDIR=/var/lib/compose md5_compat() { md5sum | cut -c -32; } quick_cat_file() { quick_cat_stdin < "$1"; } quick_cat_stdin() { local IFS=''; while read -r line; do echo "$line"; done ; } export -f quick_cat_file quick_cat_stdin md5_compat clean_cache() { local i=0 for f in $(ls -t "$CACHEDIR/"*.cache.* 2>/dev/null | tail -n +500); do ((i++)) rm -f "$f" done if (( i > 0 )); then debug "${WHITE}Cleaned cache:${NORMAL} Removed $((i)) elements (current cache size is $(du -sh "$CACHEDIR" | cut -f 1))" fi } trap_add "EXIT" clean_cache usage="$exname CHARM"' Deploy and manage a swarm of containers to provide services based on a ``compose.yml`` definition and charms from a ``charm-store``. ' export DEFAULT_COMPOSE_FILE ## ## Merge YAML files ## export _merge_yaml_common_code=" import sys import yaml try: # included in standard lib from Python 2.7 from collections import OrderedDict except ImportError: # try importing the backported drop-in replacement # it's available on PyPI from ordereddict import OrderedDict ## Ensure that there are no collision with legacy OrderedDict ## that could be used for omap for instance. class MyOrderedDict(OrderedDict): pass yaml.add_representer( MyOrderedDict, lambda cls, data: cls.represent_dict(data.items())) yaml.add_constructor( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, lambda cls, node: MyOrderedDict(cls.construct_pairs(node))) def fc(filename): with open(filename) as f: return f.read() def merge(*args): # sys.stderr.write('%r\n' % (args, )) args = [arg for arg in args if arg is not None] if len(args) == 0: return None if len(args) == 1: return args[0] if all(isinstance(arg, (int, basestring, bool)) for arg in args): return args[-1] elif all(isinstance(arg, list) for arg in args): res = [] for arg in args: for elt in arg: if elt in res: res.remove(elt) res.append(elt) return res elif all(isinstance(arg, dict) for arg in args): keys = set() for arg in args: keys |= set(arg.keys()) dct = {} for key in keys: sub_args = [] for arg in args: if key in arg: sub_args.append(arg) try: dct[key] = merge(*(a[key] for a in sub_args)) except NotImplementedError as e: raise NotImplementedError( e.args[0], '%s.%s' % (key, e.args[1]) if e.args[1] else key, e.args[2]) if dct[key] is None: del dct[key] return dct else: raise NotImplementedError( 'Unsupported types: %s' % (', '.join(list(set(arg.__class__.__name__ for arg in args)))), '', args) return None def merge_cli(*args): try: c = merge(*args) except NotImplementedError as e: sys.stderr.write('Merging Failed: %s.\n%s\n' ' Values are:\n %s\n' % (e.args[0], ' Conflicting key is %r.' % e.args[1] if e.args[1] else ' Conflict at base of structure.', '\\n '.join('v%d: %r' % (i, a) for i, a in enumerate(e.args[2])))) exit(1) if c is not None: print '%s' % yaml.dump(c, default_flow_style=False) " merge_yaml() { if ! [ -r "$state_tmpdir/merge_yaml.py" ]; then cat < "$state_tmpdir/merge_yaml.py" $_merge_yaml_common_code merge_cli(*(yaml.load(fc(f)) for f in sys.argv[1:])) EOF fi python "$state_tmpdir/merge_yaml.py" "$@" } export -f merge_yaml merge_yaml_str() { local entries="$@" if ! [ -r "$state_tmpdir/merge_yaml_str.py" ]; then cat < "$state_tmpdir/merge_yaml_str.py" $_merge_yaml_common_code merge_cli(*(yaml.load(f) for f in sys.argv[1:])) EOF fi python "$state_tmpdir/merge_yaml_str.py" "$@" } export -f merge_yaml_str yaml_key_val_str() { local entries="$@" if ! [ -r "$state_tmpdir/yaml_key_val_str.py" ]; then cat < "$state_tmpdir/yaml_key_val_str.py" $_merge_yaml_common_code print '%s' % yaml.dump({ yaml.load(sys.argv[1]): yaml.load(sys.argv[2])}, default_flow_style=False) EOF fi python "$state_tmpdir/yaml_key_val_str.py" "$@" } export -f yaml_key_val_str ## ## Docker ## docker_has_image() { local image="$1" images=$(docker images -q "$image" 2>/dev/null) || { err "docker images call has failed unexpectedly." return 1 } [ "$images" ] } export -f docker_has_image cmd_on_base_image() { local charm="$1" base_image shift base_image=$(service_base_docker_image "$charm") || return 1 docker run -i --entrypoint /bin/bash "$base_image" -c "$*" } export -f cmd_on_base_image cached_cmd_on_base_image() { local charm="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" shift if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" quick_cat_stdin < "$cache_file" return 0 fi result=$(cmd_on_base_image "$charm" "$@") || return 1 echo "$result" | tee "$cache_file" } export -f cached_cmd_on_base_image image_exposed_ports_0() { local image="$1" docker inspect --format='{{range $p, $conf := .Config.ExposedPorts}}{{$p}}{{"\x00"}}{{end}}' "$image" } export -f image_exposed_ports_0 ## ## Generic ## fn.exists() { declare -F "$1" >/dev/null } str_matches() { local str="$1" shift for pattern in "$@"; do eval "[[ \"$str\" == $pattern ]]" && return 0 done return 1 } gen_password() { local l=( {a..z} {A..Z} {0..9} ) nl="${#l[@]}" size=${1:-16} while ((size--)); do echo -n "${l[$((RANDOM * nl / 32768))]}" done echo } export -f gen_password file_put() { local TARGET="$1" mkdir -p "$(dirname "$TARGET")" && cat - > "$TARGET" } export -f file_put file_put_0() { local TARGET="$1" mkdir -p "$(dirname "$TARGET")" && cat > "$TARGET" } export -f file_put_0 fetch_file() { local src="$1" case "$src" in *"://"*) err "Unsupported target scheme." return 1 ;; *) ## Try direct if ! [ -r "$src" ]; then err "File '$src' not found/readable." return 1 fi cat "$src" || return 1 ;; esac } export -f fetch_file ## receives stdin content to decompress on stdout ## stdout content should be tar format. uncompress_file() { local filename="$1" ## Warning, the content of the file is already as stdin, the filename ## is there to hint for correct decompression. case "$filename" in *".gz") gunzip ;; *".bz2") bunzip2 ;; *) cat ;; esac } export -f uncompress_file get_file() { local src="$1" fetch_file "$src" | uncompress_file "$src" } export -f get_file ## ## Common database lib ## _clean_docker() { local _DB_NAME="$1" container_id="$2" ( set +e debug "Removing container $_DB_NAME" docker stop "$container_id" docker rm "$_DB_NAME" rm -vf "/tmp/${_DB_NAME}.state" ) } export -f _clean_docker get_service_base_image_dir_uid_gid() { local service="$1" dir="$2" uid_gid uid_gid=$(cached_cmd_on_base_image "$service" "stat -c '%u %g' '$dir'") || { debug "Failed to query '$dir' uid in ${DARKYELLOW}$service${NORMAL} base image." return 1 } info "uid and gid from ${DARKYELLOW}$service${NORMAL}:$dir is '$uid_gid'" echo "$uid_gid" } export -f get_service_base_image_dir_uid_gid are_files_locked_in_dir() { local dir="$1" device hdev ldev device=$(stat -c %d "$dir") || { err "Can't stat %d." return 1 } device=$(printf "%04x" $device) hdev=${device:0:2} ldev=${device:2:2} inodes=$(find "$dir" -printf ':%i:\n') found= while read -r inode; do debug "try inode:$inode" if [[ "$inodes" == *":$inode:"* ]]; then found=1 break fi done < <(cat /proc/locks | grep " $hdev:$ldev:" | sed -r "s/^.*$hdev:$ldev:([0-9]+).*$/\1/g") [ "$found" ] } export -f are_files_locked_in_dir export _PID="$$" ensure_db_docker_running () { _DB_NAME="db_${DB_NAME}_${_PID}" if [ -e "/tmp/${_DB_NAME}.state" ]; then export DOCKER_IP="$(cat "/tmp/${_DB_NAME}.state")" debug "Re-using previous docker/connection '$DOCKER_IP'." _set_db_params "$DOCKER_IP" return 0 fi if [ -e "/tmp/${_DB_NAME}.working" ]; then ## avoid recursive calls. if [ -z "$DOCKER_IP" ]; then err "Currently figuring up DOCKER_IP, please set it yourself before this call if needed." return 1 else debug "Ignoring recursive call." fi return 0 fi touch "/tmp/${_DB_NAME}.working" docker rm "$_DB_NAME" 2>/dev/null || true host_db_working_dir="$DATASTORE/${SERVICE_NAME}$DB_DATADIR" if is_db_locked; then info "Some process is using '$host_db_working_dir'. Trying to find a docker that would do this..." found= for docker_id in $(docker ps -q); do has_volume_mounted=$( docker inspect \ --format "{{range .Mounts}}{{if eq .Destination \"$DB_DATADIR\"}}{{.Source}}{{end}}{{end}}" \ "$docker_id") if [ "$has_volume_mounted" == "$host_db_working_dir" ]; then found="$docker_id" break fi done if [ -z "$found" ]; then err "Please shutdown any other docker using this directory." return 1 fi export container_id="$found" info "Found docker $docker_id is already running." else verb "Database is not locked." if ! docker_has_image "$DOCKER_BASE_IMAGE"; then docker pull "$DOCKER_BASE_IMAGE" fi docker_opts= if [ -f "$DB_PASSFILE" ]; then verb "Found and using '$DB_PASSFILE'." docker_opts="$db_docker_opts -v $SERVER_ROOT_PREFIX$DB_PASSFILE:$DB_PASSFILE" fi debug docker run -d \ --name "$_DB_NAME" \ $docker_opts \ -v "$host_db_working_dir:$DB_DATADIR" \ "$DOCKER_BASE_IMAGE" if ! container_id=$( docker run -d \ --name "$_DB_NAME" \ $docker_opts \ -v "$host_db_working_dir:$DB_DATADIR" \ "$DOCKER_BASE_IMAGE" ); then err "'docker run' failed !" _clean_docker "$_DB_NAME" "$container_id" rm "/tmp/${_DB_NAME}.working" return 1 fi trap_add EXIT,ERR "_clean_docker '$_DB_NAME' '$container_id'" fi if DOCKER_IP=$(wait_for_docker_ip "$container_id"); then echo "$DOCKER_IP" > "/tmp/${_DB_NAME}.state" debug "written /tmp/${_DB_NAME}.state" rm "/tmp/${_DB_NAME}.working" _set_db_params "$DOCKER_IP" return 0 else err "Db not found. Docker logs follows:" docker logs --tail=5 "$container_id" 2>&1 | prefix " | " >&2 rm "/tmp/${_DB_NAME}.working" return 1 fi } export -f ensure_db_docker_running ## Require to set $db_docker_opts if needed, and $DB_PASSFILE ## _dcmd() { local docker_opts command="$1" shift debug "Db> $command $@" if [ -f "$DB_PASSFILE" ]; then verb "Found and using '$DB_PASSFILE'." db_docker_opts="$db_docker_opts -v $SERVER_ROOT_PREFIX$DB_PASSFILE:$DB_PASSFILE" fi debug docker run -i --rm \ $db_docker_opts \ --entrypoint "$command" "$DOCKER_BASE_IMAGE" $db_cmd_opts "$@" docker run -i --rm \ $db_docker_opts \ --entrypoint "$command" "$DOCKER_BASE_IMAGE" $db_cmd_opts "$@" } export -f _dcmd ## Executes code through db dcmd () { [ "$DB_NAME" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_NAME." [ "$DB_DATADIR" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_DATADIR." [ "$DB_PASSFILE" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_PASSFILE." [ "$_PID" ] || print_syntax_error "$FUNCNAME: You must provide \$_PID." [ "$(type -t is_db_locked)" == "function" ] || print_syntax_error "$FUNCNAME: You must provide function 'is_db_locked'." [ "$(type -t _set_db_params)" == "function" ] || print_syntax_error "$FUNCNAME: You must provide function '_set_db_params'." [ "$(type -t ddb)" == "function" ] || print_syntax_error "$FUNCNAME: You must provide function 'ddb'." ensure_db_docker_running /dev/null } export -f get_docker_ip wait_docker_ip() { local name="$1" timeout="${2:-15}" timeout_count=0 docker_ip= start=$SECONDS while [ -z "$docker_ip" ]; do sleep 0.5 docker_ip=$(get_docker_ip "$name") && break elapsed=$((SECONDS - start)) if ((elapsed > timeout)); then err "${RED}timeout error${NORMAL}(${timeout}s):" \ "Could not find '$name' docker container's IP." return 1 fi [ "$elapsed" == "$old_elapsed" ] || verb "Waiting for docker $name... ($elapsed/$timeout)" old_elapsed="$elapsed" done verb "Found docker $name IP: $docker_ip" echo "$docker_ip" } export -f wait_docker_ip ## Warning: requires a ``ddb`` matching current database to be checked wait_for_docker_ip() { local name=$1 timeout=5 timeout_count=0 DOCKER_IP= DB_OK= DOCKER_IP=$(wait_docker_ip "$name" 5) || return 1 timeout_count=0 DB_OK= _set_db_params "$DOCKER_IP" while [ -z "$DB_OK" ]; do sleep 1 echo "SELECT 1;" | ddb >/dev/null && DB_OK=1 verb "[2/2] Waiting for db service from docker $name... ($[timeout_count + 1]/$timeout)" ((timeout_count++)) || true if [ "$timeout_count" == "$timeout" ]; then err "${RED}timeout error${NORMAL}(${timeout}s):"\ "Could not connect to db on $DOCKER_IP." \ "container's IP." return 1 fi done verb "[2/2] Db is ready !" echo "$DOCKER_IP" return 0 } export -f wait_for_docker_ip docker_add_host_declaration() { local src_docker=$1 domain=$2 dst_docker=$3 dst_docker_ip= dst_docker_ip=$(wait_docker_ip "$dst_docker") || exit 1 docker exec -i "$src_docker" bash < /dev/null 2>&1; then sed -ri "s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+$domain\$/$dst_docker_ip $domain/g" /etc/hosts else echo "$dst_docker_ip $domain" >> /etc/hosts fi EOF } export -f docker_add_host_declaration ## ## Internal Process ## get_docker_compose_links() { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ links charm charm_part master_charm if [ -z "$service" ]; then print_syntax_error "$FUNCNAME: Please specify a service as first argument." return 1 fi if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi master_charm=$(get_top_master_charm_for_service "$service") || return 1 deps=() while read-0 relation_name target_service relation_config tech_dep; do master_target_charm="$(get_top_master_charm_for_service "$target_service")" [ "$master_charm" == "$master_target_charm" ] && continue if [ "$tech_dep" == "reversed" ]; then deps+=("$(echo -en "$master_target_charm:\n links:\n - $master_charm")") elif [ "$tech_dep" == "True" ]; then deps+=("$(echo -en "$master_charm:\n links:\n - $master_target_charm")") fi done < <(get_compose_relations "$service") || return 1 merge_yaml_str "${deps[@]}" | tee "$cache_file" } _get_docker_compose_opts() { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ links charm charm_part master_charm if [ -z "$service" ]; then print_syntax_error "$FUNCNAME: Please specify a service as first argument." return 1 fi if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi compose_def="$(get_compose_service_def "$service")" || return 1 master_charm="$(get_top_master_charm_for_service "$service")" docker_compose_opts=$(echo "$compose_def" | shyaml get-value "docker-compose" 2>/dev/null) if [ "$docker_compose_opts" ]; then yaml_key_val_str "$master_charm" "$docker_compose_opts" fi | tee "$cache_file" } ## ## By Reading the metadata.yml, we create a docker-compose.yml mixin. ## Some metadata.yml (of subordinates) will indeed modify other ## services than themselves. _get_docker_compose_service_mixin() { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" links charm charm_part if [ -z "$service" ]; then print_syntax_error "$FUNCNAME: Please specify a service as first argument." return 1 fi if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi master_charm=$(get_top_master_charm_for_service "$service") || { err "Failed to get top master charm for service $DARKYELLOW$service$NORMAL" return 1 } ## The compose part links_yaml=$(get_docker_compose_links "$service") || return 1 docker_compose_options=$(_get_docker_compose_opts "$service") || return 1 ## the charm part #debug "Get charm name from service name $DARKYELLOW$service$NORMAL." charm=$(get_service_charm "$service") || return 1 charm_part=$(get_docker_compose_mixin_from_metadata "$charm") || return 1 ## Merge results if [ "$charm_part" ]; then charm_yaml="$(yaml_key_val_str "$master_charm" "$charm_part")" || return 1 merge_yaml_str "$links_yaml" "$charm_yaml" "$docker_compose_options" || return 1 else echo "$links_yaml" fi | tee "$cache_file" } export -f _get_docker_compose_service_mixin ## ## Get full `docker-compose.yml` format for all listed services (and ## their deps) ## ## @export ## @cache: !system !nofail +stdout get_docker_compose () { local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" entries services service start if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi ## ## Adding sub services configurations ## declare -A entries start_compilation=$SECONDS debug "Compiling 'docker-compose.yml' base for $DARKYELLOW$@$NORMAL..." for target_service in "$@"; do start=$SECONDS services=$(get_ordered_service_dependencies "$target_service") || { err "Failed to get dependencies for $DARKYELLOW$target_service$NORMAL" return 1 } debug " $DARKYELLOW$target_service$NORMAL deps:$DARKYELLOW" $services "$NORMAL$GRAY(in $((SECONDS - start))s)$NORMAL" for service in $services; do if [ "${entries[$service]}" ]; then ## Prevent double inclusion of same service if this ## service is deps of two or more of your ## requirements. continue fi ## mark the service as "loaded" as well as it's containers ## if this is a subordinate service start_service=$SECONDS entries[$service]=$(_get_docker_compose_service_mixin "$service") || { err "Failed to get service mixin for $DARKYELLOW$service$NORMAL" return 1 } debug " Applied $DARKYELLOW$service$NORMAL charm metadata mixins $GRAY(in $((SECONDS - start_service))s)$NORMAL" done debug " ..finished all mixins for $DARKYELLOW$target_service$NORMAL $GRAY(in $((SECONDS - start))s)$NORMAL" done merge_yaml_str "${entries[@]}" > "$cache_file" export _current_docker_compose="$(cat "$cache_file")" echo "$_current_docker_compose" debug " ..compilation of base 'docker-compose.yml' done $GRAY(in $((SECONDS - start_compilation))s)$NORMAL" || true # debug " ** ${WHITE}docker-compose.yml${NORMAL}:" # debug "$_current_docker_compose" } export -f get_docker_compose _get_compose_service_def_cached () { local service="$1" docker_compose="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" if [ -e "$cache_file" ]; then #debug "$FUNCNAME: STATIC cache hit" cat "$cache_file" touch "$cache_file" return 0 fi service_def_base="charm: $service" value=$(echo "$docker_compose" | shyaml get-value "$service" 2>/dev/null) merge_yaml <(echo "$service_def_base") <(echo "$value") | tee "$cache_file" } export -f _get_compose_service_def_cached ## XXXvlab: a lot to be done to cache the results get_compose_service_def () { local service="$1" docker_compose cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ result if [ -e "$cache_file" ]; then #debug "$FUNCNAME: SESSION cache hit" cat "$cache_file" return 0 fi [ -z "$service" ] && print_syntax_error "Missing service as first argument." docker_compose=$(cat "$COMPOSE_YML_FILE") || return 1 result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1 echo "$result" | tee "$cache_file" } export -f get_compose_service_def _get_service_charm_cached () { local service="$1" service_def="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit $1" cat "$cache_file" touch "$cache_file" return 0 fi charm=$(echo "$service_def" | shyaml get-value charm 2>/dev/null) if [ -z "$charm" ]; then err "Missing charm in service $DARKYELLOW$service$NORMAL definition." return 1 fi echo "$charm" | tee "$cache_file" } export -f _get_service_charm_cached get_service_charm () { local service="$1" if [ -z "$service" ]; then print_syntax_error "$FUNCNAME: Please specify a service as first argument." return 1 fi service_def=$(get_compose_service_def "$service") || return 1 _get_service_charm_cached "$service" "$service_def" } export -f get_service_charm ## built above the docker-compose abstraction, so it relies on the ## full docker-compose.yml to be already built. get_service_def () { local service="$1" def if [ -z "$_current_docker_compose" ]; then print_syntax_error "$FUNCNAME is meant to be called after"\ "\$_current_docker_compose has been calculated." fi def=$(echo "$_current_docker_compose" | shyaml get-value "$service" 2>/dev/null) if [ -z "$def" ]; then err "No definition for service $DARKYELLOW$service$NORMAL in compiled 'docker-compose.yml'." return 1 fi echo "$def" } export -f get_service_def ## Return the base docker image name of a service service_base_docker_image() { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ master_charm charm service_image service_build service_dockerfile if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi master_charm="$(get_top_master_charm_for_service "$service")" || { err "Could not compute base charm for service $DARKYELLOW$service$NORMAL." return 1 } service_def="$(get_service_def "$master_charm")" || { err "Could not get docker-compose service definition for $DARKYELLOW$master_charm$NORMAL." return 1 } service_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null) if [ "$?" != 0 ]; then service_build=$(echo "$service_def" | shyaml get-value build 2>/dev/null) if [ "$?" != 0 ]; then err "Service $DARKYELLOW$service$NORMAL has no ${WHITE}image${NORMAL} nor ${WHITE}build${NORMAL} parameter." echo "$service_def" >&2 return 1 fi service_dockerfile="$(charm.get_dir "${service_build%%/*}")"/"${service_build#*/}"/Dockerfile if ! [ -e "$service_dockerfile" ]; then err "No Dockerfile found in '$service_dockerfile' location." return 1 fi grep '^FROM' "$service_dockerfile" | xargs echo | cut -f 2 -d " " else echo "$service_image" fi | tee "$cache_file" } export -f service_base_docker_image get_charm_relation_def () { local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \ relation_def metadata if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi metadata="$(charm.metadata "$charm")" || return 1 relation_def="$(echo "$metadata" | shyaml get-value "provides.${relation_name}" 2>/dev/null)" echo "$relation_def" | tee "$cache_file" } export -f get_charm_relation_def get_charm_tech_dep_orientation_for_relation() { local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \ relation_def metadata value if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi relation_def=$(get_charm_relation_def "$charm" "$relation_name" 2>/dev/null) value=$(echo "$relation_def" | shyaml get-value 'tech-dep' 2>/dev/null) value=${value:-True} echo "$value" | tee "$cache_file" } export -f get_charm_tech_dep_orientation_for_relation ## ## Use compose file to get deps, and relation definition in metadata.yml ## for tech-dep attribute. get_service_deps() { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi ( set -o pipefail get_compose_relations "$service" | \ while read-0 relation_name target_service _relation_config tech_dep; do echo "$target_service" done | tee "$cache_file" ) || return 1 } export -f get_service_deps _rec_get_depth() { local elt=$1 dep deps max cache_file="$state_tmpdir/$FUNCNAME.cache.$1" [ "${depths[$elt]}" ] && return 0 if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" depths[$elt]=$(cat "$cache_file") return 0 fi visited[$elt]=1 #debug "Setting visited[$elt]" #debug "Asking for $DARKYELLOW$elt$NORMAL dependencies" deps=$(get_service_deps "$elt") || { debug "Failed get_service_deps $elt" return 1 } # debug "$elt deps are:" $deps max=0 for dep in $deps; do [ "${visited[$dep]}" ] && { #debug "Already computing $dep" continue } _rec_get_depth "$dep" || return 1 #debug "Requesting depth[$dep]" if (( ${depths[$dep]} > max )); then max="${depths[$dep]}" fi done # debug "Setting depth[$elt] to $((max + 1))" depths[$elt]=$((max + 1)) echo "${depths[$elt]}" > $cache_file } export -f _rec_get_depth get_ordered_service_dependencies() { local services=("$@") cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi #debug "Figuring ordered deps of $DARKYELLOW$services$NORMAL" if [ -z "${services[*]}" ]; then print_syntax_error "$FUNCNAME: no arguments" return 1 fi declare -A depths declare -A visited heads=("${services[@]}") while [ "${#heads[@]}" != 0 ]; do array_pop heads head _rec_get_depth "$head" || return 1 done i=0 while [ "${#depths[@]}" != 0 ]; do for key in "${!depths[@]}"; do value="${depths[$key]}" if [ "$value" == "$i" ]; then echo "$key" unset depths[$key] fi done i=$((i + 1)) done | tee "$cache_file" } export -f get_ordered_service_dependencies run_service_hook () { local services="$1" action="$2" loaded declare -A loaded for service in $services; do for subservice in $(get_ordered_service_dependencies "$service"); do if [ "${loaded[$subservice]}" ]; then ## Prevent double inclusion of same service if this ## service is deps of two or more of your ## requirements. continue fi charm=$(get_service_charm "$subservice") || return 1 charm.has_hook "$charm" "$action" >/dev/null || continue PROJECT_NAME=$(get_default_project_name) || return 1 Wrap -d "$YELLOW$action$NORMAL hook of charm $DARKYELLOW$charm$NORMAL" </dev/null) || { err "Missing ${WHITE}type$NORMAL option in ${WHITE}get$NORMAL config for location '$location'" return 1 } if fn.exists host_resource_get_$type; then host_resource_get_$type "$location" "$cfg" else err "Source ${WHITE}source$NORMAL type '$type' unknown for" \ "${WHITE}host-resource$NORMAL '$location' defined in" \ "$DARKYELLOW$subservice$NORMAL config." return 1 fi } export -f host_resource_get host_resource_get_git() { local location="$1" cfg="$2" branch parent url branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null) branch=${branch:-master} url=$(echo "$cfg" | shyaml get-value url 2>/dev/null) parent="$(dirname "$location")" ( mkdir -p "$parent" && cd "$parent" && git clone -b "$branch" "$url" "$(basename "$location")" ) || return 1 } export -f host_resource_get_git host_resource_get_git-sub() { local location="$1" cfg="$2" branch parent url branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null) branch=${branch:-master} url=$(echo "$cfg" | shyaml get-value url 2>/dev/null) parent="$(dirname "$location")" ( mkdir -p "$parent" && cd "$parent" && git sub clone -b "$branch" "$url" "$(basename "$location")" ) || return 1 } export -f host_resource_get_git-sub setup_host_resource () { local service="$1" service_def location get cfg service_def=$(get_compose_service_def "$subservice") || return 1 while read-0 location cfg; do ## XXXvlab: will it be a git resources always ? if [ -d "$location" -a ! -d "$location/.git" ]; then err "Hum, location '$location' does not seem to be a git directory." return 1 fi if [ -d "$location" ]; then info "host resource '$location' already set up." continue fi get=$(echo "$cfg" | shyaml get-value get 2>/dev/null) if [ -z "$get" ]; then err "No host directory '$location' found, and no ${WHITE}source$NORMAL" \ "specified for $DARKYELLOW$subservice$NORMAL." return 1 fi host_resource_get "$location" "$get" || return 1 done < <(echo "$service_def" | shyaml key-values-0 host-resources 2>/dev/null) } export -f setup_host_resource setup_host_resources () { local services="$1" loaded declare -A loaded for service in $services; do for subservice in $(get_ordered_service_dependencies "$service"); do if [ "${loaded[$subservice]}" ]; then ## Prevent double inclusion of same service if this ## service is deps of two or more of your ## requirements. continue fi setup_host_resource "$service" loaded[$subservice]=1 done done return 0 } export -f setup_host_resources relation-get () { local key="$1" cat "$RELATION_DATA_FILE" | shyaml get-value "$key" 2>/dev/null if [ "$?" != 0 ]; then err "The key $WHITE$key$NORMAL was not found in relation's data." return 1 fi } export -f relation-get relation-base-compose-get () { local key="$1" echo "$RELATION_BASE_COMPOSE_DEF" | shyaml get-value "options.$key" 2>/dev/null if [ "$?" != 0 ]; then err "The key $WHITE$key$NORMAL was not found in base service compose definition.." return 1 fi } export -f relation-base-compose-get relation-target-compose-get () { local key="$1" echo "$RELATION_BASE_COMPOSE_DEF" | shyaml get-value "options.$key" 2>/dev/null if [ "$?" != 0 ]; then err "The key $WHITE$key$NORMAL was not found in base service compose definition.." return 1 fi } export -f relation-target-compose-get relation-set () { local key="$1" value="$2" if [ -z "$RELATION_DATA_FILE" ]; then err "$FUNCNAME: relation does not seems to be correctly setup." return 1 fi if ! [ -r "$RELATION_DATA_FILE" ]; then err "$FUNCNAME: can't read relation's data." >&2 return 1 fi _config_merge "$RELATION_DATA_FILE" <(echo "$key: $value") } export -f relation-set _config_merge() { local config_filename="$1" merge_to_file="$2" touch "$config_filename" && merge_yaml "$config_filename" "$merge_to_file" > "$config_filename.tmp" || return 1 mv "$config_filename.tmp" "$config_filename" } export -f _config_merge ## XXXvlab; this can be used only in relation, I'd like to use it in init. config-add() { local metadata="$1" _config_merge "$RELATION_CONFIG" <(echo "$metadata") } export -f config-add ## XXXvlab; this can be used only in relation, I'd like to use it in init. init-config-add() { local metadata="$1" _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" <(echo "$metadata") } export -f init-config-add logstdout() { local name="$1" sed -r 's%^%'"${name}"'> %g' } export -f logstdout logstderr() { local name="$1" sed -r 's%^(.*)$%'"${RED}${name}>${NORMAL} \1"'%g' } export -f logstderr _run_service_relation () { local relation_name="$1" service="$2" target_service="$3" relation_config="$4" relation_dir services charm=$(get_service_charm "$service") || return 1 target_charm=$(get_service_charm "$target_service") || return 1 base_script_name=$(charm.has_relation_hook "$charm" "$relation_name" relation-joined) || true target_script_name=$(charm.has_relation_hook "$target_charm" "$relation_name" relation-joined) || true [ "$base_script_name" -o "$target_script_name" ] || return 0 relation_dir=$(get_relation_data_dir "$charm" "$target_charm" "$relation_name") || return 1 RELATION_DATA_FILE=$(get_relation_data_file "$charm" "$target_charm" "$relation_name" "$relation_config") || return 1 export BASE_CHARM_NAME=$charm export BASE_CHARM_PATH=$(charm.get_dir "$charm") export TARGET_CHARM_NAME=$target_charm export TARGET_CHARM_PATH=$(charm.get_dir "$target_charm") export RELATION_DATA_FILE target_errlvl=0 if [ -z "$target_script_name" ]; then verb "No relation script $DARKBLUE$relation_name$NORMAL in target $DARKYELLOW$target_charm$NORMAL." else verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \ "for target charm $DARKYELLOW$target_charm$NORMAL" RELATION_CONFIG="$relation_dir/config_provider" DOCKER_BASE_IMAGE=$(service_base_docker_image "$target_service") || return 1 export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA { ( SERVICE_NAME=$target_service SERVICE_DATASTORE="$DATASTORE/$target_charm" SERVICE_CONFIGSTORE="$CONFIGSTORE/$target_charm" export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE charm.run_relation_hook "$target_charm" "$relation_name" relation-joined echo "$?" > "$relation_dir/target_errlvl" ) | logstdout "$DARKYELLOW$target_charm$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}" } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$target_charm$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@${NORMAL}" 3>&1 1>&2 2>&3 target_errlvl="$(cat "$relation_dir/target_errlvl")" || { err "Relation script '$script_name' in $DARKYELLOW$target_charm$NORMAL" \ "failed before outputing an errorlevel." ((target_errlvl |= "1" )) } if [ -e "$RELATION_CONFIG" ]; then debug "Merging some new config info in $DARKYELLOW$target_service$NORMAL" _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" && rm "$RELATION_CONFIG" ((target_errlvl |= "$?")) fi fi if [ "$target_errlvl" == 0 ]; then errlvl=0 if [ "$base_script_name" ]; then verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \ "for charm $DARKYELLOW$charm$NORMAL" RELATION_CONFIG="$relation_dir/config_providee" RELATION_DATA="$(cat "$RELATION_DATA_FILE")" DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") || return 1 export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA { ( SERVICE_NAME=$service SERVICE_DATASTORE="$DATASTORE/$charm" SERVICE_CONFIGSTORE="$CONFIGSTORE/$charm" export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE charm.run_relation_hook "$charm" "$relation_name" relation-joined echo "$?" > "$relation_dir/errlvl" ) | logstdout "$DARKYELLOW$charm$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}" } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$charm$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@$NORMAL" 3>&1 1>&2 2>&3 errlvl="$(cat "$relation_dir/errlvl")" || { err "Relation script '$script_name' in $DARKYELLOW$charm$NORMAL" \ "failed before outputing an errorlevel." ((errlvl |= "1" )) } if [ -e "$RELATION_CONFIG" ]; then _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" && rm "$RELATION_CONFIG" ((errlvl |= "$?" )) fi if [ "$errlvl" != 0 ]; then err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$charm$NORMAL failed to run properly." fi else verb "No relation script '$script_name' in charm $DARKYELLOW$charm$NORMAL. Ignoring." fi else err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$target_charm$NORMAL failed to run properly." fi if [ "$target_errlvl" == 0 -a "$errlvl" == 0 ]; then debug "Relation $DARKBLUE$relation_name$NORMAL is established" \ "between $DARKYELLOW$service$NORMAL and $DARKYELLOW$target_service$NORMAL." return 0 else return 1 fi } export -f _run_service_relation _get_compose_relations_cached () { local compose_service_def="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ relation_name relation_def target_service if [ -e "$cache_file" ]; then #debug "$FUNCNAME: STATIC cache hit $1" cat "$cache_file" touch "$cache_file" return 0 fi ( set -o pipefail if [ "$compose_service_def" ]; then while read-0 relation_name relation_def; do ( case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in "str") target_service="$(echo "$relation_def" | shyaml get-value 2>/dev/null)" tech_dep="$(get_charm_tech_dep_orientation_for_relation "$target_service" "$relation_name")" echo -en "$relation_name\0$target_service\0\0$tech_dep\0" ;; "sequence") while read-0 target_service; do tech_dep="$(get_charm_tech_dep_orientation_for_relation "$target_service" "$relation_name")" echo -en "$relation_name\0$target_service\0\0$tech_dep\0" done < <(echo "$relation_def" | shyaml get-values-0 2>/dev/null) ;; "struct") while read-0 target_service relation_config; do tech_dep="$(get_charm_tech_dep_orientation_for_relation "$target_service" "$relation_name")" echo -en "$relation_name\0$target_service\0$relation_config\0$tech_dep\0" done < <(echo "$relation_def" | shyaml key-values-0 2>/dev/null) ;; esac ) > "$cache_file" done < <(echo "$compose_service_def" | shyaml key-values-0 relations 2>/dev/null) fi ) if [ "$?" != 0 ]; then err "Error while looking for compose relations." rm -f "$cache_file" ## no cache return 1 fi [ -e "$cache_file" ] && cat "$cache_file" return 0 } export -f _get_compose_relations_cached get_compose_relations () { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ compose_def if [ -e "$cache_file" ]; then #debug "$FUNCNAME: SESSION cache hit $1" cat "$cache_file" return 0 fi compose_def="$(get_compose_service_def "$service")" || return 1 _get_compose_relations_cached "$compose_def" > "$cache_file" if [ "$?" != 0 ]; then err "Error while looking for compose relations." rm -f "$cache_file" ## no cache return 1 fi cat "$cache_file" } export -f get_compose_relations run_service_relations () { local services="$1" loaded PROJECT_NAME=$(get_default_project_name) || return 1 export PROJECT_NAME declare -A loaded for service in $(get_ordered_service_dependencies $services); do # debug "Upping dep's relations of ${DARKYELLOW}$service${NORMAL}:" for subservice in $(get_service_deps "$service") "$service"; do [ "${loaded[$subservice]}" ] && continue export BASE_SERVICE_NAME=$service MASTER_BASE_CHARM_NAME=$(get_top_master_charm_for_service "$subservice") || return 1 RELATION_BASE_COMPOSE_DEF=$(get_compose_service_def "$subservice") || return 1 export RELATION_BASE_COMPOSE_DEF MASTER_BASE_CHARM_NAME # debug " Relations of ${DARKYELLOW}$subservice${NORMAL}:" while read-0 relation_name target_service relation_config tech_dep; do export relation_config export TARGET_SERVICE_NAME=$target_service MASTER_TARGET_CHARM_NAME=$(get_top_master_charm_for_service "$target_service") || return 1 RELATION_TARGET_COMPOSE_DEF=$(get_compose_service_def "$target_service") || return 1 export MASTER_TARGET_CHARM_NAME RELATION_TARGET_COMPOSE_DEF Wrap -d "Building $DARKYELLOW$subservice$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW$target_service$NORMAL" < "$action_errlvl_file" ) | logstdout "$DARKYELLOW$charm$NORMAL/${DARKCYAN}$action${NORMAL} ${GREEN}@${NORMAL}" } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$charm$NORMAL/${DARKCYAN}$action${NORMAL} ${RED}@$NORMAL" 3>&1 1>&2 2>&3 if ! [ -e "$action_errlvl_file" ]; then err "Action $DARKYELLOW$service$NORMAL:$DARKCYAN$action$NORMAL has failed without having time" \ "to output an errlvl" return 1 fi return "$(cat "$action_errlvl_file")" } export -f _run_service_action_direct _run_service_action_relation() { local service="$1" action="$2" charm target_charm relation_name relation_config _dummy shift; shift read-0 charm target_charm relation_name relation_config || true if read-0 _dummy || [ "$_dummy" ]; then print_syntax_error "$FUNCNAME: too many arguments in action descriptor" return 1 fi export RELATION_DATA_FILE=$(get_relation_data_file "$charm" "$target_charm" "$relation_name" "$relation_config") compose_file=$(get_compose_yml_location) || return 1 export action_errlvl_file="$state_tmpdir/action-$service-$charm-$action-errlvl" export state_tmpdir { ( set +e ## Prevents unwanted leaks from parent shell export METADATA_CONFIG=$(charm.metadata "$charm") export SERVICE_NAME=$service export RELATION_TARGET_CHARM="$target_charm" export RELATION_BASE_CHARM="$charm" export ACTION_NAME=$action export CONTAINER_NAME=$(get_top_master_charm_for_service "$service") export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME") export SERVICE_DATASTORE="$DATASTORE/$service" export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service" exname="$exname $ACTION_NAME $SERVICE_NAME" \ stdbuf -oL -eL bash -c 'charm.run_relation_action "$@"' -- "$target_charm" "$relation_name" "$action" "$@" echo "$?" > "$action_errlvl_file" ) | logstdout "$DARKYELLOW$charm$NORMAL/${DARKCYAN}$action${NORMAL} ${GREEN}@${NORMAL}" } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$charm$NORMAL/${DARKCYAN}$action${NORMAL} ${RED}@$NORMAL" 3>&1 1>&2 2>&3 if ! [ -e "$action_errlvl_file" ]; then err "Action $DARKYELLOW$service$NORMAL:$DARKCYAN$action$NORMAL has failed without having time" \ "to output an errlvl" return 1 fi return "$(cat "$action_errlvl_file")" } export -f _run_service_action_relation get_relation_data_dir() { local charm="$1" target_charm="$2" relation_name="$3" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi project=$(get_default_project_name) || return 1 relation_dir="$VARDIR/relations/$project/$charm-$target_charm/$relation_name" if ! [ -d "$relation_dir" ]; then mkdir -p "$relation_dir" || return 1 chmod go-rwx "$relation_dir" || return 1 ## protecting this directory fi echo "$relation_dir" || tee "$cache_file" } export -f get_relation_data_dir get_relation_data_file() { local charm="$1" target_charm="$2" relation_name="$3" relation_config="$4" relation_dir=$(get_relation_data_dir "$charm" "$target_charm" "$relation_name") || return 1 relation_data_file="$relation_dir/data" new= if [ -e "$relation_data_file" ]; then ## Has reference changed ? new_md5=$(echo "$relation_config" | md5_compat) if [ "$new_md5" != "$(cat "$relation_data_file.md5_ref" 2>/dev/null)" ]; then new=true fi else new=true fi if [ "$new" ]; then echo "$relation_config" > "$relation_data_file" chmod go-rwx "$relation_data_file" ## protecting this file echo "$relation_config" | md5_compat > "$relation_data_file.md5_ref" fi echo "$relation_data_file" } export -f get_relation_data_file has_service_action () { local service="$1" action="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ charm target_charm if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi charm=$(get_service_charm "$service") || return 1 ## Action directly provided ? if charm.has_direct_action "$charm" "$action" >/dev/null; then echo -en "direct\0$charm" | tee "$cache_file" return 0 fi ## Action provided by relation ? while read-0 relation_name target_service relation_config tech_dep; do target_charm=$(get_service_charm "$target_service") || return 1 if charm.has_relation_action "$target_charm" "$relation_name" "$action" >/dev/null; then echo -en "relation\0$charm\0$target_charm\0$relation_name\0$relation_config" | tee "$cache_file" return 0 fi done < <(get_compose_relations "$service") return 1 # master=$(get_top_master_charm_for_service "$charm") # [ "$master" == "$charm" ] && return 1 # has_service_action "$master" "$action" } export -f has_service_action run_service_action () { local service="$1" action="$2" shift ; shift { if ! read-0 action_type; then info "Service $DARKYELLOW$service$NORMAL does not have any action $DARKCYAN$action$NORMAL defined." info " Add an executable script to 'actions/$action' to implement action." return 1 fi Section "running $DARKYELLOW$service$NORMAL/$DARKCYAN$action$NORMAL ($action_type)"; Feed "_run_service_action_${action_type}" "$service" "$action" "$@" } < <(has_service_action "$service" "$action") } export -f run_service_action get_compose_relation_config() { local service=$1 relation_config cache_file="$state_tmpdir/$FUNCNAME.cache.$1" if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi compose_service_def=$(get_compose_service_def "$service") || return 1 echo "$compose_service_def" | shyaml get-value "relations" 2>/dev/null | tee "$cache_file" } export -f get_compose_relation_config # ## Return key-values-0 # get_compose_relation_config_for_service() { # local service=$1 relation_name=$2 relation_config # compose_service_relations=$(get_compose_relation_config "$service") || return 1 # if ! relation_config=$( # echo "$compose_service_relations" | # shyaml get-value "${relation_name}" 2>/dev/null); then # err "Couldn't find $DARKYELLOW${service}$NORMAL/${WHITE}${relation_name}$NORMAL" \ # "relation config in compose configuration." # return 1 # fi # if [ -z "$relation_config" ]; then # err "Relation ${WHITE}mysql-database$NORMAL is empty in compose configuration." # return 1 # fi # if ! echo "$relation_config" | shyaml key-values-0 2>/dev/null; then # err "No key/values in ${DARKBLUE}mysql-database$NORMAL of compose config." # return 1 # fi # } # export -f get_compose_relation_config_for_service _get_master_charm_for_service_cached () { local service="$1" charm="$2" metadata="$3" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ charm requires master_charm target_charm target_service service_def if [ -e "$cache_file" ]; then # debug "$FUNCNAME: STATIC cache hit ($1)" cat "$cache_file" touch "$cache_file" return 0 fi if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" != "True" ]; then ## just return charm name echo "$charm" | tee "$cache_file" return 0 fi ## fetch the container relation requires="$(echo "$metadata" | shyaml get-value "requires" 2>/dev/null)" if [ -z "$requires" ]; then die "Charm $DARKYELLOW$charm$NORMAL is a subordinate but does not have any 'requires' " \ "section." fi master_charm= while read-0 relation_name relation; do if [ "$(echo "$relation" | shyaml get-value "scope" 2>/dev/null)" == "container" ]; then # debug "$DARKYELLOW$service$NORMAL's relation" \ # "$DARKBLUE${relation_name}$NORMAL is a container." interface="$(echo "$relation" | shyaml get-value "interface" 2>/dev/null)" if [ -z "$interface" ]; then err "No ${WHITE}$interface${NORMAL} set for relation $relation_name." return 1 fi ## Action provided by relation ? target_service= while read-0 relation_name candidate_target_service _relation_config _tech_dep; do [ "$interface" == "$relation_name" ] && { target_service="$candidate_target_service" break } done < <(get_compose_relations "$service") if [ -z "$target_service" ]; then err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \ "${DARKYELLOW}$service$NORMAL compose definition." return 1 fi master_charm=$(get_service_charm "$target_service") || return 1 break fi done < <(echo "$requires" | shyaml key-values-0 2>/dev/null) if [ -z "$master_charm" ]; then die "Charm $DARKYELLOW$charm$NORMAL is a subordinate but does not have any relation with" \ " ${WHITE}scope${NORMAL} set to 'container'." fi echo "$master_charm" | tee "$cache_file" } export -f _get_master_charm_for_service_cached get_master_charm_for_service() { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ charm metadata result if [ -e "$cache_file" ]; then # debug "$FUNCNAME: SESSION cache hit ($*)" cat "$cache_file" return 0 fi charm=$(get_service_charm "$service") || return 1 metadata=$(charm.metadata "$charm") || { err "Failed to access metadata of charm $DARKYELLOW$charm$NORMAL" return 1 } result=$(_get_master_charm_for_service_cached "$service" "$charm" "$metadata") || return 1 echo "$result" | tee "$cache_file" } export -f get_master_charm_for_service get_top_master_charm_for_service() { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ current_service if [ -e "$cache_file" ]; then # debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi current_service="$service" while true; do master_service=$(get_master_charm_for_service "$current_service") || return 1 [ "$master_service" == "$current_service" ] && break current_service="$master_service" done echo "$current_service" | tee "$cache_file" return 0 } export -f get_top_master_charm_for_service ## ## The result is a mixin that is not always a complete valid ## docker-compose entry (thinking of subordinates). The result ## will be merge with master charms. _get_docker_compose_mixin_from_metadata_cached() { local charm="$1" metadata="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ metadata_file metadata volumes docker_compose subordinate image if [ -e "$cache_file" ]; then #debug "$FUNCNAME: STATIC cache hit $1" cat "$cache_file" touch "$cache_file" return 0 fi mixin= if [ "$metadata" ]; then ## resources to volumes volumes=$( for resource_type in data config; do while read-0 resource; do eval "echo \" - \$${resource_type^^}STORE/\$charm\$resource:\$resource:rw\"" done < <(echo "$metadata" | shyaml get-values-0 "${resource_type}-resources" 2>/dev/null) done while read-0 resource; do if [[ "$resource" == *:* ]]; then echo " - $resource:rw" else echo " - $resource:$resource:rw" fi done < <(echo "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null) while read-0 resource; do dest="$(charm.get_dir "$charm")/resources$resource" if ! [ -e "$dest" ]; then die "charm-resource: '$resource' does not exist (file: '$dest')." fi echo " - $dest:$resource:rw" done < <(echo "$metadata" | shyaml get-values-0 "charm-resources" 2>/dev/null) ) || return 1 if [ "$volumes" ]; then mixin=$(echo -en "volumes:\n$volumes") fi docker_compose=$(echo "$metadata" | shyaml get-value "docker-compose" 2>/dev/null) if [ "$docker_compose" ]; then mixin=$(merge_yaml_str "$mixin" "$docker_compose") fi if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" == "True" ]; then subordinate=true fi fi image=$(echo "$metadata" | shyaml get-value "docker-image" 2>/dev/null) image_or_build_statement= if [ "$image" ]; then if [ "$subordinate" ]; then err "Subordinate charm can not have a ${WHITE}docker-image${NORMAL} value." return 1 fi image_or_build_statement="image: $image" elif [ -d "$(charm.get_dir "$charm")/build" ]; then if [ "$subordinate" ]; then err "Subordinate charm can not have a 'build' sub directory." return 1 fi image_or_build_statement="build: $charm/build" fi if [ "$image_or_build_statement" ]; then mixin=$(merge_yaml_str "$mixin" "$image_or_build_statement") fi echo "$mixin" | tee "$cache_file" } export -f _get_docker_compose_mixin_from_metadata_cached get_docker_compose_mixin_from_metadata() { local charm="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" if [ -e "$cache_file" ]; then #debug "$FUNCNAME: SESSION cache hit ($*)" cat "$cache_file" return 0 fi metadata="$(charm.metadata "$charm")" || return 1 mixin=$(_get_docker_compose_mixin_from_metadata_cached "$charm" "$metadata") || return 1 echo "$mixin" | tee "$cache_file" } export -f get_docker_compose_mixin_from_metadata _save() { local name="$1" cat - | tee -a "$docker_compose_dir/.data/$name" } export -f _save get_default_project_name() { if [ "$DEFAULT_PROJECT_NAME" ]; then echo "$DEFAULT_PROJECT_NAME" return 0 fi compose_yml_location="$(get_compose_yml_location)" || return 1 if [ "$compose_yml_location" ]; then if normalized_path=$(readlink -e "$compose_yml_location"); then echo "$(basename "$(dirname "$normalized_path")")" return 0 fi fi echo "project" return 0 } export -f get_default_project_name launch_docker_compose() { local charm docker_compose_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX) #debug "Creating temporary docker-compose directory in '$docker_compose_tmpdir'." # trap_add EXIT "debug \"Removing temporary docker-compose directory in $docker_compose_tmpdir.\";\ # rm -rf \"$docker_compose_tmpdir\"" trap_add EXIT "rm -rf \"$docker_compose_tmpdir\"" project=$(get_default_project_name) mkdir -p "$docker_compose_tmpdir/$project" docker_compose_dir="$docker_compose_tmpdir/$project" if [ -z "$SERVICE_PACK" ]; then export SERVICE_PACK=$(get_default_target_services $SERVICE_PACK) fi get_docker_compose $SERVICE_PACK > "$docker_compose_dir/docker-compose.yml" || return 1 if [ -e "$state_tmpdir/to-merge-in-docker-compose.yml" ]; then # debug "Merging some config data in docker-compose.yml:" # debug "$(cat $state_tmpdir/to-merge-in-docker-compose.yml)" _config_merge "$docker_compose_dir/docker-compose.yml" "$state_tmpdir/to-merge-in-docker-compose.yml" || return 1 fi if [ -z "$(echo $(cat "$docker_compose_dir/docker-compose.yml"))" ]; then die "Generated 'docker-compose.yml' is unexpectedly empty." fi ## XXXvlab: could be more specific and only link the needed charms for charm in $(shyaml keys < "$docker_compose_dir/docker-compose.yml"); do if charm.exists "$charm"; then ln -sf "$(charm.get_dir "$charm")" "$docker_compose_dir/$charm" || exit 1 fi done mkdir "$docker_compose_dir/.data" { { cd "$docker_compose_dir" debug "${WHITE}docker-compose.yml$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:" debug "$(cat "$docker_compose_dir/docker-compose.yml" | prefix " $GRAY|$NORMAL ")" debug "${WHITE}Launching$NORMAL: docker-compose $@" docker-compose "$@" echo "$?" > "$docker_compose_dir/.data/errlvl" } | _save stdout } 3>&1 1>&2 2>&3 | _save stderr 3>&1 1>&2 2>&3 if tail -n 1 "$docker_compose_dir/.data/stderr" | egrep "Service .+ failed to build: Error getting container [0-9a-f]+ from driver devicemapper: (open|Error mounting) /dev/mapper/docker-.*: no such file or directory$" >/dev/null 2>&1; then err "Detected bug https://github.com/docker/docker/issues/4036 ... " err "Please re-launch your command, or switch from 'devicemapper' driver to 'overlayfs' or 'aufs'." fi docker_compose_errlvl="$(cat "$docker_compose_dir/.data/errlvl" 2>/dev/null)" if [ -z "$docker_compose_errlvl" ]; then err "Something went wrong before you could gather docker-compose errorlevel." return 1 fi return "$docker_compose_errlvl" } export -f launch_docker_compose get_compose_yml_location() { parent=$(while ! [ -e "./compose.yml" ]; do [ "$PWD" == "/" ] && exit 0 cd .. done; echo "$PWD" ) if [ "$parent" ]; then echo "$parent/compose.yml" return 0 fi if [ "$DEFAULT_COMPOSE_FILE" ]; then if ! [ -e "$DEFAULT_COMPOSE_FILE" ]; then err "No 'compose.yml' was found in current or parent dirs," \ "and \$DEFAULT_COMPOSE_FILE points to an unexistent file." die "Please provide a 'compose.yml' file." fi echo "$DEFAULT_COMPOSE_FILE" return 0 fi err "No 'compose.yml' was found in current or parent dirs, and no \$DEFAULT_COMPOSE_FILE was set." die "Please provide a 'compose.yml' file." return 1 } export -f get_compose_yml_location get_default_target_services() { local services=("$@") if [ -z "${services[*]}" ]; then if [ "$DEFAULT_SERVICES" ]; then debug "No service provided, using $WHITE\$DEFAULT_SERVICES$NORMAL variable." \ "Target services: $DARKYELLOW$DEFAULT_SERVICES$NORMAL" services="$DEFAULT_SERVICES" else err "No service provided." return 1 fi fi echo "${services[*]}" } export -f get_default_target_services get_master_services() { local loaded master_service declare -A loaded for service in "$@"; do master_service=$(get_top_master_charm_for_service "$service") || return 1 if [ "${loaded[$master_service]}" ]; then continue fi echo "$master_service" loaded["$master_service"]=1 done | xargs echo } export -f get_master_services _setup_state_dir() { export state_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX) #debug "Creating temporary state directory in '$state_tmpdir'." # trap_add EXIT "debug \"Removing temporary state directory in $state_tmpdir.\";\ # rm -rf \"$state_tmpdir\"" trap_add EXIT "rm -rf \"$state_tmpdir\"" } get_docker_compose_opts_list() { local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$action"; cat "$(which docker-compose)" | md5_compat)" if [ -e "$cache_file" ]; then cat "$cache_file" return 0 fi docker_compose_help_msg=$(docker-compose "$action" --help 2>/dev/null) || return 1 echo "$docker_compose_help_msg" | grep '^Options:' -A 20000 | tail -n +2 | egrep "^\s+-" | sed -r 's/\s+((((-[a-zA-Z]|--[a-zA-Z0-9-]+)( [A-Z=]+|=[^ ]+)?)(, )?)+)\s+.*$/\1/g' | tee "$cache_file" } _MULTIOPTION_REGEX='^((-[a-zA-Z]|--[a-zA-Z0-9-]+)(, )?)+' _MULTIOPTION_REGEX_LINE_FILTER=$_MULTIOPTION_REGEX'(\s|=)' get_docker_compose_multi_opts_list() { local action="$1" opts_list opts_list=$(get_docker_compose_opts_list "$action") || return 1 echo "$opts_list" | egrep "$_MULTIOPTION_REGEX_LINE_FILTER" | sed -r "s/^($_MULTIOPTION_REGEX)(\s|=).*$/\1/g" | tr ',' "\n" | xargs echo } get_docker_compose_single_opts_list() { local action="$1" opts_list opts_list=$(get_docker_compose_opts_list "$action") || return 1 echo "$opts_list" | egrep -v "$_MULTIOPTION_REGEX_LINE_FILTER" | tr ',' "\n" | xargs echo } _graph_service() { local service="$1" base="$1" charm=$(get_service_charm "$service") || return 1 metadata=$(charm.metadata "$charm") || return 1 subordinate=$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null) if [ "$subordinate" == "True" ]; then requires="$(echo "$metadata" | shyaml get-value "requires" 2>/dev/null)" master_charm= while read-0 relation_name relation; do [ "$(echo "$relation" | shyaml get-value "scope" 2>/dev/null)" == "container" ] || continue interface="$(echo "$relation" | shyaml get-value "interface" 2>/dev/null)" if [ -z "$interface" ]; then err "No ${WHITE}$interface${NORMAL} set for relation $relation_name." return 1 fi ## Action provided by relation ? target_service= while read-0 relation_name candidate_target_service _relation_config _tech_dep; do [ "$interface" == "$relation_name" ] && { target_service="$candidate_target_service" break } done < <(get_compose_relations "$service") if [ -z "$target_service" ]; then err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \ "${DARKYELLOW}$service$NORMAL compose definition." return 1 fi master_service="$target_service" master_charm=$(get_service_charm "$target_service") || return 1 break done < <(echo "$requires" | shyaml key-values-0 2>/dev/null) fi _graph_node_service "$service" "$base" "$charm" _graph_edge_service "$service" "$subordinate" "$master_service" } _graph_node_service() { local service="$1" base="$2" charm="$3" cat < ]; EOF } _graph_edge_service() { local service="$1" subordinate="$2" master_service="$3" while read-0 relation_name target_service relation_config tech_dep; do cat < "$(_graph_node_service_label ${target_service})" [ penwidth = $([ "$master_service" == "$target_service" ] && echo 3 || echo 2) fontsize = 16 fontcolor = "black" style = $([ "$master_service" == "$target_service" ] && echo dashed || echo "\"\"") weight = $([ "$master_service" == "$target_service" ] && echo 2.0 || echo 1.0) dir = $([ "$master_service" == "$target_service" ] && echo none || echo both) arrowtail = odot # arrowhead = dotlicurve taillabel = "$relation_name" ]; EOF done < <(get_compose_relations "$service") || return 1 } _graph_node_service_label() { local service="$1" echo "service_$service" } _graph_node_service_content() { local service="$1" charm=$(get_service_charm "$service") || return 1 cat < $service $(if [ "$charm" != "$service" ]; then cat < charm: $charm EOF2 fi) EOF } cla_contains () { local e for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done return 1 } graph() { local services=("$@") declare -A entries cat <&2 exit 1 fi fi ;; *) action_posargs+=("$1") stage="remainder" ;; esac ;; "remainder") remainder_args+=("$@") break 3;; esac shift done [ "${compose_opts[*]}" ] && debug "Main docker-compose opts: ${compose_opts[*]}" [ "${action_opts[*]}" ] && debug "Action $DARKCYAN$action$NORMAL with opts: ${action_opts[*]}" [ "${remainder_args[*]}" ] && debug "Remainder args: ${remainder_args[*]}" ## ## Actual code ## export DOCKER_DATASTORE=${DOCKER_DATASTORE:-/srv/docker-datastore} COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1 export COMPOSE_YML_FILE debug "Found $WHITE$exname$NORMAL YAML file in '$COMPOSE_YML_FILE'." output=$(shyaml get-value < "$COMPOSE_YML_FILE" 2>&1) if [ "$?" != 0 ]; then err "Invalid YAML in '$COMPOSE_YML_FILE':" while IFS='' read -r line1 && IFS='' read -r line2; do echo "$line1 $GRAY($line2)$NORMAL" done < <(echo "$output" | grep ^yaml.scanner -A 100 | sed -r 's/^ in "", //g' | sed -r 's/^yaml.scanner.[a-zA-Z]+: //g') | prefix " $GRAY|$NORMAL " exit 1 fi charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct." ## ## Get services in command line. ## is_service_action= case "$action" in up|build|start|stop|config|graph) services="$(get_default_target_services "${action_posargs[@]}")" || exit 1 orig_services="${action_posargs[@]:1}" ;; run) services="${action_posargs[0]}" ;; "") services= ;; *) if is_service_action=$(has_service_action "${action_posargs[0]}" "$action"); then { read-0 action_type case "$action_type" in "relation") read-0 _ target_service relation_name debug "Found action $DARKYELLOW${action_posargs[0]}$NORMAL/$DARKBLUE$relation_name$NORMAL/$DARKCYAN$action$NORMAL (in $DARKYELLOW$target_service$NORMAL)" ;; "direct") debug "Found action $DARKYELLOW${action_posargs[0]}$NORMAL.$DARKCYAN$action$NORMAL" ;; esac } < <(has_service_action "${action_posargs[0]}" "$action") services="${action_posargs[0]}" else services="$(get_default_target_services "${action_posargs[@]}")" fi ;; esac get_docker_compose $services >/dev/null || { ## precalculate variable \$_current_docker_compose err "Fails to compile base 'docker-compose.yml'" exit 1 } ## ## Pre-action ## full_init= case "$action" in up|run) full_init=true post_hook=true ;; "") full_init= ;; *) if [ "$is_service_action" ]; then full_init=true fi ;; esac if [ "$full_init" ]; then ## init in order if [ -z "$no_init" ]; then Section setup host resources setup_host_resources "$services" || exit 1 Section initialisation run_service_hook "$services" init || exit 1 fi ## Get relations if [ -z "$no_relations" ]; then run_service_relations "$services" || exit 1 fi fi export SERVICE_PACK="$services" ## ## Docker-compose ## case "$action" in up|start|stop|build) master_services=$(get_master_services $SERVICE_PACK) || exit 1 if [[ "$action" == "up" ]] && ! array_member action_opts -d; then ## force daemon mode for up action_opts=("-d" "${action_opts[@]}") fi launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" $master_services ;; run) master_service=$(get_master_services $SERVICE_PACK) || exit 1 launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "$master_service" "${remainder_args[@]}" ;; # enter) # master_service=$(get_master_services $SERVICE_PACK) || exit 1 # [ "${remainder_args[*]}" ] || remainder_args=("/bin/bash" "-c" "export TERM=xterm; exec bash") # docker exec -ti "${action_opts[@]}" "$master_service" "${remainder_args[@]}" # ;; graph) graph $SERVICE_PACK ;; config) ## removing the services launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" warn "Runtime configuration modification (from relations) are not included here." ;; *) if [ "$is_service_action" ]; then run_service_action "$SERVICE_PACK" "$action" "${remainder_args[@]}" else launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" fi ;; esac if [ "$post_hook" ]; then run_service_hook "$services" post_deploy || exit 1 fi