From 37d5c0a43db42b38cca645fa78aaa77d75980de0 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Mon, 26 Nov 2018 11:02:04 +0100 Subject: [PATCH] new: ``compose`` is now to be used in a docker. --- Dockerfile | 87 ++ bin/compose | 3157 ++-------------------------------------------- bin/compose-core | 3088 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 3314 insertions(+), 3018 deletions(-) create mode 100644 Dockerfile create mode 100755 bin/compose-core diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bb3b54d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,87 @@ + +FROM python:2-alpine3.7 as common + +CMD ["/bin/sh"] + +RUN apk --no-cache --update add git bash yaml sed lsof + +FROM common as builder + +RUN apk --update add curl + +## would love to use args... but I need it as an environment variable +ENV KAL_SHLIB_ARRAY_VERSION="0.1.0" \ + KAL_SHLIB_CACHE_VERSION="0.0.1" \ + KAL_SHLIB_CHARM_VERSION="0.3.3" \ + KAL_SHLIB_CMDLINE_VERSION="0.0.4" \ + KAL_SHLIB_COMMON_VERSION="0.4.9" \ + KAL_SHLIB_CONFIG_VERSION="0.0.2" \ + KAL_SHLIB_CORE_VERSION="0.7.0" \ + KAL_SHLIB_FIREWALL_VERSION="0.2.0" \ + KAL_SHLIB_OTHER_VERSION="0.2.2" \ + KAL_SHLIB_PRETTY_VERSION="0.4.2" + +ARG DOCKER_CLI_VERSION="17.06.2-ce" +ARG DOCKER_COMPOSE_VERSION="1.21.2" + +## install docker +ENV DOCKER_DOWNLOAD_URL="https://download.docker.com/linux/static/stable/x86_64/docker-$DOCKER_CLI_VERSION.tgz" +RUN mkdir -p /tmp/docker \ + && curl -L "$DOCKER_DOWNLOAD_URL" | tar -xz -C /tmp/docker \ + && mv /tmp/docker/docker/docker /usr/local/bin/ \ + && rm -rf /tmp/docker + + +## install docker-compose +# ENV DOCKER_COMPOSE_DOWNLOAD_URL="https://github.com/docker/compose/releases/download/$DOCKER_COMPOSE_VERSION/docker-compose-Linux-x86_64" +# RUN curl -L "$DOCKER_COMPOSE_DOWNLOAD_URL" > /usr/local/bin/docker-compose \ +# && chmod +x /usr/local/bin/docker-compose + + +## install kal-shlibs +RUN apk --update add binutils && \ + mkdir /tmp/kal-shlibs && cd /tmp/kal-shlibs && \ + export pkg && \ + for pkg in core common array cache charm cmdline config firewall other pretty; do \ + echo "Installing kal-shlib-$pkg" ; \ + bash -c -- 'eval curl -L http://deb.kalysto.org/pool/no-dist/kal-alpha/kal-shlib-${pkg}_\${KAL_SHLIB_${pkg^^}_VERSION}-1_all.deb' > pkg.deb || exit 1 ; \ + ar x pkg.deb || exit 1; \ + tar xf /tmp/kal-shlibs/data.tar.* -C / || exit 1; \ + rm /tmp/kal-shlibs/data.tar.* ; \ + done + + +## install shyaml +RUN apk add python-dev build-base +RUN apk add python-dev py-yaml && \ + pip install shyaml + +RUN pip install docker-compose==$DOCKER_COMPOSE_VERSION + +RUN curl -L https://git.0k.io/0k-charm.git/plain/bin/charm > /usr/local/bin/charm && \ + chmod +x /usr/local/bin/charm + +RUN curl -L https://git.0k.io/git-sub.git/plain/bin/git-sub > /usr/local/bin/git-sub && \ + chmod +x /usr/local/bin/git-sub + +RUN curl http://docker.0k.io/get/ca.0k.io.pem > /usr/local/share/ca-certificates/ca.0k.io.pem + + +FROM common + +COPY --from=builder /etc/shlib /etc/shlib +COPY --from=builder /usr/lib/shlib /usr/lib/shlib +COPY --from=builder /usr/local /usr/local + +RUN cp /usr/local/share/ca-certificates/ca.0k.io.pem /etc/ssl/ca.0k.io.pem && \ + apk add ca-certificates && update-ca-certificates && \ + mkdir -p /etc/docker/certs.d/docker.0k.io && \ + ln -sfn /etc/ssl/ca.0k.io.pem /etc/docker/certs.d/docker.0k.io/ca.crt + +## requiring ``stdbuf`` for actions +RUN apk add coreutils + +## install compose +COPY ./bin/ /usr/local/bin/ + +ENTRYPOINT ["/usr/local/bin/compose-core"] diff --git a/bin/compose b/bin/compose index 29c17be..2d09dea 100755 --- a/bin/compose +++ b/bin/compose @@ -1,3088 +1,209 @@ #!/bin/bash +## Bash wrap script to launch the ``compose`` docker with right options. ## -## TODO: -## - subordinate container should really be able to modify base image of their master -## - this could be done through docker-update -## - I'm not happy with the current build using 'build/' directory, this should be -## changed to: -## - always have a base image (specified in metadata), and always have hooks/install -## executed and merge in image (like docker-build-charm). -## - container base image is ALWAYS the image of the master container... this brings -## questions about a double way to express inheritage (through relations as it is -## implemented now, or through this base-image ?) -## - the name of the scripts for relation (aka relation_name-relation-joined) is bad as -## reading the name in a hooks/ dir, there are no way to know if we are the target or -## the base of the relation. -## - we could leverage a 'relations/' dir on the root of the charm, with both: -## 'relations/provide/relation_name' and 'relations/receive/relation_name' -## - a very bad point with the actual naming is that we can't have a providing AND -## receiving a relation with same name. -## - The cache system should keep md5 of docker-compose and other things between runs -## - The cache system should use underlying function that have only arguments inputs. -## This will allow to cache completely without issues function in time. -## - would probably need instrospection in charm custom action to know if these need -## init or relations to be set up. -## - Be clear about when the SERVICE name is used and the CHARM name is used. -## - in case of service contained in another container -## - in normal case -## - in docker-compose, can't use charm name: if we want 2 instances of the same charm -## we are stuck. What will be unique is the name of the service. -## - some relations are configured in compose.yml but should not trigger the loading -## of necessary component (for instance, apache --> log-rotate), if log-rotate is -## not there, this link should considered optional. -## - Could probably allow an unexistent charm to be populated with only "docker-image:" -## of the same name. Although this should trigger a visible warning. - - -#:- -[ -e /etc/shlib ] && . /etc/shlib || { - echo "Unsatisfied dependency. Please install 'kal-shlib-core'." - exit 1 - } -#:- - -include common -include pretty -include parse -include charm -include array -include cla - -depends shyaml docker - - -version=0.1 -usage="$exname [COMPOSE_OPTS] [ACTION [ACTION_OPTS]]" -help="\ - -$WHITE$exname$NORMAL jobs is to run various shell scripts to build -a running orchestrated and configured docker containers. These shell -scripts will have the opportunity to build a 'docker-compose.yml'. - -Once init script and relations scripts are executed, $WHITE$exname$NORMAL -delegate the launching to ${WHITE}docker-compose${NORMAL} by providing it -the final 'docker-compose.yml'. - -$WHITE$exname$NORMAL also leverage charms to offer some additional custom -actions per charm, which are simply other scripts that can be -run without launching ${WHITE}docker-compose${NORMAL}. - -In compose message, color coding is enforced as such: - - ${DARKCYAN}action$NORMAL, - - ${DARKBLUE}relation$NORMAL, - - ${DARKPINK}charm${NORMAL}, - - ${DARKYELLOW}service${NORMAL}, - - ${WHITE}option-name${NORMAL}/${WHITE}command-name${NORMAL}/${WHITE}Section-Title${NORMAL} - -$WHITE$exname$NORMAL reads '/etc/compose.conf' for global variables, and -'/etc/compose.local.conf' for local host adjustements. - -" - - -## XXXvlab: this doesn't seem to work when 'compose' is called in -## a hook of a charm. -#[[ "${BASH_SOURCE[0]}" == "" ]] && SOURCED=true -$(return >/dev/null 2>&1) && SOURCED=true - - -if [ "$UID" == 0 ]; then - CACHEDIR=${CACHEDIR:-/var/cache/compose} - VARDIR=${VARDIR:-/var/lib/compose} -else - [ "$XDG_CONFIG_HOME" ] && CACHEDIR=${CACHEDIR:-$XDG_CONFIG_HOME/compose} - [ "$XDG_DATA_HOME" ] && VARDIR=${VARDIR:-$XDG_DATA_HOME/compose} - CACHEDIR=${CACHEDIR:-$HOME/.cache/compose} - VARDIR=${VARDIR:-$HOME/.local/share/compose} -fi -export VARDIR CACHEDIR - - -md5_compat() { md5sum | cut -c -32; } -quick_cat_file() { quick_cat_stdin < "$1"; } -quick_cat_stdin() { local IFS=''; while read -r line; do echo "$line"; done ; } -export -f quick_cat_file quick_cat_stdin md5_compat - - -clean_cache() { - local i=0 - for f in $(ls -t "$CACHEDIR/"*.cache.* 2>/dev/null | tail -n +500); do - ((i++)) - rm -f "$f" - done - if (( i > 0 )); then - debug "${WHITE}Cleaned cache:${NORMAL} Removed $((i)) elements (current cache size is $(du -sh "$CACHEDIR" | cut -f 1))" - fi -} - - -usage="$exname SERVICE"' - -Deploy and manage a swarm of containers to provide services based on -a ``compose.yml`` definition and charms from a ``charm-store``. -' - -export DEFAULT_COMPOSE_FILE - -## -## Merge YAML files ## -export _merge_yaml_common_code=" - -import sys -import yaml - - -try: - # included in standard lib from Python 2.7 - from collections import OrderedDict -except ImportError: - # try importing the backported drop-in replacement - # it's available on PyPI - from ordereddict import OrderedDict - - -## Ensure that there are no collision with legacy OrderedDict -## that could be used for omap for instance. -class MyOrderedDict(OrderedDict): - pass - -yaml.add_representer( - MyOrderedDict, - lambda cls, data: cls.represent_dict(data.items())) - -yaml.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - lambda cls, node: MyOrderedDict(cls.construct_pairs(node))) - - -def fc(filename): - with open(filename) as f: - return f.read() - -def merge(*args): - # sys.stderr.write('%r\n' % (args, )) - args = [arg for arg in args if arg is not None] - if len(args) == 0: - return None - if len(args) == 1: - return args[0] - if all(isinstance(arg, (int, basestring, bool)) for arg in args): - return args[-1] - elif all(isinstance(arg, list) for arg in args): - res = [] - for arg in args: - for elt in arg: - if elt in res: - res.remove(elt) - res.append(elt) - return res - elif all(isinstance(arg, dict) for arg in args): - keys = set() - for arg in args: - keys |= set(arg.keys()) - dct = {} - for key in keys: - sub_args = [] - for arg in args: - if key in arg: - sub_args.append(arg) - try: - dct[key] = merge(*(a[key] for a in sub_args)) - except NotImplementedError as e: - raise NotImplementedError( - e.args[0], - '%s.%s' % (key, e.args[1]) if e.args[1] else key, - e.args[2]) - if dct[key] is None: - del dct[key] - return dct - else: - raise NotImplementedError( - 'Unsupported types: %s' - % (', '.join(list(set(arg.__class__.__name__ for arg in args)))), '', args) - return None - -def merge_cli(*args): - try: - c = merge(*args) - except NotImplementedError as e: - sys.stderr.write('Merging Failed: %s.\n%s\n' - ' Values are:\n %s\n' - % (e.args[0], - ' Conflicting key is %r.' % e.args[1] if e.args[1] else - ' Conflict at base of structure.', - '\\n '.join('v%d: %r' % (i, a) - for i, a in enumerate(e.args[2])))) - exit(1) - if c is not None: - print '%s' % yaml.dump(c, default_flow_style=False) - -" - - -merge_yaml() { - - if ! [ -r "$state_tmpdir/merge_yaml.py" ]; then - cat < "$state_tmpdir/merge_yaml.py" - -$_merge_yaml_common_code - -merge_cli(*(yaml.load(fc(f)) for f in sys.argv[1:])) -EOF - fi - - python "$state_tmpdir/merge_yaml.py" "$@" -} -export -f merge_yaml - - -merge_yaml_str() { - local entries="$@" - - if ! [ -r "$state_tmpdir/merge_yaml_str.py" ]; then - cat < "$state_tmpdir/merge_yaml_str.py" || return 1 - -$_merge_yaml_common_code - -merge_cli(*(yaml.load(f) for f in sys.argv[1:])) -EOF - fi - - python "$state_tmpdir/merge_yaml_str.py" "$@" -} -export -f merge_yaml_str - - -yaml_key_val_str() { - local entries="$@" - - if ! [ -r "$state_tmpdir/yaml_key_val_str.py" ]; then - cat < "$state_tmpdir/yaml_key_val_str.py" - -$_merge_yaml_common_code - -print '%s' % yaml.dump({ - yaml.load(sys.argv[1]): - yaml.load(sys.argv[2])}, default_flow_style=False) - -EOF - fi - - python "$state_tmpdir/yaml_key_val_str.py" "$@" -} -export -f yaml_key_val_str +## Launcher +## - should need minimum requirement to run +## - no shell libs +## -## -## Docker -## -docker_has_image() { - local image="$1" - images=$(docker images -q "$image" 2>/dev/null) || { - err "docker images call has failed unexpectedly." - return 1 - } - [ "$images" ] +read-0() { + local eof= IFS='' + while [ "$1" ]; do + read -r -d '' -- "$1" || eof=1 + shift + done + [ -z "$eof" ] } -export -f docker_has_image -cmd_on_base_image() { - local service="$1" base_image - shift - base_image=$(service_base_docker_image "$service") || return 1 - docker run -i --entrypoint /bin/bash "$base_image" -c "$*" +get_running_compose_containers() { + ## XXXvlab: docker bug: there will be a final newline anyway + docker ps --filter label="compose.service" --format='{{.ID}}' } -export -f cmd_on_base_image - -cached_cmd_on_base_image() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" - shift - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - quick_cat_stdin < "$cache_file" - return 0 - fi - result=$(cmd_on_base_image "$service" "$@") || return 1 - echo "$result" | tee "$cache_file" -} -export -f cached_cmd_on_base_image +get_volumes_for_container() { + local container="$1" -image_exposed_ports_0() { - local image="$1" - docker inspect --format='{{range $p, $conf := .Config.ExposedPorts}}{{$p}}{{"\x00"}}{{end}}' "$image" + docker inspect \ + --format '{{range $mount := .Mounts}}{{$mount.Source}}{{"\x00"}}{{$mount.Destination}}{{"\x00"}}{{end}}' \ + "$container" } -export -f image_exposed_ports_0 - -## -## Generic -## - -fn.exists() { - declare -F "$1" >/dev/null -} +is_volume_used() { + local volume="$1" -str_pattern_matches() { - local str="$1" - shift - for pattern in "$@"; do - eval "[[ \"$str\" == $pattern ]]" && return 0 - done + while read container_id; do + while read-0 src dst; do + [ "$src" == "$volume" ] && return 0 + done < <(get_volumes_for_container "$container_id") + done < <(get_running_compose_containers) return 1 } -str_matches() { - local str="$1" - shift - for pattern in "$@"; do - [[ "$str" == "$pattern" ]] && return 0 - done - return 1 -} - -gen_password() { - local l=( {a..z} {A..Z} {0..9} ) nl="${#l[@]}" size=${1:-16} - while ((size--)); do - echo -n "${l[$((RANDOM * nl / 32768))]}" +clean_unused_sessions() { + for f in /var/lib/compose/sessions/*; do + [ -e "$f" ] || continue + is_volume_used "$f" && continue + rm -f "$f" done - echo -} -export -f gen_password - - -file_put() { - local TARGET="$1" - mkdir -p "$(dirname "$TARGET")" && - cat - > "$TARGET" -} -export -f file_put - - -file_put_0() { - local TARGET="$1" - mkdir -p "$(dirname "$TARGET")" && - cat > "$TARGET" -} -export -f file_put_0 - - -fetch_file() { - local src="$1" - - case "$src" in - *"://"*) - err "Unsupported target scheme." - return 1 - ;; - *) - ## Try direct - if ! [ -r "$src" ]; then - err "File '$src' not found/readable." - return 1 - fi - cat "$src" || return 1 - ;; - esac -} -export -f fetch_file - - -## receives stdin content to decompress on stdout -## stdout content should be tar format. -uncompress_file() { - local filename="$1" - - ## Warning, the content of the file is already as stdin, the filename - ## is there to hint for correct decompression. - case "$filename" in - *".gz") - gunzip - ;; - *".bz2") - bunzip2 - ;; - *) - cat - ;; - esac - -} -export -f uncompress_file - - -get_file() { - local src="$1" - - fetch_file "$src" | uncompress_file "$src" -} -export -f get_file - - -## -## Common database lib -## - -_clean_docker() { - local _DB_NAME="$1" container_id="$2" - ( - set +e - debug "Removing container $_DB_NAME" - docker stop "$container_id" - docker rm "$_DB_NAME" - docker network rm "${_DB_NAME}" - rm -vf "/tmp/${_DB_NAME}.state" - ) -} -export -f _clean_docker - - -get_service_base_image_dir_uid_gid() { - local service="$1" dir="$2" uid_gid - uid_gid=$(cached_cmd_on_base_image "$service" "stat -c '%u %g' '$dir'") || { - debug "Failed to query '$dir' uid in ${DARKYELLOW}$service${NORMAL} base image." - return 1 - } - info "uid and gid from ${DARKYELLOW}$service${NORMAL}:$dir is '$uid_gid'" - echo "$uid_gid" -} -export -f get_service_base_image_dir_uid_gid - - -get_service_type() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" - if [ -z "$service" ]; then - print_syntax_error "$FUNCNAME: Please specify a service as first argument." - return 1 - fi - - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" - return 0 - fi - - charm=$(get_service_charm "$service") || return 1 - metadata=$(charm.metadata "$charm") || return 1 - printf "%s" "$metadata" | shyaml get-value type service 2>/dev/null | - tee "$cache_file" -} - - -are_files_locked_in_dir() { - local dir="$1" device hdev ldev - device=$(stat -c %d "$dir") || { - err "Can't stat %d." - return 1 - } - device=$(printf "%04x" $device) - hdev=${device:0:2} - ldev=${device:2:2} - inodes=$(find "$dir" -printf ':%i:\n') - - found= - while read -r inode; do - debug "try inode:$inode" - if [[ "$inodes" == *":$inode:"* ]]; then - found=1 - break - fi - done < <(cat /proc/locks | grep " $hdev:$ldev:" | sed -r "s/^.*$hdev:$ldev:([0-9]+).*$/\1/g") - - [ "$found" ] } -export -f are_files_locked_in_dir -export _PID="$$" -ensure_db_docker_running () { - local _STATE_FILE - - _DB_NAME="db_${DB_NAME}_${_PID}" - _STATE_FILE=/tmp/${_DB_NAME}.state - if [ -e "$_STATE_FILE" ]; then - IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$(cat "$_STATE_FILE")" - - debug "Re-using previous docker/connection '$DOCKER_IP'." - _set_db_params "$DOCKER_IP" "$DOCKER_NETWORK" - - return 0 - fi - - if [ -e "/tmp/${_DB_NAME}.working" ]; then - ## avoid recursive calls. - if [ -z "$DOCKER_IP" ]; then - err "Currently figuring up DOCKER_IP, please set it yourself before this call if needed." - return 1 - else - debug "ignoring recursive call of 'ensure_db_docker_running'." - fi - return 0 - fi - - touch "/tmp/${_DB_NAME}.working" - - docker rm "$_DB_NAME" 2>/dev/null || true - - host_db_working_dir="$DATASTORE/${SERVICE_NAME}$DB_DATADIR" - - if is_db_locked; then - info "Some process is using '$host_db_working_dir'. Trying to find a docker that would do this..." - found= - for docker_id in $(docker ps -q); do - has_volume_mounted=$( - docker inspect \ - --format "{{range .Mounts}}{{if eq .Destination \"$DB_DATADIR\"}}{{.Source}}{{end}}{{end}}" \ - "$docker_id") - if [ "$has_volume_mounted" == "$host_db_working_dir" ]; then - found="$docker_id" - break - fi - done - if [ -z "$found" ]; then - err "Please shutdown any other docker using this directory." - return 1 - fi - export container_id="$found" - info "Found docker $docker_id is already running." - else - verb "Database is not locked." - if ! docker_has_image "$DOCKER_BASE_IMAGE"; then - docker pull "$DOCKER_BASE_IMAGE" - fi - - docker_opts= - if [ -f "$DB_PASSFILE" ]; then - verb "Found and using '$DB_PASSFILE'." - docker_opts="$db_docker_opts -v $SERVER_ROOT_PREFIX$DB_PASSFILE:$DB_PASSFILE" - fi - debug docker network create "$_DB_NAME" - if ! network_id=$(docker network create "$_DB_NAME"); then - err "'docker network create' failed !" - _clean_docker "$_DB_NAME" "$container_id" - rm "/tmp/${_DB_NAME}.working" - return 1 - fi - debug docker run -d \ - --name "$_DB_NAME" \ - $docker_opts \ - --network "$_DB_NAME" \ - -v "$host_db_working_dir:$DB_DATADIR" \ - "$DOCKER_BASE_IMAGE" - if ! container_id=$( - docker run -d \ - --name "$_DB_NAME" \ - $docker_opts \ - --network "$_DB_NAME" \ - -v "$host_db_working_dir:$DB_DATADIR" \ - "$DOCKER_BASE_IMAGE" - ); then - err "'docker run' failed !" - _clean_docker "$_DB_NAME" "$container_id" - rm "/tmp/${_DB_NAME}.working" - return 1 - fi - trap_add EXIT,ERR "_clean_docker \"$_DB_NAME\" \"$container_id\"" - fi - - if docker_ip=$(wait_for_docker_ip "$container_id"); then - IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip" - echo "$docker_ip" > "$_STATE_FILE" - debug "written '$_STATE_FILE'" - rm "/tmp/${_DB_NAME}.working" - _set_db_params "$DOCKER_IP" "$DOCKER_NETWORK" - return 0 - else - errlvl="$?" - err "Db not found (errlvl: $errlvl). Tail of docker logs follows:" - docker logs --tail=5 "$container_id" 2>&1 | prefix " | " >&2 - rm "/tmp/${_DB_NAME}.working" - return "$errlvl" - fi -} -export -f ensure_db_docker_running - - -## Require to set $db_docker_opts if needed, and $DB_PASSFILE -## -_dcmd() { - local docker_opts command="$1" - shift - - debug "Db> $command $@" - - if [ -f "$DB_PASSFILE" ]; then - verb "Found and using '$DB_PASSFILE'." - db_docker_opts="$db_docker_opts -v $SERVER_ROOT_PREFIX$DB_PASSFILE:$DB_PASSFILE" - fi - - ## XXXX was here: actualy, we need only connection between this version and the client version - debug docker run -i --rm \ - $db_docker_opts \ - --entrypoint "$command" "$DOCKER_BASE_IMAGE" $db_cmd_opts "$@" - docker run -i --rm \ - $db_docker_opts \ - --entrypoint "$command" "$DOCKER_BASE_IMAGE" $db_cmd_opts "$@" -} -export -f _dcmd - - -## Executes code through db -dcmd() { - local fun - [ "$DB_NAME" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_NAME." - [ "$DB_DATADIR" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_DATADIR." - # [ "$DB_PASSFILE" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_PASSFILE." - [ "$_PID" ] || print_syntax_error "$FUNCNAME: You must provide \$_PID." - for fun in is_db_locked _set_db_params ddb; do - [ "$(type -t "$fun")" == "function" ] || - print_syntax_error "$FUNCNAME: You must provide function '$fun'." +relink_subdirs() { + local dir + for dir in "$@"; do + [ -L "$dir" ] || continue + target=$(realpath "$dir") + [ -d "$target" ] || continue + docker_run_opts+=("-v" "$target:$dir") + [ -e "$dir/metadata.yml" ] && continue + relink_subdirs "$dir"/* done - ensure_db_docker_running /dev/null 2>&1; then - echo "default:$(docker inspect --format='{{ .NetworkSettings.IPAdress }}' "$name" 2>/dev/null)" - else - format='{{range $name, $conf := .NetworkSettings.Networks}}{{$name}}{{"\x00"}}{{$conf.IPAddress}}{{"\x00"}}{{end}}' - while read-0 network_id ip; do - printf "%s:%s\n" "$network_id" "$ip" - done < <(docker inspect --format="$format" "$name") - fi } -export -f get_docker_ips - -get_docker_ip() { - local name="$1" - get_docker_ips "$name" -} -export -f get_docker_ip +mk_docker_run_options() { -wait_docker_ip() { - local name="$1" timeout="${2:-15}" timeout_count=0 docker_ip= - start=$SECONDS - while [ -z "$docker_ip" ]; do - sleep 0.5 - docker_ip=$(get_docker_ip "$name") && break - elapsed=$((SECONDS - start)) - if ((elapsed > timeout)); then - err "${RED}timeout error${NORMAL}(${timeout}s):" \ - "Could not find '$name' docker container's IP." - return 1 - fi - [ "$elapsed" == "$old_elapsed" ] || - verb "Waiting for docker $name... ($elapsed/$timeout)" - old_elapsed="$elapsed" - done - verb "Found docker $name network and IP: $docker_ip" - echo "$docker_ip" -} -export -f wait_docker_ip + docker_run_opts=("-v" "/var/run/docker.sock:/var/run/docker.sock") -DEFAULT_BASH_IMAGE=${DEFAULT_BASH_IMAGE:-docker.0k.io/bash} -wait_for_tcp_port() { - local network=$1 host_port=$2 timeout=20 - verb "Trying to connect to $host_port" - docker run --rm -i --network "$network" "$DEFAULT_BASH_IMAGE" </dev/null 2>&1 && break - sleep 0.2 - if [ "\$((SECONDS - start))" -gt "$timeout" ]; then - exit 1 - fi - done - exit 0 -EOF - if [ "$?" == 1 ]; then - err "${RED}timeout error${NORMAL}(${timeout}s):"\ - "Could not connect to $host_port." - return 1 - fi - return 0 -} -export -f wait_for_tcp_port + ## CACHE/DATA DIRS + docker_run_opts+=("-v" "/var/lib/compose:/var/lib/compose") + docker_run_opts+=("-v" "/var/cache/compose:/var/cache/compose") + docker_run_opts+=("-v" "/etc/timezone:/etc/timezone:ro") -## Warning: requires a ``ddb`` matching current database to be checked -wait_for_docker_ip() { - local name=$1 DOCKER_IP= DOCKER_NETWORK= docker_ips= docker_ip= - docker_ip=$(wait_docker_ip "$name" 5) || return 1 - IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip" - if ! str_is_ipv4 "$DOCKER_IP"; then - err "internal 'wait_docker_ip' did not return a valid IP. Returned IP is '$DOCKER_IP'." - return 1 - fi - _set_db_params "$DOCKER_IP" "$DOCKER_NETWORK" - while read-0 port; do - IFS="/" read port type <<<"$port" - [ "$type" == "tcp" ] || continue - wait_for_tcp_port "$DOCKER_NETWORK" "$DOCKER_IP:${port}" || return 17 - verb "Port $DOCKER_IP:${port} checked open." - done < <(image_exposed_ports_0 "$container_id") - ## Checking direct connection - if ! echo "SELECT 1;" | ddb >/dev/null; then - err "${RED}db connection error${NORMAL}:"\ - "Could not connect to db on $DOCKER_IP" \ - "container's IP. (IP up, TCP ports is(are) open)" - return 18 + ## current dir + if parent=$(while true; do + [ -e "./compose.yml" ] && { + echo "$PWD" + exit 0 + } + [ "$PWD" == "/" ] && exit 1 + cd .. + done + ); then + docker_path=/var/lib/compose/root/$(basename "$parent") + docker_run_opts+=("-v" "$parent:$docker_path:ro" \ + "-w" "$docker_path") fi - echo "${DOCKER_NETWORK}:${DOCKER_IP}" - return 0 -} -export -f wait_for_docker_ip - - -docker_add_host_declaration() { - local src_docker=$1 domain=$2 dst_docker=$3 dst_docker_ip= dst_docker_network - dst_docker_ip=$(wait_docker_ip "$dst_docker") || exit 1 - - IFS=: read dst_docker_ip dst_docker_network <<<"$dst_docker_ip" - - docker exec -i "$src_docker" bash < /dev/null 2>&1; then - sed -ri "s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+$domain\$/$dst_docker_ip $domain/g" /etc/hosts - else - echo "$dst_docker_ip $domain" >> /etc/hosts - fi -EOF -} -export -f docker_add_host_declaration - - -get_running_containers_for_service() { - local service="$1" - - docker ps --filter label="compose.service=$service" --format="{{.ID}}" -} -export -f get_running_containers_for_service - - -## -## Internal Process -## - -get_docker_compose_links() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ - deps master_service master_target_service _relation_name \ - target_service _relation_config tech_dep - if [ -z "$service" ]; then - print_syntax_error "$FUNCNAME: Please specify a service as first argument." - return 1 - fi - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" - return 0 - fi + ## + ## Load config files + ## - master_service=$(get_top_master_service_for_service "$service") || return 1 - ## XXXvlab: yuck, this make the assumption that next function is cached, - ## and leverage the fact that the result is stored in a file. All this only - ## to catch a failure of ``get_compose_relations``, that would go out silently. - get_compose_relations "$service" >/dev/null || return 1 ## fetch cache and fail if necessary - deps=() - while read-0 _relation_name target_service _relation_config tech_dep; do - master_target_service="$(get_top_master_service_for_service "$target_service")" || return 1 - [ "$master_service" == "$master_target_service" ] && continue - type="$(get_service_type "$target_service")" || return 1 - [ "$type" == "run-once" ] && continue - if [ "$tech_dep" == "reversed" ]; then - deps+=("$(echo -en "$master_target_service:\n links:\n - $master_service")") - elif [ "$tech_dep" == "True" ]; then - deps+=("$(echo -en "$master_service:\n links:\n - $master_target_service")") + if [ -z "$DISABLE_SYSTEM_CONFIG_FILE" ]; then + if [ -r /etc/default/charm ]; then + docker_run_opts+=("-v" "/etc/default/charm:/etc/default/charm:ro") + . /etc/default/charm fi - ## XXXvlab: an attempt to add depends_on, but this doesn't work well actually - ## as there's a circular dependency issue. We don't really want the full feature - ## of depends_on, but just to add it as targets when doing an 'up' - # deps+=("$(echo -en "$master_service:\n depends_on:\n - $master_target_service")") - done < <(get_compose_relations "$service") - merge_yaml_str "${deps[@]}" | tee "$cache_file" || return 1 - if [ "${PIPESTATUS[0]}" != 0 ]; then - rm "$cache_file" - err "Failed to merge YAML from all ${WHITE}links${NORMAL} dependencies." - return 1 - fi -} - - -_get_docker_compose_opts() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ - compose_def master_service docker_compose_opts - if [ -z "$service" ]; then - print_syntax_error "$FUNCNAME: Please specify a service as first argument." - return 1 - fi - - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" - return 0 - fi - - compose_def="$(get_compose_service_def "$service")" || return 1 - master_service="$(get_top_master_service_for_service "$service")" - docker_compose_opts=$(echo "$compose_def" | shyaml get-value "docker-compose" 2>/dev/null) - - if [ "$docker_compose_opts" ]; then - yaml_key_val_str "$master_service" "$docker_compose_opts" - fi | tee "$cache_file" - if [ "${PIPESTATUS[0]}" != 0 ]; then - rm "$cache_file" - return 1 - fi -} - - -## -## By Reading the metadata.yml, we create a docker-compose.yml mixin. -## Some metadata.yml (of subordinates) will indeed modify other -## services than themselves. -_get_docker_compose_service_mixin() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ - links_yaml base_mixin links_yaml docker_compose_options \ - charm charm_part - if [ -z "$service" ]; then - print_syntax_error "$FUNCNAME: Please specify a service as first argument." - return 1 - fi - - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" - return 0 - fi - master_service=$(get_top_master_service_for_service "$service") || { - err "Failed to get top master service for service $DARKYELLOW$service$NORMAL" - return 1 - } - - ## The compose part - - base_mixin="$master_service: - labels: - - compose.service=$service - - compose.master-service=${master_service} - - compose.project=$(get_default_project_name)" - links_yaml=$(get_docker_compose_links "$service") || return 1 - docker_compose_options=$(_get_docker_compose_opts "$service") || return 1 - - ## the charm part - - charm_part=$(get_docker_compose_mixin_from_metadata "$service") || return 1 - ## Merge results - if [ "$charm_part" ]; then - charm_yaml="$(yaml_key_val_str "$master_service" "$charm_part")" || return 1 - merge_yaml_str "$base_mixin" "$links_yaml" "$charm_yaml" "$docker_compose_options" || return 1 + ## XXXvlab: should provide YML config opportunities in possible parent dirs ? + ## userdir ? and global /etc/compose.yml ? + for cfgfile in /etc/compose.conf /etc/compose.local.conf \ + /etc/default/compose /etc/compose/local.conf; do + [ -e "$cfgfile" ] || continue + docker_run_opts+=("-v" "$cfgfile:$cfgfile:ro") + . "$cfgfile" + done + for cfgfile in /etc/default/datastore; do + [ -e "$cfgfile" ] || continue + docker_run_opts+=("-v" "$cfgfile:$cfgfile:ro") + done else - merge_yaml_str "$base_mixin" "$links_yaml" "$docker_compose_options" || return 1 - fi | tee "$cache_file" - if [ "${PIPESTATUS[0]}" != 0 ]; then - err "Failed to constitute the base YAML for service '${DARKYELLOW}$service${NORMAL}'" - rm "$cache_file" - return 1 + docker_run_opts+=("-e" "DISABLE_SYSTEM_CONFIG_FILE=$DISABLE_SYSTEM_CONFIG_FILE") fi -} -export -f _get_docker_compose_service_mixin - - -## -## Get full `docker-compose.yml` format for all listed services (and -## their deps) -## - -## @export -## @cache: !system !nofail +stdout -get_docker_compose () { - local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ - entries services service start docker_compose_services - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" - return 0 - fi ## - ## Adding sub services configurations + ## Checking vars ## - declare -A entries - start_compilation=$SECONDS - debug "Compiling 'docker-compose.yml' base for ${DARKYELLOW}$*$NORMAL..." - for target_service in "$@"; do - start=$SECONDS - services=$(get_ordered_service_dependencies "$target_service") || { - err "Failed to get dependencies for $DARKYELLOW$target_service$NORMAL" - return 1 - } - debug " $DARKYELLOW$target_service$NORMAL deps:$DARKYELLOW" $services "$NORMAL$GRAY(in $((SECONDS - start))s)$NORMAL" - for service in $services; do - - if [ "${entries[$service]}" ]; then - ## Prevent double inclusion of same service if this - ## service is deps of two or more of your - ## requirements. - continue - fi - - ## mark the service as "loaded" as well as it's containers - ## if this is a subordinate service - start_service=$SECONDS - entries[$service]=$(_get_docker_compose_service_mixin "$service") || { - err "Failed to get service mixin for $DARKYELLOW$service$NORMAL" - return 1 - } - debug " Applied $DARKYELLOW$service$NORMAL charm metadata mixins $GRAY(in $((SECONDS - start_service))s)$NORMAL" - done - debug " ..finished all mixins for $DARKYELLOW$target_service$NORMAL $GRAY(in $((SECONDS - start))s)$NORMAL" - done - - docker_compose_services=$(merge_yaml_str "${entries[@]}") || { - err "Failed to merge YAML services entries together." - return 1 + ## CHARM_STORE + CHARM_STORE=${CHARM_STORE:-/srv/charm-store} + [ -L "$CHARM_STORE" ] && { + CHARM_STORE=$(readlink "$CHARM_STORE") || exit 1 } + docker_run_opts+=("-v" "$CHARM_STORE:/srv/charm-store:ro") + relink_subdirs /srv/charm-store/* - base_v2="version: '2.0'" - merge_yaml_str "$(yaml_key_val_str "services" "$docker_compose_services")" \ - "$base_v2" > "$cache_file" || return 1 - - export _current_docker_compose="$(cat "$cache_file")" - echo "$_current_docker_compose" - debug " ..compilation of base 'docker-compose.yml' done $GRAY(in $((SECONDS - start_compilation))s)$NORMAL" || true - # debug " ** ${WHITE}docker-compose.yml${NORMAL}:" - # debug "$_current_docker_compose" -} -export -f get_docker_compose - - -_get_compose_service_def_cached () { - local service="$1" docker_compose="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" - if [ -e "$cache_file" ]; then - #debug "$FUNCNAME: STATIC cache hit" - cat "$cache_file" && - touch "$cache_file" || return 1 - return 0 + ## DEFAULT_COMPOSE_FILE + if [ "${DEFAULT_COMPOSE_FILE+x}" ]; then + DEFAULT_COMPOSE_FILE=$(realpath "$DEFAULT_COMPOSE_FILE") + dirname=$(dirname "$DEFAULT_COMPOSE_FILE")/ + if [ -e "${DEFAULT_COMPOSE_FILE}" ]; then + docker_run_opts+=("-v" "$dirname:$dirname:ro") + fi fi - value=$(echo "$docker_compose" | shyaml get-value "$service" 2>/dev/null) - [ "$value" == None ] && value="" - if ! echo "$value" | shyaml get-value "charm" >/dev/null 2>&1; then - if charm.exists "$service"; then - value=$(merge_yaml <(echo "charm: $service") <(echo "$value")) || { - err "Can't merge YAML infered 'charm: $service' with base ${DARKYELLOW}$service${NORMAL} YAML definition." - return 1 - } - else - err "No ${WHITE}charm${NORMAL} value for service $DARKYELLOW$service$NORMAL" \ - "in compose, nor same name charm found." - return 1 + + ## COMPOSE_YML_FILE + if [ "${COMPOSE_YML_FILE+x}" ]; then + if [ -e "${COMPOSE_YML_FILE}" ]; then + docker_run_opts+=("-v" "$COMPOSE_YML_FILE:/tmp/compose.yml:ro") + docker_run_opts+=("-e" "COMPOSE_YML_FILE=/tmp/compose.yml") fi fi - echo "$value" | tee "$cache_file" || return 1 - # if [ "${PIPESTATUS[0]}" != 0 ]; then - # rm "$cache_file" - # return 1 - # fi - return 0 - # if [ "${PIPESTATUS[0]}" != 0 -o \! -s "$cache_file" ]; then - # rm "$cache_file" - # err "PAS OK $service: $value" - # return 1 - # fi -} -export -f _get_compose_service_def_cached + ## DATASTORE + if [ "${DATASTORE+x}" ]; then + docker_run_opts+=("-v" "$DATASTORE:/srv/datastore/data:rw") + docker_run_opts+=("-e" "DATASTORE=/srv/datastore/data") + fi -## XXXvlab: a lot to be done to cache the results -get_compose_service_def () { - local service="$1" docker_compose cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ - result - if [ -e "$cache_file" ]; then - #debug "$FUNCNAME: SESSION cache hit" - cat "$cache_file" || return 1 - return 0 + ## CONFIGSTORE + if [ "${CONFIGSTORE+x}" ]; then + docker_run_opts+=("-v" "$CONFIGSTORE:/srv/datastore/config:rw") + docker_run_opts+=("-e" "CONFIGSTORE=/srv/datastore/config") fi - [ -z "$service" ] && print_syntax_error "Missing service as first argument." - docker_compose=$(get_compose_yml_content) || return 1 - result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1 - echo "$result" | tee "$cache_file" || return 1 -} -export -f get_compose_service_def + docker_run_opts+=("-v" "$HOME/.docker:/root/.docker") + COMPOSE_LAUNCHER_BIN=$(readlink -f "${BASH_SOURCE[0]}") -_get_service_charm_cached () { - local service="$1" service_def="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit $1" - cat "$cache_file" && - touch "$cache_file" || return 1 - return 0 - fi - charm=$(echo "$service_def" | shyaml get-value charm 2>/dev/null) - if [ -z "$charm" ]; then - err "Missing ${WHITE}charm${NORMAL} value in service $DARKYELLOW$service$NORMAL definition." - return 1 - fi - echo "$charm" | tee "$cache_file" || return 1 + filename=$(mktemp -p /tmp/ -t launch_opts-XXXXXXXXXXXXXXXX) + { + printf "%s\0" "${docker_run_opts[@]}" + } > "$filename" + sha=$(sha256sum "$filename") + sha=${sha:0:64} + dest="/var/lib/compose/sessions/$sha" + { + printf "%s\0" "-v" "$dest:$dest" + printf "%s\0" "-e" "COMPOSE_LAUNCHER_OPTS=$dest" + printf "%s\0" "-e" "COMPOSE_LAUNCHER_BIN=$COMPOSE_LAUNCHER_BIN" + } >> "$filename" + mkdir -p /var/lib/compose/sessions + mv "$filename" "$dest" + echo "$dest" } -export -f _get_service_charm_cached -get_service_charm () { - local service="$1" - if [ -z "$service" ]; then - echo ${FUNCNAME[@]} >&2 - print_syntax_error "$FUNCNAME: Please specify a service as first argument." - return 1 +run() { + docker_run_opts=() + if [ -z "$COMPOSE_LAUNCHER_OPTS" ]; then + clean_unused_sessions + COMPOSE_LAUNCHER_OPTS="$(mk_docker_run_options)" fi - service_def=$(get_compose_service_def "$service") || return 1 - _get_service_charm_cached "$service" "$service_def" -} -export -f get_service_charm + while read-0 opt; do + docker_run_opts+=("$opt") + done < <(cat "$COMPOSE_LAUNCHER_OPTS") -## built above the docker-compose abstraction, so it relies on the -## full docker-compose.yml to be already built. -get_service_def () { - local service="$1" def - if [ -z "$_current_docker_compose" ]; then - print_syntax_error "$FUNCNAME is meant to be called after"\ - "\$_current_docker_compose has been calculated." + COMPOSE_DOCKER_IMAGE=${COMPOSE_DOCKER_IMAGE:-docker.0k.io/compose} + if [ -t 1 ]; then + docker_run_opts+=("-ti") fi - - def=$(echo "$_current_docker_compose" | shyaml get-value "services.$service" 2>/dev/null) - if [ -z "$def" ]; then - err "No definition for service $DARKYELLOW$service$NORMAL in compiled 'docker-compose.yml'." - return 1 - fi - echo "$def" + exec docker run "${docker_run_opts[@]}" "${COMPOSE_DOCKER_IMAGE}" "$@" } -export -f get_service_def - - -## Return the base docker image name of a service -service_base_docker_image() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ - master_service service_def service_image service_build service_dockerfile - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" - return 0 - fi - master_service="$(get_top_master_service_for_service "$service")" || { - err "Could not compute master service for service $DARKYELLOW$service$NORMAL." - return 1 - } - service_def="$(get_service_def "$master_service")" || { - err "Could not get docker-compose service definition for $DARKYELLOW$master_service$NORMAL." - return 1 - } - service_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null) - if [ "$?" != 0 ]; then - service_build=$(echo "$service_def" | shyaml get-value build 2>/dev/null) - if [ "$?" != 0 ]; then - err "Service $DARKYELLOW$service$NORMAL has no ${WHITE}image${NORMAL} nor ${WHITE}build${NORMAL} parameter." - echo "$service_def" >&2 - return 1 - fi - service_dockerfile="${service_build}"/Dockerfile - if ! [ -e "$service_dockerfile" ]; then - err "No Dockerfile found in '$service_dockerfile' location." - return 1 - fi - - grep '^FROM' "$service_dockerfile" | xargs printf "%s " | cut -f 2 -d " " - else - echo "$service_image" - fi | tee "$cache_file" - -} -export -f service_base_docker_image - - -get_charm_relation_def () { - local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \ - relation_def metadata - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" - return 0 - fi - metadata="$(charm.metadata "$charm")" || return 1 - relation_def="$(echo "$metadata" | shyaml get-value "provides.${relation_name}" 2>/dev/null)" - echo "$relation_def" | tee "$cache_file" -} -export -f get_charm_relation_def - - -get_charm_tech_dep_orientation_for_relation() { - local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \ - relation_def value - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" - return 0 - fi - relation_def=$(get_charm_relation_def "$charm" "$relation_name" 2>/dev/null) - value=$(echo "$relation_def" | shyaml get-value 'tech-dep' 2>/dev/null) - value=${value:-True} - echo "$value" | tee "$cache_file" -} -export -f get_charm_tech_dep_orientation_for_relation - - -## -## Use compose file to get deps, and relation definition in metadata.yml -## for tech-dep attribute. -get_service_deps() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" - return 0 - fi - ( - set -o pipefail - get_compose_relations "$service" | \ - while read-0 relation_name target_service _relation_config tech_dep; do - echo "$target_service" - done | tee "$cache_file" - ) || return 1 -} -export -f get_service_deps - - -_rec_get_depth() { - local elt=$1 dep deps max cache_file="$state_tmpdir/$FUNCNAME.cache.$1" - [ "${depths[$elt]}" ] && return 0 - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - depths[$elt]=$(cat "$cache_file") - return 0 - fi - - visited[$elt]=1 - #debug "Setting visited[$elt]" - #debug "Asking for $DARKYELLOW$elt$NORMAL dependencies" - deps=$(get_service_deps "$elt") || { - debug "Failed get_service_deps $elt" - return 1 - } - # debug "$elt deps are:" $deps - max=0 - for dep in $deps; do - [ "${visited[$dep]}" ] && { - #debug "Already computing $dep" - continue - } - _rec_get_depth "$dep" || return 1 - #debug "Requesting depth[$dep]" - if (( ${depths[$dep]} > max )); then - max="${depths[$dep]}" - fi - done - # debug "Setting depth[$elt] to $((max + 1))" - depths[$elt]=$((max + 1)) - echo "${depths[$elt]}" > $cache_file -} -export -f _rec_get_depth - - -get_ordered_service_dependencies() { - local services=("$@") cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" - return 0 - fi - - #debug "Figuring ordered deps of $DARKYELLOW$services$NORMAL" - if [ -z "${services[*]}" ]; then - return 0 - # print_syntax_error "$FUNCNAME: no arguments" - # return 1 - fi - - declare -A depths - declare -A visited - heads=("${services[@]}") - while [ "${#heads[@]}" != 0 ]; do - array_pop heads head - _rec_get_depth "$head" || return 1 - done - - i=0 - while [ "${#depths[@]}" != 0 ]; do - for key in "${!depths[@]}"; do - value="${depths[$key]}" - if [ "$value" == "$i" ]; then - echo "$key" - unset depths[$key] - fi - done - i=$((i + 1)) - done | tee "$cache_file" -} -export -f get_ordered_service_dependencies - - -run_service_hook () { - local action="$1" service subservice subservices loaded - shift - declare -A loaded - for service in "$@"; do - subservices=$(get_ordered_service_dependencies "$service") || return 1 - for subservice in $subservices; do - if [ "${loaded[$subservice]}" ]; then - ## Prevent double inclusion of same service if this - ## service is deps of two or more of your - ## requirements. - continue - fi - - charm=$(get_service_charm "$subservice") || return 1 - charm.has_hook "$charm" "$action" >/dev/null || continue - - PROJECT_NAME=$(get_default_project_name) || return 1 - export MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1 - export MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1 - - Wrap -d "running $YELLOW$action$NORMAL hook of $DARKYELLOW$subservice$NORMAL in charm $DARKPINK$charm$NORMAL" </dev/null) || { - err "Missing ${WHITE}type$NORMAL option in ${WHITE}get$NORMAL config for location '$location'" - return 1 - } - if fn.exists host_resource_get_$type; then - host_resource_get_$type "$location" "$cfg" - else - err "Source ${WHITE}source$NORMAL type '$type' unknown for" \ - "${WHITE}host-resource$NORMAL '$location' defined in" \ - "$DARKYELLOW$subservice$NORMAL config." - return 1 - fi -} -export -f host_resource_get - - -host_resource_get_git() { - local location="$1" cfg="$2" branch parent url - branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null) - branch=${branch:-master} - - url=$(echo "$cfg" | shyaml get-value url 2>/dev/null) - parent="$(dirname "$location")" - ( - mkdir -p "$parent" && cd "$parent" && - git clone -b "$branch" "$url" "$(basename "$location")" - ) || return 1 -} -export -f host_resource_get_git - - -host_resource_get_git-sub() { - local location="$1" cfg="$2" branch parent url - branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null) - branch=${branch:-master} - - url=$(echo "$cfg" | shyaml get-value url 2>/dev/null) - parent="$(dirname "$location")" - ( - mkdir -p "$parent" && cd "$parent" && - git sub clone -b "$branch" "$url" "$(basename "$location")" - ) || return 1 -} -export -f host_resource_get_git-sub - - -setup_host_resource () { - local subservice="$1" service_def location get cfg - - service_def=$(get_compose_service_def "$subservice") || return 1 - while read-0 location cfg; do - ## XXXvlab: will it be a git resources always ? - if [ -d "$location" -a ! -d "$location/.git" ]; then - err "Hum, location '$location' does not seem to be a git directory." - return 1 - fi - if [ -d "$location" ]; then - info "host resource '$location' already set up." - continue - fi - get=$(echo "$cfg" | shyaml get-value get 2>/dev/null) - - if [ -z "$get" ]; then - err "No host directory '$location' found, and no ${WHITE}source$NORMAL" \ - "specified for $DARKYELLOW$subservice$NORMAL." - return 1 - fi - host_resource_get "$location" "$get" || return 1 - done < <(echo "$service_def" | shyaml key-values-0 host-resources 2>/dev/null) -} -export -f setup_host_resource - - -setup_host_resources () { - local service subservices subservice loaded - - declare -A loaded - for service in "$@"; do - subservices=$(get_ordered_service_dependencies "$service") || return 1 - for subservice in $subservices; do - if [ "${loaded[$subservice]}" ]; then - ## Prevent double inclusion of same service if this - ## service is deps of two or more of your - ## requirements. - continue - fi - setup_host_resource "$service" - loaded[$subservice]=1 - done - done - return 0 -} -export -f setup_host_resources - - -relation-get () { - local key="$1" - cat "$RELATION_DATA_FILE" | shyaml get-value "$key" 2>/dev/null - if [ "$?" != 0 ]; then - err "The key $WHITE$key$NORMAL was not found in relation's data." - return 1 - fi -} -export -f relation-get - - -relation-base-compose-get () { - local key="$1" - echo "$RELATION_BASE_COMPOSE_DEF" | shyaml get-value "options.$key" 2>/dev/null - if [ "$?" != 0 ]; then - err "The key $WHITE$key$NORMAL was not found in base service compose definition.." - return 1 - fi -} -export -f relation-base-compose-get - - -relation-target-compose-get () { - local key="$1" - echo "$RELATION_BASE_COMPOSE_DEF" | shyaml get-value "options.$key" 2>/dev/null - if [ "$?" != 0 ]; then - err "The key $WHITE$key$NORMAL was not found in base service compose definition.." - return 1 - fi -} -export -f relation-target-compose-get - - -relation-set () { - local key="$1" value="$2" - if [ -z "$RELATION_DATA_FILE" ]; then - err "$FUNCNAME: relation does not seems to be correctly setup." - return 1 - fi - - if ! [ -r "$RELATION_DATA_FILE" ]; then - err "$FUNCNAME: can't read relation's data." >&2 - return 1 - fi - - _config_merge "$RELATION_DATA_FILE" <(echo "$key: $value") -} -export -f relation-set - - -_config_merge() { - local config_filename="$1" merge_to_file="$2" - touch "$config_filename" && - merge_yaml "$config_filename" "$merge_to_file" > "$config_filename.tmp" || return 1 - mv "$config_filename.tmp" "$config_filename" -} -export -f _config_merge - - -## XXXvlab; this can be used only in relation, I'd like to use it in init. -config-add() { - local metadata="$1" - _config_merge "$RELATION_CONFIG" <(echo "$metadata") -} -export -f config-add - - -## XXXvlab; this can be used only in relation, I'd like to use it in init. -init-config-add() { - local metadata="$1" - _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" \ - <(yaml_key_val_str "services" "$metadata") -} -export -f init-config-add - - -logstdout() { - local name="$1" - sed -r 's%^%'"${name}"'> %g' -} -export -f logstdout - - -logstderr() { - local name="$1" - sed -r 's%^(.*)$%'"${RED}${name}>${NORMAL} \1"'%g' -} -export -f logstderr - - -_run_service_relation () { - local relation_name="$1" service="$2" target_service="$3" relation_config="$4" relation_dir services - - charm=$(get_service_charm "$service") || return 1 - target_charm=$(get_service_charm "$target_service") || return 1 - - base_script_name=$(charm.has_relation_hook "$charm" "$relation_name" relation-joined) || true - target_script_name=$(charm.has_relation_hook "$target_charm" "$relation_name" relation-joined) || true - [ "$base_script_name" -o "$target_script_name" ] || return 0 - - relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1 - RELATION_DATA_FILE=$(get_relation_data_file "$service" "$target_service" "$relation_name" "$relation_config") || return 1 - export BASE_SERVICE_NAME=$service - export BASE_CHARM_NAME=$charm - export BASE_CHARM_PATH=$(charm.get_dir "$charm") - export TARGET_SERVICE_NAME=$target_service - export TARGET_CHARM_NAME=$target_charm - export TARGET_CHARM_PATH=$(charm.get_dir "$target_charm") - export RELATION_DATA_FILE - target_errlvl=0 - - if [ -z "$target_script_name" ]; then - verb "No relation script $DARKBLUE$relation_name$NORMAL in target $DARKPINK$target_charm$NORMAL." - else - verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \ - "for target $DARKYELLOW$target_service$NORMAL (charm $DARKPINK$target_charm$NORMAL)" - RELATION_CONFIG="$relation_dir/config_provider" - DOCKER_BASE_IMAGE=$(service_base_docker_image "$target_service") || return 1 - export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA - { - ( - SERVICE_NAME=$target_service - SERVICE_DATASTORE="$DATASTORE/$target_service" - SERVICE_CONFIGSTORE="$CONFIGSTORE/$target_service" - export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE - charm.run_relation_hook "$target_charm" "$relation_name" relation-joined - echo "$?" > "$relation_dir/target_errlvl" - ) | logstdout "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}" - } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@${NORMAL}" 3>&1 1>&2 2>&3 - target_errlvl="$(cat "$relation_dir/target_errlvl")" || { - err "Relation script '$script_name' in $DARKPINK$target_charm$NORMAL" \ - "failed before outputing an errorlevel." - ((target_errlvl |= "1" )) - } - if [ -e "$RELATION_CONFIG" ]; then - debug "Merging some new config info in $DARKYELLOW$target_service$NORMAL" - _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" && - rm "$RELATION_CONFIG" - ((target_errlvl |= "$?")) - fi - fi - - if [ "$target_errlvl" == 0 ]; then - errlvl=0 - if [ "$base_script_name" ]; then - verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \ - "for $DARKYELLOW$service$NORMAL (charm $DARKPINK$charm$NORMAL)" - RELATION_CONFIG="$relation_dir/config_providee" - RELATION_DATA="$(cat "$RELATION_DATA_FILE")" - DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") || return 1 - export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA - { - ( - SERVICE_NAME=$service - SERVICE_DATASTORE="$DATASTORE/$service" - SERVICE_CONFIGSTORE="$CONFIGSTORE/$service" - export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE - charm.run_relation_hook "$charm" "$relation_name" relation-joined - echo "$?" > "$relation_dir/errlvl" - ) | logstdout "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}" - } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@$NORMAL" 3>&1 1>&2 2>&3 - errlvl="$(cat "$relation_dir/errlvl")" || { - err "Relation script '$script_name' in $DARKPINK$charm$NORMAL" \ - "failed before outputing an errorlevel." - ((errlvl |= "1" )) - } - if [ -e "$RELATION_CONFIG" ]; then - _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" && - rm "$RELATION_CONFIG" - ((errlvl |= "$?" )) - fi - if [ "$errlvl" != 0 ]; then - err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$service$NORMAL failed to run properly." - fi - else - verb "No relation script '$script_name' in charm $DARKPINK$charm$NORMAL. Ignoring." - fi - else - err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$target_service$NORMAL failed to run properly." - fi - - if [ "$target_errlvl" == 0 -a "$errlvl" == 0 ]; then - debug "Relation $DARKBLUE$relation_name$NORMAL is established" \ - "between $DARKYELLOW$service$NORMAL and $DARKYELLOW$target_service$NORMAL." - return 0 - else - return 1 - fi -} -export -f _run_service_relation - - -_get_compose_relations_cached () { - local compose_service_def="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ - relation_name relation_def target_service - if [ -e "$cache_file" ]; then - #debug "$FUNCNAME: STATIC cache hit $1" - cat "$cache_file" && - touch "$cache_file" || return 1 - return 0 - fi - - ( - set -o pipefail - if [ "$compose_service_def" ]; then - while read-0 relation_name relation_def; do - ## XXXvlab: could we use braces here instead of parenthesis ? - ( - case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in - "str") - target_service="$(echo "$relation_def" | shyaml get-value 2>/dev/null)" - target_charm=$(get_service_charm "$target_service") || return 1 - tech_dep="$(get_charm_tech_dep_orientation_for_relation "$target_charm" "$relation_name")" - echo -en "$relation_name\0$target_service\0\0$tech_dep\0" - ;; - "sequence") - while read-0 target_service; do - target_charm=$(get_service_charm "$target_service") || return 1 - tech_dep="$(get_charm_tech_dep_orientation_for_relation "$target_charm" "$relation_name")" - echo -en "$relation_name\0$target_service\0\0$tech_dep\0" - done < <(echo "$relation_def" | shyaml get-values-0 2>/dev/null) - ;; - "struct") - while read-0 target_service relation_config; do - target_charm=$(get_service_charm "$target_service") || return 1 - tech_dep="$(get_charm_tech_dep_orientation_for_relation "$target_charm" "$relation_name")" - echo -en "$relation_name\0$target_service\0$relation_config\0$tech_dep\0" - done < <(echo "$relation_def" | shyaml key-values-0 2>/dev/null) - ;; - esac - ) > "$cache_file" || return 1 - done < <(echo "$compose_service_def" | shyaml key-values-0 relations 2>/dev/null) - fi - ) - if [ "$?" != 0 ]; then - err "Error while looking for compose relations." - rm -f "$cache_file" ## no cache - return 1 - fi - [ -e "$cache_file" ] && cat "$cache_file" - return 0 -} -export -f _get_compose_relations_cached - - -get_compose_relations () { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ - compose_def - - if [ -e "$cache_file" ]; then - #debug "$FUNCNAME: SESSION cache hit $1" - cat "$cache_file" - return 0 - fi - - compose_def="$(get_compose_service_def "$service")" || return 1 - _get_compose_relations_cached "$compose_def" > "$cache_file" - if [ "$?" != 0 ]; then - rm -f "$cache_file" ## no cache - return 1 - fi - cat "$cache_file" -} -export -f get_compose_relations - - -get_compose_relation_def() { - local service="$1" relation="$2" relation_name target_service relation_config tech_dep - while read-0 relation_name target_service relation_config tech_dep; do - [ "$relation_name" == "$relation" ] || continue - printf "%s\0%s\0%s\0" "$target_service" "$relation_config" "$tech_dep" - done < <(get_compose_relations "$service") || return 1 -} -export -f get_compose_relation_def - - -run_service_relations () { - local service services loaded subservices subservice - - PROJECT_NAME=$(get_default_project_name) || return 1 - - export PROJECT_NAME - - declare -A loaded - subservices=$(get_ordered_service_dependencies "$@") || return 1 - for service in $subservices; do - - # debug "Upping dep's relations of ${DARKYELLOW}$service${NORMAL}:" - for subservice in $(get_service_deps "$service") "$service"; do - [ "${loaded[$subservice]}" ] && continue - - export BASE_SERVICE_NAME=$service - MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1 - MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1 - RELATION_BASE_COMPOSE_DEF=$(get_compose_service_def "$subservice") || return 1 - export RELATION_BASE_COMPOSE_DEF MASTER_BASE_{CHARM,SERVICE}_NAME - - # debug " Relations of ${DARKYELLOW}$subservice${NORMAL}:" - while read-0 relation_name target_service relation_config tech_dep; do - export relation_config - - export TARGET_SERVICE_NAME=$target_service - MASTER_TARGET_SERVICE_NAME=$(get_top_master_service_for_service "$target_service") || return 1 - MASTER_TARGET_CHARM_NAME=$(get_service_charm "$MASTER_TARGET_SERVICE_NAME") || return 1 - RELATION_TARGET_COMPOSE_DEF=$(get_compose_service_def "$target_service") || return 1 - export RELATION_TARGET_COMPOSE_DEF MASTER_TARGET_{CHARM,SERVICE}_NAME - - Wrap -d "Building $DARKYELLOW$subservice$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW$target_service$NORMAL" < "$action_errlvl_file" - ) | logstdout "$DARKYELLOW$service$NORMAL/${DARKCYAN}$action${NORMAL} ${GREEN}@${NORMAL}" - } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$service$NORMAL/${DARKCYAN}$action${NORMAL} ${RED}@$NORMAL" 3>&1 1>&2 2>&3 - if ! [ -e "$action_errlvl_file" ]; then - err "Action $DARKYELLOW$service$NORMAL:$DARKCYAN$action$NORMAL has failed without having time" \ - "to output an errlvl" - return 1 - fi - return "$(cat "$action_errlvl_file")" -} -export -f _run_service_action_direct - -_run_service_action_relation() { - local service="$1" action="$2" charm target_charm relation_name relation_config _dummy - shift; shift - - read-0 charm target_service target_charm relation_name relation_config || true - - if read-0 _dummy || [ "$_dummy" ]; then - print_syntax_error "$FUNCNAME: too many arguments in action descriptor" - return 1 - fi - - export RELATION_DATA_FILE=$(get_relation_data_file "$service" "$target_service" "$relation_name" "$relation_config") - - export action_errlvl_file="$state_tmpdir/action-$service-$charm-$action-errlvl" - export state_tmpdir - { - ( - set +e ## Prevents unwanted leaks from parent shell - export METADATA_CONFIG=$(charm.metadata "$charm") - export SERVICE_NAME=$service - export RELATION_TARGET_SERVICE="$target_service" - export RELATION_TARGET_CHARM="$target_charm" - export RELATION_BASE_SERVICE="$service" - export RELATION_BASE_CHARM="$charm" - export ACTION_NAME=$action - export CONTAINER_NAME=$(get_top_master_service_for_service "$service") - export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME") - export SERVICE_DATASTORE="$DATASTORE/$service" - export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service" - exname="$exname $ACTION_NAME $SERVICE_NAME" \ - stdbuf -oL -eL bash -c 'charm.run_relation_action "$@"' -- "$target_charm" "$relation_name" "$action" "$@" - echo "$?" > "$action_errlvl_file" - ) | logstdout "$DARKYELLOW$service$NORMAL/${DARKCYAN}$action${NORMAL} ${GREEN}@${NORMAL}" - } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$service$NORMAL/${DARKCYAN}$action${NORMAL} ${RED}@$NORMAL" 3>&1 1>&2 2>&3 - if ! [ -e "$action_errlvl_file" ]; then - err "Action $DARKYELLOW$service$NORMAL:$DARKCYAN$action$NORMAL has failed without having time" \ - "to output an errlvl" - return 1 - fi - return "$(cat "$action_errlvl_file")" -} -export -f _run_service_action_relation - - -get_relation_data_dir() { - local service="$1" target_service="$2" relation_name="$3" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" - return 0 - fi - - project=$(get_default_project_name) || return 1 - relation_dir="$VARDIR/relations/$project/${service}-${target_service}/$relation_name" - if ! [ -d "$relation_dir" ]; then - mkdir -p "$relation_dir" || return 1 - chmod go-rwx "$relation_dir" || return 1 ## protecting this directory - fi - echo "$relation_dir" || tee "$cache_file" -} -export -f get_relation_data_dir - - -get_relation_data_file() { - local service="$1" target_service="$2" relation_name="$3" relation_config="$4" - - relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1 - relation_data_file="$relation_dir/data" - - new= - if [ -e "$relation_data_file" ]; then - ## Has reference changed ? - new_md5=$(echo "$relation_config" | md5_compat) - if [ "$new_md5" != "$(cat "$relation_data_file.md5_ref" 2>/dev/null)" ]; then - new=true - fi - else - new=true - fi - - if [ "$new" ]; then - echo "$relation_config" > "$relation_data_file" - chmod go-rwx "$relation_data_file" ## protecting this file - echo "$relation_config" | md5_compat > "$relation_data_file.md5_ref" - fi - echo "$relation_data_file" -} -export -f get_relation_data_file - - -has_service_action () { - local service="$1" action="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ - charm target_charm relation_name target_service relation_config _tech_dep - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" - return 0 - fi - - charm=$(get_service_charm "$service") || return 1 - - ## Action directly provided ? - - if charm.has_direct_action "$charm" "$action" >/dev/null; then - echo -en "direct\0$charm" | tee "$cache_file" - return 0 - fi - - ## Action provided by relation ? - - while read-0 relation_name target_service relation_config _tech_dep; do - target_charm=$(get_service_charm "$target_service") || return 1 - if charm.has_relation_action "$target_charm" "$relation_name" "$action" >/dev/null; then - echo -en "relation\0$charm\0$target_service\0$target_charm\0$relation_name\0$relation_config" | tee "$cache_file" - return 0 - fi - done < <(get_compose_relations "$service") - - return 1 - # master=$(get_top_master_service_for_service "$service") - # [ "$master" == "$charm" ] && return 1 - - # has_service_action "$master" "$action" -} -export -f has_service_action - - -run_service_action () { - local service="$1" action="$2" - shift ; shift - { - if ! read-0 action_type; then - info "Service $DARKYELLOW$service$NORMAL does not have any action $DARKCYAN$action$NORMAL defined." - info " Add an executable script to 'actions/$action' to implement action." - return 1 - fi - - Section "running $DARKYELLOW$service$NORMAL/$DARKCYAN$action$NORMAL ($action_type)"; Feed - "_run_service_action_${action_type}" "$service" "$action" "$@" - - } < <(has_service_action "$service" "$action") -} -export -f run_service_action - - -get_compose_relation_config() { - local service=$1 relation_config cache_file="$state_tmpdir/$FUNCNAME.cache.$1" - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" - return 0 - fi - - compose_service_def=$(get_compose_service_def "$service") || return 1 - - echo "$compose_service_def" | shyaml get-value "relations" 2>/dev/null | tee "$cache_file" -} -export -f get_compose_relation_config - - -# ## Return key-values-0 -# get_compose_relation_config_for_service() { -# local service=$1 relation_name=$2 relation_config -# compose_service_relations=$(get_compose_relation_config "$service") || return 1 -# if ! relation_config=$( -# echo "$compose_service_relations" | -# shyaml get-value "${relation_name}" 2>/dev/null); then -# err "Couldn't find $DARKYELLOW${service}$NORMAL/${WHITE}${relation_name}$NORMAL" \ -# "relation config in compose configuration." -# return 1 -# fi -# if [ -z "$relation_config" ]; then -# err "Relation ${WHITE}mysql-database$NORMAL is empty in compose configuration." -# return 1 -# fi -# if ! echo "$relation_config" | shyaml key-values-0 2>/dev/null; then -# err "No key/values in ${DARKBLUE}mysql-database$NORMAL of compose config." -# return 1 -# fi -# } -# export -f get_compose_relation_config_for_service - - -_get_master_service_for_service_cached () { - local service="$1" charm="$2" metadata="$3" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ - charm requires master_charm target_charm target_service service_def found - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: STATIC cache hit ($1)" - cat "$cache_file" && - touch "$cache_file" || return 1 - return 0 - fi - - if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" != "True" ]; then - ## just return service name - echo "$service" | tee "$cache_file" - return 0 - fi - - ## fetch the container relation - requires="$(echo "$metadata" | shyaml get-value "requires" 2>/dev/null)" - if [ -z "$requires" ]; then - die "Charm $DARKPINK$charm$NORMAL is a subordinate but does not have any 'requires' " \ - "section." - fi - found= - while read-0 relation_name relation; do - [ "$(echo "$relation" | shyaml get-value "scope" 2>/dev/null)" == "container" ] && { - found=1 - break - } - done < <(echo "$requires" | shyaml key-values-0 2>/dev/null) - if [ -z "$found" ]; then - die "Charm $DARKPINK$charm$NORMAL is a subordinate but does not have any required relation declaration with" \ - " ${WHITE}scope${NORMAL} set to 'container'." - fi - - interface="$(echo "$relation" | shyaml get-value "interface" 2>/dev/null)" - if [ -z "$interface" ]; then - err "No ${WHITE}interface${NORMAL} set for relation $DARKBLUE$relation_name$NORMAL." - return 1 - fi - - ## Action provided by relation ? - - found= - while read-0 relation_name target_service _relation_config _tech_dep; do - [ "$interface" == "$relation_name" ] && { - found=1 - break - } - done < <(get_compose_relations "$service") - if [ -z "$found" ]; then - err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \ - "${DARKYELLOW}$service$NORMAL compose definition." - return 1 - fi - echo "$target_service" | tee "$cache_file" -} -export -f _get_master_service_for_service_cached - - -get_master_service_for_service() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ - charm metadata result - - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: SESSION cache hit ($*)" - cat "$cache_file" || return 1 - return 0 - fi - charm=$(get_service_charm "$service") || return 1 - metadata=$(charm.metadata "$charm" 2>/dev/null) || { - metadata="" - warn "No charm $DARKPINK$charm$NORMAL found." - } - result=$(_get_master_service_for_service_cached "$service" "$charm" "$metadata") || return 1 - echo "$result" | tee "$cache_file" || return 1 -} -export -f get_master_service_for_service - - -get_top_master_service_for_service() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ - current_service - - if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" - cat "$cache_file" - return 0 - fi - - current_service="$service" - while true; do - master_service=$(get_master_service_for_service "$current_service") || return 1 - [ "$master_service" == "$current_service" ] && break - current_service="$master_service" - done - echo "$current_service" | tee "$cache_file" - return 0 -} -export -f get_top_master_service_for_service - - -## -## The result is a mixin that is not always a complete valid -## docker-compose entry (thinking of subordinates). The result -## will be merge with master charms. -_get_docker_compose_mixin_from_metadata_cached() { - local service="$1" charm="$2" metadata="$3" has_build_dir="$4" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ - metadata_file metadata volumes docker_compose subordinate image - if [ -e "$cache_file" ]; then - #debug "$FUNCNAME: STATIC cache hit $1" - cat "$cache_file" && - touch "$cache_file" || return 1 - return 0 - fi - - mixin=$(echo -en "labels:\n- compose.charm=$charm") - if [ "$metadata" ]; then - ## resources to volumes - volumes=$( - for resource_type in data config; do - while read-0 resource; do - eval "echo \" - \$${resource_type^^}STORE/\$service\$resource:\$resource:rw\"" - done < <(echo "$metadata" | shyaml get-values-0 "${resource_type}-resources" 2>/dev/null) - done - while read-0 resource; do - if [[ "$resource" == /*:/*:* ]]; then - echo " - $resource" - elif [[ "$resource" == /*:/* ]]; then - echo " - $resource:rw" - elif [[ "$resource" == /*:* ]]; then - echo " - ${resource%%:*}:$resource" - elif [[ "$resource" =~ ^/[^:]+$ ]]; then - echo " - $resource:$resource:rw" - else - die "Invalid host-resource specified in 'metadata.yml'." - fi - done < <(echo "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null) - while read-0 resource; do - dest="$(charm.get_dir "$charm")/resources$resource" - if ! [ -e "$dest" ]; then - die "charm-resource: '$resource' does not exist (file: '$dest')." - fi - echo " - $dest:$resource:ro" - done < <(echo "$metadata" | shyaml get-values-0 "charm-resources" 2>/dev/null) - ) || return 1 - if [ "$volumes" ]; then - mixin=$(merge_yaml_str "$mixin" "$(echo -en "volumes:\n$volumes")") || { - err "Failed to merge mixin with ${WHITE}docker-compose${NORMAL} option" \ - "from charm ${DARKPINK}$charm$NORMAL." - return 1 - } - fi - - docker_compose=$(echo "$metadata" | shyaml get-value -y "docker-compose" 2>/dev/null) - if [ "$docker_compose" ]; then - mixin=$(merge_yaml_str "$mixin" "$docker_compose") || { - err "Failed to merge mixin with ${WHITE}docker-compose${NORMAL} option" \ - "from charm ${DARKPINK}$charm$NORMAL." - return 1 - } - fi - - if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" == "True" ]; then - subordinate=true - fi - fi - - image=$(echo "$metadata" | shyaml get-value "docker-image" 2>/dev/null) - [ "$image" == "None" ] && image="" - image_or_build_statement= - if [ "$image" ]; then - if [ "$subordinate" ]; then - err "Subordinate charm can not have a ${WHITE}docker-image${NORMAL} value." - return 1 - fi - image_or_build_statement="image: $image" - elif [ "$has_build_dir" ]; then - if [ "$subordinate" ]; then - err "Subordinate charm can not have a 'build' sub directory." - return 1 - fi - image_or_build_statement="build: $(charm.get_dir "$charm")/build" - fi - if [ "$image_or_build_statement" ]; then - mixin=$(merge_yaml_str "$mixin" "$image_or_build_statement") || { - err "Failed to merge yaml with image or build YAML statement." - return 1 - } - fi - echo "$mixin" | tee "$cache_file" -} -export -f _get_docker_compose_mixin_from_metadata_cached - - -get_docker_compose_mixin_from_metadata() { - local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" - if [ -e "$cache_file" ]; then - #debug "$FUNCNAME: SESSION cache hit ($*)" - cat "$cache_file" - return 0 - fi - charm=$(get_service_charm "$service") || return 1 - metadata="$(charm.metadata "$charm" 2>/dev/null)" || return 1 - has_build_dir= - [ -d "$(charm.get_dir "$charm")/build" ] && has_build_dir=true - mixin=$(_get_docker_compose_mixin_from_metadata_cached "$service" "$charm" "$metadata" "$has_build_dir") || return 1 - echo "$mixin" | tee "$cache_file" -} -export -f get_docker_compose_mixin_from_metadata - - -_save() { - local name="$1" - cat - | tee -a "$docker_compose_dir/.data/$name" -} -export -f _save - - -get_default_project_name() { - if [ "$DEFAULT_PROJECT_NAME" ]; then - echo "$DEFAULT_PROJECT_NAME" - return 0 - fi - compose_yml_location="$(get_compose_yml_location)" || return 1 - if [ "$compose_yml_location" ]; then - if normalized_path=$(readlink -f "$compose_yml_location"); then - echo "$(basename "$(dirname "$normalized_path")")" - return 0 - fi - fi - echo "orphan" - return 0 -} -export -f get_default_project_name - - -launch_docker_compose() { - local charm docker_compose_tmpdir docker_compose_dir - docker_compose_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX) - #debug "Creating temporary docker-compose directory in '$docker_compose_tmpdir'." - trap_add EXIT "rm -rf \"$docker_compose_tmpdir\"" - - ## docker-compose will name network from the parent dir name - project=$(get_default_project_name) - mkdir -p "$docker_compose_tmpdir/$project" - docker_compose_dir="$docker_compose_tmpdir/$project" - - get_docker_compose $SERVICE_PACK > "$docker_compose_dir/docker-compose.yml" || return 1 - if [ -e "$state_tmpdir/to-merge-in-docker-compose.yml" ]; then - # debug "Merging some config data in docker-compose.yml:" - # debug "$(cat $state_tmpdir/to-merge-in-docker-compose.yml)" - _config_merge "$docker_compose_dir/docker-compose.yml" "$state_tmpdir/to-merge-in-docker-compose.yml" || return 1 - fi - - if [ -z "$(echo $(cat "$docker_compose_dir/docker-compose.yml"))" ]; then - die "Generated 'docker-compose.yml' is unexpectedly empty." - fi - - ## XXXvlab: could be more specific and only link the needed charms - ## XXXvlab: why do we need these links ? If this is for the build command, then it is not useful anymore. - # for charm in $(shyaml keys services < "$docker_compose_dir/docker-compose.yml"); do - # if charm.exists "$charm"; then - # ln -sf "$(charm.get_dir "$charm")" "$docker_compose_dir/$charm" || exit 1 - # fi - # done - mkdir "$docker_compose_dir/.data" - - { - { - cd "$docker_compose_dir" - debug "${WHITE}docker-compose.yml$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:" - debug "$(cat "docker-compose.yml" | prefix " $GRAY|$NORMAL ")" - debug "${WHITE}Launching$NORMAL: docker-compose $@" - if [ "$DRY_COMPOSE_RUN" ]; then - echo docker-compose "$@" - else - docker-compose "$@" - fi - echo "$?" > "$docker_compose_dir/.data/errlvl" - } | _save stdout - } 3>&1 1>&2 2>&3 | _save stderr 3>&1 1>&2 2>&3 - if tail -n 1 "$docker_compose_dir/.data/stderr" | egrep "Service .+ failed to build: Error getting container [0-9a-f]+ from driver devicemapper: (open|Error mounting) /dev/mapper/docker-.*: no such file or directory$" >/dev/null 2>&1; then - err "Detected bug https://github.com/docker/docker/issues/4036 ... " - err "Please re-launch your command, or switch from 'devicemapper' driver to 'overlayfs' or 'aufs'." - fi - - docker_compose_errlvl="$(cat "$docker_compose_dir/.data/errlvl" 2>/dev/null)" - if [ -z "$docker_compose_errlvl" ]; then - err "Something went wrong before you could gather docker-compose errorlevel." - return 1 - fi - return "$docker_compose_errlvl" -} -export -f launch_docker_compose - - -get_compose_yml_location() { - if ! [ -z ${COMPOSE_YML_FILE+x} ]; then ## if set, even if empty - echo "$COMPOSE_YML_FILE" - return 0 - fi - parent=$(while ! [ -f "./compose.yml" ]; do - [ "$PWD" == "/" ] && exit 0 - cd .. - done; echo "$PWD" - ) - if [ "$parent" ]; then - echo "$parent/compose.yml" - return 0 - fi - ## XXXvlab: do we need this additional environment variable, - ## COMPOSE_YML_FILE is not sufficient ? - if [ "$DEFAULT_COMPOSE_FILE" ]; then - if ! [ -e "$DEFAULT_COMPOSE_FILE" ]; then - warn "No 'compose.yml' was found in current or parent dirs," \ - "and \$DEFAULT_COMPOSE_FILE points to an unexistent file." \ - "(${DEFAULT_COMPOSE_FILE})" - return 0 - fi - echo "$DEFAULT_COMPOSE_FILE" - return 0 - fi - warn "No 'compose.yml' was found in current or parent dirs, and no \$DEFAULT_COMPOSE_FILE was set." - return 0 -} -export -f get_compose_yml_location - - -get_compose_yml_content() { - local cache_file="$state_tmpdir/$FUNCNAME.cache" - if [ -e "$cache_file" ]; then - cat "$cache_file" && - touch "$cache_file" || return 1 - return 0 - fi - - if [ -z "$COMPOSE_YML_FILE" ]; then - COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1 - fi - if [ -e "$COMPOSE_YML_FILE" ]; then - debug "Found $WHITE$exname$NORMAL YAML file in '$COMPOSE_YML_FILE'." - COMPOSE_YML_CONTENT=$(cat "$COMPOSE_YML_FILE") || { - err "Could not read '$COMPOSE_YML_FILE'." - return 1 - } - else - debug "No compose file found. Using an empty one." - COMPOSE_YML_CONTENT="" - fi - - COMPOSE_YML_CONTENT=$(merge_yaml_str "$COMPOSE_YML_CONTENT" "${compose_contents[@]}") || return 1 - output=$(echo "$COMPOSE_YML_CONTENT"| shyaml get-value 2>&1) - if [ "$?" != 0 ]; then - outputed_something= - while IFS='' read -r line1 && IFS='' read -r line2; do - [ "$outputed_something" ] || err "Invalid YAML in '$COMPOSE_YML_FILE':" - outputed_something=true - echo "$line1 $GRAY($line2)$NORMAL" - done < <(echo "$output" | grep ^yaml.scanner -A 100 | - sed -r 's/^ in "", //g' | sed -r 's/^yaml.scanner.[a-zA-Z]+: //g') | - prefix " $GRAY|$NORMAL " - [ "$outputed_something" ] || { - err "Unexpected error while running 'shyaml get-value' on '$COMPOSE_YML_FILE':" - echo "$output" | prefix " $GRAY|$NORMAL " - } - return 1 - fi - echo "$COMPOSE_YML_CONTENT" | tee "$cache_file" || return 1 -} -export -f get_compose_yml_content - - -get_default_target_services() { - local services=("$@") - if [ -z "${services[*]}" ]; then - if [ "$DEFAULT_SERVICES" ]; then - debug "No service provided, using $WHITE\$DEFAULT_SERVICES$NORMAL variable." \ - "Target services: $DARKYELLOW$DEFAULT_SERVICES$NORMAL" - services="$DEFAULT_SERVICES" - else - err "No service provided." - return 1 - fi - fi - echo "${services[*]}" -} -export -f get_default_target_services - - -get_master_services() { - local loaded master_service service - declare -A loaded - for service in "$@"; do - master_service=$(get_top_master_service_for_service "$service") || return 1 - if [ "${loaded[$master_service]}" ]; then - continue - fi - echo "$master_service" - loaded["$master_service"]=1 - done | xargs printf "%s " - return "${PIPESTATUS[0]}" -} -export -f get_master_services - - -_setup_state_dir() { - export state_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX) - #debug "Creating temporary state directory in '$state_tmpdir'." -# trap_add EXIT "debug \"Removing temporary state directory in $state_tmpdir.\";\ -# rm -rf \"$state_tmpdir\"" - trap_add EXIT "rm -rf \"$state_tmpdir\"" -} - - -get_docker_compose_help_msg() { - local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \ - docker_compose_help_msg - if [ -e "$cache_file" ]; then - cat "$cache_file" && - touch "$cache_file" || return 1 - return 0 - fi - docker_compose_help_msg=$(docker-compose $action --help 2>/dev/null) || return 1 - echo "$docker_compose_help_msg" | - tee "$cache_file" || return 1 -} - - -get_docker_compose_usage() { - local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \ - docker_compose_help_msg - if [ -e "$cache_file" ]; then - cat "$cache_file" && - touch "$cache_file" || return 1 - return 0 - fi - docker_compose_help_msg=$(get_docker_compose_help_msg $action) || return 1 - echo "$docker_compose_help_msg" | - grep -m 1 "^Usage:" -A 10000 | - egrep -m 1 "^\$" -B 10000 | - xargs printf "%s " | - sed -r 's/^Usage: //g' | - tee "$cache_file" || return 1 -} - - -get_docker_compose_opts_help() { - local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \ - docker_compose_help_msg - if [ -e "$cache_file" ]; then - cat "$cache_file" && - touch "$cache_file" || return 1 - return 0 - fi - docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1 - echo "$docker_compose_opts_help" | - grep '^Options:' -A 20000 | - tail -n +2 | - { cat ; echo; } | - egrep -m 1 "^\S*\$" -B 10000 | - head -n -1 | - tee "$cache_file" || return 1 -} - - -get_docker_compose_commands_help() { - local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \ - docker_compose_help_msg - if [ -e "$cache_file" ]; then - cat "$cache_file" && - touch "$cache_file" || return 1 - return 0 - fi - docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1 - echo "$docker_compose_opts_help" | - grep '^Commands:' -A 20000 | - tail -n +2 | - { cat ; echo; } | - egrep -m 1 "^\S*\$" -B 10000 | - head -n -1 | - tee "$cache_file" || return 1 -} - - -get_docker_compose_opts_list() { - local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \ - docker_compose_help_msg - if [ -e "$cache_file" ]; then - cat "$cache_file" && - touch "$cache_file" || return 1 - return 0 - fi - docker_compose_opts_help=$(get_docker_compose_opts_help $action) || return 1 - echo "$docker_compose_opts_help" | - egrep "^\s+-" | - sed -r 's/\s+((((-[a-zA-Z]|--[a-zA-Z0-9-]+)( [A-Z=]+|=[^ ]+)?)(, )?)+)\s+.*$/\1/g' | - tee "$cache_file" || return 1 -} - - -options_parser() { - sed -r 's/^(\s+(((-[a-zA-Z]|--[a-zA-Z0-9-]+)([ =]([a-zA-Z_=\"\[]|\])+)?(, | )?)+)\s+)[^ ].*$/\x0\2\x0\0/g' - printf "\0" -} - - -remove_options_in_option_help_msg() { - { - read-0 null - if [ "$null" ]; then - err "options parsing error, should start with an option line." - return 1 - fi - while read-0 opt full_txt;do - multi_opts="$(printf "%s " $opt | multi_opts_filter)" - single_opts="$(printf "%s " $opt | single_opts_filter)" - for to_remove in "$@"; do - str_matches "$to_remove" $multi_opts $single_opts && { - continue 2 - } - done - echo -n "$full_txt" - done - } < <(options_parser) -} - - -_MULTIOPTION_REGEX='^((-[a-zA-Z]|--[a-zA-Z0-9-]+)(, )?)+' -_MULTIOPTION_REGEX_LINE_FILTER=$_MULTIOPTION_REGEX'(\s|=)' - - -multi_opts_filter() { - egrep "$_MULTIOPTION_REGEX_LINE_FILTER" | - sed -r "s/^($_MULTIOPTION_REGEX)(\s|=).*$/\1/g" | - tr ',' "\n" | xargs printf "%s " -} - - -single_opts_filter() { - egrep -v "$_MULTIOPTION_REGEX_LINE_FILTER" | - tr ',' "\n" | xargs printf "%s " -} - - -get_docker_compose_multi_opts_list() { - local action="$1" opts_list - opts_list=$(get_docker_compose_opts_list "$action") || return 1 - echo "$opts_list" | multi_opts_filter -} - - -get_docker_compose_single_opts_list() { - local action="$1" opts_list - opts_list=$(get_docker_compose_opts_list "$action") || return 1 - echo "$opts_list" | single_opts_filter -} - - -display_help() { - print_help - echo "${WHITE}Options${NORMAL}:" - echo " -h, --help Print this message and quit" - echo " (ignoring any other options)" - echo " -V, --version Print current version and quit" - echo " (ignoring any other options)" - echo " --dirs Display data dirs and quit" - echo " (ignoring any other options)" - echo " -v, --verbose Be more verbose" - echo " -d, --debug Print full debugging information (sets also verbose)" - echo " --dry-compose-run If docker-compose will be run, only print out what" - echo " command line will be used." - echo " --rebuild-relations-to-service, -R SERVICE" - echo " Will rebuild all relations to given service" - echo " --add-compose-content, -Y YAML" - echo " Will merge some direct YAML with the current compose" - - get_docker_compose_opts_help | remove_options_in_option_help_msg --version --help --verbose | - filter_docker_compose_help_message - echo - echo "${WHITE}Commands${NORMAL} (thanks to docker-compose):" - get_docker_compose_commands_help | sed -r "s/ ([a-z]+)(\s+)/ ${DARKCYAN}\1${NORMAL}\2/g" -} - - -_graph_service() { - local service="$1" base="$1" - - charm=$(get_service_charm "$service") || return 1 - metadata=$(charm.metadata "$charm") || return 1 - subordinate=$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null) - - if [ "$subordinate" == "True" ]; then - requires="$(echo "$metadata" | shyaml get-value "requires" 2>/dev/null)" - master_charm= - while read-0 relation_name relation; do - [ "$(echo "$relation" | shyaml get-value "scope" 2>/dev/null)" == "container" ] || continue - interface="$(echo "$relation" | shyaml get-value "interface" 2>/dev/null)" - if [ -z "$interface" ]; then - err "No ${WHITE}$interface${NORMAL} set for relation $relation_name." - return 1 - fi - - ## Action provided by relation ? - - target_service= - while read-0 relation_name candidate_target_service _relation_config _tech_dep; do - [ "$interface" == "$relation_name" ] && { - target_service="$candidate_target_service" - break - } - done < <(get_compose_relations "$service") - if [ -z "$target_service" ]; then - err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \ - "${DARKYELLOW}$service$NORMAL compose definition." - return 1 - fi - master_service="$target_service" - master_charm=$(get_service_charm "$target_service") || return 1 - break - done < <(echo "$requires" | shyaml key-values-0 2>/dev/null) - fi - - _graph_node_service "$service" "$base" "$charm" - _graph_edge_service "$service" "$subordinate" "$master_service" - -} - - -_graph_node_service() { - local service="$1" base="$2" charm="$3" - - cat < - ]; -EOF - -} - - -_graph_edge_service() { - local service="$1" subordinate="$2" master_service="$3" - while read-0 relation_name target_service relation_config tech_dep; do - cat < "$(_graph_node_service_label ${target_service})" [ - penwidth = $([ "$master_service" == "$target_service" ] && echo 3 || echo 2) - - fontsize = 16 - fontcolor = "black" - style = $([ "$master_service" == "$target_service" ] && echo dashed || echo "\"\"") - weight = $([ "$master_service" == "$target_service" ] && echo 2.0 || echo 1.0) - dir = $([ "$master_service" == "$target_service" ] && echo none || echo both) - arrowtail = odot - # arrowhead = dotlicurve - taillabel = "$relation_name" ]; -EOF - done < <(get_compose_relations "$service") || return 1 -} - - -_graph_node_service_label() { - local service="$1" - echo "service_$service" -} - - -_graph_node_service_content() { - local service="$1" - charm=$(get_service_charm "$service") || return 1 - -cat < - - - $service - - -$(if [ "$charm" != "$service" ]; then -cat < - charm: $charm - -EOF2 -fi) - -EOF -} - - -cla_contains () { - local e - for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done - return 1 -} - - -filter_docker_compose_help_message() { - cat - | - sed -r "s/docker-compose run/${DARKWHITE}compose${NORMAL} ${DARKCYAN}$action${NORMAL}/g; - s/docker-compose.yml/compose.yml/g; - s/SERVICES?/${DARKYELLOW}\0${NORMAL}/g; - s/^(\s+)\\$/\1${WHITE}\$${NORMAL}/g; - s/^(\s+)run/\1${DARKCYAN}$action${NORMAL}/g; - s/docker-compose/${DARKWHITE}compose${NORMAL}/g" -} - - -graph() { - local services=("$@") - declare -A entries - cat <&2 - exit 1 - fi - fi - ;; - *) - # echo "LOOP $1 : pos_arg: $pos_arg_ct // ${pos_args[$pos_arg_ct]}" - if [[ "${pos_args[$pos_arg_ct]}" == "[SERVICE...]" ]]; then - services_args+=("$arg") - elif [[ "${pos_args[$pos_arg_ct]}" == "SERVICE" ]]; then - services_args=("$arg") || exit 1 - stage="remainder" - else - action_posargs+=("$arg") - ((pos_arg_ct++)) - fi - ;; - esac - ;; - "remainder") - remainder_args+=("$arg") - while read-0 arg; do - remainder_args+=("$arg") - done - break 3 - ;; - esac - shift -done < <(cla.normalize "$@") - - -export compose_contents -[ "${services_args[*]}" ] && debug " ${DARKWHITE}Services:$NORMAL ${DARKYELLOW}${services_args[*]}$NORMAL" -[ "${compose_opts[*]}" ] && debug " ${DARKWHITE}Main docker-compose opts:$NORMAL ${compose_opts[*]}" -[ "${action_posargs[*]}" ] && debug " ${DARKWHITE}Main docker-compose pos args:$NORMAL ${action_posargs[*]}" -[ "${action_opts[*]}" ] && debug " ${DARKWHITE}Action $DARKCYAN$action$NORMAL with opts:$NORMAL ${action_opts[*]}" -[ "${remainder_args[*]}" ] && debug " ${DARKWHITE}Remainder args:$NORMAL ${remainder_args[*]}" - - - -aexport remainder_args - - -## -## Actual code -## - - -COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1 -COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1 -export COMPOSE_YML_FILE COMPOSE_YML_CONTENT - - -charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct." - -## -## Get services in command line. -## - -if [ -z "$is_docker_compose_action" -a "$action" ]; then - action_service=${remainder_args[0]} - remainder_args=("${remainder_args[@]:1}") - if is_service_action=$(has_service_action "$action_service" "$action"); then - { - read-0 action_type - case "$action_type" in - "relation") - read-0 _ target_service _target_charm relation_name - debug "Found action $DARKYELLOW${action_service}$NORMAL/$DARKBLUE$relation_name$NORMAL/$DARKCYAN$action$NORMAL (in $DARKYELLOW$target_service$NORMAL)" - ;; - "direct") - debug "Found action $DARKYELLOW${action_service}$NORMAL.$DARKCYAN$action$NORMAL" - ;; - esac - } < <(has_service_action "$action_service" "$action") - services_args=("$action_service") - else - die "Unknown action '${DARKCYAN}$action$NORMAL': It doesn't match any docker-compose commands nor inner charm actions." - fi -else - case "$action" in - ps) - if [ "${#services_args[@]}" == 0 ]; then - services_args=($(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys 2>/dev/null)) || true - fi - ;; - up) - if [ "${#services_args[@]}" == 0 ]; then - while read-0 service; do - type="$(get_service_type "$service")" || exit 1 - if [ "$type" != "run-once" ]; then - services_args+=("$service") - fi - done < <(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys-0 2>/dev/null) - fi - ;; - config) - services_args=("${action_posargs[@]}") - ;; - esac - if [ "$is_docker_compose_action" -a "${#services_args[@]}" -gt 0 ]; then - services=($(get_master_services "${services_args[@]}")) || exit 1 - action_posargs+=("${services[@]}") - fi -fi - - -get_docker_compose "${services_args[@]}" >/dev/null || { ## precalculate variable \$_current_docker_compose - err "Fails to compile base 'docker-compose.yml'" - exit 1 -} - -## -## Pre-action -## - - -full_init= -case "$action" in - up|run) - full_init=true - post_hook=true - ;; - ""|down|restart|logs|config|ps) - full_init= - ;; - *) - if [ "$is_service_action" ]; then - full_init=true - fi - ;; -esac - - -if [ "$full_init" ]; then - ## init in order - if [ -z "$no_init" ]; then - Section setup host resources - setup_host_resources "${services_args[@]}" || exit 1 - Section initialisation - run_service_hook init "${services_args[@]}" || exit 1 - fi - - ## Get relations - if [ -z "$no_relations" ]; then - if [ "${#rebuild_relations_to_service[@]}" != 0 ]; then - rebuild_relations_to_service=( - $(get_master_services "${rebuild_relations_to_service[@]}")) - project=$(get_default_project_name) || return 1 - for service in "${rebuild_relations_to_service[@]}"; do - for dir in "$VARDIR/relations/$project/"*"-${service}/"*; do - [ -d "$dir" ] && { - debug rm -rf "$dir" - rm -rf "$dir" - } - done - done - fi - run_service_relations "${services_args[@]}" || exit 1 - fi - - run_service_hook pre_deploy "${services_args[@]}" || exit 1 - -fi - -if [ "$action" == "run" ]; then - charm=$(get_service_charm "${services_args[0]}") || exit 1 - metadata=$(charm.metadata "$charm") || exit 1 - type="$(printf "%s" "$metadata" | shyaml get-value type 2>/dev/null)" || true - if [ "$type" == "run-once" ]; then - run_service_hook dc-pre-run "${services_args[@]}" || exit 1 - fi -fi - - -export SERVICE_PACK="${services_args[*]}" - -## -## Docker-compose -## - -case "$action" in - up|start|stop|build|run) - ## force daemon mode for up - if [[ "$action" == "up" ]] && ! array_member action_opts -d; then - action_opts+=("-d") - fi - - launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}" - ;; - logs) - if ! array_member action_opts --tail; then ## force daemon mode for up - action_opts+=("--tail" "10") - fi - launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}" - ;; - "") - launch_docker_compose "${compose_opts[@]}" - ;; - graph) - graph $SERVICE_PACK - ;; - config) - ## removing the services - services=($(get_master_services "${action_posargs[@]}")) || exit 1 - ## forcing docker-compose config to output the config file to stdout and not stderr - out=$(launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" 2>&1) || { - echo "$out" - exit 1 - } - echo "$out" - warn "Runtime configuration modification (from relations) are not included here." - ;; - down) - if ! array_member action_opts --remove-orphans; then ## force daemon mode for up - debug "Adding a default argument of '--remove-orphans'" - action_opts+=("--remove-orphans") - fi - launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" - ;; - *) - if [ "$is_service_action" ]; then - run_service_action "$SERVICE_PACK" "$action" "${remainder_args[@]}" - else - launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}" - fi - ;; -esac || exit 1 -if [ "$post_hook" -a "${#services_args[@]}" != 0 ]; then - run_service_hook post_deploy "${services_args[@]}" || exit 1 -fi +run "$@" \ No newline at end of file diff --git a/bin/compose-core b/bin/compose-core new file mode 100755 index 0000000..008dab5 --- /dev/null +++ b/bin/compose-core @@ -0,0 +1,3088 @@ +#!/bin/bash + +## +## TODO: +## - subordinate container should really be able to modify base image of their master +## - this could be done through docker-update +## - I'm not happy with the current build using 'build/' directory, this should be +## changed to: +## - always have a base image (specified in metadata), and always have hooks/install +## executed and merge in image (like docker-build-charm). +## - container base image is ALWAYS the image of the master container... this brings +## questions about a double way to express inheritage (through relations as it is +## implemented now, or through this base-image ?) +## - the name of the scripts for relation (aka relation_name-relation-joined) is bad as +## reading the name in a hooks/ dir, there are no way to know if we are the target or +## the base of the relation. +## - we could leverage a 'relations/' dir on the root of the charm, with both: +## 'relations/provide/relation_name' and 'relations/receive/relation_name' +## - a very bad point with the actual naming is that we can't have a providing AND +## receiving a relation with same name. +## - The cache system should keep md5 of docker-compose and other things between runs +## - The cache system should use underlying function that have only arguments inputs. +## This will allow to cache completely without issues function in time. +## - would probably need instrospection in charm custom action to know if these need +## init or relations to be set up. +## - Be clear about when the SERVICE name is used and the CHARM name is used. +## - in case of service contained in another container +## - in normal case +## - in docker-compose, can't use charm name: if we want 2 instances of the same charm +## we are stuck. What will be unique is the name of the service. +## - some relations are configured in compose.yml but should not trigger the loading +## of necessary component (for instance, apache --> log-rotate), if log-rotate is +## not there, this link should considered optional. +## - Could probably allow an unexistent charm to be populated with only "docker-image:" +## of the same name. Although this should trigger a visible warning. + + +#:- +[ -e /etc/shlib ] && . /etc/shlib || { + echo "Unsatisfied dependency. Please install 'kal-shlib-core'." + exit 1 + } +#:- + +include common +include pretty +include parse +include charm +include array +include cla + +depends shyaml docker + +exname="compose" +version=0.1 +usage="$exname [COMPOSE_OPTS] [ACTION [ACTION_OPTS]]" +help="\ + +$WHITE$exname$NORMAL jobs is to run various shell scripts to build +a running orchestrated and configured docker containers. These shell +scripts will have the opportunity to build a 'docker-compose.yml'. + +Once init script and relations scripts are executed, $WHITE$exname$NORMAL +delegate the launching to ${WHITE}docker-compose${NORMAL} by providing it +the final 'docker-compose.yml'. + +$WHITE$exname$NORMAL also leverage charms to offer some additional custom +actions per charm, which are simply other scripts that can be +run without launching ${WHITE}docker-compose${NORMAL}. + +In compose message, color coding is enforced as such: + - ${DARKCYAN}action$NORMAL, + - ${DARKBLUE}relation$NORMAL, + - ${DARKPINK}charm${NORMAL}, + - ${DARKYELLOW}service${NORMAL}, + - ${WHITE}option-name${NORMAL}/${WHITE}command-name${NORMAL}/${WHITE}Section-Title${NORMAL} + +$WHITE$exname$NORMAL reads '/etc/compose.conf' for global variables, and +'/etc/compose.local.conf' for local host adjustements. + +" + + +## XXXvlab: this doesn't seem to work when 'compose' is called in +## a hook of a charm. +#[[ "${BASH_SOURCE[0]}" == "" ]] && SOURCED=true +$(return >/dev/null 2>&1) && SOURCED=true + + +if [ "$UID" == 0 ]; then + CACHEDIR=${CACHEDIR:-/var/cache/compose} + VARDIR=${VARDIR:-/var/lib/compose} +else + [ "$XDG_CONFIG_HOME" ] && CACHEDIR=${CACHEDIR:-$XDG_CONFIG_HOME/compose} + [ "$XDG_DATA_HOME" ] && VARDIR=${VARDIR:-$XDG_DATA_HOME/compose} + CACHEDIR=${CACHEDIR:-$HOME/.cache/compose} + VARDIR=${VARDIR:-$HOME/.local/share/compose} +fi +export VARDIR CACHEDIR + + +md5_compat() { md5sum | cut -c -32; } +quick_cat_file() { quick_cat_stdin < "$1"; } +quick_cat_stdin() { local IFS=''; while read -r line; do echo "$line"; done ; } +export -f quick_cat_file quick_cat_stdin md5_compat + + +clean_cache() { + local i=0 + for f in $(ls -t "$CACHEDIR/"*.cache.* 2>/dev/null | tail -n +500); do + ((i++)) + rm -f "$f" + done + if (( i > 0 )); then + debug "${WHITE}Cleaned cache:${NORMAL} Removed $((i)) elements (current cache size is $(du -sh "$CACHEDIR" | cut -f 1))" + fi +} + + +usage="$exname SERVICE"' + +Deploy and manage a swarm of containers to provide services based on +a ``compose.yml`` definition and charms from a ``charm-store``. +' + +export DEFAULT_COMPOSE_FILE + +## +## Merge YAML files +## + +export _merge_yaml_common_code=" + +import sys +import yaml + + +try: + # included in standard lib from Python 2.7 + from collections import OrderedDict +except ImportError: + # try importing the backported drop-in replacement + # it's available on PyPI + from ordereddict import OrderedDict + + +## Ensure that there are no collision with legacy OrderedDict +## that could be used for omap for instance. +class MyOrderedDict(OrderedDict): + pass + +yaml.add_representer( + MyOrderedDict, + lambda cls, data: cls.represent_dict(data.items())) + +yaml.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + lambda cls, node: MyOrderedDict(cls.construct_pairs(node))) + + +def fc(filename): + with open(filename) as f: + return f.read() + +def merge(*args): + # sys.stderr.write('%r\n' % (args, )) + args = [arg for arg in args if arg is not None] + if len(args) == 0: + return None + if len(args) == 1: + return args[0] + if all(isinstance(arg, (int, basestring, bool)) for arg in args): + return args[-1] + elif all(isinstance(arg, list) for arg in args): + res = [] + for arg in args: + for elt in arg: + if elt in res: + res.remove(elt) + res.append(elt) + return res + elif all(isinstance(arg, dict) for arg in args): + keys = set() + for arg in args: + keys |= set(arg.keys()) + dct = {} + for key in keys: + sub_args = [] + for arg in args: + if key in arg: + sub_args.append(arg) + try: + dct[key] = merge(*(a[key] for a in sub_args)) + except NotImplementedError as e: + raise NotImplementedError( + e.args[0], + '%s.%s' % (key, e.args[1]) if e.args[1] else key, + e.args[2]) + if dct[key] is None: + del dct[key] + return dct + else: + raise NotImplementedError( + 'Unsupported types: %s' + % (', '.join(list(set(arg.__class__.__name__ for arg in args)))), '', args) + return None + +def merge_cli(*args): + try: + c = merge(*args) + except NotImplementedError as e: + sys.stderr.write('Merging Failed: %s.\n%s\n' + ' Values are:\n %s\n' + % (e.args[0], + ' Conflicting key is %r.' % e.args[1] if e.args[1] else + ' Conflict at base of structure.', + '\\n '.join('v%d: %r' % (i, a) + for i, a in enumerate(e.args[2])))) + exit(1) + if c is not None: + print '%s' % yaml.dump(c, default_flow_style=False) + +" + + +merge_yaml() { + + if ! [ -r "$state_tmpdir/merge_yaml.py" ]; then + cat < "$state_tmpdir/merge_yaml.py" + +$_merge_yaml_common_code + +merge_cli(*(yaml.load(fc(f)) for f in sys.argv[1:])) +EOF + fi + + python "$state_tmpdir/merge_yaml.py" "$@" +} +export -f merge_yaml + + +merge_yaml_str() { + local entries="$@" + + if ! [ -r "$state_tmpdir/merge_yaml_str.py" ]; then + cat < "$state_tmpdir/merge_yaml_str.py" || return 1 + +$_merge_yaml_common_code + +merge_cli(*(yaml.load(f) for f in sys.argv[1:])) +EOF + fi + + python "$state_tmpdir/merge_yaml_str.py" "$@" +} +export -f merge_yaml_str + + +yaml_key_val_str() { + local entries="$@" + + if ! [ -r "$state_tmpdir/yaml_key_val_str.py" ]; then + cat < "$state_tmpdir/yaml_key_val_str.py" + +$_merge_yaml_common_code + +print '%s' % yaml.dump({ + yaml.load(sys.argv[1]): + yaml.load(sys.argv[2])}, default_flow_style=False) + +EOF + fi + + python "$state_tmpdir/yaml_key_val_str.py" "$@" +} +export -f yaml_key_val_str + + +## +## Docker +## + +docker_has_image() { + local image="$1" + images=$(docker images -q "$image" 2>/dev/null) || { + err "docker images call has failed unexpectedly." + return 1 + } + [ "$images" ] +} +export -f docker_has_image + + +cmd_on_base_image() { + local service="$1" base_image + shift + base_image=$(service_base_docker_image "$service") || return 1 + docker run -i --entrypoint /bin/bash "$base_image" -c "$*" +} +export -f cmd_on_base_image + + +cached_cmd_on_base_image() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" + shift + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + quick_cat_stdin < "$cache_file" + return 0 + fi + result=$(cmd_on_base_image "$service" "$@") || return 1 + echo "$result" | tee "$cache_file" +} +export -f cached_cmd_on_base_image + + +image_exposed_ports_0() { + local image="$1" + docker inspect --format='{{range $p, $conf := .Config.ExposedPorts}}{{$p}}{{"\x00"}}{{end}}' "$image" +} +export -f image_exposed_ports_0 + +## +## Generic +## + + +fn.exists() { + declare -F "$1" >/dev/null +} + + +str_pattern_matches() { + local str="$1" + shift + for pattern in "$@"; do + eval "[[ \"$str\" == $pattern ]]" && return 0 + done + return 1 +} + + +str_matches() { + local str="$1" + shift + for pattern in "$@"; do + [[ "$str" == "$pattern" ]] && return 0 + done + return 1 +} + +gen_password() { + local l=( {a..z} {A..Z} {0..9} ) nl="${#l[@]}" size=${1:-16} + while ((size--)); do + echo -n "${l[$((RANDOM * nl / 32768))]}" + done + echo +} +export -f gen_password + + +file_put() { + local TARGET="$1" + mkdir -p "$(dirname "$TARGET")" && + cat - > "$TARGET" +} +export -f file_put + + +file_put_0() { + local TARGET="$1" + mkdir -p "$(dirname "$TARGET")" && + cat > "$TARGET" +} +export -f file_put_0 + + +fetch_file() { + local src="$1" + + case "$src" in + *"://"*) + err "Unsupported target scheme." + return 1 + ;; + *) + ## Try direct + if ! [ -r "$src" ]; then + err "File '$src' not found/readable." + return 1 + fi + cat "$src" || return 1 + ;; + esac +} +export -f fetch_file + + +## receives stdin content to decompress on stdout +## stdout content should be tar format. +uncompress_file() { + local filename="$1" + + ## Warning, the content of the file is already as stdin, the filename + ## is there to hint for correct decompression. + case "$filename" in + *".gz") + gunzip + ;; + *".bz2") + bunzip2 + ;; + *) + cat + ;; + esac + +} +export -f uncompress_file + + +get_file() { + local src="$1" + + fetch_file "$src" | uncompress_file "$src" +} +export -f get_file + + +## +## Common database lib +## + +_clean_docker() { + local _DB_NAME="$1" container_id="$2" + ( + set +e + debug "Removing container $_DB_NAME" + docker stop "$container_id" + docker rm "$_DB_NAME" + docker network rm "${_DB_NAME}" + rm -vf "/tmp/${_DB_NAME}.state" + ) +} +export -f _clean_docker + + +get_service_base_image_dir_uid_gid() { + local service="$1" dir="$2" uid_gid + uid_gid=$(cached_cmd_on_base_image "$service" "stat -c '%u %g' '$dir'") || { + debug "Failed to query '$dir' uid in ${DARKYELLOW}$service${NORMAL} base image." + return 1 + } + info "uid and gid from ${DARKYELLOW}$service${NORMAL}:$dir is '$uid_gid'" + echo "$uid_gid" +} +export -f get_service_base_image_dir_uid_gid + + +get_service_type() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" + if [ -z "$service" ]; then + print_syntax_error "$FUNCNAME: Please specify a service as first argument." + return 1 + fi + + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + charm=$(get_service_charm "$service") || return 1 + metadata=$(charm.metadata "$charm") || return 1 + printf "%s" "$metadata" | shyaml get-value type service 2>/dev/null | + tee "$cache_file" +} + + +are_files_locked_in_dir() { + local dir="$1" device hdev ldev + device=$(stat -c %d "$dir") || { + err "Can't stat %d." + return 1 + } + device=$(printf "%04x" $device) + hdev=${device:0:2} + ldev=${device:2:2} + inodes=$(find "$dir" -printf ':%i:\n') + + found= + while read -r inode; do + debug "try inode:$inode" + if [[ "$inodes" == *":$inode:"* ]]; then + found=1 + break + fi + done < <(cat /proc/locks | grep " $hdev:$ldev:" | sed -r "s/^.*$hdev:$ldev:([0-9]+).*$/\1/g") + + [ "$found" ] +} +export -f are_files_locked_in_dir + + +export _PID="$$" +ensure_db_docker_running () { + local _STATE_FILE + + _DB_NAME="db_${DB_NAME}_${_PID}" + _STATE_FILE=/tmp/${_DB_NAME}.state + if [ -e "$_STATE_FILE" ]; then + IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$(cat "$_STATE_FILE")" + + debug "Re-using previous docker/connection '$DOCKER_IP'." + _set_db_params "$DOCKER_IP" "$DOCKER_NETWORK" + + return 0 + fi + + if [ -e "/tmp/${_DB_NAME}.working" ]; then + ## avoid recursive calls. + if [ -z "$DOCKER_IP" ]; then + err "Currently figuring up DOCKER_IP, please set it yourself before this call if needed." + return 1 + else + debug "ignoring recursive call of 'ensure_db_docker_running'." + fi + return 0 + fi + + touch "/tmp/${_DB_NAME}.working" + + docker rm "$_DB_NAME" 2>/dev/null || true + + host_db_working_dir="$DATASTORE/${SERVICE_NAME}$DB_DATADIR" + + if is_db_locked; then + info "Some process is using '$host_db_working_dir'. Trying to find a docker that would do this..." + found= + for docker_id in $(docker ps -q); do + has_volume_mounted=$( + docker inspect \ + --format "{{range .Mounts}}{{if eq .Destination \"$DB_DATADIR\"}}{{.Source}}{{end}}{{end}}" \ + "$docker_id") + if [ "$has_volume_mounted" == "$host_db_working_dir" ]; then + found="$docker_id" + break + fi + done + if [ -z "$found" ]; then + err "Please shutdown any other docker using this directory." + return 1 + fi + export container_id="$found" + info "Found docker $docker_id is already running." + else + verb "Database is not locked." + if ! docker_has_image "$DOCKER_BASE_IMAGE"; then + docker pull "$DOCKER_BASE_IMAGE" + fi + + docker_opts= + if [ -f "$DB_PASSFILE" ]; then + verb "Found and using '$DB_PASSFILE'." + docker_opts="$db_docker_opts -v $SERVER_ROOT_PREFIX$DB_PASSFILE:$DB_PASSFILE" + fi + debug docker network create "$_DB_NAME" + if ! network_id=$(docker network create "$_DB_NAME"); then + err "'docker network create' failed !" + _clean_docker "$_DB_NAME" "$container_id" + rm "/tmp/${_DB_NAME}.working" + return 1 + fi + debug docker run -d \ + --name "$_DB_NAME" \ + $docker_opts \ + --network "$_DB_NAME" \ + -v "$host_db_working_dir:$DB_DATADIR" \ + "$DOCKER_BASE_IMAGE" + if ! container_id=$( + docker run -d \ + --name "$_DB_NAME" \ + $docker_opts \ + --network "$_DB_NAME" \ + -v "$host_db_working_dir:$DB_DATADIR" \ + "$DOCKER_BASE_IMAGE" + ); then + err "'docker run' failed !" + _clean_docker "$_DB_NAME" "$container_id" + rm "/tmp/${_DB_NAME}.working" + return 1 + fi + trap_add EXIT,ERR "_clean_docker \"$_DB_NAME\" \"$container_id\"" + fi + + if docker_ip=$(wait_for_docker_ip "$container_id"); then + IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip" + echo "$docker_ip" > "$_STATE_FILE" + debug "written '$_STATE_FILE'" + rm "/tmp/${_DB_NAME}.working" + _set_db_params "$DOCKER_IP" "$DOCKER_NETWORK" + return 0 + else + errlvl="$?" + err "Db not found (errlvl: $errlvl). Tail of docker logs follows:" + docker logs --tail=5 "$container_id" 2>&1 | prefix " | " >&2 + rm "/tmp/${_DB_NAME}.working" + return "$errlvl" + fi +} +export -f ensure_db_docker_running + + +## Require to set $db_docker_opts if needed, and $DB_PASSFILE +## +_dcmd() { + local docker_opts command="$1" + shift + + debug "Db> $command $@" + + if [ -f "$DB_PASSFILE" ]; then + verb "Found and using '$DB_PASSFILE'." + db_docker_opts="$db_docker_opts -v $SERVER_ROOT_PREFIX$DB_PASSFILE:$DB_PASSFILE" + fi + + ## XXXX was here: actualy, we need only connection between this version and the client version + debug docker run -i --rm \ + $db_docker_opts \ + --entrypoint "$command" "$DOCKER_BASE_IMAGE" $db_cmd_opts "$@" + docker run -i --rm \ + $db_docker_opts \ + --entrypoint "$command" "$DOCKER_BASE_IMAGE" $db_cmd_opts "$@" +} +export -f _dcmd + + +## Executes code through db +dcmd() { + local fun + [ "$DB_NAME" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_NAME." + [ "$DB_DATADIR" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_DATADIR." + # [ "$DB_PASSFILE" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_PASSFILE." + [ "$_PID" ] || print_syntax_error "$FUNCNAME: You must provide \$_PID." + for fun in is_db_locked _set_db_params ddb; do + [ "$(type -t "$fun")" == "function" ] || + print_syntax_error "$FUNCNAME: You must provide function '$fun'." + done + ensure_db_docker_running /dev/null 2>&1; then + echo "default:$(docker inspect --format='{{ .NetworkSettings.IPAdress }}' "$name" 2>/dev/null)" + else + format='{{range $name, $conf := .NetworkSettings.Networks}}{{$name}}{{"\x00"}}{{$conf.IPAddress}}{{"\x00"}}{{end}}' + while read-0 network_id ip; do + printf "%s:%s\n" "$network_id" "$ip" + done < <(docker inspect --format="$format" "$name") + fi +} +export -f get_docker_ips + + +get_docker_ip() { + local name="$1" + get_docker_ips "$name" +} +export -f get_docker_ip + + +wait_docker_ip() { + local name="$1" timeout="${2:-15}" timeout_count=0 docker_ip= + start=$SECONDS + while [ -z "$docker_ip" ]; do + sleep 0.5 + docker_ip=$(get_docker_ip "$name") && break + elapsed=$((SECONDS - start)) + if ((elapsed > timeout)); then + err "${RED}timeout error${NORMAL}(${timeout}s):" \ + "Could not find '$name' docker container's IP." + return 1 + fi + [ "$elapsed" == "$old_elapsed" ] || + verb "Waiting for docker $name... ($elapsed/$timeout)" + old_elapsed="$elapsed" + done + verb "Found docker $name network and IP: $docker_ip" + echo "$docker_ip" +} +export -f wait_docker_ip + +DEFAULT_BASH_IMAGE=${DEFAULT_BASH_IMAGE:-docker.0k.io/bash} +wait_for_tcp_port() { + local network=$1 host_port=$2 timeout=20 + verb "Trying to connect to $host_port" + docker run --rm -i --network "$network" "$DEFAULT_BASH_IMAGE" </dev/null 2>&1 && break + sleep 0.2 + if [ "\$((SECONDS - start))" -gt "$timeout" ]; then + exit 1 + fi + done + exit 0 +EOF + if [ "$?" == 1 ]; then + err "${RED}timeout error${NORMAL}(${timeout}s):"\ + "Could not connect to $host_port." + return 1 + fi + return 0 +} +export -f wait_for_tcp_port + + +## Warning: requires a ``ddb`` matching current database to be checked +wait_for_docker_ip() { + local name=$1 DOCKER_IP= DOCKER_NETWORK= docker_ips= docker_ip= + docker_ip=$(wait_docker_ip "$name" 5) || return 1 + IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip" + if ! str_is_ipv4 "$DOCKER_IP"; then + err "internal 'wait_docker_ip' did not return a valid IP. Returned IP is '$DOCKER_IP'." + return 1 + fi + _set_db_params "$DOCKER_IP" "$DOCKER_NETWORK" + while read-0 port; do + IFS="/" read port type <<<"$port" + [ "$type" == "tcp" ] || continue + wait_for_tcp_port "$DOCKER_NETWORK" "$DOCKER_IP:${port}" || return 17 + verb "Port $DOCKER_IP:${port} checked open." + done < <(image_exposed_ports_0 "$container_id") + + ## Checking direct connection + if ! echo "SELECT 1;" | ddb >/dev/null; then + err "${RED}db connection error${NORMAL}:"\ + "Could not connect to db on $DOCKER_IP" \ + "container's IP. (IP up, TCP ports is(are) open)" + return 18 + fi + echo "${DOCKER_NETWORK}:${DOCKER_IP}" + return 0 +} +export -f wait_for_docker_ip + + +docker_add_host_declaration() { + local src_docker=$1 domain=$2 dst_docker=$3 dst_docker_ip= dst_docker_network + dst_docker_ip=$(wait_docker_ip "$dst_docker") || exit 1 + + IFS=: read dst_docker_ip dst_docker_network <<<"$dst_docker_ip" + + docker exec -i "$src_docker" bash < /dev/null 2>&1; then + sed -ri "s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+$domain\$/$dst_docker_ip $domain/g" /etc/hosts + else + echo "$dst_docker_ip $domain" >> /etc/hosts + fi +EOF +} +export -f docker_add_host_declaration + + +get_running_containers_for_service() { + local service="$1" + + docker ps --filter label="compose.service=$service" --format="{{.ID}}" +} +export -f get_running_containers_for_service + + +## +## Internal Process +## + + +get_docker_compose_links() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + deps master_service master_target_service _relation_name \ + target_service _relation_config tech_dep + if [ -z "$service" ]; then + print_syntax_error "$FUNCNAME: Please specify a service as first argument." + return 1 + fi + + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + master_service=$(get_top_master_service_for_service "$service") || return 1 + ## XXXvlab: yuck, this make the assumption that next function is cached, + ## and leverage the fact that the result is stored in a file. All this only + ## to catch a failure of ``get_compose_relations``, that would go out silently. + get_compose_relations "$service" >/dev/null || return 1 ## fetch cache and fail if necessary + deps=() + while read-0 _relation_name target_service _relation_config tech_dep; do + master_target_service="$(get_top_master_service_for_service "$target_service")" || return 1 + [ "$master_service" == "$master_target_service" ] && continue + type="$(get_service_type "$target_service")" || return 1 + [ "$type" == "run-once" ] && continue + if [ "$tech_dep" == "reversed" ]; then + deps+=("$(echo -en "$master_target_service:\n links:\n - $master_service")") + elif [ "$tech_dep" == "True" ]; then + deps+=("$(echo -en "$master_service:\n links:\n - $master_target_service")") + fi + ## XXXvlab: an attempt to add depends_on, but this doesn't work well actually + ## as there's a circular dependency issue. We don't really want the full feature + ## of depends_on, but just to add it as targets when doing an 'up' + # deps+=("$(echo -en "$master_service:\n depends_on:\n - $master_target_service")") + done < <(get_compose_relations "$service") + merge_yaml_str "${deps[@]}" | tee "$cache_file" || return 1 + if [ "${PIPESTATUS[0]}" != 0 ]; then + rm "$cache_file" + err "Failed to merge YAML from all ${WHITE}links${NORMAL} dependencies." + return 1 + fi +} + + +_get_docker_compose_opts() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + compose_def master_service docker_compose_opts + if [ -z "$service" ]; then + print_syntax_error "$FUNCNAME: Please specify a service as first argument." + return 1 + fi + + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + compose_def="$(get_compose_service_def "$service")" || return 1 + master_service="$(get_top_master_service_for_service "$service")" + docker_compose_opts=$(echo "$compose_def" | shyaml get-value "docker-compose" 2>/dev/null) + + if [ "$docker_compose_opts" ]; then + yaml_key_val_str "$master_service" "$docker_compose_opts" + fi | tee "$cache_file" + if [ "${PIPESTATUS[0]}" != 0 ]; then + rm "$cache_file" + return 1 + fi +} + + +## +## By Reading the metadata.yml, we create a docker-compose.yml mixin. +## Some metadata.yml (of subordinates) will indeed modify other +## services than themselves. +_get_docker_compose_service_mixin() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + links_yaml base_mixin links_yaml docker_compose_options \ + charm charm_part + if [ -z "$service" ]; then + print_syntax_error "$FUNCNAME: Please specify a service as first argument." + return 1 + fi + + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + master_service=$(get_top_master_service_for_service "$service") || { + err "Failed to get top master service for service $DARKYELLOW$service$NORMAL" + return 1 + } + + ## The compose part + + base_mixin="$master_service: + labels: + - compose.service=$service + - compose.master-service=${master_service} + - compose.project=$(get_default_project_name)" + links_yaml=$(get_docker_compose_links "$service") || return 1 + docker_compose_options=$(_get_docker_compose_opts "$service") || return 1 + + ## the charm part + + charm_part=$(get_docker_compose_mixin_from_metadata "$service") || return 1 + + ## Merge results + if [ "$charm_part" ]; then + charm_yaml="$(yaml_key_val_str "$master_service" "$charm_part")" || return 1 + merge_yaml_str "$base_mixin" "$links_yaml" "$charm_yaml" "$docker_compose_options" || return 1 + else + merge_yaml_str "$base_mixin" "$links_yaml" "$docker_compose_options" || return 1 + fi | tee "$cache_file" + if [ "${PIPESTATUS[0]}" != 0 ]; then + err "Failed to constitute the base YAML for service '${DARKYELLOW}$service${NORMAL}'" + rm "$cache_file" + return 1 + fi + +} +export -f _get_docker_compose_service_mixin + + +## +## Get full `docker-compose.yml` format for all listed services (and +## their deps) +## + +## @export +## @cache: !system !nofail +stdout +get_docker_compose () { + local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + entries services service start docker_compose_services + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + ## + ## Adding sub services configurations + ## + + declare -A entries + start_compilation=$SECONDS + debug "Compiling 'docker-compose.yml' base for ${DARKYELLOW}$*$NORMAL..." + for target_service in "$@"; do + start=$SECONDS + services=$(get_ordered_service_dependencies "$target_service") || { + err "Failed to get dependencies for $DARKYELLOW$target_service$NORMAL" + return 1 + } + debug " $DARKYELLOW$target_service$NORMAL deps:$DARKYELLOW" $services "$NORMAL$GRAY(in $((SECONDS - start))s)$NORMAL" + for service in $services; do + + if [ "${entries[$service]}" ]; then + ## Prevent double inclusion of same service if this + ## service is deps of two or more of your + ## requirements. + continue + fi + + ## mark the service as "loaded" as well as it's containers + ## if this is a subordinate service + start_service=$SECONDS + entries[$service]=$(_get_docker_compose_service_mixin "$service") || { + err "Failed to get service mixin for $DARKYELLOW$service$NORMAL" + return 1 + } + debug " Applied $DARKYELLOW$service$NORMAL charm metadata mixins $GRAY(in $((SECONDS - start_service))s)$NORMAL" + done + debug " ..finished all mixins for $DARKYELLOW$target_service$NORMAL $GRAY(in $((SECONDS - start))s)$NORMAL" + done + + docker_compose_services=$(merge_yaml_str "${entries[@]}") || { + err "Failed to merge YAML services entries together." + return 1 + } + + base_v2="version: '2.0'" + merge_yaml_str "$(yaml_key_val_str "services" "$docker_compose_services")" \ + "$base_v2" > "$cache_file" || return 1 + + export _current_docker_compose="$(cat "$cache_file")" + echo "$_current_docker_compose" + debug " ..compilation of base 'docker-compose.yml' done $GRAY(in $((SECONDS - start_compilation))s)$NORMAL" || true + # debug " ** ${WHITE}docker-compose.yml${NORMAL}:" + # debug "$_current_docker_compose" +} +export -f get_docker_compose + + +_get_compose_service_def_cached () { + local service="$1" docker_compose="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" + if [ -e "$cache_file" ]; then + #debug "$FUNCNAME: STATIC cache hit" + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + value=$(echo "$docker_compose" | shyaml get-value "$service" 2>/dev/null) + [ "$value" == None ] && value="" + if ! echo "$value" | shyaml get-value "charm" >/dev/null 2>&1; then + if charm.exists "$service"; then + value=$(merge_yaml <(echo "charm: $service") <(echo "$value")) || { + err "Can't merge YAML infered 'charm: $service' with base ${DARKYELLOW}$service${NORMAL} YAML definition." + return 1 + } + else + err "No ${WHITE}charm${NORMAL} value for service $DARKYELLOW$service$NORMAL" \ + "in compose, nor same name charm found." + return 1 + fi + fi + echo "$value" | tee "$cache_file" || return 1 + # if [ "${PIPESTATUS[0]}" != 0 ]; then + # rm "$cache_file" + # return 1 + # fi + return 0 + # if [ "${PIPESTATUS[0]}" != 0 -o \! -s "$cache_file" ]; then + # rm "$cache_file" + # err "PAS OK $service: $value" + # return 1 + # fi +} +export -f _get_compose_service_def_cached + + +## XXXvlab: a lot to be done to cache the results +get_compose_service_def () { + local service="$1" docker_compose cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + result + if [ -e "$cache_file" ]; then + #debug "$FUNCNAME: SESSION cache hit" + cat "$cache_file" || return 1 + return 0 + fi + + [ -z "$service" ] && print_syntax_error "Missing service as first argument." + docker_compose=$(get_compose_yml_content) || return 1 + result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1 + echo "$result" | tee "$cache_file" || return 1 +} +export -f get_compose_service_def + + +_get_service_charm_cached () { + local service="$1" service_def="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit $1" + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + charm=$(echo "$service_def" | shyaml get-value charm 2>/dev/null) + if [ -z "$charm" ]; then + err "Missing ${WHITE}charm${NORMAL} value in service $DARKYELLOW$service$NORMAL definition." + return 1 + fi + echo "$charm" | tee "$cache_file" || return 1 +} +export -f _get_service_charm_cached + + +get_service_charm () { + local service="$1" + if [ -z "$service" ]; then + echo ${FUNCNAME[@]} >&2 + print_syntax_error "$FUNCNAME: Please specify a service as first argument." + return 1 + fi + service_def=$(get_compose_service_def "$service") || return 1 + _get_service_charm_cached "$service" "$service_def" +} +export -f get_service_charm + + +## built above the docker-compose abstraction, so it relies on the +## full docker-compose.yml to be already built. +get_service_def () { + local service="$1" def + if [ -z "$_current_docker_compose" ]; then + print_syntax_error "$FUNCNAME is meant to be called after"\ + "\$_current_docker_compose has been calculated." + fi + + def=$(echo "$_current_docker_compose" | shyaml get-value "services.$service" 2>/dev/null) + if [ -z "$def" ]; then + err "No definition for service $DARKYELLOW$service$NORMAL in compiled 'docker-compose.yml'." + return 1 + fi + echo "$def" +} +export -f get_service_def + + +## Return the base docker image name of a service +service_base_docker_image() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + master_service service_def service_image service_build service_dockerfile + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + master_service="$(get_top_master_service_for_service "$service")" || { + err "Could not compute master service for service $DARKYELLOW$service$NORMAL." + return 1 + } + service_def="$(get_service_def "$master_service")" || { + err "Could not get docker-compose service definition for $DARKYELLOW$master_service$NORMAL." + return 1 + } + service_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null) + if [ "$?" != 0 ]; then + service_build=$(echo "$service_def" | shyaml get-value build 2>/dev/null) + if [ "$?" != 0 ]; then + err "Service $DARKYELLOW$service$NORMAL has no ${WHITE}image${NORMAL} nor ${WHITE}build${NORMAL} parameter." + echo "$service_def" >&2 + return 1 + fi + service_dockerfile="${service_build}"/Dockerfile + if ! [ -e "$service_dockerfile" ]; then + err "No Dockerfile found in '$service_dockerfile' location." + return 1 + fi + + grep '^FROM' "$service_dockerfile" | xargs printf "%s " | cut -f 2 -d " " + else + echo "$service_image" + fi | tee "$cache_file" + +} +export -f service_base_docker_image + + +get_charm_relation_def () { + local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \ + relation_def metadata + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + metadata="$(charm.metadata "$charm")" || return 1 + relation_def="$(echo "$metadata" | shyaml get-value "provides.${relation_name}" 2>/dev/null)" + echo "$relation_def" | tee "$cache_file" +} +export -f get_charm_relation_def + + +get_charm_tech_dep_orientation_for_relation() { + local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \ + relation_def value + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + relation_def=$(get_charm_relation_def "$charm" "$relation_name" 2>/dev/null) + value=$(echo "$relation_def" | shyaml get-value 'tech-dep' 2>/dev/null) + value=${value:-True} + echo "$value" | tee "$cache_file" +} +export -f get_charm_tech_dep_orientation_for_relation + + +## +## Use compose file to get deps, and relation definition in metadata.yml +## for tech-dep attribute. +get_service_deps() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + ( + set -o pipefail + get_compose_relations "$service" | \ + while read-0 relation_name target_service _relation_config tech_dep; do + echo "$target_service" + done | tee "$cache_file" + ) || return 1 +} +export -f get_service_deps + + +_rec_get_depth() { + local elt=$1 dep deps max cache_file="$state_tmpdir/$FUNCNAME.cache.$1" + [ "${depths[$elt]}" ] && return 0 + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + depths[$elt]=$(cat "$cache_file") + return 0 + fi + + visited[$elt]=1 + #debug "Setting visited[$elt]" + #debug "Asking for $DARKYELLOW$elt$NORMAL dependencies" + deps=$(get_service_deps "$elt") || { + debug "Failed get_service_deps $elt" + return 1 + } + # debug "$elt deps are:" $deps + max=0 + for dep in $deps; do + [ "${visited[$dep]}" ] && { + #debug "Already computing $dep" + continue + } + _rec_get_depth "$dep" || return 1 + #debug "Requesting depth[$dep]" + if (( ${depths[$dep]} > max )); then + max="${depths[$dep]}" + fi + done + # debug "Setting depth[$elt] to $((max + 1))" + depths[$elt]=$((max + 1)) + echo "${depths[$elt]}" > $cache_file +} +export -f _rec_get_depth + + +get_ordered_service_dependencies() { + local services=("$@") cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + #debug "Figuring ordered deps of $DARKYELLOW$services$NORMAL" + if [ -z "${services[*]}" ]; then + return 0 + # print_syntax_error "$FUNCNAME: no arguments" + # return 1 + fi + + declare -A depths + declare -A visited + heads=("${services[@]}") + while [ "${#heads[@]}" != 0 ]; do + array_pop heads head + _rec_get_depth "$head" || return 1 + done + + i=0 + while [ "${#depths[@]}" != 0 ]; do + for key in "${!depths[@]}"; do + value="${depths[$key]}" + if [ "$value" == "$i" ]; then + echo "$key" + unset depths[$key] + fi + done + i=$((i + 1)) + done | tee "$cache_file" +} +export -f get_ordered_service_dependencies + + +run_service_hook () { + local action="$1" service subservice subservices loaded + shift + declare -A loaded + for service in "$@"; do + subservices=$(get_ordered_service_dependencies "$service") || return 1 + for subservice in $subservices; do + if [ "${loaded[$subservice]}" ]; then + ## Prevent double inclusion of same service if this + ## service is deps of two or more of your + ## requirements. + continue + fi + + charm=$(get_service_charm "$subservice") || return 1 + charm.has_hook "$charm" "$action" >/dev/null || continue + + PROJECT_NAME=$(get_default_project_name) || return 1 + export MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1 + export MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1 + + Wrap -d "running $YELLOW$action$NORMAL hook of $DARKYELLOW$subservice$NORMAL in charm $DARKPINK$charm$NORMAL" </dev/null) || { + err "Missing ${WHITE}type$NORMAL option in ${WHITE}get$NORMAL config for location '$location'" + return 1 + } + if fn.exists host_resource_get_$type; then + host_resource_get_$type "$location" "$cfg" + else + err "Source ${WHITE}source$NORMAL type '$type' unknown for" \ + "${WHITE}host-resource$NORMAL '$location' defined in" \ + "$DARKYELLOW$subservice$NORMAL config." + return 1 + fi +} +export -f host_resource_get + + +host_resource_get_git() { + local location="$1" cfg="$2" branch parent url + branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null) + branch=${branch:-master} + + url=$(echo "$cfg" | shyaml get-value url 2>/dev/null) + parent="$(dirname "$location")" + ( + mkdir -p "$parent" && cd "$parent" && + git clone -b "$branch" "$url" "$(basename "$location")" + ) || return 1 +} +export -f host_resource_get_git + + +host_resource_get_git-sub() { + local location="$1" cfg="$2" branch parent url + branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null) + branch=${branch:-master} + + url=$(echo "$cfg" | shyaml get-value url 2>/dev/null) + parent="$(dirname "$location")" + ( + mkdir -p "$parent" && cd "$parent" && + git sub clone -b "$branch" "$url" "$(basename "$location")" + ) || return 1 +} +export -f host_resource_get_git-sub + + +setup_host_resource () { + local subservice="$1" service_def location get cfg + + service_def=$(get_compose_service_def "$subservice") || return 1 + while read-0 location cfg; do + ## XXXvlab: will it be a git resources always ? + if [ -d "$location" -a ! -d "$location/.git" ]; then + err "Hum, location '$location' does not seem to be a git directory." + return 1 + fi + if [ -d "$location" ]; then + info "host resource '$location' already set up." + continue + fi + get=$(echo "$cfg" | shyaml get-value get 2>/dev/null) + + if [ -z "$get" ]; then + err "No host directory '$location' found, and no ${WHITE}source$NORMAL" \ + "specified for $DARKYELLOW$subservice$NORMAL." + return 1 + fi + host_resource_get "$location" "$get" || return 1 + done < <(echo "$service_def" | shyaml key-values-0 host-resources 2>/dev/null) +} +export -f setup_host_resource + + +setup_host_resources () { + local service subservices subservice loaded + + declare -A loaded + for service in "$@"; do + subservices=$(get_ordered_service_dependencies "$service") || return 1 + for subservice in $subservices; do + if [ "${loaded[$subservice]}" ]; then + ## Prevent double inclusion of same service if this + ## service is deps of two or more of your + ## requirements. + continue + fi + setup_host_resource "$service" + loaded[$subservice]=1 + done + done + return 0 +} +export -f setup_host_resources + + +relation-get () { + local key="$1" + cat "$RELATION_DATA_FILE" | shyaml get-value "$key" 2>/dev/null + if [ "$?" != 0 ]; then + err "The key $WHITE$key$NORMAL was not found in relation's data." + return 1 + fi +} +export -f relation-get + + +relation-base-compose-get () { + local key="$1" + echo "$RELATION_BASE_COMPOSE_DEF" | shyaml get-value "options.$key" 2>/dev/null + if [ "$?" != 0 ]; then + err "The key $WHITE$key$NORMAL was not found in base service compose definition.." + return 1 + fi +} +export -f relation-base-compose-get + + +relation-target-compose-get () { + local key="$1" + echo "$RELATION_BASE_COMPOSE_DEF" | shyaml get-value "options.$key" 2>/dev/null + if [ "$?" != 0 ]; then + err "The key $WHITE$key$NORMAL was not found in base service compose definition.." + return 1 + fi +} +export -f relation-target-compose-get + + +relation-set () { + local key="$1" value="$2" + if [ -z "$RELATION_DATA_FILE" ]; then + err "$FUNCNAME: relation does not seems to be correctly setup." + return 1 + fi + + if ! [ -r "$RELATION_DATA_FILE" ]; then + err "$FUNCNAME: can't read relation's data." >&2 + return 1 + fi + + _config_merge "$RELATION_DATA_FILE" <(echo "$key: $value") +} +export -f relation-set + + +_config_merge() { + local config_filename="$1" merge_to_file="$2" + touch "$config_filename" && + merge_yaml "$config_filename" "$merge_to_file" > "$config_filename.tmp" || return 1 + mv "$config_filename.tmp" "$config_filename" +} +export -f _config_merge + + +## XXXvlab; this can be used only in relation, I'd like to use it in init. +config-add() { + local metadata="$1" + _config_merge "$RELATION_CONFIG" <(echo "$metadata") +} +export -f config-add + + +## XXXvlab; this can be used only in relation, I'd like to use it in init. +init-config-add() { + local metadata="$1" + _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" \ + <(yaml_key_val_str "services" "$metadata") +} +export -f init-config-add + + +logstdout() { + local name="$1" + sed -r 's%^%'"${name}"'> %g' +} +export -f logstdout + + +logstderr() { + local name="$1" + sed -r 's%^(.*)$%'"${RED}${name}>${NORMAL} \1"'%g' +} +export -f logstderr + + +_run_service_relation () { + local relation_name="$1" service="$2" target_service="$3" relation_config="$4" relation_dir services + + charm=$(get_service_charm "$service") || return 1 + target_charm=$(get_service_charm "$target_service") || return 1 + + base_script_name=$(charm.has_relation_hook "$charm" "$relation_name" relation-joined) || true + target_script_name=$(charm.has_relation_hook "$target_charm" "$relation_name" relation-joined) || true + [ "$base_script_name" -o "$target_script_name" ] || return 0 + + relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1 + RELATION_DATA_FILE=$(get_relation_data_file "$service" "$target_service" "$relation_name" "$relation_config") || return 1 + export BASE_SERVICE_NAME=$service + export BASE_CHARM_NAME=$charm + export BASE_CHARM_PATH=$(charm.get_dir "$charm") + export TARGET_SERVICE_NAME=$target_service + export TARGET_CHARM_NAME=$target_charm + export TARGET_CHARM_PATH=$(charm.get_dir "$target_charm") + export RELATION_DATA_FILE + target_errlvl=0 + + if [ -z "$target_script_name" ]; then + verb "No relation script $DARKBLUE$relation_name$NORMAL in target $DARKPINK$target_charm$NORMAL." + else + verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \ + "for target $DARKYELLOW$target_service$NORMAL (charm $DARKPINK$target_charm$NORMAL)" + RELATION_CONFIG="$relation_dir/config_provider" + DOCKER_BASE_IMAGE=$(service_base_docker_image "$target_service") || return 1 + export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA + { + ( + SERVICE_NAME=$target_service + SERVICE_DATASTORE="$DATASTORE/$target_service" + SERVICE_CONFIGSTORE="$CONFIGSTORE/$target_service" + export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE + charm.run_relation_hook "$target_charm" "$relation_name" relation-joined + echo "$?" > "$relation_dir/target_errlvl" + ) | logstdout "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}" + } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@${NORMAL}" 3>&1 1>&2 2>&3 + target_errlvl="$(cat "$relation_dir/target_errlvl")" || { + err "Relation script '$script_name' in $DARKPINK$target_charm$NORMAL" \ + "failed before outputing an errorlevel." + ((target_errlvl |= "1" )) + } + if [ -e "$RELATION_CONFIG" ]; then + debug "Merging some new config info in $DARKYELLOW$target_service$NORMAL" + _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" && + rm "$RELATION_CONFIG" + ((target_errlvl |= "$?")) + fi + fi + + if [ "$target_errlvl" == 0 ]; then + errlvl=0 + if [ "$base_script_name" ]; then + verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \ + "for $DARKYELLOW$service$NORMAL (charm $DARKPINK$charm$NORMAL)" + RELATION_CONFIG="$relation_dir/config_providee" + RELATION_DATA="$(cat "$RELATION_DATA_FILE")" + DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") || return 1 + export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA + { + ( + SERVICE_NAME=$service + SERVICE_DATASTORE="$DATASTORE/$service" + SERVICE_CONFIGSTORE="$CONFIGSTORE/$service" + export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE + charm.run_relation_hook "$charm" "$relation_name" relation-joined + echo "$?" > "$relation_dir/errlvl" + ) | logstdout "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}" + } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@$NORMAL" 3>&1 1>&2 2>&3 + errlvl="$(cat "$relation_dir/errlvl")" || { + err "Relation script '$script_name' in $DARKPINK$charm$NORMAL" \ + "failed before outputing an errorlevel." + ((errlvl |= "1" )) + } + if [ -e "$RELATION_CONFIG" ]; then + _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" && + rm "$RELATION_CONFIG" + ((errlvl |= "$?" )) + fi + if [ "$errlvl" != 0 ]; then + err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$service$NORMAL failed to run properly." + fi + else + verb "No relation script '$script_name' in charm $DARKPINK$charm$NORMAL. Ignoring." + fi + else + err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$target_service$NORMAL failed to run properly." + fi + + if [ "$target_errlvl" == 0 -a "$errlvl" == 0 ]; then + debug "Relation $DARKBLUE$relation_name$NORMAL is established" \ + "between $DARKYELLOW$service$NORMAL and $DARKYELLOW$target_service$NORMAL." + return 0 + else + return 1 + fi +} +export -f _run_service_relation + + +_get_compose_relations_cached () { + local compose_service_def="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + relation_name relation_def target_service + if [ -e "$cache_file" ]; then + #debug "$FUNCNAME: STATIC cache hit $1" + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + + ( + set -o pipefail + if [ "$compose_service_def" ]; then + while read-0 relation_name relation_def; do + ## XXXvlab: could we use braces here instead of parenthesis ? + ( + case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in + "str") + target_service="$(echo "$relation_def" | shyaml get-value 2>/dev/null)" + target_charm=$(get_service_charm "$target_service") || return 1 + tech_dep="$(get_charm_tech_dep_orientation_for_relation "$target_charm" "$relation_name")" + echo -en "$relation_name\0$target_service\0\0$tech_dep\0" + ;; + "sequence") + while read-0 target_service; do + target_charm=$(get_service_charm "$target_service") || return 1 + tech_dep="$(get_charm_tech_dep_orientation_for_relation "$target_charm" "$relation_name")" + echo -en "$relation_name\0$target_service\0\0$tech_dep\0" + done < <(echo "$relation_def" | shyaml get-values-0 2>/dev/null) + ;; + "struct") + while read-0 target_service relation_config; do + target_charm=$(get_service_charm "$target_service") || return 1 + tech_dep="$(get_charm_tech_dep_orientation_for_relation "$target_charm" "$relation_name")" + echo -en "$relation_name\0$target_service\0$relation_config\0$tech_dep\0" + done < <(echo "$relation_def" | shyaml key-values-0 2>/dev/null) + ;; + esac + ) > "$cache_file" || return 1 + done < <(echo "$compose_service_def" | shyaml key-values-0 relations 2>/dev/null) + fi + ) + if [ "$?" != 0 ]; then + err "Error while looking for compose relations." + rm -f "$cache_file" ## no cache + return 1 + fi + [ -e "$cache_file" ] && cat "$cache_file" + return 0 +} +export -f _get_compose_relations_cached + + +get_compose_relations () { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + compose_def + + if [ -e "$cache_file" ]; then + #debug "$FUNCNAME: SESSION cache hit $1" + cat "$cache_file" + return 0 + fi + + compose_def="$(get_compose_service_def "$service")" || return 1 + _get_compose_relations_cached "$compose_def" > "$cache_file" + if [ "$?" != 0 ]; then + rm -f "$cache_file" ## no cache + return 1 + fi + cat "$cache_file" +} +export -f get_compose_relations + + +get_compose_relation_def() { + local service="$1" relation="$2" relation_name target_service relation_config tech_dep + while read-0 relation_name target_service relation_config tech_dep; do + [ "$relation_name" == "$relation" ] || continue + printf "%s\0%s\0%s\0" "$target_service" "$relation_config" "$tech_dep" + done < <(get_compose_relations "$service") || return 1 +} +export -f get_compose_relation_def + + +run_service_relations () { + local service services loaded subservices subservice + + PROJECT_NAME=$(get_default_project_name) || return 1 + + export PROJECT_NAME + + declare -A loaded + subservices=$(get_ordered_service_dependencies "$@") || return 1 + for service in $subservices; do + + # debug "Upping dep's relations of ${DARKYELLOW}$service${NORMAL}:" + for subservice in $(get_service_deps "$service") "$service"; do + [ "${loaded[$subservice]}" ] && continue + + export BASE_SERVICE_NAME=$service + MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1 + MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1 + RELATION_BASE_COMPOSE_DEF=$(get_compose_service_def "$subservice") || return 1 + export RELATION_BASE_COMPOSE_DEF MASTER_BASE_{CHARM,SERVICE}_NAME + + # debug " Relations of ${DARKYELLOW}$subservice${NORMAL}:" + while read-0 relation_name target_service relation_config tech_dep; do + export relation_config + + export TARGET_SERVICE_NAME=$target_service + MASTER_TARGET_SERVICE_NAME=$(get_top_master_service_for_service "$target_service") || return 1 + MASTER_TARGET_CHARM_NAME=$(get_service_charm "$MASTER_TARGET_SERVICE_NAME") || return 1 + RELATION_TARGET_COMPOSE_DEF=$(get_compose_service_def "$target_service") || return 1 + export RELATION_TARGET_COMPOSE_DEF MASTER_TARGET_{CHARM,SERVICE}_NAME + + Wrap -d "Building $DARKYELLOW$subservice$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW$target_service$NORMAL" < "$action_errlvl_file" + ) | logstdout "$DARKYELLOW$service$NORMAL/${DARKCYAN}$action${NORMAL} ${GREEN}@${NORMAL}" + } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$service$NORMAL/${DARKCYAN}$action${NORMAL} ${RED}@$NORMAL" 3>&1 1>&2 2>&3 + if ! [ -e "$action_errlvl_file" ]; then + err "Action $DARKYELLOW$service$NORMAL:$DARKCYAN$action$NORMAL has failed without having time" \ + "to output an errlvl" + return 1 + fi + return "$(cat "$action_errlvl_file")" +} +export -f _run_service_action_direct + +_run_service_action_relation() { + local service="$1" action="$2" charm target_charm relation_name relation_config _dummy + shift; shift + + read-0 charm target_service target_charm relation_name relation_config || true + + if read-0 _dummy || [ "$_dummy" ]; then + print_syntax_error "$FUNCNAME: too many arguments in action descriptor" + return 1 + fi + + export RELATION_DATA_FILE=$(get_relation_data_file "$service" "$target_service" "$relation_name" "$relation_config") + + export action_errlvl_file="$state_tmpdir/action-$service-$charm-$action-errlvl" + export state_tmpdir + { + ( + set +e ## Prevents unwanted leaks from parent shell + export METADATA_CONFIG=$(charm.metadata "$charm") + export SERVICE_NAME=$service + export RELATION_TARGET_SERVICE="$target_service" + export RELATION_TARGET_CHARM="$target_charm" + export RELATION_BASE_SERVICE="$service" + export RELATION_BASE_CHARM="$charm" + export ACTION_NAME=$action + export CONTAINER_NAME=$(get_top_master_service_for_service "$service") + export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME") + export SERVICE_DATASTORE="$DATASTORE/$service" + export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service" + exname="$exname $ACTION_NAME $SERVICE_NAME" \ + stdbuf -oL -eL bash -c 'charm.run_relation_action "$@"' -- "$target_charm" "$relation_name" "$action" "$@" + echo "$?" > "$action_errlvl_file" + ) | logstdout "$DARKYELLOW$service$NORMAL/${DARKCYAN}$action${NORMAL} ${GREEN}@${NORMAL}" + } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$service$NORMAL/${DARKCYAN}$action${NORMAL} ${RED}@$NORMAL" 3>&1 1>&2 2>&3 + if ! [ -e "$action_errlvl_file" ]; then + err "Action $DARKYELLOW$service$NORMAL:$DARKCYAN$action$NORMAL has failed without having time" \ + "to output an errlvl" + return 1 + fi + return "$(cat "$action_errlvl_file")" +} +export -f _run_service_action_relation + + +get_relation_data_dir() { + local service="$1" target_service="$2" relation_name="$3" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + project=$(get_default_project_name) || return 1 + relation_dir="$VARDIR/relations/$project/${service}-${target_service}/$relation_name" + if ! [ -d "$relation_dir" ]; then + mkdir -p "$relation_dir" || return 1 + chmod go-rwx "$relation_dir" || return 1 ## protecting this directory + fi + echo "$relation_dir" | tee "$cache_file" +} +export -f get_relation_data_dir + + +get_relation_data_file() { + local service="$1" target_service="$2" relation_name="$3" relation_config="$4" + + relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1 + relation_data_file="$relation_dir/data" + + new= + if [ -e "$relation_data_file" ]; then + ## Has reference changed ? + new_md5=$(echo "$relation_config" | md5_compat) + if [ "$new_md5" != "$(cat "$relation_data_file.md5_ref" 2>/dev/null)" ]; then + new=true + fi + else + new=true + fi + + if [ "$new" ]; then + echo "$relation_config" > "$relation_data_file" + chmod go-rwx "$relation_data_file" ## protecting this file + echo "$relation_config" | md5_compat > "$relation_data_file.md5_ref" + fi + echo "$relation_data_file" +} +export -f get_relation_data_file + + +has_service_action () { + local service="$1" action="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + charm target_charm relation_name target_service relation_config _tech_dep + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + charm=$(get_service_charm "$service") || return 1 + + ## Action directly provided ? + + if charm.has_direct_action "$charm" "$action" >/dev/null; then + echo -en "direct\0$charm" | tee "$cache_file" + return 0 + fi + + ## Action provided by relation ? + + while read-0 relation_name target_service relation_config _tech_dep; do + target_charm=$(get_service_charm "$target_service") || return 1 + if charm.has_relation_action "$target_charm" "$relation_name" "$action" >/dev/null; then + echo -en "relation\0$charm\0$target_service\0$target_charm\0$relation_name\0$relation_config" | tee "$cache_file" + return 0 + fi + done < <(get_compose_relations "$service") + + return 1 + # master=$(get_top_master_service_for_service "$service") + # [ "$master" == "$charm" ] && return 1 + + # has_service_action "$master" "$action" +} +export -f has_service_action + + +run_service_action () { + local service="$1" action="$2" + shift ; shift + { + if ! read-0 action_type; then + info "Service $DARKYELLOW$service$NORMAL does not have any action $DARKCYAN$action$NORMAL defined." + info " Add an executable script to 'actions/$action' to implement action." + return 1 + fi + + Section "running $DARKYELLOW$service$NORMAL/$DARKCYAN$action$NORMAL ($action_type)"; Feed + "_run_service_action_${action_type}" "$service" "$action" "$@" + + } < <(has_service_action "$service" "$action") +} +export -f run_service_action + + +get_compose_relation_config() { + local service=$1 relation_config cache_file="$state_tmpdir/$FUNCNAME.cache.$1" + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + compose_service_def=$(get_compose_service_def "$service") || return 1 + + echo "$compose_service_def" | shyaml get-value "relations" 2>/dev/null | tee "$cache_file" +} +export -f get_compose_relation_config + + +# ## Return key-values-0 +# get_compose_relation_config_for_service() { +# local service=$1 relation_name=$2 relation_config +# compose_service_relations=$(get_compose_relation_config "$service") || return 1 +# if ! relation_config=$( +# echo "$compose_service_relations" | +# shyaml get-value "${relation_name}" 2>/dev/null); then +# err "Couldn't find $DARKYELLOW${service}$NORMAL/${WHITE}${relation_name}$NORMAL" \ +# "relation config in compose configuration." +# return 1 +# fi +# if [ -z "$relation_config" ]; then +# err "Relation ${WHITE}mysql-database$NORMAL is empty in compose configuration." +# return 1 +# fi +# if ! echo "$relation_config" | shyaml key-values-0 2>/dev/null; then +# err "No key/values in ${DARKBLUE}mysql-database$NORMAL of compose config." +# return 1 +# fi +# } +# export -f get_compose_relation_config_for_service + + +_get_master_service_for_service_cached () { + local service="$1" charm="$2" metadata="$3" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + charm requires master_charm target_charm target_service service_def found + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: STATIC cache hit ($1)" + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + + if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" != "True" ]; then + ## just return service name + echo "$service" | tee "$cache_file" + return 0 + fi + + ## fetch the container relation + requires="$(echo "$metadata" | shyaml get-value "requires" 2>/dev/null)" + if [ -z "$requires" ]; then + die "Charm $DARKPINK$charm$NORMAL is a subordinate but does not have any 'requires' " \ + "section." + fi + found= + while read-0 relation_name relation; do + [ "$(echo "$relation" | shyaml get-value "scope" 2>/dev/null)" == "container" ] && { + found=1 + break + } + done < <(echo "$requires" | shyaml key-values-0 2>/dev/null) + if [ -z "$found" ]; then + die "Charm $DARKPINK$charm$NORMAL is a subordinate but does not have any required relation declaration with" \ + " ${WHITE}scope${NORMAL} set to 'container'." + fi + + interface="$(echo "$relation" | shyaml get-value "interface" 2>/dev/null)" + if [ -z "$interface" ]; then + err "No ${WHITE}interface${NORMAL} set for relation $DARKBLUE$relation_name$NORMAL." + return 1 + fi + + ## Action provided by relation ? + + found= + while read-0 relation_name target_service _relation_config _tech_dep; do + [ "$interface" == "$relation_name" ] && { + found=1 + break + } + done < <(get_compose_relations "$service") + if [ -z "$found" ]; then + err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \ + "${DARKYELLOW}$service$NORMAL compose definition." + return 1 + fi + echo "$target_service" | tee "$cache_file" +} +export -f _get_master_service_for_service_cached + + +get_master_service_for_service() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + charm metadata result + + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: SESSION cache hit ($*)" + cat "$cache_file" || return 1 + return 0 + fi + charm=$(get_service_charm "$service") || return 1 + metadata=$(charm.metadata "$charm" 2>/dev/null) || { + metadata="" + warn "No charm $DARKPINK$charm$NORMAL found." + } + result=$(_get_master_service_for_service_cached "$service" "$charm" "$metadata") || return 1 + echo "$result" | tee "$cache_file" || return 1 +} +export -f get_master_service_for_service + + +get_top_master_service_for_service() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ + current_service + + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + + current_service="$service" + while true; do + master_service=$(get_master_service_for_service "$current_service") || return 1 + [ "$master_service" == "$current_service" ] && break + current_service="$master_service" + done + echo "$current_service" | tee "$cache_file" + return 0 +} +export -f get_top_master_service_for_service + + +## +## The result is a mixin that is not always a complete valid +## docker-compose entry (thinking of subordinates). The result +## will be merge with master charms. +_get_docker_compose_mixin_from_metadata_cached() { + local service="$1" charm="$2" metadata="$3" has_build_dir="$4" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \ + metadata_file metadata volumes docker_compose subordinate image + if [ -e "$cache_file" ]; then + #debug "$FUNCNAME: STATIC cache hit $1" + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + + mixin=$(echo -en "labels:\n- compose.charm=$charm") + if [ "$metadata" ]; then + ## resources to volumes + volumes=$( + for resource_type in data config; do + while read-0 resource; do + eval "echo \" - \$${resource_type^^}STORE/\$service\$resource:\$resource:rw\"" + done < <(echo "$metadata" | shyaml get-values-0 "${resource_type}-resources" 2>/dev/null) + done + while read-0 resource; do + if [[ "$resource" == /*:/*:* ]]; then + echo " - $resource" + elif [[ "$resource" == /*:/* ]]; then + echo " - $resource:rw" + elif [[ "$resource" == /*:* ]]; then + echo " - ${resource%%:*}:$resource" + elif [[ "$resource" =~ ^/[^:]+$ ]]; then + echo " - $resource:$resource:rw" + else + die "Invalid host-resource specified in 'metadata.yml'." + fi + done < <(echo "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null) + while read-0 resource; do + dest="$(charm.get_dir "$charm")/resources$resource" + if ! [ -e "$dest" ]; then + die "charm-resource: '$resource' does not exist (file: '$dest')." + fi + echo " - $dest:$resource:ro" + done < <(echo "$metadata" | shyaml get-values-0 "charm-resources" 2>/dev/null) + ) || return 1 + if [ "$volumes" ]; then + mixin=$(merge_yaml_str "$mixin" "$(echo -en "volumes:\n$volumes")") || { + err "Failed to merge mixin with ${WHITE}docker-compose${NORMAL} option" \ + "from charm ${DARKPINK}$charm$NORMAL." + return 1 + } + fi + + docker_compose=$(echo "$metadata" | shyaml get-value -y "docker-compose" 2>/dev/null) + if [ "$docker_compose" ]; then + mixin=$(merge_yaml_str "$mixin" "$docker_compose") || { + err "Failed to merge mixin with ${WHITE}docker-compose${NORMAL} option" \ + "from charm ${DARKPINK}$charm$NORMAL." + return 1 + } + fi + + if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" == "True" ]; then + subordinate=true + fi + fi + + image=$(echo "$metadata" | shyaml get-value "docker-image" 2>/dev/null) + [ "$image" == "None" ] && image="" + image_or_build_statement= + if [ "$image" ]; then + if [ "$subordinate" ]; then + err "Subordinate charm can not have a ${WHITE}docker-image${NORMAL} value." + return 1 + fi + image_or_build_statement="image: $image" + elif [ "$has_build_dir" ]; then + if [ "$subordinate" ]; then + err "Subordinate charm can not have a 'build' sub directory." + return 1 + fi + image_or_build_statement="build: $(charm.get_dir "$charm")/build" + fi + if [ "$image_or_build_statement" ]; then + mixin=$(merge_yaml_str "$mixin" "$image_or_build_statement") || { + err "Failed to merge yaml with image or build YAML statement." + return 1 + } + fi + echo "$mixin" | tee "$cache_file" +} +export -f _get_docker_compose_mixin_from_metadata_cached + + +get_docker_compose_mixin_from_metadata() { + local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" + if [ -e "$cache_file" ]; then + #debug "$FUNCNAME: SESSION cache hit ($*)" + cat "$cache_file" + return 0 + fi + charm=$(get_service_charm "$service") || return 1 + metadata="$(charm.metadata "$charm" 2>/dev/null)" || return 1 + has_build_dir= + [ -d "$(charm.get_dir "$charm")/build" ] && has_build_dir=true + mixin=$(_get_docker_compose_mixin_from_metadata_cached "$service" "$charm" "$metadata" "$has_build_dir") || return 1 + echo "$mixin" | tee "$cache_file" +} +export -f get_docker_compose_mixin_from_metadata + + +_save() { + local name="$1" + cat - | tee -a "$docker_compose_dir/.data/$name" +} +export -f _save + + +get_default_project_name() { + if [ "$DEFAULT_PROJECT_NAME" ]; then + echo "$DEFAULT_PROJECT_NAME" + return 0 + fi + compose_yml_location="$(get_compose_yml_location)" || return 1 + if [ "$compose_yml_location" ]; then + if normalized_path=$(readlink -f "$compose_yml_location"); then + echo "$(basename "$(dirname "$normalized_path")")" + return 0 + fi + fi + echo "orphan" + return 0 +} +export -f get_default_project_name + + +launch_docker_compose() { + local charm docker_compose_tmpdir docker_compose_dir + docker_compose_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX) + #debug "Creating temporary docker-compose directory in '$docker_compose_tmpdir'." + trap_add EXIT "rm -rf \"$docker_compose_tmpdir\"" + + ## docker-compose will name network from the parent dir name + project=$(get_default_project_name) + mkdir -p "$docker_compose_tmpdir/$project" + docker_compose_dir="$docker_compose_tmpdir/$project" + + get_docker_compose $SERVICE_PACK > "$docker_compose_dir/docker-compose.yml" || return 1 + if [ -e "$state_tmpdir/to-merge-in-docker-compose.yml" ]; then + # debug "Merging some config data in docker-compose.yml:" + # debug "$(cat $state_tmpdir/to-merge-in-docker-compose.yml)" + _config_merge "$docker_compose_dir/docker-compose.yml" "$state_tmpdir/to-merge-in-docker-compose.yml" || return 1 + fi + + if [ -z "$(echo $(cat "$docker_compose_dir/docker-compose.yml"))" ]; then + die "Generated 'docker-compose.yml' is unexpectedly empty." + fi + + ## XXXvlab: could be more specific and only link the needed charms + ## XXXvlab: why do we need these links ? If this is for the build command, then it is not useful anymore. + # for charm in $(shyaml keys services < "$docker_compose_dir/docker-compose.yml"); do + # if charm.exists "$charm"; then + # ln -sf "$(charm.get_dir "$charm")" "$docker_compose_dir/$charm" || exit 1 + # fi + # done + mkdir "$docker_compose_dir/.data" + + { + { + cd "$docker_compose_dir" + debug "${WHITE}docker-compose.yml$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:" + debug "$(cat "docker-compose.yml" | prefix " $GRAY|$NORMAL ")" + debug "${WHITE}Launching$NORMAL: docker-compose $@" + if [ "$DRY_COMPOSE_RUN" ]; then + echo docker-compose "$@" + else + docker-compose "$@" + fi + echo "$?" > "$docker_compose_dir/.data/errlvl" + } | _save stdout + } 3>&1 1>&2 2>&3 | _save stderr 3>&1 1>&2 2>&3 + if tail -n 1 "$docker_compose_dir/.data/stderr" | egrep "Service .+ failed to build: Error getting container [0-9a-f]+ from driver devicemapper: (open|Error mounting) /dev/mapper/docker-.*: no such file or directory$" >/dev/null 2>&1; then + err "Detected bug https://github.com/docker/docker/issues/4036 ... " + err "Please re-launch your command, or switch from 'devicemapper' driver to 'overlayfs' or 'aufs'." + fi + + docker_compose_errlvl="$(cat "$docker_compose_dir/.data/errlvl" 2>/dev/null)" + if [ -z "$docker_compose_errlvl" ]; then + err "Something went wrong before you could gather docker-compose errorlevel." + return 1 + fi + return "$docker_compose_errlvl" +} +export -f launch_docker_compose + + +get_compose_yml_location() { + if ! [ -z ${COMPOSE_YML_FILE+x} ]; then ## if set, even if empty + echo "$COMPOSE_YML_FILE" + return 0 + fi + parent=$(while ! [ -f "./compose.yml" ]; do + [ "$PWD" == "/" ] && exit 0 + cd .. + done; echo "$PWD" + ) + if [ "$parent" ]; then + echo "$parent/compose.yml" + return 0 + fi + ## XXXvlab: do we need this additional environment variable, + ## COMPOSE_YML_FILE is not sufficient ? + if [ "$DEFAULT_COMPOSE_FILE" ]; then + if ! [ -e "$DEFAULT_COMPOSE_FILE" ]; then + warn "No 'compose.yml' was found in current or parent dirs," \ + "and \$DEFAULT_COMPOSE_FILE points to an unexistent file." \ + "(${DEFAULT_COMPOSE_FILE})" + return 0 + fi + echo "$DEFAULT_COMPOSE_FILE" + return 0 + fi + warn "No 'compose.yml' was found in current or parent dirs, and no \$DEFAULT_COMPOSE_FILE was set." + return 0 +} +export -f get_compose_yml_location + + +get_compose_yml_content() { + local cache_file="$state_tmpdir/$FUNCNAME.cache" + if [ -e "$cache_file" ]; then + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + + if [ -z "$COMPOSE_YML_FILE" ]; then + COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1 + fi + if [ -e "$COMPOSE_YML_FILE" ]; then + debug "Found $WHITE$exname$NORMAL YAML file in '$COMPOSE_YML_FILE'." + COMPOSE_YML_CONTENT=$(cat "$COMPOSE_YML_FILE") || { + err "Could not read '$COMPOSE_YML_FILE'." + return 1 + } + else + debug "No compose file found. Using an empty one." + COMPOSE_YML_CONTENT="" + fi + + COMPOSE_YML_CONTENT=$(merge_yaml_str "$COMPOSE_YML_CONTENT" "${compose_contents[@]}") || return 1 + output=$(echo "$COMPOSE_YML_CONTENT"| shyaml get-value 2>&1) + if [ "$?" != 0 ]; then + outputed_something= + while IFS='' read -r line1 && IFS='' read -r line2; do + [ "$outputed_something" ] || err "Invalid YAML in '$COMPOSE_YML_FILE':" + outputed_something=true + echo "$line1 $GRAY($line2)$NORMAL" + done < <(echo "$output" | grep ^yaml.scanner -A 100 | + sed -r 's/^ in "", //g' | sed -r 's/^yaml.scanner.[a-zA-Z]+: //g') | + prefix " $GRAY|$NORMAL " + [ "$outputed_something" ] || { + err "Unexpected error while running 'shyaml get-value' on '$COMPOSE_YML_FILE':" + echo "$output" | prefix " $GRAY|$NORMAL " + } + return 1 + fi + echo "$COMPOSE_YML_CONTENT" | tee "$cache_file" || return 1 +} +export -f get_compose_yml_content + + +get_default_target_services() { + local services=("$@") + if [ -z "${services[*]}" ]; then + if [ "$DEFAULT_SERVICES" ]; then + debug "No service provided, using $WHITE\$DEFAULT_SERVICES$NORMAL variable." \ + "Target services: $DARKYELLOW$DEFAULT_SERVICES$NORMAL" + services="$DEFAULT_SERVICES" + else + err "No service provided." + return 1 + fi + fi + echo "${services[*]}" +} +export -f get_default_target_services + + +get_master_services() { + local loaded master_service service + declare -A loaded + for service in "$@"; do + master_service=$(get_top_master_service_for_service "$service") || return 1 + if [ "${loaded[$master_service]}" ]; then + continue + fi + echo "$master_service" + loaded["$master_service"]=1 + done | xargs printf "%s " + return "${PIPESTATUS[0]}" +} +export -f get_master_services + + +_setup_state_dir() { + export state_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX) + #debug "Creating temporary state directory in '$state_tmpdir'." +# trap_add EXIT "debug \"Removing temporary state directory in $state_tmpdir.\";\ +# rm -rf \"$state_tmpdir\"" + trap_add EXIT "rm -rf \"$state_tmpdir\"" +} + + +get_docker_compose_help_msg() { + local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \ + docker_compose_help_msg + if [ -e "$cache_file" ]; then + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + docker_compose_help_msg=$(docker-compose $action --help 2>/dev/null) || return 1 + echo "$docker_compose_help_msg" | + tee "$cache_file" || return 1 +} + + +get_docker_compose_usage() { + local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \ + docker_compose_help_msg + if [ -e "$cache_file" ]; then + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + docker_compose_help_msg=$(get_docker_compose_help_msg $action) || return 1 + echo "$docker_compose_help_msg" | + grep -m 1 "^Usage:" -A 10000 | + egrep -m 1 "^\$" -B 10000 | + xargs printf "%s " | + sed -r 's/^Usage: //g' | + tee "$cache_file" || return 1 +} + + +get_docker_compose_opts_help() { + local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \ + docker_compose_help_msg + if [ -e "$cache_file" ]; then + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1 + echo "$docker_compose_opts_help" | + grep '^Options:' -A 20000 | + tail -n +2 | + { cat ; echo; } | + egrep -m 1 "^\S*\$" -B 10000 | + head -n -1 | + tee "$cache_file" || return 1 +} + + +get_docker_compose_commands_help() { + local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \ + docker_compose_help_msg + if [ -e "$cache_file" ]; then + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1 + echo "$docker_compose_opts_help" | + grep '^Commands:' -A 20000 | + tail -n +2 | + { cat ; echo; } | + egrep -m 1 "^\S*\$" -B 10000 | + head -n -1 | + tee "$cache_file" || return 1 +} + + +get_docker_compose_opts_list() { + local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \ + docker_compose_help_msg + if [ -e "$cache_file" ]; then + cat "$cache_file" && + touch "$cache_file" || return 1 + return 0 + fi + docker_compose_opts_help=$(get_docker_compose_opts_help $action) || return 1 + echo "$docker_compose_opts_help" | + egrep "^\s+-" | + sed -r 's/\s+((((-[a-zA-Z]|--[a-zA-Z0-9-]+)( [A-Z=]+|=[^ ]+)?)(, )?)+)\s+.*$/\1/g' | + tee "$cache_file" || return 1 +} + + +options_parser() { + sed -r 's/^(\s+(((-[a-zA-Z]|--[a-zA-Z0-9-]+)([ =]([a-zA-Z_=\"\[]|\])+)?(, | )?)+)\s+)[^ ].*$/\x0\2\x0\0/g' + printf "\0" +} + + +remove_options_in_option_help_msg() { + { + read-0 null + if [ "$null" ]; then + err "options parsing error, should start with an option line." + return 1 + fi + while read-0 opt full_txt;do + multi_opts="$(printf "%s " $opt | multi_opts_filter)" + single_opts="$(printf "%s " $opt | single_opts_filter)" + for to_remove in "$@"; do + str_matches "$to_remove" $multi_opts $single_opts && { + continue 2 + } + done + echo -n "$full_txt" + done + } < <(options_parser) +} + + +_MULTIOPTION_REGEX='^((-[a-zA-Z]|--[a-zA-Z0-9-]+)(, )?)+' +_MULTIOPTION_REGEX_LINE_FILTER=$_MULTIOPTION_REGEX'(\s|=)' + + +multi_opts_filter() { + egrep "$_MULTIOPTION_REGEX_LINE_FILTER" | + sed -r "s/^($_MULTIOPTION_REGEX)(\s|=).*$/\1/g" | + tr ',' "\n" | xargs printf "%s " +} + + +single_opts_filter() { + egrep -v "$_MULTIOPTION_REGEX_LINE_FILTER" | + tr ',' "\n" | xargs printf "%s " +} + + +get_docker_compose_multi_opts_list() { + local action="$1" opts_list + opts_list=$(get_docker_compose_opts_list "$action") || return 1 + echo "$opts_list" | multi_opts_filter +} + + +get_docker_compose_single_opts_list() { + local action="$1" opts_list + opts_list=$(get_docker_compose_opts_list "$action") || return 1 + echo "$opts_list" | single_opts_filter +} + + +display_help() { + print_help + echo "${WHITE}Options${NORMAL}:" + echo " -h, --help Print this message and quit" + echo " (ignoring any other options)" + echo " -V, --version Print current version and quit" + echo " (ignoring any other options)" + echo " --dirs Display data dirs and quit" + echo " (ignoring any other options)" + echo " -v, --verbose Be more verbose" + echo " -d, --debug Print full debugging information (sets also verbose)" + echo " --dry-compose-run If docker-compose will be run, only print out what" + echo " command line will be used." + echo " --rebuild-relations-to-service, -R SERVICE" + echo " Will rebuild all relations to given service" + echo " --add-compose-content, -Y YAML" + echo " Will merge some direct YAML with the current compose" + + get_docker_compose_opts_help | remove_options_in_option_help_msg --version --help --verbose | + filter_docker_compose_help_message + echo + echo "${WHITE}Commands${NORMAL} (thanks to docker-compose):" + get_docker_compose_commands_help | sed -r "s/ ([a-z]+)(\s+)/ ${DARKCYAN}\1${NORMAL}\2/g" +} + + +_graph_service() { + local service="$1" base="$1" + + charm=$(get_service_charm "$service") || return 1 + metadata=$(charm.metadata "$charm") || return 1 + subordinate=$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null) + + if [ "$subordinate" == "True" ]; then + requires="$(echo "$metadata" | shyaml get-value "requires" 2>/dev/null)" + master_charm= + while read-0 relation_name relation; do + [ "$(echo "$relation" | shyaml get-value "scope" 2>/dev/null)" == "container" ] || continue + interface="$(echo "$relation" | shyaml get-value "interface" 2>/dev/null)" + if [ -z "$interface" ]; then + err "No ${WHITE}$interface${NORMAL} set for relation $relation_name." + return 1 + fi + + ## Action provided by relation ? + + target_service= + while read-0 relation_name candidate_target_service _relation_config _tech_dep; do + [ "$interface" == "$relation_name" ] && { + target_service="$candidate_target_service" + break + } + done < <(get_compose_relations "$service") + if [ -z "$target_service" ]; then + err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \ + "${DARKYELLOW}$service$NORMAL compose definition." + return 1 + fi + master_service="$target_service" + master_charm=$(get_service_charm "$target_service") || return 1 + break + done < <(echo "$requires" | shyaml key-values-0 2>/dev/null) + fi + + _graph_node_service "$service" "$base" "$charm" + _graph_edge_service "$service" "$subordinate" "$master_service" + +} + + +_graph_node_service() { + local service="$1" base="$2" charm="$3" + + cat < + ]; +EOF + +} + + +_graph_edge_service() { + local service="$1" subordinate="$2" master_service="$3" + while read-0 relation_name target_service relation_config tech_dep; do + cat < "$(_graph_node_service_label ${target_service})" [ + penwidth = $([ "$master_service" == "$target_service" ] && echo 3 || echo 2) + + fontsize = 16 + fontcolor = "black" + style = $([ "$master_service" == "$target_service" ] && echo dashed || echo "\"\"") + weight = $([ "$master_service" == "$target_service" ] && echo 2.0 || echo 1.0) + dir = $([ "$master_service" == "$target_service" ] && echo none || echo both) + arrowtail = odot + # arrowhead = dotlicurve + taillabel = "$relation_name" ]; +EOF + done < <(get_compose_relations "$service") || return 1 +} + + +_graph_node_service_label() { + local service="$1" + echo "service_$service" +} + + +_graph_node_service_content() { + local service="$1" + charm=$(get_service_charm "$service") || return 1 + +cat < + + + $service + + +$(if [ "$charm" != "$service" ]; then +cat < + charm: $charm + +EOF2 +fi) + +EOF +} + + +cla_contains () { + local e + for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done + return 1 +} + + +filter_docker_compose_help_message() { + cat - | + sed -r "s/docker-compose run/${DARKWHITE}compose${NORMAL} ${DARKCYAN}$action${NORMAL}/g; + s/docker-compose.yml/compose.yml/g; + s/SERVICES?/${DARKYELLOW}\0${NORMAL}/g; + s/^(\s+)\\$/\1${WHITE}\$${NORMAL}/g; + s/^(\s+)run/\1${DARKCYAN}$action${NORMAL}/g; + s/docker-compose/${DARKWHITE}compose${NORMAL}/g" +} + + +graph() { + local services=("$@") + declare -A entries + cat <&2 + exit 1 + fi + fi + ;; + *) + # echo "LOOP $1 : pos_arg: $pos_arg_ct // ${pos_args[$pos_arg_ct]}" + if [[ "${pos_args[$pos_arg_ct]}" == "[SERVICE...]" ]]; then + services_args+=("$arg") + elif [[ "${pos_args[$pos_arg_ct]}" == "SERVICE" ]]; then + services_args=("$arg") || exit 1 + stage="remainder" + else + action_posargs+=("$arg") + ((pos_arg_ct++)) + fi + ;; + esac + ;; + "remainder") + remainder_args+=("$arg") + while read-0 arg; do + remainder_args+=("$arg") + done + break 3 + ;; + esac + shift +done < <(cla.normalize "$@") + + +export compose_contents +[ "${services_args[*]}" ] && debug " ${DARKWHITE}Services:$NORMAL ${DARKYELLOW}${services_args[*]}$NORMAL" +[ "${compose_opts[*]}" ] && debug " ${DARKWHITE}Main docker-compose opts:$NORMAL ${compose_opts[*]}" +[ "${action_posargs[*]}" ] && debug " ${DARKWHITE}Main docker-compose pos args:$NORMAL ${action_posargs[*]}" +[ "${action_opts[*]}" ] && debug " ${DARKWHITE}Action $DARKCYAN$action$NORMAL with opts:$NORMAL ${action_opts[*]}" +[ "${remainder_args[*]}" ] && debug " ${DARKWHITE}Remainder args:$NORMAL ${remainder_args[*]}" + + + +aexport remainder_args + + +## +## Actual code +## + + +COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1 +COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1 +export COMPOSE_YML_FILE COMPOSE_YML_CONTENT + + +charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct." + +## +## Get services in command line. +## + +if [ -z "$is_docker_compose_action" -a "$action" ]; then + action_service=${remainder_args[0]} + remainder_args=("${remainder_args[@]:1}") + if is_service_action=$(has_service_action "$action_service" "$action"); then + { + read-0 action_type + case "$action_type" in + "relation") + read-0 _ target_service _target_charm relation_name + debug "Found action $DARKYELLOW${action_service}$NORMAL/$DARKBLUE$relation_name$NORMAL/$DARKCYAN$action$NORMAL (in $DARKYELLOW$target_service$NORMAL)" + ;; + "direct") + debug "Found action $DARKYELLOW${action_service}$NORMAL.$DARKCYAN$action$NORMAL" + ;; + esac + } < <(has_service_action "$action_service" "$action") + services_args=("$action_service") + else + die "Unknown action '${DARKCYAN}$action$NORMAL': It doesn't match any docker-compose commands nor inner charm actions." + fi +else + case "$action" in + ps) + if [ "${#services_args[@]}" == 0 ]; then + services_args=($(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys 2>/dev/null)) || true + fi + ;; + up) + if [ "${#services_args[@]}" == 0 ]; then + while read-0 service; do + type="$(get_service_type "$service")" || exit 1 + if [ "$type" != "run-once" ]; then + services_args+=("$service") + fi + done < <(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys-0 2>/dev/null) + fi + ;; + config) + services_args=("${action_posargs[@]}") + ;; + esac + if [ "$is_docker_compose_action" -a "${#services_args[@]}" -gt 0 ]; then + services=($(get_master_services "${services_args[@]}")) || exit 1 + action_posargs+=("${services[@]}") + fi +fi + + +get_docker_compose "${services_args[@]}" >/dev/null || { ## precalculate variable \$_current_docker_compose + err "Fails to compile base 'docker-compose.yml'" + exit 1 +} + +## +## Pre-action +## + + +full_init= +case "$action" in + up|run) + full_init=true + post_hook=true + ;; + ""|down|restart|logs|config|ps) + full_init= + ;; + *) + if [ "$is_service_action" ]; then + full_init=true + fi + ;; +esac + + +if [ "$full_init" ]; then + ## init in order + if [ -z "$no_init" ]; then + Section setup host resources + setup_host_resources "${services_args[@]}" || exit 1 + Section initialisation + run_service_hook init "${services_args[@]}" || exit 1 + fi + + ## Get relations + if [ -z "$no_relations" ]; then + if [ "${#rebuild_relations_to_service[@]}" != 0 ]; then + rebuild_relations_to_service=( + $(get_master_services "${rebuild_relations_to_service[@]}")) + project=$(get_default_project_name) || return 1 + for service in "${rebuild_relations_to_service[@]}"; do + for dir in "$VARDIR/relations/$project/"*"-${service}/"*; do + [ -d "$dir" ] && { + debug rm -rf "$dir" + rm -rf "$dir" + } + done + done + fi + run_service_relations "${services_args[@]}" || exit 1 + fi + + run_service_hook pre_deploy "${services_args[@]}" || exit 1 + +fi + +if [ "$action" == "run" ]; then + charm=$(get_service_charm "${services_args[0]}") || exit 1 + metadata=$(charm.metadata "$charm") || exit 1 + type="$(printf "%s" "$metadata" | shyaml get-value type 2>/dev/null)" || true + if [ "$type" == "run-once" ]; then + run_service_hook dc-pre-run "${services_args[@]}" || exit 1 + fi +fi + + +export SERVICE_PACK="${services_args[*]}" + +## +## Docker-compose +## + +case "$action" in + up|start|stop|build|run) + ## force daemon mode for up + if [[ "$action" == "up" ]] && ! array_member action_opts -d; then + action_opts+=("-d") + fi + + launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}" + ;; + logs) + if ! array_member action_opts --tail; then ## force daemon mode for up + action_opts+=("--tail" "10") + fi + launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}" + ;; + "") + launch_docker_compose "${compose_opts[@]}" + ;; + graph) + graph $SERVICE_PACK + ;; + config) + ## removing the services + services=($(get_master_services "${action_posargs[@]}")) || exit 1 + ## forcing docker-compose config to output the config file to stdout and not stderr + out=$(launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" 2>&1) || { + echo "$out" + exit 1 + } + echo "$out" + warn "Runtime configuration modification (from relations) are not included here." + ;; + down) + if ! array_member action_opts --remove-orphans; then ## force daemon mode for up + debug "Adding a default argument of '--remove-orphans'" + action_opts+=("--remove-orphans") + fi + launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" + ;; + *) + if [ "$is_service_action" ]; then + run_service_action "$SERVICE_PACK" "$action" "${remainder_args[@]}" + else + launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}" + fi + ;; +esac || exit 1 + + +if [ "$post_hook" -a "${#services_args[@]}" != 0 ]; then + run_service_hook post_deploy "${services_args[@]}" || exit 1 +fi