new: charm can now declare relations

A relation can be 'recommended', 'required', 'optional'. It can also auto-pair.

bin/compose-core

@@ -813,10 +813,6 @@ get_docker_compose_links() {
     fi
 
     master_service=$(get_top_master_service_for_service "$service") || return 1
-    ## XXXvlab: yuck, this make the assumption that next function is cached,
-    ## and leverage the fact that the result is stored in a file. All this only
-    ## to catch a failure of ``get_compose_relations``, that would go out silently.
-    get_compose_relations "$service" >/dev/null || return 1  ## fetch cache and fail if necessary
     deps=()
     while read-0 _relation_name target_service _relation_config tech_dep; do
         master_target_service="$(get_top_master_service_for_service "$target_service")" || return 1
@@ -832,7 +828,7 @@ get_docker_compose_links() {
         ## as there's a circular dependency issue. We don't really want the full feature
         ## of depends_on, but just to add it as targets when doing an 'up'
         # deps+=("$(echo -en "$master_service:\n  depends_on:\n    - $master_target_service")")
-    done < <(get_compose_relations "$service")
+    done < <(get_service_relations "$service")
     merge_yaml_str "${deps[@]}" | tee "$cache_file" || return 1
     if [ "${PIPESTATUS[0]}" != 0 ]; then
         rm "$cache_file"
@@ -1204,7 +1200,7 @@ get_service_deps() {
     fi
     (
         set -o pipefail
-        get_compose_relations "$service" | \
+        get_service_relations "$service" | \
             while read-0 relation_name target_service _relation_config tech_dep; do
                 echo "$target_service"
             done | tee "$cache_file"
@@ -1692,6 +1688,400 @@ get_compose_relations () {
 export -f get_compose_relations
 
 
+get_service_relations () {
+    local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
+          s rn ts rc td
+    if [ -e "$cache_file" ]; then
+        #debug "$FUNCNAME: SESSION cache hit $1"
+        cat "$cache_file"
+        return 0
+    fi
+
+    if [ -z "$ALL_RELATIONS" ]; then
+        err "Can't access global \$ALL_RELATIONS"
+        return 1
+    fi
+
+    while read-0 s rn ts rc td; do
+        [[ "$s" == "$service" ]] || continue
+        printf "%s\0" "$rn" "$ts" "$rc" "$td"
+    done < <(cat "$ALL_RELATIONS") > "$cache_file"
+
+    if [ "$?" != 0 ]; then
+        rm -f "$cache_file"  ## no cache
+        return 1
+    fi
+    cat "$cache_file"
+}
+export -f get_service_relations
+
+
+get_service_relation() {
+    local service="$1" relation="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
+          rn ts rc td
+    if [ -e "$cache_file" ]; then
+        #debug "$FUNCNAME: SESSION cache hit $1"
+        cat "$cache_file"
+        return 0
+    fi
+
+    while read-0 rn ts rc td; do
+        [ "$relation" == "$rn" ] && {
+            printf "%s\0" "$ts" "$rc" "$td"
+            break
+        }
+    done < <(get_service_relations "$service") > "$cache_file"
+    if [ "$?" != 0 ]; then
+        rm -f "$cache_file"  ## no cache
+        return 1
+    fi
+    cat "$cache_file"
+}
+
+
+_get_charm_metadata_uses() {
+    local metadata="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
+    if [ -e "$cache_file" ]; then
+        #debug "$FUNCNAME: SESSION cache hit $1"
+        cat "$cache_file" || return 1
+        return 0
+    fi
+
+    printf "%s" "$metadata" | { shyaml key-values-0 uses 2>/dev/null || true; } | tee "$cache_file"
+}
+export -f _get_charm_metadata_uses
+
+
+_get_service_metadata() {
+    local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
+          charm
+
+    if [ -e "$cache_file" ]; then
+        #debug "$FUNCNAME: SESSION cache hit $1"
+        cat "$cache_file"
+        return 0
+    fi
+
+    charm="$(get_service_charm "$service")" || return 1
+    charm.metadata "$charm" > "$cache_file"
+    if [ "$?" != 0 ]; then
+        rm -f "$cache_file"  ## no cache
+        return 1
+    fi
+    cat "$cache_file"
+}
+export -f _get_service_metadata
+
+
+_get_service_uses() {
+    local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
+          metadata
+
+    if [ -e "$cache_file" ]; then
+        #debug "$FUNCNAME: SESSION cache hit $1"
+        cat "$cache_file"
+        return 0
+    fi
+
+    metadata="$(_get_service_metadata "$service")" || return 1
+    _get_charm_metadata_uses "$metadata" > "$cache_file"
+    if [ "$?" != 0 ]; then
+        rm -f "$cache_file"  ## no cache
+        return 1
+    fi
+    cat "$cache_file"
+}
+export -f _get_service_uses
+
+
+_get_services_uses() {
+    local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
+          service rn rd
+
+    if [ -e "$cache_file" ]; then
+        #debug "$FUNCNAME: SESSION cache hit $1"
+        cat "$cache_file"
+        return 0
+    fi
+
+    for service in "$@"; do
+        _get_service_uses "$service" | while read-0 rn rd; do
+            printf "%s\0" "$service" "$rn" "$rd"
+        done
+        [ "${PIPESTATUS[0]}" == 0 ] || {
+            return 1
+        }
+    done > "${cache_file}.wip"
+    mv "${cache_file}"{.wip,} &&
+    cat "$cache_file" || return 1
+}
+export -f _get_services_uses
+
+
+_get_services_provides() {
+    local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
+          service rn rd
+
+    if [ -e "$cache_file" ]; then
+        #debug "$FUNCNAME: SESSION cache hit $1"
+        cat "$cache_file"
+        return 0
+    fi
+
+    ## YYY: replace the inner loop by a cached function
+    for service in "$@"; do
+        metadata="$(_get_service_metadata "$service")" || return 1
+
+        while read-0 rn rd; do
+            printf "%s\0" "$service" "$rn" "$rd"
+        done < <(printf "%s" "$metadata" | shyaml key-values-0 provides 2>/dev/null)
+    done > "$cache_file"
+    if [ "$?" != 0 ]; then
+        rm -f "$cache_file"  ## no cache
+        return 1
+    fi
+    cat "$cache_file"
+}
+export -f _get_services_provides
+
+
+_get_services_providing() {
+    local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
+          relation="$1"
+    shift   ## services is "$@"
+    if [ -e "$cache_file" ]; then
+        #debug "$FUNCNAME: SESSION cache hit $1"
+        cat "$cache_file"
+        return 0
+    fi
+
+    while read-0 service relation_name relation_def; do
+        [ "$relation_name" == "$relation" ] || continue
+        printf "%s\0" "$service" "$relation_def"
+    done < <(_get_services_provides "$@") > "$cache_file"
+
+    if [ "$?" != 0 ]; then
+        rm -f "$cache_file"  ## no cache
+        return 1
+    fi
+    cat "$cache_file"
+}
+export -f _get_services_provides
+
+
+get_all_relations () {
+    local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
+          services
+
+    if [ -e "${cache_file}" ]; then
+        #debug "$FUNCNAME: SESSION cache hit $1"
+        cat "${cache_file}"
+        return 0
+    fi
+
+    declare -A services
+    services_uses=()
+    ## XXXvlab: bwerk, leveraging cache to be able to get the errorlevel here.
+    _get_services_uses "$@" || return 1
+    array_read-0 services_uses < <(_get_services_uses "$@")
+    services_provides=()
+    ## XXXvlab: bwerk, leveraging cache to be able to get the errorlevel here.
+    _get_services_provides "$@" || return 1
+    array_read-0 services_provides < <(_get_services_provides "$@")
+
+    for service in "$@"; do
+        services[$service]=1
+    done
+
+    all_services=("$@")
+    while [ "${#all_services[@]}" != 0 ]; do
+        array_pop all_services service
+        while read-0 relation_name ts relation_config tech_dep; do
+            printf "%s\0" "$service" "$relation_name" "$ts" "$relation_config" "$tech_dep"
+
+            ## adding target services ?
+            [ "${services[$ts]}" ] && continue
+            array_read-0 services_uses < <(_get_services_uses "$ts")
+            all_services+=("$ts")
+            services[$ts]=1
+        done < <(get_compose_relations "$service")
+    done > "${cache_file}.wip"
+
+    while true; do
+        changed=
+        new_services_uses=()
+        summon=()
+        required=()
+        recommended=()
+        optional=()
+        while [ "${#services_uses[@]}" != 0 ]; do
+            service="${services_uses[0]}"
+            relation_name="${services_uses[1]}"
+            relation_def="${services_uses[2]}"
+            services_uses=("${services_uses[@]:3}")
+
+            default_options=$(printf "%s" "$relation_def" | shyaml -y get-value "default-options" 2>/dev/null)
+            ## is this "use" declaration satisfied ?
+            found=
+            while read-0 s rn ts rc td; do
+                if [ -z "$found" -a "$service" == "$s" -a "$relation_name" == "$rn" ]; then
+                    if [ "$default_options" ]; then
+                        rc=$(merge_yaml_str "$default_options" "$rc") || return 1
+                    fi
+                    found="$ts"
+                fi
+                printf "%s\0" "$s" "$rn" "$ts" "$rc" "$td"
+            done < "${cache_file}.wip" > "${cache_file}.wip.new"
+            mv "${cache_file}.wip.new" "${cache_file}.wip"
+            if [ "$found" ]; then  ## this "use" declaration was satisfied
+                debug "${DARKYELLOW}$service${NORMAL} use declaration for relation " \
+                      "${DARKBLUE}$relation_name${NORMAL} is satisfied with ${DARKYELLOW}$found${NORMAL}"
+                continue
+            fi
+
+            auto=$(echo "$relation_def" | shyaml get-value auto pair 2>/dev/null)
+            case "$auto" in
+                "pair")
+                    service_list=()
+                    array_read-0 service_list < <(array_keys_to_stdin services)
+                    providers=()
+                    array_read-0 providers providers_def < <(_get_services_providing "$relation_name" "${service_list[@]}")
+                    if [ "${#providers[@]}" == 1 ]; then
+                        ts="${providers[0]}"
+                        rd_provider="${providers_def[0]}"
+                        rc_provider=$(printf "%s" "$rd_provider" | shyaml -y get-value "default-options" 2>/dev/null)
+                        ## YYYvlab: should be seen even in no debug mode no ?
+                        debug "Auto-pairs ${DARKYELLOW}$service${NORMAL}" \
+                              "--${DARKBLUE}$relation_name${NORMAL}--> ${DARKYELLOW}$ts${NORMAL}"
+                        rc=$(printf "%s" "$relation_def" | shyaml -y get-value "default-options" 2>/dev/null)
+                        td=$(echo "$rd_provider" | shyaml get-value 'tech-dep' 2>/dev/null)
+                        td=${td:-True}
+                        rc=$(merge_yaml_str "$rc_provider" "$rc") || return 1
+                        printf "%s\0" "$service" "$relation_name" "$ts" "$rc" "$td" >> "${cache_file}.wip"
+
+                        ## Adding service
+                        [ "${services[$ts]}" ] && continue
+                        array_read-0 new_services_uses < <(_get_services_uses "$ts")
+                        services[$ts]=1
+                        changed=1
+                        continue
+                    elif [ "${#providers[@]}" -gt 1 ]; then
+                        msg=""
+                        warn "No auto-pairing ${DARKYELLOW}$service${NORMAL}" \
+                             "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
+                             "(> 1 provider)."
+                        continue
+                    else
+                        :  ## Do nothing
+                    fi
+                    ;;
+                "summon")
+                    summon+=("$service" "$relation_name" "$relation_def")
+                    ;;
+                ""|null|disable|disabled)
+                    :
+                    ;;
+                *)
+                    err "Invalid ${WHITE}auto${NORMAL} value '$auto'."
+                    return 1
+                    ;;
+            esac
+            constraint=$(echo "$relation_def" | shyaml get-value constraint auto-pair 2>/dev/null)
+            case "$constraint" in
+                "required")
+                    required+=("$service" "$relation_name" "$relation_def")
+                    ;;
+                "recommended")
+                    recommended+=("$service" "$relation_name" "$relation_def")
+                    ;;
+                "optional")
+                    optional+=("$service" "$relation_name" "$relation_def")
+                    ;;
+                *)
+                    err "Invalid ${WHITE}constraint${NORMAL} value '$contraint'."
+                    return 1
+                    ;;
+            esac
+            new_services_uses+=("$service" "$relation_name" "$relation_def")  ## re-queue it
+        done
+        services_uses=("${new_services_uses[@]}")
+
+        if [ "$changed" ]; then
+            continue
+        fi
+        ## situation is stable
+
+        if [ "${#summon[@]}" != 0 ]; then
+            die "Summon code not implemented yet"
+            continue
+        fi
+        [ "$NO_CONSTRAINT_CHECK" ] && break
+        if [ "${#required[@]}" != 0 ]; then
+            echo "$(_display_solves required)" | sed -r "s/^/${RED}||${NORMAL} /g" >&2
+            err "Required relations not satisfied"
+            return 1
+        fi
+        if [ "${#recommended[@]}" != 0 ]; then
+            ## make recommendation
+            echo "$(_display_solves recommended)" | sed -r "s/^/${YELLOW}||${NORMAL} /g" >&2
+        fi
+        if [ "${#optional[@]}" != 0 ]; then
+            ## inform about options
+            echo "$(_display_solves optional)"  | sed -r "s/^/${BLUE}||${NORMAL} /g" >&2
+        fi
+        # if [ "${#required[@]}" != 0 ]; then
+        #     err "Required relations not satisfied"
+        #     return 1
+        # fi
+        if [ "${#recommended[@]}" != 0 ]; then
+            warn "Recommended relations not satisfied"
+        fi
+        break
+    done
+    if [ "$?" != 0 ]; then
+        rm -f "${cache_file}"{,.wip,.wip.new}  ## no cache
+        return 1
+    fi
+    export ALL_RELATIONS="$cache_file"
+    mv "${cache_file}"{.wip,}
+    cat "$cache_file"
+}
+export -f get_all_relations
+
+
+_display_solves() {
+    local array_name="$1" by_relation msg
+    ## inform about options
+    msg=""
+    declare -A by_relation
+    while read-0 service relation_name relation_def; do
+        solves=$(printf "%s" "$relation_def" | shyaml -y get-value solves 2>/dev/null);
+        auto=$(printf "%s" "$relation_def" | shyaml get-value auto 2>/dev/null);
+        if [ -z "$solves" ]; then
+            continue
+        fi
+        by_relation[$relation_name]+=$(printf "\n  %s" "${DARKYELLOW}$service$NORMAL for:")
+        if [ "$auto" == "pair" ]; then
+            requirement="add provider in cluster to auto-pair"
+        else
+            requirement="add explicit relation"
+        fi
+        while read-0 name def; do
+            by_relation[$relation_name]+=$(printf "\n    - ${DARKCYAN}%-15s${NORMAL} %s (%s)" "$name" "$def" "$requirement")
+        done < <(printf "%s" "$solves" | shyaml key-values-0)
+    done < <(array_values_to_stdin "$array_name")
+
+    while read-0 relation_name message; do
+        msg+="$(printf "\n${DARKBLUE}%s$NORMAL provider is $array_name by%s" \
+                        "$relation_name" "$message" )"
+    done < <(array_kv_to_stdin by_relation)
+
+    if [ "$msg" ]; then
+        printf "%s\n" "${msg:1}"
+    fi
+}
+
+
 get_compose_relation_def() {
     local service="$1" relation="$2" relation_name target_service relation_config tech_dep
     while read-0 relation_name target_service relation_config tech_dep; do
@@ -1736,7 +2126,7 @@ run_service_relations () {
                 Wrap -d "Building $DARKYELLOW$subservice$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW$target_service$NORMAL" <<EOF || return 1
 _run_service_relation "$relation_name" "$subservice" "$target_service" "\$relation_config"
 EOF
-            done < <(get_compose_relations "$subservice") || return 1
+            done < <(get_service_relations "$subservice") || return 1
             loaded[$subservice]=1
         done
     done
@@ -1899,7 +2289,7 @@ has_service_action () {
             echo -en "relation\0$charm\0$target_service\0$target_charm\0$relation_name\0$relation_config" | tee "$cache_file"
             return 0
         fi
-    done < <(get_compose_relations "$service")
+    done < <(get_service_relations "$service")
 
     return 1
     # master=$(get_top_master_service_for_service "$service")
@@ -1965,6 +2355,23 @@ export -f get_compose_relation_config
 # }
 # export -f get_compose_relation_config_for_service
 
+_get_container_relation() {
+    local metadata=$1 found relation_name relation_def
+
+    found=
+    while read-0 relation_name relation_def; do
+        [ "$(echo "$relation_def" | shyaml get-value "scope" 2>/dev/null)" == "container" ] && {
+            found="$relation_name"
+            break
+        }
+    done < <(_get_charm_metadata_uses "$metadata")
+    if [ -z "$found" ]; then
+        die "Charm $DARKPINK$charm$NORMAL is a subordinate but does not have any required relation declaration with" \
+            "${WHITE}scope${NORMAL} set to 'container'."
+    fi
+    printf "%s" "$found"
+}
+
 
 _get_master_service_for_service_cached () {
     local service="$1" charm="$2" metadata="$3" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
@@ -1982,42 +2389,13 @@ _get_master_service_for_service_cached () {
         return 0
     fi
 
-    ## fetch the container relation
-    requires="$(echo "$metadata" | shyaml get-value "requires" 2>/dev/null)"
-    if [ -z "$requires" ]; then
-        die "Charm $DARKPINK$charm$NORMAL is a subordinate but does not have any 'requires' " \
-            "section."
-    fi
-    found=
-    while read-0 relation_name relation; do
-        [ "$(echo "$relation" | shyaml get-value "scope" 2>/dev/null)" == "container" ] && {
-            found=1
-            break
-        }
-    done < <(echo "$requires" | shyaml key-values-0 2>/dev/null)
-    if [ -z "$found" ]; then
-        die "Charm $DARKPINK$charm$NORMAL is a subordinate but does not have any required relation declaration with" \
-            " ${WHITE}scope${NORMAL} set to 'container'."
-    fi
-
-    interface="$(echo "$relation" | shyaml get-value "interface" 2>/dev/null)"
-    if [ -z "$interface" ]; then
-        err "No ${WHITE}interface${NORMAL} set for relation $DARKBLUE$relation_name$NORMAL."
-        return 1
-    fi
-
     ## Action provided by relation ?
-
-    found=
-    while read-0 relation_name target_service _relation_config _tech_dep; do
-        [ "$interface" == "$relation_name" ] && {
-            found=1
-            break
-        }
-    done < <(get_compose_relations "$service")
-    if [ -z "$found" ]; then
-        err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \
+    container_relation=$(_get_container_relation "$metadata")
+    read-0 target_service _ _ < <(get_service_relation "$service" "$container_relation")
+    if [ -z "$target_service" ]; then
+        err "Couldn't find ${WHITE}relations.${container_relation}${NORMAL} in" \
             "${DARKYELLOW}$service$NORMAL compose definition."
+        err ${FUNCNAME[@]}
         return 1
     fi
     echo "$target_service" | tee "$cache_file"
@@ -2757,7 +3135,7 @@ _graph_service() {
                     target_service="$candidate_target_service"
                     break
                 }
-            done < <(get_compose_relations "$service")
+            done < <(get_service_relations "$service")
             if [ -z "$target_service" ]; then
                 err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \
                     "${DARKYELLOW}$service$NORMAL compose definition."
@@ -2809,7 +3187,7 @@ _graph_edge_service() {
     # arrowhead = dotlicurve
     taillabel = "$relation_name" ];
 EOF
-    done < <(get_compose_relations "$service") || return 1
+    done < <(get_service_relations "$service") || return 1
 }
 
 
@@ -3162,27 +3540,44 @@ if [ -z "$is_docker_compose_action" -a "$action" ]; then
     fi
 else
     case "$action" in
-        ps)
+        ps|up)
             if [ "${#services_args[@]}" == 0 ]; then
-                services_args=($(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys 2>/dev/null)) || true
-            fi
-            ;;
-        up)
-            if [ "${#services_args[@]}" == 0 ]; then
-                while read-0 service; do
-                    type="$(get_service_type "$service")" || exit 1
-                    if [ "$type" != "run-once" ]; then
-                        services_args+=("$service")
-                    fi
-                done < <(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys-0 2>/dev/null)
+                array_read-0 services_args < <(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys-0 2>/dev/null)
             fi
             ;;
         config)
             services_args=("${action_posargs[@]}")
             ;;
     esac
-    if [ "$is_docker_compose_action" -a "${#services_args[@]}" -gt 0 ]; then
-        services=($(get_master_services "${services_args[@]}")) || exit 1
+fi
+
+
+NO_CONSTRAINT_CHECK=
+case "$action" in
+    ""|down|restart|logs|config|ps)
+        NO_CONSTRAINT_CHECK=True
+        ;;
+    *)
+        if [ "$is_service_action" ]; then
+            NO_CONSTRAINT_CHECK=True
+        fi
+        ;;
+esac
+
+
+get_all_relations "${services_args[@]}" >/dev/null || exit 1
+
+if [ "$is_docker_compose_action" -a "${#services_args[@]}" -gt 0 ]; then
+    services=($(get_master_services "${services_args[@]}")) || exit 1
+    if [ "$action" == "up" ]; then
+        ## remove run-once
+        for service in "${services_args[@]}"; do
+            type="$(get_service_type "$service")" || exit 1
+            if [ "$type" != "run-once" ]; then
+                action_posargs+=("$service")
+            fi
+        done
+    else
         action_posargs+=("${services[@]}")
     fi
 fi

test/base

@@ -547,6 +547,8 @@ EOF2
 . "$tprog"
 
 _setup_state_dir
+get_all_relations www >/dev/null || exit 3
+
 test "\$(get_master_service_for_service www)" == "www"
 
 ## -- subordinate
@@ -556,9 +558,9 @@ mkdir -p $test_tmpdir/{www,mysql}
 touch $test_tmpdir/mysql/metadata.yml
 cat <<EOF2 > $test_tmpdir/www/metadata.yml
 subordinate: true
-requires:
-  a-label-for-relation:
-    interface: a-name-relation
+uses:
+  a-name-relation:
+    constraint: required
     scope: container
 
 EOF2
@@ -575,6 +577,8 @@ EOF2
 . "$tprog"
 
 _setup_state_dir
+get_all_relations www >/dev/null || exit 3
+
 COMPOSE_YML_FILE=$test_tmpdir/compose.yml
 test "\$(get_master_service_for_service www)" == "mysql"
 EOF
@@ -609,6 +613,7 @@ EOF2
 . "$tprog"
 
 _setup_state_dir
+get_all_relations www >/dev/null || exit 3
 
 out=\$(_get_docker_compose_service_mixin www | shyaml get-value www.volumes)
 [[ "\$out" == "\
@@ -644,6 +649,8 @@ EOF2
 
 _setup_state_dir
 COMPOSE_YML_FILE=$test_tmpdir/compose.yml
+get_all_relations www >/dev/null || exit 3
+
 
 out="\$(_get_docker_compose_service_mixin www)" || exit 1
 [ "\$out" == "www:
@@ -671,9 +678,9 @@ data-resources:
   - /tmp/a
 config-resources:
   - /tmp/b
-requires:
+uses:
   a-name-relation:
-    interface: a-name-relation
+    constraint: required
     scope: container
 EOF2
 
@@ -690,6 +697,9 @@ EOF2
 
 _setup_state_dir
 COMPOSE_YML_FILE=$test_tmpdir/compose.yml
+
+get_all_relations www >/dev/null || exit 3
+
 out="\$(_get_docker_compose_service_mixin www)" || exit 1
 expected="mysql:
   labels:
@@ -730,6 +740,7 @@ EOF2
 . "$tprog"
 
 _setup_state_dir
+get_all_relations www >/dev/null || exit 3
 out=\$(get_docker_compose www)
 echo "OUT:"
 echo "\$out"
@@ -775,8 +786,11 @@ EOF2
 COMPOSE_YML_FILE=$test_tmpdir/compose.yml
 
 _setup_state_dir
+get_all_relations web_site >/dev/null || exit 3
+
 
-out=\$(get_docker_compose www | shyaml get-value services.www.volumes)
+out=\$(get_docker_compose www) || exit 4
+out=\$(printf "%s" "\$out" | shyaml get-value services.www.volumes) || exit 5
 test "\$out" == "\\
 - /www/tmp/a:/tmp/a:rw
 - /www/tmp/b:/tmp/b:rw" || {
@@ -784,7 +798,8 @@ test "\$out" == "\\
     exit 1
 }
 
-out=\$(get_docker_compose_links web_site | shyaml get-value web_site.links)
+out=\$(get_docker_compose_links web_site) || exit 6
+out=\$(printf "%s" "\$out" | shyaml get-value web_site.links) || exit 7
 test "\$out" == "- mysql" || {
     echo -e "** get_docker_compose_links web_site:\n\$out"
     exit 1
@@ -827,9 +842,9 @@ data-resources:
   - /tmp/a
 config-resources:
   - /tmp/b
-requires:
-  my-db-connection:
-    interface: db-connection
+uses:
+  db-connection:
+    constraint: required
     scope: container
 EOF2
 
@@ -856,6 +871,7 @@ EOF2
 COMPOSE_YML_FILE=$test_tmpdir/compose.yml
 
 _setup_state_dir
+get_all_relations web_site >/dev/null || exit 3
 
 # should fail because of missing relations
 ! get_docker_compose www || exit 1
@@ -882,9 +898,9 @@ data-resources:
   - /tmp/a
 config-resources:
   - /tmp/b
-requires:
-  my-db-connection:
-    interface: db-connection
+uses:
+  db-connection:
+    constraint: required
     scope: container
 docker-compose:
   volumes:
@@ -918,6 +934,7 @@ EOF2
 COMPOSE_YML_FILE=$test_tmpdir/compose.yml
 
 _setup_state_dir
+get_all_relations web_site >/dev/null || exit 3
 
 # should fail because of missing relations
 #! get_docker_compose www || exit 1
@@ -1049,9 +1066,9 @@ EOF
 #   - /tmp/a
 # config-resources:
 #   - /tmp/b
-# requires:
-#   my-db-connection:
-#     interface: db-connection
+# uses:
+#   db-connection:
+#     constraint: required
 #     scope: container
 # EOF2
 
@@ -1125,6 +1142,7 @@ EOF2
 . "$tprog"
 COMPOSE_YML_FILE=$test_tmpdir/compose.yml
 _setup_state_dir
+get_all_relations app >/dev/null || exit 3
 
 out=\$(get_docker_compose_links "app")
 test "\$out" == "app:
@@ -1158,6 +1176,7 @@ EOF2
 . "$tprog"
 COMPOSE_YML_FILE=$test_tmpdir/compose.yml
 _setup_state_dir
+get_all_relations web_site >/dev/null || exit 3
 
 out=\$(get_charm_relation_def "www" "web-proxy") || exit 1
 test "\$out" == "tech-dep: reversed" || {
@@ -1200,9 +1219,9 @@ data-resources:
   - /tmp/a
 config-resources:
   - /tmp/b
-requires:
-  my-db-connection:
-    interface: db-connection
+uses:
+  db-connection:
+    constraint: required
     scope: container
 docker-compose:
   volumes:

test/relations

@@ -0,0 +1,533 @@
+#!/usr/bin/env bash-shlib
+# -*- mode: shell-script -*-
+
+include shunit
+
+depends sed grep git mkdir readlink
+
+export -f matches
+export grep
+
+tmp=/tmp
+tprog="../bin/compose-core"
+tprog=$(readlink -f $tprog)
+
+
+export PATH=".:$PATH"
+short_tprog=$(basename "$tprog")
+
+
+##
+## Convenience function
+##
+
+init_test() {
+    test_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX)
+    cd "$test_tmpdir"
+    export CACHEDIR="$test_tmpdir/.cache"
+    export VARDIR="$test_tmpdir/.var"
+    mkdir -p "$CACHEDIR"
+}
+
+
+
+tear_test() {
+    rm -rf "$test_tmpdir"
+}
+
+
+##
+## Tests
+##
+
+
+function test_all_relations_simple {
+
+    init_test
+
+    export DISABLE_SYSTEM_CONFIG_FILE=true
+
+    assert_list <<EOF
+
+### Testing simple case
+
+## -- simple case with only one service with one relation, no uses/provides
+
+export CHARM_STORE=$test_tmpdir
+mkdir -p $test_tmpdir/{www,mysql}
+cat <<EOF2 > $test_tmpdir/www/metadata.yml
+EOF2
+
+touch $test_tmpdir/mysql/metadata.yml
+
+cat <<EOF2 > $test_tmpdir/compose.yml
+www:
+  charm: www
+  relations:
+    mysql-db:
+      mysql:
+        label: value
+EOF2
+
+. "$tprog"
+
+_setup_state_dir
+COMPOSE_YML_FILE=$test_tmpdir/compose.yml
+
+out=\$(set -o pipefail
+  get_all_relations www mysql | tr '\0' '\n') || exit 3
+expected="www
+mysql-db
+mysql
+label: value
+
+True"
+
+[[ "\$out" == "\$expected" ]] || {
+    echo "doesn't end with: \$expected" >&2
+    echo "\$out"
+    exit 1
+}
+
+
+## -- simple case, 1 uses, already present
+
+export CHARM_STORE=$test_tmpdir
+mkdir -p $test_tmpdir/{www,mysql}
+cat <<EOF2 > $test_tmpdir/www/metadata.yml
+uses:
+  mysql-db:
+    constraint: optional
+    default-options:
+       label: from-default-options
+       label2: from-default-options
+EOF2
+
+touch $test_tmpdir/mysql/metadata.yml
+
+cat <<EOF2 > $test_tmpdir/compose.yml
+www:
+  charm: www
+  relations:
+    mysql-db:
+      mysql:
+        label: value
+EOF2
+
+. "$tprog"
+
+_setup_state_dir
+COMPOSE_YML_FILE=$test_tmpdir/compose.yml
+
+out=\$(set -o pipefail
+  get_all_relations www mysql | tr '\0' '\n') || exit 3
+expected="www
+mysql-db
+mysql
+label: value
+label2: from-default-options
+True"
+
+[[ "\$out" == "\$expected" ]] || {
+    echo "doesn't end with: \$expected" >&2
+    echo --- OUT: >&2
+    echo "\$out" >&2
+    exit 1
+}
+
+
+## -- simple case, 1 uses, optional, not present
+
+export CHARM_STORE=$test_tmpdir
+mkdir -p $test_tmpdir/{www,mysql}
+cat <<EOF2 > $test_tmpdir/www/metadata.yml
+uses:
+  mysql-db:
+    constraint: optional
+    default-options:
+       label: from-default-options
+       label2: from-default-options
+EOF2
+
+touch $test_tmpdir/mysql/metadata.yml
+
+cat <<EOF2 > $test_tmpdir/compose.yml
+www:
+  charm: www
+EOF2
+
+. "$tprog"
+
+_setup_state_dir
+COMPOSE_YML_FILE=$test_tmpdir/compose.yml
+
+
+out=\$(set -o pipefail
+  get_all_relations www mysql 2>&1 >/dev/null) || exit 3
+expected=""
+
+[[ "\$out" == "\$expected" ]] || {
+    echo "doesn't end with: \$expected" >&2
+    echo "\$out"
+    exit 1
+}
+
+
+
+out=\$(set -o pipefail
+  get_all_relations www mysql | tr '\0' '\n') || exit 3
+expected=""
+
+[[ "\$out" == "\$expected" ]] || {
+    echo "doesn't end with: \$expected" >&2
+    echo "\$out"
+    exit 1
+}
+
+
+## -- simple case, 1 uses, optional with solves, not present
+
+export CHARM_STORE=$test_tmpdir
+mkdir -p $test_tmpdir/{www,mysql}
+cat <<EOF2 > $test_tmpdir/www/metadata.yml
+uses:
+  mysql-db:
+    constraint: optional
+    solves:
+       missing-feature: foo
+    default-options:
+       label: from-default-options
+       label2: from-default-options
+EOF2
+
+touch $test_tmpdir/mysql/metadata.yml
+
+cat <<EOF2 > $test_tmpdir/compose.yml
+www:
+  charm: www
+EOF2
+
+. "$tprog"
+
+_setup_state_dir
+COMPOSE_YML_FILE=$test_tmpdir/compose.yml
+
+
+out=\$(set -o pipefail
+  get_all_relations www mysql 2>&1 >/dev/null) || exit 3
+expected="Notice"
+
+[[ "\$out" == *"\$expected"* ]] || {
+    echo "doesn't contain: \$expected" >&2
+    echo "\$out"
+    exit 1
+}
+
+
+
+
+## -- simple case, 1 uses, auto-pair, present
+
+export CHARM_STORE=$test_tmpdir
+mkdir -p $test_tmpdir/{www,mysql}
+cat <<EOF2 > $test_tmpdir/www/metadata.yml
+uses:
+  mysql-db:
+    constraint: optional
+    auto: pair
+    default-options:
+       label: from-default-options
+       label2: from-default-options
+EOF2
+
+touch $test_tmpdir/mysql/metadata.yml
+
+cat <<EOF2 > $test_tmpdir/compose.yml
+www:
+  charm: www
+  relations:
+    mysql-db:
+      mysql:
+        label: value
+EOF2
+
+. "$tprog"
+
+_setup_state_dir
+COMPOSE_YML_FILE=$test_tmpdir/compose.yml
+
+out=\$(set -o pipefail
+  get_all_relations www mysql | tr '\0' '\n') || exit 3
+expected="www
+mysql-db
+mysql
+label: value
+label2: from-default-options
+True"
+
+[[ "\$out" == "\$expected" ]] || {
+    echo "doesn't end with: \$expected" >&2
+    echo "\$out"
+    exit 1
+}
+
+
+
+## -- simple case, 1 uses, auto-pair, not present, no provider
+
+export CHARM_STORE=$test_tmpdir
+mkdir -p $test_tmpdir/{www,mysql}
+cat <<EOF2 > $test_tmpdir/www/metadata.yml
+uses:
+  mysql-db:
+    constraint: optional
+    auto: pair
+    default-options:
+       label: from-default-options
+       label2: from-default-options
+EOF2
+
+touch $test_tmpdir/mysql/metadata.yml
+
+cat <<EOF2 > $test_tmpdir/compose.yml
+www:
+  charm: www
+EOF2
+
+. "$tprog"
+
+_setup_state_dir
+COMPOSE_YML_FILE=$test_tmpdir/compose.yml
+
+out=\$(set -o pipefail
+  get_all_relations www mysql | tr '\0' '\n') || exit 3
+expected=""
+
+[[ "\$out" == "\$expected" ]] || {
+    echo "doesn't end with: \$expected" >&2
+    echo "\$out"
+    exit 1
+}
+
+
+## -- simple case, 1 uses, auto-pair, not present, 1 provider
+
+export CHARM_STORE=$test_tmpdir
+mkdir -p $test_tmpdir/{www,mysql}
+cat <<EOF2 > $test_tmpdir/www/metadata.yml
+uses:
+  mysql-db:
+    constraint: optional
+    auto: pair
+    default-options:
+       label: from-default-options
+       label2: from-default-options
+EOF2
+
+cat <<EOF2 > $test_tmpdir/mysql/metadata.yml
+provides:
+  mysql-db:
+EOF2
+
+cat <<EOF2 > $test_tmpdir/compose.yml
+www:
+  charm: www
+EOF2
+
+. "$tprog"
+
+_setup_state_dir
+COMPOSE_YML_FILE=$test_tmpdir/compose.yml
+
+out=\$(set -o pipefail
+  get_all_relations www mysql | tr '\0' '\n') || exit 3
+expected="www
+mysql-db
+mysql
+label: from-default-options
+label2: from-default-options
+True"
+
+[[ "\$out" == "\$expected" ]] || {
+    echo "--- EXPECTED" >&2
+    echo "\$expected" >&2
+    echo "--- OUT" >&2
+    echo "\$out" >&2
+
+    exit 1
+}
+
+
+## -- simple case, 1 uses, auto-pair, not present, 2 providers
+
+export CHARM_STORE=$test_tmpdir
+mkdir -p $test_tmpdir/{www,mysql,mysql2}
+cat <<EOF2 > $test_tmpdir/www/metadata.yml
+uses:
+  mysql-db:
+    constraint: optional
+    auto: pair
+    default-options:
+       label: from-default-options
+       label2: from-default-options
+EOF2
+
+cat <<EOF2 > $test_tmpdir/mysql/metadata.yml
+provides:
+  mysql-db:
+EOF2
+
+cat <<EOF2 > $test_tmpdir/mysql2/metadata.yml
+provides:
+  mysql-db:
+EOF2
+
+cat <<EOF2 > $test_tmpdir/compose.yml
+www:
+  charm: www
+EOF2
+
+. "$tprog"
+
+_setup_state_dir
+COMPOSE_YML_FILE=$test_tmpdir/compose.yml
+
+out=\$(set -o pipefail
+  get_all_relations www mysql mysql2 2>&1 >/dev/null) || exit 3
+
+expected="> 1"
+
+[[ "\$out" == *"\$expected"* ]] || {
+    echo "doesn't contain: \$expected" >&2
+    echo "\$out"
+    exit 1
+}
+
+# Remember, it is cached
+out=\$(set -o pipefail
+  get_all_relations www mysql mysql2 | tr '\0' '\n') || exit 3
+expected=""
+
+[[ "\$out" == "\$expected" ]] || {
+    echo "doesn't end with: \$expected" >&2
+    echo "\$out"
+    exit 1
+}
+
+
+EOF
+
+    tear_test
+}
+
+
+
+function test_all_relations_missing {
+
+    init_test
+
+    export DISABLE_SYSTEM_CONFIG_FILE=true
+
+    assert_list <<EOF
+
+### Testing missing services at first call
+
+## -- 1 service, connection to another, no use/provide
+
+export CHARM_STORE=$test_tmpdir
+mkdir -p $test_tmpdir/{www,mysql}
+cat <<EOF2 > $test_tmpdir/www/metadata.yml
+EOF2
+
+cat <<EOF2 > $test_tmpdir/mysql/metadata.yml
+EOF2
+
+cat <<EOF2 > $test_tmpdir/compose.yml
+www:
+  relations:
+    mysql-db: mysql
+EOF2
+
+. "$tprog"
+
+_setup_state_dir
+COMPOSE_YML_FILE=$test_tmpdir/compose.yml
+
+out=\$(set -o pipefail
+  get_all_relations www | tr '\0' '\n') || exit 3
+expected="www
+mysql-db
+mysql
+
+True"
+
+[[ "\$out" == "\$expected" ]] || {
+    echo "doesn't end with: \$expected" >&2
+    echo "OUT:" >&2
+    echo "\$out" >&2
+    exit 1
+}
+
+
+## -- 1 service, connection to another that auto-pair with first
+
+export CHARM_STORE=$test_tmpdir
+mkdir -p $test_tmpdir/{www,mysql}
+cat <<EOF2 > $test_tmpdir/www/metadata.yml
+provides:
+  www-proxy:
+EOF2
+
+cat <<EOF2 > $test_tmpdir/mysql/metadata.yml
+uses:
+  www-proxy:
+    constraint: optional
+    auto: pair
+EOF2
+
+cat <<EOF2 > $test_tmpdir/compose.yml
+www:
+  relations:
+    mysql-db: mysql
+EOF2
+
+. "$tprog"
+
+_setup_state_dir
+COMPOSE_YML_FILE=$test_tmpdir/compose.yml
+
+out=\$(set -o pipefail
+  get_all_relations www | tr '\0' '\n') || exit 3
+expected="\
+www
+mysql-db
+mysql
+
+True
+mysql
+www-proxy
+www
+
+True"
+
+[[ "\$out" == "\$expected" ]] || {
+    echo "DIFF:" >&2
+    diff -u <(echo "\$expected") <(echo "\$out") >&2
+    exit 1
+}
+
+
+
+
+
+EOF
+
+    tear_test
+}
+
+
+
+
+
+
+continue_on_error="0" testbench $*

test/uses

@@ -0,0 +1,200 @@
+#!/usr/bin/env bash-shlib
+# -*- mode: shell-script -*-
+
+include shunit
+
+depends sed grep git mkdir readlink
+
+export -f matches
+export grep
+
+tmp=/tmp
+tprog="../bin/compose-core"
+tprog=$(readlink -f $tprog)
+
+
+export PATH=".:$PATH"
+short_tprog=$(basename "$tprog")
+
+
+##
+## Convenience function
+##
+
+init_test() {
+    test_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX)
+    cd "$test_tmpdir"
+    export CACHEDIR="$test_tmpdir/.cache"
+    export VARDIR="$test_tmpdir/.var"
+    mkdir -p "$CACHEDIR"
+}
+
+
+
+tear_test() {
+    rm -rf "$test_tmpdir"
+}
+
+
+##
+## Tests
+##
+
+
+function test_uses {
+
+    init_test
+
+    export CHARM_STORE=$test_tmpdir
+    mkdir -p  $test_tmpdir/www/actions
+    cat <<EOF2 > $test_tmpdir/www/metadata.yml
+docker-image: bar   ## required as we want relation to know the base image
+uses:
+  myrelation: myrelationdef
+EOF2
+
+    cat <<EOF2 > $test_tmpdir/compose.yml
+web_site:
+  charm: www
+EOF2
+
+    export DISABLE_SYSTEM_CONFIG_FILE=true
+
+    assert_list <<EOF
+
+### Testing _get_services_uses
+
+## -- simple case with only one service with one relation
+
+cd "$test_tmpdir"
+. "$tprog" || exit 1
+
+_setup_state_dir
+
+
+out=\$(_get_services_uses web_site | tr '\0' ':')
+expected="web_site:myrelation:myrelationdef:"
+
+[[ "\$out" == "\$expected" ]] || {
+    echo "doesn't end with: \$expected" >&2
+    echo "\$out"
+    exit 1
+}
+
+## -- service not defined in compose, but has charm
+
+cd "$test_tmpdir"
+. "$tprog" || exit 1
+
+_setup_state_dir
+
+out=\$(_get_services_uses www | tr '\0' ':') || exit 2
+expected="www:myrelation:myrelationdef:"
+
+[[ "\$out" == "\$expected" ]] || {
+    echo "doesn't end with: \$expected" >&2
+    echo "\$out"
+    exit 1
+}
+
+
+## -- service not defined in compose, nor has charm should fail
+
+cd "$test_tmpdir"
+. "$tprog" || exit 1
+
+_setup_state_dir
+
+_get_services_uses xxx || exit 0
+
+echo "Expected it to fail"
+exit 1
+
+EOF
+
+    tear_test
+}
+
+
+function test_provides {
+
+    init_test
+
+    export CHARM_STORE=$test_tmpdir
+    mkdir -p  $test_tmpdir/www/actions
+    cat <<EOF2 > $test_tmpdir/www/metadata.yml
+docker-image: bar   ## required as we want relation to know the base image
+provides:
+  myrelation: myrelationdef
+EOF2
+
+    cat <<EOF2 > $test_tmpdir/compose.yml
+web_site:
+  charm: www
+EOF2
+
+    export DISABLE_SYSTEM_CONFIG_FILE=true
+
+    assert_list <<EOF
+
+### Testing _get_services_provides
+
+## -- simple case with only one service with one relation
+
+cd "$test_tmpdir"
+. "$tprog" || exit 1
+
+_setup_state_dir
+
+
+out=\$(_get_services_provides web_site | tr '\0' ':')
+expected="web_site:myrelation:myrelationdef:"
+
+[[ "\$out" == "\$expected" ]] || {
+    echo "doesn't end with: \$expected" >&2
+    echo "\$out"
+    exit 1
+}
+
+
+## -- service not defined in compose, but has charm
+
+cd "$test_tmpdir"
+. "$tprog" || exit 1
+
+_setup_state_dir
+
+
+out=\$(_get_services_provides www | tr '\0' ':')
+expected="www:myrelation:myrelationdef:"
+
+[[ "\$out" == "\$expected" ]] || {
+    echo "doesn't end with: \$expected" >&2
+    echo "\$out"
+    exit 1
+}
+
+
+## -- service not defined in compose, nor has charm should fail
+
+cd "$test_tmpdir"
+. "$tprog" || exit 1
+
+_setup_state_dir
+
+out=\$(set -o pipefail
+  _get_provides_uses xxx | tr '\0' :) || exit 0
+
+echo "Expected it to fail"
+exit 1
+
+
+EOF
+
+    tear_test
+}
+
+
+
+
+continue_on_error="0" testbench $*