You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

4228 lines
139 KiB

  1. #!/bin/bash
  2. ##
  3. ## TODO:
  4. ## - subordinate container should really be able to modify base image of their master
  5. ## - this could be done through docker-update
  6. ## - I'm not happy with the current build using 'build/' directory, this should be
  7. ## changed to:
  8. ## - always have a base image (specified in metadata), and always have hooks/install
  9. ## executed and merge in image (like docker-build-charm).
  10. ## - container base image is ALWAYS the image of the master container... this brings
  11. ## questions about a double way to express inheritage (through relations as it is
  12. ## implemented now, or through this base-image ?)
  13. ## - the name of the scripts for relation (aka relation_name-relation-joined) is bad as
  14. ## reading the name in a hooks/ dir, there are no way to know if we are the target or
  15. ## the base of the relation.
  16. ## - we could leverage a 'relations/' dir on the root of the charm, with both:
  17. ## 'relations/provide/relation_name' and 'relations/receive/relation_name'
  18. ## - a very bad point with the actual naming is that we can't have a providing AND
  19. ## receiving a relation with same name.
  20. ## - The cache system should keep md5 of docker-compose and other things between runs
  21. ## - The cache system should use underlying function that have only arguments inputs.
  22. ## This will allow to cache completely without issues function in time.
  23. ## - would probably need instrospection in charm custom action to know if these need
  24. ## init or relations to be set up.
  25. ## - Be clear about when the SERVICE name is used and the CHARM name is used.
  26. ## - in case of service contained in another container
  27. ## - in normal case
  28. ## - in docker-compose, can't use charm name: if we want 2 instances of the same charm
  29. ## we are stuck. What will be unique is the name of the service.
  30. ## - some relations are configured in compose.yml but should not trigger the loading
  31. ## of necessary component (for instance, apache --> log-rotate), if log-rotate is
  32. ## not there, this link should considered optional.
  33. ## - Could probably allow an unexistent charm to be populated with only "docker-image:"
  34. ## of the same name. Although this should trigger a visible warning.
  35. #:-
  36. [ -e /etc/shlib ] && . /etc/shlib || {
  37. echo "Unsatisfied dependency. Please install 'kal-shlib-core'."
  38. exit 1
  39. }
  40. #:-
  41. include common
  42. include pretty
  43. include parse
  44. include charm
  45. include array
  46. include cla
  47. depends shyaml docker
  48. exname="compose"
  49. version=0.1
  50. usage="$exname [COMPOSE_OPTS] [ACTION [ACTION_OPTS]]"
  51. help="\
  52. $WHITE$exname$NORMAL jobs is to run various shell scripts to build
  53. a running orchestrated and configured docker containers. These shell
  54. scripts will have the opportunity to build a 'docker-compose.yml'.
  55. Once init script and relations scripts are executed, $WHITE$exname$NORMAL
  56. delegate the launching to ${WHITE}docker-compose${NORMAL} by providing it
  57. the final 'docker-compose.yml'.
  58. $WHITE$exname$NORMAL also leverage charms to offer some additional custom
  59. actions per charm, which are simply other scripts that can be
  60. run without launching ${WHITE}docker-compose${NORMAL}.
  61. In compose message, color coding is enforced as such:
  62. - ${DARKCYAN}action$NORMAL,
  63. - ${DARKBLUE}relation$NORMAL,
  64. - ${DARKPINK}charm${NORMAL},
  65. - ${DARKYELLOW}service${NORMAL},
  66. - ${WHITE}option-name${NORMAL}/${WHITE}command-name${NORMAL}/${WHITE}Section-Title${NORMAL}
  67. $WHITE$exname$NORMAL reads '/etc/compose.conf' for global variables, and
  68. '/etc/compose.local.conf' for local host adjustements.
  69. "
  70. ## XXXvlab: this doesn't seem to work when 'compose' is called in
  71. ## a hook of a charm.
  72. #[[ "${BASH_SOURCE[0]}" == "" ]] && SOURCED=true
  73. $(return >/dev/null 2>&1) && SOURCED=true
  74. if [ "$UID" == 0 ]; then
  75. CACHEDIR=${CACHEDIR:-/var/cache/compose}
  76. VARDIR=${VARDIR:-/var/lib/compose}
  77. else
  78. [ "$XDG_CONFIG_HOME" ] && CACHEDIR=${CACHEDIR:-$XDG_CONFIG_HOME/compose}
  79. [ "$XDG_DATA_HOME" ] && VARDIR=${VARDIR:-$XDG_DATA_HOME/compose}
  80. CACHEDIR=${CACHEDIR:-$HOME/.cache/compose}
  81. VARDIR=${VARDIR:-$HOME/.local/share/compose}
  82. fi
  83. export VARDIR CACHEDIR
  84. md5_compat() { md5sum | cut -c -32; }
  85. quick_cat_file() { quick_cat_stdin < "$1"; }
  86. quick_cat_stdin() { local IFS=''; while read -r line; do echo "$line"; done ; }
  87. export -f quick_cat_file quick_cat_stdin md5_compat
  88. clean_cache() {
  89. local i=0
  90. for f in $(ls -t "$CACHEDIR/"*.cache.* 2>/dev/null | tail -n +500); do
  91. ((i++))
  92. rm -f "$f"
  93. done
  94. if (( i > 0 )); then
  95. debug "${WHITE}Cleaned cache:${NORMAL} Removed $((i)) elements (current cache size is $(du -sh "$CACHEDIR" | cut -f 1))"
  96. fi
  97. }
  98. export DEFAULT_COMPOSE_FILE
  99. ##
  100. ## Merge YAML files
  101. ##
  102. export _merge_yaml_common_code="
  103. import sys
  104. import yaml
  105. try:
  106. from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
  107. except ImportError: ## pragma: no cover
  108. sys.stderr.write('YAML code in pure python\n')
  109. exit(1)
  110. from yaml import SafeLoader, SafeDumper
  111. class MySafeLoader(SafeLoader): pass
  112. class MySafeDumper(SafeDumper): pass
  113. try:
  114. # included in standard lib from Python 2.7
  115. from collections import OrderedDict
  116. except ImportError:
  117. # try importing the backported drop-in replacement
  118. # it's available on PyPI
  119. from ordereddict import OrderedDict
  120. ## Ensure that there are no collision with legacy OrderedDict
  121. ## that could be used for omap for instance.
  122. class MyOrderedDict(OrderedDict):
  123. pass
  124. MySafeDumper.add_representer(
  125. MyOrderedDict,
  126. lambda cls, data: cls.represent_dict(data.items()))
  127. def construct_omap(cls, node):
  128. cls.flatten_mapping(node)
  129. return MyOrderedDict(cls.construct_pairs(node))
  130. MySafeLoader.add_constructor(
  131. yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
  132. construct_omap)
  133. ##
  134. ## Support local and global objects
  135. ##
  136. class EncapsulatedNode(object): pass
  137. def mk_encapsulated_node(s, node):
  138. method = 'construct_%s' % (node.id, )
  139. data = getattr(s, method)(node)
  140. class _E(data.__class__, EncapsulatedNode):
  141. pass
  142. _E.__name__ = str(node.tag)
  143. _E._node = node
  144. return _E(data)
  145. def represent_encapsulated_node(s, o):
  146. value = s.represent_data(o.__class__.__bases__[0](o))
  147. value.tag = o.__class__.__name__
  148. return value
  149. MySafeDumper.add_multi_representer(EncapsulatedNode,
  150. represent_encapsulated_node)
  151. MySafeLoader.add_constructor(None, mk_encapsulated_node)
  152. def fc(filename):
  153. with open(filename) as f:
  154. return f.read()
  155. def merge(*args):
  156. # sys.stderr.write('%r\n' % (args, ))
  157. args = [arg for arg in args if arg is not None]
  158. if len(args) == 0:
  159. return None
  160. if len(args) == 1:
  161. return args[0]
  162. if all(isinstance(arg, (int, basestring, bool, float)) for arg in args):
  163. return args[-1]
  164. elif all(isinstance(arg, list) for arg in args):
  165. res = []
  166. for arg in args:
  167. for elt in arg:
  168. if elt in res:
  169. res.remove(elt)
  170. res.append(elt)
  171. return res
  172. elif all(isinstance(arg, dict) for arg in args):
  173. keys = set()
  174. for arg in args:
  175. keys |= set(arg.keys())
  176. dct = {}
  177. for key in keys:
  178. sub_args = []
  179. for arg in args:
  180. if key in arg:
  181. sub_args.append(arg)
  182. try:
  183. dct[key] = merge(*(a[key] for a in sub_args))
  184. except NotImplementedError as e:
  185. raise NotImplementedError(
  186. e.args[0],
  187. '%s.%s' % (key, e.args[1]) if e.args[1] else key,
  188. e.args[2])
  189. if dct[key] is None:
  190. del dct[key]
  191. return dct
  192. else:
  193. raise NotImplementedError(
  194. 'Unsupported types: %s'
  195. % (', '.join(list(set(arg.__class__.__name__ for arg in args)))), '', args)
  196. return None
  197. def merge_cli(*args):
  198. try:
  199. c = merge(*args)
  200. except NotImplementedError as e:
  201. sys.stderr.write('Merging Failed: %s.\n%s\n'
  202. ' Values are:\n %s\n'
  203. % (e.args[0],
  204. ' Conflicting key is %r.' % e.args[1] if e.args[1] else
  205. ' Conflict at base of structure.',
  206. '\\n '.join('v%d: %r' % (i, a)
  207. for i, a in enumerate(e.args[2]))))
  208. exit(1)
  209. if c is not None:
  210. print '%s' % yaml.dump(c, default_flow_style=False, Dumper=MySafeDumper)
  211. "
  212. merge_yaml() {
  213. if ! [ -r "$state_tmpdir/merge_yaml.py" ]; then
  214. cat <<EOF > "$state_tmpdir/merge_yaml.py"
  215. $_merge_yaml_common_code
  216. merge_cli(*(yaml.load(fc(f), Loader=MySafeLoader) for f in sys.argv[1:]))
  217. EOF
  218. fi
  219. python "$state_tmpdir/merge_yaml.py" "$@"
  220. }
  221. export -f merge_yaml
  222. merge_yaml_str() {
  223. local entries="$@"
  224. if ! [ -r "$state_tmpdir/merge_yaml_str.py" ]; then
  225. cat <<EOF > "$state_tmpdir/merge_yaml_str.py" || return 1
  226. $_merge_yaml_common_code
  227. merge_cli(*(yaml.load(f, Loader=MySafeLoader) for f in sys.argv[1:]))
  228. EOF
  229. fi
  230. if ! python "$state_tmpdir/merge_yaml_str.py" "$@"; then
  231. err "Failed to merge yaml strings:"
  232. local s
  233. for s in "$@"; do
  234. printf " - \n"
  235. printf "%s\n" "$s" | prefix " ${GRAY}|$NORMAL "
  236. done >&2
  237. return 1
  238. fi
  239. }
  240. export -f merge_yaml_str
  241. yaml_get_values() {
  242. local sep=${1:-$'\n'} value input type first elt
  243. input=$(cat -)
  244. if [ -z "$input" ] || [ "$input" == "None" ]; then
  245. return 0
  246. fi
  247. type=$(e "$input" | shyaml get-type)
  248. value=
  249. case "$type" in
  250. "sequence")
  251. first=1
  252. while read-0 elt; do
  253. elt="$(e "$elt" | yaml_get_interpret)" || return 1
  254. [ "$elt" ] || continue
  255. if [ "$first" ]; then
  256. first=
  257. else
  258. value+="$sep"
  259. fi
  260. first=
  261. value+="$elt"
  262. done < <(e "$input" | shyaml -y get-values-0)
  263. ;;
  264. "struct")
  265. while read-0 val; do
  266. value+=$'\n'"$(e "$val" | yaml_get_interpret)" || return 1
  267. done < <(e "$input" | shyaml -y values-0)
  268. ;;
  269. "NoneType")
  270. value=""
  271. ;;
  272. "str"|*)
  273. value+="$(e "$input" | yaml_get_interpret)"
  274. ;;
  275. esac
  276. e "$value"
  277. }
  278. export -f yaml_get_values
  279. yaml_key_val_str() {
  280. local entries="$@"
  281. if ! [ -r "$state_tmpdir/yaml_key_val_str.py" ]; then
  282. cat <<EOF > "$state_tmpdir/yaml_key_val_str.py"
  283. $_merge_yaml_common_code
  284. print '%s' % yaml.dump(
  285. {
  286. yaml.load(sys.argv[1], Loader=MySafeLoader):
  287. yaml.load(sys.argv[2], Loader=MySafeLoader)
  288. },
  289. default_flow_style=False,
  290. Dumper=MySafeDumper,
  291. )
  292. EOF
  293. fi
  294. python "$state_tmpdir/yaml_key_val_str.py" "$@"
  295. }
  296. export -f yaml_key_val_str
  297. ##
  298. ## Docker
  299. ##
  300. docker_has_image() {
  301. local image="$1"
  302. images=$(docker images -q "$image" 2>/dev/null) || {
  303. err "docker images call has failed unexpectedly."
  304. return 1
  305. }
  306. [ "$images" ]
  307. }
  308. export -f docker_has_image
  309. docker_image_id() {
  310. local image="$1"
  311. image_id=$(docker inspect "$image" --format='{{.Id}}') || return 1
  312. echo "$image_id" # | tee "$cache_file"
  313. }
  314. export -f docker_image_id
  315. cached_cmd_on_image() {
  316. local image="$1" cache_file
  317. image_id=$(docker_image_id "$image") || return 1
  318. cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  319. if [ -e "$cache_file" ]; then
  320. # debug "$FUNCNAME: cache hit ($*)"
  321. quick_cat_stdin < "$cache_file"
  322. return 0
  323. fi
  324. shift
  325. out=$(docker run -i --rm --entrypoint /bin/sh "$image_id" -c "$*") || return 1
  326. echo "$out" | tee "$cache_file"
  327. }
  328. export -f cached_cmd_on_image
  329. cmd_on_base_image() {
  330. local service="$1" base_image
  331. shift
  332. base_image=$(service_base_docker_image "$service") || return 1
  333. docker run -i --rm --entrypoint /bin/bash "$base_image" -c "$*"
  334. }
  335. export -f cmd_on_base_image
  336. cached_cmd_on_base_image() {
  337. local service="$1" base_image cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  338. shift
  339. if [ -e "$cache_file" ]; then
  340. # debug "$FUNCNAME: cache hit ($*)"
  341. quick_cat_stdin < "$cache_file"
  342. return 0
  343. fi
  344. base_image=$(service_base_docker_image "$service") || return 1
  345. if ! docker_has_image "$base_image"; then
  346. docker pull "$base_image" 1>&2
  347. fi
  348. result=$(cached_cmd_on_image "$base_image" "$@") || return 1
  349. echo "$result" | tee "$cache_file"
  350. }
  351. export -f cached_cmd_on_base_image
  352. docker_update() {
  353. ## YYY: warning, we a storing important information in cache, cache can
  354. ## be removed.
  355. ## We want here to cache the last script on given service whatever that script was
  356. local service="$1" script="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$1" \
  357. previous_base_image stored_image_id
  358. shift
  359. shift
  360. ## this will build it if necessary
  361. base_image=$(service_base_docker_image "$service") || return 1
  362. ## XXXvlab: there are probably ways to avoid rebuilding that each time
  363. image_id="$(docker_image_id "$base_image")" || return 1
  364. if [ -e "$cache_file" ]; then
  365. info "Cache file exists"
  366. read-0 previous_base_image stored_image_id < <(cat "$cache_file")
  367. info "previous: $previous_base_image"
  368. info "stored: $stored_image_id"
  369. else
  370. info "No cache file $cache_file"
  371. previous_base_image=""
  372. fi
  373. if [ "$previous_base_image" -a "$stored_image_id" == "$image_id" ]; then
  374. info "Resetting $base_image to $previous_base_image"
  375. docker tag "$previous_base_image" "$base_image" || return 1
  376. image_id="$(docker_image_id "$base_image")" || return 1
  377. else
  378. previous_base_image="$image_id"
  379. fi
  380. info "Updating base image: $base_image (hash: $image_id)"
  381. echo "$script" | dupd --debug -u "$base_image" -- "$@" || {
  382. err "Failed updating base image"
  383. return 1
  384. }
  385. new_image_id="$(docker_image_id "$base_image")"
  386. [ "$new_image_id" == "$previous_base_image" ] && {
  387. err "Image was not updated correctly (same id)."
  388. return 1
  389. }
  390. printf "%s\0" "$previous_base_image" "$new_image_id" > "$cache_file"
  391. info "Wrote cache file $cache_file"
  392. }
  393. export -f docker_update
  394. image_exposed_ports_0() {
  395. local image="$1"
  396. docker inspect --format='{{range $p, $conf := .Config.ExposedPorts}}{{$p}}{{"\x00"}}{{end}}' "$image"
  397. }
  398. export -f image_exposed_ports_0
  399. ## feature not yet included in docker: https://github.com/moby/moby/issues/16079
  400. docker_image_export_dir() {
  401. local image="$1" src="$2" dst="$3" container_id
  402. (
  403. container_id=$(docker create "$image") || exit 1
  404. trap_add EXIT,ERR "docker rm \"$container_id\" >/dev/null"
  405. docker cp "$container_id":"$src" "$dst"
  406. )
  407. }
  408. export -f docker_image_export_dir
  409. service_base_image_export_dir() {
  410. local service="$1" src="$2" dst="$3" base_image
  411. shift
  412. base_image=$(service_base_docker_image "$service") || return 1
  413. if ! docker_has_image "$base_image"; then
  414. docker pull "$base_image"
  415. fi
  416. docker_image_export_dir "$base_image" "$src" "$dst"
  417. }
  418. export -f service_base_image_export_dir
  419. ##
  420. ## Generic
  421. ##
  422. fn.exists() {
  423. declare -F "$1" >/dev/null
  424. }
  425. str_pattern_matches() {
  426. local str="$1"
  427. shift
  428. for pattern in "$@"; do
  429. eval "[[ \"$str\" == $pattern ]]" && return 0
  430. done
  431. return 1
  432. }
  433. str_matches() {
  434. local str="$1"
  435. shift
  436. for pattern in "$@"; do
  437. [[ "$str" == "$pattern" ]] && return 0
  438. done
  439. return 1
  440. }
  441. gen_password() {
  442. local l=( {a..z} {A..Z} {0..9} ) nl="${#l[@]}" size=${1:-16}
  443. while ((size--)); do
  444. echo -n "${l[$((RANDOM * nl / 32768))]}"
  445. done
  446. echo
  447. }
  448. export -f gen_password
  449. file_put() {
  450. local TARGET="$1"
  451. mkdir -p "$(dirname "$TARGET")" &&
  452. cat - > "$TARGET"
  453. }
  454. export -f file_put
  455. file_put_0() {
  456. local TARGET="$1"
  457. mkdir -p "$(dirname "$TARGET")" &&
  458. cat > "$TARGET"
  459. }
  460. export -f file_put_0
  461. fetch_file() {
  462. local src="$1"
  463. case "$src" in
  464. *"://"*)
  465. err "Unsupported target scheme."
  466. return 1
  467. ;;
  468. *)
  469. ## Try direct
  470. if ! [ -r "$src" ]; then
  471. err "File '$src' not found/readable."
  472. return 1
  473. fi
  474. cat "$src" || return 1
  475. ;;
  476. esac
  477. }
  478. export -f fetch_file
  479. ## receives stdin content to decompress on stdout
  480. ## stdout content should be tar format.
  481. uncompress_file() {
  482. local filename="$1"
  483. ## Warning, the content of the file is already as stdin, the filename
  484. ## is there to hint for correct decompression.
  485. case "$filename" in
  486. *".gz")
  487. gunzip
  488. ;;
  489. *".bz2")
  490. bunzip2
  491. ;;
  492. *)
  493. cat
  494. ;;
  495. esac
  496. }
  497. export -f uncompress_file
  498. get_file() {
  499. local src="$1"
  500. fetch_file "$src" | uncompress_file "$src"
  501. }
  502. export -f get_file
  503. ##
  504. ## Common database lib
  505. ##
  506. _clean_docker() {
  507. local _DB_NAME="$1" container_id="$2"
  508. (
  509. set +e
  510. debug "Removing container $_DB_NAME"
  511. docker stop "$container_id"
  512. docker rm "$_DB_NAME"
  513. docker network rm "${_DB_NAME}"
  514. rm -vf "$state_tmpdir/${_DB_NAME}.state"
  515. )
  516. }
  517. export -f _clean_docker
  518. get_service_base_image_dir_uid_gid() {
  519. local service="$1" dir="$2" uid_gid
  520. uid_gid=$(cached_cmd_on_base_image "$service" "stat -c '%u %g' '$dir'") || {
  521. debug "Failed to query '$dir' uid in ${DARKYELLOW}$service${NORMAL} base image."
  522. return 1
  523. }
  524. info "uid and gid from ${DARKYELLOW}$service${NORMAL}:$dir is '$uid_gid'"
  525. echo "$uid_gid"
  526. }
  527. export -f get_service_base_image_dir_uid_gid
  528. get_service_type() {
  529. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  530. if [ -z "$service" ]; then
  531. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  532. return 1
  533. fi
  534. if [ -e "$cache_file" ]; then
  535. # debug "$FUNCNAME: cache hit ($*)"
  536. cat "$cache_file"
  537. return 0
  538. fi
  539. charm=$(get_service_charm "$service") || return 1
  540. metadata=$(charm.metadata "$charm") || return 1
  541. printf "%s" "$metadata" | shyaml get-value type service 2>/dev/null |
  542. tee "$cache_file"
  543. }
  544. are_files_locked_in_dir() {
  545. local dir="$1" device hdev ldev
  546. device=$(stat -c %d "$dir") || {
  547. err "Can't stat '$dir'."
  548. return 1
  549. }
  550. device=$(printf "%04x" $device)
  551. hdev=${device:0:2}
  552. ldev=${device:2:2}
  553. inodes=$(find "$dir" -printf ':%i:\n')
  554. found=
  555. while read -r inode; do
  556. debug "try inode:$inode"
  557. if [[ "$inodes" == *":$inode:"* ]]; then
  558. found=1
  559. break
  560. fi
  561. done < <(cat /proc/locks | grep " $hdev:$ldev:" | sed -r "s/^.*$hdev:$ldev:([0-9]+).*$/\1/g")
  562. [ "$found" ]
  563. }
  564. export -f are_files_locked_in_dir
  565. export _PID="$$"
  566. ensure_db_docker_running () {
  567. local _STATE_FILE
  568. _DB_NAME="db_${DB_NAME}_${_PID}"
  569. _STATE_FILE="$state_tmpdir/${_DB_NAME}.state"
  570. if [ -e "$_STATE_FILE" ]; then
  571. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$(cat "$_STATE_FILE")"
  572. debug "Re-using previous docker/connection '$DOCKER_IP'."
  573. _set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  574. return 0
  575. fi
  576. if [ -e "$state_tmpdir/${_DB_NAME}.working" ]; then
  577. ## avoid recursive calls.
  578. if [ -z "$DOCKER_IP" ]; then
  579. err "Currently figuring up DOCKER_IP, please set it yourself before this call if needed."
  580. return 1
  581. else
  582. debug "ignoring recursive call of 'ensure_db_docker_running'."
  583. fi
  584. return 0
  585. fi
  586. touch "$state_tmpdir/${_DB_NAME}.working"
  587. docker rm "$_DB_NAME" 2>/dev/null || true
  588. host_db_working_dir="$HOST_DATASTORE/${SERVICE_NAME}$DB_DATADIR"
  589. if is_db_locked; then
  590. info "Some process is using '$host_db_working_dir'. Trying to find a docker that would do this..."
  591. found=
  592. for docker_id in $(docker ps -q); do
  593. has_volume_mounted=$(
  594. docker inspect \
  595. --format "{{range .Mounts}}{{if eq .Destination \"$DB_DATADIR\"}}{{.Source}}{{end}}{{end}}" \
  596. "$docker_id")
  597. if [ "$has_volume_mounted" == "$host_db_working_dir" ]; then
  598. found="$docker_id"
  599. break
  600. fi
  601. done
  602. if [ -z "$found" ]; then
  603. err "Please shutdown any other docker using this directory."
  604. return 1
  605. fi
  606. export container_id="$found"
  607. info "Found docker $docker_id is already running."
  608. else
  609. verb "Database is not locked."
  610. if ! docker_has_image "$DOCKER_BASE_IMAGE"; then
  611. docker pull "$DOCKER_BASE_IMAGE"
  612. fi
  613. _set_server_db_params
  614. debug docker network create "$_DB_NAME"
  615. if ! network_id=$(docker network create "$_DB_NAME"); then
  616. err "'docker network create $_DB_NAME' failed !"
  617. _clean_docker "$_DB_NAME" "$container_id"
  618. rm "$state_tmpdir/${_DB_NAME}.working"
  619. return 1
  620. fi
  621. debug docker run -d \
  622. --name "$_DB_NAME" \
  623. "${server_docker_opts[@]}" \
  624. --network "$_DB_NAME" \
  625. -v "$host_db_working_dir:$DB_DATADIR" \
  626. "$DOCKER_BASE_IMAGE"
  627. if ! container_id=$(
  628. docker run -d \
  629. --name "$_DB_NAME" \
  630. "${server_docker_opts[@]}" \
  631. --network "$_DB_NAME" \
  632. -v "$host_db_working_dir:$DB_DATADIR" \
  633. "$DOCKER_BASE_IMAGE"
  634. ); then
  635. err "'docker run' failed !"
  636. _clean_docker "$_DB_NAME" "$container_id"
  637. rm "$state_tmpdir/${_DB_NAME}.working"
  638. return 1
  639. fi
  640. trap_add EXIT,ERR "_clean_docker \"$_DB_NAME\" \"$container_id\""
  641. fi
  642. if docker_ip=$(wait_for_docker_ip "$container_id"); then
  643. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip"
  644. echo "$docker_ip" > "$_STATE_FILE"
  645. debug "written '$_STATE_FILE'"
  646. rm "$state_tmpdir/${_DB_NAME}.working"
  647. _set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  648. return 0
  649. else
  650. errlvl="$?"
  651. err "Db not found (errlvl: $errlvl). Tail of docker logs follows:"
  652. docker logs --tail=5 "$container_id" 2>&1 | prefix " | " >&2
  653. rm "$state_tmpdir/${_DB_NAME}.working"
  654. return "$errlvl"
  655. fi
  656. }
  657. export -f ensure_db_docker_running
  658. ## Require to set $db_docker_opts if needed, and $DB_PASSFILE
  659. ##
  660. _dcmd() {
  661. local docker_opts command="$1"
  662. shift
  663. debug "Db> $command $@"
  664. if [ "$HOST_DB_PASSFILE" -a -f "$LOCAL_DB_PASSFILE" -a "$CLIENT_DB_PASSFILE" ]; then
  665. verb "Found and using '$HOST_DB_PASSFILE' as '$CLIENT_DB_PASSFILE'."
  666. docker_opts=("${db_docker_opts[@]}" "-v" "$HOST_DB_PASSFILE:$CLIENT_DB_PASSFILE")
  667. else
  668. docker_opts=("${db_docker_opts[@]}")
  669. fi
  670. ## XXXX was here: actualy, we need only connection between this version and the client version
  671. debug docker run -i --rm \
  672. "${docker_opts[@]}" \
  673. --entrypoint "$command" "$DOCKER_BASE_IMAGE" "${db_cmd_opts[@]}" "$@"
  674. docker run -i --rm \
  675. "${docker_opts[@]}" \
  676. --entrypoint "$command" "$DOCKER_BASE_IMAGE" "${db_cmd_opts[@]}" "$@"
  677. }
  678. export -f _dcmd
  679. ## Executes code through db
  680. dcmd() {
  681. local fun
  682. [ "$DB_NAME" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_NAME."
  683. [ "$DB_DATADIR" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_DATADIR."
  684. # [ "$DB_PASSFILE" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_PASSFILE."
  685. [ "$_PID" ] || print_syntax_error "$FUNCNAME: You must provide \$_PID."
  686. for fun in is_db_locked _set_db_params ddb; do
  687. [ "$(type -t "$fun")" == "function" ] ||
  688. print_syntax_error "$FUNCNAME: You must provide function '$fun'."
  689. done
  690. ensure_db_docker_running </dev/null || return 1
  691. _dcmd "$@"
  692. }
  693. export -f dcmd
  694. get_docker_ips() {
  695. local name="$1" ip format network_id
  696. if ! docker inspect --format='{{ .NetworkSettings.Networks }}' "$name" >/dev/null 2>&1; then
  697. echo "default:$(docker inspect --format='{{ .NetworkSettings.IPAdress }}' "$name" 2>/dev/null)"
  698. else
  699. format='{{range $name, $conf := .NetworkSettings.Networks}}{{$name}}{{"\x00"}}{{$conf.IPAddress}}{{"\x00"}}{{end}}'
  700. while read-0 network_id ip; do
  701. printf "%s:%s\n" "$network_id" "$ip"
  702. done < <(docker inspect --format="$format" "$name")
  703. fi
  704. }
  705. export -f get_docker_ips
  706. get_docker_ip() {
  707. local name="$1"
  708. get_docker_ips "$name"
  709. }
  710. export -f get_docker_ip
  711. wait_docker_ip() {
  712. local name="$1" timeout="${2:-15}" timeout_count=0 docker_ip=
  713. start=$SECONDS
  714. while [ -z "$docker_ip" ]; do
  715. sleep 0.5
  716. docker_ip=$(get_docker_ip "$name") && break
  717. elapsed=$((SECONDS - start))
  718. if ((elapsed > timeout)); then
  719. err "${RED}timeout error${NORMAL}(${timeout}s):" \
  720. "Could not find '$name' docker container's IP."
  721. return 1
  722. fi
  723. [ "$elapsed" == "$old_elapsed" ] ||
  724. verb "Waiting for docker $name... ($elapsed/$timeout)"
  725. old_elapsed="$elapsed"
  726. done
  727. verb "Found docker $name network and IP: $docker_ip"
  728. echo "$docker_ip"
  729. }
  730. export -f wait_docker_ip
  731. wait_for_tcp_port() {
  732. local network=$1 host_port=$2 timeout=60
  733. verb "Trying to connect to $host_port"
  734. bash_image=${DEFAULT_BASH_IMAGE:-docker.0k.io/bash}
  735. echo docker run --rm -i --network "$network" "$bash_image" >&2
  736. docker run --rm -i --network "$network" "$bash_image" <<EOF
  737. start=\$SECONDS
  738. while true; do
  739. timeout 1 bash -c "</dev/tcp/${host_port/://}" >/dev/null 2>&1 && break
  740. sleep 0.2
  741. if [ "\$((SECONDS - start))" -gt "$timeout" ]; then
  742. exit 1
  743. fi
  744. done
  745. exit 0
  746. EOF
  747. if [ "$?" != 0 ]; then
  748. err "${RED}timeout error${NORMAL}(${timeout}s):"\
  749. "Could not connect to $host_port."
  750. return 1
  751. fi
  752. return 0
  753. }
  754. export -f wait_for_tcp_port
  755. ## Warning: requires a ``ddb`` matching current database to be checked
  756. wait_for_docker_ip() {
  757. local name=$1 DOCKER_IP= DOCKER_NETWORK= docker_ips= docker_ip= elapsed timeout=10
  758. docker_ip=$(wait_docker_ip "$name" 5) || return 1
  759. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip"
  760. if ! str_is_ipv4 "$DOCKER_IP"; then
  761. err "internal 'wait_docker_ip' did not return a valid IP. Returned IP is '$DOCKER_IP'."
  762. return 1
  763. fi
  764. _set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  765. while read-0 port; do
  766. IFS="/" read port type <<<"$port"
  767. [ "$type" == "tcp" ] || continue
  768. wait_for_tcp_port "$DOCKER_NETWORK" "$DOCKER_IP:${port}" || return 17
  769. info "Host/Port $DOCKER_IP:${port} checked ${GREEN}open${NORMAL}."
  770. ## XXXvlab: what to do with more than one port ?
  771. break
  772. done < <(image_exposed_ports_0 "$container_id")
  773. ## Checking direct connection
  774. timeout=30
  775. start=$SECONDS
  776. while true; do
  777. if err=$(echo "$check_command" | ddb 2>&1 >/dev/null); then
  778. break
  779. fi
  780. if ! [[ "$err" == *"the database system is starting up" ]]; then
  781. err "${RED}db connection error${NORMAL}:" \
  782. "Could not connect to db on $DOCKER_IP container's IP."
  783. echo " Note: IP up, TCP ports is(are) open" >&2
  784. if [ "$err" ]; then
  785. echo " Error:" >&2
  786. printf "%s\n" "$err" | prefix " ${RED}!${NORMAL} " >&2
  787. fi
  788. return 18
  789. fi
  790. debug "Got 'database system is starting up' error."
  791. elapsed=$((SECONDS - start))
  792. if ((elapsed > timeout)); then
  793. err "${RED}db connection error${NORMAL}:"\
  794. "Could not connect to db on $DOCKER_IP" \
  795. "container's IP. (IP up, TCP ports is(are) open, sql answer after ${timeout}s)"
  796. return 1
  797. fi
  798. sleep 0.2
  799. done
  800. echo "${DOCKER_NETWORK}:${DOCKER_IP}"
  801. return 0
  802. }
  803. export -f wait_for_docker_ip
  804. docker_add_host_declaration() {
  805. local src_docker=$1 domain=$2 dst_docker=$3 dst_docker_ip= dst_docker_network
  806. dst_docker_ip=$(wait_docker_ip "$dst_docker") || exit 1
  807. IFS=: read dst_docker_ip dst_docker_network <<<"$dst_docker_ip"
  808. docker exec -i "$src_docker" bash <<EOF
  809. if cat /etc/hosts | grep -E "^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+$domain\$" > /dev/null 2>&1; then
  810. sed -ri "s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+$domain\$/$dst_docker_ip $domain/g" /etc/hosts
  811. else
  812. echo "$dst_docker_ip $domain" >> /etc/hosts
  813. fi
  814. EOF
  815. }
  816. export -f docker_add_host_declaration
  817. get_running_containers_for_service() {
  818. local service="$1"
  819. docker ps --filter label="compose.service=$service" --format="{{.ID}}"
  820. }
  821. export -f get_running_containers_for_service
  822. get_container_network_ips() {
  823. local container="$1"
  824. docker inspect "$container" \
  825. --format='{{range $key, $val :=.NetworkSettings.Networks}}{{$key}}{{"\x00"}}{{$val.IPAddress}}{{"\x00"}}{{end}}'
  826. }
  827. export -f get_container_network_ips
  828. get_container_network_ip() {
  829. local container="$1"
  830. while read-0 network ip; do
  831. printf "%s\0" "$network" "$ip"
  832. break
  833. done < <(get_container_network_ips "$container")
  834. }
  835. export -f get_container_network_ip
  836. ##
  837. ## Internal Process
  838. ##
  839. get_docker_compose_links() {
  840. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  841. deps master_service master_target_service _relation_name \
  842. target_service _relation_config tech_dep
  843. if [ -z "$service" ]; then
  844. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  845. return 1
  846. fi
  847. if [ -e "$cache_file" ]; then
  848. # debug "$FUNCNAME: cache hit ($*)"
  849. cat "$cache_file"
  850. return 0
  851. fi
  852. master_service=$(get_top_master_service_for_service "$service") || return 1
  853. deps=()
  854. while read-0 _relation_name target_service _relation_config tech_dep; do
  855. master_target_service="$(get_top_master_service_for_service "$target_service")" || return 1
  856. [ "$master_service" == "$master_target_service" ] && continue
  857. type="$(get_service_type "$target_service")" || return 1
  858. [ "$type" == "run-once" ] && continue
  859. if [ "$tech_dep" == "reversed" ]; then
  860. deps+=("$(echo -en "$master_target_service:\n links:\n - $master_service")")
  861. elif [ "$tech_dep" == "True" ]; then
  862. deps+=("$(echo -en "$master_service:\n links:\n - $master_target_service")")
  863. fi
  864. ## XXXvlab: an attempt to add depends_on, but this doesn't work well actually
  865. ## as there's a circular dependency issue. We don't really want the full feature
  866. ## of depends_on, but just to add it as targets when doing an 'up'
  867. # deps+=("$(echo -en "$master_service:\n depends_on:\n - $master_target_service")")
  868. done < <(get_service_relations "$service")
  869. merge_yaml_str "${deps[@]}" | tee "$cache_file" || return 1
  870. if [ "${PIPESTATUS[0]}" != 0 ]; then
  871. rm "$cache_file"
  872. err "Failed to merge YAML from all ${WHITE}links${NORMAL} dependencies."
  873. return 1
  874. fi
  875. }
  876. _get_docker_compose_opts() {
  877. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  878. compose_def master_service docker_compose_opts
  879. if [ -z "$service" ]; then
  880. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  881. return 1
  882. fi
  883. if [ -e "$cache_file" ]; then
  884. # debug "$FUNCNAME: cache hit ($*)"
  885. cat "$cache_file"
  886. return 0
  887. fi
  888. compose_def="$(get_compose_service_def "$service")" || return 1
  889. master_service="$(get_top_master_service_for_service "$service")"
  890. if docker_compose_opts=$(echo "$compose_def" | shyaml get-value -y "docker-compose" 2>/dev/null); then
  891. yaml_key_val_str "$master_service" "$docker_compose_opts"
  892. fi | tee "$cache_file"
  893. if [ "${PIPESTATUS[0]}" != 0 ]; then
  894. rm "$cache_file"
  895. return 1
  896. fi
  897. }
  898. ##
  899. ## By Reading the metadata.yml, we create a docker-compose.yml mixin.
  900. ## Some metadata.yml (of subordinates) will indeed modify other
  901. ## services than themselves.
  902. _get_docker_compose_service_mixin() {
  903. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  904. links_yaml base_mixin links_yaml docker_compose_options \
  905. charm charm_part
  906. if [ -z "$service" ]; then
  907. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  908. return 1
  909. fi
  910. if [ -e "$cache_file" ]; then
  911. # debug "$FUNCNAME: cache hit ($*)"
  912. cat "$cache_file"
  913. return 0
  914. fi
  915. master_service=$(get_top_master_service_for_service "$service") || {
  916. err "Failed to get top master service for service $DARKYELLOW$service$NORMAL"
  917. return 1
  918. }
  919. ## The compose part
  920. base_mixin="$master_service:
  921. labels:
  922. - compose.service=$service
  923. - compose.master-service=${master_service}
  924. - compose.project=$(get_default_project_name)"
  925. links_yaml=$(get_docker_compose_links "$service") || return 1
  926. docker_compose_options=$(_get_docker_compose_opts "$service") || return 1
  927. ## the charm part
  928. charm_part=$(get_docker_compose_mixin_from_metadata "$service") || return 1
  929. ## Merge results
  930. if [ "$charm_part" ]; then
  931. charm_yaml="$(yaml_key_val_str "$master_service" "$charm_part")" || return 1
  932. merge_yaml_str "$base_mixin" "$links_yaml" "$charm_yaml" "$docker_compose_options" || return 1
  933. else
  934. merge_yaml_str "$base_mixin" "$links_yaml" "$docker_compose_options" || return 1
  935. fi | tee "$cache_file"
  936. if [ "${PIPESTATUS[0]}" != 0 ]; then
  937. err "Failed to constitute the base YAML for service '${DARKYELLOW}$service${NORMAL}'"
  938. rm "$cache_file"
  939. return 1
  940. fi
  941. }
  942. export -f _get_docker_compose_service_mixin
  943. ##
  944. ## Get full `docker-compose.yml` format for all listed services (and
  945. ## their deps)
  946. ##
  947. ## @export
  948. ## @cache: !system !nofail +stdout
  949. get_docker_compose () {
  950. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  951. entries services service start docker_compose_services
  952. if [ -e "$cache_file" ]; then
  953. # debug "$FUNCNAME: cache hit ($*)"
  954. cat "$cache_file"
  955. return 0
  956. fi
  957. ##
  958. ## Adding sub services configurations
  959. ##
  960. declare -A entries
  961. start_compilation=$SECONDS
  962. debug "Compiling 'docker-compose.yml' base for ${DARKYELLOW}$*$NORMAL..."
  963. for target_service in "$@"; do
  964. start=$SECONDS
  965. services=($(get_ordered_service_dependencies "$target_service")) || {
  966. err "Failed to get dependencies for $DARKYELLOW$target_service$NORMAL"
  967. return 1
  968. }
  969. if [ "$DEBUG" ]; then
  970. debug " $DARKYELLOW$target_service$NORMAL deps:$DARKYELLOW" \
  971. "${services[@]::$((${#services[@]} - 1))}" \
  972. "$NORMAL$GRAY(in $((SECONDS - start))s)$NORMAL"
  973. fi
  974. for service in "${services[@]}"; do
  975. if [ "${entries[$service]}" ]; then
  976. ## Prevent double inclusion of same service if this
  977. ## service is deps of two or more of your
  978. ## requirements.
  979. continue
  980. fi
  981. ## mark the service as "loaded" as well as it's containers
  982. ## if this is a subordinate service
  983. start_service=$SECONDS
  984. entries[$service]=$(_get_docker_compose_service_mixin "$service") || {
  985. err "Failed to get service mixin for $DARKYELLOW$service$NORMAL"
  986. return 1
  987. }
  988. debug " Applied $DARKYELLOW$service$NORMAL charm metadata mixins $GRAY(in $((SECONDS - start_service))s)$NORMAL"
  989. done
  990. debug " ..finished all mixins for $DARKYELLOW$target_service$NORMAL $GRAY(in $((SECONDS - start))s)$NORMAL"
  991. done
  992. docker_compose_services=$(merge_yaml_str "${entries[@]}") || {
  993. err "Failed to merge YAML services entries together."
  994. return 1
  995. }
  996. base_v2="version: '2.0'"
  997. merge_yaml_str "$(yaml_key_val_str "services" "$docker_compose_services")" \
  998. "$base_v2" > "$cache_file" || return 1
  999. export _current_docker_compose="$(cat "$cache_file")"
  1000. echo "$_current_docker_compose"
  1001. debug " ..compilation of base 'docker-compose.yml' done $GRAY(in $((SECONDS - start_compilation))s)$NORMAL" || true
  1002. # debug " ** ${WHITE}docker-compose.yml${NORMAL}:"
  1003. # debug "$_current_docker_compose"
  1004. }
  1005. export -f get_docker_compose
  1006. _get_compose_service_def_cached () {
  1007. local service="$1" docker_compose="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  1008. if [ -e "$cache_file" ]; then
  1009. #debug "$FUNCNAME: STATIC cache hit"
  1010. cat "$cache_file" &&
  1011. touch "$cache_file" || return 1
  1012. return 0
  1013. fi
  1014. value=$(echo "$docker_compose" | shyaml get-value "${service//./\\.}" 2>/dev/null)
  1015. [ "$value" == None ] && value=""
  1016. if ! echo "$value" | shyaml get-value "charm" >/dev/null 2>&1; then
  1017. if charm.exists "$service"; then
  1018. value=$(merge_yaml <(echo "charm: $service") <(echo "$value")) || {
  1019. err "Can't merge YAML infered 'charm: $service' with base ${DARKYELLOW}$service${NORMAL} YAML definition."
  1020. return 1
  1021. }
  1022. else
  1023. err "No ${WHITE}charm${NORMAL} value for service $DARKYELLOW$service$NORMAL" \
  1024. "in compose, nor same name charm found."
  1025. return 1
  1026. fi
  1027. fi
  1028. echo "$value" | tee "$cache_file" || return 1
  1029. # if [ "${PIPESTATUS[0]}" != 0 ]; then
  1030. # rm "$cache_file"
  1031. # return 1
  1032. # fi
  1033. return 0
  1034. # if [ "${PIPESTATUS[0]}" != 0 -o \! -s "$cache_file" ]; then
  1035. # rm "$cache_file"
  1036. # err "PAS OK $service: $value"
  1037. # return 1
  1038. # fi
  1039. }
  1040. export -f _get_compose_service_def_cached
  1041. ## XXXvlab: a lot to be done to cache the results
  1042. get_compose_service_def () {
  1043. local service="$1" docker_compose cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1044. result
  1045. if [ -e "$cache_file" ]; then
  1046. #debug "$FUNCNAME: SESSION cache hit"
  1047. cat "$cache_file" || return 1
  1048. return 0
  1049. fi
  1050. [ -z "$service" ] && print_syntax_error "Missing service as first argument."
  1051. docker_compose=$(get_compose_yml_content) || return 1
  1052. result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1
  1053. charm=$(echo "$result" | shyaml get-value charm 2>/dev/null) || return 1
  1054. metadata=$(charm.metadata "$charm") || return 1
  1055. if default_options=$(printf "%s" "$metadata" | shyaml -y -q get-value default-options); then
  1056. default_options=$(yaml_key_val_str "options" "$default_options") || return 1
  1057. result=$(merge_yaml_str "$default_options" "$result") || return 1
  1058. fi
  1059. echo "$result" | tee "$cache_file" || return 1
  1060. }
  1061. export -f get_compose_service_def
  1062. _get_service_charm_cached () {
  1063. local service="$1" service_def="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  1064. if [ -e "$cache_file" ]; then
  1065. # debug "$FUNCNAME: cache hit $1"
  1066. cat "$cache_file" &&
  1067. touch "$cache_file" || return 1
  1068. return 0
  1069. fi
  1070. charm=$(echo "$service_def" | shyaml get-value charm 2>/dev/null)
  1071. if [ -z "$charm" ]; then
  1072. err "Missing ${WHITE}charm${NORMAL} value in service $DARKYELLOW$service$NORMAL definition."
  1073. return 1
  1074. fi
  1075. echo "$charm" | tee "$cache_file" || return 1
  1076. }
  1077. export -f _get_service_charm_cached
  1078. get_service_charm () {
  1079. local service="$1"
  1080. if [ -z "$service" ]; then
  1081. echo ${FUNCNAME[@]} >&2
  1082. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  1083. return 1
  1084. fi
  1085. service_def=$(get_compose_service_def "$service") || return 1
  1086. _get_service_charm_cached "$service" "$service_def"
  1087. }
  1088. export -f get_service_charm
  1089. ## built above the docker-compose abstraction, so it relies on the
  1090. ## full docker-compose.yml to be already built.
  1091. get_service_def () {
  1092. local service="$1" def
  1093. if [ -z "$_current_docker_compose" ]; then
  1094. print_syntax_error "$FUNCNAME is meant to be called after"\
  1095. "\$_current_docker_compose has been calculated."
  1096. fi
  1097. def=$(echo "$_current_docker_compose" | shyaml get-value "services.${service//./\\.}" 2>/dev/null)
  1098. if [ -z "$def" ]; then
  1099. err "No definition for service $DARKYELLOW$service$NORMAL in compiled 'docker-compose.yml'."
  1100. return 1
  1101. fi
  1102. echo "$def"
  1103. }
  1104. export -f get_service_def
  1105. ## Return the base docker image name of a service
  1106. service_base_docker_image() {
  1107. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1108. master_service service_def service_image service_build service_dockerfile
  1109. if [ -e "$cache_file" ]; then
  1110. # debug "$FUNCNAME: cache hit ($*)"
  1111. cat "$cache_file"
  1112. return 0
  1113. fi
  1114. master_service="$(get_top_master_service_for_service "$service")" || {
  1115. err "Could not compute master service for service $DARKYELLOW$service$NORMAL."
  1116. return 1
  1117. }
  1118. service_def="$(get_service_def "$master_service")" || {
  1119. err "Could not get docker-compose service definition for $DARKYELLOW$master_service$NORMAL."
  1120. return 1
  1121. }
  1122. service_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null)
  1123. if [ "$?" != 0 ]; then
  1124. ## According to https://stackoverflow.com/questions/32230577 , if there's a build,
  1125. ## then the builded image will get name ${project}_${service}
  1126. project=$(get_default_project_name) || return 1
  1127. image_name="${project}_${service}"
  1128. if ! docker_has_image "$image_name"; then
  1129. service_build=$(echo "$service_def" | shyaml get-value build 2>/dev/null)
  1130. if [ "$?" != 0 ]; then
  1131. err "Service $DARKYELLOW$service$NORMAL has no ${WHITE}image${NORMAL} nor ${WHITE}build${NORMAL} parameter."
  1132. echo "$service_def" >&2
  1133. return 1
  1134. fi
  1135. docker build "$service_build" -t "${project}_${service}" >&2 || {
  1136. err "Failed to build image for ${DARKYELLOW}$service${NORMAL}."
  1137. return 1
  1138. }
  1139. fi
  1140. printf "%s" "${project}_${service}"
  1141. else
  1142. printf "%s" "${service_image}"
  1143. fi | tee "$cache_file"
  1144. if [ "${PIPESTATUS[0]}" != 0 ]; then
  1145. rm "$cache_file"
  1146. return 1
  1147. fi
  1148. }
  1149. export -f service_base_docker_image
  1150. get_charm_relation_def () {
  1151. local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1152. relation_def metadata
  1153. if [ -e "$cache_file" ]; then
  1154. # debug "$FUNCNAME: cache hit ($*)"
  1155. cat "$cache_file"
  1156. return 0
  1157. fi
  1158. metadata="$(charm.metadata "$charm")" || return 1
  1159. relation_def="$(echo "$metadata" | shyaml get-value "provides.${relation_name}" 2>/dev/null)"
  1160. echo "$relation_def" | tee "$cache_file"
  1161. }
  1162. export -f get_charm_relation_def
  1163. get_charm_tech_dep_orientation_for_relation() {
  1164. local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1165. relation_def value
  1166. if [ -e "$cache_file" ]; then
  1167. # debug "$FUNCNAME: cache hit ($*)"
  1168. cat "$cache_file"
  1169. return 0
  1170. fi
  1171. relation_def=$(get_charm_relation_def "$charm" "$relation_name" 2>/dev/null)
  1172. value=$(echo "$relation_def" | shyaml get-value 'tech-dep' 2>/dev/null)
  1173. value=${value:-True}
  1174. printf "%s" "$value" | tee "$cache_file"
  1175. }
  1176. export -f get_charm_tech_dep_orientation_for_relation
  1177. get_service_relation_tech_dep() {
  1178. local service="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1179. charm tech_dep
  1180. if [ -e "$cache_file" ]; then
  1181. # debug "$FUNCNAME: cache hit ($*)"
  1182. cat "$cache_file"
  1183. return 0
  1184. fi
  1185. charm=$(get_service_charm "$service") || return 1
  1186. tech_dep="$(get_charm_tech_dep_orientation_for_relation "$charm" "$relation_name")" || return 1
  1187. printf "%s" "$tech_dep" | tee "$cache_file"
  1188. }
  1189. export -f get_service_relation_tech_dep
  1190. ##
  1191. ## Use compose file to get deps, and relation definition in metadata.yml
  1192. ## for tech-dep attribute.
  1193. get_service_deps() {
  1194. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$ALL_RELATIONS")"
  1195. if [ -e "$cache_file" ]; then
  1196. # debug "$FUNCNAME: cache hit ($*)"
  1197. cat "$cache_file"
  1198. return 0
  1199. fi
  1200. (
  1201. set -o pipefail
  1202. get_service_relations "$service" | \
  1203. while read-0 relation_name target_service _relation_config tech_dep; do
  1204. echo "$target_service"
  1205. done | tee "$cache_file"
  1206. ) || return 1
  1207. }
  1208. export -f get_service_deps
  1209. ## XXXvlab: cache was disabled because improper. Indeed, this needs to cache
  1210. ## 'depths' full state. Second, it should be
  1211. _rec_get_depth() {
  1212. local elt=$1 dep deps max
  1213. [ "${depths[$elt]}" ] && return 0
  1214. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$(declare -pA depths)" "$ALL_RELATIONS")"
  1215. if [ -e "$cache_file.depths" ]; then
  1216. #debug "$FUNCNAME: cache hit ($*) - $cache_file.depths"
  1217. while read-0 k v; do
  1218. depths["$k"]="$v"
  1219. done < "$cache_file.depths"
  1220. while read-0 k v; do
  1221. visited["$k"]="$v"
  1222. done < "$cache_file.visited"
  1223. return 0
  1224. fi
  1225. visited[$elt]=1
  1226. #debug "Setting visited[$elt]"
  1227. #debug "Asking for $DARKYELLOW$elt$NORMAL dependencies"
  1228. deps=$(get_service_deps "$elt") || {
  1229. debug "Failed get_service_deps $elt"
  1230. return 1
  1231. }
  1232. # debug "$elt deps are:" $deps
  1233. max=0
  1234. for dep in $deps; do
  1235. [ "${visited[$dep]}" ] && {
  1236. #debug "Already computing $dep"
  1237. continue
  1238. }
  1239. _rec_get_depth "$dep" || return 1
  1240. #debug "Requesting depth[$dep]"
  1241. if (( ${depths[$dep]} > max )); then
  1242. max="${depths[$dep]}"
  1243. fi
  1244. done
  1245. # debug "Setting depth[$elt] to $((max + 1))"
  1246. depths[$elt]=$((max + 1))
  1247. array_kv_to_stdin depths > "$cache_file.depths"
  1248. array_kv_to_stdin visited > "$cache_file.visited"
  1249. # debug "DEPTHS: $(declare -pA depths)"
  1250. # debug "$FUNCNAME: caching hit ($*) - $cache_file"
  1251. }
  1252. export -f _rec_get_depth
  1253. get_ordered_service_dependencies() {
  1254. local services=("$@") cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$ALL_RELATIONS")" \
  1255. i value key heads depths visited
  1256. if [ -e "$cache_file" ]; then
  1257. # debug "$FUNCNAME: cache hit ($*)"
  1258. cat "$cache_file"
  1259. return 0
  1260. fi
  1261. #debug "Figuring ordered deps of $DARKYELLOW${services[@]}$NORMAL"
  1262. if [ -z "${services[*]}" ]; then
  1263. return 0
  1264. # print_syntax_error "$FUNCNAME: no arguments"
  1265. # return 1
  1266. fi
  1267. declare -A depths
  1268. declare -A visited
  1269. heads=("${services[@]}")
  1270. while [ "${#heads[@]}" != 0 ]; do
  1271. array_pop heads head
  1272. _rec_get_depth "$head" || return 1
  1273. done
  1274. i=0
  1275. while [ "${#depths[@]}" != 0 ]; do
  1276. for key in "${!depths[@]}"; do
  1277. value="${depths[$key]}"
  1278. if [ "$value" == "$i" ]; then
  1279. echo "$key"
  1280. unset depths[$key]
  1281. fi
  1282. done
  1283. ((i++))
  1284. done | tee "$cache_file"
  1285. }
  1286. export -f get_ordered_service_dependencies
  1287. run_service_hook () {
  1288. local action="$1" service subservice subservices loaded
  1289. shift
  1290. declare -A loaded
  1291. for service in "$@"; do
  1292. subservices=$(get_ordered_service_dependencies "$service") || return 1
  1293. for subservice in $subservices; do
  1294. if [ "${loaded[$subservice]}" ]; then
  1295. ## Prevent double inclusion of same service if this
  1296. ## service is deps of two or more of your
  1297. ## requirements.
  1298. continue
  1299. fi
  1300. charm=$(get_service_charm "$subservice") || return 1
  1301. charm.has_hook "$charm" "$action" >/dev/null || continue
  1302. PROJECT_NAME=$(get_default_project_name) || return 1
  1303. MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
  1304. MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1
  1305. DOCKER_BASE_IMAGE=$(service_base_docker_image "$MASTER_BASE_SERVICE_NAME") || return 1
  1306. Wrap "${wrap_opts[@]}" -d "running $YELLOW$action$NORMAL hook of $DARKYELLOW$subservice$NORMAL in charm $DARKPINK$charm$NORMAL" <<EOF || return 1
  1307. export DOCKER_BASE_IMAGE="$DOCKER_BASE_IMAGE"
  1308. export SERVICE_NAME=$subservice
  1309. export IMAGE_NAME=$(echo "${PROJECT_NAME}" | tr -d "_-")_\${SERVICE_NAME}
  1310. export CONTAINER_NAME=\${IMAGE_NAME}_1
  1311. export CHARM_NAME="$charm"
  1312. export PROJECT_NAME="$PROJECT_NAME"
  1313. export SERVICE_DATASTORE="$DATASTORE/$subservice"
  1314. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$subservice"
  1315. export MASTER_BASE_SERVICE_NAME="$MASTER_BASE_SERVICE_NAME"
  1316. export MASTER_BASE_CHARM_NAME="$MASTER_BASE_CHARM_NAME"
  1317. charm.run_hook "local" "$charm" "$action"
  1318. EOF
  1319. loaded[$subservice]=1
  1320. done
  1321. done
  1322. return 0
  1323. }
  1324. host_resource_get() {
  1325. local location="$1" cfg="$2"
  1326. type=$(echo "$cfg" | shyaml get-value type 2>/dev/null) || {
  1327. err "Missing ${WHITE}type$NORMAL option in ${WHITE}get$NORMAL config for location '$location'"
  1328. return 1
  1329. }
  1330. if fn.exists host_resource_get_$type; then
  1331. host_resource_get_$type "$location" "$cfg"
  1332. else
  1333. err "Source ${WHITE}source$NORMAL type '$type' unknown for" \
  1334. "${WHITE}host-resource$NORMAL '$location' defined in" \
  1335. "$DARKYELLOW$subservice$NORMAL config."
  1336. return 1
  1337. fi
  1338. }
  1339. export -f host_resource_get
  1340. host_resource_get_git() {
  1341. local location="$1" cfg="$2" branch parent url
  1342. branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null)
  1343. branch=${branch:-master}
  1344. url=$(echo "$cfg" | shyaml get-value url 2>/dev/null)
  1345. parent="$(dirname "$location")"
  1346. (
  1347. mkdir -p "$parent" && cd "$parent" &&
  1348. git clone -b "$branch" "$url" "$(basename "$location")"
  1349. ) || return 1
  1350. }
  1351. export -f host_resource_get_git
  1352. host_resource_get_git-sub() {
  1353. local location="$1" cfg="$2" branch parent url
  1354. branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null)
  1355. branch=${branch:-master}
  1356. url=$(echo "$cfg" | shyaml get-value url 2>/dev/null)
  1357. parent="$(dirname "$location")"
  1358. (
  1359. mkdir -p "$parent" && cd "$parent" &&
  1360. git sub clone -b "$branch" "$url" "$(basename "$location")"
  1361. ) || return 1
  1362. }
  1363. export -f host_resource_get_git-sub
  1364. setup_host_resource () {
  1365. local subservice="$1" service_def location get cfg
  1366. service_def=$(get_compose_service_def "$subservice") || return 1
  1367. while read-0 location cfg; do
  1368. ## XXXvlab: will it be a git resources always ?
  1369. if [ -d "$location" -a ! -d "$location/.git" ]; then
  1370. err "Hum, location '$location' does not seem to be a git directory."
  1371. return 1
  1372. fi
  1373. if [ -d "$location" ]; then
  1374. info "host resource '$location' already set up."
  1375. continue
  1376. fi
  1377. get=$(echo "$cfg" | shyaml get-value get 2>/dev/null)
  1378. if [ -z "$get" ]; then
  1379. err "No host directory '$location' found, and no ${WHITE}source$NORMAL" \
  1380. "specified for $DARKYELLOW$subservice$NORMAL."
  1381. return 1
  1382. fi
  1383. host_resource_get "$location" "$get" || return 1
  1384. done < <(echo "$service_def" | shyaml key-values-0 host-resources 2>/dev/null)
  1385. }
  1386. export -f setup_host_resource
  1387. setup_host_resources () {
  1388. local service subservices subservice loaded
  1389. declare -A loaded
  1390. for service in "$@"; do
  1391. subservices=$(get_ordered_service_dependencies "$service") || return 1
  1392. for subservice in $subservices; do
  1393. if [ "${loaded[$subservice]}" ]; then
  1394. ## Prevent double inclusion of same service if this
  1395. ## service is deps of two or more of your
  1396. ## requirements.
  1397. continue
  1398. fi
  1399. setup_host_resource "$subservice" || return 1
  1400. loaded[$subservice]=1
  1401. done
  1402. done
  1403. return 0
  1404. }
  1405. export -f setup_host_resources
  1406. ## Works on stdin
  1407. cfg-get-value () {
  1408. local key="$1" out
  1409. if [ -z "$key" ]; then
  1410. yaml_get_interpret || return 1
  1411. return 0
  1412. fi
  1413. if ! out=$(shyaml -y get-value "$key" 2>/dev/null); then
  1414. err "The key $WHITE$key$NORMAL was not found in relation's data."
  1415. return 1
  1416. fi
  1417. echo "$out" | yaml_get_interpret
  1418. }
  1419. export -f cfg-get-value
  1420. relation-get () {
  1421. cfg-get-value "$1" < "$RELATION_DATA_FILE"
  1422. }
  1423. export -f relation-get
  1424. expand_vars() {
  1425. local unlikely_prefix="UNLIKELY_PREFIX"
  1426. content=$(cat -)
  1427. ## find first identifier not in content
  1428. remaining_lines=$(echo "$content" | grep "^$unlikely_prefix")
  1429. while [ "$(echo "$remaining_lines" | grep "^$unlikely_prefix$")" ]; do
  1430. size_prefix="${#unlikely_prefix}"
  1431. first_matching=$(echo "$remaining_lines" |
  1432. grep -v "^$unlikely_prefix$" |
  1433. uniq -w "$((size_prefix + 1))" -c |
  1434. sort -rn |
  1435. head -n 1)
  1436. first_matching=${first_matching#"${x%%[![:space:]]*}"}
  1437. first_matching="${first_matching#* }"
  1438. next_char=${first_matching:$size_prefix:1}
  1439. if [ "$next_char" != "0" ]; then
  1440. unlikely_prefix+="0"
  1441. else
  1442. unlikely_prefix+="1"
  1443. fi
  1444. remaining_lines=$(echo "$remaining_lines" | grep "^$unlikely_prefix")
  1445. done
  1446. eval "cat <<$unlikely_prefix
  1447. $content
  1448. $unlikely_prefix"
  1449. }
  1450. export -f expand_vars
  1451. yaml_get_interpret() {
  1452. local content tag
  1453. content=$(cat -)
  1454. tag=$(echo "$content" | shyaml -y get-value) || return 1
  1455. tag="${tag%% *}"
  1456. content=$(echo "$content" | shyaml get-value) || return 1
  1457. if ! [ "${tag:0:1}" == "!" ]; then
  1458. echo "$content" || return 1
  1459. return 0
  1460. fi
  1461. case "$tag" in
  1462. "!bash-stdout")
  1463. echo "$content" | bash || {
  1464. err "shell code didn't end with errorlevel 0"
  1465. return 1
  1466. }
  1467. ;;
  1468. "!var-expand")
  1469. echo "$content" | expand_vars || {
  1470. err "shell expansion failed"
  1471. return 1
  1472. }
  1473. ;;
  1474. "!file-content")
  1475. source=$(echo "$content" | expand_vars) || {
  1476. err "shell expansion failed"
  1477. return 1
  1478. }
  1479. cat "$source" || return 1
  1480. ;;
  1481. *)
  1482. err "Invalid object tag ${WHITE}$tag${NORMAL}"
  1483. return 1
  1484. ;;
  1485. esac
  1486. }
  1487. export -f yaml_get_interpret
  1488. options-get () {
  1489. local key="$1" out
  1490. service_def=$(get_compose_service_def "$SERVICE_NAME") || return 1
  1491. if ! out=$(echo "$service_def" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1492. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  1493. return 1
  1494. fi
  1495. echo "$out" | yaml_get_interpret
  1496. }
  1497. export -f options-get
  1498. relation-base-compose-get () {
  1499. local key="$1" out
  1500. if ! out=$(echo "$RELATION_BASE_COMPOSE_DEF" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1501. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  1502. return 1
  1503. fi
  1504. echo "$out" | yaml_get_interpret
  1505. }
  1506. export -f relation-base-compose-get
  1507. relation-target-compose-get () {
  1508. local key="$1" out
  1509. if ! out=$(echo "$RELATION_TARGET_COMPOSE_DEF" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1510. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  1511. return 1
  1512. fi
  1513. echo "$out" | yaml_get_interpret
  1514. }
  1515. export -f relation-target-compose-get
  1516. relation-set () {
  1517. local key="$1" value="$2"
  1518. if [ -z "$RELATION_DATA_FILE" ]; then
  1519. err "$FUNCNAME: relation does not seems to be correctly setup."
  1520. return 1
  1521. fi
  1522. if ! [ -r "$RELATION_DATA_FILE" ]; then
  1523. err "$FUNCNAME: can't read relation's data." >&2
  1524. return 1
  1525. fi
  1526. _config_merge "$RELATION_DATA_FILE" <(yaml_key_val_str "$key" "$value")
  1527. }
  1528. export -f relation-set
  1529. _config_merge() {
  1530. local config_filename="$1" mixin="$2"
  1531. touch "$config_filename" &&
  1532. merge_yaml "$config_filename" "$mixin" > "$config_filename.tmp" || return 1
  1533. mv "$config_filename.tmp" "$config_filename"
  1534. }
  1535. export -f _config_merge
  1536. ## XXXvlab; this can be used only in relation, I'd like to use it in init.
  1537. config-add() {
  1538. local metadata="$1"
  1539. _config_merge "$RELATION_CONFIG" <(echo "$metadata")
  1540. }
  1541. export -f config-add
  1542. ## XXXvlab; this can be used only in relation, I'd like to use it in init.
  1543. init-config-add() {
  1544. local metadata="$1"
  1545. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" \
  1546. <(yaml_key_val_str "services" "$metadata")
  1547. }
  1548. export -f init-config-add
  1549. docker_get_uid() {
  1550. local service="$1" user="$2" uid
  1551. uid=$(cached_cmd_on_base_image "$service" "id -u \"$user\"") || {
  1552. debug "Failed to query for '$user' uid in ${DARKYELLOW}$service${NORMAL} base image."
  1553. return 1
  1554. }
  1555. info "uid from ${DARKYELLOW}$service${NORMAL} for user '$user' is '$uid'"
  1556. echo "$uid"
  1557. }
  1558. export -f docker_get_uid
  1559. docker_get_uid_gid() {
  1560. local service="$1" user="$2" group="$3" uid
  1561. uid_gid=$(cached_cmd_on_base_image "$service" "id -u \"$user\"; id -g \"$group\"") || {
  1562. debug "Failed to query for '$user' uid in ${DARKYELLOW}$service${NORMAL} base image."
  1563. return 1
  1564. }
  1565. info "uid from ${DARKYELLOW}$service${NORMAL} for user '$user' is '$uid_gid'"
  1566. echo "$uid_gid"
  1567. }
  1568. export -f docker_get_uid_gid
  1569. logstdout() {
  1570. local name="$1"
  1571. sed -r 's%^%'"${name}"'> %g'
  1572. }
  1573. export -f logstdout
  1574. logstderr() {
  1575. local name="$1"
  1576. sed -r 's%^(.*)$%'"${RED}${name}>${NORMAL} \1"'%g'
  1577. }
  1578. export -f logstderr
  1579. _run_service_relation () {
  1580. local relation_name="$1" service="$2" target_service="$3" relation_config="$4" relation_dir services
  1581. charm=$(get_service_charm "$service") || return 1
  1582. target_charm=$(get_service_charm "$target_service") || return 1
  1583. base_script_name=$(charm.has_relation_hook "$charm" "$relation_name" relation-joined) || true
  1584. target_script_name=$(charm.has_relation_hook "$target_charm" "$relation_name" relation-joined) || true
  1585. [ "$base_script_name" -o "$target_script_name" ] || return 0
  1586. relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1
  1587. RELATION_DATA_FILE=$(get_relation_data_file "$service" "$target_service" "$relation_name" "$relation_config") || return 1
  1588. export BASE_SERVICE_NAME=$service
  1589. export BASE_CHARM_NAME=$charm
  1590. export BASE_CHARM_PATH=$(charm.get_dir "$charm")
  1591. export TARGET_SERVICE_NAME=$target_service
  1592. export TARGET_CHARM_NAME=$target_charm
  1593. export TARGET_CHARM_PATH=$(charm.get_dir "$target_charm")
  1594. export RELATION_DATA_FILE
  1595. target_errlvl=0
  1596. if [ -z "$target_script_name" ]; then
  1597. verb "No relation script $DARKBLUE$relation_name$NORMAL in target $DARKPINK$target_charm$NORMAL."
  1598. else
  1599. verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \
  1600. "for target $DARKYELLOW$target_service$NORMAL (charm $DARKPINK$target_charm$NORMAL)"
  1601. RELATION_CONFIG="$relation_dir/config_provider"
  1602. DOCKER_BASE_IMAGE=$(service_base_docker_image "$target_service") || return 1
  1603. export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA
  1604. {
  1605. (
  1606. SERVICE_NAME=$target_service
  1607. SERVICE_DATASTORE="$DATASTORE/$target_service"
  1608. SERVICE_CONFIGSTORE="$CONFIGSTORE/$target_service"
  1609. export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE
  1610. charm.run_relation_hook local "$target_charm" "$relation_name" relation-joined
  1611. echo "$?" > "$relation_dir/target_errlvl"
  1612. ) | logstdout "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}"
  1613. } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@${NORMAL}" 3>&1 1>&2 2>&3
  1614. target_errlvl="$(cat "$relation_dir/target_errlvl")" || {
  1615. err "Relation script '$script_name' in $DARKPINK$target_charm$NORMAL" \
  1616. "failed before outputing an errorlevel."
  1617. ((target_errlvl |= "1" ))
  1618. }
  1619. if [ -e "$RELATION_CONFIG" ]; then
  1620. debug "Merging some new config info in $DARKYELLOW$target_service$NORMAL"
  1621. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" &&
  1622. rm "$RELATION_CONFIG"
  1623. ((target_errlvl |= "$?"))
  1624. fi
  1625. fi
  1626. if [ "$target_errlvl" == 0 ]; then
  1627. errlvl=0
  1628. if [ "$base_script_name" ]; then
  1629. verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \
  1630. "for $DARKYELLOW$service$NORMAL (charm $DARKPINK$charm$NORMAL)"
  1631. RELATION_CONFIG="$relation_dir/config_providee"
  1632. RELATION_DATA="$(cat "$RELATION_DATA_FILE")"
  1633. DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") || return 1
  1634. export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA
  1635. {
  1636. (
  1637. SERVICE_NAME=$service
  1638. SERVICE_DATASTORE="$DATASTORE/$service"
  1639. SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  1640. export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE
  1641. charm.run_relation_hook local "$charm" "$relation_name" relation-joined
  1642. echo "$?" > "$relation_dir/errlvl"
  1643. ) | logstdout "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}"
  1644. } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@$NORMAL" 3>&1 1>&2 2>&3
  1645. errlvl="$(cat "$relation_dir/errlvl")" || {
  1646. err "Relation script '$script_name' in $DARKPINK$charm$NORMAL" \
  1647. "failed before outputing an errorlevel."
  1648. ((errlvl |= "1" ))
  1649. }
  1650. if [ -e "$RELATION_CONFIG" ]; then
  1651. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" &&
  1652. rm "$RELATION_CONFIG"
  1653. ((errlvl |= "$?" ))
  1654. fi
  1655. if [ "$errlvl" != 0 ]; then
  1656. err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$service$NORMAL failed to run properly."
  1657. fi
  1658. else
  1659. verb "No relation script '$script_name' in charm $DARKPINK$charm$NORMAL. Ignoring."
  1660. fi
  1661. else
  1662. err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$target_service$NORMAL failed to run properly."
  1663. fi
  1664. if [ "$target_errlvl" == 0 -a "$errlvl" == 0 ]; then
  1665. debug "Relation $DARKBLUE$relation_name$NORMAL is established" \
  1666. "between $DARKYELLOW$service$NORMAL and $DARKYELLOW$target_service$NORMAL."
  1667. return 0
  1668. else
  1669. return 1
  1670. fi
  1671. }
  1672. export -f _run_service_relation
  1673. _get_compose_relations_cached () {
  1674. local compose_service_def="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  1675. relation_name relation_def target_service
  1676. if [ -e "$cache_file" ]; then
  1677. #debug "$FUNCNAME: STATIC cache hit $1"
  1678. cat "$cache_file" &&
  1679. touch "$cache_file" || return 1
  1680. return 0
  1681. fi
  1682. (
  1683. set -o pipefail
  1684. if [ "$compose_service_def" ]; then
  1685. while read-0 relation_name relation_def; do
  1686. ## XXXvlab: could we use braces here instead of parenthesis ?
  1687. (
  1688. case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in
  1689. "str")
  1690. target_service="$(echo "$relation_def" | shyaml get-value 2>/dev/null)" || return 1
  1691. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  1692. printf "%s\0" "$relation_name" "$target_service" "" "$tech_dep"
  1693. ;;
  1694. "sequence")
  1695. while read-0 target_service; do
  1696. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  1697. printf "%s\0" "$relation_name" "$target_service" "" "$tech_dep"
  1698. done < <(echo "$relation_def" | shyaml get-values-0 2>/dev/null)
  1699. ;;
  1700. "struct")
  1701. while read-0 target_service relation_config; do
  1702. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  1703. printf "%s\0" "$relation_name" "$target_service" "$relation_config" "$tech_dep"
  1704. done < <(echo "$relation_def" | shyaml key-values-0 2>/dev/null)
  1705. ;;
  1706. esac
  1707. ) </dev/null >> "$cache_file" || return 1
  1708. done < <(echo "$compose_service_def" | shyaml key-values-0 relations 2>/dev/null)
  1709. fi
  1710. )
  1711. if [ "$?" != 0 ]; then
  1712. err "Error while looking for compose relations."
  1713. rm -f "$cache_file" ## no cache
  1714. return 1
  1715. fi
  1716. [ -e "$cache_file" ] && cat "$cache_file"
  1717. return 0
  1718. }
  1719. export -f _get_compose_relations_cached
  1720. get_compose_relations () {
  1721. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1722. compose_def
  1723. if [ -e "$cache_file" ]; then
  1724. #debug "$FUNCNAME: SESSION cache hit $1"
  1725. cat "$cache_file"
  1726. return 0
  1727. fi
  1728. compose_def="$(get_compose_service_def "$service")" || return 1
  1729. _get_compose_relations_cached "$compose_def" > "$cache_file"
  1730. if [ "$?" != 0 ]; then
  1731. rm -f "$cache_file" ## no cache
  1732. return 1
  1733. fi
  1734. cat "$cache_file"
  1735. }
  1736. export -f get_compose_relations
  1737. get_service_relations () {
  1738. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$ALL_RELATIONS")" \
  1739. s rn ts rc td
  1740. if [ -e "$cache_file" ]; then
  1741. #debug "$FUNCNAME: SESSION cache hit $1"
  1742. cat "$cache_file"
  1743. return 0
  1744. fi
  1745. if [ -z "$ALL_RELATIONS" ]; then
  1746. err "Can't access global \$ALL_RELATIONS"
  1747. return 1
  1748. fi
  1749. while read-0 s rn ts rc td; do
  1750. [[ "$s" == "$service" ]] || continue
  1751. printf "%s\0" "$rn" "$ts" "$rc" "$td"
  1752. done < <(cat "$ALL_RELATIONS") > "$cache_file"
  1753. if [ "$?" != 0 ]; then
  1754. rm -f "$cache_file" ## no cache
  1755. return 1
  1756. fi
  1757. cat "$cache_file"
  1758. }
  1759. export -f get_service_relations
  1760. get_service_relation() {
  1761. local service="$1" relation="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1762. rn ts rc td
  1763. if [ -e "$cache_file" ]; then
  1764. #debug "$FUNCNAME: SESSION cache hit $1"
  1765. cat "$cache_file"
  1766. return 0
  1767. fi
  1768. while read-0 rn ts rc td; do
  1769. [ "$relation" == "$rn" ] && {
  1770. printf "%s\0" "$ts" "$rc" "$td"
  1771. break
  1772. }
  1773. done < <(get_service_relations "$service") > "$cache_file"
  1774. if [ "$?" != 0 ]; then
  1775. rm -f "$cache_file" ## no cache
  1776. return 1
  1777. fi
  1778. cat "$cache_file"
  1779. }
  1780. export -f get_service_relation
  1781. _get_charm_metadata_uses() {
  1782. local metadata="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  1783. if [ -e "$cache_file" ]; then
  1784. #debug "$FUNCNAME: SESSION cache hit $1"
  1785. cat "$cache_file" || return 1
  1786. return 0
  1787. fi
  1788. printf "%s" "$metadata" | { shyaml key-values-0 uses 2>/dev/null || true; } | tee "$cache_file"
  1789. }
  1790. export -f _get_charm_metadata_uses
  1791. _get_service_metadata() {
  1792. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1793. charm
  1794. if [ -e "$cache_file" ]; then
  1795. #debug "$FUNCNAME: SESSION cache hit $1"
  1796. cat "$cache_file"
  1797. return 0
  1798. fi
  1799. charm="$(get_service_charm "$service")" || return 1
  1800. charm.metadata "$charm" > "$cache_file"
  1801. if [ "$?" != 0 ]; then
  1802. rm -f "$cache_file" ## no cache
  1803. return 1
  1804. fi
  1805. cat "$cache_file"
  1806. }
  1807. export -f _get_service_metadata
  1808. _get_service_uses() {
  1809. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1810. metadata
  1811. if [ -e "$cache_file" ]; then
  1812. #debug "$FUNCNAME: SESSION cache hit $1"
  1813. cat "$cache_file"
  1814. return 0
  1815. fi
  1816. metadata="$(_get_service_metadata "$service")" || return 1
  1817. _get_charm_metadata_uses "$metadata" > "$cache_file"
  1818. if [ "$?" != 0 ]; then
  1819. rm -f "$cache_file" ## no cache
  1820. return 1
  1821. fi
  1822. cat "$cache_file"
  1823. }
  1824. export -f _get_service_uses
  1825. _get_services_uses() {
  1826. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  1827. service rn rd
  1828. if [ -e "$cache_file" ]; then
  1829. #debug "$FUNCNAME: SESSION cache hit $1"
  1830. cat "$cache_file"
  1831. return 0
  1832. fi
  1833. for service in "$@"; do
  1834. _get_service_uses "$service" | while read-0 rn rd; do
  1835. printf "%s\0" "$service" "$rn" "$rd"
  1836. done
  1837. [ "${PIPESTATUS[0]}" == 0 ] || {
  1838. return 1
  1839. }
  1840. done > "${cache_file}.wip"
  1841. mv "${cache_file}"{.wip,} &&
  1842. cat "$cache_file" || return 1
  1843. }
  1844. export -f _get_services_uses
  1845. _get_provides_provides() {
  1846. local provides="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  1847. service rn rd
  1848. if [ -e "$cache_file" ]; then
  1849. # debug "$FUNCNAME: CACHEDIR cache hit $1"
  1850. cat "$cache_file"
  1851. return 0
  1852. fi
  1853. type=$(printf "%s" "$provides" | shyaml get-type)
  1854. case "$type" in
  1855. sequence)
  1856. while read-0 prov; do
  1857. printf "%s\0" "$prov" ""
  1858. done < <(echo "$provides" | shyaml get-values-0)
  1859. ;;
  1860. struct)
  1861. printf "%s" "$provides" | shyaml key-values-0
  1862. ;;
  1863. str)
  1864. printf "%s\0" "$(echo "$provides" | shyaml get-value)" ""
  1865. ;;
  1866. *)
  1867. err "Unexpected type '$type' for provider identifier in charm '$charm'."
  1868. return 1
  1869. esac | tee "$cache_file"
  1870. return "${PIPESTATUS[0]}"
  1871. }
  1872. _get_metadata_provides() {
  1873. local metadata="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  1874. service rn rd
  1875. if [ -e "$cache_file" ]; then
  1876. #debug "$FUNCNAME: CACHEDIR cache hit"
  1877. cat "$cache_file"
  1878. return 0
  1879. fi
  1880. provides=$(printf "%s" "$metadata" | shyaml get-value -y -q provides "")
  1881. [ "$provides" -a "$provides" != "''" ] || { touch "$cache_file"; return 0; }
  1882. _get_provides_provides "$provides" | tee "$cache_file"
  1883. return "${PIPESTATUS[0]}"
  1884. }
  1885. _get_services_provides() {
  1886. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  1887. service rn rd
  1888. if [ -e "$cache_file" ]; then
  1889. #debug "$FUNCNAME: SESSION cache hit $1"
  1890. cat "$cache_file"
  1891. return 0
  1892. fi
  1893. ## YYY: replace the inner loop by a cached function
  1894. for service in "$@"; do
  1895. metadata="$(_get_service_metadata "$service")" || return 1
  1896. while read-0 rn rd; do
  1897. printf "%s\0" "$service" "$rn" "$rd"
  1898. done < <(_get_metadata_provides "$metadata")
  1899. done > "$cache_file"
  1900. if [ "$?" != 0 ]; then
  1901. rm -f "$cache_file" ## no cache
  1902. return 1
  1903. fi
  1904. cat "$cache_file"
  1905. }
  1906. export -f _get_services_provides
  1907. _get_charm_provides() {
  1908. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(charm.store_metadata_hash)"
  1909. if [ -e "$cache_file" ]; then
  1910. #debug "$FUNCNAME: SESSION cache hit"
  1911. cat "$cache_file"
  1912. return 0
  1913. fi
  1914. start="$SECONDS"
  1915. debug "Getting charm provider list..."
  1916. while read-0 charm _ realpath metadata; do
  1917. metadata="$(charm.metadata "$charm")" || continue
  1918. # echo "reading $charm" >&2
  1919. while read-0 rn rd; do
  1920. printf "%s\0" "$charm" "$rn" "$rd"
  1921. done < <(_get_metadata_provides "$metadata")
  1922. done < <(charm.ls) | tee "$cache_file"
  1923. errlvl="${PIPESTATUS[0]}"
  1924. debug " ..charm provider list done $GRAY(in $((SECONDS - start))s)$NORMAL"
  1925. return "$errlvl"
  1926. }
  1927. _get_charm_providing() {
  1928. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  1929. relation="$1"
  1930. if [ -e "$cache_file" ]; then
  1931. #debug "$FUNCNAME: SESSION cache hit $1"
  1932. cat "$cache_file"
  1933. return 0
  1934. fi
  1935. while read-0 charm relation_name relation_def; do
  1936. [ "$relation_name" == "$relation" ] || continue
  1937. printf "%s\0" "$charm" "$relation_def"
  1938. done < <(_get_charm_provides) > "$cache_file"
  1939. if [ "$?" != 0 ]; then
  1940. rm -f "$cache_file" ## no cache
  1941. return 1
  1942. fi
  1943. cat "$cache_file"
  1944. }
  1945. _get_services_providing() {
  1946. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  1947. relation="$1"
  1948. shift ## services is "$@"
  1949. if [ -e "$cache_file" ]; then
  1950. #debug "$FUNCNAME: SESSION cache hit $1"
  1951. cat "$cache_file"
  1952. return 0
  1953. fi
  1954. while read-0 service relation_name relation_def; do
  1955. [ "$relation_name" == "$relation" ] || continue
  1956. printf "%s\0" "$service" "$relation_def"
  1957. done < <(_get_services_provides "$@") > "$cache_file"
  1958. if [ "$?" != 0 ]; then
  1959. rm -f "$cache_file" ## no cache
  1960. return 1
  1961. fi
  1962. cat "$cache_file"
  1963. }
  1964. export -f _get_services_provides
  1965. _out_new_relation_from_defs() {
  1966. local service="$1" rn="$2" ts="$3" prov_def="$4" rel_def="$5" rc td rc_prov
  1967. rc_prov=$(printf "%s" "$prov_def" | shyaml -y get-value "default-options" 2>/dev/null)
  1968. ## YYYvlab: should be seen even in no debug mode no ?
  1969. rc=$(printf "%s" "$rel_def" | shyaml -y get-value "default-options" 2>/dev/null)
  1970. td=$(echo "$prov_def" | shyaml get-value 'tech-dep' 2>/dev/null)
  1971. td=${td:-True}
  1972. rc=$(merge_yaml_str "$rc_prov" "$rc") || return 1
  1973. printf "%s\0" "$service" "$relation_name" "$ts" "$rc" "$td"
  1974. }
  1975. get_all_relations () {
  1976. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$(declare -p without_relations)")" \
  1977. services
  1978. if [ -e "${cache_file}" ]; then
  1979. #debug "$FUNCNAME: SESSION cache hit $1"
  1980. cat "${cache_file}"
  1981. return 0
  1982. fi
  1983. declare -A services
  1984. services_uses=()
  1985. ## XXXvlab: bwerk, leveraging cache to be able to get the errorlevel here.
  1986. _get_services_uses "$@" || return 1
  1987. array_read-0 services_uses < <(_get_services_uses "$@")
  1988. services_provides=()
  1989. ## XXXvlab: bwerk, leveraging cache to be able to get the errorlevel here.
  1990. _get_services_provides "$@" || return 1
  1991. array_read-0 services_provides < <(_get_services_provides "$@")
  1992. for service in "$@"; do
  1993. services[$service]=1
  1994. done
  1995. all_services=("$@")
  1996. while [ "${#all_services[@]}" != 0 ]; do
  1997. array_pop all_services service
  1998. while read-0 relation_name ts relation_config tech_dep; do
  1999. [ "${without_relations[$service:$relation_name]}" ] && {
  2000. debug "Ignoring compose $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> ${DARKYELLOW}$ts$NORMAL"
  2001. continue
  2002. }
  2003. printf "%s\0" "$service" "$relation_name" "$ts" "$relation_config" "$tech_dep"
  2004. ## adding target services ?
  2005. [ "${services[$ts]}" ] && continue
  2006. array_read-0 services_uses < <(_get_services_uses "$ts")
  2007. all_services+=("$ts")
  2008. services[$ts]=1
  2009. done < <(get_compose_relations "$service")
  2010. done > "${cache_file}.wip"
  2011. while true; do
  2012. changed=
  2013. new_services_uses=()
  2014. summon=()
  2015. required=()
  2016. recommended=()
  2017. optional=()
  2018. while [ "${#services_uses[@]}" != 0 ]; do
  2019. service="${services_uses[0]}"
  2020. relation_name="${services_uses[1]}"
  2021. relation_def="${services_uses[2]}"
  2022. services_uses=("${services_uses[@]:3}")
  2023. [ "${without_relations[$service:$relation_name]}" ] && {
  2024. debug "Skipping $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW*$NORMAL"
  2025. continue
  2026. }
  2027. default_options=$(printf "%s" "$relation_def" | shyaml -y get-value "default-options" 2>/dev/null)
  2028. ## is this "use" declaration satisfied ?
  2029. found=
  2030. while read-0 s rn ts rc td; do
  2031. if [ -z "$found" -a "$service" == "$s" -a "$relation_name" == "$rn" ]; then
  2032. if [ "$default_options" ]; then
  2033. rc=$(merge_yaml_str "$default_options" "$rc") || return 1
  2034. fi
  2035. found="$ts"
  2036. fi
  2037. printf "%s\0" "$s" "$rn" "$ts" "$rc" "$td"
  2038. done < "${cache_file}.wip" > "${cache_file}.wip.new"
  2039. mv "${cache_file}.wip.new" "${cache_file}.wip"
  2040. if [ "$found" ]; then ## this "use" declaration was satisfied
  2041. debug "${DARKYELLOW}$service${NORMAL} use declaration for relation" \
  2042. "${DARKBLUE}$relation_name${NORMAL} is satisfied with ${DARKYELLOW}$found${NORMAL}"
  2043. continue
  2044. fi
  2045. auto=$(echo "$relation_def" | shyaml get-value auto pair 2>/dev/null)
  2046. case "$auto" in
  2047. "pair"|"summon")
  2048. service_list=()
  2049. array_read-0 service_list < <(array_keys_to_stdin services)
  2050. providers=()
  2051. array_read-0 providers providers_def < <(_get_services_providing "$relation_name" "${service_list[@]}")
  2052. if [ "${#providers[@]}" == 1 ]; then
  2053. ts="${providers[0]}"
  2054. debug "Auto-pairs ${DARKYELLOW}$service${NORMAL}" \
  2055. "--${DARKBLUE}$relation_name${NORMAL}--> ${DARKYELLOW}$ts${NORMAL}"
  2056. _out_new_relation_from_defs "$service" "$relation_name" "$ts" \
  2057. "${providers_def[0]}" "$relation_def" \
  2058. >> "${cache_file}.wip"
  2059. ## Adding service
  2060. [ "${services[$ts]}" ] && continue
  2061. array_read-0 new_services_uses < <(_get_services_uses "$ts")
  2062. services[$ts]=1
  2063. changed=1
  2064. continue
  2065. elif [ "${#providers[@]}" -gt 1 ]; then
  2066. msg=""
  2067. warn "No auto-pairing ${DARKYELLOW}$service${NORMAL}" \
  2068. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  2069. "(> 1 provider)."
  2070. continue
  2071. else
  2072. if [ "$auto" == "summon" ]; then
  2073. summon+=("$service" "$relation_name" "$relation_def")
  2074. fi
  2075. fi
  2076. ;;
  2077. ""|null|disable|disabled)
  2078. :
  2079. ;;
  2080. *)
  2081. err "Invalid ${WHITE}auto${NORMAL} value '$auto'."
  2082. return 1
  2083. ;;
  2084. esac
  2085. constraint=$(echo "$relation_def" | shyaml get-value constraint auto-pair 2>/dev/null)
  2086. case "$constraint" in
  2087. "required")
  2088. required+=("$service" "$relation_name" "$relation_def")
  2089. ;;
  2090. "recommended")
  2091. recommended+=("$service" "$relation_name" "$relation_def")
  2092. ;;
  2093. "optional")
  2094. optional+=("$service" "$relation_name" "$relation_def")
  2095. ;;
  2096. *)
  2097. err "Invalid ${WHITE}constraint${NORMAL} value '$constraint'."
  2098. return 1
  2099. ;;
  2100. esac
  2101. new_services_uses+=("$service" "$relation_name" "$relation_def") ## re-queue it
  2102. done
  2103. services_uses=("${new_services_uses[@]}")
  2104. if [ "$changed" ]; then
  2105. continue
  2106. fi
  2107. ## situation is stable
  2108. if [ "${#summon[@]}" != 0 ]; then
  2109. while [ "${#summon[@]}" != 0 ]; do
  2110. service="${summon[0]}"
  2111. relation_name="${summon[1]}"
  2112. relation_def="${summon[2]}"
  2113. summon=("${summon[@]:3}")
  2114. providers=()
  2115. providers_def=()
  2116. array_read-0 providers providers_def < <(_get_charm_providing "$relation_name" "${service_list[@]}")
  2117. if [ "${#providers[@]}" == 0 ]; then
  2118. die "Summoning a ${DARKBLUE}$relation_name${NORMAL} provider failed: none were found in charm store."
  2119. fi
  2120. if [ "${#providers[@]}" -gt 1 ]; then
  2121. warn "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  2122. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  2123. "(> 1 provider). Choosing first."
  2124. fi
  2125. ts="${providers[0]}"
  2126. ## YYYvlab: should be seen even in no debug mode no ?
  2127. debug "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  2128. "--${DARKBLUE}$relation_name${NORMAL}--> ${DARKYELLOW}$ts${NORMAL}"
  2129. _out_new_relation_from_defs "$service" "$relation_name" "$ts" \
  2130. "${providers_def[0]}" "$relation_def" \
  2131. >> "${cache_file}.wip"
  2132. ## Adding service
  2133. [ "${services[$ts]}" ] && continue
  2134. array_read-0 services_uses < <(_get_services_uses "$ts")
  2135. services[$ts]=1
  2136. changed=1
  2137. done
  2138. continue
  2139. fi
  2140. [ "$NO_CONSTRAINT_CHECK" ] && break
  2141. if [ "${#required[@]}" != 0 ]; then
  2142. echo "$(_display_solves required)" | sed -r "s/^/${RED}||${NORMAL} /g" >&2
  2143. err "Required relations not satisfied"
  2144. return 1
  2145. fi
  2146. if [ "${#recommended[@]}" != 0 ]; then
  2147. ## make recommendation
  2148. echo "$(_display_solves recommended)" | sed -r "s/^/${YELLOW}||${NORMAL} /g" >&2
  2149. fi
  2150. if [ "${#optional[@]}" != 0 ]; then
  2151. ## inform about options
  2152. echo "$(_display_solves optional)" | sed -r "s/^/${BLUE}||${NORMAL} /g" >&2
  2153. fi
  2154. # if [ "${#required[@]}" != 0 ]; then
  2155. # err "Required relations not satisfied"
  2156. # return 1
  2157. # fi
  2158. if [ "${#recommended[@]}" != 0 ]; then
  2159. warn "Recommended relations not satisfied"
  2160. fi
  2161. break
  2162. done
  2163. if [ "$?" != 0 ]; then
  2164. rm -f "${cache_file}"{,.wip,.wip.new} ## no cache
  2165. return 1
  2166. fi
  2167. export ALL_RELATIONS="$cache_file"
  2168. mv "${cache_file}"{.wip,}
  2169. cat "$cache_file"
  2170. }
  2171. export -f get_all_relations
  2172. _display_solves() {
  2173. local array_name="$1" by_relation msg
  2174. ## inform about options
  2175. msg=""
  2176. declare -A by_relation
  2177. while read-0 service relation_name relation_def; do
  2178. solves=$(printf "%s" "$relation_def" | shyaml -y get-value solves 2>/dev/null);
  2179. auto=$(printf "%s" "$relation_def" | shyaml get-value auto 2>/dev/null);
  2180. if [ -z "$solves" ]; then
  2181. continue
  2182. fi
  2183. by_relation[$relation_name]+=$(printf "\n %s" "${DARKYELLOW}$service$NORMAL for:")
  2184. if [ "$auto" == "pair" ]; then
  2185. requirement="add provider in cluster to auto-pair"
  2186. else
  2187. requirement="add explicit relation"
  2188. fi
  2189. while read-0 name def; do
  2190. by_relation[$relation_name]+=$(printf "\n - ${DARKCYAN}%-15s${NORMAL} %s (%s)" "$name" "$def" "$requirement")
  2191. done < <(printf "%s" "$solves" | shyaml key-values-0)
  2192. done < <(array_values_to_stdin "$array_name")
  2193. while read-0 relation_name message; do
  2194. msg+="$(printf "\n${DARKBLUE}%s$NORMAL provider is $array_name by%s" \
  2195. "$relation_name" "$message" )"
  2196. done < <(array_kv_to_stdin by_relation)
  2197. if [ "$msg" ]; then
  2198. printf "%s\n" "${msg:1}"
  2199. fi
  2200. }
  2201. get_compose_relation_def() {
  2202. local service="$1" relation="$2" relation_name target_service relation_config tech_dep
  2203. while read-0 relation_name target_service relation_config tech_dep; do
  2204. [ "$relation_name" == "$relation" ] || continue
  2205. printf "%s\0%s\0%s\0" "$target_service" "$relation_config" "$tech_dep"
  2206. done < <(get_compose_relations "$service") || return 1
  2207. }
  2208. export -f get_compose_relation_def
  2209. run_service_relations () {
  2210. local service services loaded subservices subservice
  2211. PROJECT_NAME=$(get_default_project_name) || return 1
  2212. export PROJECT_NAME
  2213. declare -A loaded
  2214. subservices=$(get_ordered_service_dependencies "$@") || return 1
  2215. for service in $subservices; do
  2216. # debug "Upping dep's relations of ${DARKYELLOW}$service${NORMAL}:"
  2217. for subservice in $(get_service_deps "$service") "$service"; do
  2218. [ "${loaded[$subservice]}" ] && continue
  2219. export BASE_SERVICE_NAME=$service
  2220. MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
  2221. MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1
  2222. RELATION_BASE_COMPOSE_DEF=$(get_compose_service_def "$subservice") || return 1
  2223. export RELATION_BASE_COMPOSE_DEF MASTER_BASE_{CHARM,SERVICE}_NAME
  2224. # debug " Relations of ${DARKYELLOW}$subservice${NORMAL}:"
  2225. while read-0 relation_name target_service relation_config tech_dep; do
  2226. export relation_config
  2227. export TARGET_SERVICE_NAME=$target_service
  2228. MASTER_TARGET_SERVICE_NAME=$(get_top_master_service_for_service "$target_service") || return 1
  2229. MASTER_TARGET_CHARM_NAME=$(get_service_charm "$MASTER_TARGET_SERVICE_NAME") || return 1
  2230. RELATION_TARGET_COMPOSE_DEF=$(get_compose_service_def "$target_service") || return 1
  2231. export RELATION_TARGET_COMPOSE_DEF MASTER_TARGET_{CHARM,SERVICE}_NAME
  2232. Wrap "${wrap_opts[@]}" -d "Building $DARKYELLOW$subservice$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW$target_service$NORMAL" <<EOF || return 1
  2233. _run_service_relation "$relation_name" "$subservice" "$target_service" "\$relation_config"
  2234. EOF
  2235. done < <(get_service_relations "$subservice") || return 1
  2236. loaded[$subservice]=1
  2237. done
  2238. done
  2239. }
  2240. export -f run_service_relations
  2241. _run_service_action_direct() {
  2242. local service="$1" action="$2" charm _dummy
  2243. shift; shift
  2244. read-0 charm || true ## against 'set -e' that could be setup in parent scripts
  2245. if read-0 _dummy || [ "$_dummy" ]; then
  2246. print_syntax_error "$FUNCNAME: too many arguments in action descriptor"
  2247. return 1
  2248. fi
  2249. export action_errlvl_file="$state_tmpdir/action-$service-$charm-$action-errlvl"
  2250. export state_tmpdir
  2251. (
  2252. set +e ## Prevents unwanted leaks from parent shell
  2253. export COMPOSE_CONFIG=$(get_compose_yml_content)
  2254. export METADATA_CONFIG=$(charm.metadata "$charm")
  2255. export SERVICE_NAME=$service
  2256. export ACTION_NAME=$action
  2257. export CONTAINER_NAME=$(get_top_master_service_for_service "$service")
  2258. export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME")
  2259. export SERVICE_DATASTORE="$DATASTORE/$service"
  2260. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  2261. exname="$exname $ACTION_NAME $SERVICE_NAME" \
  2262. stdbuf -oL -eL bash -c 'charm.run_direct_action "$@"' -- "$charm" "$action" "$@"
  2263. )
  2264. }
  2265. export -f _run_service_action_direct
  2266. _run_service_action_relation() {
  2267. local service="$1" action="$2" charm target_charm relation_name relation_config _dummy
  2268. shift; shift
  2269. read-0 charm target_service target_charm relation_name relation_config || true
  2270. if read-0 _dummy || [ "$_dummy" ]; then
  2271. print_syntax_error "$FUNCNAME: too many arguments in action descriptor"
  2272. return 1
  2273. fi
  2274. export RELATION_DATA_FILE=$(get_relation_data_file "$service" "$target_service" "$relation_name" "$relation_config")
  2275. export action_errlvl_file="$state_tmpdir/action-$service-$charm-$action-errlvl"
  2276. export state_tmpdir
  2277. {
  2278. (
  2279. set +e ## Prevents unwanted leaks from parent shell
  2280. export METADATA_CONFIG=$(charm.metadata "$charm")
  2281. export SERVICE_NAME=$service
  2282. export RELATION_TARGET_SERVICE="$target_service"
  2283. export RELATION_TARGET_CHARM="$target_charm"
  2284. export RELATION_BASE_SERVICE="$service"
  2285. export RELATION_BASE_CHARM="$charm"
  2286. export ACTION_NAME=$action
  2287. export CONTAINER_NAME=$(get_top_master_service_for_service "$service")
  2288. export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME")
  2289. export SERVICE_DATASTORE="$DATASTORE/$service"
  2290. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  2291. exname="$exname $ACTION_NAME $SERVICE_NAME" \
  2292. stdbuf -oL -eL bash -c 'charm.run_relation_action "$@"' -- "$target_charm" "$relation_name" "$action" "$@"
  2293. echo "$?" > "$action_errlvl_file"
  2294. ) | logstdout "$DARKYELLOW$service$NORMAL/${DARKCYAN}$action${NORMAL} ${GREEN}@${NORMAL}"
  2295. } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$service$NORMAL/${DARKCYAN}$action${NORMAL} ${RED}@$NORMAL" 3>&1 1>&2 2>&3
  2296. if ! [ -e "$action_errlvl_file" ]; then
  2297. err "Action $DARKYELLOW$service$NORMAL:$DARKCYAN$action$NORMAL has failed without having time" \
  2298. "to output an errlvl"
  2299. return 1
  2300. fi
  2301. return "$(cat "$action_errlvl_file")"
  2302. }
  2303. export -f _run_service_action_relation
  2304. get_relation_data_dir() {
  2305. local service="$1" target_service="$2" relation_name="$3" \
  2306. cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  2307. if [ -e "$cache_file" ]; then
  2308. # debug "$FUNCNAME: cache hit ($*)"
  2309. cat "$cache_file"
  2310. return 0
  2311. fi
  2312. project=$(get_default_project_name) || return 1
  2313. relation_dir="$VARDIR/relations/$project/${service}-${target_service}/$relation_name"
  2314. if ! [ -d "$relation_dir" ]; then
  2315. mkdir -p "$relation_dir" || return 1
  2316. chmod go-rwx "$relation_dir" || return 1 ## protecting this directory
  2317. fi
  2318. echo "$relation_dir" | tee "$cache_file"
  2319. }
  2320. export -f get_relation_data_dir
  2321. get_relation_data_file() {
  2322. local service="$1" target_service="$2" relation_name="$3" relation_config="$4"
  2323. relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1
  2324. relation_data_file="$relation_dir/data"
  2325. new=
  2326. if [ -e "$relation_data_file" ]; then
  2327. ## Has reference changed ?
  2328. new_md5=$(echo "$relation_config" | md5_compat)
  2329. if [ "$new_md5" != "$(cat "$relation_data_file.md5_ref" 2>/dev/null)" ]; then
  2330. new=true
  2331. fi
  2332. else
  2333. new=true
  2334. fi
  2335. if [ "$new" ]; then
  2336. echo "$relation_config" > "$relation_data_file"
  2337. chmod go-rwx "$relation_data_file" ## protecting this file
  2338. echo "$relation_config" | md5_compat > "$relation_data_file.md5_ref"
  2339. fi
  2340. echo "$relation_data_file"
  2341. }
  2342. export -f get_relation_data_file
  2343. has_service_action () {
  2344. local service="$1" action="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2345. charm target_charm relation_name target_service relation_config _tech_dep
  2346. if [ -e "$cache_file" ]; then
  2347. # debug "$FUNCNAME: cache hit ($*)"
  2348. cat "$cache_file"
  2349. return 0
  2350. fi
  2351. charm=$(get_service_charm "$service") || return 1
  2352. ## Action directly provided ?
  2353. if charm.has_direct_action "$charm" "$action" >/dev/null; then
  2354. echo -en "direct\0$charm" | tee "$cache_file"
  2355. return 0
  2356. fi
  2357. ## Action provided by relation ?
  2358. while read-0 relation_name target_service relation_config _tech_dep; do
  2359. target_charm=$(get_service_charm "$target_service") || return 1
  2360. if charm.has_relation_action "$target_charm" "$relation_name" "$action" >/dev/null; then
  2361. echo -en "relation\0$charm\0$target_service\0$target_charm\0$relation_name\0$relation_config" | tee "$cache_file"
  2362. return 0
  2363. fi
  2364. done < <(get_service_relations "$service")
  2365. return 1
  2366. # master=$(get_top_master_service_for_service "$service")
  2367. # [ "$master" == "$charm" ] && return 1
  2368. # has_service_action "$master" "$action"
  2369. }
  2370. export -f has_service_action
  2371. run_service_action () {
  2372. local service="$1" action="$2"
  2373. shift ; shift
  2374. {
  2375. if ! read-0 action_type; then
  2376. info "Service $DARKYELLOW$service$NORMAL does not have any action $DARKCYAN$action$NORMAL defined."
  2377. info " Add an executable script to 'actions/$action' to implement action."
  2378. return 1
  2379. fi
  2380. "_run_service_action_${action_type}" "$service" "$action" "$@"
  2381. } < <(has_service_action "$service" "$action")
  2382. }
  2383. export -f run_service_action
  2384. get_compose_relation_config() {
  2385. local service=$1 relation_config cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  2386. if [ -e "$cache_file" ]; then
  2387. # debug "$FUNCNAME: cache hit ($*)"
  2388. cat "$cache_file"
  2389. return 0
  2390. fi
  2391. compose_service_def=$(get_compose_service_def "$service") || return 1
  2392. echo "$compose_service_def" | shyaml get-value "relations" 2>/dev/null | tee "$cache_file"
  2393. }
  2394. export -f get_compose_relation_config
  2395. # ## Return key-values-0
  2396. # get_compose_relation_config_for_service() {
  2397. # local service=$1 relation_name=$2 relation_config
  2398. # compose_service_relations=$(get_compose_relation_config "$service") || return 1
  2399. # if ! relation_config=$(
  2400. # echo "$compose_service_relations" |
  2401. # shyaml get-value "${relation_name}" 2>/dev/null); then
  2402. # err "Couldn't find $DARKYELLOW${service}$NORMAL/${WHITE}${relation_name}$NORMAL" \
  2403. # "relation config in compose configuration."
  2404. # return 1
  2405. # fi
  2406. # if [ -z "$relation_config" ]; then
  2407. # err "Relation ${WHITE}mysql-database$NORMAL is empty in compose configuration."
  2408. # return 1
  2409. # fi
  2410. # if ! echo "$relation_config" | shyaml key-values-0 2>/dev/null; then
  2411. # err "No key/values in ${DARKBLUE}mysql-database$NORMAL of compose config."
  2412. # return 1
  2413. # fi
  2414. # }
  2415. # export -f get_compose_relation_config_for_service
  2416. _get_container_relation() {
  2417. local metadata=$1 found relation_name relation_def
  2418. found=
  2419. while read-0 relation_name relation_def; do
  2420. [ "$(echo "$relation_def" | shyaml get-value "scope" 2>/dev/null)" == "container" ] && {
  2421. found="$relation_name"
  2422. break
  2423. }
  2424. done < <(_get_charm_metadata_uses "$metadata")
  2425. if [ -z "$found" ]; then
  2426. die "Charm $DARKPINK$charm$NORMAL is a subordinate but does not have any required relation declaration with" \
  2427. "${WHITE}scope${NORMAL} set to 'container'."
  2428. fi
  2429. printf "%s" "$found"
  2430. }
  2431. _get_master_service_for_service_cached () {
  2432. local service="$1" charm="$2" metadata="$3" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  2433. charm requires master_charm target_charm target_service service_def found
  2434. if [ -e "$cache_file" ]; then
  2435. # debug "$FUNCNAME: STATIC cache hit ($1)"
  2436. cat "$cache_file" &&
  2437. touch "$cache_file" || return 1
  2438. return 0
  2439. fi
  2440. if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" != "True" ]; then
  2441. ## just return service name
  2442. echo "$service" | tee "$cache_file"
  2443. return 0
  2444. fi
  2445. ## Action provided by relation ?
  2446. container_relation=$(_get_container_relation "$metadata") || exit 1
  2447. read-0 target_service _ _ < <(get_service_relation "$service" "$container_relation")
  2448. if [ -z "$target_service" ]; then
  2449. err "Couldn't find ${WHITE}relations.${container_relation}${NORMAL} in" \
  2450. "${DARKYELLOW}$service$NORMAL compose definition."
  2451. err ${FUNCNAME[@]}
  2452. return 1
  2453. fi
  2454. echo "$target_service" | tee "$cache_file"
  2455. }
  2456. export -f _get_master_service_for_service_cached
  2457. get_master_service_for_service() {
  2458. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2459. charm metadata result
  2460. if [ -e "$cache_file" ]; then
  2461. # debug "$FUNCNAME: SESSION cache hit ($*)"
  2462. cat "$cache_file" || return 1
  2463. return 0
  2464. fi
  2465. charm=$(get_service_charm "$service") || return 1
  2466. metadata=$(charm.metadata "$charm" 2>/dev/null) || {
  2467. metadata=""
  2468. warn "No charm $DARKPINK$charm$NORMAL found."
  2469. }
  2470. result=$(_get_master_service_for_service_cached "$service" "$charm" "$metadata") || return 1
  2471. echo "$result" | tee "$cache_file" || return 1
  2472. }
  2473. export -f get_master_service_for_service
  2474. get_top_master_service_for_service() {
  2475. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2476. current_service
  2477. if [ -e "$cache_file" ]; then
  2478. # debug "$FUNCNAME: cache hit ($*)"
  2479. cat "$cache_file"
  2480. return 0
  2481. fi
  2482. current_service="$service"
  2483. while true; do
  2484. master_service=$(get_master_service_for_service "$current_service") || return 1
  2485. [ "$master_service" == "$current_service" ] && break
  2486. current_service="$master_service"
  2487. done
  2488. echo "$current_service" | tee "$cache_file"
  2489. return 0
  2490. }
  2491. export -f get_top_master_service_for_service
  2492. ##
  2493. ## The result is a mixin that is not always a complete valid
  2494. ## docker-compose entry (thinking of subordinates). The result
  2495. ## will be merge with master charms.
  2496. _get_docker_compose_mixin_from_metadata_cached() {
  2497. local service="$1" charm="$2" metadata="$3" \
  2498. has_build_dir="$4" \
  2499. cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  2500. metadata_file metadata volumes docker_compose subordinate image mixin mixins
  2501. if [ -e "$cache_file" ]; then
  2502. #debug "$FUNCNAME: STATIC cache hit $1"
  2503. cat "$cache_file" &&
  2504. touch "$cache_file" || return 1
  2505. return 0
  2506. fi
  2507. mixins=("$(echo -en "labels:\n- compose.charm=$charm")")
  2508. if [ "$metadata" ]; then
  2509. ## resources to volumes
  2510. volumes=$(
  2511. for resource_type in data config; do
  2512. while read-0 resource; do
  2513. eval "echo \" - \$HOST_${resource_type^^}STORE/\$service\$resource:\$resource:rw\""
  2514. done < <(echo "$metadata" | shyaml get-values-0 "${resource_type}-resources" 2>/dev/null)
  2515. done
  2516. while read-0 resource; do
  2517. if [[ "$resource" == /*:/*:* ]]; then
  2518. echo " - $resource"
  2519. elif [[ "$resource" == /*:/* ]]; then
  2520. echo " - $resource:rw"
  2521. elif [[ "$resource" == /*:* ]]; then
  2522. echo " - ${resource%%:*}:$resource"
  2523. elif [[ "$resource" =~ ^/[^:]+$ ]]; then
  2524. echo " - $resource:$resource:rw"
  2525. else
  2526. die "Invalid host-resource specified in 'metadata.yml'."
  2527. fi
  2528. done < <(printf "%s" "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null)
  2529. while read-0 resource; do
  2530. dest="$(charm.get_dir "$charm")/resources$resource"
  2531. if ! [ -e "$dest" ]; then
  2532. die "charm-resource: '$resource' does not exist (file: '$dest')."
  2533. fi
  2534. echo " - $dest:$resource:ro"
  2535. done < <(echo "$metadata" | shyaml get-values-0 "charm-resources" 2>/dev/null)
  2536. ) || return 1
  2537. if [ "$volumes" ]; then
  2538. mixins+=("volumes:"$'\n'"$volumes")
  2539. fi
  2540. type="$(printf "%s" "$metadata" | shyaml get-value type 2>/dev/null)" || true
  2541. if [ "$type" != "run-once" ]; then
  2542. mixins+=("restart: unless-stopped")
  2543. fi
  2544. docker_compose=$(printf "%s" "$metadata" | shyaml get-value -y "docker-compose" 2>/dev/null) || true
  2545. if [ "$docker_compose" ]; then
  2546. mixins+=("$docker_compose")
  2547. fi
  2548. if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" == "True" ]; then
  2549. subordinate=true
  2550. fi
  2551. fi
  2552. image=$(echo "$metadata" | shyaml get-value "docker-image" 2>/dev/null)
  2553. [ "$image" == "None" ] && image=""
  2554. if [ "$image" ]; then
  2555. if [ "$subordinate" ]; then
  2556. err "Subordinate charm can not have a ${WHITE}docker-image${NORMAL} value."
  2557. return 1
  2558. fi
  2559. mixins+=("image: $image")
  2560. elif [ "$has_build_dir" ]; then
  2561. if [ "$subordinate" ]; then
  2562. err "Subordinate charm can not have a 'build' sub directory."
  2563. return 1
  2564. fi
  2565. mixins+=("build: $(charm.get_dir "$charm")/build")
  2566. fi
  2567. mixin=$(merge_yaml_str "${mixins[@]}") || {
  2568. err "Failed to merge mixins from ${DARKPINK}${charm}${NORMAL} metadata."
  2569. return 1
  2570. }
  2571. echo "$mixin" | tee "$cache_file"
  2572. }
  2573. export -f _get_docker_compose_mixin_from_metadata_cached
  2574. get_docker_compose_mixin_from_metadata() {
  2575. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  2576. if [ -e "$cache_file" ]; then
  2577. #debug "$FUNCNAME: SESSION cache hit ($*)"
  2578. cat "$cache_file"
  2579. return 0
  2580. fi
  2581. charm=$(get_service_charm "$service") || return 1
  2582. metadata="$(charm.metadata "$charm" 2>/dev/null)" || return 1
  2583. has_build_dir=
  2584. [ -d "$(charm.get_dir "$charm")/build" ] && has_build_dir=true
  2585. mixin=$(_get_docker_compose_mixin_from_metadata_cached "$service" "$charm" "$metadata" "$has_build_dir") || return 1
  2586. echo "$mixin" | tee "$cache_file"
  2587. }
  2588. export -f get_docker_compose_mixin_from_metadata
  2589. _save() {
  2590. local name="$1"
  2591. cat - | tee -a "$docker_compose_dir/.data/$name"
  2592. }
  2593. export -f _save
  2594. get_default_project_name() {
  2595. if [ "$DEFAULT_PROJECT_NAME" ]; then
  2596. echo "$DEFAULT_PROJECT_NAME"
  2597. return 0
  2598. fi
  2599. compose_yml_location="$(get_compose_yml_location)" || return 1
  2600. if [ "$compose_yml_location" ]; then
  2601. if normalized_path=$(readlink -f "$compose_yml_location"); then
  2602. name="$(basename "$(dirname "$normalized_path")")"
  2603. echo "${name%%-deploy}"
  2604. return 0
  2605. fi
  2606. fi
  2607. echo "orphan"
  2608. return 0
  2609. }
  2610. export -f get_default_project_name
  2611. get_running_compose_containers() {
  2612. ## XXXvlab: docker bug: there will be a final newline anyway
  2613. docker ps --filter label="compose.service" --format='{{.ID}}'
  2614. }
  2615. export -f get_running_compose_containers
  2616. get_volumes_for_container() {
  2617. local container="$1"
  2618. docker inspect \
  2619. --format '{{range $mount := .Mounts}}{{$mount.Source}}{{"\x00"}}{{$mount.Destination}}{{"\x00"}}{{end}}' \
  2620. "$container"
  2621. }
  2622. export -f get_volumes_for_container
  2623. is_volume_used() {
  2624. local volume="$1" container_id src dst
  2625. while read -r container_id; do
  2626. while read-0 src dst; do
  2627. [[ "$src/" == "$volume"/* ]] && return 0
  2628. done < <(get_volumes_for_container "$container_id")
  2629. done < <(get_running_compose_containers)
  2630. return 1
  2631. }
  2632. export -f is_volume_used
  2633. clean_unused_docker_compose() {
  2634. for f in /var/lib/compose/docker-compose/*; do
  2635. [ -e "$f" ] || continue
  2636. is_volume_used "$f" && continue
  2637. debug "Cleaning unused docker-compose ${f##*/}"
  2638. rm -rf "$f" || return 1
  2639. done
  2640. return 0
  2641. }
  2642. export -f clean_unused_docker_compose
  2643. stdin_get_hash() {
  2644. local sha
  2645. sha=$(sha256sum) || return 1
  2646. sha=${sha:0:64}
  2647. echo "$sha"
  2648. }
  2649. export -f stdin_get_hash
  2650. file_get_hash() {
  2651. stdin_get_hash < "$1" || return 1
  2652. }
  2653. export -f file_get_hash
  2654. docker_compose_store() {
  2655. local file="$1" sha
  2656. sha=$(file_get_hash "$file") || return 1
  2657. project=$(get_default_project_name) || return 1
  2658. dst="/var/lib/compose/docker-compose/$sha/$project"
  2659. mkdir -p "$dst" || return 1
  2660. cat <<EOF > "$dst/.env" || return 1
  2661. DOCKER_COMPOSE_PATH=$dst
  2662. EOF
  2663. cp "$file" "$dst/docker-compose.yml" || return 1
  2664. mkdir -p "$dst/bin" || return 1
  2665. cat <<EOF > "$dst/bin/dc" || return 1
  2666. #!/bin/bash
  2667. $(declare -f read-0)
  2668. docker_run_opts=()
  2669. while read-0 opt; do
  2670. docker_run_opts+=("\$opt")
  2671. done < <(cat "$COMPOSE_LAUNCHER_OPTS")
  2672. docker_run_opts+=(
  2673. "-w" "$dst"
  2674. "--entrypoint" "/usr/local/bin/docker-compose"
  2675. )
  2676. [ -t 1 ] && {
  2677. docker_run_opts+=("-ti")
  2678. }
  2679. exec docker run --rm "\${docker_run_opts[@]}" "${COMPOSE_DOCKER_IMAGE:-docker.0k.io/compose}" "\$@"
  2680. EOF
  2681. chmod +x "$dst/bin/dc" || return 1
  2682. printf "%s" "$sha"
  2683. }
  2684. export -f docker_compose_store
  2685. launch_docker_compose() {
  2686. local charm docker_compose_tmpdir docker_compose_dir
  2687. docker_compose_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX)
  2688. #debug "Creating temporary docker-compose directory in '$docker_compose_tmpdir'."
  2689. trap_add EXIT "rm -rf \"$docker_compose_tmpdir\""
  2690. ## docker-compose will name network from the parent dir name
  2691. project=$(get_default_project_name)
  2692. mkdir -p "$docker_compose_tmpdir/$project"
  2693. docker_compose_dir="$docker_compose_tmpdir/$project"
  2694. get_docker_compose $SERVICE_PACK > "$docker_compose_dir/docker-compose.yml" || return 1
  2695. if [ -e "$state_tmpdir/to-merge-in-docker-compose.yml" ]; then
  2696. # debug "Merging some config data in docker-compose.yml:"
  2697. # debug "$(cat $state_tmpdir/to-merge-in-docker-compose.yml)"
  2698. _config_merge "$docker_compose_dir/docker-compose.yml" "$state_tmpdir/to-merge-in-docker-compose.yml" || return 1
  2699. fi
  2700. if [ -z "$(echo $(cat "$docker_compose_dir/docker-compose.yml"))" ]; then
  2701. die "Generated 'docker-compose.yml' is unexpectedly empty."
  2702. fi
  2703. ## XXXvlab: could be more specific and only link the needed charms
  2704. ## XXXvlab: why do we need these links ? If this is for the build command, then it is not useful anymore.
  2705. # for charm in $(shyaml keys services < "$docker_compose_dir/docker-compose.yml"); do
  2706. # if charm.exists "$charm"; then
  2707. # ln -sf "$(charm.get_dir "$charm")" "$docker_compose_dir/$charm" || exit 1
  2708. # fi
  2709. # done
  2710. mkdir "$docker_compose_dir/.data"
  2711. if [ -z "$COMPOSE_DISABLE_DOCKER_COMPOSE_STORE" ]; then
  2712. sha=$(docker_compose_store "$docker_compose_dir/docker-compose.yml") || return 1
  2713. fi
  2714. {
  2715. {
  2716. {
  2717. if [ -z "$COMPOSE_DISABLE_DOCKER_COMPOSE_STORE" ]; then
  2718. cd "/var/lib/compose/docker-compose/$sha/$project"
  2719. else
  2720. cd "$docker_compose_dir"
  2721. fi
  2722. if [ -f ".env" ]; then
  2723. debug "${WHITE}.env$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:"
  2724. debug "$(cat ".env" | prefix " $GRAY|$NORMAL ")"
  2725. fi
  2726. debug "${WHITE}docker-compose.yml$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:"
  2727. debug "$(cat "docker-compose.yml" | prefix " $GRAY|$NORMAL ")"
  2728. debug "${WHITE}Launching$NORMAL: docker-compose $@"
  2729. if [ "$DRY_COMPOSE_RUN" ]; then
  2730. echo docker-compose "$@"
  2731. else
  2732. docker-compose "$@"
  2733. fi
  2734. echo "$?" > "$docker_compose_dir/.data/errlvl"
  2735. } | _save stdout
  2736. } 3>&1 1>&2 2>&3 | _save stderr
  2737. } 3>&1 1>&2 2>&3
  2738. if tail -n 1 "$docker_compose_dir/.data/stderr" | egrep "Service .+ failed to build: Error getting container [0-9a-f]+ from driver devicemapper: (open|Error mounting) /dev/mapper/docker-.*: no such file or directory$" >/dev/null 2>&1; then
  2739. err "Detected bug https://github.com/docker/docker/issues/4036 ... "
  2740. err "Please re-launch your command, or switch from 'devicemapper' driver to 'overlayfs' or 'aufs'."
  2741. fi
  2742. docker_compose_errlvl="$(cat "$docker_compose_dir/.data/errlvl" 2>/dev/null)"
  2743. if [ -z "$docker_compose_errlvl" ]; then
  2744. err "Something went wrong before you could gather docker-compose errorlevel."
  2745. return 1
  2746. fi
  2747. return "$docker_compose_errlvl"
  2748. }
  2749. export -f launch_docker_compose
  2750. get_compose_yml_location() {
  2751. if ! [ -z ${COMPOSE_YML_FILE+x} ]; then ## if set, even if empty
  2752. echo "$COMPOSE_YML_FILE"
  2753. return 0
  2754. fi
  2755. parent=$(while ! [ -f "./compose.yml" ]; do
  2756. [ "$PWD" == "/" ] && exit 0
  2757. cd ..
  2758. done; echo "$PWD"
  2759. )
  2760. if [ "$parent" ]; then
  2761. echo "$parent/compose.yml"
  2762. return 0
  2763. fi
  2764. ## XXXvlab: do we need this additional environment variable,
  2765. ## COMPOSE_YML_FILE is not sufficient ?
  2766. if [ "$DEFAULT_COMPOSE_FILE" ]; then
  2767. if ! [ -e "$DEFAULT_COMPOSE_FILE" ]; then
  2768. warn "No 'compose.yml' was found in current or parent dirs," \
  2769. "and \$DEFAULT_COMPOSE_FILE points to an unexistent file." \
  2770. "(${DEFAULT_COMPOSE_FILE})"
  2771. return 0
  2772. fi
  2773. echo "$DEFAULT_COMPOSE_FILE"
  2774. return 0
  2775. fi
  2776. warn "No 'compose.yml' was found in current or parent dirs, and no \$DEFAULT_COMPOSE_FILE was set."
  2777. return 0
  2778. }
  2779. export -f get_compose_yml_location
  2780. get_compose_yml_content() {
  2781. local cache_file="$state_tmpdir/$FUNCNAME.cache"
  2782. if [ -e "$cache_file" ]; then
  2783. cat "$cache_file" &&
  2784. touch "$cache_file" || return 1
  2785. return 0
  2786. fi
  2787. if [ -z "$COMPOSE_YML_FILE" ]; then
  2788. COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1
  2789. fi
  2790. if [ -e "$COMPOSE_YML_FILE" ]; then
  2791. debug "Found $WHITE$exname$NORMAL YAML file in '$COMPOSE_YML_FILE'."
  2792. COMPOSE_YML_CONTENT=$(cat "$COMPOSE_YML_FILE") || {
  2793. err "Could not read '$COMPOSE_YML_FILE'."
  2794. return 1
  2795. }
  2796. else
  2797. debug "No compose file found. Using an empty one."
  2798. COMPOSE_YML_CONTENT=""
  2799. fi
  2800. COMPOSE_YML_CONTENT=$(merge_yaml_str "$COMPOSE_YML_CONTENT" "${compose_contents[@]}") || return 1
  2801. output=$(echo "$COMPOSE_YML_CONTENT"| shyaml get-value 2>&1)
  2802. if [ "$?" != 0 ]; then
  2803. outputed_something=
  2804. while IFS='' read -r line1 && IFS='' read -r line2; do
  2805. [ "$outputed_something" ] || err "Invalid YAML in '$COMPOSE_YML_FILE':"
  2806. outputed_something=true
  2807. echo "$line1 $GRAY($line2)$NORMAL"
  2808. done < <(echo "$output" | grep ^yaml.scanner -A 100 |
  2809. sed -r 's/^ in "<stdin>", //g' | sed -r 's/^yaml.scanner.[a-zA-Z]+: //g') |
  2810. prefix " $GRAY|$NORMAL "
  2811. [ "$outputed_something" ] || {
  2812. err "Unexpected error while running 'shyaml get-value' on '$COMPOSE_YML_FILE':"
  2813. echo "$output" | prefix " $GRAY|$NORMAL "
  2814. }
  2815. return 1
  2816. fi
  2817. echo "$COMPOSE_YML_CONTENT" | tee "$cache_file" || return 1
  2818. }
  2819. export -f get_compose_yml_content
  2820. get_default_target_services() {
  2821. local services=("$@")
  2822. if [ -z "${services[*]}" ]; then
  2823. if [ "$DEFAULT_SERVICES" ]; then
  2824. debug "No service provided, using $WHITE\$DEFAULT_SERVICES$NORMAL variable." \
  2825. "Target services: $DARKYELLOW$DEFAULT_SERVICES$NORMAL"
  2826. services="$DEFAULT_SERVICES"
  2827. else
  2828. err "No service provided."
  2829. return 1
  2830. fi
  2831. fi
  2832. echo "${services[*]}"
  2833. }
  2834. export -f get_default_target_services
  2835. get_master_services() {
  2836. local loaded master_service service
  2837. declare -A loaded
  2838. for service in "$@"; do
  2839. master_service=$(get_top_master_service_for_service "$service") || return 1
  2840. if [ "${loaded[$master_service]}" ]; then
  2841. continue
  2842. fi
  2843. echo "$master_service"
  2844. loaded["$master_service"]=1
  2845. done | nspc
  2846. return "${PIPESTATUS[0]}"
  2847. }
  2848. export -f get_master_services
  2849. get_current_docker_container_id() {
  2850. local line
  2851. line=$(cat "/proc/self/cpuset") || return 1
  2852. [[ "$line" == *docker* ]] || return 1
  2853. echo "${line##*/}"
  2854. }
  2855. export -f get_current_docker_container_id
  2856. ## if we are in a docker compose, we might want to know what is the
  2857. ## real host path of some local paths.
  2858. get_host_path() {
  2859. local path="$1"
  2860. path=$(realpath "$path") || return 1
  2861. container_id=$(get_current_docker_container_id) || {
  2862. print "%s" "$path"
  2863. return 0
  2864. }
  2865. biggest_dst=
  2866. current_src=
  2867. while read-0 src dst; do
  2868. [[ "$path" == "$dst"* ]] || continue
  2869. if [[ "${#biggest_dst}" < "${#dst}" ]]; then
  2870. biggest_dst="$dst"
  2871. current_src="$src"
  2872. fi
  2873. done < <(get_volumes_for_container "$container_id")
  2874. if [ "$current_src" ]; then
  2875. printf "%s" "$current_src"
  2876. else
  2877. return 1
  2878. fi
  2879. }
  2880. export -f get_host_path
  2881. _setup_state_dir() {
  2882. export state_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX)
  2883. #debug "Creating temporary state directory in '$state_tmpdir'."
  2884. # trap_add EXIT "debug \"Removing temporary state directory in $state_tmpdir.\";\
  2885. # rm -rf \"$state_tmpdir\""
  2886. trap_add EXIT "rm -rf \"$state_tmpdir\""
  2887. }
  2888. get_docker_compose_help_msg() {
  2889. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  2890. docker_compose_help_msg
  2891. if [ -e "$cache_file" ]; then
  2892. cat "$cache_file" &&
  2893. touch "$cache_file" || return 1
  2894. return 0
  2895. fi
  2896. docker_compose_help_msg=$(docker-compose $action --help 2>/dev/null) || return 1
  2897. echo "$docker_compose_help_msg" |
  2898. tee "$cache_file" || return 1
  2899. }
  2900. get_docker_compose_usage() {
  2901. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  2902. docker_compose_help_msg
  2903. if [ -e "$cache_file" ]; then
  2904. cat "$cache_file" &&
  2905. touch "$cache_file" || return 1
  2906. return 0
  2907. fi
  2908. docker_compose_help_msg=$(get_docker_compose_help_msg $action) || return 1
  2909. echo "$docker_compose_help_msg" |
  2910. grep -m 1 "^Usage:" -A 10000 |
  2911. egrep -m 1 "^\$" -B 10000 |
  2912. nspc |
  2913. sed -r 's/^Usage: //g' |
  2914. tee "$cache_file" || return 1
  2915. }
  2916. get_docker_compose_opts_help() {
  2917. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  2918. docker_compose_help_msg
  2919. if [ -e "$cache_file" ]; then
  2920. cat "$cache_file" &&
  2921. touch "$cache_file" || return 1
  2922. return 0
  2923. fi
  2924. docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1
  2925. echo "$docker_compose_opts_help" |
  2926. grep '^Options:' -A 20000 |
  2927. tail -n +2 |
  2928. { cat ; echo; } |
  2929. egrep -m 1 "^\S*\$" -B 10000 |
  2930. head -n -1 |
  2931. tee "$cache_file" || return 1
  2932. }
  2933. get_docker_compose_commands_help() {
  2934. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  2935. docker_compose_help_msg
  2936. if [ -e "$cache_file" ]; then
  2937. cat "$cache_file" &&
  2938. touch "$cache_file" || return 1
  2939. return 0
  2940. fi
  2941. docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1
  2942. echo "$docker_compose_opts_help" |
  2943. grep '^Commands:' -A 20000 |
  2944. tail -n +2 |
  2945. { cat ; echo; } |
  2946. egrep -m 1 "^\S*\$" -B 10000 |
  2947. head -n -1 |
  2948. tee "$cache_file" || return 1
  2949. }
  2950. get_docker_compose_opts_list() {
  2951. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  2952. docker_compose_help_msg
  2953. if [ -e "$cache_file" ]; then
  2954. cat "$cache_file" &&
  2955. touch "$cache_file" || return 1
  2956. return 0
  2957. fi
  2958. docker_compose_opts_help=$(get_docker_compose_opts_help $action) || return 1
  2959. echo "$docker_compose_opts_help" |
  2960. egrep "^\s+-" |
  2961. sed -r 's/\s+((((-[a-zA-Z]|--[a-zA-Z0-9-]+)( [A-Z=]+|=[^ ]+)?)(, )?)+)\s+.*$/\1/g' |
  2962. tee "$cache_file" || return 1
  2963. }
  2964. options_parser() {
  2965. sed -r 's/^(\s+(((-[a-zA-Z]|--[a-zA-Z0-9-]+)([ =]([a-zA-Z_=\"\[]|\])+)?(, | )?)+)\s+)[^ ].*$/\x0\2\x0\0/g'
  2966. printf "\0"
  2967. }
  2968. remove_options_in_option_help_msg() {
  2969. {
  2970. read-0 null
  2971. if [ "$null" ]; then
  2972. err "options parsing error, should start with an option line."
  2973. return 1
  2974. fi
  2975. while read-0 opt full_txt;do
  2976. multi_opts="$(printf "%s " $opt | multi_opts_filter)"
  2977. single_opts="$(printf "%s " $opt | single_opts_filter)"
  2978. for to_remove in "$@"; do
  2979. str_matches "$to_remove" $multi_opts $single_opts && {
  2980. continue 2
  2981. }
  2982. done
  2983. echo -n "$full_txt"
  2984. done
  2985. } < <(options_parser)
  2986. }
  2987. _MULTIOPTION_REGEX='^((-[a-zA-Z]|--[a-zA-Z0-9-]+)(, )?)+'
  2988. _MULTIOPTION_REGEX_LINE_FILTER=$_MULTIOPTION_REGEX'(\s|=)'
  2989. multi_opts_filter() {
  2990. egrep "$_MULTIOPTION_REGEX_LINE_FILTER" |
  2991. sed -r "s/^($_MULTIOPTION_REGEX)(\s|=).*$/\1/g" |
  2992. tr ',' "\n" | nspc
  2993. }
  2994. single_opts_filter() {
  2995. egrep -v "$_MULTIOPTION_REGEX_LINE_FILTER" |
  2996. tr ',' "\n" | nspc
  2997. }
  2998. get_docker_compose_multi_opts_list() {
  2999. local action="$1" opts_list
  3000. opts_list=$(get_docker_compose_opts_list "$action") || return 1
  3001. echo "$opts_list" | multi_opts_filter
  3002. }
  3003. get_docker_compose_single_opts_list() {
  3004. local action="$1" opts_list
  3005. opts_list=$(get_docker_compose_opts_list "$action") || return 1
  3006. echo "$opts_list" | single_opts_filter
  3007. }
  3008. display_commands_help() {
  3009. local charm_actions
  3010. echo
  3011. echo "${WHITE}Commands${NORMAL} (thanks to docker-compose):"
  3012. get_docker_compose_commands_help | sed -r "s/ ([a-z]+)(\s+)/ ${DARKCYAN}\1${NORMAL}\2/g"
  3013. charm_actions_help=$(get_docker_charm_action_help) || return 1
  3014. if [ "$charm_actions_help" ]; then
  3015. echo
  3016. echo "${WHITE}Charm actions${NORMAL}:"
  3017. printf "%s\n" "$charm_actions_help" | \
  3018. sed -r "s/^ ([a-z0-9-]+)(\s+)([a-z0-9-]+)(\s+)/ ${DARKCYAN}\1${NORMAL}\2${DARKYELLOW}\3${NORMAL}\4/g"
  3019. fi
  3020. }
  3021. get_docker_charm_action_help() {
  3022. local services service charm relation_name target_service relation_config \
  3023. target_charm
  3024. services=($(get_compose_yml_content | shyaml keys 2>/dev/null))
  3025. for service in "${services[@]}"; do
  3026. out=$(
  3027. charm=$(get_service_charm "$service") || return 1
  3028. for action in $(charm.ls_direct_actions "$charm"); do
  3029. printf " %-28s %s\n" "$action $service" "Direct action from ${DARKPINK}$charm${NORMAL}"
  3030. done
  3031. while read-0 relation_name target_service _relation_config _tech_dep; do
  3032. target_charm=$(get_service_charm "$target_service") || return 1
  3033. for action in $(charm.ls_relation_actions "$target_charm" "$relation_name"); do
  3034. printf " %-28s %s\n" "$action $service" "Indirect action from ${DARKPINK}$target_charm${NORMAL}"
  3035. done
  3036. done < <(get_compose_relations "$service")
  3037. )
  3038. if [ "$out" ]; then
  3039. echo " for ${DARKYELLOW}$service${NORMAL}:"
  3040. printf "%s\n" "$out"
  3041. fi
  3042. done
  3043. }
  3044. display_help() {
  3045. print_help
  3046. echo "${WHITE}Usage${NORMAL}:"
  3047. echo " $usage"
  3048. echo "${WHITE}Options${NORMAL}:"
  3049. echo " -h, --help Print this message and quit"
  3050. echo " (ignoring any other options)"
  3051. echo " -V, --version Print current version and quit"
  3052. echo " (ignoring any other options)"
  3053. echo " --dirs Display data dirs and quit"
  3054. echo " (ignoring any other options)"
  3055. echo " -v, --verbose Be more verbose"
  3056. echo " -q, --quiet Be quiet"
  3057. echo " -d, --debug Print full debugging information (sets also verbose)"
  3058. echo " --dry-compose-run If docker-compose will be run, only print out what"
  3059. echo " command line will be used."
  3060. echo " --no-relations Do not run any relation script"
  3061. echo " --no-hooks Do not run any init script"
  3062. echo " --no-init Do not run any init script"
  3063. echo " --without-relation RELATION "
  3064. echo " Do not run given relation"
  3065. echo " -R, --rebuild-relations-to-service SERVICE"
  3066. echo " Will rebuild all relations to given service"
  3067. echo " --add-compose-content, -Y YAML"
  3068. echo " Will merge some direct YAML with the current compose"
  3069. get_docker_compose_opts_help | remove_options_in_option_help_msg --version --help --verbose |
  3070. filter_docker_compose_help_message
  3071. display_commands_help
  3072. }
  3073. _graph_service() {
  3074. local service="$1" base="$1"
  3075. charm=$(get_service_charm "$service") || return 1
  3076. metadata=$(charm.metadata "$charm") || return 1
  3077. subordinate=$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)
  3078. if [ "$subordinate" == "True" ]; then
  3079. requires="$(echo "$metadata" | shyaml get-value "requires" 2>/dev/null)"
  3080. master_charm=
  3081. while read-0 relation_name relation; do
  3082. [ "$(echo "$relation" | shyaml get-value "scope" 2>/dev/null)" == "container" ] || continue
  3083. interface="$(echo "$relation" | shyaml get-value "interface" 2>/dev/null)"
  3084. if [ -z "$interface" ]; then
  3085. err "No ${WHITE}$interface${NORMAL} set for relation $relation_name."
  3086. return 1
  3087. fi
  3088. ## Action provided by relation ?
  3089. target_service=
  3090. while read-0 relation_name candidate_target_service _relation_config _tech_dep; do
  3091. [ "$interface" == "$relation_name" ] && {
  3092. target_service="$candidate_target_service"
  3093. break
  3094. }
  3095. done < <(get_service_relations "$service")
  3096. if [ -z "$target_service" ]; then
  3097. err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \
  3098. "${DARKYELLOW}$service$NORMAL compose definition."
  3099. return 1
  3100. fi
  3101. master_service="$target_service"
  3102. master_charm=$(get_service_charm "$target_service") || return 1
  3103. break
  3104. done < <(echo "$requires" | shyaml key-values-0 2>/dev/null)
  3105. fi
  3106. _graph_node_service "$service" "$base" "$charm"
  3107. _graph_edge_service "$service" "$subordinate" "$master_service"
  3108. }
  3109. _graph_node_service() {
  3110. local service="$1" base="$2" charm="$3"
  3111. cat <<EOF
  3112. "$(_graph_node_service_label ${service})" [
  3113. style = "filled, $([ "$subordinate" == "True" ] && echo "dashed" || echo "bold")"
  3114. penwidth = $([ "$subordinate" == "True" ] && echo "3" || echo "5")
  3115. color = $([ "$base" ] && echo "blue" || echo "black")
  3116. fillcolor = "white"
  3117. fontname = "Courier New"
  3118. shape = "Mrecord"
  3119. label =<$(_graph_node_service_content "$service")>
  3120. ];
  3121. EOF
  3122. }
  3123. _graph_edge_service() {
  3124. local service="$1" subordinate="$2" master_service="$3"
  3125. while read-0 relation_name target_service relation_config tech_dep; do
  3126. cat <<EOF
  3127. "$(_graph_node_service_label ${service})" -> "$(_graph_node_service_label ${target_service})" [
  3128. penwidth = $([ "$master_service" == "$target_service" ] && echo 3 || echo 2)
  3129. fontsize = 16
  3130. fontcolor = "black"
  3131. style = $([ "$master_service" == "$target_service" ] && echo dashed || echo "\"\"")
  3132. weight = $([ "$master_service" == "$target_service" ] && echo 2.0 || echo 1.0)
  3133. dir = $([ "$master_service" == "$target_service" ] && echo none || echo both)
  3134. arrowtail = odot
  3135. # arrowhead = dotlicurve
  3136. taillabel = "$relation_name" ];
  3137. EOF
  3138. done < <(get_service_relations "$service") || return 1
  3139. }
  3140. _graph_node_service_label() {
  3141. local service="$1"
  3142. echo "service_$service"
  3143. }
  3144. _graph_node_service_content() {
  3145. local service="$1"
  3146. charm=$(get_service_charm "$service") || return 1
  3147. cat <<EOF
  3148. <table border="0" cellborder="0" cellpadding="3" bgcolor="white">
  3149. <tr>
  3150. <td bgcolor="black" align="center" colspan="2">
  3151. <font color="white">$service</font>
  3152. </td>
  3153. </tr>
  3154. $(if [ "$charm" != "$service" ]; then
  3155. cat <<EOF2
  3156. <tr>
  3157. <td align="left" port="r0">charm: $charm</td>
  3158. </tr>
  3159. EOF2
  3160. fi)
  3161. </table>
  3162. EOF
  3163. }
  3164. cla_contains () {
  3165. local e
  3166. for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done
  3167. return 1
  3168. }
  3169. filter_docker_compose_help_message() {
  3170. cat - |
  3171. sed -r "s/docker-compose run/${DARKWHITE}compose${NORMAL} ${DARKCYAN}$action${NORMAL}/g;
  3172. s/docker-compose.yml/compose.yml/g;
  3173. s/SERVICES?/${DARKYELLOW}\0${NORMAL}/g;
  3174. s/^(\s+)\\$/\1${WHITE}\$${NORMAL}/g;
  3175. s/^(\s+)run/\1${DARKCYAN}$action${NORMAL}/g;
  3176. s/docker-compose/${DARKWHITE}compose${NORMAL}/g"
  3177. }
  3178. graph() {
  3179. local services=("$@")
  3180. declare -A entries
  3181. cat <<EOF
  3182. digraph g {
  3183. graph [
  3184. fontsize=30
  3185. labelloc="t"
  3186. label=""
  3187. splines=true
  3188. overlap=false
  3189. #rankdir = "LR"
  3190. ];
  3191. ratio = auto;
  3192. EOF
  3193. for target_service in "$@"; do
  3194. services=$(get_ordered_service_dependencies "$target_service") || return 1
  3195. for service in $services; do
  3196. [ "${entries[$service]}" ] && continue || entries[$service]=1
  3197. if cla_contains "$service" "${services[@]}"; then
  3198. base=true
  3199. else
  3200. base=
  3201. fi
  3202. _graph_service "$service" "$base"
  3203. done
  3204. done
  3205. echo "}"
  3206. }
  3207. cached_wget() {
  3208. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  3209. url="$1"
  3210. if [ -e "$cache_file" ]; then
  3211. cat "$cache_file"
  3212. touch "$cache_file"
  3213. return 0
  3214. fi
  3215. wget -O- "${url}" |
  3216. tee "$cache_file"
  3217. if [ "${PIPESTATUS[0]}" != 0 ]; then
  3218. rm "$cache_file"
  3219. die "Unable to fetch '$url'."
  3220. return 1
  3221. fi
  3222. }
  3223. export -f cached_wget
  3224. [ "$SOURCED" ] && return 0
  3225. trap_add "EXIT" clean_cache
  3226. if [ -z "$DISABLE_SYSTEM_CONFIG_FILE" ]; then
  3227. if [ -r /etc/default/charm ]; then
  3228. . /etc/default/charm
  3229. fi
  3230. if [ -r "/etc/default/$exname" ]; then
  3231. . "/etc/default/$exname"
  3232. fi
  3233. ## XXXvlab: should provide YML config opportunities in possible parent dirs ?
  3234. ## userdir ? and global /etc/compose.yml ?
  3235. for cfgfile in /etc/compose.conf /etc/compose.local.conf \
  3236. /etc/default/compose /etc/compose/local.conf; do
  3237. [ -e "$cfgfile" ] || continue
  3238. . "$cfgfile" || die "Loading config file '$cfgfile' failed."
  3239. done
  3240. fi
  3241. _setup_state_dir
  3242. mkdir -p "$CACHEDIR" || exit 1
  3243. ##
  3244. ## Argument parsing
  3245. ##
  3246. wrap_opts=()
  3247. services=()
  3248. remainder_args=()
  3249. compose_opts=()
  3250. compose_contents=()
  3251. action_opts=()
  3252. services_args=()
  3253. pos_arg_ct=0
  3254. no_hooks=
  3255. no_init=
  3256. action=
  3257. stage="main" ## switches from 'main', to 'action', 'remainder'
  3258. is_docker_compose_action=
  3259. rebuild_relations_to_service=()
  3260. declare -A without_relations
  3261. DC_MATCH_MULTI=$(get_docker_compose_multi_opts_list) &&
  3262. DC_MATCH_SINGLE=$(get_docker_compose_single_opts_list) || exit 1
  3263. while read-0 arg; do
  3264. case "$stage" in
  3265. "main")
  3266. case "$arg" in
  3267. --help|-h)
  3268. no_init=true ; no_hooks=true ; no_relations=true
  3269. display_help
  3270. exit 0
  3271. ;;
  3272. --verbose|-v)
  3273. export VERBOSE=true
  3274. compose_opts+=("--verbose")
  3275. ;;
  3276. --quiet|-q)
  3277. export QUIET=true
  3278. export wrap_opts+=("-q")
  3279. ;;
  3280. --version|-V)
  3281. print_version
  3282. docker-compose --version
  3283. docker --version
  3284. exit 0
  3285. ;;
  3286. -f|--file)
  3287. read-0 value
  3288. [ -e "$value" ] || die "File $value doesn't exists"
  3289. export COMPOSE_YML_FILE="$value"
  3290. shift
  3291. ;;
  3292. -p|--project-name)
  3293. read-0 value
  3294. export DEFAULT_PROJECT_NAME="$value"
  3295. compose_opts+=("--project-name $value")
  3296. shift
  3297. ;;
  3298. --no-relations)
  3299. export no_relations=true
  3300. ;;
  3301. --without-relation)
  3302. read-0 value
  3303. without_relations["$value"]=1
  3304. shift
  3305. ;;
  3306. --no-hooks)
  3307. export no_hooks=true
  3308. ;;
  3309. --no-init)
  3310. export no_init=true
  3311. ;;
  3312. --rebuild-relations-to-service|-R)
  3313. read-0 value
  3314. rebuild_relations_to_service+=("$value")
  3315. shift
  3316. ;;
  3317. --debug|-d)
  3318. export DEBUG=true
  3319. export VERBOSE=true
  3320. #compose_opts+=("--verbose" "--log-level" "DEBUG")
  3321. ;;
  3322. --add-compose-content|-Y)
  3323. read-0 value
  3324. compose_contents+=("$value")
  3325. shift
  3326. ;;
  3327. --dirs)
  3328. echo "CACHEDIR: $CACHEDIR"
  3329. echo "VARDIR: $VARDIR"
  3330. exit 0
  3331. ;;
  3332. --dry-compose-run)
  3333. export DRY_COMPOSE_RUN=true
  3334. ;;
  3335. --*|-*)
  3336. if str_pattern_matches "$arg" $DC_MATCH_MULTI; then
  3337. read-0 value
  3338. compose_opts+=("$arg" "$value")
  3339. shift;
  3340. elif str_pattern_matches "$arg" $DC_MATCH_SINGLE; then
  3341. compose_opts+=("$arg")
  3342. else
  3343. err "Unknown option '$arg'. Please check help:"
  3344. display_help >&2
  3345. exit 1
  3346. fi
  3347. ;;
  3348. *)
  3349. action="$arg"
  3350. stage="action"
  3351. if DC_USAGE=$(get_docker_compose_usage "$action"); then
  3352. is_docker_compose_action=true
  3353. DC_MATCH_MULTI=$(get_docker_compose_multi_opts_list "$action") &&
  3354. DC_MATCH_SINGLE="$(get_docker_compose_single_opts_list "$action")"
  3355. if [ "$DC_MATCH_MULTI" ]; then
  3356. DC_MATCH_SINGLE="$DC_MATCH_SINGLE $(echo "$DC_MATCH_MULTI" | sed -r 's/( |$)/=\* /g')"
  3357. fi
  3358. pos_args=($(echo "$DC_USAGE" | sed -r 's/\[-[^]]+\] ?//g;s/\[options\] ?//g'))
  3359. pos_args=("${pos_args[@]:1}")
  3360. # echo "USAGE: $DC_USAGE"
  3361. # echo "pos_args: ${pos_args[@]}"
  3362. # echo "MULTI: $DC_MATCH_MULTI"
  3363. # echo "SINGLE: $DC_MATCH_SINGLE"
  3364. # exit 1
  3365. else
  3366. stage="remainder"
  3367. fi
  3368. ;;
  3369. esac
  3370. ;;
  3371. "action") ## Only for docker-compose actions
  3372. case "$arg" in
  3373. --help|-h)
  3374. no_init=true ; no_hooks=true ; no_relations=true
  3375. action_opts+=("$arg")
  3376. ;;
  3377. --*|-*)
  3378. if [ "$is_docker_compose_action" ]; then
  3379. if str_pattern_matches "$arg" $DC_MATCH_MULTI; then
  3380. read-0 value
  3381. action_opts+=("$arg" "$value")
  3382. shift
  3383. elif str_pattern_matches "$arg" $DC_MATCH_SINGLE; then
  3384. action_opts+=("$arg")
  3385. else
  3386. err "Unknown option '$arg'. Please check '${DARKCYAN}$action${NORMAL}' help:"
  3387. docker-compose "$action" --help |
  3388. filter_docker_compose_help_message >&2
  3389. exit 1
  3390. fi
  3391. fi
  3392. ;;
  3393. *)
  3394. # echo "LOOP $1 : pos_arg: $pos_arg_ct // ${pos_args[$pos_arg_ct]}"
  3395. if [[ "${pos_args[$pos_arg_ct]}" == "[SERVICE...]" ]]; then
  3396. services_args+=("$arg")
  3397. elif [[ "${pos_args[$pos_arg_ct]}" == "SERVICE" ]]; then
  3398. services_args=("$arg") || exit 1
  3399. stage="remainder"
  3400. else
  3401. action_posargs+=("$arg")
  3402. ((pos_arg_ct++))
  3403. fi
  3404. ;;
  3405. esac
  3406. ;;
  3407. "remainder")
  3408. remainder_args+=("$arg")
  3409. while read-0 arg; do
  3410. remainder_args+=("$arg")
  3411. done
  3412. break 3
  3413. ;;
  3414. esac
  3415. shift
  3416. done < <(cla.normalize "$@")
  3417. export compose_contents
  3418. [ "${services_args[*]}" ] && debug " ${DARKWHITE}Services:$NORMAL ${DARKYELLOW}${services_args[*]}$NORMAL"
  3419. [ "${compose_opts[*]}" ] && debug " ${DARKWHITE}Main docker-compose opts:$NORMAL ${compose_opts[*]}"
  3420. [ "${action_posargs[*]}" ] && debug " ${DARKWHITE}Main docker-compose pos args:$NORMAL ${action_posargs[*]}"
  3421. [ "${action_opts[*]}" ] && debug " ${DARKWHITE}Action $DARKCYAN$action$NORMAL with opts:$NORMAL ${action_opts[*]}"
  3422. [ "${remainder_args[*]}" ] && debug " ${DARKWHITE}Remainder args:$NORMAL ${remainder_args[*]}"
  3423. aexport remainder_args
  3424. ##
  3425. ## Actual code
  3426. ##
  3427. COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1
  3428. COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1
  3429. export COMPOSE_YML_FILE COMPOSE_YML_CONTENT
  3430. charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct."
  3431. ##
  3432. ## Get services in command line.
  3433. ##
  3434. if [ -z "$is_docker_compose_action" -a "$action" ]; then
  3435. action_service=${remainder_args[0]}
  3436. if [ -z "$action_service" ]; then
  3437. err "No such command or action: ${DARKCYAN}$action${NORMAL}"
  3438. display_commands_help
  3439. exit 1
  3440. fi
  3441. remainder_args=("${remainder_args[@]:1}")
  3442. if has_service_action "$action_service" "$action" >/dev/null; then
  3443. is_service_action=true
  3444. {
  3445. read-0 action_type
  3446. case "$action_type" in
  3447. "relation")
  3448. read-0 _ target_service _target_charm relation_name
  3449. debug "Found action $DARKYELLOW${action_service}$NORMAL/$DARKBLUE$relation_name$NORMAL/$DARKCYAN$action$NORMAL (in $DARKYELLOW$target_service$NORMAL)"
  3450. ;;
  3451. "direct")
  3452. debug "Found action $DARKYELLOW${action_service}$NORMAL.$DARKCYAN$action$NORMAL"
  3453. ;;
  3454. esac
  3455. } < <(has_service_action "$action_service" "$action")
  3456. services_args=("$action_service")
  3457. else
  3458. die "Unknown action '${DARKCYAN}$action$NORMAL': It doesn't match any docker-compose commands nor inner charm actions."
  3459. fi
  3460. else
  3461. case "$action" in
  3462. ps|up)
  3463. if [ "${#services_args[@]}" == 0 ]; then
  3464. array_read-0 services_args < <(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys-0 2>/dev/null)
  3465. fi
  3466. ;;
  3467. config)
  3468. services_args=("${action_posargs[@]}")
  3469. ;;
  3470. esac
  3471. fi
  3472. NO_CONSTRAINT_CHECK=True
  3473. case "$action" in
  3474. up)
  3475. NO_CONSTRAINT_CHECK=
  3476. ;;
  3477. esac
  3478. get_all_relations "${services_args[@]}" >/dev/null || exit 1
  3479. if [ "$is_docker_compose_action" -a "${#services_args[@]}" -gt 0 ]; then
  3480. services=($(get_master_services "${services_args[@]}")) || exit 1
  3481. if [ "$action" == "up" ]; then
  3482. declare -A seen
  3483. for service in $(get_ordered_service_dependencies "${services_args[@]}"); do
  3484. mservice=$(get_master_service_for_service "$service")
  3485. [ "${seen[$mservice]}" ] && continue
  3486. type="$(get_service_type "$mservice")" || exit 1
  3487. ## remove run-once
  3488. [ "$type" == "run-once" ] && continue
  3489. seen[$mservice]=1
  3490. action_posargs+=("$mservice")
  3491. done
  3492. else
  3493. action_posargs+=("${services[@]}")
  3494. fi
  3495. ## Get rid of subordinates
  3496. action_posargs=($(get_master_services "${action_posargs[@]}")) || exit 1
  3497. fi
  3498. get_docker_compose "${services_args[@]}" >/dev/null || { ## precalculate variable \$_current_docker_compose
  3499. err "Fails to compile base 'docker-compose.yml'"
  3500. exit 1
  3501. }
  3502. ##
  3503. ## Pre-action
  3504. ##
  3505. full_init=
  3506. case "$action" in
  3507. up|run)
  3508. full_init=true
  3509. post_hook=true
  3510. ;;
  3511. ""|down|restart|logs|config|ps)
  3512. full_init=
  3513. ;;
  3514. *)
  3515. if [ "$is_service_action" ]; then
  3516. full_init=true
  3517. fi
  3518. ;;
  3519. esac
  3520. if [ -n "$full_init" ]; then
  3521. ## init in order
  3522. if [[ -z "$no_init" && -z "$no_hooks" ]]; then
  3523. Section setup host resources
  3524. setup_host_resources "${services_args[@]}" || exit 1
  3525. Section initialisation
  3526. run_service_hook init "${services_args[@]}" || exit 1
  3527. fi
  3528. ## Get relations
  3529. if [[ -z "$no_relations" && -z "$no_hooks" ]]; then
  3530. if [ "${#rebuild_relations_to_service[@]}" != 0 ]; then
  3531. rebuild_relations_to_service=$(get_master_services "${rebuild_relations_to_service[@]}") || return 1
  3532. rebuild_relations_to_service=($rebuild_relations_to_service)
  3533. project=$(get_default_project_name) || return 1
  3534. for service in "${rebuild_relations_to_service[@]}"; do
  3535. for dir in "$VARDIR/relations/$project/"*"-${service}/"*; do
  3536. [ -d "$dir" ] && {
  3537. debug rm -rf "$dir"
  3538. rm -rf "$dir"
  3539. }
  3540. done
  3541. done
  3542. fi
  3543. run_service_relations "${services_args[@]}" || exit 1
  3544. fi
  3545. run_service_hook pre_deploy "${services_args[@]}" || exit 1
  3546. fi
  3547. if [ "$action" == "run" -a "${#services_args}" != 0 ]; then
  3548. charm=$(get_service_charm "${services_args[0]}") || exit 1
  3549. metadata=$(charm.metadata "$charm") || exit 1
  3550. SERVICE_TYPE="$(printf "%s" "$metadata" | shyaml get-value type 2>/dev/null)" || true
  3551. if [ "$SERVICE_TYPE" == "run-once" ]; then
  3552. run_service_hook dc-pre-run "${services_args[@]}" || exit 1
  3553. fi
  3554. fi
  3555. export SERVICE_PACK="${services_args[*]}"
  3556. ##
  3557. ## Docker-compose
  3558. ##
  3559. case "$action" in
  3560. up|start|stop|build|run)
  3561. ## force daemon mode for up
  3562. if [[ "$action" == "up" ]]; then
  3563. if ! array_member action_opts -d; then
  3564. action_opts+=("-d")
  3565. fi
  3566. if ! array_member action_opts --remove-orphans; then
  3567. action_opts+=("--remove-orphans")
  3568. fi
  3569. fi
  3570. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  3571. ;;
  3572. logs)
  3573. if ! array_member action_opts --tail; then ## force daemon mode for up
  3574. action_opts+=("--tail" "10")
  3575. fi
  3576. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  3577. ;;
  3578. "")
  3579. launch_docker_compose "${compose_opts[@]}"
  3580. ;;
  3581. graph)
  3582. graph $SERVICE_PACK
  3583. ;;
  3584. config)
  3585. ## removing the services
  3586. services=($(get_master_services "${action_posargs[@]}")) || exit 1
  3587. ## forcing docker-compose config to output the config file to stdout and not stderr
  3588. out=$(launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" 2>&1) || {
  3589. echo "$out"
  3590. exit 1
  3591. }
  3592. echo "$out"
  3593. warn "Runtime configuration modification (from relations) are not included here."
  3594. ;;
  3595. down)
  3596. if ! array_member action_opts --remove-orphans; then ## force daemon mode for up
  3597. debug "Adding a default argument of '--remove-orphans'"
  3598. action_opts+=("--remove-orphans")
  3599. fi
  3600. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}"
  3601. ;;
  3602. *)
  3603. if [ "$is_service_action" ]; then
  3604. run_service_action "$SERVICE_PACK" "$action" "${remainder_args[@]}"
  3605. else
  3606. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  3607. fi
  3608. ;;
  3609. esac || exit 1
  3610. if [ "$post_hook" -a "${#services_args[@]}" != 0 ]; then
  3611. run_service_hook post_deploy "${services_args[@]}" || exit 1
  3612. fi
  3613. if [ "$action" == "run" -a "${#services_args}" != 0 ]; then
  3614. if [ "$SERVICE_TYPE" == "run-once" ]; then
  3615. run_service_hook dc-post-run "${services_args[@]}" || exit 1
  3616. fi
  3617. fi
  3618. clean_unused_docker_compose || return 1