You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

4760 lines
158 KiB

  1. #!/bin/bash
  2. ##
  3. ## TODO:
  4. ## - subordinate container should really be able to modify base image of their master
  5. ## - this could be done through docker-update
  6. ## - I'm not happy with the current build using 'build/' directory, this should be
  7. ## changed to:
  8. ## - always have a base image (specified in metadata), and always have hooks/install
  9. ## executed and merge in image (like docker-build-charm).
  10. ## - container base image is ALWAYS the image of the master container... this brings
  11. ## questions about a double way to express inheritage (through relations as it is
  12. ## implemented now, or through this base-image ?)
  13. ## - the name of the scripts for relation (aka relation_name-relation-joined) is bad as
  14. ## reading the name in a hooks/ dir, there are no way to know if we are the target or
  15. ## the base of the relation.
  16. ## - we could leverage a 'relations/' dir on the root of the charm, with both:
  17. ## 'relations/provide/relation_name' and 'relations/receive/relation_name'
  18. ## - a very bad point with the actual naming is that we can't have a providing AND
  19. ## receiving a relation with same name.
  20. ## - The cache system should keep md5 of docker-compose and other things between runs
  21. ## - The cache system should use underlying function that have only arguments inputs.
  22. ## This will allow to cache completely without issues function in time.
  23. ## - would probably need instrospection in charm custom action to know if these need
  24. ## init or relations to be set up.
  25. ## - Be clear about when the SERVICE name is used and the CHARM name is used.
  26. ## - in case of service contained in another container
  27. ## - in normal case
  28. ## - in docker-compose, can't use charm name: if we want 2 instances of the same charm
  29. ## we are stuck. What will be unique is the name of the service.
  30. ## - some relations are configured in compose.yml but should not trigger the loading
  31. ## of necessary component (for instance, apache --> log-rotate), if log-rotate is
  32. ## not there, this link should considered optional.
  33. ## - Could probably allow an unexistent charm to be populated with only "docker-image:"
  34. ## of the same name. Although this should trigger a visible warning.
  35. #:-
  36. [ -e /etc/shlib ] && . /etc/shlib || {
  37. echo "Unsatisfied dependency. Please install 'kal-shlib-core'."
  38. exit 1
  39. }
  40. #:-
  41. include common
  42. include pretty
  43. include parse
  44. include charm
  45. include array
  46. include cla
  47. include docker
  48. depends shyaml docker
  49. exname="compose"
  50. version=0.1
  51. usage="$exname [COMPOSE_OPTS] [ACTION [ACTION_OPTS]]"
  52. help="\
  53. $WHITE$exname$NORMAL jobs is to run various shell scripts to build
  54. a running orchestrated and configured docker containers. These shell
  55. scripts will have the opportunity to build a 'docker-compose.yml'.
  56. Once init script and relations scripts are executed, $WHITE$exname$NORMAL
  57. delegate the launching to ${WHITE}docker-compose${NORMAL} by providing it
  58. the final 'docker-compose.yml'.
  59. $WHITE$exname$NORMAL also leverage charms to offer some additional custom
  60. actions per charm, which are simply other scripts that can be
  61. run without launching ${WHITE}docker-compose${NORMAL}.
  62. In compose message, color coding is enforced as such:
  63. - ${DARKCYAN}action$NORMAL,
  64. - ${DARKBLUE}relation$NORMAL,
  65. - ${DARKPINK}charm${NORMAL},
  66. - ${DARKYELLOW}service${NORMAL},
  67. - ${WHITE}option-name${NORMAL}/${WHITE}command-name${NORMAL}/${WHITE}Section-Title${NORMAL}
  68. $WHITE$exname$NORMAL reads '/etc/compose.conf' for global variables, and
  69. '/etc/compose.local.conf' for local host adjustements.
  70. "
  71. ## XXXvlab: this doesn't seem to work when 'compose' is called in
  72. ## a hook of a charm.
  73. #[[ "${BASH_SOURCE[0]}" == "" ]] && SOURCED=true
  74. $(return >/dev/null 2>&1) && SOURCED=true
  75. errlvl() { return "${1:-1}"; }
  76. export -f errlvl
  77. if [ "$UID" == 0 ]; then
  78. CACHEDIR=${CACHEDIR:-/var/cache/compose}
  79. VARDIR=${VARDIR:-/var/lib/compose}
  80. else
  81. [ "$XDG_CONFIG_HOME" ] && CACHEDIR=${CACHEDIR:-$XDG_CONFIG_HOME/compose}
  82. [ "$XDG_DATA_HOME" ] && VARDIR=${VARDIR:-$XDG_DATA_HOME/compose}
  83. CACHEDIR=${CACHEDIR:-$HOME/.cache/compose}
  84. VARDIR=${VARDIR:-$HOME/.local/share/compose}
  85. fi
  86. export VARDIR CACHEDIR
  87. md5_compat() { md5sum | cut -c -32; }
  88. quick_cat_file() { quick_cat_stdin < "$1"; }
  89. quick_cat_stdin() { local IFS=''; while read -r line; do echo "$line"; done ; }
  90. export -f quick_cat_file quick_cat_stdin md5_compat
  91. read-0-err() {
  92. local ret="$1" eof="" idx=0 last=
  93. read -r -- "${ret?}" <<<"0"
  94. shift
  95. while [ "$1" ]; do
  96. last=$idx
  97. read -r -d '' -- "$1" || {
  98. ## Put this last value in ${!ret}
  99. eof="$1"
  100. read -r -- "$ret" <<<"${!eof}"
  101. break
  102. }
  103. ((idx++))
  104. shift
  105. done
  106. [ -z "$eof" ] || {
  107. if [ "$last" != 0 ]; then
  108. echo "Error: read-0-err couldn't fill all value" >&2
  109. read -r -- "$ret" <<<"127"
  110. else
  111. if [ -z "${!ret}" ]; then
  112. echo "Error: last value is not a number, did you finish with an errorlevel ?" >&2
  113. read -r -- "$ret" <<<"126"
  114. fi
  115. fi
  116. false
  117. }
  118. }
  119. wyq() {
  120. local exp="$1"
  121. yq e -e -0 "$1"
  122. printf "%s" "$?"
  123. }
  124. wyq-r() {
  125. local exp="$1"
  126. yq e -e -0 -r=false "$1"
  127. printf "%s" "$?"
  128. }
  129. clean_cache() {
  130. local i=0
  131. for f in $(ls -t "$CACHEDIR/"*.cache.* 2>/dev/null | tail -n +500); do
  132. ((i++))
  133. rm -f "$f"
  134. done
  135. if (( i > 0 )); then
  136. debug "${WHITE}Cleaned cache:${NORMAL} Removed $((i)) elements (current cache size is $(du -sh "$CACHEDIR" | cut -f 1))"
  137. fi
  138. }
  139. export DEFAULT_COMPOSE_FILE
  140. ##
  141. ## Merge YAML files
  142. ##
  143. export _merge_yaml_common_code="
  144. import sys
  145. import yaml
  146. try:
  147. from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
  148. except ImportError: ## pragma: no cover
  149. sys.stderr.write('YAML code in pure python\n')
  150. exit(1)
  151. from yaml import SafeLoader, SafeDumper
  152. class MySafeLoader(SafeLoader): pass
  153. class MySafeDumper(SafeDumper): pass
  154. try:
  155. # included in standard lib from Python 2.7
  156. from collections import OrderedDict
  157. except ImportError:
  158. # try importing the backported drop-in replacement
  159. # it's available on PyPI
  160. from ordereddict import OrderedDict
  161. ## Ensure that there are no collision with legacy OrderedDict
  162. ## that could be used for omap for instance.
  163. class MyOrderedDict(OrderedDict):
  164. pass
  165. MySafeDumper.add_representer(
  166. MyOrderedDict,
  167. lambda cls, data: cls.represent_dict(data.items()))
  168. def construct_omap(cls, node):
  169. cls.flatten_mapping(node)
  170. return MyOrderedDict(cls.construct_pairs(node))
  171. MySafeLoader.add_constructor(
  172. yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
  173. construct_omap)
  174. ##
  175. ## Support local and global objects
  176. ##
  177. class EncapsulatedNode(object): pass
  178. def mk_encapsulated_node(s, node):
  179. method = 'construct_%s' % (node.id, )
  180. data = getattr(s, method)(node)
  181. class _E(data.__class__, EncapsulatedNode):
  182. pass
  183. _E.__name__ = str(node.tag)
  184. _E._node = node
  185. return _E(data)
  186. def represent_encapsulated_node(s, o):
  187. value = s.represent_data(o.__class__.__bases__[0](o))
  188. value.tag = o.__class__.__name__
  189. return value
  190. MySafeDumper.add_multi_representer(EncapsulatedNode,
  191. represent_encapsulated_node)
  192. MySafeLoader.add_constructor(None, mk_encapsulated_node)
  193. def fc(filename):
  194. with open(filename) as f:
  195. return f.read()
  196. def merge(*args):
  197. # sys.stderr.write('%r\n' % (args, ))
  198. args = [arg for arg in args if arg is not None]
  199. if len(args) == 0:
  200. return None
  201. if len(args) == 1:
  202. return args[0]
  203. if all(isinstance(arg, (int, basestring, bool, float)) for arg in args):
  204. return args[-1]
  205. elif all(isinstance(arg, list) for arg in args):
  206. res = []
  207. for arg in args:
  208. for elt in arg:
  209. if elt in res:
  210. res.remove(elt)
  211. res.append(elt)
  212. return res
  213. elif all(isinstance(arg, dict) for arg in args):
  214. keys = set()
  215. for arg in args:
  216. keys |= set(arg.keys())
  217. dct = {}
  218. for key in keys:
  219. sub_args = []
  220. for arg in args:
  221. if key in arg:
  222. sub_args.append(arg)
  223. try:
  224. dct[key] = merge(*(a[key] for a in sub_args))
  225. except NotImplementedError as e:
  226. raise NotImplementedError(
  227. e.args[0],
  228. '%s.%s' % (key, e.args[1]) if e.args[1] else key,
  229. e.args[2])
  230. if dct[key] is None:
  231. del dct[key]
  232. return dct
  233. else:
  234. raise NotImplementedError(
  235. 'Unsupported types: %s'
  236. % (', '.join(list(set(arg.__class__.__name__ for arg in args)))), '', args)
  237. return None
  238. def merge_cli(*args):
  239. try:
  240. c = merge(*args)
  241. except NotImplementedError as e:
  242. sys.stderr.write('Merging Failed: %s.\n%s\n'
  243. ' Values are:\n %s\n'
  244. % (e.args[0],
  245. ' Conflicting key is %r.' % e.args[1] if e.args[1] else
  246. ' Conflict at base of structure.',
  247. '\\n '.join('v%d: %r' % (i, a)
  248. for i, a in enumerate(e.args[2]))))
  249. exit(1)
  250. if c is not None:
  251. print '%s' % yaml.dump(c, default_flow_style=False, Dumper=MySafeDumper)
  252. "
  253. merge_yaml() {
  254. if ! [ -r "$state_tmpdir/merge_yaml.py" ]; then
  255. cat <<EOF > "$state_tmpdir/merge_yaml.py"
  256. $_merge_yaml_common_code
  257. merge_cli(*(yaml.load(fc(f), Loader=MySafeLoader) for f in sys.argv[1:]))
  258. EOF
  259. fi
  260. python "$state_tmpdir/merge_yaml.py" "$@"
  261. }
  262. export -f merge_yaml
  263. merge_yaml_str() {
  264. local entries="$@"
  265. if ! [ -r "$state_tmpdir/merge_yaml_str.py" ]; then
  266. cat <<EOF > "$state_tmpdir/merge_yaml_str.py" || return 1
  267. $_merge_yaml_common_code
  268. merge_cli(*(yaml.load(f, Loader=MySafeLoader) for f in sys.argv[1:]))
  269. EOF
  270. fi
  271. if ! python "$state_tmpdir/merge_yaml_str.py" "$@"; then
  272. err "Failed to merge yaml strings:"
  273. local s
  274. for s in "$@"; do
  275. printf " - \n"
  276. printf "%s\n" "$s" | prefix " ${GRAY}|$NORMAL "
  277. done >&2
  278. return 1
  279. fi
  280. }
  281. export -f merge_yaml_str
  282. yaml_get_values() {
  283. local sep=${1:-$'\n'} value input type first elt
  284. input=$(cat -)
  285. if [ -z "$input" ] || [ "$input" == "None" ]; then
  286. return 0
  287. fi
  288. type=$(e "$input" | shyaml get-type)
  289. value=
  290. case "$type" in
  291. "sequence")
  292. first=1
  293. while read-0 elt; do
  294. elt="$(e "$elt" | yaml_get_interpret)" || return 1
  295. [ "$elt" ] || continue
  296. if [ "$first" ]; then
  297. first=
  298. else
  299. value+="$sep"
  300. fi
  301. first=
  302. value+="$elt"
  303. done < <(e "$input" | shyaml -y get-values-0)
  304. ;;
  305. "struct")
  306. while read-0 val; do
  307. value+=$'\n'"$(e "$val" | yaml_get_interpret)" || return 1
  308. done < <(e "$input" | shyaml -y values-0)
  309. ;;
  310. "NoneType")
  311. value=""
  312. ;;
  313. "str"|*)
  314. value+="$(e "$input" | yaml_get_interpret)"
  315. ;;
  316. esac
  317. e "$value"
  318. }
  319. export -f yaml_get_values
  320. yaml_key_val_str() {
  321. local entries="$@"
  322. if ! [ -r "$state_tmpdir/yaml_key_val_str.py" ]; then
  323. cat <<EOF > "$state_tmpdir/yaml_key_val_str.py"
  324. $_merge_yaml_common_code
  325. print '%s' % yaml.dump(
  326. {
  327. yaml.load(sys.argv[1], Loader=MySafeLoader):
  328. yaml.load(sys.argv[2], Loader=MySafeLoader)
  329. },
  330. default_flow_style=False,
  331. Dumper=MySafeDumper,
  332. )
  333. EOF
  334. fi
  335. python "$state_tmpdir/yaml_key_val_str.py" "$@"
  336. }
  337. export -f yaml_key_val_str
  338. ##
  339. ## Docker
  340. ##
  341. docker_has_image() {
  342. local image="$1"
  343. images=$(docker images -q "$image" 2>/dev/null) || {
  344. err "docker images call has failed unexpectedly."
  345. return 1
  346. }
  347. [ -n "$images" ]
  348. }
  349. export -f docker_has_image
  350. docker_image_id() {
  351. local image="$1"
  352. image_id=$(docker inspect "$image" --format='{{.Id}}') || return 1
  353. echo "$image_id" # | tee "$cache_file"
  354. }
  355. export -f docker_image_id
  356. cached_cmd_on_image() {
  357. local image="$1" cache_file
  358. image_id=$(docker_image_id "$image") || return 1
  359. cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  360. if [ -e "$cache_file" ]; then
  361. # debug "$FUNCNAME: cache hit ($*)"
  362. quick_cat_stdin < "$cache_file"
  363. return 0
  364. fi
  365. shift
  366. out=$(docker run -i --rm --entrypoint /bin/sh "$image_id" -c "$*") || return 1
  367. echo "$out" | tee "$cache_file"
  368. }
  369. export -f cached_cmd_on_image
  370. cmd_on_base_image() {
  371. local service="$1" base_image
  372. shift
  373. base_image=$(service_base_docker_image "$service") || return 1
  374. docker run -i --rm --entrypoint /bin/bash "$base_image" -c "$*"
  375. }
  376. export -f cmd_on_base_image
  377. cached_cmd_on_base_image() {
  378. local service="$1" base_image cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  379. shift
  380. if [ -e "$cache_file" ]; then
  381. # debug "$FUNCNAME: cache hit ($*)"
  382. quick_cat_stdin < "$cache_file"
  383. return 0
  384. fi
  385. base_image=$(service_base_docker_image "$service") || return 1
  386. if ! docker_has_image "$base_image"; then
  387. docker pull "$base_image" 1>&2
  388. fi
  389. result=$(cached_cmd_on_image "$base_image" "$@") || return 1
  390. echo "$result" | tee "$cache_file"
  391. }
  392. export -f cached_cmd_on_base_image
  393. docker_update() {
  394. ## YYY: warning, we a storing important information in cache, cache can
  395. ## be removed.
  396. ## We want here to cache the last script on given service whatever that script was
  397. local service="$1" script="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$1" \
  398. previous_base_image stored_image_id
  399. shift
  400. shift
  401. ## this will build it if necessary
  402. base_image=$(service_base_docker_image "$service") || return 1
  403. ## XXXvlab: there are probably ways to avoid rebuilding that each time
  404. image_id="$(docker_image_id "$base_image")" || return 1
  405. if [ -e "$cache_file" ]; then
  406. info "Cache file exists"
  407. read-0 previous_base_image stored_image_id < <(cat "$cache_file")
  408. info "previous: $previous_base_image"
  409. info "stored: $stored_image_id"
  410. else
  411. info "No cache file $cache_file"
  412. previous_base_image=""
  413. fi
  414. if [ "$previous_base_image" -a "$stored_image_id" == "$image_id" ]; then
  415. info "Resetting $base_image to $previous_base_image"
  416. docker tag "$previous_base_image" "$base_image" || return 1
  417. image_id="$(docker_image_id "$base_image")" || return 1
  418. else
  419. previous_base_image="$image_id"
  420. fi
  421. info "Updating base image: $base_image (hash: $image_id)"
  422. echo "$script" | dupd --debug -u "$base_image" -- "$@" || {
  423. err "Failed updating base image"
  424. return 1
  425. }
  426. new_image_id="$(docker_image_id "$base_image")"
  427. [ "$new_image_id" == "$previous_base_image" ] && {
  428. err "Image was not updated correctly (same id)."
  429. return 1
  430. }
  431. printf "%s\0" "$previous_base_image" "$new_image_id" > "$cache_file"
  432. info "Wrote cache file $cache_file"
  433. }
  434. export -f docker_update
  435. image_exposed_ports_0() {
  436. local image="$1"
  437. docker inspect --format='{{range $p, $conf := .Config.ExposedPorts}}{{$p}}{{"\x00"}}{{end}}' "$image"
  438. }
  439. export -f image_exposed_ports_0
  440. ## feature not yet included in docker: https://github.com/moby/moby/issues/16079
  441. docker_image_export_dir() {
  442. local image="$1" src="$2" dst="$3" container_id
  443. (
  444. container_id=$(docker create "$image") || exit 1
  445. trap_add EXIT,ERR "docker rm \"$container_id\" >/dev/null"
  446. docker cp "$container_id":"$src" "$dst"
  447. )
  448. }
  449. export -f docker_image_export_dir
  450. service_base_image_export_dir() {
  451. local service="$1" src="$2" dst="$3" base_image
  452. shift
  453. base_image=$(service_base_docker_image "$service") || return 1
  454. if ! docker_has_image "$base_image"; then
  455. docker pull "$base_image"
  456. fi
  457. docker_image_export_dir "$base_image" "$src" "$dst"
  458. }
  459. export -f service_base_image_export_dir
  460. service_base_image_id() {
  461. local service="$1" src="$2" dst="$3" base_image
  462. shift
  463. base_image=$(service_base_docker_image "$service") || return 1
  464. if ! docker_has_image "$base_image"; then
  465. docker pull "$base_image"
  466. fi
  467. docker inspect "$base_image" --format="{{ .Id }}"
  468. }
  469. export -f service_base_image_id
  470. ##
  471. ## Generic
  472. ##
  473. fn.exists() {
  474. declare -F "$1" >/dev/null
  475. }
  476. str_pattern_matches() {
  477. local str="$1"
  478. shift
  479. for pattern in "$@"; do
  480. eval "[[ \"$str\" == $pattern ]]" && return 0
  481. done
  482. return 1
  483. }
  484. str_matches() {
  485. local str="$1"
  486. shift
  487. for pattern in "$@"; do
  488. [[ "$str" == "$pattern" ]] && return 0
  489. done
  490. return 1
  491. }
  492. gen_password() {
  493. local l=( {a..z} {A..Z} {0..9} ) nl="${#l[@]}" size=${1:-16}
  494. while ((size--)); do
  495. echo -n "${l[$((RANDOM * nl / 32768))]}"
  496. done
  497. echo
  498. }
  499. export -f gen_password
  500. file_put() {
  501. local TARGET="$1"
  502. mkdir -p "$(dirname "$TARGET")" &&
  503. cat - > "$TARGET"
  504. }
  505. export -f file_put
  506. file_put_0() {
  507. local TARGET="$1"
  508. mkdir -p "$(dirname "$TARGET")" &&
  509. cat > "$TARGET"
  510. }
  511. export -f file_put_0
  512. fetch_file() {
  513. local src="$1"
  514. case "$src" in
  515. *"://"*)
  516. err "Unsupported target scheme."
  517. return 1
  518. ;;
  519. *)
  520. ## Try direct
  521. if ! [ -r "$src" ]; then
  522. err "File '$src' not found/readable."
  523. return 1
  524. fi
  525. cat "$src" || return 1
  526. ;;
  527. esac
  528. }
  529. export -f fetch_file
  530. ## receives stdin content to decompress on stdout
  531. ## stdout content should be tar format.
  532. uncompress_file() {
  533. local filename="$1"
  534. ## Warning, the content of the file is already as stdin, the filename
  535. ## is there to hint for correct decompression.
  536. case "$filename" in
  537. *".gz")
  538. gunzip
  539. ;;
  540. *".bz2")
  541. bunzip2
  542. ;;
  543. *)
  544. cat
  545. ;;
  546. esac
  547. }
  548. export -f uncompress_file
  549. get_file() {
  550. local src="$1"
  551. fetch_file "$src" | uncompress_file "$src"
  552. }
  553. export -f get_file
  554. ##
  555. ## Common database lib
  556. ##
  557. _clean_docker() {
  558. local _DB_NAME="$1" container_id="$2"
  559. (
  560. set +e
  561. debug "Removing container $_DB_NAME"
  562. docker stop "$container_id"
  563. docker rm "$_DB_NAME"
  564. docker network rm "${_DB_NAME}"
  565. rm -vf "$state_tmpdir/${_DB_NAME}.state"
  566. ) >&2
  567. }
  568. export -f _clean_docker
  569. get_service_base_image_dir_uid_gid() {
  570. local service="$1" dir="$2" uid_gid
  571. uid_gid=$(cached_cmd_on_base_image "$service" "stat -c '%u %g' '$dir'") || {
  572. debug "Failed to query '$dir' uid in ${DARKYELLOW}$service${NORMAL} base image."
  573. return 1
  574. }
  575. info "uid and gid from ${DARKYELLOW}$service${NORMAL}:$dir is '$uid_gid'"
  576. echo "$uid_gid"
  577. }
  578. export -f get_service_base_image_dir_uid_gid
  579. get_service_type() {
  580. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  581. if [ -z "$service" ]; then
  582. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  583. return 1
  584. fi
  585. if [ -e "$cache_file" ]; then
  586. # debug "$FUNCNAME: cache hit ($*)"
  587. cat "$cache_file"
  588. return 0
  589. fi
  590. master_target_service="$(get_top_master_service_for_service "$service")" || return 1
  591. charm=$(get_service_charm "$master_target_service") || return 1
  592. metadata=$(charm.metadata "$charm") || return 1
  593. printf "%s" "$metadata" | shyaml get-value type service 2>/dev/null |
  594. tee "$cache_file"
  595. }
  596. export -f get_service_type
  597. are_files_locked_in_dir() {
  598. local dir="$1" device hdev ldev
  599. device=$(stat -c %d "$dir") || {
  600. err "Can't stat '$dir'."
  601. return 1
  602. }
  603. device=$(printf "%04x" $device)
  604. hdev=${device:0:2}
  605. ldev=${device:2:2}
  606. inodes=$(find "$dir" -printf ':%i:\n')
  607. found=
  608. while read -r inode; do
  609. debug "try inode:$inode"
  610. if [[ "$inodes" == *":$inode:"* ]]; then
  611. found=1
  612. break
  613. fi
  614. done < <(cat /proc/locks | grep " $hdev:$ldev:" | sed -r "s/^.*$hdev:$ldev:([0-9]+).*$/\1/g")
  615. [ "$found" ]
  616. }
  617. export -f are_files_locked_in_dir
  618. set_db_params() {
  619. local docker_ip="$1" docker_network="$2"
  620. if [ -z "$DB_PARAMS_LOADED" ]; then
  621. DB_PARAMS_LOADED=1
  622. _set_db_params "$docker_ip" "$docker_network"
  623. fi
  624. }
  625. export -f set_db_params
  626. export _PID="$$"
  627. ensure_db_docker_running () {
  628. local _STATE_FILE errlvl project
  629. _DB_NAME="db_${DB_NAME}_${_PID}"
  630. _STATE_FILE="$state_tmpdir/${_DB_NAME}.state"
  631. if [ -e "$_STATE_FILE" ]; then
  632. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$(cat "$_STATE_FILE")"
  633. debug "Re-using previous docker/connection '$DOCKER_IP'."
  634. set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  635. return 0
  636. fi
  637. if [ -e "$state_tmpdir/${_DB_NAME}.working" ]; then
  638. ## avoid recursive calls.
  639. if [ -z "$DOCKER_IP" ]; then
  640. err "Currently figuring up DOCKER_IP, please set it yourself before this call if needed."
  641. return 1
  642. else
  643. debug "ignoring recursive call of 'ensure_db_docker_running'."
  644. fi
  645. return 0
  646. fi
  647. touch "$state_tmpdir/${_DB_NAME}.working"
  648. docker rm "$_DB_NAME" 2>/dev/null || true
  649. host_db_working_dir="$HOST_DATASTORE/${SERVICE_NAME}$DB_DATADIR"
  650. if is_db_locked; then
  651. info "Some process is using '$host_db_working_dir'. Trying to find a docker that would do this..."
  652. found=
  653. for docker_id in $(docker ps -q); do
  654. has_volume_mounted=$(
  655. docker inspect \
  656. --format "{{range .Mounts}}{{if eq .Destination \"$DB_DATADIR\"}}{{.Source}}{{end}}{{end}}" \
  657. "$docker_id")
  658. if [ "$has_volume_mounted" == "$host_db_working_dir" ]; then
  659. info "docker '$docker_id' uses '$has_volume_mounted'."
  660. project=$(docker inspect "$docker_id" \
  661. --format "{{index .Config.Labels \"compose.project\" }}") || continue
  662. info "docker '$docker_id' is from project '$project' (current project is '$PROJECT_NAME')."
  663. [ "$project" == "$PROJECT_NAME" ] || continue
  664. found="$docker_id"
  665. break
  666. fi
  667. done
  668. if [ -z "$found" ]; then
  669. err "Please shutdown any other docker using this directory."
  670. return 1
  671. fi
  672. export container_id="$found"
  673. info "Found docker $docker_id is already running."
  674. else
  675. verb "Database is not locked."
  676. if ! docker_has_image "$DOCKER_BASE_IMAGE"; then
  677. docker pull "$DOCKER_BASE_IMAGE"
  678. fi
  679. _set_server_db_params || return 1
  680. debug docker network create "$_DB_NAME"
  681. if ! network_id=$(docker network create "$_DB_NAME"); then
  682. err "'docker network create $_DB_NAME' failed !"
  683. _clean_docker "$_DB_NAME" "$container_id"
  684. rm "$state_tmpdir/${_DB_NAME}.working"
  685. return 1
  686. fi
  687. debug docker run -d \
  688. --name "$_DB_NAME" \
  689. "${server_docker_opts[@]}" \
  690. --network "$_DB_NAME" \
  691. -v "$host_db_working_dir:$DB_DATADIR" \
  692. "$DOCKER_BASE_IMAGE"
  693. if ! container_id=$(
  694. docker run -d \
  695. --name "$_DB_NAME" \
  696. "${server_docker_opts[@]}" \
  697. --network "$_DB_NAME" \
  698. -v "$host_db_working_dir:$DB_DATADIR" \
  699. "$DOCKER_BASE_IMAGE"
  700. ); then
  701. err "'docker run' failed !"
  702. _clean_docker "$_DB_NAME" "$container_id"
  703. rm "$state_tmpdir/${_DB_NAME}.working"
  704. return 1
  705. fi
  706. trap_add EXIT,ERR "_clean_docker \"$_DB_NAME\" \"$container_id\""
  707. fi
  708. if docker_ip=$(wait_for_docker_ip "$container_id"); then
  709. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip"
  710. echo "$docker_ip" > "$_STATE_FILE"
  711. debug "written '$_STATE_FILE'"
  712. rm "$state_tmpdir/${_DB_NAME}.working"
  713. set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  714. return 0
  715. else
  716. errlvl="$?"
  717. err "Db not found (errlvl: $errlvl). Tail of docker logs follows:"
  718. docker logs --tail=5 "$container_id" 2>&1 | prefix " | " >&2
  719. rm "$state_tmpdir/${_DB_NAME}.working"
  720. return "$errlvl"
  721. fi
  722. }
  723. export -f ensure_db_docker_running
  724. ## Require to set $db_docker_opts if needed, and $DB_PASSFILE
  725. ##
  726. _dcmd() {
  727. local docker_opts command="$1"
  728. shift
  729. debug "Db> $command $@"
  730. if [ "$HOST_DB_PASSFILE" -a -f "$LOCAL_DB_PASSFILE" -a "$CLIENT_DB_PASSFILE" ]; then
  731. verb "Found and using '$HOST_DB_PASSFILE' as '$CLIENT_DB_PASSFILE'."
  732. docker_opts=("${db_docker_opts[@]}" "-v" "$HOST_DB_PASSFILE:$CLIENT_DB_PASSFILE")
  733. else
  734. docker_opts=("${db_docker_opts[@]}")
  735. fi
  736. ## XXXX was here: actualy, we need only connection between this version and the client version
  737. debug docker run -i --rm \
  738. "${docker_opts[@]}" \
  739. --entrypoint "$command" "$DOCKER_BASE_IMAGE" "${db_cmd_opts[@]}" "$@"
  740. docker run -i --rm \
  741. "${docker_opts[@]}" \
  742. --entrypoint "$command" "$DOCKER_BASE_IMAGE" "${db_cmd_opts[@]}" "$@"
  743. }
  744. export -f _dcmd
  745. ## Executes code through db
  746. dcmd() {
  747. local fun
  748. [ "$DB_NAME" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_NAME."
  749. [ "$DB_DATADIR" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_DATADIR."
  750. # [ "$DB_PASSFILE" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_PASSFILE."
  751. [ "$_PID" ] || print_syntax_error "$FUNCNAME: You must provide \$_PID."
  752. for fun in is_db_locked _set_db_params ddb; do
  753. [ "$(type -t "$fun")" == "function" ] ||
  754. print_syntax_error "$FUNCNAME: You must provide function '$fun'."
  755. done
  756. ensure_db_docker_running </dev/null || return 1
  757. _dcmd "$@"
  758. }
  759. export -f dcmd
  760. get_docker_ips() {
  761. local name="$1" ip format network_id
  762. if ! docker inspect --format='{{ .NetworkSettings.Networks }}' "$name" >/dev/null 2>&1; then
  763. echo "default:$(docker inspect --format='{{ .NetworkSettings.IPAddress }}' "$name" 2>/dev/null)"
  764. else
  765. format='{{range $name, $conf := .NetworkSettings.Networks}}{{$name}}{{"\x00"}}{{$conf.IPAddress}}{{"\x00"}}{{end}}'
  766. while read-0 network_id ip; do
  767. printf "%s:%s\n" "$network_id" "$ip"
  768. done < <(docker inspect --format="$format" "$name")
  769. fi
  770. }
  771. export -f get_docker_ips
  772. get_docker_ip() {
  773. local name="$1"
  774. get_docker_ips "$name"
  775. }
  776. export -f get_docker_ip
  777. wait_docker_ip() {
  778. local name="$1" timeout="${2:-15}" timeout_count=0 docker_ip=
  779. start=$SECONDS
  780. while [ -z "$docker_ip" ]; do
  781. sleep 0.5
  782. docker_ip=$(get_docker_ip "$name") && break
  783. elapsed=$((SECONDS - start))
  784. if ((elapsed > timeout)); then
  785. err "${RED}timeout error${NORMAL}(${timeout}s):" \
  786. "Could not find '$name' docker container's IP."
  787. return 1
  788. fi
  789. [ "$elapsed" == "$old_elapsed" ] ||
  790. verb "Waiting for docker $name... ($elapsed/$timeout)"
  791. old_elapsed="$elapsed"
  792. done
  793. verb "Found docker $name network and IP: $docker_ip"
  794. echo "$docker_ip"
  795. }
  796. export -f wait_docker_ip
  797. wait_for_tcp_port() {
  798. local network=$1 host_port=$2 timeout=${3:-60}
  799. verb "Trying to connect to $host_port"
  800. bash_image=${DEFAULT_BASH_IMAGE:-docker.0k.io/bash}
  801. #echo docker run --rm -i --network "$network" "$bash_image" >&2
  802. docker run --rm -i --network "$network" "$bash_image" <<EOF
  803. start=\$SECONDS
  804. while true; do
  805. timeout 1 bash -c "</dev/tcp/${host_port/://}" >/dev/null 2>&1 && break
  806. sleep 0.2
  807. if [ "\$((SECONDS - start))" -gt "$timeout" ]; then
  808. exit 1
  809. fi
  810. done
  811. exit 0
  812. EOF
  813. if [ "$?" != 0 ]; then
  814. err "${RED}timeout error${NORMAL}(${timeout}s):"\
  815. "Could not connect to $host_port."
  816. return 1
  817. fi
  818. return 0
  819. }
  820. export -f wait_for_tcp_port
  821. ## Warning: requires a ``ddb`` matching current database to be checked
  822. wait_for_docker_ip() {
  823. local name=$1 DOCKER_IP= DOCKER_NETWORK= docker_ips= docker_ip= elapsed timeout=10
  824. docker_ip=$(wait_docker_ip "$name" 5) || return 1
  825. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip"
  826. if ! str_is_ipv4 "$DOCKER_IP"; then
  827. err "internal 'wait_docker_ip' did not return a valid IP. Returned IP is '$DOCKER_IP'."
  828. return 1
  829. fi
  830. set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  831. while read-0 port; do
  832. IFS="/" read port type <<<"$port"
  833. [ "$type" == "tcp" ] || continue
  834. wait_for_tcp_port "$DOCKER_NETWORK" "$DOCKER_IP:${port}" || return 17
  835. info "Host/Port $DOCKER_IP:${port} checked ${GREEN}open${NORMAL}."
  836. ## XXXvlab: what to do with more than one port ?
  837. break
  838. done < <(image_exposed_ports_0 "$container_id")
  839. ## Checking direct connection
  840. timeout=120
  841. start=$SECONDS
  842. while true; do
  843. if err=$(echo "$check_command" | ddb 2>&1 >/dev/null); then
  844. break
  845. fi
  846. if ! [[ "$err" == *"the database system is starting up" ]]; then
  847. err "${RED}db connection error${NORMAL}:" \
  848. "Could not connect to db on $DOCKER_IP container's IP."
  849. echo " Note: IP up, TCP ports is(are) open" >&2
  850. if [ "$err" ]; then
  851. echo " Error:" >&2
  852. printf "%s\n" "$err" | prefix " ${RED}!${NORMAL} " >&2
  853. fi
  854. return 18
  855. fi
  856. debug "Got 'database system is starting up' error."
  857. elapsed=$((SECONDS - start))
  858. if ((elapsed > timeout)); then
  859. err "${RED}db connection error${NORMAL}:"\
  860. "Could not connect to db on $DOCKER_IP" \
  861. "container's IP. (IP up, TCP ports is(are) open, sql answer after ${timeout}s)"
  862. return 1
  863. fi
  864. sleep 0.2
  865. done
  866. echo "${DOCKER_NETWORK}:${DOCKER_IP}"
  867. return 0
  868. }
  869. export -f wait_for_docker_ip
  870. docker_add_host_declaration() {
  871. local src_docker=$1 domain=$2 dst_docker=$3 dst_docker_ip= dst_docker_network
  872. dst_docker_ip=$(wait_docker_ip "$dst_docker") || exit 1
  873. IFS=: read dst_docker_ip dst_docker_network <<<"$dst_docker_ip"
  874. docker exec -i "$src_docker" bash <<EOF
  875. if cat /etc/hosts | grep -E "^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+$domain\$" > /dev/null 2>&1; then
  876. sed -ri "s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+$domain\$/$dst_docker_ip $domain/g" /etc/hosts
  877. else
  878. echo "$dst_docker_ip $domain" >> /etc/hosts
  879. fi
  880. EOF
  881. }
  882. export -f docker_add_host_declaration
  883. get_running_containers_for_service() {
  884. local service="$1" project="$2"
  885. project=${project:-$PROJECT_NAME}
  886. [ -n "$project" ] || {
  887. err "No project name was defined yet."
  888. return 1
  889. }
  890. docker ps \
  891. --filter label="compose.project=$project" \
  892. --filter label="compose.master-service=$service" \
  893. --format="{{.ID}}"
  894. }
  895. export -f get_running_containers_for_service
  896. get_container_network_ips() {
  897. local container="$1"
  898. docker inspect "$container" \
  899. --format='{{range $key, $val :=.NetworkSettings.Networks}}{{$key}}{{"\x00"}}{{$val.IPAddress}}{{"\x00"}}{{end}}'
  900. }
  901. export -f get_container_network_ips
  902. get_container_network_ip() {
  903. local container="$1"
  904. while read-0 network ip; do
  905. printf "%s\0" "$network" "$ip"
  906. break
  907. done < <(get_container_network_ips "$container")
  908. }
  909. export -f get_container_network_ip
  910. ##
  911. ## Internal Process
  912. ##
  913. get_docker_compose_links() {
  914. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  915. deps master_service master_target_service _relation_name \
  916. target_service _relation_config tech_dep
  917. if [ -z "$service" ]; then
  918. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  919. return 1
  920. fi
  921. if [ -e "$cache_file" ]; then
  922. # debug "$FUNCNAME: cache hit ($*)"
  923. cat "$cache_file"
  924. return 0
  925. fi
  926. master_service=$(get_top_master_service_for_service "$service") || return 1
  927. deps=()
  928. while read-0 _relation_name target_service _relation_config tech_dep; do
  929. master_target_service="$(get_top_master_service_for_service "$target_service")" || return 1
  930. [ "$master_service" == "$master_target_service" ] && continue
  931. type="$(get_service_type "$target_service")" || return 1
  932. [ "$type" == "stub" ] && continue
  933. [ "$type" == "run-once" ] && continue
  934. if [ "$tech_dep" == "reversed" ]; then
  935. deps+=("$(echo -en "$master_target_service:\n links:\n - $master_service")")
  936. elif [ "$tech_dep" == "True" ]; then
  937. deps+=("$(echo -en "$master_service:\n links:\n - $master_target_service")")
  938. fi
  939. ## XXXvlab: an attempt to add depends_on, but this doesn't work well actually
  940. ## as there's a circular dependency issue. We don't really want the full feature
  941. ## of depends_on, but just to add it as targets when doing an 'up'
  942. # deps+=("$(echo -en "$master_service:\n depends_on:\n - $master_target_service")")
  943. done < <(get_service_relations "$service")
  944. merge_yaml_str "${deps[@]}" | tee "$cache_file" || return 1
  945. if [ "${PIPESTATUS[0]}" != 0 ]; then
  946. rm "$cache_file"
  947. err "Failed to merge YAML from all ${WHITE}links${NORMAL} dependencies."
  948. return 1
  949. fi
  950. }
  951. _get_docker_compose_opts() {
  952. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  953. compose_def master_service docker_compose_opts
  954. if [ -z "$service" ]; then
  955. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  956. return 1
  957. fi
  958. if [ -e "$cache_file" ]; then
  959. # debug "$FUNCNAME: cache hit ($*)"
  960. cat "$cache_file"
  961. return 0
  962. fi
  963. compose_def="$(get_compose_service_def "$service")" || return 1
  964. master_service="$(get_top_master_service_for_service "$service")"
  965. if docker_compose_opts=$(echo "$compose_def" | shyaml get-value -y "docker-compose" 2>/dev/null); then
  966. yaml_key_val_str "$master_service" "$docker_compose_opts"
  967. fi | tee "$cache_file"
  968. if [ "${PIPESTATUS[0]}" != 0 ]; then
  969. rm "$cache_file"
  970. return 1
  971. fi
  972. }
  973. ##
  974. ## By Reading the metadata.yml, we create a docker-compose.yml mixin.
  975. ## Some metadata.yml (of subordinates) will indeed modify other
  976. ## services than themselves.
  977. _get_docker_compose_service_mixin() {
  978. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  979. links_yaml base_mixin links_yaml docker_compose_options \
  980. charm charm_part
  981. if [ -z "$service" ]; then
  982. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  983. return 1
  984. fi
  985. if [ -e "$cache_file" ]; then
  986. # debug "$FUNCNAME: cache hit ($*)"
  987. cat "$cache_file"
  988. return 0
  989. fi
  990. type=$(get_service_type "$service") || return 1
  991. [ "$type" == "stub" ] && return 0
  992. master_service=$(get_top_master_service_for_service "$service") || {
  993. err "Failed to get top master service for service $DARKYELLOW$service$NORMAL"
  994. return 1
  995. }
  996. ## The compose part
  997. base_mixin="$master_service:
  998. labels:
  999. - compose.service=$service
  1000. - compose.master-service=${master_service}
  1001. - compose.project=$(get_default_project_name)"
  1002. links_yaml=$(get_docker_compose_links "$service") || return 1
  1003. docker_compose_options=$(_get_docker_compose_opts "$service") || return 1
  1004. ## the charm part
  1005. charm_part=$(get_docker_compose_mixin_from_metadata "$service") || return 1
  1006. ## Merge results
  1007. if [ "$charm_part" ]; then
  1008. charm_yaml="$(yaml_key_val_str "$master_service" "$charm_part")" || return 1
  1009. merge_yaml_str "$base_mixin" "$links_yaml" "$charm_yaml" "$docker_compose_options" || return 1
  1010. else
  1011. merge_yaml_str "$base_mixin" "$links_yaml" "$docker_compose_options" || return 1
  1012. fi | tee "$cache_file"
  1013. if [ "${PIPESTATUS[0]}" != 0 ]; then
  1014. err "Failed to constitute the base YAML for service '${DARKYELLOW}$service${NORMAL}'"
  1015. rm "$cache_file"
  1016. return 1
  1017. fi
  1018. }
  1019. export -f _get_docker_compose_service_mixin
  1020. ##
  1021. ## Get full `docker-compose.yml` format for all listed services (and
  1022. ## their deps)
  1023. ##
  1024. ## @export
  1025. ## @cache: !system !nofail +stdout
  1026. get_docker_compose () {
  1027. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  1028. entries services service start docker_compose_services
  1029. if [ -e "$cache_file" ]; then
  1030. # debug "$FUNCNAME: cache hit ($*)"
  1031. cat "$cache_file"
  1032. return 0
  1033. fi
  1034. ##
  1035. ## Adding sub services configurations
  1036. ##
  1037. declare -A entries
  1038. start_compilation=$SECONDS
  1039. debug "Compiling 'docker-compose.yml' base for ${DARKYELLOW}$*$NORMAL..."
  1040. for target_service in "$@"; do
  1041. start=$SECONDS
  1042. services=($(get_ordered_service_dependencies "$target_service")) || {
  1043. err "Failed to get dependencies for $DARKYELLOW$target_service$NORMAL"
  1044. return 1
  1045. }
  1046. if [ "$DEBUG" ]; then
  1047. debug " $DARKYELLOW$target_service$NORMAL deps:$DARKYELLOW" \
  1048. "${services[@]::$((${#services[@]} - 1))}" \
  1049. "$NORMAL$GRAY(in $((SECONDS - start))s)$NORMAL"
  1050. fi
  1051. for service in "${services[@]}"; do
  1052. if [ "${entries[$service]}" ]; then
  1053. ## Prevent double inclusion of same service if this
  1054. ## service is deps of two or more of your
  1055. ## requirements.
  1056. continue
  1057. fi
  1058. ## mark the service as "loaded" as well as it's containers
  1059. ## if this is a subordinate service
  1060. start_service=$SECONDS
  1061. entries[$service]=$(_get_docker_compose_service_mixin "$service") || {
  1062. err "Failed to get service mixin for $DARKYELLOW$service$NORMAL"
  1063. return 1
  1064. }
  1065. debug " Applied $DARKYELLOW$service$NORMAL charm metadata mixins $GRAY(in $((SECONDS - start_service))s)$NORMAL"
  1066. done
  1067. debug " ..finished all mixins for $DARKYELLOW$target_service$NORMAL $GRAY(in $((SECONDS - start))s)$NORMAL"
  1068. done
  1069. docker_compose_services=$(merge_yaml_str "${entries[@]}") || {
  1070. err "Failed to merge YAML services entries together."
  1071. return 1
  1072. }
  1073. base_v2="version: '2.0'"
  1074. merge_yaml_str "$(yaml_key_val_str "services" "$docker_compose_services")" \
  1075. "$base_v2" > "$cache_file" || return 1
  1076. export _current_docker_compose="$(cat "$cache_file")"
  1077. echo "$_current_docker_compose"
  1078. debug " ..compilation of base 'docker-compose.yml' done $GRAY(in $((SECONDS - start_compilation))s)$NORMAL" || true
  1079. # debug " ** ${WHITE}docker-compose.yml${NORMAL}:"
  1080. # debug "$_current_docker_compose"
  1081. }
  1082. export -f get_docker_compose
  1083. _get_compose_service_def_cached () {
  1084. local service="$1" docker_compose="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  1085. if [ -e "$cache_file" ]; then
  1086. #debug "$FUNCNAME: STATIC cache hit"
  1087. cat "$cache_file" &&
  1088. touch "$cache_file" || return 1
  1089. return 0
  1090. fi
  1091. value=$(echo "$docker_compose" | shyaml get-value "${service//./\\.}" 2>/dev/null)
  1092. [ "$value" == None ] && value=""
  1093. if ! echo "$value" | shyaml get-value "charm" >/dev/null 2>&1; then
  1094. if charm.exists "$service"; then
  1095. value=$(merge_yaml <(echo "charm: $service") <(echo "$value")) || {
  1096. err "Can't merge YAML infered 'charm: $service' with base ${DARKYELLOW}$service${NORMAL} YAML definition."
  1097. return 1
  1098. }
  1099. else
  1100. err "No ${WHITE}charm${NORMAL} value for service $DARKYELLOW$service$NORMAL" \
  1101. "in compose, nor same name charm found."
  1102. return 1
  1103. fi
  1104. fi
  1105. echo "$value" | tee "$cache_file" || return 1
  1106. # if [ "${PIPESTATUS[0]}" != 0 ]; then
  1107. # rm "$cache_file"
  1108. # return 1
  1109. # fi
  1110. return 0
  1111. # if [ "${PIPESTATUS[0]}" != 0 -o \! -s "$cache_file" ]; then
  1112. # rm "$cache_file"
  1113. # err "PAS OK $service: $value"
  1114. # return 1
  1115. # fi
  1116. }
  1117. export -f _get_compose_service_def_cached
  1118. ## XXXvlab: a lot to be done to cache the results
  1119. get_compose_service_def () {
  1120. local service="$1" docker_compose cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1121. result
  1122. if [ -e "$cache_file" ]; then
  1123. #debug "$FUNCNAME: SESSION cache hit"
  1124. cat "$cache_file" || return 1
  1125. return 0
  1126. fi
  1127. [ -z "$service" ] && print_syntax_error "Missing service as first argument."
  1128. docker_compose=$(get_compose_yml_content) || return 1
  1129. result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1
  1130. charm=$(echo "$result" | shyaml get-value charm 2>/dev/null) || return 1
  1131. metadata=$(charm.metadata "$charm") || return 1
  1132. if default_options=$(printf "%s" "$metadata" | shyaml -y -q get-value default-options); then
  1133. default_options=$(yaml_key_val_str "options" "$default_options") || return 1
  1134. result=$(merge_yaml_str "$default_options" "$result") || return 1
  1135. fi
  1136. echo "$result" | tee "$cache_file" || return 1
  1137. }
  1138. export -f get_compose_service_def
  1139. _get_service_charm_cached () {
  1140. local service="$1" service_def="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  1141. if [ -e "$cache_file" ]; then
  1142. # debug "$FUNCNAME: cache hit $1"
  1143. cat "$cache_file" &&
  1144. touch "$cache_file" || return 1
  1145. return 0
  1146. fi
  1147. charm=$(echo "$service_def" | shyaml get-value charm 2>/dev/null)
  1148. if [ -z "$charm" ]; then
  1149. err "Missing ${WHITE}charm${NORMAL} value in service $DARKYELLOW$service$NORMAL definition."
  1150. return 1
  1151. fi
  1152. echo "$charm" | tee "$cache_file" || return 1
  1153. }
  1154. export -f _get_service_charm_cached
  1155. get_service_charm () {
  1156. local service="$1"
  1157. if [ -z "$service" ]; then
  1158. echo ${FUNCNAME[@]} >&2
  1159. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  1160. return 1
  1161. fi
  1162. service_def=$(get_compose_service_def "$service") || return 1
  1163. _get_service_charm_cached "$service" "$service_def"
  1164. }
  1165. export -f get_service_charm
  1166. ## built above the docker-compose abstraction, so it relies on the
  1167. ## full docker-compose.yml to be already built.
  1168. get_service_def () {
  1169. local service="$1" def
  1170. if [ -z "$_current_docker_compose" ]; then
  1171. print_syntax_error "$FUNCNAME is meant to be called after"\
  1172. "\$_current_docker_compose has been calculated."
  1173. fi
  1174. def=$(echo "$_current_docker_compose" | shyaml get-value "services.${service//./\\.}" 2>/dev/null)
  1175. if [ -z "$def" ]; then
  1176. err "No definition for service $DARKYELLOW$service$NORMAL in compiled 'docker-compose.yml'."
  1177. return 1
  1178. fi
  1179. echo "$def"
  1180. }
  1181. export -f get_service_def
  1182. ## Return the base docker image name of a service
  1183. service_base_docker_image() {
  1184. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1185. master_service service_def service_image service_build service_dockerfile
  1186. if [ -e "$cache_file" ]; then
  1187. # debug "$FUNCNAME: cache hit ($*)"
  1188. cat "$cache_file"
  1189. return 0
  1190. fi
  1191. master_service="$(get_top_master_service_for_service "$service")" || {
  1192. err "Could not compute master service for service $DARKYELLOW$service$NORMAL."
  1193. return 1
  1194. }
  1195. service_def="$(get_service_def "$master_service")" || {
  1196. err "Could not get docker-compose service definition for $DARKYELLOW$master_service$NORMAL."
  1197. return 1
  1198. }
  1199. service_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null)
  1200. if [ "$?" != 0 ]; then
  1201. ## According to https://stackoverflow.com/questions/32230577 , if there's a build,
  1202. ## then the builded image will get name ${project}_${service}
  1203. project=$(get_default_project_name) || return 1
  1204. image_name="${project}_${service}"
  1205. if ! docker_has_image "$image_name"; then
  1206. service_build=$(echo "$service_def" | shyaml get-value build 2>/dev/null)
  1207. if [ "$?" != 0 ]; then
  1208. err "Service $DARKYELLOW$service$NORMAL has no ${WHITE}image${NORMAL} nor ${WHITE}build${NORMAL} parameter."
  1209. echo "$service_def" >&2
  1210. return 1
  1211. fi
  1212. docker build "$service_build" -t "${project}_${service}" >&2 || {
  1213. err "Failed to build image for ${DARKYELLOW}$service${NORMAL}."
  1214. return 1
  1215. }
  1216. fi
  1217. printf "%s" "${project}_${service}"
  1218. else
  1219. printf "%s" "${service_image}"
  1220. fi | tee "$cache_file"
  1221. if [ "${PIPESTATUS[0]}" != 0 ]; then
  1222. rm "$cache_file"
  1223. return 1
  1224. fi
  1225. }
  1226. export -f service_base_docker_image
  1227. get_charm_relation_def () {
  1228. local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1229. relation_def metadata
  1230. if [ -e "$cache_file" ]; then
  1231. # debug "$FUNCNAME: cache hit ($*)"
  1232. cat "$cache_file"
  1233. return 0
  1234. fi
  1235. metadata="$(charm.metadata "$charm")" || return 1
  1236. relation_def="$(echo "$metadata" | shyaml get-value "provides.${relation_name}" 2>/dev/null)"
  1237. echo "$relation_def" | tee "$cache_file"
  1238. }
  1239. export -f get_charm_relation_def
  1240. get_charm_tech_dep_orientation_for_relation() {
  1241. local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1242. relation_def value
  1243. if [ -e "$cache_file" ]; then
  1244. # debug "$FUNCNAME: cache hit ($*)"
  1245. cat "$cache_file"
  1246. return 0
  1247. fi
  1248. relation_def=$(get_charm_relation_def "$charm" "$relation_name" 2>/dev/null)
  1249. value=$(echo "$relation_def" | shyaml get-value 'tech-dep' 2>/dev/null)
  1250. value=${value:-True}
  1251. printf "%s" "$value" | tee "$cache_file"
  1252. }
  1253. export -f get_charm_tech_dep_orientation_for_relation
  1254. get_service_relation_tech_dep() {
  1255. local service="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1256. charm tech_dep
  1257. if [ -e "$cache_file" ]; then
  1258. # debug "$FUNCNAME: cache hit ($*)"
  1259. cat "$cache_file"
  1260. return 0
  1261. fi
  1262. charm=$(get_service_charm "$service") || return 1
  1263. tech_dep="$(get_charm_tech_dep_orientation_for_relation "$charm" "$relation_name")" || return 1
  1264. printf "%s" "$tech_dep" | tee "$cache_file"
  1265. }
  1266. export -f get_service_relation_tech_dep
  1267. ##
  1268. ## Use compose file to get deps, and relation definition in metadata.yml
  1269. ## for tech-dep attribute.
  1270. get_service_deps() {
  1271. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$ALL_RELATIONS")"
  1272. if [ -e "$cache_file" ]; then
  1273. # debug "$FUNCNAME: cache hit ($*)"
  1274. cat "$cache_file"
  1275. return 0
  1276. fi
  1277. (
  1278. set -o pipefail
  1279. get_service_relations "$service" | \
  1280. while read-0 relation_name target_service _relation_config tech_dep; do
  1281. echo "$target_service"
  1282. done | tee "$cache_file"
  1283. ) || return 1
  1284. }
  1285. export -f get_service_deps
  1286. ## XXXvlab: cache was disabled because improper. Indeed, this needs to cache
  1287. ## 'depths' full state. Second, it should be
  1288. _rec_get_depth() {
  1289. local elt=$1 dep deps max
  1290. [ "${depths[$elt]}" ] && return 0
  1291. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$(declare -pA depths)" "$ALL_RELATIONS")"
  1292. if [ -e "$cache_file.depths" ]; then
  1293. #debug "$FUNCNAME: cache hit ($*) - $cache_file.depths"
  1294. while read-0 k v; do
  1295. depths["$k"]="$v"
  1296. done < "$cache_file.depths"
  1297. while read-0 k v; do
  1298. visited["$k"]="$v"
  1299. done < "$cache_file.visited"
  1300. return 0
  1301. fi
  1302. visited[$elt]=1
  1303. #debug "Setting visited[$elt]"
  1304. #debug "Asking for $DARKYELLOW$elt$NORMAL dependencies"
  1305. deps=$(get_service_deps "$elt") || {
  1306. debug "Failed get_service_deps $elt"
  1307. return 1
  1308. }
  1309. # debug "$elt deps are:" $deps
  1310. max=0
  1311. for dep in $deps; do
  1312. [ "${visited[$dep]}" ] && {
  1313. #debug "Already computing $dep"
  1314. continue
  1315. }
  1316. _rec_get_depth "$dep" || return 1
  1317. #debug "Requesting depth[$dep]"
  1318. if (( ${depths[$dep]} > max )); then
  1319. max="${depths[$dep]}"
  1320. fi
  1321. done
  1322. # debug "Setting depth[$elt] to $((max + 1))"
  1323. depths[$elt]=$((max + 1))
  1324. array_kv_to_stdin depths > "$cache_file.depths"
  1325. array_kv_to_stdin visited > "$cache_file.visited"
  1326. # debug "DEPTHS: $(declare -pA depths)"
  1327. # debug "$FUNCNAME: caching hit ($*) - $cache_file"
  1328. }
  1329. export -f _rec_get_depth
  1330. get_ordered_service_dependencies() {
  1331. local services=("$@") cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$ALL_RELATIONS")" \
  1332. i value key heads depths visited
  1333. if [ -e "$cache_file" ]; then
  1334. # debug "$FUNCNAME: cache hit ($*)"
  1335. cat "$cache_file"
  1336. return 0
  1337. fi
  1338. #debug "Figuring ordered deps of $DARKYELLOW${services[@]}$NORMAL"
  1339. if [ -z "${services[*]}" ]; then
  1340. return 0
  1341. # print_syntax_error "$FUNCNAME: no arguments"
  1342. # return 1
  1343. fi
  1344. declare -A depths
  1345. declare -A visited
  1346. heads=("${services[@]}")
  1347. while [ "${#heads[@]}" != 0 ]; do
  1348. array_pop heads head
  1349. _rec_get_depth "$head" || return 1
  1350. done
  1351. i=0
  1352. while [ "${#depths[@]}" != 0 ]; do
  1353. for key in "${!depths[@]}"; do
  1354. value="${depths[$key]}"
  1355. if [ "$value" == "$i" ]; then
  1356. echo "$key"
  1357. unset depths[$key]
  1358. fi
  1359. done
  1360. ((i++))
  1361. done | tee "$cache_file"
  1362. }
  1363. export -f get_ordered_service_dependencies
  1364. run_service_hook () {
  1365. local action="$1" service subservice subservices loaded
  1366. shift
  1367. declare -A loaded
  1368. for service in "$@"; do
  1369. subservices=$(get_ordered_service_dependencies "$service") || return 1
  1370. for subservice in $subservices; do
  1371. if [ "${loaded[$subservice]}" ]; then
  1372. ## Prevent double inclusion of same service if this
  1373. ## service is deps of two or more of your
  1374. ## requirements.
  1375. continue
  1376. fi
  1377. charm=$(get_service_charm "$subservice") || return 1
  1378. charm.has_hook "$charm" "$action" >/dev/null || continue
  1379. type=$(get_service_type "$subservice") || return 1
  1380. PROJECT_NAME=$(get_default_project_name) || return 1
  1381. MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
  1382. MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1
  1383. if [ "$type" != "stub" ]; then
  1384. DOCKER_BASE_IMAGE=$(service_base_docker_image "$MASTER_BASE_SERVICE_NAME") || return 1
  1385. fi
  1386. Wrap "${wrap_opts[@]}" -d "running $YELLOW$action$NORMAL hook of $DARKYELLOW$subservice$NORMAL in charm $DARKPINK$charm$NORMAL" <<EOF || return 1
  1387. export DOCKER_BASE_IMAGE="$DOCKER_BASE_IMAGE"
  1388. export SERVICE_NAME=$subservice
  1389. export IMAGE_NAME=$(echo "${PROJECT_NAME}" | tr -d "_-")_\${SERVICE_NAME}
  1390. export CONTAINER_NAME=\${IMAGE_NAME}_1
  1391. export CHARM_NAME="$charm"
  1392. export PROJECT_NAME="$PROJECT_NAME"
  1393. export SERVICE_DATASTORE="$DATASTORE/$subservice"
  1394. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$subservice"
  1395. export MASTER_BASE_SERVICE_NAME="$MASTER_BASE_SERVICE_NAME"
  1396. export MASTER_BASE_CHARM_NAME="$MASTER_BASE_CHARM_NAME"
  1397. charm.run_hook "local" "$charm" "$action"
  1398. EOF
  1399. loaded[$subservice]=1
  1400. done
  1401. done
  1402. return 0
  1403. }
  1404. host_resource_get() {
  1405. local location="$1" cfg="$2"
  1406. type=$(echo "$cfg" | shyaml get-value type 2>/dev/null) || {
  1407. err "Missing ${WHITE}type$NORMAL option in ${WHITE}get$NORMAL config for location '$location'"
  1408. return 1
  1409. }
  1410. if fn.exists host_resource_get_$type; then
  1411. host_resource_get_$type "$location" "$cfg"
  1412. else
  1413. err "Source ${WHITE}source$NORMAL type '$type' unknown for" \
  1414. "${WHITE}host-resource$NORMAL '$location' defined in" \
  1415. "$DARKYELLOW$subservice$NORMAL config."
  1416. return 1
  1417. fi
  1418. }
  1419. export -f host_resource_get
  1420. host_resource_get_git() {
  1421. local location="$1" cfg="$2" branch parent url
  1422. branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null)
  1423. branch=${branch:-master}
  1424. url=$(echo "$cfg" | shyaml get-value url 2>/dev/null)
  1425. parent="$(dirname "$location")"
  1426. (
  1427. mkdir -p "$parent" && cd "$parent" &&
  1428. git clone -b "$branch" "$url" "$(basename "$location")"
  1429. ) || return 1
  1430. }
  1431. export -f host_resource_get_git
  1432. host_resource_get_git-sub() {
  1433. local location="$1" cfg="$2" branch parent url
  1434. branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null)
  1435. branch=${branch:-master}
  1436. url=$(echo "$cfg" | shyaml get-value url 2>/dev/null)
  1437. parent="$(dirname "$location")"
  1438. (
  1439. mkdir -p "$parent" && cd "$parent" &&
  1440. git sub clone -b "$branch" "$url" "$(basename "$location")"
  1441. ) || return 1
  1442. }
  1443. export -f host_resource_get_git-sub
  1444. setup_host_resource () {
  1445. local subservice="$1" service_def location get cfg
  1446. service_def=$(get_compose_service_def "$subservice") || return 1
  1447. while read-0 location cfg; do
  1448. ## XXXvlab: will it be a git resources always ?
  1449. if [ -d "$location" -a ! -d "$location/.git" ]; then
  1450. err "Hum, location '$location' does not seem to be a git directory."
  1451. return 1
  1452. fi
  1453. if [ -d "$location" ]; then
  1454. info "host resource '$location' already set up."
  1455. continue
  1456. fi
  1457. get=$(echo "$cfg" | shyaml get-value get 2>/dev/null)
  1458. if [ -z "$get" ]; then
  1459. err "No host directory '$location' found, and no ${WHITE}source$NORMAL" \
  1460. "specified for $DARKYELLOW$subservice$NORMAL."
  1461. return 1
  1462. fi
  1463. host_resource_get "$location" "$get" || return 1
  1464. done < <(echo "$service_def" | shyaml key-values-0 host-resources 2>/dev/null)
  1465. }
  1466. export -f setup_host_resource
  1467. setup_host_resources () {
  1468. local service subservices subservice loaded
  1469. declare -A loaded
  1470. for service in "$@"; do
  1471. subservices=$(get_ordered_service_dependencies "$service") || return 1
  1472. for subservice in $subservices; do
  1473. if [ "${loaded[$subservice]}" ]; then
  1474. ## Prevent double inclusion of same service if this
  1475. ## service is deps of two or more of your
  1476. ## requirements.
  1477. continue
  1478. fi
  1479. setup_host_resource "$subservice" || return 1
  1480. loaded[$subservice]=1
  1481. done
  1482. done
  1483. return 0
  1484. }
  1485. export -f setup_host_resources
  1486. ## Works on stdin
  1487. cfg-get-value () {
  1488. local key="$1" out
  1489. if [ -z "$key" ]; then
  1490. yaml_get_interpret || return 1
  1491. return 0
  1492. fi
  1493. if ! out=$(shyaml -y get-value "$key" 2>/dev/null); then
  1494. err "The key $WHITE$key$NORMAL was not found in relation's data."
  1495. return 1
  1496. fi
  1497. echo "$out" | yaml_get_interpret
  1498. }
  1499. export -f cfg-get-value
  1500. relation-get () {
  1501. cfg-get-value "$1" < "$RELATION_DATA_FILE"
  1502. }
  1503. export -f relation-get
  1504. expand_vars() {
  1505. local unlikely_prefix="UNLIKELY_PREFIX"
  1506. content=$(cat -)
  1507. ## find first identifier not in content
  1508. remaining_lines=$(echo "$content" | grep "^$unlikely_prefix")
  1509. while [ "$(echo "$remaining_lines" | grep "^$unlikely_prefix$")" ]; do
  1510. size_prefix="${#unlikely_prefix}"
  1511. first_matching=$(echo "$remaining_lines" |
  1512. grep -v "^$unlikely_prefix$" |
  1513. uniq -w "$((size_prefix + 1))" -c |
  1514. sort -rn |
  1515. head -n 1)
  1516. first_matching=${first_matching#"${x%%[![:space:]]*}"}
  1517. first_matching="${first_matching#* }"
  1518. next_char=${first_matching:$size_prefix:1}
  1519. if [ "$next_char" != "0" ]; then
  1520. unlikely_prefix+="0"
  1521. else
  1522. unlikely_prefix+="1"
  1523. fi
  1524. remaining_lines=$(echo "$remaining_lines" | grep "^$unlikely_prefix")
  1525. done
  1526. eval "cat <<$unlikely_prefix
  1527. $content
  1528. $unlikely_prefix"
  1529. }
  1530. export -f expand_vars
  1531. yaml_get_interpret() {
  1532. local content tag
  1533. content=$(cat -)
  1534. tag=$(echo "$content" | shyaml -y get-value) || return 1
  1535. tag="${tag%% *}"
  1536. content=$(echo "$content" | shyaml get-value) || return 1
  1537. if ! [ "${tag:0:1}" == "!" ]; then
  1538. echo "$content" || return 1
  1539. return 0
  1540. fi
  1541. case "$tag" in
  1542. "!bash-stdout")
  1543. echo "$content" | bash || {
  1544. err "shell code didn't end with errorlevel 0"
  1545. return 1
  1546. }
  1547. ;;
  1548. "!var-expand")
  1549. echo "$content" | expand_vars || {
  1550. err "shell expansion failed"
  1551. return 1
  1552. }
  1553. ;;
  1554. "!file-content")
  1555. source=$(echo "$content" | expand_vars) || {
  1556. err "shell expansion failed"
  1557. return 1
  1558. }
  1559. cat "$source" || return 1
  1560. ;;
  1561. *)
  1562. err "Invalid object tag ${WHITE}$tag${NORMAL}"
  1563. return 1
  1564. ;;
  1565. esac
  1566. }
  1567. export -f yaml_get_interpret
  1568. options-get () {
  1569. local key="$1" out
  1570. service_def=$(get_compose_service_def "$SERVICE_NAME") || return 1
  1571. if ! out=$(echo "$service_def" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1572. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  1573. return 1
  1574. fi
  1575. echo "$out" | yaml_get_interpret
  1576. }
  1577. export -f options-get
  1578. relation-base-compose-get () {
  1579. local key="$1" out
  1580. if ! out=$(echo "$RELATION_BASE_COMPOSE_DEF" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1581. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  1582. return 1
  1583. fi
  1584. echo "$out" | yaml_get_interpret
  1585. }
  1586. export -f relation-base-compose-get
  1587. relation-target-compose-get () {
  1588. local key="$1" out
  1589. if ! out=$(echo "$RELATION_TARGET_COMPOSE_DEF" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1590. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  1591. return 1
  1592. fi
  1593. echo "$out" | yaml_get_interpret
  1594. }
  1595. export -f relation-target-compose-get
  1596. relation-set () {
  1597. local key="$1" value="$2"
  1598. if [ -z "$RELATION_DATA_FILE" ]; then
  1599. err "$FUNCNAME: relation does not seems to be correctly setup."
  1600. return 1
  1601. fi
  1602. if ! [ -r "$RELATION_DATA_FILE" ]; then
  1603. err "$FUNCNAME: can't read relation's data." >&2
  1604. return 1
  1605. fi
  1606. _config_merge "$RELATION_DATA_FILE" <(yaml_key_val_str "$key" "$value")
  1607. }
  1608. export -f relation-set
  1609. _config_merge() {
  1610. local config_filename="$1" mixin="$2"
  1611. touch "$config_filename" &&
  1612. merge_yaml "$config_filename" "$mixin" > "$config_filename.tmp" || return 1
  1613. mv "$config_filename.tmp" "$config_filename"
  1614. }
  1615. export -f _config_merge
  1616. ## XXXvlab; this can be used only in relation, I'd like to use it in init.
  1617. config-add() {
  1618. local metadata="$1"
  1619. _config_merge "$RELATION_CONFIG" <(echo "$metadata")
  1620. }
  1621. export -f config-add
  1622. ## XXXvlab; this can be used only in relation, I'd like to use it in init.
  1623. init-config-add() {
  1624. local metadata="$1"
  1625. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" \
  1626. <(yaml_key_val_str "services" "$metadata")
  1627. }
  1628. export -f init-config-add
  1629. docker_get_uid() {
  1630. local service="$1" user="$2" uid
  1631. uid=$(cached_cmd_on_base_image "$service" "id -u \"$user\"") || {
  1632. debug "Failed to query for '$user' uid in ${DARKYELLOW}$service${NORMAL} base image."
  1633. return 1
  1634. }
  1635. info "uid from ${DARKYELLOW}$service${NORMAL} for user '$user' is '$uid'"
  1636. echo "$uid"
  1637. }
  1638. export -f docker_get_uid
  1639. docker_get_uid_gid() {
  1640. local service="$1" user="$2" group="$3" uid
  1641. uid_gid=$(cached_cmd_on_base_image "$service" "id -u \"$user\"; id -g \"$group\"") || {
  1642. debug "Failed to query for '$user' uid in ${DARKYELLOW}$service${NORMAL} base image."
  1643. return 1
  1644. }
  1645. info "uid from ${DARKYELLOW}$service${NORMAL} for user '$user' is '$uid_gid'"
  1646. echo "$uid_gid"
  1647. }
  1648. export -f docker_get_uid_gid
  1649. logstdout() {
  1650. local name="$1"
  1651. sed -r 's%^%'"${name}"'> %g'
  1652. }
  1653. export -f logstdout
  1654. logstderr() {
  1655. local name="$1"
  1656. sed -r 's%^(.*)$%'"${RED}${name}>${NORMAL} \1"'%g'
  1657. }
  1658. export -f logstderr
  1659. _run_service_relation () {
  1660. local relation_name="$1" service="$2" target_service="$3" relation_config="$4" relation_dir services
  1661. local errlvl
  1662. charm=$(get_service_charm "$service") || return 1
  1663. target_charm=$(get_service_charm "$target_service") || return 1
  1664. base_script_name=$(charm.has_relation_hook "$charm" "$relation_name" relation-joined) || true
  1665. target_script_name=$(charm.has_relation_hook "$target_charm" "$relation_name" relation-joined) || true
  1666. [ -n "$base_script_name" ] || [ -n "$target_script_name" ] || return 0
  1667. relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1
  1668. RELATION_DATA_FILE=$(get_relation_data_file "$service" "$target_service" "$relation_name" "$relation_config") || return 1
  1669. export BASE_SERVICE_NAME=$service
  1670. export BASE_CHARM_NAME=$charm
  1671. export BASE_CHARM_PATH=$(charm.get_dir "$charm")
  1672. export TARGET_SERVICE_NAME=$target_service
  1673. export TARGET_CHARM_NAME=$target_charm
  1674. export TARGET_CHARM_PATH=$(charm.get_dir "$target_charm")
  1675. export RELATION_DATA_FILE
  1676. target_errlvl=0
  1677. if [ -z "$target_script_name" ]; then
  1678. verb "No relation script $DARKBLUE$relation_name$NORMAL in target $DARKPINK$target_charm$NORMAL."
  1679. else
  1680. verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \
  1681. "for target $DARKYELLOW$target_service$NORMAL (charm $DARKPINK$target_charm$NORMAL)"
  1682. RELATION_CONFIG="$relation_dir/config_provider"
  1683. type=$(get_service_type "$target_service") || return 1
  1684. if [ "$type" != "stub" ]; then
  1685. DOCKER_BASE_IMAGE=$(service_base_docker_image "$target_service") || return 1
  1686. fi
  1687. export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA
  1688. {
  1689. (
  1690. SERVICE_NAME=$target_service
  1691. SERVICE_DATASTORE="$DATASTORE/$target_service"
  1692. SERVICE_CONFIGSTORE="$CONFIGSTORE/$target_service"
  1693. export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE
  1694. charm.run_relation_hook local "$target_charm" "$relation_name" relation-joined
  1695. echo "$?" > "$relation_dir/target_errlvl"
  1696. ) | logstdout "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}"
  1697. } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@${NORMAL}" 3>&1 1>&2 2>&3
  1698. target_errlvl="$(cat "$relation_dir/target_errlvl")" || {
  1699. err "Relation script '$script_name' in $DARKPINK$target_charm$NORMAL" \
  1700. "failed before outputing an errorlevel."
  1701. ((target_errlvl |= "1" ))
  1702. }
  1703. if [ -e "$RELATION_CONFIG" ]; then
  1704. debug "Merging some new config info in $DARKYELLOW$target_service$NORMAL"
  1705. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" &&
  1706. rm "$RELATION_CONFIG"
  1707. ((target_errlvl |= "$?"))
  1708. fi
  1709. fi
  1710. if [ "$target_errlvl" == 0 ]; then
  1711. errlvl=0
  1712. if [ "$base_script_name" ]; then
  1713. verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \
  1714. "for $DARKYELLOW$service$NORMAL (charm $DARKPINK$charm$NORMAL)"
  1715. RELATION_CONFIG="$relation_dir/config_providee"
  1716. RELATION_DATA="$(cat "$RELATION_DATA_FILE")"
  1717. DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") || return 1
  1718. export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA
  1719. {
  1720. (
  1721. SERVICE_NAME=$service
  1722. SERVICE_DATASTORE="$DATASTORE/$service"
  1723. SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  1724. export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE
  1725. charm.run_relation_hook local "$charm" "$relation_name" relation-joined
  1726. echo "$?" > "$relation_dir/errlvl"
  1727. ) | logstdout "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}"
  1728. } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@$NORMAL" 3>&1 1>&2 2>&3
  1729. errlvl="$(cat "$relation_dir/errlvl")" || {
  1730. err "Relation script '$script_name' in $DARKPINK$charm$NORMAL" \
  1731. "failed before outputing an errorlevel."
  1732. ((errlvl |= "1" ))
  1733. }
  1734. if [ -e "$RELATION_CONFIG" ]; then
  1735. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" &&
  1736. rm "$RELATION_CONFIG"
  1737. ((errlvl |= "$?" ))
  1738. fi
  1739. if [ "$errlvl" != 0 ]; then
  1740. err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$service$NORMAL failed to run properly."
  1741. fi
  1742. else
  1743. verb "No relation script '$script_name' in charm $DARKPINK$charm$NORMAL. Ignoring."
  1744. fi
  1745. else
  1746. err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$target_service$NORMAL failed to run properly."
  1747. fi
  1748. if [ "$target_errlvl" == 0 -a "$errlvl" == 0 ]; then
  1749. debug "Relation $DARKBLUE$relation_name$NORMAL is established" \
  1750. "between $DARKYELLOW$service$NORMAL and $DARKYELLOW$target_service$NORMAL."
  1751. return 0
  1752. else
  1753. return 1
  1754. fi
  1755. }
  1756. export -f _run_service_relation
  1757. _get_compose_relations_cached () {
  1758. local compose_service_def="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  1759. relation_name relation_def target_service
  1760. if [ -e "$cache_file" ]; then
  1761. #debug "$FUNCNAME: STATIC cache hit $1"
  1762. cat "$cache_file" &&
  1763. touch "$cache_file" || return 1
  1764. return 0
  1765. fi
  1766. (
  1767. set -o pipefail
  1768. if [ "$compose_service_def" ]; then
  1769. while read-0 relation_name relation_def; do
  1770. ## XXXvlab: could we use braces here instead of parenthesis ?
  1771. (
  1772. case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in
  1773. "str")
  1774. target_service="$(echo "$relation_def" | shyaml get-value 2>/dev/null)" || return 1
  1775. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  1776. printf "%s\0" "$relation_name" "$target_service" "" "$tech_dep"
  1777. ;;
  1778. "sequence")
  1779. while read-0 target_service; do
  1780. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  1781. printf "%s\0" "$relation_name" "$target_service" "" "$tech_dep"
  1782. done < <(echo "$relation_def" | shyaml get-values-0 2>/dev/null)
  1783. ;;
  1784. "struct")
  1785. while read-0 target_service relation_config; do
  1786. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  1787. printf "%s\0" "$relation_name" "$target_service" "$relation_config" "$tech_dep"
  1788. done < <(echo "$relation_def" | shyaml key-values-0 2>/dev/null)
  1789. ;;
  1790. esac
  1791. ) </dev/null >> "$cache_file" || return 1
  1792. done < <(echo "$compose_service_def" | shyaml key-values-0 relations 2>/dev/null)
  1793. fi
  1794. )
  1795. if [ "$?" != 0 ]; then
  1796. err "Error while looking for compose relations."
  1797. rm -f "$cache_file" ## no cache
  1798. return 1
  1799. fi
  1800. [ -e "$cache_file" ] && cat "$cache_file"
  1801. return 0
  1802. }
  1803. export -f _get_compose_relations_cached
  1804. get_compose_relations () {
  1805. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1806. compose_def
  1807. if [ -e "$cache_file" ]; then
  1808. #debug "$FUNCNAME: SESSION cache hit $1"
  1809. cat "$cache_file"
  1810. return 0
  1811. fi
  1812. compose_def="$(get_compose_service_def "$service")" || return 1
  1813. _get_compose_relations_cached "$compose_def" > "$cache_file"
  1814. if [ "$?" != 0 ]; then
  1815. rm -f "$cache_file" ## no cache
  1816. return 1
  1817. fi
  1818. cat "$cache_file"
  1819. }
  1820. export -f get_compose_relations
  1821. get_all_services() {
  1822. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$ALL_RELATIONS")" \
  1823. s rn ts rc td services
  1824. if [ -e "$cache_file" ]; then
  1825. #debug "$FUNCNAME: SESSION cache hit $1"
  1826. cat "$cache_file"
  1827. return 0
  1828. fi
  1829. if [ -z "$ALL_RELATIONS" ]; then
  1830. err "Can't access global \$ALL_RELATIONS"
  1831. return 1
  1832. fi
  1833. declare -A services
  1834. while read-0 s _ ts _ _; do
  1835. for service in "$s" "$ts"; do
  1836. [ "${services[$service]}" ] && continue
  1837. services["$service"]=1
  1838. echo "$service"
  1839. done
  1840. done < <(cat "$ALL_RELATIONS") > "$cache_file"
  1841. cat "$cache_file"
  1842. }
  1843. export -f get_all_services
  1844. get_service_relations () {
  1845. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$ALL_RELATIONS")" \
  1846. s rn ts rc td
  1847. if [ -e "$cache_file" ]; then
  1848. #debug "$FUNCNAME: SESSION cache hit $1"
  1849. cat "$cache_file"
  1850. return 0
  1851. fi
  1852. if [ -z "$ALL_RELATIONS" ]; then
  1853. err "Can't access global \$ALL_RELATIONS"
  1854. return 1
  1855. fi
  1856. while read-0 s rn ts rc td; do
  1857. [[ "$s" == "$service" ]] || continue
  1858. printf "%s\0" "$rn" "$ts" "$rc" "$td"
  1859. done < <(cat "$ALL_RELATIONS") > "$cache_file"
  1860. cat "$cache_file"
  1861. }
  1862. export -f get_service_relations
  1863. get_service_relation() {
  1864. local service="$1" relation="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1865. rn ts rc td
  1866. if [ -e "$cache_file" ]; then
  1867. #debug "$FUNCNAME: SESSION cache hit $1"
  1868. cat "$cache_file"
  1869. return 0
  1870. fi
  1871. while read-0 rn ts rc td; do
  1872. [ "$relation" == "$rn" ] && {
  1873. printf "%s\0" "$ts" "$rc" "$td"
  1874. break
  1875. }
  1876. done < <(get_service_relations "$service") > "$cache_file"
  1877. if [ "$?" != 0 ]; then
  1878. rm -f "$cache_file" ## no cache
  1879. return 1
  1880. fi
  1881. cat "$cache_file"
  1882. }
  1883. export -f get_service_relation
  1884. export TRAVERSE_SEPARATOR=:
  1885. ## Traverse on first service satisfying relation
  1886. service:traverse() {
  1887. local service_path="$1"
  1888. {
  1889. SEPARATOR=:
  1890. read -d "$TRAVERSE_SEPARATOR" service
  1891. while read -d "$TRAVERSE_SEPARATOR" relation; do
  1892. ## XXXvlab: Take only first service
  1893. if ! read-0 ts _ _ < <(get_service_relation "${service}" "${relation}"); then
  1894. err "Couldn't find relation ${DARKCYAN}${relation}${NORMAL}" \
  1895. "from ${DARKYELLOW}$service${NORMAL}."
  1896. return 1
  1897. fi
  1898. service="$ts"
  1899. done
  1900. echo "$service"
  1901. } < <(e "${service_path}${TRAVERSE_SEPARATOR}")
  1902. }
  1903. export -f service:traverse
  1904. service:relation-file() {
  1905. local service_path="$1" relation service relation_file
  1906. if ! [[ "$service_path" == *"$TRAVERSE_SEPARATOR"* ]]; then
  1907. err "Invalid argument '$service_path'." \
  1908. "Must provide a service path (no '${TRAVERSE_SEPARATOR}' found)."
  1909. return 1
  1910. fi
  1911. relation="${service_path##*${TRAVERSE_SEPARATOR}}"
  1912. service=$(service:traverse "${service_path%${TRAVERSE_SEPARATOR}*}") || return 1
  1913. if ! read-0 ts rc _ < <(get_service_relation "${service}" "${relation}"); then
  1914. err "Couldn't find relation ${DARKCYAN}${relation}${NORMAL}" \
  1915. "from ${DARKYELLOW}$service${NORMAL}."
  1916. return 1
  1917. fi
  1918. relation_dir=$(get_relation_data_dir "$service" "$ts" "$relation") || {
  1919. err "Failed to find relation file"
  1920. return 1
  1921. }
  1922. relation_file="$relation_dir/data"
  1923. if ! [ -e "$relation_file" ]; then
  1924. e "$rc" > "$relation_file"
  1925. chmod go-rwx "$relation_file" ## protecting this file
  1926. fi
  1927. echo "$relation_file"
  1928. }
  1929. export -f service:relation-file
  1930. service:relation-options() {
  1931. local service_path="$1" relation_file
  1932. relation_file=$(service:relation-file "$service_path") || {
  1933. err "Failed to find relation file"
  1934. return 1
  1935. }
  1936. cat "$relation_file"
  1937. }
  1938. export -f service:relation-options
  1939. relation:get() {
  1940. local service_path="$1" query="$2" relation_file
  1941. relation_file=$(service:relation-file "$service_path") || {
  1942. err "Failed to find relation file"
  1943. return 1
  1944. }
  1945. cfg-get-value "$query" < "$relation_file"
  1946. }
  1947. export -f relation:get
  1948. _get_charm_metadata_uses() {
  1949. local metadata="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  1950. if [ -e "$cache_file" ]; then
  1951. #debug "$FUNCNAME: SESSION cache hit $1"
  1952. cat "$cache_file" || return 1
  1953. return 0
  1954. fi
  1955. printf "%s" "$metadata" | { shyaml key-values-0 uses 2>/dev/null || true; } | tee "$cache_file"
  1956. }
  1957. export -f _get_charm_metadata_uses
  1958. _get_service_metadata() {
  1959. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1960. charm
  1961. if [ -e "$cache_file" ]; then
  1962. #debug "$FUNCNAME: SESSION cache hit $1"
  1963. cat "$cache_file"
  1964. return 0
  1965. fi
  1966. charm="$(get_service_charm "$service")" || return 1
  1967. charm.metadata "$charm" > "$cache_file"
  1968. if [ "$?" != 0 ]; then
  1969. rm -f "$cache_file" ## no cache
  1970. return 1
  1971. fi
  1972. cat "$cache_file"
  1973. }
  1974. export -f _get_service_metadata
  1975. _get_service_uses() {
  1976. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1977. metadata
  1978. if [ -e "$cache_file" ]; then
  1979. #debug "$FUNCNAME: SESSION cache hit $1"
  1980. cat "$cache_file"
  1981. return 0
  1982. fi
  1983. metadata="$(_get_service_metadata "$service")" || return 1
  1984. _get_charm_metadata_uses "$metadata" > "$cache_file"
  1985. if [ "$?" != 0 ]; then
  1986. rm -f "$cache_file" ## no cache
  1987. return 1
  1988. fi
  1989. cat "$cache_file"
  1990. }
  1991. export -f _get_service_uses
  1992. _get_services_uses() {
  1993. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  1994. service rn rd
  1995. if [ -e "$cache_file" ]; then
  1996. #debug "$FUNCNAME: SESSION cache hit $1"
  1997. cat "$cache_file"
  1998. return 0
  1999. fi
  2000. for service in "$@"; do
  2001. _get_service_uses "$service" | while read-0 rn rd; do
  2002. printf "%s\0" "$service" "$rn" "$rd"
  2003. done
  2004. [ "${PIPESTATUS[0]}" == 0 ] || {
  2005. return 1
  2006. }
  2007. done > "${cache_file}.wip"
  2008. mv "${cache_file}"{.wip,} &&
  2009. cat "$cache_file" || return 1
  2010. }
  2011. export -f _get_services_uses
  2012. _get_provides_provides() {
  2013. local provides="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2014. service rn rd
  2015. if [ -e "$cache_file" ]; then
  2016. # debug "$FUNCNAME: CACHEDIR cache hit $1"
  2017. cat "$cache_file"
  2018. return 0
  2019. fi
  2020. type=$(printf "%s" "$provides" | shyaml get-type)
  2021. case "$type" in
  2022. sequence)
  2023. while read-0 prov; do
  2024. printf "%s\0" "$prov" ""
  2025. done < <(echo "$provides" | shyaml get-values-0)
  2026. ;;
  2027. struct)
  2028. printf "%s" "$provides" | shyaml key-values-0
  2029. ;;
  2030. str)
  2031. printf "%s\0" "$(echo "$provides" | shyaml get-value)" ""
  2032. ;;
  2033. *)
  2034. err "Unexpected type '$type' for provider identifier in charm '$charm'."
  2035. return 1
  2036. esac | tee "$cache_file"
  2037. return "${PIPESTATUS[0]}"
  2038. }
  2039. _get_metadata_provides() {
  2040. local metadata="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2041. service rn rd
  2042. if [ -e "$cache_file" ]; then
  2043. #debug "$FUNCNAME: CACHEDIR cache hit"
  2044. cat "$cache_file"
  2045. return 0
  2046. fi
  2047. provides=$(printf "%s" "$metadata" | shyaml get-value -y -q provides "")
  2048. [ "$provides" -a "$provides" != "''" ] || { touch "$cache_file"; return 0; }
  2049. _get_provides_provides "$provides" | tee "$cache_file"
  2050. return "${PIPESTATUS[0]}"
  2051. }
  2052. _get_services_provides() {
  2053. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2054. service rn rd
  2055. if [ -e "$cache_file" ]; then
  2056. #debug "$FUNCNAME: SESSION cache hit $1"
  2057. cat "$cache_file"
  2058. return 0
  2059. fi
  2060. ## YYY: replace the inner loop by a cached function
  2061. for service in "$@"; do
  2062. metadata="$(_get_service_metadata "$service")" || return 1
  2063. while read-0 rn rd; do
  2064. printf "%s\0" "$service" "$rn" "$rd"
  2065. done < <(_get_metadata_provides "$metadata")
  2066. done > "$cache_file"
  2067. if [ "$?" != 0 ]; then
  2068. rm -f "$cache_file" ## no cache
  2069. return 1
  2070. fi
  2071. cat "$cache_file"
  2072. }
  2073. export -f _get_services_provides
  2074. _get_charm_provides() {
  2075. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(charm.store_metadata_hash)" errlvl
  2076. if [ -e "$cache_file" ]; then
  2077. #debug "$FUNCNAME: SESSION cache hit"
  2078. cat "$cache_file"
  2079. return 0
  2080. fi
  2081. start="$SECONDS"
  2082. debug "Getting charm provider list..."
  2083. while read-0 charm _ realpath metadata; do
  2084. metadata="$(charm.metadata "$charm")" || continue
  2085. # echo "reading $charm" >&2
  2086. while read-0 rn rd; do
  2087. printf "%s\0" "$charm" "$rn" "$rd"
  2088. done < <(_get_metadata_provides "$metadata")
  2089. done < <(charm.ls) | tee "$cache_file"
  2090. errlvl="${PIPESTATUS[0]}"
  2091. debug " ..charm provider list done $GRAY(in $((SECONDS - start))s)$NORMAL"
  2092. return "$errlvl"
  2093. }
  2094. _get_charm_providing() {
  2095. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2096. relation="$1"
  2097. if [ -e "$cache_file" ]; then
  2098. #debug "$FUNCNAME: SESSION cache hit $1"
  2099. cat "$cache_file"
  2100. return 0
  2101. fi
  2102. while read-0 charm relation_name relation_def; do
  2103. [ "$relation_name" == "$relation" ] || continue
  2104. printf "%s\0" "$charm" "$relation_def"
  2105. done < <(_get_charm_provides) > "$cache_file"
  2106. if [ "$?" != 0 ]; then
  2107. rm -f "$cache_file" ## no cache
  2108. return 1
  2109. fi
  2110. cat "$cache_file"
  2111. }
  2112. _get_services_providing() {
  2113. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2114. relation="$1"
  2115. shift ## services is "$@"
  2116. if [ -e "$cache_file" ]; then
  2117. #debug "$FUNCNAME: SESSION cache hit $1"
  2118. cat "$cache_file"
  2119. return 0
  2120. fi
  2121. while read-0 service relation_name relation_def; do
  2122. [ "$relation_name" == "$relation" ] || continue
  2123. printf "%s\0" "$service" "$relation_def"
  2124. done < <(_get_services_provides "$@") > "$cache_file"
  2125. if [ "$?" != 0 ]; then
  2126. rm -f "$cache_file" ## no cache
  2127. return 1
  2128. fi
  2129. cat "$cache_file"
  2130. }
  2131. export -f _get_services_provides
  2132. _out_new_relation_from_defs() {
  2133. local service="$1" rn="$2" ts="$3" prov_def="$4" rel_def="$5" rc td rc_prov
  2134. rc_prov=$(printf "%s" "$prov_def" | shyaml -y get-value "default-options" 2>/dev/null)
  2135. ## YYYvlab: should be seen even in no debug mode no ?
  2136. rc=$(printf "%s" "$rel_def" | shyaml -y get-value "default-options" 2>/dev/null)
  2137. td=$(echo "$prov_def" | shyaml get-value 'tech-dep' 2>/dev/null)
  2138. td=${td:-True}
  2139. rc=$(merge_yaml_str "$rc_prov" "$rc") || return 1
  2140. after=$(_out_after_value_from_def "$service" "$rn" "$rel_def") || return 1
  2141. printf "%s\0" "$after" "$service" "$relation_name" "$ts" "$rc" "$td"
  2142. }
  2143. _out_after_value_from_def() {
  2144. local service="$1" relation_name="$2" relation_def="$3" after_t after
  2145. if after_t=$(echo "$relation_def" | shyaml get-type after 2>/dev/null); then
  2146. case "$after_t" in
  2147. sequence)
  2148. after="$(echo "$relation_def" | shyaml get-values after 2>/dev/null)" || return 1
  2149. after=",$service:${after//$'\n'/,$service:},"
  2150. ;;
  2151. struct)
  2152. err "Invalid type for ${WHITE}after${NORMAL}'s value in ${DARKBLUE}$relation_name${NORMAL}'s definition."
  2153. return 1
  2154. ;;
  2155. str)
  2156. after=",$service:$(echo "$relation_def" | shyaml get-value after "" 2>/dev/null)," || return 1
  2157. ;;
  2158. esac
  2159. else
  2160. after=""
  2161. fi
  2162. e "$after"
  2163. }
  2164. get_all_relations () {
  2165. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$(declare -p without_relations)")" \
  2166. services
  2167. if [ -n "$ALL_RELATIONS" ]; then
  2168. cat "$ALL_RELATIONS" || return 1
  2169. return 0
  2170. fi
  2171. if [ -e "${cache_file}" ]; then
  2172. #debug "$FUNCNAME: SESSION cache hit $1"
  2173. export ALL_RELATIONS="$cache_file"
  2174. cat "${cache_file}"
  2175. return 0
  2176. fi
  2177. declare -A services
  2178. services_uses=()
  2179. ## XXXvlab: bwerk, leveraging cache to be able to get the errorlevel here.
  2180. _get_services_uses "$@" || return 1
  2181. array_read-0 services_uses < <(_get_services_uses "$@")
  2182. services_provides=()
  2183. ## XXXvlab: bwerk, leveraging cache to be able to get the errorlevel here.
  2184. _get_services_provides "$@" || return 1
  2185. array_read-0 services_provides < <(_get_services_provides "$@")
  2186. for service in "$@"; do
  2187. services[$service]=1
  2188. done
  2189. all_services=("$@")
  2190. while [ "${#all_services[@]}" != 0 ]; do
  2191. array_pop all_services service
  2192. while read-0 relation_name ts relation_config tech_dep; do
  2193. [ "${without_relations[$service:$relation_name]}" ] && {
  2194. debug "Ignoring compose $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> ${DARKYELLOW}$ts$NORMAL"
  2195. continue
  2196. }
  2197. ## First is priority, that can be adjusted in second step
  2198. printf "%s\0" "" "$service" "$relation_name" "$ts" "$relation_config" "$tech_dep"
  2199. ## adding target services ?
  2200. [ "${services[$ts]}" ] && continue
  2201. array_read-0 services_uses < <(_get_services_uses "$ts")
  2202. all_services+=("$ts")
  2203. services[$ts]=1
  2204. done < <(get_compose_relations "$service")
  2205. done > "${cache_file}.wip"
  2206. while true; do
  2207. changed=
  2208. new_services_uses=()
  2209. summon=()
  2210. required=()
  2211. recommended=()
  2212. optional=()
  2213. while [ "${#services_uses[@]}" != 0 ]; do
  2214. service="${services_uses[0]}"
  2215. relation_name="${services_uses[1]}"
  2216. relation_def="${services_uses[2]}"
  2217. services_uses=("${services_uses[@]:3}")
  2218. [ "${without_relations[$service:$relation_name]}" ] && {
  2219. debug "Skipping $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW*$NORMAL"
  2220. continue
  2221. }
  2222. default_options=$(printf "%s" "$relation_def" | shyaml -y get-value "default-options" 2>/dev/null)
  2223. after=$(_out_after_value_from_def "$service" "$relation_name" "$relation_def") || return 1
  2224. ## is this "use" declaration satisfied ?
  2225. found=
  2226. while read-0 p s rn ts rc td; do
  2227. if [ -z "$found" -a "$service" == "$s" -a "$relation_name" == "$rn" ]; then
  2228. if [ "$default_options" ]; then
  2229. rc=$(merge_yaml_str "$default_options" "$rc") || return 1
  2230. fi
  2231. found="$ts"
  2232. p="$after"
  2233. fi
  2234. printf "%s\0" "$p" "$s" "$rn" "$ts" "$rc" "$td"
  2235. done < "${cache_file}.wip" > "${cache_file}.wip.new"
  2236. mv "${cache_file}.wip.new" "${cache_file}.wip"
  2237. if [ "$found" ]; then ## this "use" declaration was satisfied
  2238. debug "${DARKYELLOW}$service${NORMAL} use declaration for relation" \
  2239. "${DARKBLUE}$relation_name${NORMAL} is satisfied with ${DARKYELLOW}$found${NORMAL}"
  2240. continue
  2241. fi
  2242. auto=$(echo "$relation_def" | shyaml get-value auto pair 2>/dev/null)
  2243. auto=${auto:-pair}
  2244. case "$auto" in
  2245. "pair"|"summon")
  2246. service_list=()
  2247. array_read-0 service_list < <(array_keys_to_stdin services)
  2248. providers=()
  2249. providers_def=()
  2250. array_read-0 providers providers_def < <(_get_services_providing "$relation_name" "${service_list[@]}")
  2251. if [ "${#providers[@]}" == 1 ]; then
  2252. ts="${providers[0]}"
  2253. debug "Auto-pairs ${DARKYELLOW}$service${NORMAL}" \
  2254. "--${DARKBLUE}$relation_name${NORMAL}--> ${DARKYELLOW}$ts${NORMAL}"
  2255. _out_new_relation_from_defs "$service" "$relation_name" "$ts" \
  2256. "${providers_def[0]}" "$relation_def" \
  2257. >> "${cache_file}.wip"
  2258. ## Adding service
  2259. [ "${services[$ts]}" ] && continue
  2260. array_read-0 new_services_uses < <(_get_services_uses "$ts")
  2261. services[$ts]=1
  2262. changed=1
  2263. continue
  2264. elif [ "${#providers[@]}" -gt 1 ]; then
  2265. msg=""
  2266. warn "No auto-pairing ${DARKYELLOW}$service${NORMAL}" \
  2267. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  2268. "(> 1 provider)."
  2269. continue
  2270. else
  2271. if [ "$auto" == "summon" ]; then
  2272. summon+=("$service" "$relation_name" "$relation_def")
  2273. fi
  2274. fi
  2275. ;;
  2276. null|disable|disabled)
  2277. :
  2278. ;;
  2279. *)
  2280. err "Invalid ${WHITE}auto${NORMAL} value '$auto'."
  2281. return 1
  2282. ;;
  2283. esac
  2284. constraint=$(echo "$relation_def" | shyaml get-value constraint 2>/dev/null)
  2285. constraint=${constraint:-optional}
  2286. case "$constraint" in
  2287. "required")
  2288. required+=("$service" "$relation_name" "$relation_def")
  2289. ;;
  2290. "recommended")
  2291. recommended+=("$service" "$relation_name" "$relation_def")
  2292. ;;
  2293. "optional")
  2294. optional+=("$service" "$relation_name" "$relation_def")
  2295. ;;
  2296. *)
  2297. err "Invalid ${WHITE}constraint${NORMAL} value '$constraint'."
  2298. return 1
  2299. ;;
  2300. esac
  2301. new_services_uses+=("$service" "$relation_name" "$relation_def") ## re-queue it
  2302. done
  2303. services_uses=("${new_services_uses[@]}")
  2304. if [ "$changed" ]; then
  2305. continue
  2306. fi
  2307. ## situation is stable
  2308. if [ "${#summon[@]}" != 0 ]; then
  2309. declare -A summon_requeued=()
  2310. while [ "${#summon[@]}" != 0 ]; do
  2311. service="${summon[0]}"
  2312. relation_name="${summon[1]}"
  2313. relation_def="${summon[2]}"
  2314. summon=("${summon[@]:3}")
  2315. providers=()
  2316. providers_def=()
  2317. array_read-0 providers providers_def < <(_get_charm_providing "$relation_name" "${service_list[@]}")
  2318. if [ "${#providers[@]}" == 0 ]; then
  2319. err "Summoning a ${DARKBLUE}$relation_name${NORMAL} provider failed: none were found in charm store."
  2320. return 1
  2321. fi
  2322. if [ "${#providers[@]}" -gt 1 ]; then
  2323. if [ -z "${summon_requeued[$service/$relation_name]}" ]; then
  2324. debug "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  2325. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  2326. "(> 1 provider). Requeing."
  2327. summon+=("$service" "$relation_name" "$relation_def") ## re-queue it
  2328. summon_requeued["$service/$relation_name"]=1
  2329. continue
  2330. else
  2331. warn "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  2332. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  2333. "(> 1 provider). Choosing first."
  2334. fi
  2335. fi
  2336. ts="${providers[0]}"
  2337. ## YYYvlab: should be seen even in no debug mode no ?
  2338. debug "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  2339. "--${DARKBLUE}$relation_name${NORMAL}--> ${DARKYELLOW}$ts${NORMAL}"
  2340. _out_new_relation_from_defs "$service" "$relation_name" "$ts" \
  2341. "${providers_def[0]}" "$relation_def" \
  2342. >> "${cache_file}.wip"
  2343. ## Adding service
  2344. [ "${services[$ts]}" ] && continue
  2345. array_read-0 services_uses < <(_get_services_uses "$ts")
  2346. services[$ts]=1
  2347. changed=1
  2348. continue 2
  2349. done
  2350. continue
  2351. fi
  2352. [ "$NO_CONSTRAINT_CHECK" ] && break
  2353. if [ "${#required[@]}" != 0 ]; then
  2354. echo "$(_display_solves required)" | sed -r "s/^/${RED}||${NORMAL} /g" >&2
  2355. err "Required relations not satisfied"
  2356. return 1
  2357. fi
  2358. if [ "${#recommended[@]}" != 0 ]; then
  2359. ## make recommendation
  2360. echo "$(_display_solves recommended)" | sed -r "s/^/${YELLOW}||${NORMAL} /g" >&2
  2361. fi
  2362. if [ -z "$QUIET" ]; then
  2363. if [ "${#optional[@]}" != 0 ]; then
  2364. ## inform about options
  2365. echo "$(_display_solves optional)" | sed -r "s/^/${BLUE}||${NORMAL} /g" >&2
  2366. fi
  2367. fi
  2368. # if [ "${#required[@]}" != 0 ]; then
  2369. # err "Required relations not satisfied"
  2370. # return 1
  2371. # fi
  2372. if [ "${#recommended[@]}" != 0 ]; then
  2373. warn "Recommended relations not satisfied"
  2374. fi
  2375. break
  2376. done
  2377. if [ "$?" != 0 ]; then
  2378. rm -f "${cache_file}"{,.wip,.wip.new} ## no cache
  2379. return 1
  2380. fi
  2381. ##
  2382. ## Sort relations thanks to uses =metadata.yml= relations.
  2383. ##
  2384. mv "${cache_file}.wip"{,.in} &&
  2385. rm -f "${cache_file}.wip.final" &&
  2386. touch "${cache_file}.wip.final" || {
  2387. err "Unexpected error when mangling cache files."
  2388. return 1
  2389. }
  2390. declare -A relation_done=()
  2391. while true; do
  2392. had_remaining_relation=
  2393. had_new_relation=
  2394. while read-0 p s rn ts rc td; do
  2395. if [ -z "$p" ] || [ "$p" == "," ]; then
  2396. relation_done["$s:$rn"]=1
  2397. # printf " .. %-30s %-30s %-30s\n" "--" "$s" "$rn" >&2
  2398. printf "%s\0" "$s" "$rn" "$ts" "$rc" "$td" >> "${cache_file}.wip.final"
  2399. had_new_relation=1
  2400. else
  2401. # printf " !! %-30s %-30s %-30s\n" "$p" "$s" "$rn" >&2
  2402. printf "%s\0" "$p" "$s" "$rn" "$ts" "$rc" "$td" >> "${cache_file}.wip.out"
  2403. had_remaining_relation=1
  2404. fi
  2405. done < "${cache_file}.wip.in"
  2406. [ -z "$had_remaining_relation" ] && break
  2407. mv "${cache_file}.wip."{out,in}
  2408. while read-0 p s rn ts rc td; do
  2409. for rel in "${!relation_done[@]}"; do
  2410. p="${p//,$rel,/,}"
  2411. done
  2412. # printf " CC %-30s %-30s %-30s\n" "$p" "$s" "$rn" >&2
  2413. if [ -z "$had_new_relation" ]; then
  2414. err "${DARKYELLOW}$s${NORMAL} --${DARKBLUE}$rn${NORMAL}--> ${DARKYELLOW}$ts${NORMAL} missing required ${WHITE}after${NORMAL} relations:"
  2415. for rel in ${p//,/ }; do
  2416. rel_s=${rel%%:*}
  2417. rel_r=${rel##*:}
  2418. echo " - ${DARKYELLOW}$rel_s${NORMAL} --${DARKBLUE}$rel_r${NORMAL}--> ${DARKGRAY}*${NORMAL}" >&2
  2419. done
  2420. else
  2421. printf "%s\0" "$p" "$s" "$rn" "$ts" "$rc" "$td" >> "${cache_file}.wip.out"
  2422. fi
  2423. done < "${cache_file}.wip.in"
  2424. if [ -z "$had_new_relation" ]; then
  2425. rm -f "${cache_file}"{,.wip{,new,in,out,final}} ## no cache
  2426. return 1
  2427. fi
  2428. mv "${cache_file}.wip."{out,in}
  2429. done
  2430. export ALL_RELATIONS="$cache_file"
  2431. mv "${cache_file}"{.wip.final,} || return 1
  2432. cat "$cache_file"
  2433. }
  2434. export -f get_all_relations
  2435. _display_solves() {
  2436. local array_name="$1" by_relation msg
  2437. ## inform about options
  2438. msg=""
  2439. declare -A by_relation
  2440. while read-0 service relation_name relation_def; do
  2441. solves=$(printf "%s" "$relation_def" | shyaml -y get-value solves 2>/dev/null);
  2442. auto=$(printf "%s" "$relation_def" | shyaml get-value auto 2>/dev/null);
  2443. if [ -z "$solves" ]; then
  2444. continue
  2445. fi
  2446. by_relation[$relation_name]+=$(printf "\n %s" "${DARKYELLOW}$service$NORMAL for:")
  2447. if [ "$auto" == "pair" ]; then
  2448. requirement="add provider in cluster to auto-pair"
  2449. else
  2450. requirement="add explicit relation"
  2451. fi
  2452. while read-0 name def; do
  2453. by_relation[$relation_name]+=$(printf "\n - ${DARKCYAN}%-15s${NORMAL} %s (%s)" "$name" "$def" "$requirement")
  2454. done < <(printf "%s" "$solves" | shyaml key-values-0)
  2455. done < <(array_values_to_stdin "$array_name")
  2456. while read-0 relation_name message; do
  2457. msg+="$(printf "\n${DARKBLUE}%s$NORMAL provider is $array_name by%s" \
  2458. "$relation_name" "$message" )"
  2459. done < <(array_kv_to_stdin by_relation)
  2460. if [ "$msg" ]; then
  2461. printf "%s\n" "${msg:1}"
  2462. fi
  2463. }
  2464. get_compose_relation_def() {
  2465. local service="$1" relation="$2" relation_name target_service relation_config tech_dep
  2466. while read-0 relation_name target_service relation_config tech_dep; do
  2467. [ "$relation_name" == "$relation" ] || continue
  2468. printf "%s\0%s\0%s\0" "$target_service" "$relation_config" "$tech_dep"
  2469. done < <(get_compose_relations "$service") || return 1
  2470. }
  2471. export -f get_compose_relation_def
  2472. run_service_relations () {
  2473. local service services loaded subservices subservice
  2474. PROJECT_NAME=$(get_default_project_name) || return 1
  2475. export PROJECT_NAME
  2476. declare -A loaded
  2477. subservices=$(get_ordered_service_dependencies "$@") || return 1
  2478. for service in $subservices; do
  2479. # debug "Upping dep's relations of ${DARKYELLOW}$service${NORMAL}:"
  2480. for subservice in $(get_service_deps "$service") "$service"; do
  2481. [ "${loaded[$subservice]}" ] && continue
  2482. export BASE_SERVICE_NAME=$service
  2483. MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
  2484. MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1
  2485. RELATION_BASE_COMPOSE_DEF=$(get_compose_service_def "$subservice") || return 1
  2486. export RELATION_BASE_COMPOSE_DEF MASTER_BASE_{CHARM,SERVICE}_NAME
  2487. # debug " Relations of ${DARKYELLOW}$subservice${NORMAL}:"
  2488. while read-0 relation_name target_service relation_config tech_dep; do
  2489. [ "${without_relations[$service:$relation_name]}" ] && {
  2490. debug "Skipping $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW*$NORMAL"
  2491. continue
  2492. }
  2493. export relation_config
  2494. export TARGET_SERVICE_NAME=$target_service
  2495. MASTER_TARGET_SERVICE_NAME=$(get_top_master_service_for_service "$target_service") || return 1
  2496. MASTER_TARGET_CHARM_NAME=$(get_service_charm "$MASTER_TARGET_SERVICE_NAME") || return 1
  2497. RELATION_TARGET_COMPOSE_DEF=$(get_compose_service_def "$target_service") || return 1
  2498. export RELATION_TARGET_COMPOSE_DEF MASTER_TARGET_{CHARM,SERVICE}_NAME
  2499. Wrap "${wrap_opts[@]}" -d "Building $DARKYELLOW$subservice$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW$target_service$NORMAL" <<EOF || return 1
  2500. _run_service_relation "$relation_name" "$subservice" "$target_service" "\$relation_config"
  2501. EOF
  2502. done < <(get_service_relations "$subservice") || return 1
  2503. loaded[$subservice]=1
  2504. done
  2505. done
  2506. }
  2507. export -f run_service_relations
  2508. _run_service_action_direct() {
  2509. local service="$1" action="$2" charm _dummy project_name
  2510. shift; shift
  2511. read-0 charm action_script_path || true ## against 'set -e' that could be setup in parent scripts
  2512. if read-0 _dummy || [ "$_dummy" ]; then
  2513. print_syntax_error "$FUNCNAME: too many arguments in action descriptor"
  2514. return 1
  2515. fi
  2516. project_name=$(get_default_project_name) || return 1
  2517. export PROJECT_NAME="$project_name"
  2518. export state_tmpdir
  2519. (
  2520. set +e ## Prevents unwanted leaks from parent shell
  2521. export COMPOSE_CONFIG=$(get_compose_yml_content)
  2522. export METADATA_CONFIG=$(charm.metadata "$charm")
  2523. export SERVICE_NAME=$service
  2524. export ACTION_NAME=$action
  2525. export ACTION_SCRIPT_PATH="$action_script_path"
  2526. export CONTAINER_NAME=$(get_top_master_service_for_service "$service")
  2527. export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME")
  2528. export SERVICE_DATASTORE="$DATASTORE/$service"
  2529. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  2530. exname="$exname $ACTION_NAME $SERVICE_NAME" \
  2531. stdbuf -oL -eL bash -c 'charm.run_direct_action "$@"' -- "$charm" "$action" "$@"
  2532. ) 0<&6 ## inject general stdin
  2533. }
  2534. export -f _run_service_action_direct
  2535. _run_service_action_relation() {
  2536. local service="$1" action="$2" charm target_charm relation_name relation_config _dummy
  2537. shift; shift
  2538. read-0 charm target_service target_charm relation_name relation_config action_script_path || true
  2539. if read-0 _dummy || [ "$_dummy" ]; then
  2540. print_syntax_error "$FUNCNAME: too many arguments in action descriptor"
  2541. return 1
  2542. fi
  2543. export action_errlvl_file="$state_tmpdir/action-$service-$charm-$action-errlvl"
  2544. export state_tmpdir
  2545. (
  2546. set +e ## Prevents unwanted leaks from parent shell
  2547. export METADATA_CONFIG=$(charm.metadata "$charm")
  2548. export SERVICE_NAME=$service
  2549. export RELATION_TARGET_SERVICE="$target_service"
  2550. export RELATION_TARGET_CHARM="$target_charm"
  2551. export RELATION_BASE_SERVICE="$service"
  2552. export RELATION_BASE_CHARM="$charm"
  2553. export ACTION_NAME=$action
  2554. export ACTION_SCRIPT_PATH="$action_script_path"
  2555. export CONTAINER_NAME=$(get_top_master_service_for_service "$service")
  2556. export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME")
  2557. export SERVICE_DATASTORE="$DATASTORE/$service"
  2558. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  2559. exname="$exname $ACTION_NAME $SERVICE_NAME" \
  2560. stdbuf -oL -eL bash -c 'charm.run_relation_action "$@"' -- "$target_charm" "$relation_name" "$action" "$@"
  2561. ) 0<&6 ## inject general stdin
  2562. }
  2563. export -f _run_service_action_relation
  2564. get_relation_data_dir() {
  2565. local service="$1" target_service="$2" relation_name="$3" \
  2566. cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  2567. if [ -e "$cache_file" ]; then
  2568. # debug "$FUNCNAME: cache hit ($*)"
  2569. cat "$cache_file"
  2570. return 0
  2571. fi
  2572. local project relation_dir
  2573. project=${PROJECT_NAME}
  2574. if [ -z "$project" ]; then
  2575. project=$(get_default_project_name) || return 1
  2576. fi
  2577. relation_dir="$VARDIR/relations/$project/${service}-${target_service}/$relation_name"
  2578. if ! [ -d "$relation_dir" ]; then
  2579. mkdir -p "$relation_dir" || return 1
  2580. chmod go-rwx "$relation_dir" || return 1 ## protecting this directory
  2581. fi
  2582. echo "$relation_dir" | tee "$cache_file"
  2583. }
  2584. export -f get_relation_data_dir
  2585. get_relation_data_file() {
  2586. local service="$1" target_service="$2" relation_name="$3" relation_config="$4"
  2587. relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1
  2588. relation_data_file="$relation_dir/data"
  2589. new=
  2590. if [ -e "$relation_data_file" ]; then
  2591. ## Has reference changed ?
  2592. new_md5=$(e "$relation_config" | md5_compat)
  2593. if [ "$new_md5" != "$(cat "$relation_data_file.md5_ref" 2>/dev/null)" ]; then
  2594. new=true
  2595. fi
  2596. else
  2597. new=true
  2598. fi
  2599. if [ "$new" ]; then
  2600. e "$relation_config" > "$relation_data_file"
  2601. chmod go-rwx "$relation_data_file" ## protecting this file
  2602. e "$relation_config" | md5_compat > "$relation_data_file.md5_ref"
  2603. fi
  2604. echo "$relation_data_file"
  2605. }
  2606. export -f get_relation_data_file
  2607. has_service_action () {
  2608. local service="$1" action="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2609. charm target_charm relation_name target_service relation_config _tech_dep \
  2610. path
  2611. if [ -e "$cache_file" ]; then
  2612. # debug "$FUNCNAME: cache hit ($*)"
  2613. cat "$cache_file"
  2614. return 0
  2615. fi
  2616. charm=$(get_service_charm "$service") || return 1
  2617. ## Action directly provided ?
  2618. if path=$(charm.has_direct_action "$charm" "$action"); then
  2619. p0 "direct" "$charm" "$path" | tee "$cache_file"
  2620. return 0
  2621. fi
  2622. ## Action provided by relation ?
  2623. while read-0 relation_name target_service relation_config _tech_dep; do
  2624. target_charm=$(get_service_charm "$target_service") || return 1
  2625. if path=$(charm.has_relation_action "$target_charm" "$relation_name" "$action"); then
  2626. p0 "relation" "$charm" "$target_service" "$target_charm" "$relation_name" "$relation_config" "$path" | tee "$cache_file"
  2627. return 0
  2628. fi
  2629. done < <(get_service_relations "$service")
  2630. return 1
  2631. # master=$(get_top_master_service_for_service "$service")
  2632. # [ "$master" == "$charm" ] && return 1
  2633. # has_service_action "$master" "$action"
  2634. }
  2635. export -f has_service_action
  2636. run_service_action () {
  2637. local service="$1" action="$2" errlvl
  2638. shift ; shift
  2639. exec 6<&0 ## saving stdin
  2640. {
  2641. if ! read-0 action_type; then
  2642. info "Service $DARKYELLOW$service$NORMAL does not have any action $DARKCYAN$action$NORMAL defined."
  2643. info " Add an executable script to 'actions/$action' to implement action."
  2644. return 1
  2645. fi
  2646. "_run_service_action_${action_type}" "$service" "$action" "$@"
  2647. errlvl="$?"
  2648. } < <(has_service_action "$service" "$action")
  2649. exec 0<&6 6<&- ## restoring stdin
  2650. return "$errlvl"
  2651. }
  2652. export -f run_service_action
  2653. get_compose_relation_config() {
  2654. local service=$1 relation_config cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  2655. if [ -e "$cache_file" ]; then
  2656. # debug "$FUNCNAME: cache hit ($*)"
  2657. cat "$cache_file"
  2658. return 0
  2659. fi
  2660. compose_service_def=$(get_compose_service_def "$service") || return 1
  2661. echo "$compose_service_def" | shyaml get-value "relations" 2>/dev/null | tee "$cache_file"
  2662. }
  2663. export -f get_compose_relation_config
  2664. # ## Return key-values-0
  2665. # get_compose_relation_config_for_service() {
  2666. # local service=$1 relation_name=$2 relation_config
  2667. # compose_service_relations=$(get_compose_relation_config "$service") || return 1
  2668. # if ! relation_config=$(
  2669. # echo "$compose_service_relations" |
  2670. # shyaml get-value "${relation_name}" 2>/dev/null); then
  2671. # err "Couldn't find $DARKYELLOW${service}$NORMAL/${WHITE}${relation_name}$NORMAL" \
  2672. # "relation config in compose configuration."
  2673. # return 1
  2674. # fi
  2675. # if [ -z "$relation_config" ]; then
  2676. # err "Relation ${WHITE}mysql-database$NORMAL is empty in compose configuration."
  2677. # return 1
  2678. # fi
  2679. # if ! echo "$relation_config" | shyaml key-values-0 2>/dev/null; then
  2680. # err "No key/values in ${DARKBLUE}mysql-database$NORMAL of compose config."
  2681. # return 1
  2682. # fi
  2683. # }
  2684. # export -f get_compose_relation_config_for_service
  2685. _get_container_relation() {
  2686. local metadata=$1 found relation_name relation_def
  2687. found=
  2688. while read-0 relation_name relation_def; do
  2689. [ "$(echo "$relation_def" | shyaml get-value "scope" 2>/dev/null)" == "container" ] && {
  2690. found="$relation_name"
  2691. break
  2692. }
  2693. done < <(_get_charm_metadata_uses "$metadata")
  2694. if [ -z "$found" ]; then
  2695. err "Charm $DARKPINK$charm$NORMAL is a subordinate but does not have any required relation declaration with" \
  2696. "${WHITE}scope${NORMAL} set to 'container'."
  2697. return 1
  2698. fi
  2699. printf "%s" "$found"
  2700. }
  2701. _get_master_service_for_service_cached () {
  2702. local service="$1" charm="$2" metadata="$3" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  2703. charm requires master_charm target_charm target_service service_def found
  2704. if [ -e "$cache_file" ]; then
  2705. # debug "$FUNCNAME: STATIC cache hit ($1)"
  2706. cat "$cache_file" &&
  2707. touch "$cache_file" || return 1
  2708. return 0
  2709. fi
  2710. if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" != "True" ]; then
  2711. ## just return service name
  2712. echo "$service" | tee "$cache_file"
  2713. return 0
  2714. fi
  2715. ## Action provided by relation ?
  2716. container_relation=$(_get_container_relation "$metadata") || return 1
  2717. read-0 target_service _ _ < <(get_service_relation "$service" "$container_relation")
  2718. if [ -z "$target_service" ]; then
  2719. err "Couldn't find ${WHITE}relations.${container_relation}${NORMAL} in" \
  2720. "${DARKYELLOW}$service$NORMAL compose definition."
  2721. err ${FUNCNAME[@]}
  2722. return 1
  2723. fi
  2724. echo "$target_service" | tee "$cache_file"
  2725. }
  2726. export -f _get_master_service_for_service_cached
  2727. get_master_service_for_service() {
  2728. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2729. charm metadata result
  2730. if [ -e "$cache_file" ]; then
  2731. # debug "$FUNCNAME: SESSION cache hit ($*)"
  2732. cat "$cache_file" || return 1
  2733. return 0
  2734. fi
  2735. charm=$(get_service_charm "$service") || return 1
  2736. metadata=$(charm.metadata "$charm" 2>/dev/null) || {
  2737. metadata=""
  2738. warn "No charm $DARKPINK$charm$NORMAL found."
  2739. }
  2740. result=$(_get_master_service_for_service_cached "$service" "$charm" "$metadata") || return 1
  2741. echo "$result" | tee "$cache_file" || return 1
  2742. }
  2743. export -f get_master_service_for_service
  2744. get_top_master_service_for_service() {
  2745. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2746. current_service
  2747. if [ -e "$cache_file" ]; then
  2748. # debug "$FUNCNAME: cache hit ($*)"
  2749. cat "$cache_file"
  2750. return 0
  2751. fi
  2752. current_service="$service"
  2753. while true; do
  2754. master_service=$(get_master_service_for_service "$current_service") || return 1
  2755. [ "$master_service" == "$current_service" ] && break
  2756. current_service="$master_service"
  2757. done
  2758. echo "$current_service" | tee "$cache_file"
  2759. return 0
  2760. }
  2761. export -f get_top_master_service_for_service
  2762. ##
  2763. ## The result is a mixin that is not always a complete valid
  2764. ## docker-compose entry (thinking of subordinates). The result
  2765. ## will be merge with master charms.
  2766. _get_docker_compose_mixin_from_metadata_cached() {
  2767. local service="$1" charm="$2" metadata="$3" \
  2768. has_build_dir="$4" \
  2769. cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  2770. metadata_file metadata volumes docker_compose subordinate image \
  2771. mixin mixins tmemory memory limit docker_memory
  2772. if [ -e "$cache_file" ]; then
  2773. #debug "$FUNCNAME: STATIC cache hit $1"
  2774. cat "$cache_file" &&
  2775. touch "$cache_file" || return 1
  2776. return 0
  2777. fi
  2778. mixins=("$(echo -en "labels:\n- compose.charm=$charm")")
  2779. if [ "$metadata" ]; then
  2780. ## resources to volumes
  2781. volumes=$(
  2782. for resource_type in data config; do
  2783. while read-0 resource; do
  2784. eval "echo \" - \$HOST_${resource_type^^}STORE/\$service\$resource:\$resource:rw\""
  2785. done < <(echo "$metadata" | shyaml get-values-0 "${resource_type}-resources" 2>/dev/null)
  2786. done
  2787. while read-0 resource; do
  2788. if [[ "$resource" == /*:/*:* ]]; then
  2789. echo " - $resource"
  2790. elif [[ "$resource" == /*:/* ]]; then
  2791. echo " - $resource:rw"
  2792. elif [[ "$resource" == /*:* ]]; then
  2793. echo " - ${resource%%:*}:$resource"
  2794. elif [[ "$resource" =~ ^/[^:]+$ ]]; then
  2795. echo " - $resource:$resource:rw"
  2796. else
  2797. die "Invalid host-resource specified in 'metadata.yml'."
  2798. fi
  2799. done < <(printf "%s" "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null)
  2800. while read-0 resource; do
  2801. dest="$(charm.get_dir "$charm")/resources$resource"
  2802. if ! [ -e "$dest" ]; then
  2803. die "charm-resource: '$resource' does not exist (file: '$dest')."
  2804. fi
  2805. echo " - $dest:$resource:ro"
  2806. done < <(echo "$metadata" | shyaml get-values-0 "charm-resources" 2>/dev/null)
  2807. ) || return 1
  2808. if [ "$volumes" ]; then
  2809. mixins+=("volumes:"$'\n'"$volumes")
  2810. fi
  2811. type="$(printf "%s" "$metadata" | shyaml get-value type 2>/dev/null)" || true
  2812. if [ "$type" != "run-once" ]; then
  2813. mixins+=("restart: unless-stopped")
  2814. fi
  2815. docker_compose=$(printf "%s" "$metadata" | shyaml get-value -y "docker-compose" 2>/dev/null) || true
  2816. if [ "$docker_compose" ]; then
  2817. mixins+=("$docker_compose")
  2818. fi
  2819. if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" == "True" ]; then
  2820. subordinate=true
  2821. fi
  2822. fi
  2823. image=$(echo "$metadata" | shyaml get-value "docker-image" 2>/dev/null)
  2824. [ "$image" == "None" ] && image=""
  2825. if [ -n "$image" ]; then
  2826. if [ -n "$subordinate" ]; then
  2827. err "Subordinate charm can not have a ${WHITE}docker-image${NORMAL} value."
  2828. return 1
  2829. fi
  2830. mixins+=("image: $image")
  2831. elif [ "$has_build_dir" ]; then
  2832. if [ "$subordinate" ]; then
  2833. err "Subordinate charm can not have a 'build' sub directory."
  2834. return 1
  2835. fi
  2836. mixins+=("build: $(charm.get_dir "$charm")/build")
  2837. fi
  2838. limit=$(e "$metadata" | yq '.limit' 2>/dev/null) || return 1
  2839. [ "$limit" == "null" ] && limit=""
  2840. if [ -n "$limit" ]; then
  2841. if ! read-0-err E tmemory memory < <(e "$limit" | wyq ".memory | type, .memory") ||
  2842. [ "$E" != 0 ]; then
  2843. err "Unexpected error in ${DARKPINK}$charm${NORMAL}'s metadata when parsing ${WHITE}.limit${NORMAL}"
  2844. return 1
  2845. fi
  2846. case "$tmemory" in
  2847. '!!str'|'!!int')
  2848. docker_memory="$(e "$memory" | numfmt --from iec)" || {
  2849. err "Invalid format specified for .limit.memory: '$memory'."
  2850. return 1
  2851. }
  2852. ;;
  2853. '!!float')
  2854. err "Unexpected value in ${DARKPINK}$charm${NORMAL}'s metadata for ${WHITE}.limit.memory${NORMAL}."
  2855. echo " You need to specify a unit (like 'K', 'M', 'G' ...)." >&2
  2856. return 1
  2857. ;;
  2858. '!!null')
  2859. :
  2860. ;;
  2861. *)
  2862. err "Unexpected type '${tmemory#!!}' in ${DARKPINK}$charm${NORMAL}'s metadata" \
  2863. "for ${WHITE}.limit.memory${NORMAL}."
  2864. echo " You need to check ${DARKPINK}$charm${NORMAL}'s metadata " \
  2865. "for ${WHITE}.limit.memory${NORMAL} and provide a valid value" >&2
  2866. echo " Example values: '1.5G', '252M', ..." >&2
  2867. return 1
  2868. ;;
  2869. esac
  2870. if [ -n "$docker_memory" ]; then
  2871. if [[ "$docker_memory" -lt 6291456 ]]; then
  2872. err "Can't limit service to lower than 6M."
  2873. echo " Specified limit of $memory (=$docker_memory) is lower than docker's min limit of 6M (=6291456)." >&2
  2874. echo " The provided limit to memory is lower than minimum memory for a container." >&2
  2875. echo " Please remove memory limit in ${DARKPINK}$charm${NORMAL}'s metadata or raise it." >&2
  2876. return 1
  2877. fi
  2878. mixins+=(
  2879. "mem_limit: $docker_memory"
  2880. "memswap_limit: $docker_memory"
  2881. )
  2882. fi
  2883. fi
  2884. ## Final merging
  2885. mixin=$(merge_yaml_str "${mixins[@]}") || {
  2886. err "Failed to merge mixins from ${DARKPINK}${charm}${NORMAL} metadata."
  2887. return 1
  2888. }
  2889. echo "$mixin" | tee "$cache_file"
  2890. }
  2891. export -f _get_docker_compose_mixin_from_metadata_cached
  2892. get_docker_compose_mixin_from_metadata() {
  2893. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  2894. if [ -e "$cache_file" ]; then
  2895. #debug "$FUNCNAME: SESSION cache hit ($*)"
  2896. cat "$cache_file"
  2897. return 0
  2898. fi
  2899. charm=$(get_service_charm "$service") || return 1
  2900. metadata="$(charm.metadata "$charm" 2>/dev/null)" || return 1
  2901. has_build_dir=
  2902. [ -d "$(charm.get_dir "$charm")/build" ] && has_build_dir=true
  2903. mixin=$(_get_docker_compose_mixin_from_metadata_cached "$service" "$charm" "$metadata" "$has_build_dir") || return 1
  2904. echo "$mixin" | tee "$cache_file"
  2905. }
  2906. export -f get_docker_compose_mixin_from_metadata
  2907. _save() {
  2908. local name="$1"
  2909. cat - | tee -a "$docker_compose_dir/.data/$name"
  2910. }
  2911. export -f _save
  2912. get_default_project_name() {
  2913. if [ -n "$DEFAULT_PROJECT_NAME" ]; then
  2914. echo "$DEFAULT_PROJECT_NAME"
  2915. return 0
  2916. fi
  2917. local normalized_path compose_yml_location name
  2918. compose_yml_location="$(get_compose_yml_location)" || return 1
  2919. if [ -n "$compose_yml_location" ]; then
  2920. if normalized_path=$(readlink -f "$compose_yml_location"); then
  2921. name="${normalized_path%/*}" ## dirname
  2922. name="${name##*/}" ## basename
  2923. name="${name%%-deploy}" ## remove any '-deploy'
  2924. name="${name,,}" ## lowercase
  2925. e "$name"
  2926. return 0
  2927. fi
  2928. fi
  2929. echo "orphan"
  2930. return 0
  2931. }
  2932. export -f get_default_project_name
  2933. get_running_compose_containers() {
  2934. ## XXXvlab: docker bug: there will be a final newline anyway
  2935. docker ps --filter label="compose.service" --format='{{.ID}}'
  2936. }
  2937. export -f get_running_compose_containers
  2938. get_healthy_container_ip_for_service () {
  2939. local service="$1" port="$2" timeout=${3:-60}
  2940. local containers container container_network container_ip
  2941. containers="$(get_running_containers_for_service "$service")"
  2942. if [ -z "$containers" ]; then
  2943. err "No containers running for service $DARKYELLOW$service$NORMAL."
  2944. return 1
  2945. fi
  2946. ## XXXvlab: taking first container is probably not a good idea
  2947. container="$(echo "$containers" | head -n 1)"
  2948. ## XXXvlab: taking first ip is probably not a good idea
  2949. read-0 container_network container_ip < <(get_container_network_ip "$container")
  2950. if [ -z "$container_ip" ]; then
  2951. err "Can't get container's IP. You should check health of" \
  2952. "${DARKYELLOW}$service${NORMAL}'s container."
  2953. return 1
  2954. fi
  2955. wait_for_tcp_port "$container_network" "$container_ip:$port" "$timeout" || {
  2956. err "TCP port of ${DARKYELLOW}$service${NORMAL}'s container doesn't seem open"
  2957. echo " Please check that container is healthy. Here are last logs:" >&2
  2958. docker logs "$container" --tail=10 | prefix " ${GRAY}|${NORMAL} " >&2
  2959. return 1
  2960. }
  2961. info "Host/Port ${container_ip}:${port} checked ${GREEN}open${NORMAL}."
  2962. echo "$container_network:$container_ip"
  2963. }
  2964. export -f get_healthy_container_ip_for_service
  2965. switch_to_relation_service() {
  2966. local relation="$1"
  2967. ## XXXvlab: can't get real config here
  2968. if ! read-0 ts _ _ < <(get_service_relation "$SERVICE_NAME" "$relation"); then
  2969. err "Couldn't find relation ${DARKCYAN}$relation${NORMAL}."
  2970. return 1
  2971. fi
  2972. export SERVICE_NAME="$ts"
  2973. export SERVICE_DATASTORE="$DATASTORE/$SERVICE_NAME"
  2974. DOCKER_BASE_IMAGE=$(service_base_docker_image "$SERVICE_NAME")
  2975. export DOCKER_BASE_IMAGE
  2976. target_charm=$(get_service_charm "$ts") || return 1
  2977. target_charm_path=$(charm.get_dir "$target_charm") || return 1
  2978. cd "$target_charm_path"
  2979. }
  2980. export -f switch_to_relation_service
  2981. get_volumes_for_container() {
  2982. local container="$1"
  2983. docker inspect \
  2984. --format '{{range $mount := .Mounts}}{{$mount.Source}}{{"\x00"}}{{$mount.Destination}}{{"\x00"}}{{end}}' \
  2985. "$container"
  2986. }
  2987. export -f get_volumes_for_container
  2988. is_volume_used() {
  2989. local volume="$1" container_id src dst
  2990. while read -r container_id; do
  2991. while read-0 src dst; do
  2992. [[ "$src/" == "$volume"/* ]] && return 0
  2993. done < <(get_volumes_for_container "$container_id")
  2994. done < <(get_running_compose_containers)
  2995. return 1
  2996. }
  2997. export -f is_volume_used
  2998. clean_unused_docker_compose() {
  2999. for f in /var/lib/compose/docker-compose/*; do
  3000. [ -e "$f" ] || continue
  3001. is_volume_used "$f" && continue
  3002. debug "Cleaning unused docker-compose ${f##*/}"
  3003. rm -rf "$f" || return 1
  3004. done
  3005. return 0
  3006. }
  3007. export -f clean_unused_docker_compose
  3008. stdin_get_hash() {
  3009. local sha
  3010. sha=$(sha256sum) || return 1
  3011. sha=${sha:0:64}
  3012. echo "$sha"
  3013. }
  3014. export -f stdin_get_hash
  3015. file_get_hash() {
  3016. stdin_get_hash < "$1" || return 1
  3017. }
  3018. export -f file_get_hash
  3019. docker_compose_store() {
  3020. local file="$1" sha
  3021. sha=$(file_get_hash "$file") || return 1
  3022. project=$(get_default_project_name) || return 1
  3023. dst="/var/lib/compose/docker-compose/$sha/$project"
  3024. mkdir -p "$dst" || return 1
  3025. cat <<EOF > "$dst/.env" || return 1
  3026. DOCKER_COMPOSE_PATH=$dst
  3027. COMPOSE_HTTP_TIMEOUT=7200
  3028. EOF
  3029. cp "$file" "$dst/docker-compose.yml" || return 1
  3030. mkdir -p "$dst/bin" || return 1
  3031. cat <<EOF > "$dst/bin/dc" || return 1
  3032. #!/bin/bash
  3033. $(declare -f read-0)
  3034. docker_run_opts=()
  3035. while read-0 opt; do
  3036. if [[ "\$opt" == "!env:"* ]]; then
  3037. opt="\${opt##!env:}"
  3038. var="\${opt%%=*}"
  3039. value="\${opt#*=}"
  3040. export "\$var"="\$value"
  3041. else
  3042. docker_run_opts+=("\$opt")
  3043. fi
  3044. done < <(cat "$COMPOSE_LAUNCHER_OPTS")
  3045. docker_run_opts+=(
  3046. "-w" "$dst"
  3047. "--entrypoint" "/usr/local/bin/docker-compose"
  3048. )
  3049. [ -t 1 ] && {
  3050. docker_run_opts+=("-ti")
  3051. }
  3052. exec docker run --rm "\${docker_run_opts[@]}" "\${COMPOSE_DOCKER_IMAGE:-docker.0k.io/compose}" "\$@"
  3053. EOF
  3054. chmod +x "$dst/bin/dc" || return 1
  3055. printf "%s" "$sha"
  3056. }
  3057. export -f docker_compose_store
  3058. launch_docker_compose() {
  3059. local charm docker_compose_tmpdir docker_compose_dir
  3060. docker_compose_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX)
  3061. #debug "Creating temporary docker-compose directory in '$docker_compose_tmpdir'."
  3062. trap_add EXIT "rm -rf \"$docker_compose_tmpdir\""
  3063. ## docker-compose will name network from the parent dir name
  3064. project=$(get_default_project_name)
  3065. mkdir -p "$docker_compose_tmpdir/$project"
  3066. docker_compose_dir="$docker_compose_tmpdir/$project"
  3067. get_docker_compose $SERVICE_PACK > "$docker_compose_dir/docker-compose.yml" || return 1
  3068. if [ -e "$state_tmpdir/to-merge-in-docker-compose.yml" ]; then
  3069. # debug "Merging some config data in docker-compose.yml:"
  3070. # debug "$(cat $state_tmpdir/to-merge-in-docker-compose.yml)"
  3071. _config_merge "$docker_compose_dir/docker-compose.yml" "$state_tmpdir/to-merge-in-docker-compose.yml" || return 1
  3072. fi
  3073. if [ -z "$(echo $(cat "$docker_compose_dir/docker-compose.yml"))" ]; then
  3074. die "Generated 'docker-compose.yml' is unexpectedly empty."
  3075. fi
  3076. ## XXXvlab: could be more specific and only link the needed charms
  3077. ## XXXvlab: why do we need these links ? If this is for the build command, then it is not useful anymore.
  3078. # for charm in $(shyaml keys services < "$docker_compose_dir/docker-compose.yml"); do
  3079. # if charm.exists "$charm"; then
  3080. # ln -sf "$(charm.get_dir "$charm")" "$docker_compose_dir/$charm" || exit 1
  3081. # fi
  3082. # done
  3083. mkdir "$docker_compose_dir/.data"
  3084. if [ -z "$COMPOSE_DISABLE_DOCKER_COMPOSE_STORE" ]; then
  3085. sha=$(docker_compose_store "$docker_compose_dir/docker-compose.yml") || return 1
  3086. fi
  3087. {
  3088. {
  3089. {
  3090. if [ -z "$COMPOSE_DISABLE_DOCKER_COMPOSE_STORE" ]; then
  3091. cd "/var/lib/compose/docker-compose/$sha/$project" || return 1
  3092. else
  3093. cd "$docker_compose_dir" || return 1
  3094. fi
  3095. if [ -f ".env" ]; then
  3096. debug "${WHITE}.env$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:"
  3097. debug "$(cat ".env" | prefix " $GRAY|$NORMAL ")"
  3098. fi
  3099. debug "${WHITE}docker-compose.yml$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:"
  3100. debug "$(cat "docker-compose.yml" | prefix " $GRAY|$NORMAL ")"
  3101. debug "${WHITE}Launching$NORMAL: docker-compose $@"
  3102. if [ "$DRY_COMPOSE_RUN" ]; then
  3103. echo docker-compose "$@"
  3104. else
  3105. docker-compose "$@"
  3106. fi
  3107. echo "$?" > "$docker_compose_dir/.data/errlvl"
  3108. } | _save stdout
  3109. } 3>&1 1>&2 2>&3 | _save stderr
  3110. } 3>&1 1>&2 2>&3
  3111. if tail -n 1 "$docker_compose_dir/.data/stderr" | grep -E "Service .+ failed to build: Error getting container [0-9a-f]+ from driver devicemapper: (open|Error mounting) /dev/mapper/docker-.*: no such file or directory$" >/dev/null 2>&1; then
  3112. err "Detected bug https://github.com/docker/docker/issues/4036 ... "
  3113. err "Please re-launch your command, or switch from 'devicemapper' driver to 'overlayfs' or 'aufs'."
  3114. fi
  3115. docker_compose_errlvl="$(cat "$docker_compose_dir/.data/errlvl" 2>/dev/null)"
  3116. if [ -z "$docker_compose_errlvl" ]; then
  3117. err "Something went wrong before you could gather docker-compose errorlevel."
  3118. return 1
  3119. fi
  3120. return "$docker_compose_errlvl"
  3121. }
  3122. export -f launch_docker_compose
  3123. get_compose_yml_location() {
  3124. if ! [ -z ${COMPOSE_YML_FILE+x} ]; then ## if set, even if empty
  3125. echo "$COMPOSE_YML_FILE"
  3126. return 0
  3127. fi
  3128. parent=$(while ! [ -f "./compose.yml" ]; do
  3129. [ "$PWD" == "/" ] && exit 0
  3130. cd ..
  3131. done; echo "$PWD"
  3132. )
  3133. if [ "$parent" ]; then
  3134. echo "$parent/compose.yml"
  3135. return 0
  3136. fi
  3137. ## XXXvlab: do we need this additional environment variable,
  3138. ## COMPOSE_YML_FILE is not sufficient ?
  3139. if [ "$DEFAULT_COMPOSE_FILE" ]; then
  3140. if ! [ -e "$DEFAULT_COMPOSE_FILE" ]; then
  3141. warn "No 'compose.yml' was found in current or parent dirs," \
  3142. "and \$DEFAULT_COMPOSE_FILE points to an unexistent file." \
  3143. "(${DEFAULT_COMPOSE_FILE})"
  3144. return 0
  3145. fi
  3146. echo "$DEFAULT_COMPOSE_FILE"
  3147. return 0
  3148. fi
  3149. warn "No 'compose.yml' was found in current or parent dirs, and no \$DEFAULT_COMPOSE_FILE was set."
  3150. return 0
  3151. }
  3152. export -f get_compose_yml_location
  3153. get_compose_yml_content() {
  3154. local cache_file="$state_tmpdir/$FUNCNAME.cache"
  3155. if [ -e "$cache_file" ]; then
  3156. cat "$cache_file" &&
  3157. touch "$cache_file" || return 1
  3158. return 0
  3159. fi
  3160. if [ -z "$COMPOSE_YML_FILE" ]; then
  3161. COMPOSE_YML_FILE=$(get_compose_yml_location) || return 1
  3162. fi
  3163. if [ -e "$COMPOSE_YML_FILE" ]; then
  3164. debug "Found $WHITE$exname$NORMAL YAML file in '$COMPOSE_YML_FILE'."
  3165. COMPOSE_YML_CONTENT=$(cat "$COMPOSE_YML_FILE") || {
  3166. err "Could not read '$COMPOSE_YML_FILE'."
  3167. return 1
  3168. }
  3169. else
  3170. debug "No compose file found. Using an empty one."
  3171. COMPOSE_YML_CONTENT=""
  3172. fi
  3173. COMPOSE_YML_CONTENT=$(merge_yaml_str "$COMPOSE_YML_CONTENT" "${compose_contents[@]}") || return 1
  3174. output=$(echo "$COMPOSE_YML_CONTENT"| shyaml get-value 2>&1)
  3175. if [ "$?" != 0 ]; then
  3176. outputed_something=
  3177. while IFS='' read -r line1 && IFS='' read -r line2; do
  3178. [ "$outputed_something" ] || err "Invalid YAML in '$COMPOSE_YML_FILE':"
  3179. outputed_something=true
  3180. echo "$line1 $GRAY($line2)$NORMAL"
  3181. done < <(echo "$output" | grep ^yaml.scanner -A 100 |
  3182. sed -r 's/^ in "<stdin>", //g' | sed -r 's/^yaml.scanner.[a-zA-Z]+: //g') |
  3183. prefix " $GRAY|$NORMAL "
  3184. [ "$outputed_something" ] || {
  3185. err "Unexpected error while running 'shyaml get-value' on '$COMPOSE_YML_FILE':"
  3186. echo "$output" | prefix " $GRAY|$NORMAL "
  3187. }
  3188. return 1
  3189. fi
  3190. echo "$COMPOSE_YML_CONTENT" | tee "$cache_file" || return 1
  3191. }
  3192. export -f get_compose_yml_content
  3193. get_default_target_services() {
  3194. local services=("$@")
  3195. if [ -z "${services[*]}" ]; then
  3196. if [ "$DEFAULT_SERVICES" ]; then
  3197. debug "No service provided, using $WHITE\$DEFAULT_SERVICES$NORMAL variable." \
  3198. "Target services: $DARKYELLOW$DEFAULT_SERVICES$NORMAL"
  3199. services="$DEFAULT_SERVICES"
  3200. else
  3201. err "No service provided."
  3202. return 1
  3203. fi
  3204. fi
  3205. echo "${services[*]}"
  3206. }
  3207. export -f get_default_target_services
  3208. get_master_services() {
  3209. local loaded master_service service
  3210. declare -A loaded
  3211. for service in "$@"; do
  3212. master_service=$(get_top_master_service_for_service "$service") || return 1
  3213. if [ "${loaded[$master_service]}" ]; then
  3214. continue
  3215. fi
  3216. echo "$master_service"
  3217. loaded["$master_service"]=1
  3218. done | nspc
  3219. return "${PIPESTATUS[0]}"
  3220. }
  3221. export -f get_master_services
  3222. get_current_docker_container_id() {
  3223. local line
  3224. line=$(cat "/proc/self/cpuset") || return 1
  3225. [[ "$line" == *docker* ]] || return 1
  3226. echo "${line##*/}"
  3227. }
  3228. export -f get_current_docker_container_id
  3229. ## if we are in a docker compose, we might want to know what is the
  3230. ## real host path of some local paths.
  3231. get_host_path() {
  3232. local path="$1"
  3233. path=$(realpath "$path") || return 1
  3234. container_id=$(get_current_docker_container_id) || {
  3235. print "%s" "$path"
  3236. return 0
  3237. }
  3238. biggest_dst=
  3239. current_src=
  3240. while read-0 src dst; do
  3241. [[ "$path" == "$dst"* ]] || continue
  3242. if [[ "${#biggest_dst}" < "${#dst}" ]]; then
  3243. biggest_dst="$dst"
  3244. current_src="$src"
  3245. fi
  3246. done < <(get_volumes_for_container "$container_id")
  3247. if [ "$current_src" ]; then
  3248. printf "%s" "$current_src"
  3249. else
  3250. return 1
  3251. fi
  3252. }
  3253. export -f get_host_path
  3254. _setup_state_dir() {
  3255. export state_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX)
  3256. #debug "Creating temporary state directory in '$state_tmpdir'."
  3257. # trap_add EXIT "debug \"Removing temporary state directory in $state_tmpdir.\";\
  3258. # rm -rf \"$state_tmpdir\""
  3259. trap_add EXIT "rm -rf \"$state_tmpdir\""
  3260. }
  3261. get_docker_compose_help_msg() {
  3262. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  3263. docker_compose_help_msg
  3264. if [ -e "$cache_file" ]; then
  3265. cat "$cache_file" &&
  3266. touch "$cache_file" || return 1
  3267. return 0
  3268. fi
  3269. docker_compose_help_msg=$(docker-compose $action --help 2>/dev/null) || return 1
  3270. echo "$docker_compose_help_msg" |
  3271. tee "$cache_file" || return 1
  3272. }
  3273. get_docker_compose_usage() {
  3274. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  3275. docker_compose_help_msg
  3276. if [ -e "$cache_file" ]; then
  3277. cat "$cache_file" &&
  3278. touch "$cache_file" || return 1
  3279. return 0
  3280. fi
  3281. docker_compose_help_msg=$(get_docker_compose_help_msg $action) || return 1
  3282. echo "$docker_compose_help_msg" |
  3283. grep -m 1 "^Usage:" -A 10000 |
  3284. egrep -m 1 "^\$" -B 10000 |
  3285. nspc |
  3286. sed -r 's/^Usage: //g' |
  3287. tee "$cache_file" || return 1
  3288. }
  3289. get_docker_compose_opts_help() {
  3290. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  3291. docker_compose_help_msg
  3292. if [ -e "$cache_file" ]; then
  3293. cat "$cache_file" &&
  3294. touch "$cache_file" || return 1
  3295. return 0
  3296. fi
  3297. docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1
  3298. echo "$docker_compose_opts_help" |
  3299. grep '^Options:' -A 20000 |
  3300. tail -n +2 |
  3301. { cat ; echo; } |
  3302. egrep -m 1 "^\S*\$" -B 10000 |
  3303. head -n -1 |
  3304. tee "$cache_file" || return 1
  3305. }
  3306. get_docker_compose_commands_help() {
  3307. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  3308. docker_compose_help_msg
  3309. if [ -e "$cache_file" ]; then
  3310. cat "$cache_file" &&
  3311. touch "$cache_file" || return 1
  3312. return 0
  3313. fi
  3314. docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1
  3315. echo "$docker_compose_opts_help" |
  3316. grep '^Commands:' -A 20000 |
  3317. tail -n +2 |
  3318. { cat ; echo; } |
  3319. egrep -m 1 "^\S*\$" -B 10000 |
  3320. head -n -1 |
  3321. tee "$cache_file" || return 1
  3322. }
  3323. get_docker_compose_opts_list() {
  3324. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  3325. docker_compose_help_msg
  3326. if [ -e "$cache_file" ]; then
  3327. cat "$cache_file" &&
  3328. touch "$cache_file" || return 1
  3329. return 0
  3330. fi
  3331. docker_compose_opts_help=$(get_docker_compose_opts_help $action) || return 1
  3332. echo "$docker_compose_opts_help" |
  3333. egrep "^\s+-" |
  3334. sed -r 's/\s+((((-[a-zA-Z]|--[a-zA-Z0-9-]+)( [A-Z=]+|=[^ ]+)?)(, )?)+)\s+.*$/\1/g' |
  3335. tee "$cache_file" || return 1
  3336. }
  3337. options_parser() {
  3338. sed -r 's/^(\s+(((-[a-zA-Z]|--[a-zA-Z0-9-]+)([ =]([a-zA-Z_=\"\[]|\])+)?(, | )?)+)\s+)[^ ].*$/\x0\2\x0\0/g'
  3339. printf "\0"
  3340. }
  3341. remove_options_in_option_help_msg() {
  3342. {
  3343. read-0 null
  3344. if [ "$null" ]; then
  3345. err "options parsing error, should start with an option line."
  3346. return 1
  3347. fi
  3348. while read-0 opt full_txt;do
  3349. multi_opts="$(printf "%s " $opt | multi_opts_filter)"
  3350. single_opts="$(printf "%s " $opt | single_opts_filter)"
  3351. for to_remove in "$@"; do
  3352. str_matches "$to_remove" $multi_opts $single_opts && {
  3353. continue 2
  3354. }
  3355. done
  3356. echo -n "$full_txt"
  3357. done
  3358. } < <(options_parser)
  3359. }
  3360. _MULTIOPTION_REGEX='^((-[a-zA-Z]|--[a-zA-Z0-9-]+)(, )?)+'
  3361. _MULTIOPTION_REGEX_LINE_FILTER=$_MULTIOPTION_REGEX'(\s|=)'
  3362. multi_opts_filter() {
  3363. egrep "$_MULTIOPTION_REGEX_LINE_FILTER" |
  3364. sed -r "s/^($_MULTIOPTION_REGEX)(\s|=).*$/\1/g" |
  3365. tr ',' "\n" | nspc
  3366. }
  3367. single_opts_filter() {
  3368. egrep -v "$_MULTIOPTION_REGEX_LINE_FILTER" |
  3369. tr ',' "\n" | nspc
  3370. }
  3371. get_docker_compose_multi_opts_list() {
  3372. local action="$1" opts_list
  3373. opts_list=$(get_docker_compose_opts_list "$action") || return 1
  3374. echo "$opts_list" | multi_opts_filter
  3375. }
  3376. get_docker_compose_single_opts_list() {
  3377. local action="$1" opts_list
  3378. opts_list=$(get_docker_compose_opts_list "$action") || return 1
  3379. echo "$opts_list" | single_opts_filter
  3380. }
  3381. display_commands_help() {
  3382. local charm_actions
  3383. echo
  3384. echo "${WHITE}Commands${NORMAL} (thanks to docker-compose):"
  3385. get_docker_compose_commands_help | sed -r "s/ ([a-z]+)(\s+)/ ${DARKCYAN}\1${NORMAL}\2/g"
  3386. charm_actions_help=$(get_docker_charm_action_help) || return 1
  3387. if [ "$charm_actions_help" ]; then
  3388. echo
  3389. echo "${WHITE}Charm actions${NORMAL}:"
  3390. printf "%s\n" "$charm_actions_help" | \
  3391. sed -r "s/^ ([a-z0-9-]+)(\s+)([a-z0-9-]+)(\s+)/ ${DARKCYAN}\1${NORMAL}\2${DARKYELLOW}\3${NORMAL}\4/g"
  3392. fi
  3393. }
  3394. get_docker_charm_action() {
  3395. local services service charm relation_name target_service relation_config \
  3396. target_charm
  3397. services=($(get_compose_yml_content | yq -r 'keys().[]' 2>/dev/null)) || return 1
  3398. NO_CONSTRAINT_CHECK=1 get_all_relations "${services[@]}" >/dev/null || return 1
  3399. services=($(get_all_services)) || return 1
  3400. for service in "${services[@]}"; do
  3401. printf "%s:\n" "$service"
  3402. charm=$(get_service_charm "$service") || return 1
  3403. for action in $(charm.ls_direct_actions "$charm"); do
  3404. printf " %s:\n" "$action"
  3405. printf " type: %s\n" "direct"
  3406. done
  3407. while read-0 relation_name target_service _relation_config _tech_dep; do
  3408. target_charm=$(get_service_charm "$target_service") || return 1
  3409. for action in $(charm.ls_relation_actions "$target_charm" "$relation_name"); do
  3410. printf " %s:\n" "$action"
  3411. printf " type: %s\n" "indirect"
  3412. printf " inherited: %s\n" "$target_charm"
  3413. done
  3414. done < <(get_service_relations "$service")
  3415. done
  3416. }
  3417. export -f get_docker_charm_action
  3418. get_docker_charm_action_help() {
  3419. local services service charm relation_name target_service relation_config \
  3420. target_charm
  3421. services=($(get_compose_yml_content | shyaml keys 2>/dev/null))
  3422. NO_CONSTRAINT_CHECK=1 get_all_relations "${services[@]}" >/dev/null || return 1
  3423. for service in "${services[@]}"; do
  3424. out=$(
  3425. charm=$(get_service_charm "$service") || return 1
  3426. for action in $(charm.ls_direct_actions "$charm"); do
  3427. printf " %-28s %s\n" "$action $service" "Direct action from ${DARKPINK}$charm${NORMAL}"
  3428. done
  3429. while read-0 relation_name target_service _relation_config _tech_dep; do
  3430. target_charm=$(get_service_charm "$target_service") || return 1
  3431. for action in $(charm.ls_relation_actions "$target_charm" "$relation_name"); do
  3432. printf " %-28s %s\n" "$action $service" "Indirect action from ${DARKPINK}$target_charm${NORMAL}"
  3433. done
  3434. done < <(get_service_relations "$service")
  3435. )
  3436. if [ "$out" ]; then
  3437. echo " for ${DARKYELLOW}$service${NORMAL}:"
  3438. printf "%s\n" "$out"
  3439. fi
  3440. done
  3441. }
  3442. display_help() {
  3443. print_help
  3444. echo "${WHITE}Usage${NORMAL}:"
  3445. echo " $usage"
  3446. echo "${WHITE}Options${NORMAL}:"
  3447. echo " -h, --help Print this message and quit"
  3448. echo " (ignoring any other options)"
  3449. echo " -V, --version Print current version and quit"
  3450. echo " (ignoring any other options)"
  3451. echo " --dirs Display data dirs and quit"
  3452. echo " (ignoring any other options)"
  3453. echo " --get-project-name Display project name and quit"
  3454. echo " (ignoring any other options)"
  3455. echo " --get-available-actions Display all available actions and quit"
  3456. echo " (ignoring any other options)"
  3457. echo " -v, --verbose Be more verbose"
  3458. echo " -q, --quiet Be quiet"
  3459. echo " -d, --debug Print full debugging information (sets also verbose)"
  3460. echo " --dry-compose-run If docker-compose will be run, only print out what"
  3461. echo " command line will be used."
  3462. echo " --no-relations Do not run any relation script"
  3463. echo " --no-hooks Do not run any hook script"
  3464. echo " --no-init Do not run any init script"
  3465. echo " --no-post-deploy Do not run any post-deploy script"
  3466. echo " --no-pre-deploy Do not run any pre-deploy script"
  3467. echo " --without-relation RELATION "
  3468. echo " Do not run given relation"
  3469. echo " -R, --rebuild-relations-to-service SERVICE"
  3470. echo " Will rebuild all relations to given service"
  3471. echo " --add-compose-content, -Y YAML"
  3472. echo " Will merge some direct YAML with the current compose"
  3473. echo " -c, --color Force color mode (default is to detect if in tty mode)"
  3474. get_docker_compose_opts_help | remove_options_in_option_help_msg --version --help --verbose |
  3475. filter_docker_compose_help_message
  3476. display_commands_help
  3477. }
  3478. _graph_service() {
  3479. local service="$1" base="$1"
  3480. charm=$(get_service_charm "$service") || return 1
  3481. metadata=$(charm.metadata "$charm") || return 1
  3482. subordinate=$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)
  3483. if [ "$subordinate" == "True" ]; then
  3484. requires="$(echo "$metadata" | shyaml get-value "requires" 2>/dev/null)"
  3485. master_charm=
  3486. while read-0 relation_name relation; do
  3487. [ "$(echo "$relation" | shyaml get-value "scope" 2>/dev/null)" == "container" ] || continue
  3488. interface="$(echo "$relation" | shyaml get-value "interface" 2>/dev/null)"
  3489. if [ -z "$interface" ]; then
  3490. err "No ${WHITE}$interface${NORMAL} set for relation $relation_name."
  3491. return 1
  3492. fi
  3493. ## Action provided by relation ?
  3494. target_service=
  3495. while read-0 relation_name candidate_target_service _relation_config _tech_dep; do
  3496. [ "$interface" == "$relation_name" ] && {
  3497. target_service="$candidate_target_service"
  3498. break
  3499. }
  3500. done < <(get_service_relations "$service")
  3501. if [ -z "$target_service" ]; then
  3502. err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \
  3503. "${DARKYELLOW}$service$NORMAL compose definition."
  3504. return 1
  3505. fi
  3506. master_service="$target_service"
  3507. master_charm=$(get_service_charm "$target_service") || return 1
  3508. break
  3509. done < <(echo "$requires" | shyaml key-values-0 2>/dev/null)
  3510. fi
  3511. _graph_node_service "$service" "$base" "$charm"
  3512. _graph_edge_service "$service" "$subordinate" "$master_service"
  3513. }
  3514. _graph_node_service() {
  3515. local service="$1" base="$2" charm="$3"
  3516. cat <<EOF
  3517. "$(_graph_node_service_label ${service})" [
  3518. style = "filled, $([ "$subordinate" == "True" ] && echo "dashed" || echo "bold")"
  3519. penwidth = $([ "$subordinate" == "True" ] && echo "3" || echo "5")
  3520. color = $([ "$base" ] && echo "blue" || echo "black")
  3521. fillcolor = "white"
  3522. fontname = "Courier New"
  3523. shape = "Mrecord"
  3524. label =<$(_graph_node_service_content "$service")>
  3525. ];
  3526. EOF
  3527. }
  3528. _graph_edge_service() {
  3529. local service="$1" subordinate="$2" master_service="$3"
  3530. while read-0 relation_name target_service relation_config tech_dep; do
  3531. cat <<EOF
  3532. "$(_graph_node_service_label ${service})" -> "$(_graph_node_service_label ${target_service})" [
  3533. penwidth = $([ "$master_service" == "$target_service" ] && echo 3 || echo 2)
  3534. fontsize = 16
  3535. fontcolor = "black"
  3536. style = $([ "$master_service" == "$target_service" ] && echo dashed || echo "\"\"")
  3537. weight = $([ "$master_service" == "$target_service" ] && echo 2.0 || echo 1.0)
  3538. dir = $([ "$master_service" == "$target_service" ] && echo none || echo both)
  3539. arrowtail = odot
  3540. # arrowhead = dotlicurve
  3541. taillabel = "$relation_name" ];
  3542. EOF
  3543. done < <(get_service_relations "$service") || return 1
  3544. }
  3545. _graph_node_service_label() {
  3546. local service="$1"
  3547. echo "service_$service"
  3548. }
  3549. _graph_node_service_content() {
  3550. local service="$1"
  3551. charm=$(get_service_charm "$service") || return 1
  3552. cat <<EOF
  3553. <table border="0" cellborder="0" cellpadding="3" bgcolor="white">
  3554. <tr>
  3555. <td bgcolor="black" align="center" colspan="2">
  3556. <font color="white">$service</font>
  3557. </td>
  3558. </tr>
  3559. $(if [ "$charm" != "$service" ]; then
  3560. cat <<EOF2
  3561. <tr>
  3562. <td align="left" port="r0">charm: $charm</td>
  3563. </tr>
  3564. EOF2
  3565. fi)
  3566. </table>
  3567. EOF
  3568. }
  3569. cla_contains () {
  3570. local e
  3571. for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done
  3572. return 1
  3573. }
  3574. filter_docker_compose_help_message() {
  3575. cat - |
  3576. sed -r "s/docker-compose run/${DARKWHITE}compose${NORMAL} ${DARKCYAN}$action${NORMAL}/g;
  3577. s/docker-compose.yml/compose.yml/g;
  3578. s/SERVICES?/${DARKYELLOW}\0${NORMAL}/g;
  3579. s/^(\s+)\\$/\1${WHITE}\$${NORMAL}/g;
  3580. s/^(\s+)run/\1${DARKCYAN}$action${NORMAL}/g;
  3581. s/docker-compose/${DARKWHITE}compose${NORMAL}/g"
  3582. }
  3583. graph() {
  3584. local services=("$@")
  3585. declare -A entries
  3586. cat <<EOF
  3587. digraph g {
  3588. graph [
  3589. fontsize=30
  3590. labelloc="t"
  3591. label=""
  3592. splines=true
  3593. overlap=false
  3594. #rankdir = "LR"
  3595. ];
  3596. ratio = auto;
  3597. EOF
  3598. for target_service in "$@"; do
  3599. services=$(get_ordered_service_dependencies "$target_service") || return 1
  3600. for service in $services; do
  3601. [ "${entries[$service]}" ] && continue || entries[$service]=1
  3602. if cla_contains "$service" "${services[@]}"; then
  3603. base=true
  3604. else
  3605. base=
  3606. fi
  3607. _graph_service "$service" "$base"
  3608. done
  3609. done
  3610. echo "}"
  3611. }
  3612. cached_wget() {
  3613. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(p0 "$@" | md5_compat)" \
  3614. url="$1"
  3615. if [ -e "$cache_file" ]; then
  3616. cat "$cache_file"
  3617. touch "$cache_file"
  3618. return 0
  3619. fi
  3620. wget -O- "${url}" |
  3621. tee "$cache_file"
  3622. if [ "${PIPESTATUS[0]}" != 0 ]; then
  3623. rm "$cache_file"
  3624. die "Unable to fetch '$url'."
  3625. return 1
  3626. fi
  3627. }
  3628. export -f cached_wget
  3629. [ "$SOURCED" ] && return 0
  3630. trap_add "EXIT" clean_cache
  3631. if [ -z "$DISABLE_SYSTEM_CONFIG_FILE" ]; then
  3632. if [ -r /etc/default/charm ]; then
  3633. . /etc/default/charm
  3634. fi
  3635. if [ -r "/etc/default/$exname" ]; then
  3636. . "/etc/default/$exname"
  3637. fi
  3638. ## XXXvlab: should provide YML config opportunities in possible parent dirs ?
  3639. ## userdir ? and global /etc/compose.yml ?
  3640. for cfgfile in /etc/compose.conf /etc/compose.local.conf \
  3641. /etc/default/compose /etc/compose/local.conf; do
  3642. [ -e "$cfgfile" ] || continue
  3643. . "$cfgfile" || die "Loading config file '$cfgfile' failed."
  3644. done
  3645. fi
  3646. _setup_state_dir
  3647. mkdir -p "$CACHEDIR" || exit 1
  3648. log () { cat; }
  3649. export -f log
  3650. ##
  3651. ## Argument parsing
  3652. ##
  3653. wrap_opts=()
  3654. services=()
  3655. remainder_args=()
  3656. compose_opts=()
  3657. compose_contents=()
  3658. action_opts=()
  3659. services_args=()
  3660. pos_arg_ct=0
  3661. no_hooks=
  3662. no_init=
  3663. action=
  3664. stage="main" ## switches from 'main', to 'action', 'remainder'
  3665. is_docker_compose_action=
  3666. rebuild_relations_to_service=()
  3667. color=
  3668. declare -A without_relations
  3669. DC_MATCH_MULTI=$(get_docker_compose_multi_opts_list) &&
  3670. DC_MATCH_SINGLE=$(get_docker_compose_single_opts_list) || exit 1
  3671. while read-0 arg; do
  3672. case "$stage" in
  3673. "main")
  3674. case "$arg" in
  3675. --help|-h)
  3676. no_init=true ; no_hooks=true ; no_relations=true
  3677. display_help
  3678. exit 0
  3679. ;;
  3680. --verbose|-v)
  3681. export VERBOSE=true
  3682. compose_opts+=("--verbose")
  3683. ;;
  3684. --quiet|-q)
  3685. export QUIET=true
  3686. export wrap_opts+=("-q")
  3687. log () { cat >&2; }
  3688. export -f log
  3689. ;;
  3690. --version|-V)
  3691. print_version
  3692. docker-compose --version
  3693. docker --version
  3694. exit 0
  3695. ;;
  3696. -f|--file)
  3697. read-0 value
  3698. [ -e "$value" ] || die "File $value doesn't exists"
  3699. export COMPOSE_YML_FILE="$value"
  3700. shift
  3701. ;;
  3702. -p|--project-name)
  3703. read-0 value
  3704. export DEFAULT_PROJECT_NAME="$value"
  3705. compose_opts+=("--project-name $value")
  3706. shift
  3707. ;;
  3708. --color|-c)
  3709. if [ "$color" == "0" ]; then
  3710. err "Conflicting option --color with previous --no-ansi."
  3711. exit 1
  3712. fi
  3713. color=1
  3714. ansi_color yes
  3715. ;;
  3716. --no-ansi)
  3717. if [ "$color" == "1" ]; then
  3718. err "Conflicting option --no-ansi with previous --color."
  3719. exit 1
  3720. fi
  3721. color=0
  3722. ansi_color no
  3723. compose_opts+=("--no-ansi")
  3724. ;;
  3725. --no-relations)
  3726. export no_relations=true
  3727. ;;
  3728. --without-relation)
  3729. read-0 value
  3730. without_relations["$value"]=1
  3731. shift
  3732. ;;
  3733. --no-hooks)
  3734. export no_hooks=true
  3735. ;;
  3736. --no-init)
  3737. export no_init=true
  3738. ;;
  3739. --no-post-deploy)
  3740. export no_post_deploy=true
  3741. ;;
  3742. --no-pre-deploy)
  3743. export no_pre_deploy=true
  3744. ;;
  3745. --rebuild-relations-to-service|-R)
  3746. read-0 value
  3747. rebuild_relations_to_service+=("$value")
  3748. shift
  3749. ;;
  3750. --debug|-d)
  3751. export DEBUG=true
  3752. export VERBOSE=true
  3753. #compose_opts+=("--verbose" "--log-level" "DEBUG")
  3754. ;;
  3755. --add-compose-content|-Y)
  3756. read-0 value
  3757. compose_contents+=("$value")
  3758. shift
  3759. ;;
  3760. --dirs)
  3761. echo "CACHEDIR: $CACHEDIR"
  3762. echo "VARDIR: $VARDIR"
  3763. exit 0
  3764. ;;
  3765. --get-project-name)
  3766. project=$(get_default_project_name) || exit 1
  3767. echo "$project"
  3768. exit 0
  3769. ;;
  3770. --get-available-actions)
  3771. get_docker_charm_action
  3772. exit $?
  3773. ;;
  3774. --dry-compose-run)
  3775. export DRY_COMPOSE_RUN=true
  3776. ;;
  3777. --*|-*)
  3778. if str_pattern_matches "$arg" $DC_MATCH_MULTI; then
  3779. read-0 value
  3780. compose_opts+=("$arg" "$value")
  3781. shift;
  3782. elif str_pattern_matches "$arg" $DC_MATCH_SINGLE; then
  3783. compose_opts+=("$arg")
  3784. else
  3785. err "Unknown option '$arg'. Please check help:"
  3786. display_help >&2
  3787. exit 1
  3788. fi
  3789. ;;
  3790. *)
  3791. action="$arg"
  3792. stage="action"
  3793. if DC_USAGE=$(get_docker_compose_usage "$action"); then
  3794. is_docker_compose_action=true
  3795. DC_MATCH_MULTI=$(get_docker_compose_multi_opts_list "$action") &&
  3796. DC_MATCH_SINGLE="$(get_docker_compose_single_opts_list "$action")"
  3797. if [ "$DC_MATCH_MULTI" ]; then
  3798. DC_MATCH_SINGLE="$DC_MATCH_SINGLE $(echo "$DC_MATCH_MULTI" | sed -r 's/( |$)/=\* /g')"
  3799. fi
  3800. pos_args=($(echo "$DC_USAGE" | sed -r 's/\[-[^]]+\] ?//g;s/\[options\] ?//g'))
  3801. pos_args=("${pos_args[@]:1}")
  3802. # echo "USAGE: $DC_USAGE"
  3803. # echo "pos_args: ${pos_args[@]}"
  3804. # echo "MULTI: $DC_MATCH_MULTI"
  3805. # echo "SINGLE: $DC_MATCH_SINGLE"
  3806. # exit 1
  3807. else
  3808. stage="remainder"
  3809. fi
  3810. ;;
  3811. esac
  3812. ;;
  3813. "action") ## Only for docker-compose actions
  3814. case "$arg" in
  3815. --help|-h)
  3816. no_init=true ; no_hooks=true ; no_relations=true
  3817. action_opts+=("$arg")
  3818. ;;
  3819. --*|-*)
  3820. if [ "$is_docker_compose_action" ]; then
  3821. if str_pattern_matches "$arg" $DC_MATCH_MULTI; then
  3822. read-0 value
  3823. action_opts+=("$arg" "$value")
  3824. shift
  3825. elif str_pattern_matches "$arg" $DC_MATCH_SINGLE; then
  3826. action_opts+=("$arg")
  3827. else
  3828. err "Unknown option '$arg'. Please check '${DARKCYAN}$action${NORMAL}' help:"
  3829. docker-compose "$action" --help |
  3830. filter_docker_compose_help_message >&2
  3831. exit 1
  3832. fi
  3833. fi
  3834. ;;
  3835. *)
  3836. # echo "LOOP $1 : pos_arg: $pos_arg_ct // ${pos_args[$pos_arg_ct]}"
  3837. if [[ "${pos_args[$pos_arg_ct]}" == "[SERVICE...]" ]]; then
  3838. services_args+=("$arg")
  3839. elif [[ "${pos_args[$pos_arg_ct]}" == "SERVICE" ]]; then
  3840. services_args=("$arg") || exit 1
  3841. stage="remainder"
  3842. else
  3843. action_posargs+=("$arg")
  3844. ((pos_arg_ct++))
  3845. fi
  3846. ;;
  3847. esac
  3848. ;;
  3849. "remainder")
  3850. remainder_args+=("$arg")
  3851. while read-0 arg; do
  3852. remainder_args+=("$arg")
  3853. done
  3854. break 3
  3855. ;;
  3856. esac
  3857. shift
  3858. done < <(cla.normalize "$@")
  3859. export compose_contents
  3860. [ "${services_args[*]}" ] && debug " ${DARKWHITE}Services:$NORMAL ${DARKYELLOW}${services_args[*]}$NORMAL"
  3861. [ "${compose_opts[*]}" ] && debug " ${DARKWHITE}Main docker-compose opts:$NORMAL ${compose_opts[*]}"
  3862. [ "${action_posargs[*]}" ] && debug " ${DARKWHITE}Main docker-compose pos args:$NORMAL ${action_posargs[*]}"
  3863. [ "${action_opts[*]}" ] && debug " ${DARKWHITE}Action $DARKCYAN$action$NORMAL with opts:$NORMAL ${action_opts[*]}"
  3864. [ "${remainder_args[*]}" ] && debug " ${DARKWHITE}Remainder args:$NORMAL ${remainder_args[*]}"
  3865. aexport remainder_args
  3866. ##
  3867. ## Actual code
  3868. ##
  3869. COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1
  3870. COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1
  3871. export COMPOSE_YML_FILE COMPOSE_YML_CONTENT
  3872. charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct."
  3873. ##
  3874. ## Get services in command line.
  3875. ##
  3876. if [ -z "$is_docker_compose_action" -a "$action" ]; then
  3877. action_service=${remainder_args[0]}
  3878. if [ -z "$action_service" ]; then
  3879. err "No such command or action: ${DARKCYAN}$action${NORMAL}"
  3880. display_commands_help
  3881. exit 1
  3882. fi
  3883. ## Required by has_service_action
  3884. array_read-0 services_args < <(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys-0 2>/dev/null)
  3885. NO_CONSTRAINT_CHECK=1 get_all_relations "${services_args[@]}" >/dev/null || exit 1
  3886. remainder_args=("${remainder_args[@]:1}")
  3887. if has_service_action "$action_service" "$action" >/dev/null; then
  3888. is_service_action=true
  3889. services_args=("$action_service")
  3890. {
  3891. read-0 action_type
  3892. case "$action_type" in
  3893. "relation")
  3894. read-0 _ target_service _target_charm relation_name _ action_script_path
  3895. debug "Found action $DARKYELLOW${action_service}$NORMAL/$DARKBLUE$relation_name$NORMAL/$DARKCYAN$action$NORMAL (in $DARKYELLOW$target_service$NORMAL)"
  3896. services_args+=("$target_service")
  3897. ;;
  3898. "direct")
  3899. read-0 _ action_script_path
  3900. debug "Found action $DARKYELLOW${action_service}$NORMAL.$DARKCYAN$action$NORMAL"
  3901. ;;
  3902. esac
  3903. } < <(has_service_action "$action_service" "$action")
  3904. get_all_relations "${services_args[@]}" >/dev/null || {
  3905. echo " Hint: if this is unexpected, you can try to delete caches, and re-run the command." >&2
  3906. exit 1
  3907. }
  3908. ## Divert logging to stdout to stderr
  3909. log () { cat >&2; }
  3910. export -f log
  3911. else
  3912. die "Unknown action '${DARKCYAN}$action$NORMAL': It doesn't match any docker-compose commands nor inner charm actions."
  3913. fi
  3914. else
  3915. case "$action" in
  3916. ps|up)
  3917. if [ "${#services_args[@]}" == 0 ]; then
  3918. array_read-0 services_args < <(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys-0 2>/dev/null)
  3919. fi
  3920. ;;
  3921. config)
  3922. services_args=("${action_posargs[@]}")
  3923. ;;
  3924. esac
  3925. fi
  3926. export COMPOSE_ACTION="$action"
  3927. NO_CONSTRAINT_CHECK=True
  3928. case "$action" in
  3929. up)
  3930. NO_CONSTRAINT_CHECK=
  3931. ;;
  3932. esac
  3933. get_all_relations "${services_args[@]}" >/dev/null || exit 1
  3934. if [ "$is_docker_compose_action" -a "${#services_args[@]}" -gt 0 ]; then
  3935. services=($(get_master_services "${services_args[@]}")) || exit 1
  3936. if [ "$action" == "up" ]; then
  3937. declare -A seen
  3938. for service in $(get_ordered_service_dependencies "${services_args[@]}"); do
  3939. mservice=$(get_master_service_for_service "$service") || exit 1
  3940. [ "${seen[$mservice]}" ] && continue
  3941. type="$(get_service_type "$mservice")" || exit 1
  3942. ## remove run-once
  3943. [ "$type" == "run-once" ] && continue
  3944. [ "$type" == "stub" ] && continue
  3945. seen[$mservice]=1
  3946. action_posargs+=("$mservice")
  3947. done
  3948. else
  3949. action_posargs+=("${services[@]}")
  3950. fi
  3951. ## Get rid of subordinates
  3952. action_posargs=($(get_master_services "${action_posargs[@]}")) || exit 1
  3953. fi
  3954. get_docker_compose "${services_args[@]}" >/dev/null || { ## precalculate variable \$_current_docker_compose
  3955. err "Fails to compile base 'docker-compose.yml'"
  3956. exit 1
  3957. }
  3958. ##
  3959. ## Pre-action
  3960. ##
  3961. full_init=
  3962. case "$action" in
  3963. up|run)
  3964. full_init=true
  3965. post_hook=true
  3966. ;;
  3967. ""|down|restart|logs|config|ps)
  3968. full_init=
  3969. ;;
  3970. *)
  3971. if [ "$is_service_action" ]; then
  3972. full_init=true
  3973. keywords=($(egrep "^#*\s*compose:" "$action_script_path" | cut -f 2- -d:))
  3974. for keyword in "${keywords[@]}"; do
  3975. case "$keyword" in
  3976. no-hooks)
  3977. full_init=
  3978. ;;
  3979. hooks)
  3980. full_init=true
  3981. ;;
  3982. esac
  3983. done
  3984. fi
  3985. ;;
  3986. esac
  3987. if [ -n "$full_init" ]; then
  3988. ## init in order
  3989. if [[ -z "$no_init" && -z "$no_hooks" ]]; then
  3990. Section setup host resources
  3991. setup_host_resources "${services_args[@]}" || exit 1
  3992. Section initialisation
  3993. run_service_hook init "${services_args[@]}" || exit 1
  3994. fi
  3995. ## Get relations
  3996. if [[ -z "$no_relations" && -z "$no_hooks" ]]; then
  3997. if [ "${#rebuild_relations_to_service[@]}" != 0 ]; then
  3998. rebuild_relations_to_service=$(get_master_services "${rebuild_relations_to_service[@]}") || return 1
  3999. rebuild_relations_to_service=($rebuild_relations_to_service)
  4000. project=$(get_default_project_name) || return 1
  4001. for service in "${rebuild_relations_to_service[@]}"; do
  4002. for dir in "$VARDIR/relations/$project/"*"-${service}/"*; do
  4003. [ -d "$dir" ] && {
  4004. debug rm -rf "$dir"
  4005. rm -rf "$dir"
  4006. }
  4007. done
  4008. done
  4009. fi
  4010. run_service_relations "${services_args[@]}" || exit 1
  4011. fi
  4012. if [[ -z "$no_pre_deploy" && -z "$no_hooks" ]]; then
  4013. run_service_hook pre_deploy "${services_args[@]}" || exit 1
  4014. fi
  4015. fi | log
  4016. if [ "${PIPESTATUS[0]}" != 0 ]; then
  4017. exit 1
  4018. fi
  4019. if [ "$action" == "run" ] && [ "${#services_args}" != 0 ]; then
  4020. charm=$(get_service_charm "${services_args[0]}") || exit 1
  4021. metadata=$(charm.metadata "$charm") || exit 1
  4022. SERVICE_TYPE="$(printf "%s" "$metadata" | shyaml get-value type 2>/dev/null)" || true
  4023. if [ "$SERVICE_TYPE" == "run-once" ]; then
  4024. run_service_hook dc-pre-run "${services_args[@]}" || exit 1
  4025. fi
  4026. fi
  4027. export SERVICE_PACK="${services_args[*]}"
  4028. ##
  4029. ## Docker-compose
  4030. ##
  4031. errlvl="0"
  4032. case "$action" in
  4033. up|start|stop|build|run)
  4034. ## force daemon mode for up
  4035. if [[ "$action" == "up" ]]; then
  4036. if ! array_member action_opts -d; then
  4037. action_opts+=("-d")
  4038. fi
  4039. if ! array_member action_opts --remove-orphans; then
  4040. action_opts+=("--remove-orphans")
  4041. fi
  4042. fi
  4043. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  4044. ;;
  4045. logs)
  4046. if ! array_member action_opts --tail; then ## force daemon mode for up
  4047. action_opts+=("--tail" "10")
  4048. fi
  4049. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  4050. ;;
  4051. "")
  4052. launch_docker_compose "${compose_opts[@]}"
  4053. ;;
  4054. graph)
  4055. graph $SERVICE_PACK
  4056. ;;
  4057. config)
  4058. ## removing the services
  4059. services=($(get_master_services "${action_posargs[@]}")) || exit 1
  4060. ## forcing docker-compose config to output the config file to stdout and not stderr
  4061. out=$(launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" 2>&1) || {
  4062. echo "$out"
  4063. exit 1
  4064. }
  4065. echo "$out"
  4066. warn "Runtime configuration modification (from relations) are not included here."
  4067. ;;
  4068. down)
  4069. if ! array_member action_opts --remove-orphans; then ## force daemon mode for up
  4070. debug "Adding a default argument of '--remove-orphans'"
  4071. action_opts+=("--remove-orphans")
  4072. fi
  4073. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}"
  4074. ;;
  4075. *)
  4076. if [ "$is_service_action" ]; then
  4077. run_service_action "$action_service" "$action" "${remainder_args[@]}"
  4078. errlvl="$?"
  4079. errlvl "$errlvl"
  4080. else
  4081. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  4082. fi
  4083. ;;
  4084. esac || exit 1
  4085. if [ "$post_hook" -a "${#services_args[@]}" != 0 -a -z "$no_hooks" -a -z "$no_post_deploy" ]; then
  4086. run_service_hook post_deploy "${services_args[@]}" || exit 1
  4087. fi
  4088. if [ "$action" == "run" -a "${#services_args}" != 0 ]; then
  4089. if [ "$SERVICE_TYPE" == "run-once" ]; then
  4090. run_service_hook dc-post-run "${services_args[@]}" || exit 1
  4091. fi
  4092. fi
  4093. clean_unused_docker_compose || exit 1
  4094. exit "$errlvl"