You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

5185 lines
174 KiB

  1. #!/bin/bash
  2. ##
  3. ## TODO:
  4. ## - subordinate container should really be able to modify base image of their master
  5. ## - this could be done through docker-update
  6. ## - I'm not happy with the current build using 'build/' directory, this should be
  7. ## changed to:
  8. ## - always have a base image (specified in metadata), and always have hooks/install
  9. ## executed and merge in image (like docker-build-charm).
  10. ## - container base image is ALWAYS the image of the master container... this brings
  11. ## questions about a double way to express inheritage (through relations as it is
  12. ## implemented now, or through this base-image ?)
  13. ## - the name of the scripts for relation (aka relation_name-relation-joined) is bad as
  14. ## reading the name in a hooks/ dir, there are no way to know if we are the target or
  15. ## the base of the relation.
  16. ## - we could leverage a 'relations/' dir on the root of the charm, with both:
  17. ## 'relations/provide/relation_name' and 'relations/receive/relation_name'
  18. ## - a very bad point with the actual naming is that we can't have a providing AND
  19. ## receiving a relation with same name.
  20. ## - The cache system should keep md5 of docker-compose and other things between runs
  21. ## - The cache system should use underlying function that have only arguments inputs.
  22. ## This will allow to cache completely without issues function in time.
  23. ## - would probably need instrospection in charm custom action to know if these need
  24. ## init or relations to be set up.
  25. ## - Be clear about when the SERVICE name is used and the CHARM name is used.
  26. ## - in case of service contained in another container
  27. ## - in normal case
  28. ## - in docker-compose, can't use charm name: if we want 2 instances of the same charm
  29. ## we are stuck. What will be unique is the name of the service.
  30. ## - some relations are configured in compose.yml but should not trigger the loading
  31. ## of necessary component (for instance, apache --> log-rotate), if log-rotate is
  32. ## not there, this link should considered optional.
  33. ## - Could probably allow an unexistent charm to be populated with only "docker-image:"
  34. ## of the same name. Although this should trigger a visible warning.
  35. #:-
  36. [ -e /etc/shlib ] && . /etc/shlib || {
  37. echo "Unsatisfied dependency. Please install 'kal-shlib-core'."
  38. exit 1
  39. }
  40. #:-
  41. include common
  42. include pretty
  43. include parse
  44. include charm
  45. include array
  46. include cla
  47. include docker
  48. depends shyaml docker
  49. exname="compose"
  50. version=0.1
  51. usage="$exname [COMPOSE_OPTS] [ACTION [ACTION_OPTS]]"
  52. help="\
  53. $WHITE$exname$NORMAL jobs is to run various shell scripts to build
  54. a running orchestrated and configured docker containers. These shell
  55. scripts will have the opportunity to build a 'docker-compose.yml'.
  56. Once init script and relations scripts are executed, $WHITE$exname$NORMAL
  57. delegate the launching to ${WHITE}docker-compose${NORMAL} by providing it
  58. the final 'docker-compose.yml'.
  59. $WHITE$exname$NORMAL also leverage charms to offer some additional custom
  60. actions per charm, which are simply other scripts that can be
  61. run without launching ${WHITE}docker-compose${NORMAL}.
  62. In compose message, color coding is enforced as such:
  63. - ${DARKCYAN}action$NORMAL,
  64. - ${DARKBLUE}relation$NORMAL,
  65. - ${DARKPINK}charm${NORMAL},
  66. - ${DARKYELLOW}service${NORMAL},
  67. - ${WHITE}option-name${NORMAL}/${WHITE}command-name${NORMAL}/${WHITE}Section-Title${NORMAL}
  68. $WHITE$exname$NORMAL reads '/etc/compose.conf' for global variables, and
  69. '/etc/compose.local.conf' for local host adjustements.
  70. "
  71. ## XXXvlab: this doesn't seem to work when 'compose' is called in
  72. ## a hook of a charm.
  73. #[[ "${BASH_SOURCE[0]}" == "" ]] && SOURCED=true
  74. $(return >/dev/null 2>&1) && SOURCED=true
  75. errlvl() { return "${1:-1}"; }
  76. export -f errlvl
  77. if [ "$UID" == 0 ]; then
  78. CACHEDIR=${CACHEDIR:-/var/cache/compose}
  79. VARDIR=${VARDIR:-/var/lib/compose}
  80. else
  81. [ "$XDG_CONFIG_HOME" ] && CACHEDIR=${CACHEDIR:-$XDG_CONFIG_HOME/compose}
  82. [ "$XDG_DATA_HOME" ] && VARDIR=${VARDIR:-$XDG_DATA_HOME/compose}
  83. CACHEDIR=${CACHEDIR:-$HOME/.cache/compose}
  84. VARDIR=${VARDIR:-$HOME/.local/share/compose}
  85. fi
  86. export VARDIR CACHEDIR
  87. md5_compat() { md5sum | cut -c -32; }
  88. quick_cat_file() { quick_cat_stdin < "$1"; }
  89. quick_cat_stdin() { local IFS=''; while read -r line; do echo "$line"; done ; }
  90. export -f quick_cat_file quick_cat_stdin md5_compat
  91. read-0-err() {
  92. local ret="$1" eof="" idx=0 last=
  93. read -r -- "${ret?}" <<<"0"
  94. shift
  95. while [ "$1" ]; do
  96. last=$idx
  97. read -r -d '' -- "$1" || {
  98. ## Put this last value in ${!ret}
  99. eof="$1"
  100. read -r -- "$ret" <<<"${!eof}"
  101. break
  102. }
  103. ((idx++))
  104. shift
  105. done
  106. [ -z "$eof" ] || {
  107. if [ "$last" != 0 ]; then
  108. echo "Error: read-0-err couldn't fill all value" >&2
  109. read -r -- "$ret" <<<"127"
  110. else
  111. if [ -z "${!ret}" ]; then
  112. echo "Error: last value is not a number, did you finish with an errorlevel ?" >&2
  113. read -r -- "$ret" <<<"126"
  114. fi
  115. fi
  116. false
  117. }
  118. }
  119. export -f read-0-err
  120. p-err() {
  121. "$@"
  122. echo "$?"
  123. }
  124. export -f p-err
  125. wyq() {
  126. local exp="$1"
  127. yq e -e -0 "$1"
  128. printf "%s" "$?"
  129. }
  130. wyq-r() {
  131. local exp="$1"
  132. yq e -e -0 -r=false "$1"
  133. printf "%s" "$?"
  134. }
  135. err-d () {
  136. local msg="$*"
  137. shift
  138. err "$msg"
  139. print:traceback 1
  140. }
  141. export -f err-d
  142. print:traceback() {
  143. local omit_level="${1:-0}"
  144. if [ -z "$DEBUG" ]; then
  145. echo " Note: traceback available if you provide {--debug|-d} option." >&2
  146. return 0
  147. fi
  148. echo "${WHITE}Traceback (most recent call last):${NORMAL}" >&2
  149. local i
  150. for ((i=${#FUNCNAME[@]} - 1; i > "$omit_level"; i--)); do
  151. local file="${BASH_SOURCE[$i]}"
  152. local line="${BASH_LINENO[$i - 1]}"
  153. local func="${FUNCNAME[$i]}"
  154. if [[ -f "$file" ]]; then
  155. # Get total number of lines in the file
  156. local total_lines
  157. total_lines=$(wc -l < "$file")
  158. # Calculate start and end lines for context
  159. local start_line=$((line - 2))
  160. local end_line=$((line + 2))
  161. [[ $start_line -lt 1 ]] && start_line=1
  162. [[ $end_line -gt $total_lines ]] && end_line=$total_lines
  163. # Extract context lines
  164. mapfile -s $((start_line - 1)) -n $((end_line - start_line + 1)) context_lines < "$file"
  165. # Calculate minimal indentation
  166. local min_indent=9999
  167. for line_text in "${context_lines[@]}"; do
  168. if [[ -n "$line_text" ]]; then
  169. # Get leading whitespace
  170. local leading_whitespace="${line_text%%[![:space:]]*}"
  171. local indent=${#leading_whitespace}
  172. if [[ $indent -lt $min_indent ]]; then
  173. min_indent=$indent
  174. fi
  175. fi
  176. done
  177. # Remove minimal indentation from each line
  178. for idx in "${!context_lines[@]}"; do
  179. context_lines[$idx]="${context_lines[$idx]:$min_indent}"
  180. done
  181. else
  182. context_lines=("<source unavailable>")
  183. min_indent=0
  184. start_line=1
  185. end_line=1
  186. fi
  187. # Print the traceback frame
  188. echo " File \"$file\", line $line, in ${WHITE}$func${NORMAL}:"
  189. # Print the context with line numbers
  190. local current_line=$start_line
  191. for context_line in "${context_lines[@]}"; do
  192. context_line="${context_line%$'\n'}"
  193. if [[ $current_line -eq $line ]]; then
  194. echo " ${DARKYELLOW}$current_line${NORMAL} ${context_line}"
  195. else
  196. echo " ${DARKGRAY}$current_line${NORMAL} ${context_line}"
  197. fi
  198. ((current_line++))
  199. done
  200. done >&2
  201. }
  202. export -f print:traceback
  203. clean_cache() {
  204. local i=0
  205. for f in $(ls -t "$CACHEDIR/"*.cache.* 2>/dev/null | tail -n +500); do
  206. ((i++))
  207. rm -f "$f"
  208. done
  209. if (( i > 0 )); then
  210. debug "${WHITE}Cleaned cache:${NORMAL} Removed $((i)) elements (current cache size is $(du -sh "$CACHEDIR" | cut -f 1))"
  211. fi
  212. }
  213. export DEFAULT_COMPOSE_FILE
  214. ##
  215. ## Merge YAML files
  216. ##
  217. export _merge_yaml_common_code="
  218. import sys
  219. import yaml
  220. try:
  221. from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
  222. except ImportError: ## pragma: no cover
  223. sys.stderr.write('YAML code in pure python\n')
  224. exit(1)
  225. from yaml import SafeLoader, SafeDumper
  226. class MySafeLoader(SafeLoader): pass
  227. class MySafeDumper(SafeDumper): pass
  228. try:
  229. # included in standard lib from Python 2.7
  230. from collections import OrderedDict
  231. except ImportError:
  232. # try importing the backported drop-in replacement
  233. # it's available on PyPI
  234. from ordereddict import OrderedDict
  235. ## Ensure that there are no collision with legacy OrderedDict
  236. ## that could be used for omap for instance.
  237. class MyOrderedDict(OrderedDict):
  238. pass
  239. MySafeDumper.add_representer(
  240. MyOrderedDict,
  241. lambda cls, data: cls.represent_dict(data.items()))
  242. def construct_omap(cls, node):
  243. cls.flatten_mapping(node)
  244. return MyOrderedDict(cls.construct_pairs(node))
  245. MySafeLoader.add_constructor(
  246. yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
  247. construct_omap)
  248. ##
  249. ## Support local and global objects
  250. ##
  251. class EncapsulatedNode(object): pass
  252. def mk_encapsulated_node(s, node):
  253. method = 'construct_%s' % (node.id, )
  254. data = getattr(s, method)(node)
  255. class _E(data.__class__, EncapsulatedNode):
  256. pass
  257. _E.__name__ = str(node.tag)
  258. _E._node = node
  259. return _E(data)
  260. def represent_encapsulated_node(s, o):
  261. value = s.represent_data(o.__class__.__bases__[0](o))
  262. value.tag = o.__class__.__name__
  263. return value
  264. MySafeDumper.add_multi_representer(EncapsulatedNode,
  265. represent_encapsulated_node)
  266. MySafeLoader.add_constructor(None, mk_encapsulated_node)
  267. def fc(filename):
  268. with open(filename) as f:
  269. return f.read()
  270. def merge(*args):
  271. # sys.stderr.write('%r\n' % (args, ))
  272. args = [arg for arg in args if arg is not None]
  273. if len(args) == 0:
  274. return None
  275. if len(args) == 1:
  276. return args[0]
  277. if all(isinstance(arg, (int, basestring, bool, float)) for arg in args):
  278. return args[-1]
  279. elif all(isinstance(arg, list) for arg in args):
  280. res = []
  281. for arg in args:
  282. for elt in arg:
  283. if elt in res:
  284. res.remove(elt)
  285. res.append(elt)
  286. return res
  287. elif all(isinstance(arg, dict) for arg in args):
  288. keys = set()
  289. for arg in args:
  290. keys |= set(arg.keys())
  291. dct = {}
  292. for key in keys:
  293. sub_args = []
  294. for arg in args:
  295. if key in arg:
  296. sub_args.append(arg)
  297. try:
  298. dct[key] = merge(*(a[key] for a in sub_args))
  299. except NotImplementedError as e:
  300. raise NotImplementedError(
  301. e.args[0],
  302. '%s.%s' % (key, e.args[1]) if e.args[1] else key,
  303. e.args[2])
  304. if dct[key] is None:
  305. del dct[key]
  306. return dct
  307. else:
  308. raise NotImplementedError(
  309. 'Unsupported types: %s'
  310. % (', '.join(list(set(arg.__class__.__name__ for arg in args)))), '', args)
  311. return None
  312. def merge_cli(*args):
  313. try:
  314. c = merge(*args)
  315. except NotImplementedError as e:
  316. sys.stderr.write('Merging Failed: %s.\n%s\n'
  317. ' Values are:\n %s\n'
  318. % (e.args[0],
  319. ' Conflicting key is %r.' % e.args[1] if e.args[1] else
  320. ' Conflict at base of structure.',
  321. '\\n '.join('v%d: %r' % (i, a)
  322. for i, a in enumerate(e.args[2]))))
  323. exit(1)
  324. if c is not None:
  325. print '%s' % yaml.dump(c, default_flow_style=False, Dumper=MySafeDumper)
  326. "
  327. merge_yaml() {
  328. if ! [ -r "$state_tmpdir/merge_yaml.py" ]; then
  329. cat <<EOF > "$state_tmpdir/merge_yaml.py"
  330. $_merge_yaml_common_code
  331. merge_cli(*(yaml.load(fc(f), Loader=MySafeLoader) for f in sys.argv[1:]))
  332. EOF
  333. fi
  334. python "$state_tmpdir/merge_yaml.py" "$@"
  335. }
  336. export -f merge_yaml
  337. merge_yaml_str() {
  338. local entries="$@"
  339. if ! [ -r "$state_tmpdir/merge_yaml_str.py" ]; then
  340. cat <<EOF > "$state_tmpdir/merge_yaml_str.py" || return 1
  341. $_merge_yaml_common_code
  342. merge_cli(*(yaml.load(f, Loader=MySafeLoader) for f in sys.argv[1:]))
  343. EOF
  344. fi
  345. if ! python "$state_tmpdir/merge_yaml_str.py" "$@"; then
  346. err "Failed to merge yaml strings:"
  347. local s
  348. for s in "$@"; do
  349. printf " - \n"
  350. printf "%s\n" "$s" | prefix " ${GRAY}|$NORMAL "
  351. done >&2
  352. return 1
  353. fi
  354. }
  355. export -f merge_yaml_str
  356. yaml_get_values() {
  357. local sep=${1:-$'\n'} value input type first elt
  358. input=$(cat -)
  359. if [ -z "$input" ] || [[ "$input" =~ ^None|null$ ]]; then
  360. return 0
  361. fi
  362. type=$(e "$input" | shyaml get-type)
  363. value=
  364. case "$type" in
  365. "sequence")
  366. first=1
  367. while read-0 elt; do
  368. elt="$(e "$elt" | yaml_get_interpret)" || return 1
  369. [ "$elt" ] || continue
  370. if [ "$first" ]; then
  371. first=
  372. else
  373. value+="$sep"
  374. fi
  375. first=
  376. value+="$elt"
  377. done < <(e "$input" | shyaml -y get-values-0)
  378. ;;
  379. "struct")
  380. while read-0 val; do
  381. value+=$'\n'"$(e "$val" | yaml_get_interpret)" || return 1
  382. done < <(e "$input" | shyaml -y values-0)
  383. ;;
  384. "NoneType")
  385. value=""
  386. ;;
  387. "str"|*)
  388. value+="$(e "$input" | yaml_get_interpret)"
  389. ;;
  390. esac
  391. e "$value"
  392. }
  393. export -f yaml_get_values
  394. yaml_key_val_str() {
  395. local entries="$@"
  396. if ! [ -r "$state_tmpdir/yaml_key_val_str.py" ]; then
  397. cat <<EOF > "$state_tmpdir/yaml_key_val_str.py"
  398. $_merge_yaml_common_code
  399. print '%s' % yaml.dump(
  400. {
  401. yaml.load(sys.argv[1], Loader=MySafeLoader):
  402. yaml.load(sys.argv[2], Loader=MySafeLoader)
  403. },
  404. default_flow_style=False,
  405. Dumper=MySafeDumper,
  406. )
  407. EOF
  408. fi
  409. python "$state_tmpdir/yaml_key_val_str.py" "$@"
  410. }
  411. export -f yaml_key_val_str
  412. ##
  413. ## Docker
  414. ##
  415. docker_has_image() {
  416. local image="$1"
  417. images=$(docker images -q "$image" 2>/dev/null) || {
  418. err "docker images call has failed unexpectedly."
  419. return 1
  420. }
  421. [ -n "$images" ]
  422. }
  423. export -f docker_has_image
  424. docker_image_id() {
  425. local image="$1"
  426. image_id=$(docker inspect "$image" --format='{{.Id}}') || return 1
  427. echo "$image_id" # | tee "$cache_file"
  428. }
  429. export -f docker_image_id
  430. cached_cmd_on_image() {
  431. local image="$1" cache_file
  432. image_id=$(docker_image_id "$image") || return 1
  433. cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  434. if [ -e "$cache_file" ]; then
  435. # debug "$FUNCNAME: cache hit ($*)"
  436. quick_cat_stdin < "$cache_file"
  437. return 0
  438. fi
  439. shift
  440. out=$(docker run -i --rm --entrypoint /bin/sh "$image_id" -c "$*") || return 1
  441. echo "$out" | tee "$cache_file"
  442. }
  443. export -f cached_cmd_on_image
  444. cmd_on_base_image() {
  445. local service="$1" base_image
  446. shift
  447. base_image=$(service_ensure_image_ready "$service") || return 1
  448. docker run -i --rm --entrypoint /bin/bash "$base_image" -c "$*"
  449. }
  450. export -f cmd_on_base_image
  451. cached_cmd_on_base_image() {
  452. local service="$1" base_image cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  453. shift
  454. if [ -e "$cache_file" ]; then
  455. # debug "$FUNCNAME: cache hit ($*)"
  456. quick_cat_stdin < "$cache_file"
  457. return 0
  458. fi
  459. base_image=$(service_ensure_image_ready "$service") || return 1
  460. result=$(cached_cmd_on_image "$base_image" "$@") || return 1
  461. echo "$result" | tee "$cache_file"
  462. }
  463. export -f cached_cmd_on_base_image
  464. docker_update() {
  465. ## YYY: warning, we a storing important information in cache, cache can
  466. ## be removed.
  467. ## We want here to cache the last script on given service whatever that script was
  468. local service="$1" script="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$1" \
  469. previous_base_image stored_image_id
  470. shift
  471. shift
  472. ## this will build it if necessary
  473. base_image=$(service_ensure_image_ready "$service") || return 1
  474. ## XXXvlab: there are probably ways to avoid rebuilding that each time
  475. image_id="$(docker_image_id "$base_image")" || return 1
  476. if [ -e "$cache_file" ]; then
  477. info "Cache file exists"
  478. read-0 previous_base_image stored_image_id < <(cat "$cache_file")
  479. info "previous: $previous_base_image"
  480. info "stored: $stored_image_id"
  481. else
  482. info "No cache file $cache_file"
  483. previous_base_image=""
  484. fi
  485. if [ "$previous_base_image" -a "$stored_image_id" == "$image_id" ]; then
  486. info "Resetting $base_image to $previous_base_image"
  487. docker tag "$previous_base_image" "$base_image" || return 1
  488. image_id="$(docker_image_id "$base_image")" || return 1
  489. else
  490. previous_base_image="$image_id"
  491. fi
  492. info "Updating base image: $base_image (hash: $image_id)"
  493. echo "$script" | dupd --debug -u "$base_image" -- "$@" || {
  494. err "Failed updating base image"
  495. return 1
  496. }
  497. new_image_id="$(docker_image_id "$base_image")"
  498. [ "$new_image_id" == "$previous_base_image" ] && {
  499. err "Image was not updated correctly (same id)."
  500. return 1
  501. }
  502. printf "%s\0" "$previous_base_image" "$new_image_id" > "$cache_file"
  503. info "Wrote cache file $cache_file"
  504. }
  505. export -f docker_update
  506. image_exposed_ports_0() {
  507. local image="$1"
  508. docker inspect --format='{{range $p, $conf := .Config.ExposedPorts}}{{$p}}{{"\x00"}}{{end}}' "$image"
  509. }
  510. export -f image_exposed_ports_0
  511. ## feature not yet included in docker: https://github.com/moby/moby/issues/16079
  512. docker_image_export_dir() {
  513. local image="$1" src="$2" dst="$3" container_id
  514. (
  515. container_id=$(docker create "$image") || exit 1
  516. trap_add EXIT,ERR "docker rm \"$container_id\" >/dev/null"
  517. docker cp "$container_id":"$src" "$dst"
  518. )
  519. }
  520. export -f docker_image_export_dir
  521. service_base_image_export_dir() {
  522. local service="$1" src="$2" dst="$3" base_image
  523. shift
  524. base_image=$(service_ensure_image_ready "$service") || return 1
  525. docker_image_export_dir "$base_image" "$src" "$dst"
  526. }
  527. export -f service_base_image_export_dir
  528. service_base_image_id() {
  529. local service="$1" src="$2" dst="$3" base_image
  530. shift
  531. base_image=$(service_ensure_image_ready "$service") || return 1
  532. docker inspect "$base_image" --format="{{ .Id }}"
  533. }
  534. export -f service_base_image_id
  535. ##
  536. ## Generic
  537. ##
  538. fn.exists() {
  539. declare -F "$1" >/dev/null
  540. }
  541. str_pattern_matches() {
  542. local str="$1"
  543. shift
  544. for pattern in "$@"; do
  545. eval "[[ \"$str\" == $pattern ]]" && return 0
  546. done
  547. return 1
  548. }
  549. str_matches() {
  550. local str="$1"
  551. shift
  552. for pattern in "$@"; do
  553. [[ "$str" == "$pattern" ]] && return 0
  554. done
  555. return 1
  556. }
  557. gen_password() {
  558. local l=( {a..z} {A..Z} {0..9} ) nl="${#l[@]}" size=${1:-16}
  559. while ((size--)); do
  560. echo -n "${l[$((RANDOM * nl / 32768))]}"
  561. done
  562. echo
  563. }
  564. export -f gen_password
  565. file_put() {
  566. local TARGET="$1"
  567. mkdir -p "$(dirname "$TARGET")" &&
  568. cat - > "$TARGET"
  569. }
  570. export -f file_put
  571. file_put_0() {
  572. local TARGET="$1"
  573. mkdir -p "$(dirname "$TARGET")" &&
  574. cat > "$TARGET"
  575. }
  576. export -f file_put_0
  577. fetch_file() {
  578. local src="$1"
  579. case "$src" in
  580. *"://"*)
  581. err "Unsupported target scheme."
  582. return 1
  583. ;;
  584. *)
  585. ## Try direct
  586. if ! [ -r "$src" ]; then
  587. err "File '$src' not found/readable."
  588. return 1
  589. fi
  590. cat "$src" || return 1
  591. ;;
  592. esac
  593. }
  594. export -f fetch_file
  595. ## receives stdin content to decompress on stdout
  596. ## stdout content should be tar format.
  597. uncompress_file() {
  598. local filename="$1"
  599. ## Warning, the content of the file is already as stdin, the filename
  600. ## is there to hint for correct decompression.
  601. case "$filename" in
  602. *".gz")
  603. gunzip
  604. ;;
  605. *".bz2")
  606. bunzip2
  607. ;;
  608. *)
  609. cat
  610. ;;
  611. esac
  612. }
  613. export -f uncompress_file
  614. get_file() {
  615. local src="$1"
  616. fetch_file "$src" | uncompress_file "$src"
  617. }
  618. export -f get_file
  619. ##
  620. ## Common database lib
  621. ##
  622. _clean_docker() {
  623. local _DB_NAME="$1" container_id="$2"
  624. (
  625. set +e
  626. debug "Removing container $_DB_NAME"
  627. docker stop "$container_id"
  628. docker rm "$_DB_NAME"
  629. docker network rm "${_DB_NAME}"
  630. rm -vf "$state_tmpdir/${_DB_NAME}.state"
  631. ) >&2
  632. }
  633. export -f _clean_docker
  634. get_service_base_image_dir_uid_gid() {
  635. local service="$1" dir="$2" uid_gid
  636. uid_gid=$(cached_cmd_on_base_image "$service" "stat -c '%u %g' '$dir'") || {
  637. debug "Failed to query '$dir' uid in ${DARKYELLOW}$service${NORMAL} base image."
  638. return 1
  639. }
  640. info "uid and gid from ${DARKYELLOW}$service${NORMAL}:$dir is '$uid_gid'"
  641. echo "$uid_gid"
  642. }
  643. export -f get_service_base_image_dir_uid_gid
  644. get_service_type() {
  645. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  646. if [ -z "$service" ]; then
  647. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  648. return 1
  649. fi
  650. if [ -e "$cache_file" ]; then
  651. # debug "$FUNCNAME: cache hit ($*)"
  652. cat "$cache_file"
  653. return 0
  654. fi
  655. master_target_service="$(get_top_master_service_for_service "$service")" || return 1
  656. charm=$(get_service_charm "$master_target_service") || return 1
  657. metadata=$(charm.metadata "$charm") || return 1
  658. printf "%s" "$metadata" | shyaml get-value type service 2>/dev/null |
  659. tee "$cache_file"
  660. }
  661. export -f get_service_type
  662. are_files_locked_in_dir() {
  663. local dir="$1" device hdev ldev
  664. device=$(stat -c %d "$dir") || {
  665. err "Can't stat '$dir'."
  666. return 1
  667. }
  668. device=$(printf "%04x" $device)
  669. hdev=${device:0:2}
  670. ldev=${device:2:2}
  671. inodes=$(find "$dir" -printf ':%i:\n')
  672. found=
  673. while read -r inode; do
  674. debug "try inode:$inode"
  675. if [[ "$inodes" == *":$inode:"* ]]; then
  676. found=1
  677. break
  678. fi
  679. done < <(cat /proc/locks | grep " $hdev:$ldev:" | sed -r "s/^.*$hdev:$ldev:([0-9]+).*$/\1/g")
  680. [ "$found" ]
  681. }
  682. export -f are_files_locked_in_dir
  683. set_db_params() {
  684. local docker_ip="$1" docker_network="$2"
  685. if [ -z "$DB_PARAMS_LOADED" ]; then
  686. DB_PARAMS_LOADED=1
  687. _set_db_params "$docker_ip" "$docker_network"
  688. fi
  689. }
  690. export -f set_db_params
  691. export _PID="$$"
  692. ensure_db_docker_running () {
  693. local _STATE_FILE errlvl project
  694. _DB_NAME="db_${DB_NAME}_${_PID}"
  695. _STATE_FILE="$state_tmpdir/${_DB_NAME}.state"
  696. if [ -e "$_STATE_FILE" ]; then
  697. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$(cat "$_STATE_FILE")"
  698. debug "Re-using previous docker/connection '$DOCKER_IP'."
  699. set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  700. return 0
  701. fi
  702. if [ -e "$state_tmpdir/${_DB_NAME}.working" ]; then
  703. ## avoid recursive calls.
  704. if [ -z "$DOCKER_IP" ]; then
  705. err "Currently figuring up DOCKER_IP, please set it yourself before this call if needed."
  706. return 1
  707. else
  708. debug "ignoring recursive call of 'ensure_db_docker_running'."
  709. fi
  710. return 0
  711. fi
  712. touch "$state_tmpdir/${_DB_NAME}.working"
  713. docker rm "$_DB_NAME" 2>/dev/null || true
  714. host_db_working_dir="$HOST_DATASTORE/${SERVICE_NAME}$DB_DATADIR"
  715. if is_db_locked; then
  716. info "Some process is using '$host_db_working_dir'. Trying to find a docker that would do this..."
  717. found=
  718. for docker_id in $(docker ps -q); do
  719. has_volume_mounted=$(
  720. docker inspect \
  721. --format "{{range .Mounts}}{{if eq .Destination \"$DB_DATADIR\"}}{{.Source}}{{end}}{{end}}" \
  722. "$docker_id")
  723. if [ "$has_volume_mounted" == "$host_db_working_dir" ]; then
  724. info "docker '$docker_id' uses '$has_volume_mounted'."
  725. project=$(docker inspect "$docker_id" \
  726. --format "{{index .Config.Labels \"compose.project\" }}") || continue
  727. info "docker '$docker_id' is from project '$project' (current project is '$PROJECT_NAME')."
  728. [ "$project" == "$PROJECT_NAME" ] || continue
  729. found="$docker_id"
  730. break
  731. fi
  732. done
  733. if [ -z "$found" ]; then
  734. err "Please shutdown any other docker using this directory."
  735. return 1
  736. fi
  737. export container_id="$found"
  738. info "Found docker $docker_id is already running."
  739. else
  740. verb "Database is not locked."
  741. if ! docker_has_image "$DOCKER_BASE_IMAGE"; then
  742. err "Unexpected missing docker image $DOCKER_BASE_IMAGE."
  743. return 1
  744. fi
  745. _set_server_db_params || return 1
  746. debug docker network create "$_DB_NAME"
  747. if ! network_id=$(docker network create "$_DB_NAME"); then
  748. err "'docker network create $_DB_NAME' failed !"
  749. _clean_docker "$_DB_NAME" "$container_id"
  750. rm "$state_tmpdir/${_DB_NAME}.working"
  751. return 1
  752. fi
  753. debug docker run -d \
  754. --name "$_DB_NAME" \
  755. "${server_docker_opts[@]}" \
  756. --network "$_DB_NAME" \
  757. -v "$host_db_working_dir:$DB_DATADIR" \
  758. "$DOCKER_BASE_IMAGE"
  759. if ! container_id=$(
  760. docker run -d \
  761. --name "$_DB_NAME" \
  762. "${server_docker_opts[@]}" \
  763. --network "$_DB_NAME" \
  764. -v "$host_db_working_dir:$DB_DATADIR" \
  765. "$DOCKER_BASE_IMAGE"
  766. ); then
  767. err "'docker run' failed !"
  768. _clean_docker "$_DB_NAME" "$container_id"
  769. rm "$state_tmpdir/${_DB_NAME}.working"
  770. return 1
  771. fi
  772. trap_add EXIT,ERR "_clean_docker \"$_DB_NAME\" \"$container_id\""
  773. fi
  774. if docker_ip=$(wait_for_docker_ip "$container_id"); then
  775. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip"
  776. echo "$docker_ip" > "$_STATE_FILE"
  777. debug "written '$_STATE_FILE'"
  778. rm "$state_tmpdir/${_DB_NAME}.working"
  779. set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  780. return 0
  781. else
  782. errlvl="$?"
  783. err "Db not found (errlvl: $errlvl). Tail of docker logs follows:"
  784. docker logs --tail=5 "$container_id" 2>&1 | prefix " | " >&2
  785. rm "$state_tmpdir/${_DB_NAME}.working"
  786. return "$errlvl"
  787. fi
  788. }
  789. export -f ensure_db_docker_running
  790. ## Require to set $db_docker_opts if needed, and $DB_PASSFILE
  791. ##
  792. _dcmd() {
  793. local docker_opts command="$1"
  794. shift
  795. debug "Db> $command $@"
  796. if [ "$HOST_DB_PASSFILE" -a -f "$LOCAL_DB_PASSFILE" -a "$CLIENT_DB_PASSFILE" ]; then
  797. verb "Found and using '$HOST_DB_PASSFILE' as '$CLIENT_DB_PASSFILE'."
  798. docker_opts=("${db_docker_opts[@]}" "-v" "$HOST_DB_PASSFILE:$CLIENT_DB_PASSFILE")
  799. else
  800. docker_opts=("${db_docker_opts[@]}")
  801. fi
  802. ## XXXX was here: actualy, we need only connection between this version and the client version
  803. debug docker run -i --rm \
  804. "${docker_opts[@]}" \
  805. --entrypoint "$command" "$DOCKER_BASE_IMAGE" "${db_cmd_opts[@]}" "$@"
  806. docker run -i --rm \
  807. "${docker_opts[@]}" \
  808. --entrypoint "$command" "$DOCKER_BASE_IMAGE" "${db_cmd_opts[@]}" "$@"
  809. }
  810. export -f _dcmd
  811. ## Executes code through db
  812. dcmd() {
  813. local fun
  814. [ "$DB_NAME" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_NAME."
  815. [ "$DB_DATADIR" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_DATADIR."
  816. # [ "$DB_PASSFILE" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_PASSFILE."
  817. [ "$_PID" ] || print_syntax_error "$FUNCNAME: You must provide \$_PID."
  818. for fun in is_db_locked _set_db_params ddb; do
  819. [ "$(type -t "$fun")" == "function" ] ||
  820. print_syntax_error "$FUNCNAME: You must provide function '$fun'."
  821. done
  822. ensure_db_docker_running </dev/null || return 1
  823. _dcmd "$@"
  824. }
  825. export -f dcmd
  826. get_docker_ips() {
  827. local name="$1" ip format network_id
  828. if ! docker inspect --format='{{ .NetworkSettings.Networks }}' "$name" >/dev/null 2>&1; then
  829. echo "default:$(docker inspect --format='{{ .NetworkSettings.IPAddress }}' "$name" 2>/dev/null)"
  830. else
  831. format='{{range $name, $conf := .NetworkSettings.Networks}}{{$name}}{{"\x00"}}{{$conf.IPAddress}}{{"\x00"}}{{end}}'
  832. while read-0 network_id ip; do
  833. printf "%s:%s\n" "$network_id" "$ip"
  834. done < <(docker inspect --format="$format" "$name")
  835. fi
  836. }
  837. export -f get_docker_ips
  838. get_docker_ip() {
  839. local name="$1"
  840. get_docker_ips "$name"
  841. }
  842. export -f get_docker_ip
  843. wait_docker_ip() {
  844. local name="$1" timeout="${2:-15}" timeout_count=0 docker_ip=
  845. start=$SECONDS
  846. while [ -z "$docker_ip" ]; do
  847. sleep 0.5
  848. docker_ip=$(get_docker_ip "$name") && break
  849. elapsed=$((SECONDS - start))
  850. if ((elapsed > timeout)); then
  851. err "${RED}timeout error${NORMAL}(${timeout}s):" \
  852. "Could not find '$name' docker container's IP."
  853. return 1
  854. fi
  855. [ "$elapsed" == "$old_elapsed" ] ||
  856. verb "Waiting for docker $name... ($elapsed/$timeout)"
  857. old_elapsed="$elapsed"
  858. done
  859. verb "Found docker $name network and IP: $docker_ip"
  860. echo "$docker_ip"
  861. }
  862. export -f wait_docker_ip
  863. wait_for_tcp_port() {
  864. local network=$1 host_port=$2 timeout=${3:-60}
  865. verb "Trying to connect to $host_port"
  866. bash_image=${DEFAULT_BASH_IMAGE:-docker.0k.io/bash}
  867. #echo docker run --rm -i --network "$network" "$bash_image" >&2
  868. docker run --rm -i --network "$network" "$bash_image" <<EOF
  869. start=\$SECONDS
  870. while true; do
  871. timeout 1 bash -c "</dev/tcp/${host_port/://}" >/dev/null 2>&1 && break
  872. sleep 0.2
  873. if [ "\$((SECONDS - start))" -gt "$timeout" ]; then
  874. exit 1
  875. fi
  876. done
  877. exit 0
  878. EOF
  879. if [ "$?" != 0 ]; then
  880. err "${RED}timeout error${NORMAL}(${timeout}s):"\
  881. "Could not connect to $host_port."
  882. return 1
  883. fi
  884. return 0
  885. }
  886. export -f wait_for_tcp_port
  887. ## Warning: requires a ``ddb`` matching current database to be checked
  888. wait_for_docker_ip() {
  889. local name=$1 DOCKER_IP= DOCKER_NETWORK= docker_ips= docker_ip= elapsed timeout=10
  890. docker_ip=$(wait_docker_ip "$name" 5) || return 1
  891. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip"
  892. if ! str_is_ipv4 "$DOCKER_IP"; then
  893. err "internal 'wait_docker_ip' did not return a valid IP. Returned IP is '$DOCKER_IP'."
  894. return 1
  895. fi
  896. set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  897. while read-0 port; do
  898. IFS="/" read port type <<<"$port"
  899. [ "$type" == "tcp" ] || continue
  900. wait_for_tcp_port "$DOCKER_NETWORK" "$DOCKER_IP:${port}" || return 17
  901. info "Host/Port $DOCKER_IP:${port} checked ${GREEN}open${NORMAL}."
  902. ## XXXvlab: what to do with more than one port ?
  903. break
  904. done < <(image_exposed_ports_0 "$container_id")
  905. ## Checking direct connection
  906. timeout=120
  907. start=$SECONDS
  908. while true; do
  909. if err=$(echo "$check_command" | ddb 2>&1 >/dev/null); then
  910. break
  911. fi
  912. if ! [[ "$err" == *"the database system is starting up" ]]; then
  913. err "${RED}db connection error${NORMAL}:" \
  914. "Could not connect to db on $DOCKER_IP container's IP."
  915. echo " Note: IP up, TCP ports is(are) open" >&2
  916. if [ "$err" ]; then
  917. echo " Error:" >&2
  918. printf "%s\n" "$err" | prefix " ${RED}!${NORMAL} " >&2
  919. fi
  920. return 18
  921. fi
  922. debug "Got 'database system is starting up' error."
  923. elapsed=$((SECONDS - start))
  924. if ((elapsed > timeout)); then
  925. err "${RED}db connection error${NORMAL}:"\
  926. "Could not connect to db on $DOCKER_IP" \
  927. "container's IP. (IP up, TCP ports is(are) open, sql answer after ${timeout}s)"
  928. return 1
  929. fi
  930. sleep 0.2
  931. done
  932. echo "${DOCKER_NETWORK}:${DOCKER_IP}"
  933. return 0
  934. }
  935. export -f wait_for_docker_ip
  936. docker_add_host_declaration() {
  937. local src_docker=$1 domain=$2 dst_docker=$3 dst_docker_ip= dst_docker_network
  938. dst_docker_ip=$(wait_docker_ip "$dst_docker") || exit 1
  939. IFS=: read dst_docker_ip dst_docker_network <<<"$dst_docker_ip"
  940. docker exec -i "$src_docker" bash <<EOF
  941. if cat /etc/hosts | grep -E "^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+$domain\$" > /dev/null 2>&1; then
  942. sed -ri "s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+$domain\$/$dst_docker_ip $domain/g" /etc/hosts
  943. else
  944. echo "$dst_docker_ip $domain" >> /etc/hosts
  945. fi
  946. EOF
  947. }
  948. export -f docker_add_host_declaration
  949. get_running_containers_for_service() {
  950. local service="$1" project="$2"
  951. project=${project:-$PROJECT_NAME}
  952. [ -n "$project" ] || {
  953. err "No project name was defined yet."
  954. return 1
  955. }
  956. docker ps \
  957. --filter label="compose.project=$project" \
  958. --filter label="compose.master-service=$service" \
  959. --format="{{.ID}}"
  960. }
  961. export -f get_running_containers_for_service
  962. get_container_network_ips() {
  963. local container="$1"
  964. docker inspect "$container" \
  965. --format='{{range $key, $val :=.NetworkSettings.Networks}}{{$key}}{{"\x00"}}{{$val.IPAddress}}{{"\x00"}}{{end}}'
  966. }
  967. export -f get_container_network_ips
  968. get_container_network_ip() {
  969. local container="$1"
  970. while read-0 network ip; do
  971. printf "%s\0" "$network" "$ip"
  972. break
  973. done < <(get_container_network_ips "$container")
  974. }
  975. export -f get_container_network_ip
  976. ##
  977. ## Internal Process
  978. ##
  979. get_docker_compose_links() {
  980. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  981. deps master_service master_target_service _relation_name \
  982. target_service _relation_config tech_dep
  983. if [ -z "$service" ]; then
  984. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  985. return 1
  986. fi
  987. if [ -e "$cache_file" ]; then
  988. # debug "$FUNCNAME: cache hit ($*)"
  989. cat "$cache_file"
  990. return 0
  991. fi
  992. master_service=$(get_top_master_service_for_service "$service") || return 1
  993. deps=()
  994. while read-0 _relation_name target_service _relation_config tech_dep; do
  995. master_target_service="$(get_top_master_service_for_service "$target_service")" || return 1
  996. [ "$master_service" == "$master_target_service" ] && continue
  997. type="$(get_service_type "$target_service")" || return 1
  998. [ "$type" == "stub" ] && continue
  999. [ "$type" == "run-once" ] && continue
  1000. if [ "$tech_dep" == "reversed" ]; then
  1001. deps+=("$(echo -en "$master_target_service:\n links:\n - $master_service")")
  1002. elif [[ "$tech_dep" =~ ^(True|true)$ ]]; then
  1003. deps+=("$(echo -en "$master_service:\n links:\n - $master_target_service")")
  1004. fi
  1005. ## XXXvlab: an attempt to add depends_on, but this doesn't work well actually
  1006. ## as there's a circular dependency issue. We don't really want the full feature
  1007. ## of depends_on, but just to add it as targets when doing an 'up'
  1008. # deps+=("$(echo -en "$master_service:\n depends_on:\n - $master_target_service")")
  1009. done < <(get_service_relations "$service")
  1010. merge_yaml_str "${deps[@]}" | tee "$cache_file" || return 1
  1011. if [ "${PIPESTATUS[0]}" != 0 ]; then
  1012. rm "$cache_file"
  1013. err "Failed to merge YAML from all ${WHITE}links${NORMAL} dependencies."
  1014. return 1
  1015. fi
  1016. }
  1017. _get_docker_compose_opts() {
  1018. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1019. compose_def master_service docker_compose_opts
  1020. if [ -z "$service" ]; then
  1021. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  1022. return 1
  1023. fi
  1024. if [ -e "$cache_file" ]; then
  1025. # debug "$FUNCNAME: cache hit ($*)"
  1026. cat "$cache_file"
  1027. return 0
  1028. fi
  1029. compose_def="$(get_compose_service_def "$service")" || return 1
  1030. master_service="$(get_top_master_service_for_service "$service")"
  1031. if docker_compose_opts=$(echo "$compose_def" | shyaml get-value -y "docker-compose" 2>/dev/null); then
  1032. yaml_key_val_str "$master_service" "$docker_compose_opts"
  1033. fi | tee "$cache_file"
  1034. if [ "${PIPESTATUS[0]}" != 0 ]; then
  1035. rm "$cache_file"
  1036. return 1
  1037. fi
  1038. }
  1039. ##
  1040. ## By Reading the metadata.yml, we create a docker-compose.yml mixin.
  1041. ## Some metadata.yml (of subordinates) will indeed modify other
  1042. ## services than themselves.
  1043. _get_docker_compose_service_mixin() {
  1044. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1045. links_yaml base_mixin links_yaml docker_compose_options \
  1046. charm charm_part
  1047. if [ -z "$service" ]; then
  1048. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  1049. return 1
  1050. fi
  1051. if [ -e "$cache_file" ]; then
  1052. # debug "$FUNCNAME: cache hit ($*)"
  1053. cat "$cache_file"
  1054. return 0
  1055. fi
  1056. type=$(get_service_type "$service") || return 1
  1057. [ "$type" == "stub" ] && return 0
  1058. master_service=$(get_top_master_service_for_service "$service") || {
  1059. err "Failed to get top master service for service $DARKYELLOW$service$NORMAL"
  1060. return 1
  1061. }
  1062. ## The compose part
  1063. base_mixin="$master_service:
  1064. labels:
  1065. - compose.service=$service
  1066. - compose.master-service=${master_service}
  1067. - compose.project=$(get_default_project_name)"
  1068. links_yaml=$(get_docker_compose_links "$service") || return 1
  1069. docker_compose_options=$(_get_docker_compose_opts "$service") || return 1
  1070. ## the charm part
  1071. charm_part=$(get_docker_compose_mixin_from_metadata "$service") || return 1
  1072. ## Merge results
  1073. if [ "$charm_part" ]; then
  1074. charm_yaml="$(yaml_key_val_str "$master_service" "$charm_part")" || return 1
  1075. merge_yaml_str "$base_mixin" "$links_yaml" "$charm_yaml" "$docker_compose_options" || return 1
  1076. else
  1077. merge_yaml_str "$base_mixin" "$links_yaml" "$docker_compose_options" || return 1
  1078. fi | tee "$cache_file"
  1079. if [ "${PIPESTATUS[0]}" != 0 ]; then
  1080. err "Failed to constitute the base YAML for service '${DARKYELLOW}$service${NORMAL}'"
  1081. rm "$cache_file"
  1082. return 1
  1083. fi
  1084. }
  1085. export -f _get_docker_compose_service_mixin
  1086. ##
  1087. ## Get full `docker-compose.yml` format for all listed services (and
  1088. ## their deps)
  1089. ##
  1090. ## @export
  1091. ## @cache: !system !nofail +stdout
  1092. get_docker_compose () {
  1093. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  1094. entries services service start docker_compose_services
  1095. if [ -e "$cache_file" ]; then
  1096. # debug "$FUNCNAME: cache hit ($*)"
  1097. cat "$cache_file"
  1098. return 0
  1099. fi
  1100. ##
  1101. ## Adding sub services configurations
  1102. ##
  1103. declare -A entries
  1104. start_compilation=$SECONDS
  1105. debug "Compiling 'docker-compose.yml' base for ${DARKYELLOW}$*$NORMAL..."
  1106. for target_service in "$@"; do
  1107. start=$SECONDS
  1108. services=($(get_ordered_service_dependencies "$target_service")) || {
  1109. err "Failed to get dependencies for $DARKYELLOW$target_service$NORMAL"
  1110. return 1
  1111. }
  1112. if [ "$DEBUG" ]; then
  1113. debug " $DARKYELLOW$target_service$NORMAL deps:$DARKYELLOW" \
  1114. "${services[@]::$((${#services[@]} - 1))}" \
  1115. "$NORMAL$GRAY(in $((SECONDS - start))s)$NORMAL"
  1116. fi
  1117. for service in "${services[@]}"; do
  1118. if [ "${entries[$service]}" ]; then
  1119. ## Prevent double inclusion of same service if this
  1120. ## service is deps of two or more of your
  1121. ## requirements.
  1122. continue
  1123. fi
  1124. ## mark the service as "loaded" as well as it's containers
  1125. ## if this is a subordinate service
  1126. start_service=$SECONDS
  1127. entries[$service]=$(_get_docker_compose_service_mixin "$service") || {
  1128. err "Failed to get service mixin for $DARKYELLOW$service$NORMAL"
  1129. return 1
  1130. }
  1131. debug " Applied $DARKYELLOW$service$NORMAL charm metadata mixins $GRAY(in $((SECONDS - start_service))s)$NORMAL"
  1132. done
  1133. debug " ..finished all mixins for $DARKYELLOW$target_service$NORMAL $GRAY(in $((SECONDS - start))s)$NORMAL"
  1134. done
  1135. docker_compose_services=$(merge_yaml_str "${entries[@]}") || {
  1136. err "Failed to merge YAML services entries together."
  1137. return 1
  1138. }
  1139. base_v2="version: '2.1'"
  1140. merge_yaml_str "$(yaml_key_val_str "services" "$docker_compose_services")" \
  1141. "$base_v2" > "$cache_file" || return 1
  1142. export _CURRENT_DOCKER_COMPOSE="$cache_file"
  1143. cat "$_CURRENT_DOCKER_COMPOSE"
  1144. debug " ..compilation of base 'docker-compose.yml' done $GRAY(in $((SECONDS - start_compilation))s)$NORMAL" || true
  1145. # debug " ** ${WHITE}docker-compose.yml${NORMAL}:"
  1146. # debug "$_current_docker_compose"
  1147. }
  1148. export -f get_docker_compose
  1149. _get_compose_service_def_cached () {
  1150. local service="$1" docker_compose="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  1151. if [ -e "$cache_file" ]; then
  1152. #debug "$FUNCNAME: STATIC cache hit"
  1153. cat "$cache_file" &&
  1154. touch "$cache_file" || return 1
  1155. return 0
  1156. fi
  1157. value=$(echo "$docker_compose" | shyaml get-value "${service//./\\.}" 2>/dev/null)
  1158. [ "$value" == None ] && value=""
  1159. if ! echo "$value" | shyaml get-value "charm" >/dev/null 2>&1; then
  1160. if charm.exists "$service"; then
  1161. value=$(merge_yaml <(echo "charm: $service") <(echo "$value")) || {
  1162. err "Can't merge YAML infered 'charm: $service' with base ${DARKYELLOW}$service${NORMAL} YAML definition."
  1163. return 1
  1164. }
  1165. else
  1166. err "No ${WHITE}charm${NORMAL} value for service $DARKYELLOW$service$NORMAL" \
  1167. "in compose, nor same name charm found."
  1168. return 1
  1169. fi
  1170. fi
  1171. echo "$value" | tee "$cache_file" || return 1
  1172. # if [ "${PIPESTATUS[0]}" != 0 ]; then
  1173. # rm "$cache_file"
  1174. # return 1
  1175. # fi
  1176. return 0
  1177. # if [ "${PIPESTATUS[0]}" != 0 -o \! -s "$cache_file" ]; then
  1178. # rm "$cache_file"
  1179. # err "PAS OK $service: $value"
  1180. # return 1
  1181. # fi
  1182. }
  1183. export -f _get_compose_service_def_cached
  1184. ## XXXvlab: a lot to be done to cache the results
  1185. get_compose_service_def () {
  1186. local service="$1" docker_compose cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1187. result
  1188. if [ -e "$cache_file" ]; then
  1189. #debug "$FUNCNAME: SESSION cache hit"
  1190. cat "$cache_file" || return 1
  1191. return 0
  1192. fi
  1193. [ -z "$service" ] && print_syntax_error "Missing service as first argument."
  1194. docker_compose=$(get_compose_yml_content) || return 1
  1195. result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1
  1196. charm=$(echo "$result" | shyaml get-value charm 2>/dev/null) || return 1
  1197. metadata=$(charm.metadata "$charm") || return 1
  1198. if default_options=$(printf "%s" "$metadata" | shyaml -y -q get-value default-options); then
  1199. default_options=$(yaml_key_val_str "options" "$default_options") || return 1
  1200. result=$(merge_yaml_str "$default_options" "$result") || return 1
  1201. fi
  1202. echo "$result" | tee "$cache_file" || return 1
  1203. }
  1204. export -f get_compose_service_def
  1205. _get_service_charm_cached () {
  1206. local service="$1" service_def="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  1207. if [ -e "$cache_file" ]; then
  1208. # debug "$FUNCNAME: cache hit $1"
  1209. cat "$cache_file" &&
  1210. touch "$cache_file" || return 1
  1211. return 0
  1212. fi
  1213. charm=$(echo "$service_def" | shyaml get-value charm 2>/dev/null)
  1214. if [ -z "$charm" ]; then
  1215. err "Missing ${WHITE}charm${NORMAL} value in service $DARKYELLOW$service$NORMAL definition."
  1216. return 1
  1217. fi
  1218. echo "$charm" | tee "$cache_file" || return 1
  1219. }
  1220. export -f _get_service_charm_cached
  1221. get_service_charm () {
  1222. local service="$1"
  1223. if [ -z "$service" ]; then
  1224. echo ${FUNCNAME[@]} >&2
  1225. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  1226. return 1
  1227. fi
  1228. service_def=$(get_compose_service_def "$service") || return 1
  1229. _get_service_charm_cached "$service" "$service_def"
  1230. }
  1231. export -f get_service_charm
  1232. ## built above the docker-compose abstraction, so it relies on the
  1233. ## full docker-compose.yml to be already built.
  1234. get_service_def () {
  1235. local service="$1" def
  1236. if [ -z "$_CURRENT_DOCKER_COMPOSE" ]; then
  1237. err "${FUNCNAME[0]} is meant to be called after"\
  1238. "\$_CURRENT_DOCKER_COMPOSE has been calculated."
  1239. echo " Called by:" >&2
  1240. printf " - %s\n" "${FUNCNAME[@]:1}" >&2
  1241. return 1
  1242. fi
  1243. def=$(cat "$_CURRENT_DOCKER_COMPOSE" | shyaml get-value "services.${service//./\\.}" 2>/dev/null)
  1244. if [ -z "$def" ]; then
  1245. err "No definition for service $DARKYELLOW$service$NORMAL in compiled 'docker-compose.yml'."
  1246. return 1
  1247. fi
  1248. echo "$def"
  1249. }
  1250. export -f get_service_def
  1251. get_build_hash() {
  1252. local dir="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$1")" hash
  1253. if [ -e "$cache_file" ]; then
  1254. # debug "$FUNCNAME: cache hit ($*)"
  1255. cat "$cache_file"
  1256. return 0
  1257. fi
  1258. ## Check that there's a Dockerfile in this directory
  1259. if [ ! -e "$dir/Dockerfile" ]; then
  1260. err "No 'Dockerfile' found in '$dir'."
  1261. return 1
  1262. fi
  1263. ## use find to md5sum all files in the directory and make a final hash
  1264. hash=$(set -o pipefail; cd "$dir"; env -i find "." -type f -exec md5sum {} \; |
  1265. sort | md5sum | awk '{print $1}') || {
  1266. err "Failed to get hash for '$dir'."
  1267. return 1
  1268. }
  1269. printf "%s" "$hash" | tee "$cache_file"
  1270. return $?
  1271. }
  1272. export -f get_build_hash
  1273. ### Query/Get cached image from registry
  1274. ##
  1275. ## Returns on stdout the name of the image if found, or an empty string if not
  1276. cache:image:registry:get() {
  1277. local charm="$1" hash="$2" service="$3"
  1278. local charm_image_name="cache/charm/$charm"
  1279. local charm_image="$charm_image_name:$hash"
  1280. Elt "pulling ${DARKPINK}$charm${NORMAL} image from $COMPOSE_DOCKER_REGISTRY" >&2
  1281. if out=$(docker pull "$COMPOSE_DOCKER_REGISTRY/$charm_image" 2>&1); then
  1282. docker tag "$COMPOSE_DOCKER_REGISTRY/$charm_image" "$charm_image" || {
  1283. err "Failed set image '$COMPOSE_DOCKER_REGISTRY/$charm_image' as '$charm_image'" \
  1284. "for ${DARKYELLOW}$service${NORMAL}."
  1285. return 1
  1286. }
  1287. print_info "found" >&2
  1288. print_status success >&2
  1289. Feed >&2
  1290. printf "%s" "$charm_image" | tee "$cache_file"
  1291. return $?
  1292. fi
  1293. if [[ "$out" != *"manifest unknown"* ]] && [[ "$out" != *"not found"* ]]; then
  1294. print_status failure >&2
  1295. Feed >&2
  1296. err "Failed to pull image '$COMPOSE_DOCKER_REGISTRY/$charm_image'" \
  1297. "for ${DARKYELLOW}$service${NORMAL}:"
  1298. e "$out"$'\n' | prefix " ${GRAY}|${NORMAL} " >&2
  1299. return 1
  1300. fi
  1301. print_info "not found" >&2
  1302. if test "$type_method" = "long"; then
  1303. __status="[${NOOP}ABSENT${NORMAL}]"
  1304. else
  1305. echo -n "${NOOP}"
  1306. shift; shift;
  1307. echo -n "$*${NORMAL}"
  1308. fi >&2
  1309. Feed >&2
  1310. }
  1311. export -f cache:image:registry:get
  1312. ### Store cached image on registry
  1313. ##
  1314. ## Returns nothing
  1315. cache:image:registry:put() {
  1316. if [ -n "$COMPOSE_DOCKER_REGISTRY" ] && [ -n "$COMPOSE_PUSH_TO_REGISTRY" ]; then
  1317. local charm="$1" hash="$2" service="$3"
  1318. local charm_image_name="cache/charm/$charm"
  1319. local charm_image="$charm_image_name:$hash"
  1320. Wrap -d "pushing ${DARKPINK}$charm${NORMAL} image to $COMPOSE_DOCKER_REGISTRY" <<EOF || return 1
  1321. docker tag "$charm_image" "$COMPOSE_DOCKER_REGISTRY/$charm_image" &&
  1322. docker push "$COMPOSE_DOCKER_REGISTRY/$charm_image"
  1323. EOF
  1324. fi >&2
  1325. }
  1326. export -f cache:image:registry:put
  1327. ### Produce docker cached charm image 'cache/charm/$charm:$hash'
  1328. ##
  1329. ## Either by fetching it from a registry or by building it from a
  1330. ## Dockerfile.
  1331. cache:image:produce() {
  1332. local type="$1" src="$2" charm="$3" hash="$4" service="$5"
  1333. local charm_image_name="cache/charm/$charm"
  1334. local charm_image="$charm_image_name:$hash"
  1335. case "$type" in
  1336. fetch)
  1337. local specified_image="$src"
  1338. ## will not pull upstream image if already present locally
  1339. if ! docker_has_image "${specified_image}"; then
  1340. if ! out=$(docker pull "${specified_image}" 2>&1); then
  1341. err "Failed to pull image '$specified_image' for ${DARKYELLOW}$service${NORMAL}:"
  1342. echo "$out" | prefix " | " >&2
  1343. return 1
  1344. fi
  1345. fi
  1346. # specified_image_id=$(docker_image_id "$specified_image") || return 1
  1347. # charm_image_id=
  1348. # if docker_has_image "${image_dst}"; then
  1349. # charm_image_id=$(docker_image_id "${image_dst}") || return 1
  1350. # fi
  1351. # if [ "$specified_image_id" != "$charm_image_id" ]; then
  1352. docker tag "$specified_image" "${charm_image}" || return 1
  1353. # fi
  1354. ;;
  1355. build)
  1356. local service_build="$src"
  1357. build_opts=()
  1358. if [ "$COMPOSE_ACTION" == "build" ]; then
  1359. while read-0 arg; do
  1360. case "$arg" in
  1361. -t|--tag)
  1362. ## XXXvlab: doesn't seem to be actually a valid option
  1363. if [ -n "$COMPOSE_PUSH_TO_REGISTRY" ]; then
  1364. err "You can't use -t|--tag option when pushing to a registry."
  1365. exit 1
  1366. fi
  1367. has_named_image=true
  1368. read-0 val ## should always be okay because already checked
  1369. build_opts+=("$arg" "$val")
  1370. ;;
  1371. --help|-h)
  1372. docker-compose "$action" --help |
  1373. filter_docker_compose_help_message >&2
  1374. exit 0
  1375. ;;
  1376. --*|-*)
  1377. if str_pattern_matches "$arg" $DC_MATCH_MULTI; then
  1378. read-0 value
  1379. build_opts+=("$arg" "$value")
  1380. shift
  1381. elif str_pattern_matches "$arg" $DC_MATCH_SINGLE; then
  1382. build_opts+=("$arg")
  1383. else
  1384. err "Unexpected error while parsing a second time the build arguments."
  1385. fi
  1386. ;;
  1387. *)
  1388. ## Already parsed
  1389. build_opts+=("$arg")
  1390. ;;
  1391. esac
  1392. done < <(cla.normalize "${action_opts[@]}")
  1393. fi
  1394. if [ -z "$has_named_image" ]; then
  1395. build_opts+=(-t "${charm_image}")
  1396. fi
  1397. Wrap -v -d "Building ${DARKPINK}$charm${NORMAL}:$hash image" -- \
  1398. docker build "$service_build" -t "${charm_image}" "${build_opts[@]}" >&2 || {
  1399. err "Failed to build image '${charm_image}' for ${DARKYELLOW}$service${NORMAL}."
  1400. return 1
  1401. }
  1402. if [ -n "$has_named_image" ]; then
  1403. exit 0
  1404. fi
  1405. ;;
  1406. *)
  1407. err "Unknown type '$type'."
  1408. return 1
  1409. ;;
  1410. esac
  1411. }
  1412. export -f cache:image:produce
  1413. service_ensure_image_ready() {
  1414. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1415. master_service service_def service_image service_build service_dockerfile image \
  1416. specified_image specified_image_id charm_image_name hash \
  1417. service_quoted
  1418. if [ -e "$cache_file" ]; then
  1419. #debug "$FUNCNAME: cache hit ($*)"
  1420. cat "$cache_file"
  1421. return 0
  1422. fi
  1423. if [ -z "$_CURRENT_DOCKER_COMPOSE" ]; then
  1424. err "${FUNCNAME[0]} is meant to be called after"\
  1425. "\$_CURRENT_DOCKER_COMPOSE has been calculated."
  1426. echo " Called by:" >&2
  1427. printf " - %s\n" "${FUNCNAME[@]:1}" >&2
  1428. return 1
  1429. fi
  1430. master_service="$(get_top_master_service_for_service "$service")" || {
  1431. err "Could not compute master service for service $DARKYELLOW$service$NORMAL."
  1432. return 1
  1433. }
  1434. if [ "$master_service" != "$service" ]; then
  1435. image=$(service_ensure_image_ready "$master_service") || return 1
  1436. printf "%s" "$image" | tee "$cache_file"
  1437. return $?
  1438. fi
  1439. ## check if \$_CURRENT_DOCKER_COMPOSE's service def is already correctly setup
  1440. local charm="$(get_service_charm "$service")" || return 1
  1441. local charm_image_name="cache/charm/$charm" || return 1
  1442. local service_def="$(get_service_def "$service")" || {
  1443. err "Could not get docker-compose service definition for $DARKYELLOW$service$NORMAL."
  1444. return 1
  1445. }
  1446. local service_quoted=${service//./\\.}
  1447. if specified_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null); then
  1448. if [ "$specified_image" == "$charm_image_name"* ]; then
  1449. ## Assume we already did the change
  1450. printf "%s" "$specified_image" | tee "$cache_file"
  1451. return 0
  1452. fi
  1453. if [[ "$specified_image" == "${COMPOSE_DOCKER_REGISTRY}/"* ]]; then
  1454. if ! docker_has_image "${specified_image}"; then
  1455. Wrap "${wrap_opts[@]}" \
  1456. -v -d "pulling ${DARKPINK}$charm${NORMAL}'s specified image from $COMPOSE_DOCKER_REGISTRY" -- \
  1457. docker pull "${specified_image}" >&2 || return 1
  1458. else
  1459. if [ -n "$DEBUG" ]; then
  1460. Elt "using local ${DARKPINK}$charm${NORMAL}'s specified image from $COMPOSE_DOCKER_REGISTRY" >&2
  1461. print_status noop >&2
  1462. Feed >&2
  1463. fi
  1464. fi
  1465. ## Already on the cache server
  1466. printf "%s" "$specified_image" | tee "$cache_file"
  1467. return 0
  1468. fi
  1469. src="$specified_image"
  1470. hash=$(echo "$specified_image" | md5sum | cut -f 1 -d " ") || return 1
  1471. type=fetch
  1472. ## replace image by charm image
  1473. yq -i ".services.[\"${service_quoted}\"].image = \"${charm_image_name}:${hash}\"" \
  1474. "$_CURRENT_DOCKER_COMPOSE" || return 1
  1475. else
  1476. if ! src=$(echo "$service_def" | shyaml get-value build 2>/dev/null); then
  1477. err "Service $DARKYELLOW$service$NORMAL has no ${WHITE}image${NORMAL} nor ${WHITE}build${NORMAL} parameter."
  1478. echo "$service_def" >&2
  1479. return 1
  1480. fi
  1481. ## According to https://stackoverflow.com/questions/32230577 , if there's a build,
  1482. ## then the built image will get name ${project}_${service}
  1483. hash=$(get_build_hash "$src") || return 1
  1484. type=build
  1485. ## delete build key from service_def and add image to charm_image_name
  1486. yq -i "del(.services.[\"${service_quoted}\"].build) |
  1487. .services.[\"${service_quoted}\"].image = \"${charm_image_name}:${hash}\"" \
  1488. "$_CURRENT_DOCKER_COMPOSE" || return 1
  1489. fi
  1490. if [ "$COMPOSE_ACTION" != "build" ] && docker_has_image "${charm_image_name}:${hash}"; then
  1491. if [ -n "$DEBUG" ]; then
  1492. Elt "using ${DARKPINK}$charm${NORMAL}'s image from local cache" >&2
  1493. print_status noop >&2
  1494. Feed >&2
  1495. fi
  1496. cache:image:registry:put "$charm" "$hash" "$service" || return 1
  1497. printf "%s" "${charm_image_name}:${hash}" | tee "$cache_file"
  1498. return $?
  1499. fi
  1500. ## Can we pull it ? Let's check on $COMPOSE_DOCKER_REGISTRY
  1501. if [ "$COMPOSE_ACTION" != "build" ] && [ -n "$COMPOSE_DOCKER_REGISTRY" ]; then
  1502. img=$(cache:image:registry:get "$charm" "$hash" "$service") || {
  1503. err "Failed to get image '$charm_image_name:$hash' from registry for ${DARKYELLOW}$service${NORMAL}."
  1504. return 1
  1505. }
  1506. [ -n "$img" ] && {
  1507. printf "%s" "$img" | tee "$cache_file"
  1508. return $?
  1509. }
  1510. fi
  1511. cache:image:produce "$type" "$src" "$charm" "$hash" "$service" || return 1
  1512. cache:image:registry:put "$charm" "$hash" "$service" || return 1
  1513. printf "%s" "${charm_image_name}:$hash" | tee "$cache_file"
  1514. return $?
  1515. }
  1516. export -f service_ensure_image_ready
  1517. get_charm_relation_def () {
  1518. local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1519. relation_def metadata
  1520. if [ -e "$cache_file" ]; then
  1521. # debug "$FUNCNAME: cache hit ($*)"
  1522. cat "$cache_file"
  1523. return 0
  1524. fi
  1525. metadata="$(charm.metadata "$charm")" || return 1
  1526. relation_def="$(echo "$metadata" | shyaml get-value "provides.${relation_name}" 2>/dev/null)"
  1527. echo "$relation_def" | tee "$cache_file"
  1528. }
  1529. export -f get_charm_relation_def
  1530. get_charm_tech_dep_orientation_for_relation() {
  1531. local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1532. relation_def value
  1533. if [ -e "$cache_file" ]; then
  1534. # debug "$FUNCNAME: cache hit ($*)"
  1535. cat "$cache_file"
  1536. return 0
  1537. fi
  1538. relation_def=$(get_charm_relation_def "$charm" "$relation_name" 2>/dev/null)
  1539. value=$(echo "$relation_def" | shyaml get-value 'tech-dep' 2>/dev/null)
  1540. value=${value:-True}
  1541. printf "%s" "$value" | tee "$cache_file"
  1542. }
  1543. export -f get_charm_tech_dep_orientation_for_relation
  1544. get_service_relation_tech_dep() {
  1545. local service="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1546. charm tech_dep
  1547. if [ -e "$cache_file" ]; then
  1548. # debug "$FUNCNAME: cache hit ($*)"
  1549. cat "$cache_file"
  1550. return 0
  1551. fi
  1552. charm=$(get_service_charm "$service") || return 1
  1553. tech_dep="$(get_charm_tech_dep_orientation_for_relation "$charm" "$relation_name")" || return 1
  1554. printf "%s" "$tech_dep" | tee "$cache_file"
  1555. }
  1556. export -f get_service_relation_tech_dep
  1557. ##
  1558. ## Use compose file to get deps, and relation definition in metadata.yml
  1559. ## for tech-dep attribute.
  1560. get_service_deps() {
  1561. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH")"
  1562. if [ -e "$cache_file" ]; then
  1563. # debug "$FUNCNAME: cache hit ($*)"
  1564. cat "$cache_file"
  1565. return 0
  1566. fi
  1567. (
  1568. set -o pipefail
  1569. get_service_relations "$service" | \
  1570. while read-0 relation_name target_service _relation_config tech_dep; do
  1571. echo "$target_service"
  1572. done | tee "$cache_file"
  1573. ) || return 1
  1574. }
  1575. export -f get_service_deps
  1576. ## XXXvlab: cache was disabled because improper. Indeed, this needs to cache
  1577. ## 'depths' full state. Second, it should be
  1578. _rec_get_depth() {
  1579. local elt=$1 dep deps max
  1580. [ "${depths[$elt]}" ] && return 0
  1581. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$(declare -pA depths)" "$GLOBAL_ALL_RELATIONS_HASH")"
  1582. if [ -e "$cache_file.depths" ]; then
  1583. #debug "$FUNCNAME: cache hit ($*) - $cache_file.depths"
  1584. while read-0 k v; do
  1585. depths["$k"]="$v"
  1586. done < "$cache_file.depths"
  1587. while read-0 k v; do
  1588. visited["$k"]="$v"
  1589. done < "$cache_file.visited"
  1590. return 0
  1591. fi
  1592. visited[$elt]=1
  1593. #debug "Setting visited[$elt]"
  1594. #debug "Asking for $DARKYELLOW$elt$NORMAL dependencies"
  1595. deps=$(get_service_deps "$elt") || {
  1596. debug "Failed get_service_deps $elt"
  1597. return 1
  1598. }
  1599. # debug "$elt deps are:" $deps
  1600. max=0
  1601. for dep in $deps; do
  1602. [ "${visited[$dep]}" ] && {
  1603. #debug "Already computing $dep"
  1604. continue
  1605. }
  1606. _rec_get_depth "$dep" || return 1
  1607. #debug "Requesting depth[$dep]"
  1608. if (( ${depths[$dep]} > max )); then
  1609. max="${depths[$dep]}"
  1610. fi
  1611. done
  1612. # debug "Setting depth[$elt] to $((max + 1))"
  1613. depths[$elt]=$((max + 1))
  1614. array_kv_to_stdin depths > "$cache_file.depths"
  1615. array_kv_to_stdin visited > "$cache_file.visited"
  1616. # debug "DEPTHS: $(declare -pA depths)"
  1617. # debug "$FUNCNAME: caching hit ($*) - $cache_file"
  1618. }
  1619. export -f _rec_get_depth
  1620. get_ordered_service_dependencies() {
  1621. local services=("$@") cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH")" \
  1622. i value key heads depths visited
  1623. if [ -e "$cache_file" ]; then
  1624. # debug "$FUNCNAME: cache hit ($*)"
  1625. cat "$cache_file"
  1626. return 0
  1627. fi
  1628. #debug "Figuring ordered deps of $DARKYELLOW${services[@]}$NORMAL"
  1629. if [ -z "${services[*]}" ]; then
  1630. return 0
  1631. # print_syntax_error "$FUNCNAME: no arguments"
  1632. # return 1
  1633. fi
  1634. declare -A depths
  1635. declare -A visited
  1636. heads=("${services[@]}")
  1637. while [ "${#heads[@]}" != 0 ]; do
  1638. array_pop heads head
  1639. _rec_get_depth "$head" || return 1
  1640. done
  1641. i=0
  1642. while [ "${#depths[@]}" != 0 ]; do
  1643. for key in "${!depths[@]}"; do
  1644. value="${depths[$key]}"
  1645. if [ "$value" == "$i" ]; then
  1646. echo "$key"
  1647. unset depths[$key]
  1648. fi
  1649. done
  1650. ((i++))
  1651. done | tee "$cache_file"
  1652. }
  1653. export -f get_ordered_service_dependencies
  1654. run_service_acquire_images () {
  1655. local service subservice subservices loaded
  1656. declare -A loaded
  1657. for service in "$@"; do
  1658. subservices=$(get_ordered_service_dependencies "$service") || return 1
  1659. for subservice in $subservices; do
  1660. if [ "${loaded[$subservice]}" ]; then
  1661. ## Prevent double inclusion of same service if this
  1662. ## service is deps of two or more of your
  1663. ## requirements.
  1664. continue
  1665. fi
  1666. type=$(get_service_type "$subservice") || return 1
  1667. MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
  1668. if [ "$type" != "stub" ]; then
  1669. DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$MASTER_BASE_SERVICE_NAME") || return 1
  1670. fi
  1671. loaded[$subservice]=1
  1672. done
  1673. done
  1674. return 0
  1675. }
  1676. run_service_hook () {
  1677. local action="$1" service subservice subservices loaded
  1678. shift
  1679. declare -A loaded
  1680. for service in "$@"; do
  1681. subservices=$(get_ordered_service_dependencies "$service") || return 1
  1682. for subservice in $subservices; do
  1683. if [ "${loaded[$subservice]}" ]; then
  1684. ## Prevent double inclusion of same service if this
  1685. ## service is deps of two or more of your
  1686. ## requirements.
  1687. continue
  1688. fi
  1689. charm=$(get_service_charm "$subservice") || return 1
  1690. charm.has_hook "$charm" "$action" >/dev/null || continue
  1691. type=$(get_service_type "$subservice") || return 1
  1692. PROJECT_NAME=$(get_default_project_name) || return 1
  1693. MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
  1694. MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1
  1695. if [ "$type" != "stub" ]; then
  1696. DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$MASTER_BASE_SERVICE_NAME") || return 1
  1697. fi
  1698. Wrap "${wrap_opts[@]}" -d "running $YELLOW$action$NORMAL hook of $DARKYELLOW$subservice$NORMAL in charm $DARKPINK$charm$NORMAL" <<EOF || return 1
  1699. export DOCKER_BASE_IMAGE="$DOCKER_BASE_IMAGE"
  1700. export SERVICE_NAME=$subservice
  1701. export IMAGE_NAME=$(echo "${PROJECT_NAME}" | tr -d "_-")_\${SERVICE_NAME}
  1702. export CONTAINER_NAME=\${IMAGE_NAME}_1
  1703. export CHARM_NAME="$charm"
  1704. export PROJECT_NAME="$PROJECT_NAME"
  1705. export SERVICE_DATASTORE="$DATASTORE/$subservice"
  1706. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$subservice"
  1707. export MASTER_BASE_SERVICE_NAME="$MASTER_BASE_SERVICE_NAME"
  1708. export MASTER_BASE_CHARM_NAME="$MASTER_BASE_CHARM_NAME"
  1709. charm.run_hook "local" "$charm" "$action"
  1710. EOF
  1711. loaded[$subservice]=1
  1712. done
  1713. done
  1714. return 0
  1715. }
  1716. host_resource_get() {
  1717. local location="$1" cfg="$2"
  1718. type=$(echo "$cfg" | shyaml get-value type 2>/dev/null) || {
  1719. err "Missing ${WHITE}type$NORMAL option in ${WHITE}get$NORMAL config for location '$location'"
  1720. return 1
  1721. }
  1722. if fn.exists host_resource_get_$type; then
  1723. host_resource_get_$type "$location" "$cfg"
  1724. else
  1725. err "Source ${WHITE}source$NORMAL type '$type' unknown for" \
  1726. "${WHITE}host-resource$NORMAL '$location' defined in" \
  1727. "$DARKYELLOW$subservice$NORMAL config."
  1728. return 1
  1729. fi
  1730. }
  1731. export -f host_resource_get
  1732. host_resource_get_git() {
  1733. local location="$1" cfg="$2" branch parent url
  1734. branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null)
  1735. branch=${branch:-master}
  1736. url=$(echo "$cfg" | shyaml get-value url 2>/dev/null)
  1737. parent="$(dirname "$location")"
  1738. (
  1739. mkdir -p "$parent" && cd "$parent" &&
  1740. git clone -b "$branch" "$url" "$(basename "$location")"
  1741. ) || return 1
  1742. }
  1743. export -f host_resource_get_git
  1744. host_resource_get_git-sub() {
  1745. local location="$1" cfg="$2" branch parent url
  1746. branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null)
  1747. branch=${branch:-master}
  1748. url=$(echo "$cfg" | shyaml get-value url 2>/dev/null)
  1749. parent="$(dirname "$location")"
  1750. (
  1751. mkdir -p "$parent" && cd "$parent" &&
  1752. git sub clone -b "$branch" "$url" "$(basename "$location")"
  1753. ) || return 1
  1754. }
  1755. export -f host_resource_get_git-sub
  1756. setup_host_resource () {
  1757. local subservice="$1" service_def location get cfg
  1758. service_def=$(get_compose_service_def "$subservice") || return 1
  1759. while read-0 location cfg; do
  1760. ## XXXvlab: will it be a git resources always ?
  1761. if [ -d "$location" -a ! -d "$location/.git" ]; then
  1762. err "Hum, location '$location' does not seem to be a git directory."
  1763. return 1
  1764. fi
  1765. if [ -d "$location" ]; then
  1766. info "host resource '$location' already set up."
  1767. continue
  1768. fi
  1769. get=$(echo "$cfg" | shyaml get-value get 2>/dev/null)
  1770. if [ -z "$get" ]; then
  1771. err "No host directory '$location' found, and no ${WHITE}source$NORMAL" \
  1772. "specified for $DARKYELLOW$subservice$NORMAL."
  1773. return 1
  1774. fi
  1775. host_resource_get "$location" "$get" || return 1
  1776. done < <(echo "$service_def" | shyaml key-values-0 host-resources 2>/dev/null)
  1777. }
  1778. export -f setup_host_resource
  1779. setup_host_resources () {
  1780. local service subservices subservice loaded
  1781. declare -A loaded
  1782. for service in "$@"; do
  1783. subservices=$(get_ordered_service_dependencies "$service") || return 1
  1784. for subservice in $subservices; do
  1785. if [ "${loaded[$subservice]}" ]; then
  1786. ## Prevent double inclusion of same service if this
  1787. ## service is deps of two or more of your
  1788. ## requirements.
  1789. continue
  1790. fi
  1791. setup_host_resource "$subservice" || return 1
  1792. loaded[$subservice]=1
  1793. done
  1794. done
  1795. return 0
  1796. }
  1797. export -f setup_host_resources
  1798. ## Works on stdin
  1799. cfg-get-value () {
  1800. local key="$1" out
  1801. if [ -z "$key" ]; then
  1802. yaml_get_interpret || return 1
  1803. return 0
  1804. fi
  1805. if ! out=$(shyaml -y get-value "$key" 2>/dev/null); then
  1806. err "The key $WHITE$key$NORMAL was not found in relation's data."
  1807. return 1
  1808. fi
  1809. printf "%s\n" "$out" | yaml_get_interpret
  1810. }
  1811. export -f cfg-get-value
  1812. relation-get () {
  1813. cfg-get-value "$1" < "$RELATION_DATA_FILE"
  1814. }
  1815. export -f relation-get
  1816. expand_vars() {
  1817. local unlikely_prefix="UNLIKELY_PREFIX"
  1818. content=$(cat -)
  1819. ## find first identifier not in content
  1820. remaining_lines=$(echo "$content" | grep "^$unlikely_prefix")
  1821. while [ "$(echo "$remaining_lines" | grep "^$unlikely_prefix$")" ]; do
  1822. size_prefix="${#unlikely_prefix}"
  1823. first_matching=$(echo "$remaining_lines" |
  1824. grep -v "^$unlikely_prefix$" |
  1825. uniq -w "$((size_prefix + 1))" -c |
  1826. sort -rn |
  1827. head -n 1)
  1828. first_matching=${first_matching#"${x%%[![:space:]]*}"}
  1829. first_matching="${first_matching#* }"
  1830. next_char=${first_matching:$size_prefix:1}
  1831. if [ "$next_char" != "0" ]; then
  1832. unlikely_prefix+="0"
  1833. else
  1834. unlikely_prefix+="1"
  1835. fi
  1836. remaining_lines=$(echo "$remaining_lines" | grep "^$unlikely_prefix")
  1837. done
  1838. eval "cat <<$unlikely_prefix
  1839. $content
  1840. $unlikely_prefix"
  1841. }
  1842. export -f expand_vars
  1843. yaml_get_interpret() {
  1844. local content tag
  1845. content=$(cat -)
  1846. tag=$(echo "$content" | shyaml get-type) || return 1
  1847. content=$(echo "$content" | shyaml get-value) || return 1
  1848. if ! [ "${tag:0:1}" == "!" ]; then
  1849. echo "$content" || return 1
  1850. return 0
  1851. fi
  1852. case "$tag" in
  1853. "!bash-stdout")
  1854. echo "$content" | bash || {
  1855. err "shell code didn't end with errorlevel 0"
  1856. return 1
  1857. }
  1858. ;;
  1859. "!var-expand")
  1860. echo "$content" | expand_vars || {
  1861. err "shell expansion failed"
  1862. return 1
  1863. }
  1864. ;;
  1865. "!file-content")
  1866. source=$(echo "$content" | expand_vars) || {
  1867. err "shell expansion failed"
  1868. return 1
  1869. }
  1870. cat "$source" || return 1
  1871. ;;
  1872. *)
  1873. err "Invalid object tag ${WHITE}$tag${NORMAL}"
  1874. return 1
  1875. ;;
  1876. esac
  1877. }
  1878. export -f yaml_get_interpret
  1879. options-get () {
  1880. local key="$1" out
  1881. service_def=$(get_compose_service_def "$SERVICE_NAME") || return 1
  1882. if ! out=$(echo "$service_def" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1883. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  1884. return 1
  1885. fi
  1886. echo "$out" | yaml_get_interpret
  1887. }
  1888. export -f options-get
  1889. relation-base-compose-get () {
  1890. local key="$1" out
  1891. if ! out=$(echo "$RELATION_BASE_COMPOSE_DEF" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1892. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  1893. return 1
  1894. fi
  1895. echo "$out" | yaml_get_interpret
  1896. }
  1897. export -f relation-base-compose-get
  1898. relation-target-compose-get () {
  1899. local key="$1" out
  1900. if ! out=$(echo "$RELATION_TARGET_COMPOSE_DEF" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1901. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  1902. return 1
  1903. fi
  1904. echo "$out" | yaml_get_interpret
  1905. }
  1906. export -f relation-target-compose-get
  1907. relation-set () {
  1908. local key="$1" value="$2"
  1909. if [ -z "$RELATION_DATA_FILE" ]; then
  1910. err "$FUNCNAME: relation does not seems to be correctly setup."
  1911. return 1
  1912. fi
  1913. if ! [ -r "$RELATION_DATA_FILE" ]; then
  1914. err "$FUNCNAME: can't read relation's data." >&2
  1915. return 1
  1916. fi
  1917. _config_merge "$RELATION_DATA_FILE" <(yaml_key_val_str "$key" "$value")
  1918. }
  1919. export -f relation-set
  1920. _config_merge() {
  1921. local config_filename="$1" mixin="$2"
  1922. touch "$config_filename" &&
  1923. merge_yaml "$config_filename" "$mixin" > "$config_filename.tmp" || return 1
  1924. mv "$config_filename.tmp" "$config_filename"
  1925. }
  1926. export -f _config_merge
  1927. ## XXXvlab; this can be used only in relation, I'd like to use it in init.
  1928. config-add() {
  1929. local metadata="$1"
  1930. _config_merge "$RELATION_CONFIG" <(echo "$metadata")
  1931. }
  1932. export -f config-add
  1933. ## XXXvlab; this can be used only in relation, I'd like to use it in init.
  1934. init-config-add() {
  1935. local metadata="$1"
  1936. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" \
  1937. <(yaml_key_val_str "services" "$metadata")
  1938. }
  1939. export -f init-config-add
  1940. docker_get_uid() {
  1941. local service="$1" user="$2" uid
  1942. uid=$(cached_cmd_on_base_image "$service" "id -u \"$user\"") || {
  1943. debug "Failed to query for '$user' uid in ${DARKYELLOW}$service${NORMAL} base image."
  1944. return 1
  1945. }
  1946. info "uid from ${DARKYELLOW}$service${NORMAL} for user '$user' is '$uid'"
  1947. echo "$uid"
  1948. }
  1949. export -f docker_get_uid
  1950. docker_get_uid_gid() {
  1951. local service="$1" user="$2" group="$3" uid
  1952. uid_gid=$(cached_cmd_on_base_image "$service" "id -u \"$user\"; id -g \"$group\"") || {
  1953. debug "Failed to query for '$user' uid in ${DARKYELLOW}$service${NORMAL} base image."
  1954. return 1
  1955. }
  1956. info "uid from ${DARKYELLOW}$service${NORMAL} for user '$user' is '$uid_gid'"
  1957. echo "$uid_gid"
  1958. }
  1959. export -f docker_get_uid_gid
  1960. logstdout() {
  1961. local name="$1"
  1962. sed -r 's%^%'"${name}"'> %g'
  1963. }
  1964. export -f logstdout
  1965. logstderr() {
  1966. local name="$1"
  1967. sed -r 's%^(.*)$%'"${RED}${name}>${NORMAL} \1"'%g'
  1968. }
  1969. export -f logstderr
  1970. _run_service_relation () {
  1971. local relation_name="$1" service="$2" target_service="$3" relation_config="$4" relation_dir services
  1972. local errlvl
  1973. charm=$(get_service_charm "$service") || return 1
  1974. target_charm=$(get_service_charm "$target_service") || return 1
  1975. base_script_name=$(charm.has_relation_hook "$charm" "$relation_name" relation-joined) || true
  1976. target_script_name=$(charm.has_relation_hook "$target_charm" "$relation_name" relation-joined) || true
  1977. [ -n "$base_script_name" ] || [ -n "$target_script_name" ] || return 0
  1978. relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1
  1979. RELATION_DATA_FILE=$(get_relation_data_file "$service" "$target_service" "$relation_name" "$relation_config") || return 1
  1980. export BASE_SERVICE_NAME=$service
  1981. export BASE_CHARM_NAME=$charm
  1982. export BASE_CHARM_PATH=$(charm.get_dir "$charm")
  1983. export TARGET_SERVICE_NAME=$target_service
  1984. export TARGET_CHARM_NAME=$target_charm
  1985. export TARGET_CHARM_PATH=$(charm.get_dir "$target_charm")
  1986. export RELATION_DATA_FILE
  1987. target_errlvl=0
  1988. if [ -z "$target_script_name" ]; then
  1989. verb "No relation script $DARKBLUE$relation_name$NORMAL in target $DARKPINK$target_charm$NORMAL."
  1990. else
  1991. verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \
  1992. "for target $DARKYELLOW$target_service$NORMAL (charm $DARKPINK$target_charm$NORMAL)"
  1993. RELATION_CONFIG="$relation_dir/config_provider"
  1994. type=$(get_service_type "$target_service") || return 1
  1995. if [ "$type" != "stub" ]; then
  1996. DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$target_service") || return 1
  1997. fi
  1998. export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA
  1999. {
  2000. (
  2001. SERVICE_NAME=$target_service
  2002. SERVICE_DATASTORE="$DATASTORE/$target_service"
  2003. SERVICE_CONFIGSTORE="$CONFIGSTORE/$target_service"
  2004. export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE
  2005. charm.run_relation_hook local "$target_charm" "$relation_name" relation-joined
  2006. echo "$?" > "$relation_dir/target_errlvl"
  2007. ) | logstdout "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}"
  2008. } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@${NORMAL}" 3>&1 1>&2 2>&3
  2009. target_errlvl="$(cat "$relation_dir/target_errlvl")" || {
  2010. err "Relation script '$script_name' in $DARKPINK$target_charm$NORMAL" \
  2011. "failed before outputing an errorlevel."
  2012. ((target_errlvl |= "1" ))
  2013. }
  2014. if [ -e "$RELATION_CONFIG" ]; then
  2015. debug "Merging some new config info in $DARKYELLOW$target_service$NORMAL"
  2016. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" &&
  2017. rm "$RELATION_CONFIG"
  2018. ((target_errlvl |= "$?"))
  2019. fi
  2020. fi
  2021. if [ "$target_errlvl" == 0 ]; then
  2022. errlvl=0
  2023. if [ "$base_script_name" ]; then
  2024. verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \
  2025. "for $DARKYELLOW$service$NORMAL (charm $DARKPINK$charm$NORMAL)"
  2026. RELATION_CONFIG="$relation_dir/config_providee"
  2027. RELATION_DATA="$(cat "$RELATION_DATA_FILE")"
  2028. DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$service") || return 1
  2029. export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA
  2030. {
  2031. (
  2032. SERVICE_NAME=$service
  2033. SERVICE_DATASTORE="$DATASTORE/$service"
  2034. SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  2035. export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE
  2036. charm.run_relation_hook local "$charm" "$relation_name" relation-joined
  2037. echo "$?" > "$relation_dir/errlvl"
  2038. ) | logstdout "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}"
  2039. } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@$NORMAL" 3>&1 1>&2 2>&3
  2040. errlvl="$(cat "$relation_dir/errlvl")" || {
  2041. err "Relation script '$script_name' in $DARKPINK$charm$NORMAL" \
  2042. "failed before outputing an errorlevel."
  2043. ((errlvl |= "1" ))
  2044. }
  2045. if [ -e "$RELATION_CONFIG" ]; then
  2046. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" &&
  2047. rm "$RELATION_CONFIG"
  2048. ((errlvl |= "$?" ))
  2049. fi
  2050. if [ "$errlvl" != 0 ]; then
  2051. err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$service$NORMAL failed to run properly."
  2052. fi
  2053. else
  2054. verb "No relation script '$script_name' in charm $DARKPINK$charm$NORMAL. Ignoring."
  2055. fi
  2056. else
  2057. err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$target_service$NORMAL failed to run properly."
  2058. fi
  2059. if [ "$target_errlvl" == 0 -a "$errlvl" == 0 ]; then
  2060. debug "Relation $DARKBLUE$relation_name$NORMAL is established" \
  2061. "between $DARKYELLOW$service$NORMAL and $DARKYELLOW$target_service$NORMAL."
  2062. return 0
  2063. else
  2064. return 1
  2065. fi
  2066. }
  2067. export -f _run_service_relation
  2068. _get_compose_relations_cached () {
  2069. local compose_service_def="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  2070. relation_name relation_def target_service
  2071. if [ -e "$cache_file" ]; then
  2072. #debug "$FUNCNAME: STATIC cache hit $1"
  2073. cat "$cache_file" &&
  2074. touch "$cache_file" || return 1
  2075. return 0
  2076. fi
  2077. (
  2078. set -o pipefail
  2079. if [ "$compose_service_def" ]; then
  2080. while read-0 relation_name relation_def; do
  2081. ## XXXvlab: could we use braces here instead of parenthesis ?
  2082. (
  2083. case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in
  2084. "str")
  2085. target_service="$(echo "$relation_def" | shyaml get-value 2>/dev/null)" || return 1
  2086. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  2087. printf "%s\0" "$relation_name" "$target_service" "" "$tech_dep"
  2088. ;;
  2089. "sequence")
  2090. while read-0 target_service; do
  2091. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  2092. printf "%s\0" "$relation_name" "$target_service" "" "$tech_dep"
  2093. done < <(echo "$relation_def" | shyaml get-values-0 2>/dev/null)
  2094. ;;
  2095. "struct")
  2096. while read-0 target_service relation_config; do
  2097. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  2098. printf "%s\0" "$relation_name" "$target_service" "$relation_config" "$tech_dep"
  2099. done < <(echo "$relation_def" | shyaml key-values-0 2>/dev/null)
  2100. ;;
  2101. esac
  2102. ) </dev/null >> "$cache_file" || return 1
  2103. done < <(echo "$compose_service_def" | shyaml key-values-0 relations 2>/dev/null)
  2104. fi
  2105. )
  2106. if [ "$?" != 0 ]; then
  2107. err "Error while looking for compose relations."
  2108. rm -f "$cache_file" ## no cache
  2109. return 1
  2110. fi
  2111. [ -e "$cache_file" ] && cat "$cache_file"
  2112. return 0
  2113. }
  2114. export -f _get_compose_relations_cached
  2115. get_compose_relations () {
  2116. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2117. compose_def
  2118. if [ -e "$cache_file" ]; then
  2119. #debug "$FUNCNAME: SESSION cache hit $1"
  2120. cat "$cache_file"
  2121. return 0
  2122. fi
  2123. compose_def="$(get_compose_service_def "$service")" || return 1
  2124. _get_compose_relations_cached "$compose_def" > "$cache_file"
  2125. if [ "$?" != 0 ]; then
  2126. rm -f "$cache_file" ## no cache
  2127. return 1
  2128. fi
  2129. cat "$cache_file"
  2130. }
  2131. export -f get_compose_relations
  2132. get_all_services() {
  2133. local cache_file="$state_tmpdir/$FUNCNAME.cache.$GLOBAL_ALL_RELATIONS_HASH" \
  2134. s rn ts rc td services service
  2135. if [ -e "$cache_file" ]; then
  2136. #debug "$FUNCNAME: SESSION cache hit $1"
  2137. cat "$cache_file"
  2138. return 0
  2139. fi
  2140. if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
  2141. err-d "Can't access global \$GLOBAL_ALL_RELATIONS"
  2142. return 1
  2143. fi
  2144. declare -A services
  2145. while read-0 s _ ts _ _; do
  2146. for service in "$s" "$ts"; do
  2147. [ "${services[$service]}" ] && continue
  2148. services["$service"]=1
  2149. echo "$service"
  2150. done
  2151. done < "$GLOBAL_ALL_RELATIONS" > "$cache_file"
  2152. cat "$cache_file"
  2153. }
  2154. export -f get_all_services
  2155. get_service_relations () {
  2156. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH")" \
  2157. s rn ts rc td
  2158. if [ -e "$cache_file" ]; then
  2159. #debug "$FUNCNAME: SESSION cache hit $1"
  2160. cat "$cache_file"
  2161. return 0
  2162. fi
  2163. if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
  2164. err-d "Can't access global \$GLOBAL_ALL_RELATIONS"
  2165. return 1
  2166. fi
  2167. while read-0 s rn ts rc td; do
  2168. [[ "$s" == "$service" ]] || continue
  2169. printf "%s\0" "$rn" "$ts" "$rc" "$td"
  2170. done < <(cat "$GLOBAL_ALL_RELATIONS") > "$cache_file"
  2171. cat "$cache_file"
  2172. }
  2173. export -f get_service_relations
  2174. get_service_relation() {
  2175. local service="$1" relation="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  2176. rn ts rc td
  2177. if [ -e "$cache_file" ]; then
  2178. #debug "$FUNCNAME: SESSION cache hit $1"
  2179. cat "$cache_file"
  2180. return 0
  2181. fi
  2182. while read-0-err E rn ts rc td; do
  2183. [ "$relation" == "$rn" ] && {
  2184. printf "%s\0" "$ts" "$rc" "$td"
  2185. break
  2186. }
  2187. done < <(p-err get_service_relations "$service") > "${cache_file}.wip"
  2188. if [ "$?" != 0 ]; then
  2189. return 1
  2190. fi
  2191. if [ "$E" != 0 ]; then
  2192. return 1
  2193. fi
  2194. mv "${cache_file}"{.wip,} || return 1
  2195. cat "$cache_file"
  2196. }
  2197. export -f get_service_relation
  2198. ## From a service and a relation, get all relations targeting given
  2199. ## service with given relation.
  2200. ##
  2201. ## Returns a NUL separated list of couple of:
  2202. ## (base_service, relation_config)
  2203. ##
  2204. get_service_incoming_relations() {
  2205. local service="$1" relation="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH")" \
  2206. s rn ts rc td
  2207. if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
  2208. err-d "Can't access global \$GLOBAL_ALL_RELATIONS"
  2209. return 1
  2210. fi
  2211. if [ -e "$cache_file" ]; then
  2212. #debug "$FUNCNAME: SESSION cache hit $1"
  2213. cat "$cache_file"
  2214. return 0
  2215. fi
  2216. if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
  2217. err "Can't access global \$GLOBAL_ALL_RELATIONS"
  2218. return 1
  2219. fi
  2220. while read-0 s rn ts rc _td; do
  2221. [[ "$ts" == "$service" ]] || continue
  2222. [[ "$rn" == "$relation" ]] || continue
  2223. relation_data_file=$(get_relation_data_file "$s" "$ts" "$rn" "$rc") || return 1
  2224. printf "%s\0" "$s" "$(cat "$relation_data_file")"
  2225. done < "$GLOBAL_ALL_RELATIONS" > "$cache_file"
  2226. cat "$cache_file"
  2227. }
  2228. export -f get_service_incoming_relations
  2229. export TRAVERSE_SEPARATOR=:
  2230. ## Traverse on first service satisfying relation
  2231. service:traverse() {
  2232. local service_path="$1"
  2233. {
  2234. SEPARATOR=:
  2235. read -d "$TRAVERSE_SEPARATOR" service
  2236. while read -d "$TRAVERSE_SEPARATOR" relation; do
  2237. ## XXXvlab: Take only first service
  2238. if ! read-0 ts _ _ < <(get_service_relation "${service}" "${relation}"); then
  2239. err "Couldn't find relation ${DARKCYAN}${relation}${NORMAL}" \
  2240. "from ${DARKYELLOW}$service${NORMAL}."
  2241. return 1
  2242. fi
  2243. service="$ts"
  2244. done
  2245. echo "$service"
  2246. } < <(e "${service_path}${TRAVERSE_SEPARATOR}")
  2247. }
  2248. export -f service:traverse
  2249. service:relation-file() {
  2250. local service_path="$1" relation service relation_file
  2251. if ! [[ "$service_path" == *"$TRAVERSE_SEPARATOR"* ]]; then
  2252. err "Invalid argument '$service_path'." \
  2253. "Must provide a service path (no '${TRAVERSE_SEPARATOR}' found)."
  2254. return 1
  2255. fi
  2256. relation="${service_path##*${TRAVERSE_SEPARATOR}}"
  2257. service=$(service:traverse "${service_path%${TRAVERSE_SEPARATOR}*}") || return 1
  2258. if ! read-0 ts rc _ < <(get_service_relation "${service}" "${relation}"); then
  2259. err "Couldn't find relation ${DARKCYAN}${relation}${NORMAL}" \
  2260. "from ${DARKYELLOW}$service${NORMAL}."
  2261. return 1
  2262. fi
  2263. relation_dir=$(get_relation_data_dir "$service" "$ts" "$relation") || {
  2264. err "Failed to find relation file"
  2265. return 1
  2266. }
  2267. relation_file="$relation_dir/data"
  2268. if ! [ -e "$relation_file" ]; then
  2269. e "$rc" > "$relation_file"
  2270. chmod go-rwx "$relation_file" ## protecting this file
  2271. fi
  2272. echo "$relation_file"
  2273. }
  2274. export -f service:relation-file
  2275. service:relation-options() {
  2276. local service_path="$1" relation_file
  2277. relation_file=$(service:relation-file "$service_path") || {
  2278. err "Failed to find relation file"
  2279. return 1
  2280. }
  2281. cat "$relation_file"
  2282. }
  2283. export -f service:relation-options
  2284. relation:get() {
  2285. local service_path="$1" query="$2" relation_file
  2286. relation_file=$(service:relation-file "$service_path") || {
  2287. err "Failed to find relation file"
  2288. return 1
  2289. }
  2290. cfg-get-value "$query" < "$relation_file"
  2291. }
  2292. export -f relation:get
  2293. _get_charm_metadata_uses() {
  2294. local metadata="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  2295. if [ -e "$cache_file" ]; then
  2296. #debug "$FUNCNAME: SESSION cache hit $1"
  2297. cat "$cache_file" || return 1
  2298. return 0
  2299. fi
  2300. printf "%s" "$metadata" | { shyaml key-values-0 uses 2>/dev/null || true; } | tee "$cache_file"
  2301. }
  2302. export -f _get_charm_metadata_uses
  2303. _get_service_metadata() {
  2304. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2305. charm
  2306. if [ -e "$cache_file" ]; then
  2307. #debug "$FUNCNAME: SESSION cache hit $1"
  2308. cat "$cache_file"
  2309. return 0
  2310. fi
  2311. charm="$(get_service_charm "$service")" || return 1
  2312. charm.metadata "$charm" > "$cache_file"
  2313. if [ "$?" != 0 ]; then
  2314. rm -f "$cache_file" ## no cache
  2315. return 1
  2316. fi
  2317. cat "$cache_file"
  2318. }
  2319. export -f _get_service_metadata
  2320. _get_service_uses() {
  2321. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2322. metadata
  2323. if [ -e "$cache_file" ]; then
  2324. #debug "$FUNCNAME: SESSION cache hit $1"
  2325. cat "$cache_file"
  2326. return 0
  2327. fi
  2328. metadata="$(_get_service_metadata "$service")" || return 1
  2329. _get_charm_metadata_uses "$metadata" > "$cache_file"
  2330. if [ "$?" != 0 ]; then
  2331. rm -f "$cache_file" ## no cache
  2332. return 1
  2333. fi
  2334. cat "$cache_file"
  2335. }
  2336. export -f _get_service_uses
  2337. _get_services_uses() {
  2338. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2339. service rn rd
  2340. if [ -e "$cache_file" ]; then
  2341. #debug "$FUNCNAME: SESSION cache hit $1"
  2342. cat "$cache_file"
  2343. return 0
  2344. fi
  2345. for service in "$@"; do
  2346. _get_service_uses "$service" | while read-0 rn rd; do
  2347. printf "%s\0" "$service" "$rn" "$rd"
  2348. done
  2349. [ "${PIPESTATUS[0]}" == 0 ] || {
  2350. return 1
  2351. }
  2352. done > "${cache_file}.wip"
  2353. mv "${cache_file}"{.wip,} &&
  2354. cat "$cache_file" || return 1
  2355. }
  2356. export -f _get_services_uses
  2357. _get_provides_provides() {
  2358. local provides="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2359. service rn rd
  2360. if [ -e "$cache_file" ]; then
  2361. # debug "$FUNCNAME: CACHEDIR cache hit $1"
  2362. cat "$cache_file"
  2363. return 0
  2364. fi
  2365. type=$(printf "%s" "$provides" | shyaml get-type)
  2366. case "$type" in
  2367. sequence)
  2368. while read-0 prov; do
  2369. printf "%s\0" "$prov" ""
  2370. done < <(echo "$provides" | shyaml get-values-0)
  2371. ;;
  2372. struct)
  2373. printf "%s" "$provides" | shyaml key-values-0
  2374. ;;
  2375. str)
  2376. printf "%s\0" "$(echo "$provides" | shyaml get-value)" ""
  2377. ;;
  2378. *)
  2379. err "Unexpected type '$type' for provider identifier in charm '$charm'."
  2380. return 1
  2381. esac | tee "$cache_file"
  2382. return "${PIPESTATUS[0]}"
  2383. }
  2384. _get_metadata_provides() {
  2385. local metadata="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2386. service rn rd
  2387. if [ -e "$cache_file" ]; then
  2388. #debug "$FUNCNAME: CACHEDIR cache hit"
  2389. cat "$cache_file"
  2390. return 0
  2391. fi
  2392. provides=$(printf "%s" "$metadata" | shyaml -q get-value -y provides "")
  2393. [ "$provides" -a "$provides" != "''" ] || { touch "$cache_file"; return 0; }
  2394. _get_provides_provides "$provides" | tee "$cache_file"
  2395. return "${PIPESTATUS[0]}"
  2396. }
  2397. _get_services_provides() {
  2398. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2399. service rn rd
  2400. if [ -e "$cache_file" ]; then
  2401. #debug "$FUNCNAME: SESSION cache hit $1"
  2402. cat "$cache_file"
  2403. return 0
  2404. fi
  2405. ## YYY: replace the inner loop by a cached function
  2406. for service in "$@"; do
  2407. metadata="$(_get_service_metadata "$service")" || return 1
  2408. while read-0 rn rd; do
  2409. printf "%s\0" "$service" "$rn" "$rd"
  2410. done < <(_get_metadata_provides "$metadata")
  2411. done > "$cache_file"
  2412. if [ "$?" != 0 ]; then
  2413. rm -f "$cache_file" ## no cache
  2414. return 1
  2415. fi
  2416. cat "$cache_file"
  2417. }
  2418. export -f _get_services_provides
  2419. _get_charm_provides() {
  2420. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(charm.store_metadata_hash)" errlvl
  2421. if [ -e "$cache_file" ]; then
  2422. #debug "$FUNCNAME: SESSION cache hit"
  2423. cat "$cache_file"
  2424. return 0
  2425. fi
  2426. start="$SECONDS"
  2427. debug "Getting charm provider list..."
  2428. while read-0 charm _ realpath metadata; do
  2429. metadata="$(charm.metadata "$charm")" || continue
  2430. # echo "reading $charm" >&2
  2431. while read-0 rn rd; do
  2432. printf "%s\0" "$charm" "$rn" "$rd"
  2433. done < <(_get_metadata_provides "$metadata")
  2434. done < <(charm.ls) | tee "$cache_file"
  2435. errlvl="${PIPESTATUS[0]}"
  2436. debug " ..charm provider list done $GRAY(in $((SECONDS - start))s)$NORMAL"
  2437. return "$errlvl"
  2438. }
  2439. _get_charm_providing() {
  2440. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2441. relation="$1"
  2442. if [ -e "$cache_file" ]; then
  2443. #debug "$FUNCNAME: SESSION cache hit $1"
  2444. cat "$cache_file"
  2445. return 0
  2446. fi
  2447. while read-0 charm relation_name relation_def; do
  2448. [ "$relation_name" == "$relation" ] || continue
  2449. printf "%s\0" "$charm" "$relation_def"
  2450. done < <(_get_charm_provides) > "$cache_file"
  2451. if [ "$?" != 0 ]; then
  2452. rm -f "$cache_file" ## no cache
  2453. return 1
  2454. fi
  2455. cat "$cache_file"
  2456. }
  2457. _get_services_providing() {
  2458. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2459. relation="$1"
  2460. shift ## services is "$@"
  2461. if [ -e "$cache_file" ]; then
  2462. #debug "$FUNCNAME: SESSION cache hit $1"
  2463. cat "$cache_file"
  2464. return 0
  2465. fi
  2466. while read-0 service relation_name relation_def; do
  2467. [ "$relation_name" == "$relation" ] || continue
  2468. printf "%s\0" "$service" "$relation_def"
  2469. done < <(_get_services_provides "$@") > "$cache_file"
  2470. if [ "$?" != 0 ]; then
  2471. rm -f "$cache_file" ## no cache
  2472. return 1
  2473. fi
  2474. cat "$cache_file"
  2475. }
  2476. export -f _get_services_provides
  2477. _out_new_relation_from_defs() {
  2478. local service="$1" rn="$2" ts="$3" prov_def="$4" rel_def="$5" rc td rc_prov
  2479. rc_prov=$(printf "%s" "$prov_def" | shyaml -y get-value "default-options" 2>/dev/null)
  2480. ## YYYvlab: should be seen even in no debug mode no ?
  2481. rc=$(printf "%s" "$rel_def" | shyaml -y get-value "default-options" 2>/dev/null)
  2482. td=$(echo "$prov_def" | shyaml get-value 'tech-dep' 2>/dev/null)
  2483. td=${td:-True}
  2484. rc=$(merge_yaml_str "$rc_prov" "$rc") || return 1
  2485. after=$(_out_after_value_from_def "$service" "$rn" "$rel_def") || return 1
  2486. printf "%s\0" "$after" "$service" "$relation_name" "$ts" "$rc" "$td"
  2487. }
  2488. _out_after_value_from_def() {
  2489. local service="$1" relation_name="$2" relation_def="$3" after_t after
  2490. if after_t=$(echo "$relation_def" | shyaml get-type after 2>/dev/null); then
  2491. case "$after_t" in
  2492. sequence)
  2493. after="$(echo "$relation_def" | shyaml get-values after 2>/dev/null)" || return 1
  2494. after=",$service:${after//$'\n'/,$service:},"
  2495. ;;
  2496. struct)
  2497. err "Invalid type for ${WHITE}after${NORMAL}'s value in ${DARKBLUE}$relation_name${NORMAL}'s definition."
  2498. return 1
  2499. ;;
  2500. str)
  2501. after=",$service:$(echo "$relation_def" | shyaml get-value after "" 2>/dev/null)," || return 1
  2502. ;;
  2503. esac
  2504. else
  2505. after=""
  2506. fi
  2507. e "$after"
  2508. }
  2509. get_all_relations () {
  2510. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$(declare -p without_relations)")" \
  2511. services
  2512. if [ -n "$GLOBAL_ALL_RELATIONS" ]; then
  2513. cat "$GLOBAL_ALL_RELATIONS" || return 1
  2514. return 0
  2515. fi
  2516. if [ -e "${cache_file}" ]; then
  2517. #debug "$FUNCNAME: SESSION cache hit $1"
  2518. export GLOBAL_ALL_RELATIONS="$cache_file"
  2519. cat "${cache_file}"
  2520. return 0
  2521. fi
  2522. declare -A services
  2523. services_uses=()
  2524. ## XXXvlab: bwerk, leveraging cache to be able to get the errorlevel here.
  2525. _get_services_uses "$@" >/dev/null || return 1
  2526. array_read-0 services_uses < <(_get_services_uses "$@")
  2527. services_provides=()
  2528. ## XXXvlab: bwerk, leveraging cache to be able to get the errorlevel here.
  2529. _get_services_provides "$@" >/dev/null || return 1
  2530. array_read-0 services_provides < <(_get_services_provides "$@")
  2531. for service in "$@"; do
  2532. services[$service]=1
  2533. done
  2534. all_services=("$@")
  2535. while [ "${#all_services[@]}" != 0 ]; do
  2536. array_pop all_services service
  2537. while read-0-err E relation_name ts relation_config tech_dep; do
  2538. [ "${without_relations[$service:$relation_name]}" ] && {
  2539. debug "Ignoring compose $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> ${DARKYELLOW}$ts$NORMAL"
  2540. continue
  2541. }
  2542. ## First is priority, that can be adjusted in second step
  2543. printf "%s\0" "" "$service" "$relation_name" "$ts" "$relation_config" "$tech_dep"
  2544. ## adding target services ?
  2545. [ "${services[$ts]}" ] && continue
  2546. array_read-0 services_uses < <(_get_services_uses "$ts")
  2547. all_services+=("$ts")
  2548. services[$ts]=1
  2549. done < <(p-err get_compose_relations "$service")
  2550. if [ "$E" != 0 ]; then
  2551. err "Failed to get relations for $DARKYELLOW$service$NORMAL."
  2552. return 1
  2553. fi
  2554. done > "${cache_file}.wip"
  2555. while true; do
  2556. changed=
  2557. new_services_uses=()
  2558. summon=()
  2559. required=()
  2560. recommended=()
  2561. optional=()
  2562. while [ "${#services_uses[@]}" != 0 ]; do
  2563. service="${services_uses[0]}"
  2564. relation_name="${services_uses[1]}"
  2565. relation_def="${services_uses[2]}"
  2566. services_uses=("${services_uses[@]:3}")
  2567. [ "${without_relations[$service:$relation_name]}" ] && {
  2568. debug "Skipping $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW*$NORMAL"
  2569. continue
  2570. }
  2571. default_options=$(printf "%s" "$relation_def" | shyaml -y get-value "default-options" 2>/dev/null)
  2572. after=$(_out_after_value_from_def "$service" "$relation_name" "$relation_def") || return 1
  2573. ## is this "use" declaration satisfied ?
  2574. found=
  2575. while read-0 p s rn ts rc td; do
  2576. if [ -z "$found" -a "$service" == "$s" -a "$relation_name" == "$rn" ]; then
  2577. if [ "$default_options" ]; then
  2578. rc=$(merge_yaml_str "$default_options" "$rc") || return 1
  2579. fi
  2580. found="$ts"
  2581. p="$after"
  2582. fi
  2583. printf "%s\0" "$p" "$s" "$rn" "$ts" "$rc" "$td"
  2584. done < "${cache_file}.wip" > "${cache_file}.wip.new"
  2585. mv "${cache_file}.wip.new" "${cache_file}.wip"
  2586. if [ "$found" ]; then ## this "use" declaration was satisfied
  2587. debug "${DARKYELLOW}$service${NORMAL} use declaration for relation" \
  2588. "${DARKBLUE}$relation_name${NORMAL} is satisfied with ${DARKYELLOW}$found${NORMAL}"
  2589. continue
  2590. fi
  2591. auto=$(echo "$relation_def" | shyaml get-value auto pair 2>/dev/null)
  2592. auto=${auto:-pair}
  2593. case "$auto" in
  2594. "pair"|"summon")
  2595. service_list=()
  2596. array_read-0 service_list < <(array_keys_to_stdin services)
  2597. providers=()
  2598. providers_def=()
  2599. array_read-0 providers providers_def < <(_get_services_providing "$relation_name" "${service_list[@]}")
  2600. if [ "${#providers[@]}" == 1 ]; then
  2601. ts="${providers[0]}"
  2602. debug "Auto-pairs ${DARKYELLOW}$service${NORMAL}" \
  2603. "--${DARKBLUE}$relation_name${NORMAL}--> ${DARKYELLOW}$ts${NORMAL}"
  2604. _out_new_relation_from_defs "$service" "$relation_name" "$ts" \
  2605. "${providers_def[0]}" "$relation_def" \
  2606. >> "${cache_file}.wip" || return 1
  2607. ## Adding service
  2608. [ "${services[$ts]}" ] && continue
  2609. array_read-0 new_services_uses < <(_get_services_uses "$ts")
  2610. services[$ts]=1
  2611. changed=1
  2612. continue
  2613. fi
  2614. if [ "${#providers[@]}" -gt 1 ]; then
  2615. msg=""
  2616. warn "No auto-pairing ${DARKYELLOW}$service${NORMAL}" \
  2617. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  2618. "(> 1 provider)."
  2619. elif [ "$auto" == "summon" ]; then ## no provider
  2620. summon+=("$service" "$relation_name" "$relation_def")
  2621. fi
  2622. ;;
  2623. null|disable|disabled)
  2624. :
  2625. ;;
  2626. *)
  2627. err "Invalid ${WHITE}auto${NORMAL} value '$auto'."
  2628. return 1
  2629. ;;
  2630. esac
  2631. constraint=$(echo "$relation_def" | shyaml get-value constraint 2>/dev/null)
  2632. constraint=${constraint:-optional}
  2633. case "$constraint" in
  2634. "required")
  2635. required+=("$service" "$relation_name" "$relation_def")
  2636. ;;
  2637. "recommended")
  2638. recommended+=("$service" "$relation_name" "$relation_def")
  2639. ;;
  2640. "optional")
  2641. optional+=("$service" "$relation_name" "$relation_def")
  2642. ;;
  2643. *)
  2644. err "Invalid ${WHITE}constraint${NORMAL} value '$constraint'."
  2645. return 1
  2646. ;;
  2647. esac
  2648. new_services_uses+=("$service" "$relation_name" "$relation_def") ## re-queue it
  2649. done
  2650. services_uses=("${new_services_uses[@]}")
  2651. if [ "$changed" ]; then
  2652. continue
  2653. fi
  2654. ## situation is stable
  2655. if [ "${#summon[@]}" != 0 ]; then
  2656. declare -A summon_requeued=()
  2657. while [ "${#summon[@]}" != 0 ]; do
  2658. service="${summon[0]}"
  2659. relation_name="${summon[1]}"
  2660. relation_def="${summon[2]}"
  2661. summon=("${summon[@]:3}")
  2662. providers=()
  2663. providers_def=()
  2664. array_read-0 providers providers_def < <(_get_charm_providing "$relation_name" "${service_list[@]}")
  2665. if [ "${#providers[@]}" == 0 ]; then
  2666. err "Summoning a ${DARKBLUE}$relation_name${NORMAL} provider failed: none were found in charm store."
  2667. return 1
  2668. fi
  2669. if [ "${#providers[@]}" -gt 1 ]; then
  2670. if [ -z "${summon_requeued[$service/$relation_name]}" ]; then
  2671. debug "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  2672. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  2673. "(> 1 provider). Requeing."
  2674. summon+=("$service" "$relation_name" "$relation_def") ## re-queue it
  2675. summon_requeued["$service/$relation_name"]=1
  2676. continue
  2677. else
  2678. warn "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  2679. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  2680. "(> 1 provider). Choosing first."
  2681. fi
  2682. fi
  2683. ts="${providers[0]}"
  2684. ## YYYvlab: should be seen even in no debug mode no ?
  2685. debug "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  2686. "--${DARKBLUE}$relation_name${NORMAL}--> ${DARKYELLOW}$ts${NORMAL}"
  2687. _out_new_relation_from_defs "$service" "$relation_name" "$ts" \
  2688. "${providers_def[0]}" "$relation_def" \
  2689. >> "${cache_file}.wip" || return 1
  2690. ## Adding service
  2691. [ "${services[$ts]}" ] && continue
  2692. array_read-0 services_uses < <(_get_services_uses "$ts")
  2693. services[$ts]=1
  2694. changed=1
  2695. continue 2
  2696. done
  2697. continue
  2698. fi
  2699. [ "$NO_CONSTRAINT_CHECK" ] && break
  2700. if [ "${#required[@]}" != 0 ]; then
  2701. echo "$(_display_solves required)" | sed -r "s/^/${RED}||${NORMAL} /g" >&2
  2702. err "Required relations not satisfied"
  2703. return 1
  2704. fi
  2705. if [ "${#recommended[@]}" != 0 ]; then
  2706. ## make recommendation
  2707. echo "$(_display_solves recommended)" | sed -r "s/^/${YELLOW}||${NORMAL} /g" >&2
  2708. fi
  2709. if [ -z "$QUIET" ]; then
  2710. if [ "${#optional[@]}" != 0 ]; then
  2711. ## inform about options
  2712. echo "$(_display_solves optional)" | sed -r "s/^/${BLUE}||${NORMAL} /g" >&2
  2713. fi
  2714. fi
  2715. # if [ "${#required[@]}" != 0 ]; then
  2716. # err "Required relations not satisfied"
  2717. # return 1
  2718. # fi
  2719. if [ "${#recommended[@]}" != 0 ]; then
  2720. warn "Recommended relations not satisfied"
  2721. fi
  2722. break
  2723. done
  2724. if [ "$?" != 0 ]; then
  2725. rm -f "${cache_file}"{,.wip,.wip.new} ## no cache
  2726. return 1
  2727. fi
  2728. ##
  2729. ## Sort relations thanks to uses =metadata.yml= relations.
  2730. ##
  2731. mv "${cache_file}.wip"{,.in} &&
  2732. rm -f "${cache_file}.wip.final" &&
  2733. touch "${cache_file}.wip.final" || {
  2734. err "Unexpected error when mangling cache files."
  2735. return 1
  2736. }
  2737. declare -A relation_done=()
  2738. while true; do
  2739. had_remaining_relation=
  2740. had_new_relation=
  2741. while read-0 p s rn ts rc td; do
  2742. if [ -z "$p" ] || [ "$p" == "," ]; then
  2743. relation_done["$s:$rn"]=1
  2744. # printf " .. %-30s %-30s %-30s\n" "--" "$s" "$rn" >&2
  2745. printf "%s\0" "$s" "$rn" "$ts" "$rc" "$td" >> "${cache_file}.wip.final"
  2746. had_new_relation=1
  2747. else
  2748. # printf " !! %-30s %-30s %-30s\n" "$p" "$s" "$rn" >&2
  2749. printf "%s\0" "$p" "$s" "$rn" "$ts" "$rc" "$td" >> "${cache_file}.wip.out"
  2750. had_remaining_relation=1
  2751. fi
  2752. done < "${cache_file}.wip.in"
  2753. [ -z "$had_remaining_relation" ] && break
  2754. mv "${cache_file}.wip."{out,in}
  2755. while read-0 p s rn ts rc td; do
  2756. for rel in "${!relation_done[@]}"; do
  2757. p="${p//,$rel,/,}"
  2758. done
  2759. # printf " CC %-30s %-30s %-30s\n" "$p" "$s" "$rn" >&2
  2760. if [ -z "$had_new_relation" ]; then
  2761. err "${DARKYELLOW}$s${NORMAL} --${DARKBLUE}$rn${NORMAL}--> ${DARKYELLOW}$ts${NORMAL} missing required ${WHITE}after${NORMAL} relations:"
  2762. for rel in ${p//,/ }; do
  2763. rel_s=${rel%%:*}
  2764. rel_r=${rel##*:}
  2765. echo " - ${DARKYELLOW}$rel_s${NORMAL} --${DARKBLUE}$rel_r${NORMAL}--> ${DARKGRAY}*${NORMAL}" >&2
  2766. done
  2767. else
  2768. printf "%s\0" "$p" "$s" "$rn" "$ts" "$rc" "$td" >> "${cache_file}.wip.out"
  2769. fi
  2770. done < "${cache_file}.wip.in"
  2771. if [ -z "$had_new_relation" ]; then
  2772. rm -f "${cache_file}"{,.wip{,new,in,out,final}} ## no cache
  2773. return 1
  2774. fi
  2775. mv "${cache_file}.wip."{out,in}
  2776. done
  2777. mv "${cache_file}"{.wip.final,} || return 1
  2778. export GLOBAL_ALL_RELATIONS="$cache_file"
  2779. GLOBAL_ALL_RELATIONS_HASH=$(hash_get < "$cache_file") || return 1
  2780. export GLOBAL_ALL_RELATIONS_HASH
  2781. cat "$cache_file"
  2782. }
  2783. export -f get_all_relations
  2784. _display_solves() {
  2785. local array_name="$1" by_relation msg
  2786. ## inform about options
  2787. msg=""
  2788. declare -A by_relation
  2789. while read-0 service relation_name relation_def; do
  2790. solves=$(printf "%s" "$relation_def" | shyaml -y get-value solves 2>/dev/null);
  2791. auto=$(printf "%s" "$relation_def" | shyaml get-value auto 2>/dev/null);
  2792. if [ -z "$solves" ]; then
  2793. continue
  2794. fi
  2795. by_relation[$relation_name]+=$(printf "\n %s" "${DARKYELLOW}$service$NORMAL for:")
  2796. if [ "$auto" == "pair" ]; then
  2797. requirement="add provider in cluster to auto-pair"
  2798. else
  2799. requirement="add explicit relation"
  2800. fi
  2801. while read-0 name def; do
  2802. by_relation[$relation_name]+=$(printf "\n - ${DARKCYAN}%-15s${NORMAL} %s (%s)" "$name" "$def" "$requirement")
  2803. done < <(printf "%s" "$solves" | shyaml key-values-0)
  2804. done < <(array_values_to_stdin "$array_name")
  2805. while read-0 relation_name message; do
  2806. msg+="$(printf "\n${DARKBLUE}%s$NORMAL provider is $array_name by%s" \
  2807. "$relation_name" "$message" )"
  2808. done < <(array_kv_to_stdin by_relation)
  2809. if [ "$msg" ]; then
  2810. printf "%s\n" "${msg:1}"
  2811. fi
  2812. }
  2813. get_compose_relation_def() {
  2814. local service="$1" relation="$2" relation_name target_service relation_config tech_dep
  2815. while read-0 relation_name target_service relation_config tech_dep; do
  2816. [ "$relation_name" == "$relation" ] || continue
  2817. printf "%s\0%s\0%s\0" "$target_service" "$relation_config" "$tech_dep"
  2818. done < <(get_compose_relations "$service") || return 1
  2819. }
  2820. export -f get_compose_relation_def
  2821. run_service_relations () {
  2822. local service services loaded subservices subservice
  2823. PROJECT_NAME=$(get_default_project_name) || return 1
  2824. export PROJECT_NAME
  2825. declare -A loaded
  2826. subservices=$(get_ordered_service_dependencies "$@") || return 1
  2827. for service in $subservices; do
  2828. # debug "Upping dep's relations of ${DARKYELLOW}$service${NORMAL}:"
  2829. for subservice in $(get_service_deps "$service") "$service"; do
  2830. [ "${loaded[$subservice]}" ] && continue
  2831. export BASE_SERVICE_NAME=$service
  2832. MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
  2833. MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1
  2834. RELATION_BASE_COMPOSE_DEF=$(get_compose_service_def "$subservice") || return 1
  2835. export RELATION_BASE_COMPOSE_DEF MASTER_BASE_{CHARM,SERVICE}_NAME
  2836. # debug " Relations of ${DARKYELLOW}$subservice${NORMAL}:"
  2837. while read-0 relation_name target_service relation_config tech_dep; do
  2838. [ "${without_relations[$service:$relation_name]}" ] && {
  2839. debug "Skipping $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW*$NORMAL"
  2840. continue
  2841. }
  2842. export relation_config
  2843. export TARGET_SERVICE_NAME=$target_service
  2844. MASTER_TARGET_SERVICE_NAME=$(get_top_master_service_for_service "$target_service") || return 1
  2845. MASTER_TARGET_CHARM_NAME=$(get_service_charm "$MASTER_TARGET_SERVICE_NAME") || return 1
  2846. RELATION_TARGET_COMPOSE_DEF=$(get_compose_service_def "$target_service") || return 1
  2847. export RELATION_TARGET_COMPOSE_DEF MASTER_TARGET_{CHARM,SERVICE}_NAME
  2848. Wrap "${wrap_opts[@]}" -d "building $DARKYELLOW$subservice$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW$target_service$NORMAL" <<EOF || return 1
  2849. _run_service_relation "$relation_name" "$subservice" "$target_service" "\$relation_config"
  2850. EOF
  2851. done < <(get_service_relations "$subservice") || return 1
  2852. loaded[$subservice]=1
  2853. done
  2854. done
  2855. }
  2856. export -f run_service_relations
  2857. _run_service_action_direct() {
  2858. local service="$1" action="$2" charm _dummy project_name
  2859. shift; shift
  2860. read-0 charm action_script_path || true ## against 'set -e' that could be setup in parent scripts
  2861. if read-0 _dummy || [ "$_dummy" ]; then
  2862. print_syntax_error "$FUNCNAME: too many arguments in action descriptor"
  2863. return 1
  2864. fi
  2865. project_name=$(get_default_project_name) || return 1
  2866. export PROJECT_NAME="$project_name"
  2867. export state_tmpdir
  2868. (
  2869. set +e ## Prevents unwanted leaks from parent shell
  2870. export COMPOSE_CONFIG=$(get_compose_yml_content)
  2871. export METADATA_CONFIG=$(charm.metadata "$charm")
  2872. export SERVICE_NAME=$service
  2873. export ACTION_NAME=$action
  2874. export ACTION_SCRIPT_PATH="$action_script_path"
  2875. export CONTAINER_NAME=$(get_top_master_service_for_service "$service")
  2876. export DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$CONTAINER_NAME")
  2877. export SERVICE_DATASTORE="$DATASTORE/$service"
  2878. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  2879. exname="$exname $ACTION_NAME $SERVICE_NAME" \
  2880. stdbuf -oL -eL bash -c 'charm.run_direct_action "$@"' -- "$charm" "$action" "$@"
  2881. ) 0<&6 ## inject general stdin
  2882. }
  2883. export -f _run_service_action_direct
  2884. _run_service_action_relation() {
  2885. local service="$1" action="$2" charm target_charm relation_name relation_config _dummy
  2886. shift; shift
  2887. read-0 charm target_service target_charm relation_name relation_config action_script_path || true
  2888. if read-0 _dummy || [ "$_dummy" ]; then
  2889. print_syntax_error "$FUNCNAME: too many arguments in action descriptor"
  2890. return 1
  2891. fi
  2892. export action_errlvl_file="$state_tmpdir/action-$service-$charm-$action-errlvl"
  2893. export state_tmpdir
  2894. (
  2895. set +e ## Prevents unwanted leaks from parent shell
  2896. export METADATA_CONFIG=$(charm.metadata "$charm")
  2897. export SERVICE_NAME=$service
  2898. export RELATION_TARGET_SERVICE="$target_service"
  2899. export RELATION_TARGET_CHARM="$target_charm"
  2900. export RELATION_BASE_SERVICE="$service"
  2901. export RELATION_BASE_CHARM="$charm"
  2902. export ACTION_NAME=$action
  2903. export ACTION_SCRIPT_PATH="$action_script_path"
  2904. export CONTAINER_NAME=$(get_top_master_service_for_service "$service")
  2905. export DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$CONTAINER_NAME")
  2906. export SERVICE_DATASTORE="$DATASTORE/$service"
  2907. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  2908. exname="$exname $ACTION_NAME $SERVICE_NAME" \
  2909. stdbuf -oL -eL bash -c 'charm.run_relation_action "$@"' -- "$target_charm" "$relation_name" "$action" "$@"
  2910. ) 0<&6 ## inject general stdin
  2911. }
  2912. export -f _run_service_action_relation
  2913. get_relation_data_dir() {
  2914. local service="$1" target_service="$2" relation_name="$3" \
  2915. cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  2916. if [ -e "$cache_file" ]; then
  2917. # debug "$FUNCNAME: cache hit ($*)"
  2918. cat "$cache_file"
  2919. return 0
  2920. fi
  2921. local project relation_dir
  2922. project=${PROJECT_NAME}
  2923. if [ -z "$project" ]; then
  2924. project=$(get_default_project_name) || return 1
  2925. fi
  2926. relation_dir="$VARDIR/relations/$project/${service}-${target_service}/$relation_name"
  2927. if ! [ -d "$relation_dir" ]; then
  2928. mkdir -p "$relation_dir" || return 1
  2929. chmod go-rwx "$relation_dir" || return 1 ## protecting this directory
  2930. fi
  2931. echo "$relation_dir" | tee "$cache_file"
  2932. }
  2933. export -f get_relation_data_dir
  2934. get_relation_data_file() {
  2935. local service="$1" target_service="$2" relation_name="$3" relation_config="$4" \
  2936. new new_md5 relation_dir relation_data_file
  2937. relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1
  2938. relation_data_file="$relation_dir/data"
  2939. new=
  2940. if [ -e "$relation_data_file" ]; then
  2941. ## Has reference changed ?
  2942. new_md5=$(e "$relation_config" | md5_compat)
  2943. if [ "$new_md5" != "$(cat "$relation_data_file.md5_ref" 2>/dev/null)" ]; then
  2944. new=true
  2945. fi
  2946. else
  2947. new=true
  2948. fi
  2949. if [ -n "$new" ]; then
  2950. OLDUMASK=$(umask)
  2951. umask 0077
  2952. e "$relation_config" > "$relation_data_file"
  2953. umask "$OLDUMASK"
  2954. e "$relation_config" | md5_compat > "$relation_data_file.md5_ref"
  2955. fi
  2956. echo "$relation_data_file"
  2957. }
  2958. export -f get_relation_data_file
  2959. has_service_action () {
  2960. local service="$1" action="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2961. charm target_charm relation_name target_service relation_config _tech_dep \
  2962. path
  2963. if [ -e "$cache_file" ]; then
  2964. # debug "$FUNCNAME: cache hit ($*)"
  2965. cat "$cache_file"
  2966. return 0
  2967. fi
  2968. charm=$(get_service_charm "$service") || return 1
  2969. ## Action directly provided ?
  2970. if path=$(charm.has_direct_action "$charm" "$action"); then
  2971. p0 "direct" "$charm" "$path" | tee "$cache_file"
  2972. return 0
  2973. fi
  2974. ## Action provided by relation ?
  2975. while read-0 relation_name target_service relation_config _tech_dep; do
  2976. target_charm=$(get_service_charm "$target_service") || return 1
  2977. if path=$(charm.has_relation_action "$target_charm" "$relation_name" "$action"); then
  2978. p0 "relation" "$charm" "$target_service" "$target_charm" "$relation_name" "$relation_config" "$path" | tee "$cache_file"
  2979. return 0
  2980. fi
  2981. done < <(get_service_relations "$service")
  2982. return 1
  2983. # master=$(get_top_master_service_for_service "$service")
  2984. # [ "$master" == "$charm" ] && return 1
  2985. # has_service_action "$master" "$action"
  2986. }
  2987. export -f has_service_action
  2988. run_service_action () {
  2989. local service="$1" action="$2" errlvl
  2990. shift ; shift
  2991. exec 6<&0 ## saving stdin
  2992. {
  2993. if ! read-0 action_type; then
  2994. info "Service $DARKYELLOW$service$NORMAL does not have any action $DARKCYAN$action$NORMAL defined."
  2995. info " Add an executable script to 'actions/$action' to implement action."
  2996. return 1
  2997. fi
  2998. "_run_service_action_${action_type}" "$service" "$action" "$@"
  2999. errlvl="$?"
  3000. } < <(has_service_action "$service" "$action")
  3001. exec 0<&6 6<&- ## restoring stdin
  3002. return "$errlvl"
  3003. }
  3004. export -f run_service_action
  3005. get_compose_relation_config() {
  3006. local service=$1 relation_config cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  3007. if [ -e "$cache_file" ]; then
  3008. # debug "$FUNCNAME: cache hit ($*)"
  3009. cat "$cache_file"
  3010. return 0
  3011. fi
  3012. compose_service_def=$(get_compose_service_def "$service") || return 1
  3013. echo "$compose_service_def" | shyaml get-value "relations" 2>/dev/null | tee "$cache_file"
  3014. }
  3015. export -f get_compose_relation_config
  3016. # ## Return key-values-0
  3017. # get_compose_relation_config_for_service() {
  3018. # local service=$1 relation_name=$2 relation_config
  3019. # compose_service_relations=$(get_compose_relation_config "$service") || return 1
  3020. # if ! relation_config=$(
  3021. # echo "$compose_service_relations" |
  3022. # shyaml get-value "${relation_name}" 2>/dev/null); then
  3023. # err "Couldn't find $DARKYELLOW${service}$NORMAL/${WHITE}${relation_name}$NORMAL" \
  3024. # "relation config in compose configuration."
  3025. # return 1
  3026. # fi
  3027. # if [ -z "$relation_config" ]; then
  3028. # err "Relation ${WHITE}mysql-database$NORMAL is empty in compose configuration."
  3029. # return 1
  3030. # fi
  3031. # if ! echo "$relation_config" | shyaml key-values-0 2>/dev/null; then
  3032. # err "No key/values in ${DARKBLUE}mysql-database$NORMAL of compose config."
  3033. # return 1
  3034. # fi
  3035. # }
  3036. # export -f get_compose_relation_config_for_service
  3037. _get_container_relation() {
  3038. local metadata=$1 found relation_name relation_def
  3039. found=
  3040. while read-0 relation_name relation_def; do
  3041. [ "$(echo "$relation_def" | shyaml get-value "scope" 2>/dev/null)" == "container" ] && {
  3042. found="$relation_name"
  3043. break
  3044. }
  3045. done < <(_get_charm_metadata_uses "$metadata")
  3046. if [ -z "$found" ]; then
  3047. err "Charm $DARKPINK$charm$NORMAL is a subordinate but does not have any required relation declaration with" \
  3048. "${WHITE}scope${NORMAL} set to 'container'."
  3049. return 1
  3050. fi
  3051. printf "%s" "$found"
  3052. }
  3053. _get_master_service_for_service_cached () {
  3054. local service="$1" charm="$2" metadata="$3" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  3055. charm requires master_charm target_charm target_service service_def found
  3056. if [ -e "$cache_file" ]; then
  3057. # debug "$FUNCNAME: STATIC cache hit ($1)"
  3058. cat "$cache_file" &&
  3059. touch "$cache_file" || return 1
  3060. return 0
  3061. fi
  3062. if ! [[ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" =~ ^True|true$ ]]; then
  3063. ## just return service name
  3064. echo "$service" | tee "$cache_file"
  3065. return 0
  3066. fi
  3067. ## Action provided by relation ?
  3068. container_relation=$(_get_container_relation "$metadata") || return 1
  3069. read-0 target_service _ _ < <(get_service_relation "$service" "$container_relation")
  3070. if [ -z "$target_service" ]; then
  3071. err "Couldn't find ${WHITE}relations.${container_relation}${NORMAL} in" \
  3072. "${DARKYELLOW}$service$NORMAL compose definition."
  3073. err ${FUNCNAME[@]}
  3074. return 1
  3075. fi
  3076. echo "$target_service" | tee "$cache_file"
  3077. }
  3078. export -f _get_master_service_for_service_cached
  3079. get_master_service_for_service() {
  3080. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  3081. charm metadata result
  3082. if [ -e "$cache_file" ]; then
  3083. # debug "$FUNCNAME: SESSION cache hit ($*)"
  3084. cat "$cache_file" || return 1
  3085. return 0
  3086. fi
  3087. charm=$(get_service_charm "$service") || return 1
  3088. metadata=$(charm.metadata "$charm" 2>/dev/null) || {
  3089. metadata=""
  3090. warn "No charm $DARKPINK$charm$NORMAL found."
  3091. }
  3092. result=$(_get_master_service_for_service_cached "$service" "$charm" "$metadata") || return 1
  3093. echo "$result" | tee "$cache_file" || return 1
  3094. }
  3095. export -f get_master_service_for_service
  3096. get_top_master_service_for_service() {
  3097. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  3098. current_service
  3099. if [ -e "$cache_file" ]; then
  3100. # debug "$FUNCNAME: cache hit ($*)"
  3101. cat "$cache_file"
  3102. return 0
  3103. fi
  3104. current_service="$service"
  3105. while true; do
  3106. master_service=$(get_master_service_for_service "$current_service") || return 1
  3107. [ "$master_service" == "$current_service" ] && break
  3108. current_service="$master_service"
  3109. done
  3110. echo "$current_service" | tee "$cache_file"
  3111. return 0
  3112. }
  3113. export -f get_top_master_service_for_service
  3114. ##
  3115. ## The result is a mixin that is not always a complete valid
  3116. ## docker-compose entry (thinking of subordinates). The result
  3117. ## will be merge with master charms.
  3118. _get_docker_compose_mixin_from_metadata_cached() {
  3119. local service="$1" charm="$2" metadata="$3" \
  3120. has_build_dir="$4" \
  3121. cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  3122. metadata_file metadata volumes docker_compose subordinate image \
  3123. mixin mixins tmemory memory limit docker_memory
  3124. if [ -e "$cache_file" ]; then
  3125. #debug "$FUNCNAME: STATIC cache hit $1"
  3126. cat "$cache_file" &&
  3127. touch "$cache_file" || return 1
  3128. return 0
  3129. fi
  3130. mixins=("$(echo -en "labels:\n- compose.charm=$charm")")
  3131. if [ "$metadata" ]; then
  3132. ## resources to volumes
  3133. volumes=$(
  3134. for resource_type in data config; do
  3135. while read-0 resource; do
  3136. eval "echo \" - \$HOST_${resource_type^^}STORE/\$service\$resource:\$resource:rw\""
  3137. done < <(echo "$metadata" | shyaml get-values-0 "${resource_type}-resources" 2>/dev/null)
  3138. done
  3139. while read-0 resource; do
  3140. if [[ "$resource" == /*:/*:* ]]; then
  3141. echo " - $resource"
  3142. elif [[ "$resource" == /*:/* ]]; then
  3143. echo " - $resource:rw"
  3144. elif [[ "$resource" == /*:* ]]; then
  3145. echo " - ${resource%%:*}:$resource"
  3146. elif [[ "$resource" =~ ^/[^:]+$ ]]; then
  3147. echo " - $resource:$resource:rw"
  3148. else
  3149. die "Invalid host-resource specified in 'metadata.yml'."
  3150. fi
  3151. done < <(printf "%s" "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null)
  3152. while read-0 resource; do
  3153. dest="$(charm.get_dir "$charm")/resources$resource"
  3154. if ! [ -e "$dest" ]; then
  3155. die "charm-resource: '$resource' does not exist (file: '$dest')."
  3156. fi
  3157. echo " - $dest:$resource:ro"
  3158. done < <(echo "$metadata" | shyaml get-values-0 "charm-resources" 2>/dev/null)
  3159. ) || return 1
  3160. if [ "$volumes" ]; then
  3161. mixins+=("volumes:"$'\n'"$volumes")
  3162. fi
  3163. type="$(printf "%s" "$metadata" | shyaml get-value type 2>/dev/null)" || true
  3164. if [ "$type" != "run-once" ]; then
  3165. mixins+=("restart: unless-stopped")
  3166. fi
  3167. docker_compose=$(printf "%s" "$metadata" | shyaml get-value -y "docker-compose" 2>/dev/null) || true
  3168. if [ "$docker_compose" ]; then
  3169. mixins+=("$docker_compose")
  3170. fi
  3171. if [[ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" =~ ^True|true$ ]]; then
  3172. subordinate=true
  3173. fi
  3174. fi
  3175. image=$(echo "$metadata" | shyaml get-value "docker-image" 2>/dev/null)
  3176. [ "$image" == "None" ] && image=""
  3177. if [ -n "$image" ]; then
  3178. if [ -n "$subordinate" ]; then
  3179. err "Subordinate charm can not have a ${WHITE}docker-image${NORMAL} value."
  3180. return 1
  3181. fi
  3182. mixins+=("image: $image")
  3183. elif [ "$has_build_dir" ]; then
  3184. if [ "$subordinate" ]; then
  3185. err "Subordinate charm can not have a 'build' sub directory."
  3186. return 1
  3187. fi
  3188. mixins+=("build: $(charm.get_dir "$charm")/build")
  3189. fi
  3190. limit=$(e "$metadata" | yq '.limit' 2>/dev/null) || return 1
  3191. [ "$limit" == "null" ] && limit=""
  3192. if [ -n "$limit" ]; then
  3193. if ! read-0-err E tmemory memory < <(e "$limit" | wyq ".memory | type, .memory") ||
  3194. [ "$E" != 0 ]; then
  3195. err "Unexpected error in ${DARKPINK}$charm${NORMAL}'s metadata when parsing ${WHITE}.limit${NORMAL}"
  3196. return 1
  3197. fi
  3198. case "$tmemory" in
  3199. '!!str'|'!!int')
  3200. docker_memory="$(e "$memory" | numfmt --from iec)" || {
  3201. err "Invalid format specified for .limit.memory: '$memory'."
  3202. return 1
  3203. }
  3204. ;;
  3205. '!!float')
  3206. err "Unexpected value in ${DARKPINK}$charm${NORMAL}'s metadata for ${WHITE}.limit.memory${NORMAL}."
  3207. echo " You need to specify a unit (like 'K', 'M', 'G' ...)." >&2
  3208. return 1
  3209. ;;
  3210. '!!null')
  3211. :
  3212. ;;
  3213. *)
  3214. err "Unexpected type '${tmemory#!!}' in ${DARKPINK}$charm${NORMAL}'s metadata" \
  3215. "for ${WHITE}.limit.memory${NORMAL}."
  3216. echo " You need to check ${DARKPINK}$charm${NORMAL}'s metadata " \
  3217. "for ${WHITE}.limit.memory${NORMAL} and provide a valid value" >&2
  3218. echo " Example values: '1.5G', '252M', ..." >&2
  3219. return 1
  3220. ;;
  3221. esac
  3222. if [ -n "$docker_memory" ]; then
  3223. if [[ "$docker_memory" -lt 6291456 ]]; then
  3224. err "Can't limit service to lower than 6M."
  3225. echo " Specified limit of $memory (=$docker_memory) is lower than docker's min limit of 6M (=6291456)." >&2
  3226. echo " The provided limit to memory is lower than minimum memory for a container." >&2
  3227. echo " Please remove memory limit in ${DARKPINK}$charm${NORMAL}'s metadata or raise it." >&2
  3228. return 1
  3229. fi
  3230. mixins+=(
  3231. "mem_limit: $docker_memory"
  3232. "memswap_limit: $docker_memory"
  3233. )
  3234. fi
  3235. fi
  3236. ## Final merging
  3237. mixin=$(merge_yaml_str "${mixins[@]}") || {
  3238. err "Failed to merge mixins from ${DARKPINK}${charm}${NORMAL} metadata."
  3239. return 1
  3240. }
  3241. echo "$mixin" | tee "$cache_file"
  3242. }
  3243. export -f _get_docker_compose_mixin_from_metadata_cached
  3244. get_docker_compose_mixin_from_metadata() {
  3245. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  3246. if [ -e "$cache_file" ]; then
  3247. #debug "$FUNCNAME: SESSION cache hit ($*)"
  3248. cat "$cache_file"
  3249. return 0
  3250. fi
  3251. charm=$(get_service_charm "$service") || return 1
  3252. metadata="$(charm.metadata "$charm" 2>/dev/null)" || return 1
  3253. has_build_dir=
  3254. [ -d "$(charm.get_dir "$charm")/build" ] && has_build_dir=true
  3255. mixin=$(_get_docker_compose_mixin_from_metadata_cached "$service" "$charm" "$metadata" "$has_build_dir") || return 1
  3256. echo "$mixin" | tee "$cache_file"
  3257. }
  3258. export -f get_docker_compose_mixin_from_metadata
  3259. _save() {
  3260. local name="$1"
  3261. cat - | tee -a "$docker_compose_dir/.data/$name"
  3262. }
  3263. export -f _save
  3264. get_default_project_name() {
  3265. if [ -n "$DEFAULT_PROJECT_NAME" ]; then
  3266. echo "$DEFAULT_PROJECT_NAME"
  3267. return 0
  3268. fi
  3269. local normalized_path compose_yml_location name
  3270. compose_yml_location="$(get_compose_yml_location)" || return 1
  3271. if [ -n "$compose_yml_location" ]; then
  3272. if normalized_path=$(readlink -f "$compose_yml_location"); then
  3273. name="${normalized_path%/*}" ## dirname
  3274. name="${name##*/}" ## basename
  3275. name="${name%%-deploy}" ## remove any '-deploy'
  3276. name="${name,,}" ## lowercase
  3277. e "$name"
  3278. return 0
  3279. fi
  3280. fi
  3281. echo "orphan"
  3282. return 0
  3283. }
  3284. export -f get_default_project_name
  3285. get_running_compose_containers() {
  3286. ## XXXvlab: docker bug: there will be a final newline anyway
  3287. docker ps --filter label="compose.service" --format='{{.ID}}'
  3288. }
  3289. export -f get_running_compose_containers
  3290. get_healthy_container_ip_for_service () {
  3291. local service="$1" port="$2" timeout=${3:-60}
  3292. local containers container container_network container_ip
  3293. containers="$(get_running_containers_for_service "$service")"
  3294. if [ -z "$containers" ]; then
  3295. err "No containers running for service $DARKYELLOW$service$NORMAL."
  3296. return 1
  3297. fi
  3298. ## XXXvlab: taking first container is probably not a good idea
  3299. container="$(echo "$containers" | head -n 1)"
  3300. ## XXXvlab: taking first ip is probably not a good idea
  3301. read-0 container_network container_ip < <(get_container_network_ip "$container")
  3302. if [ -z "$container_ip" ]; then
  3303. err "Can't get container's IP. You should check health of" \
  3304. "${DARKYELLOW}$service${NORMAL}'s container."
  3305. return 1
  3306. fi
  3307. wait_for_tcp_port "$container_network" "$container_ip:$port" "$timeout" || {
  3308. err "TCP port of ${DARKYELLOW}$service${NORMAL}'s container doesn't seem open"
  3309. echo " Please check that container is healthy. Here are last logs:" >&2
  3310. docker logs "$container" --tail=10 | prefix " ${GRAY}|${NORMAL} " >&2
  3311. return 1
  3312. }
  3313. info "Host/Port ${container_ip}:${port} checked ${GREEN}open${NORMAL}."
  3314. echo "$container_network:$container_ip"
  3315. }
  3316. export -f get_healthy_container_ip_for_service
  3317. switch_to_relation_service() {
  3318. local relation="$1"
  3319. ## XXXvlab: can't get real config here
  3320. if ! read-0 ts _ _ < <(get_service_relation "$SERVICE_NAME" "$relation"); then
  3321. err "Couldn't find relation ${DARKCYAN}$relation${NORMAL}."
  3322. return 1
  3323. fi
  3324. export SERVICE_NAME="$ts"
  3325. export SERVICE_DATASTORE="$DATASTORE/$SERVICE_NAME"
  3326. DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$SERVICE_NAME")
  3327. export DOCKER_BASE_IMAGE
  3328. target_charm=$(get_service_charm "$ts") || return 1
  3329. target_charm_path=$(charm.get_dir "$target_charm") || return 1
  3330. cd "$target_charm_path"
  3331. }
  3332. export -f switch_to_relation_service
  3333. get_volumes_for_container() {
  3334. local container="$1"
  3335. docker inspect \
  3336. --format '{{range $mount := .Mounts}}{{$mount.Source}}{{"\x00"}}{{$mount.Destination}}{{"\x00"}}{{end}}' \
  3337. "$container"
  3338. }
  3339. export -f get_volumes_for_container
  3340. is_volume_used() {
  3341. local volume="$1" container_id src dst
  3342. while read -r container_id; do
  3343. while read-0 src dst; do
  3344. [[ "$src/" == "$volume"/* ]] && return 0
  3345. done < <(get_volumes_for_container "$container_id")
  3346. done < <(get_running_compose_containers)
  3347. return 1
  3348. }
  3349. export -f is_volume_used
  3350. clean_unused_docker_compose() {
  3351. for f in /var/lib/compose/docker-compose/*; do
  3352. [ -e "$f" ] || continue
  3353. is_volume_used "$f" && continue
  3354. debug "Cleaning unused docker-compose ${f##*/}"
  3355. rm -rf "$f" || return 1
  3356. done
  3357. return 0
  3358. }
  3359. export -f clean_unused_docker_compose
  3360. docker_compose_store() {
  3361. local file="$1" sha
  3362. sha=$(hash_get 64 < "$file") || return 1
  3363. project=$(get_default_project_name) || return 1
  3364. dst="/var/lib/compose/docker-compose/$sha/$project"
  3365. mkdir -p "$dst" || return 1
  3366. cat <<EOF > "$dst/.env" || return 1
  3367. DOCKER_COMPOSE_PATH=$dst
  3368. COMPOSE_HTTP_TIMEOUT=7200
  3369. EOF
  3370. cp "$file" "$dst/docker-compose.yml" || return 1
  3371. mkdir -p "$dst/bin" || return 1
  3372. cat <<EOF > "$dst/bin/dc" || return 1
  3373. #!/bin/bash
  3374. $(declare -f read-0)
  3375. docker_run_opts=()
  3376. while read-0 opt; do
  3377. if [[ "\$opt" == "!env:"* ]]; then
  3378. opt="\${opt##!env:}"
  3379. var="\${opt%%=*}"
  3380. value="\${opt#*=}"
  3381. export "\$var"="\$value"
  3382. else
  3383. docker_run_opts+=("\$opt")
  3384. fi
  3385. done < <(cat "$COMPOSE_LAUNCHER_OPTS")
  3386. docker_run_opts+=(
  3387. "-w" "$dst"
  3388. "--entrypoint" "/usr/local/bin/docker-compose"
  3389. )
  3390. [ -t 1 ] && {
  3391. docker_run_opts+=("-ti")
  3392. }
  3393. exec docker run --rm "\${docker_run_opts[@]}" "\${COMPOSE_DOCKER_IMAGE:-docker.0k.io/compose}" "\$@"
  3394. EOF
  3395. chmod +x "$dst/bin/dc" || return 1
  3396. printf "%s" "$sha"
  3397. }
  3398. export -f docker_compose_store
  3399. launch_docker_compose() {
  3400. local charm docker_compose_tmpdir docker_compose_dir
  3401. docker_compose_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX)
  3402. #debug "Creating temporary docker-compose directory in '$docker_compose_tmpdir'."
  3403. trap_add EXIT "rm -rf \"$docker_compose_tmpdir\""
  3404. ## docker-compose will name network from the parent dir name
  3405. project=$(get_default_project_name)
  3406. mkdir -p "$docker_compose_tmpdir/$project"
  3407. docker_compose_dir="$docker_compose_tmpdir/$project"
  3408. if [ -z "$_CURRENT_DOCKER_COMPOSE" ]; then
  3409. err "${FUNCNAME[0]} is meant to be called after"\
  3410. "\$_CURRENT_DOCKER_COMPOSE has been calculated."
  3411. echo " Called by:" >&2
  3412. printf " - %s\n" "${FUNCNAME[@]:1}" >&2
  3413. return 1
  3414. fi
  3415. cat "$_CURRENT_DOCKER_COMPOSE" > "$docker_compose_dir/docker-compose.yml" || return 1
  3416. if [ -e "$state_tmpdir/to-merge-in-docker-compose.yml" ]; then
  3417. # debug "Merging some config data in docker-compose.yml:"
  3418. # debug "$(cat $state_tmpdir/to-merge-in-docker-compose.yml)"
  3419. _config_merge "$docker_compose_dir/docker-compose.yml" "$state_tmpdir/to-merge-in-docker-compose.yml" || return 1
  3420. fi
  3421. if [ -z "$(echo $(cat "$docker_compose_dir/docker-compose.yml"))" ]; then
  3422. die "Generated 'docker-compose.yml' is unexpectedly empty."
  3423. fi
  3424. ## XXXvlab: could be more specific and only link the needed charms
  3425. ## XXXvlab: why do we need these links ? If this is for the build command, then it is not useful anymore.
  3426. # for charm in $(shyaml keys services < "$docker_compose_dir/docker-compose.yml"); do
  3427. # if charm.exists "$charm"; then
  3428. # ln -sf "$(charm.get_dir "$charm")" "$docker_compose_dir/$charm" || exit 1
  3429. # fi
  3430. # done
  3431. mkdir "$docker_compose_dir/.data"
  3432. if [ -z "$COMPOSE_DISABLE_DOCKER_COMPOSE_STORE" ]; then
  3433. sha=$(docker_compose_store "$docker_compose_dir/docker-compose.yml") || return 1
  3434. fi
  3435. {
  3436. {
  3437. {
  3438. if [ -z "$COMPOSE_DISABLE_DOCKER_COMPOSE_STORE" ]; then
  3439. cd "/var/lib/compose/docker-compose/$sha/$project" || return 1
  3440. else
  3441. cd "$docker_compose_dir" || return 1
  3442. fi
  3443. if [ -f ".env" ]; then
  3444. debug "${WHITE}.env$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:"
  3445. debug "$(cat ".env" | prefix " $GRAY|$NORMAL ")"
  3446. fi
  3447. debug "${WHITE}docker-compose.yml$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:"
  3448. debug "$(cat "docker-compose.yml" | prefix " $GRAY|$NORMAL ")"
  3449. debug "${WHITE}Launching$NORMAL: docker-compose $@"
  3450. if [ "$DRY_COMPOSE_RUN" ]; then
  3451. echo docker-compose "$@"
  3452. else
  3453. docker-compose "$@"
  3454. fi
  3455. echo "$?" > "$docker_compose_dir/.data/errlvl"
  3456. } | _save stdout
  3457. } 3>&1 1>&2 2>&3 | _save stderr
  3458. } 3>&1 1>&2 2>&3
  3459. if tail -n 1 "$docker_compose_dir/.data/stderr" | grep -E "Service .+ failed to build: Error getting container [0-9a-f]+ from driver devicemapper: (open|Error mounting) /dev/mapper/docker-.*: no such file or directory$" >/dev/null 2>&1; then
  3460. err "Detected bug https://github.com/docker/docker/issues/4036 ... "
  3461. err "Please re-launch your command, or switch from 'devicemapper' driver to 'overlayfs' or 'aufs'."
  3462. fi
  3463. docker_compose_errlvl="$(cat "$docker_compose_dir/.data/errlvl" 2>/dev/null)"
  3464. if [ -z "$docker_compose_errlvl" ]; then
  3465. err "Something went wrong before you could gather docker-compose errorlevel."
  3466. return 1
  3467. fi
  3468. return "$docker_compose_errlvl"
  3469. }
  3470. export -f launch_docker_compose
  3471. get_compose_yml_location() {
  3472. if ! [ -z ${COMPOSE_YML_FILE+x} ]; then ## if set, even if empty
  3473. echo "$COMPOSE_YML_FILE"
  3474. return 0
  3475. fi
  3476. parent=$(while ! [ -f "./compose.yml" ]; do
  3477. [ "$PWD" == "/" ] && exit 0
  3478. cd ..
  3479. done; echo "$PWD"
  3480. )
  3481. if [ "$parent" ]; then
  3482. echo "$parent/compose.yml"
  3483. return 0
  3484. fi
  3485. ## XXXvlab: do we need this additional environment variable,
  3486. ## COMPOSE_YML_FILE is not sufficient ?
  3487. if [ "$DEFAULT_COMPOSE_FILE" ]; then
  3488. if ! [ -e "$DEFAULT_COMPOSE_FILE" ]; then
  3489. warn "No 'compose.yml' was found in current or parent dirs," \
  3490. "and \$DEFAULT_COMPOSE_FILE points to an unexistent file." \
  3491. "(${DEFAULT_COMPOSE_FILE})"
  3492. return 0
  3493. fi
  3494. echo "$DEFAULT_COMPOSE_FILE"
  3495. return 0
  3496. fi
  3497. warn "No 'compose.yml' was found in current or parent dirs, and no \$DEFAULT_COMPOSE_FILE was set."
  3498. return 0
  3499. }
  3500. export -f get_compose_yml_location
  3501. get_compose_yml_content() {
  3502. local cache_file="$state_tmpdir/$FUNCNAME.cache"
  3503. if [ -e "$cache_file" ]; then
  3504. cat "$cache_file" &&
  3505. touch "$cache_file" || return 1
  3506. return 0
  3507. fi
  3508. if [ -z "$COMPOSE_YML_FILE" ]; then
  3509. COMPOSE_YML_FILE=$(get_compose_yml_location) || return 1
  3510. fi
  3511. if [ -e "$COMPOSE_YML_FILE" ]; then
  3512. debug "Found $WHITE$exname$NORMAL YAML file in '$COMPOSE_YML_FILE'."
  3513. COMPOSE_YML_CONTENT=$(cat "$COMPOSE_YML_FILE") || {
  3514. err "Could not read '$COMPOSE_YML_FILE'."
  3515. return 1
  3516. }
  3517. else
  3518. debug "No compose file found. Using an empty one."
  3519. COMPOSE_YML_CONTENT=""
  3520. fi
  3521. COMPOSE_YML_CONTENT=$(merge_yaml_str "$COMPOSE_YML_CONTENT" "${compose_contents[@]}") || return 1
  3522. output=$(echo "$COMPOSE_YML_CONTENT"| shyaml get-value 2>&1)
  3523. if [ "$?" != 0 ]; then
  3524. outputed_something=
  3525. while IFS='' read -r line1 && IFS='' read -r line2; do
  3526. [ "$outputed_something" ] || err "Invalid YAML in '$COMPOSE_YML_FILE':"
  3527. outputed_something=true
  3528. echo "$line1 $GRAY($line2)$NORMAL"
  3529. done < <(echo "$output" | grep ^yaml.scanner -A 100 |
  3530. sed -r 's/^ in "<stdin>", //g' | sed -r 's/^yaml.scanner.[a-zA-Z]+: //g') |
  3531. prefix " $GRAY|$NORMAL "
  3532. [ "$outputed_something" ] || {
  3533. err "Unexpected error while running 'shyaml get-value' on '$COMPOSE_YML_FILE':"
  3534. echo "$output" | prefix " $GRAY|$NORMAL "
  3535. }
  3536. return 1
  3537. fi
  3538. echo "$COMPOSE_YML_CONTENT" | tee "$cache_file" || return 1
  3539. }
  3540. export -f get_compose_yml_content
  3541. get_default_target_services() {
  3542. local services=("$@")
  3543. if [ -z "${services[*]}" ]; then
  3544. if [ "$DEFAULT_SERVICES" ]; then
  3545. debug "No service provided, using $WHITE\$DEFAULT_SERVICES$NORMAL variable." \
  3546. "Target services: $DARKYELLOW$DEFAULT_SERVICES$NORMAL"
  3547. services="$DEFAULT_SERVICES"
  3548. else
  3549. err "No service provided."
  3550. return 1
  3551. fi
  3552. fi
  3553. echo "${services[*]}"
  3554. }
  3555. export -f get_default_target_services
  3556. get_master_services() {
  3557. local loaded master_service service
  3558. declare -A loaded
  3559. for service in "$@"; do
  3560. master_service=$(get_top_master_service_for_service "$service") || return 1
  3561. if [ "${loaded[$master_service]}" ]; then
  3562. continue
  3563. fi
  3564. echo "$master_service"
  3565. loaded["$master_service"]=1
  3566. done | nspc
  3567. return "${PIPESTATUS[0]}"
  3568. }
  3569. export -f get_master_services
  3570. get_current_docker_container_id() {
  3571. local line
  3572. line=$(cat "/proc/self/cpuset") || return 1
  3573. [[ "$line" == *docker* ]] || return 1
  3574. echo "${line##*/}"
  3575. }
  3576. export -f get_current_docker_container_id
  3577. ## if we are in a docker compose, we might want to know what is the
  3578. ## real host path of some local paths.
  3579. get_host_path() {
  3580. local path="$1"
  3581. path=$(realpath "$path") || return 1
  3582. container_id=$(get_current_docker_container_id) || {
  3583. print "%s" "$path"
  3584. return 0
  3585. }
  3586. biggest_dst=
  3587. current_src=
  3588. while read-0 src dst; do
  3589. [[ "$path" == "$dst"* ]] || continue
  3590. if [[ "${#biggest_dst}" < "${#dst}" ]]; then
  3591. biggest_dst="$dst"
  3592. current_src="$src"
  3593. fi
  3594. done < <(get_volumes_for_container "$container_id")
  3595. if [ "$current_src" ]; then
  3596. printf "%s" "$current_src"
  3597. else
  3598. return 1
  3599. fi
  3600. }
  3601. export -f get_host_path
  3602. _setup_state_dir() {
  3603. export state_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX)
  3604. #debug "Creating temporary state directory in '$state_tmpdir'."
  3605. # trap_add EXIT "debug \"Removing temporary state directory in $state_tmpdir.\";\
  3606. # rm -rf \"$state_tmpdir\""
  3607. trap_add EXIT "rm -rf \"$state_tmpdir\""
  3608. }
  3609. get_docker_compose_help_msg() {
  3610. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  3611. docker_compose_help_msg
  3612. if [ -e "$cache_file" ]; then
  3613. cat "$cache_file" &&
  3614. touch "$cache_file" || return 1
  3615. return 0
  3616. fi
  3617. docker_compose_help_msg=$(docker-compose $action --help 2>/dev/null) || return 1
  3618. echo "$docker_compose_help_msg" |
  3619. tee "$cache_file" || return 1
  3620. }
  3621. get_docker_compose_usage() {
  3622. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  3623. docker_compose_help_msg
  3624. if [ -e "$cache_file" ]; then
  3625. cat "$cache_file" &&
  3626. touch "$cache_file" || return 1
  3627. return 0
  3628. fi
  3629. docker_compose_help_msg=$(get_docker_compose_help_msg $action) || return 1
  3630. echo "$docker_compose_help_msg" |
  3631. grep -m 1 "^Usage:" -A 10000 |
  3632. egrep -m 1 "^\$" -B 10000 |
  3633. nspc |
  3634. sed -r 's/^Usage: //g' |
  3635. tee "$cache_file" || return 1
  3636. }
  3637. get_docker_compose_opts_help() {
  3638. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  3639. docker_compose_help_msg
  3640. if [ -e "$cache_file" ]; then
  3641. cat "$cache_file" &&
  3642. touch "$cache_file" || return 1
  3643. return 0
  3644. fi
  3645. docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1
  3646. echo "$docker_compose_opts_help" |
  3647. grep '^Options:' -A 20000 |
  3648. tail -n +2 |
  3649. { cat ; echo; } |
  3650. egrep -m 1 "^\S*\$" -B 10000 |
  3651. head -n -1 |
  3652. tee "$cache_file" || return 1
  3653. }
  3654. get_docker_compose_commands_help() {
  3655. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  3656. docker_compose_help_msg
  3657. if [ -e "$cache_file" ]; then
  3658. cat "$cache_file" &&
  3659. touch "$cache_file" || return 1
  3660. return 0
  3661. fi
  3662. docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1
  3663. echo "$docker_compose_opts_help" |
  3664. grep '^Commands:' -A 20000 |
  3665. tail -n +2 |
  3666. { cat ; echo; } |
  3667. egrep -m 1 "^\S*\$" -B 10000 |
  3668. head -n -1 |
  3669. tee "$cache_file" || return 1
  3670. }
  3671. get_docker_compose_opts_list() {
  3672. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  3673. docker_compose_help_msg
  3674. if [ -e "$cache_file" ]; then
  3675. cat "$cache_file" &&
  3676. touch "$cache_file" || return 1
  3677. return 0
  3678. fi
  3679. docker_compose_opts_help=$(get_docker_compose_opts_help $action) || return 1
  3680. echo "$docker_compose_opts_help" |
  3681. egrep "^\s+-" |
  3682. sed -r 's/\s+((((-[a-zA-Z]|--[a-zA-Z0-9-]+)( [A-Z=]+|=[^ ]+)?)(, )?)+)\s+.*$/\1/g' |
  3683. tee "$cache_file" || return 1
  3684. }
  3685. options_parser() {
  3686. sed -r 's/^(\s+(((-[a-zA-Z]|--[a-zA-Z0-9-]+)([ =]([a-zA-Z_=\"\[]|\])+)?(, | )?)+)\s+)[^ ].*$/\x0\2\x0\0/g'
  3687. printf "\0"
  3688. }
  3689. remove_options_in_option_help_msg() {
  3690. {
  3691. read-0 null
  3692. if [ "$null" ]; then
  3693. err "options parsing error, should start with an option line."
  3694. return 1
  3695. fi
  3696. while read-0 opt full_txt;do
  3697. multi_opts="$(printf "%s " $opt | multi_opts_filter)"
  3698. single_opts="$(printf "%s " $opt | single_opts_filter)"
  3699. for to_remove in "$@"; do
  3700. str_matches "$to_remove" $multi_opts $single_opts && {
  3701. continue 2
  3702. }
  3703. done
  3704. echo -n "$full_txt"
  3705. done
  3706. } < <(options_parser)
  3707. }
  3708. _MULTIOPTION_REGEX='^((-[a-zA-Z]|--[a-zA-Z0-9-]+)(, )?)+'
  3709. _MULTIOPTION_REGEX_LINE_FILTER=$_MULTIOPTION_REGEX'(\s|=)'
  3710. multi_opts_filter() {
  3711. egrep "$_MULTIOPTION_REGEX_LINE_FILTER" |
  3712. sed -r "s/^($_MULTIOPTION_REGEX)(\s|=).*$/\1/g" |
  3713. tr ',' "\n" | nspc
  3714. }
  3715. single_opts_filter() {
  3716. egrep -v "$_MULTIOPTION_REGEX_LINE_FILTER" |
  3717. tr ',' "\n" | nspc
  3718. }
  3719. get_docker_compose_multi_opts_list() {
  3720. local action="$1" opts_list
  3721. opts_list=$(get_docker_compose_opts_list "$action") || return 1
  3722. echo "$opts_list" | multi_opts_filter
  3723. }
  3724. get_docker_compose_single_opts_list() {
  3725. local action="$1" opts_list
  3726. opts_list=$(get_docker_compose_opts_list "$action") || return 1
  3727. echo "$opts_list" | single_opts_filter
  3728. }
  3729. display_commands_help() {
  3730. local charm_actions
  3731. echo
  3732. echo "${WHITE}Commands${NORMAL} (thanks to docker-compose):"
  3733. get_docker_compose_commands_help | sed -r "s/ ([a-z]+)(\s+)/ ${DARKCYAN}\1${NORMAL}\2/g"
  3734. charm_actions_help=$(get_docker_charm_action_help) || return 1
  3735. if [ "$charm_actions_help" ]; then
  3736. echo
  3737. echo "${WHITE}Charm actions${NORMAL}:"
  3738. printf "%s\n" "$charm_actions_help" | \
  3739. sed -r "s/^ ([a-z0-9-]+)(\s+)([a-z0-9-]+)(\s+)/ ${DARKCYAN}\1${NORMAL}\2${DARKYELLOW}\3${NORMAL}\4/g"
  3740. fi
  3741. }
  3742. get_docker_charm_action() {
  3743. local services service charm relation_name target_service relation_config \
  3744. target_charm
  3745. services=($(get_compose_yml_content | yq -r 'keys().[]' 2>/dev/null)) || return 1
  3746. NO_CONSTRAINT_CHECK=1 get_all_relations "${services[@]}" >/dev/null || return 1
  3747. services=($(get_all_services)) || return 1
  3748. for service in "${services[@]}"; do
  3749. printf "%s:\n" "$service"
  3750. charm=$(get_service_charm "$service") || return 1
  3751. for action in $(charm.ls_direct_actions "$charm"); do
  3752. printf " %s:\n" "$action"
  3753. printf " type: %s\n" "direct"
  3754. done
  3755. while read-0 relation_name target_service _relation_config _tech_dep; do
  3756. target_charm=$(get_service_charm "$target_service") || return 1
  3757. for action in $(charm.ls_relation_actions "$target_charm" "$relation_name"); do
  3758. printf " %s:\n" "$action"
  3759. printf " type: %s\n" "indirect"
  3760. printf " inherited: %s\n" "$target_charm"
  3761. done
  3762. done < <(get_service_relations "$service")
  3763. done
  3764. }
  3765. export -f get_docker_charm_action
  3766. get_docker_charm_action_help() {
  3767. local services service charm relation_name target_service relation_config \
  3768. target_charm
  3769. services=($(get_compose_yml_content | shyaml keys 2>/dev/null))
  3770. NO_CONSTRAINT_CHECK=1 get_all_relations "${services[@]}" >/dev/null || return 1
  3771. for service in "${services[@]}"; do
  3772. out=$(
  3773. charm=$(get_service_charm "$service") || return 1
  3774. for action in $(charm.ls_direct_actions "$charm"); do
  3775. printf " %-28s %s\n" "$action $service" "Direct action from ${DARKPINK}$charm${NORMAL}"
  3776. done
  3777. while read-0 relation_name target_service _relation_config _tech_dep; do
  3778. target_charm=$(get_service_charm "$target_service") || return 1
  3779. for action in $(charm.ls_relation_actions "$target_charm" "$relation_name"); do
  3780. printf " %-28s %s\n" "$action $service" "Indirect action from ${DARKPINK}$target_charm${NORMAL}"
  3781. done
  3782. done < <(get_service_relations "$service")
  3783. )
  3784. if [ "$out" ]; then
  3785. echo " for ${DARKYELLOW}$service${NORMAL}:"
  3786. printf "%s\n" "$out"
  3787. fi
  3788. done
  3789. }
  3790. display_help() {
  3791. print_help
  3792. echo "${WHITE}Usage${NORMAL}:"
  3793. echo " $usage"
  3794. echo " $usage cache {clean|clear}"
  3795. echo "${WHITE}Options${NORMAL}:"
  3796. echo " -h, --help Print this message and quit"
  3797. echo " (ignoring any other options)"
  3798. echo " -V, --version Print current version and quit"
  3799. echo " (ignoring any other options)"
  3800. echo " --dirs Display data dirs and quit"
  3801. echo " (ignoring any other options)"
  3802. echo " --get-project-name Display project name and quit"
  3803. echo " (ignoring any other options)"
  3804. echo " --get-available-actions Display all available actions and quit"
  3805. echo " (ignoring any other options)"
  3806. echo " -v, --verbose Be more verbose"
  3807. echo " -q, --quiet Be quiet"
  3808. echo " -d, --debug Print full debugging information (sets also verbose)"
  3809. echo " --dry-compose-run If docker-compose will be run, only print out what"
  3810. echo " command line will be used."
  3811. echo " --no-relations Do not run any relation script"
  3812. echo " --no-hooks Do not run any hook script"
  3813. echo " --no-init Do not run any init script"
  3814. echo " --no-post-deploy Do not run any post-deploy script"
  3815. echo " --no-pre-deploy Do not run any pre-deploy script"
  3816. echo " --without-relation RELATION "
  3817. echo " Do not run given relation"
  3818. echo " -R, --rebuild-relations-to-service SERVICE"
  3819. echo " Will rebuild all relations to given service"
  3820. echo " --add-compose-content, -Y YAML"
  3821. echo " Will merge some direct YAML with the current compose"
  3822. echo " -c, --color Force color mode (default is to detect if in tty mode)"
  3823. echo " --push-builds Will push cached docker images to docker cache registry"
  3824. get_docker_compose_opts_help | remove_options_in_option_help_msg --version --help --verbose |
  3825. filter_docker_compose_help_message
  3826. display_commands_help
  3827. }
  3828. _graph_service() {
  3829. local service="$1" base="$1"
  3830. charm=$(get_service_charm "$service") || return 1
  3831. metadata=$(charm.metadata "$charm") || return 1
  3832. subordinate=$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)
  3833. if [[ "$subordinate" =~ ^True|true$ ]]; then
  3834. requires="$(echo "$metadata" | shyaml get-value "requires" 2>/dev/null)"
  3835. master_charm=
  3836. while read-0 relation_name relation; do
  3837. [ "$(echo "$relation" | shyaml get-value "scope" 2>/dev/null)" == "container" ] || continue
  3838. interface="$(echo "$relation" | shyaml get-value "interface" 2>/dev/null)"
  3839. if [ -z "$interface" ]; then
  3840. err "No ${WHITE}$interface${NORMAL} set for relation $relation_name."
  3841. return 1
  3842. fi
  3843. ## Action provided by relation ?
  3844. target_service=
  3845. while read-0 relation_name candidate_target_service _relation_config _tech_dep; do
  3846. [ "$interface" == "$relation_name" ] && {
  3847. target_service="$candidate_target_service"
  3848. break
  3849. }
  3850. done < <(get_service_relations "$service")
  3851. if [ -z "$target_service" ]; then
  3852. err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \
  3853. "${DARKYELLOW}$service$NORMAL compose definition."
  3854. return 1
  3855. fi
  3856. master_service="$target_service"
  3857. master_charm=$(get_service_charm "$target_service") || return 1
  3858. break
  3859. done < <(echo "$requires" | shyaml key-values-0 2>/dev/null)
  3860. fi
  3861. _graph_node_service "$service" "$base" "$charm"
  3862. _graph_edge_service "$service" "$subordinate" "$master_service"
  3863. }
  3864. _graph_node_service() {
  3865. local service="$1" base="$2" charm="$3"
  3866. cat <<EOF
  3867. "$(_graph_node_service_label ${service})" [
  3868. style = "filled, $([[ "$subordinate" =~ ^True|true$ ]] && echo "dashed" || echo "bold")"
  3869. penwidth = $([[ "$subordinate" =~ ^True|true$ ]] && echo "3" || echo "5")
  3870. color = $([ "$base" ] && echo "blue" || echo "black")
  3871. fillcolor = "white"
  3872. fontname = "Courier New"
  3873. shape = "Mrecord"
  3874. label =<$(_graph_node_service_content "$service")>
  3875. ];
  3876. EOF
  3877. }
  3878. _graph_edge_service() {
  3879. local service="$1" subordinate="$2" master_service="$3"
  3880. while read-0 relation_name target_service relation_config tech_dep; do
  3881. cat <<EOF
  3882. "$(_graph_node_service_label ${service})" -> "$(_graph_node_service_label ${target_service})" [
  3883. penwidth = $([ "$master_service" == "$target_service" ] && echo 3 || echo 2)
  3884. fontsize = 16
  3885. fontcolor = "black"
  3886. style = $([ "$master_service" == "$target_service" ] && echo dashed || echo "\"\"")
  3887. weight = $([ "$master_service" == "$target_service" ] && echo 2.0 || echo 1.0)
  3888. dir = $([ "$master_service" == "$target_service" ] && echo none || echo both)
  3889. arrowtail = odot
  3890. # arrowhead = dotlicurve
  3891. taillabel = "$relation_name" ];
  3892. EOF
  3893. done < <(get_service_relations "$service") || return 1
  3894. }
  3895. _graph_node_service_label() {
  3896. local service="$1"
  3897. echo "service_$service"
  3898. }
  3899. _graph_node_service_content() {
  3900. local service="$1"
  3901. charm=$(get_service_charm "$service") || return 1
  3902. cat <<EOF
  3903. <table border="0" cellborder="0" cellpadding="3" bgcolor="white">
  3904. <tr>
  3905. <td bgcolor="black" align="center" colspan="2">
  3906. <font color="white">$service</font>
  3907. </td>
  3908. </tr>
  3909. $(if [ "$charm" != "$service" ]; then
  3910. cat <<EOF2
  3911. <tr>
  3912. <td align="left" port="r0">charm: $charm</td>
  3913. </tr>
  3914. EOF2
  3915. fi)
  3916. </table>
  3917. EOF
  3918. }
  3919. cla_contains () {
  3920. local e
  3921. for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done
  3922. return 1
  3923. }
  3924. filter_docker_compose_help_message() {
  3925. cat - |
  3926. sed -r "s/docker-compose run/${DARKWHITE}compose${NORMAL} ${DARKCYAN}$action${NORMAL}/g;
  3927. s/docker-compose.yml/compose.yml/g;
  3928. s/SERVICES?/${DARKYELLOW}\0${NORMAL}/g;
  3929. s/^(\s+)\\$/\1${WHITE}\$${NORMAL}/g;
  3930. s/^(\s+)run/\1${DARKCYAN}$action${NORMAL}/g;
  3931. s/docker-compose/${DARKWHITE}compose${NORMAL}/g"
  3932. }
  3933. graph() {
  3934. local services=("$@")
  3935. declare -A entries
  3936. cat <<EOF
  3937. digraph g {
  3938. graph [
  3939. fontsize=30
  3940. labelloc="t"
  3941. label=""
  3942. splines=true
  3943. overlap=false
  3944. #rankdir = "LR"
  3945. ];
  3946. ratio = auto;
  3947. EOF
  3948. for target_service in "$@"; do
  3949. services=$(get_ordered_service_dependencies "$target_service") || return 1
  3950. for service in $services; do
  3951. [ "${entries[$service]}" ] && continue || entries[$service]=1
  3952. if cla_contains "$service" "${services[@]}"; then
  3953. base=true
  3954. else
  3955. base=
  3956. fi
  3957. _graph_service "$service" "$base"
  3958. done
  3959. done
  3960. echo "}"
  3961. }
  3962. cached_wget() {
  3963. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(p0 "$@" | md5_compat)" \
  3964. url="$1"
  3965. if [ -e "$cache_file" ]; then
  3966. cat "$cache_file"
  3967. touch "$cache_file"
  3968. return 0
  3969. fi
  3970. wget -O- "${url}" |
  3971. tee "$cache_file"
  3972. if [ "${PIPESTATUS[0]}" != 0 ]; then
  3973. rm "$cache_file"
  3974. die "Unable to fetch '$url'."
  3975. return 1
  3976. fi
  3977. }
  3978. export -f cached_wget
  3979. [ "$SOURCED" ] && return 0
  3980. trap_add "EXIT" clean_cache
  3981. export COMPOSE_DOCKER_REGISTRY="${COMPOSE_DOCKER_REGISTRY:-docker.0k.io}"
  3982. if [ -z "$DISABLE_SYSTEM_CONFIG_FILE" ]; then
  3983. if [ -r /etc/default/charm ]; then
  3984. . "/etc/default/charm"
  3985. fi
  3986. if [ -r "/etc/default/$exname" ]; then
  3987. . "/etc/default/$exname"
  3988. fi
  3989. ## XXXvlab: should provide YML config opportunities in possible parent dirs ?
  3990. ## userdir ? and global /etc/compose.yml ?
  3991. for cfgfile in /etc/compose.conf /etc/compose.local.conf \
  3992. /etc/default/compose /etc/compose/local.conf; do
  3993. [ -e "$cfgfile" ] || continue
  3994. . "$cfgfile" || die "Loading config file '$cfgfile' failed."
  3995. done
  3996. fi
  3997. _setup_state_dir
  3998. mkdir -p "$CACHEDIR" || exit 1
  3999. log () { cat; }
  4000. export -f log
  4001. ##
  4002. ## Argument parsing
  4003. ##
  4004. wrap_opts=()
  4005. services=()
  4006. remainder_args=()
  4007. compose_opts=()
  4008. compose_contents=()
  4009. action_opts=()
  4010. services_args=()
  4011. pos_arg_ct=0
  4012. no_hooks=
  4013. no_init=
  4014. action=
  4015. stage="main" ## switches from 'main', to 'action', 'remainder'
  4016. is_docker_compose_action=
  4017. rebuild_relations_to_service=()
  4018. color=
  4019. declare -A without_relations
  4020. DC_MATCH_MULTI=$(get_docker_compose_multi_opts_list) &&
  4021. DC_MATCH_SINGLE=$(get_docker_compose_single_opts_list) || exit 1
  4022. while read-0 arg; do
  4023. case "$stage" in
  4024. "main")
  4025. case "$arg" in
  4026. --help|-h)
  4027. no_init=true ; no_hooks=true ; no_relations=true
  4028. display_help
  4029. exit 0
  4030. ;;
  4031. --verbose|-v)
  4032. export VERBOSE=true
  4033. compose_opts+=("--verbose")
  4034. ;;
  4035. --quiet|-q)
  4036. export QUIET=true
  4037. export wrap_opts+=("-q")
  4038. log () { cat >&2; }
  4039. export -f log
  4040. ;;
  4041. --version|-V)
  4042. print_version
  4043. docker-compose --version
  4044. docker --version
  4045. exit 0
  4046. ;;
  4047. -f|--file)
  4048. read-0 value
  4049. [ -e "$value" ] || die "File $value doesn't exists"
  4050. export COMPOSE_YML_FILE="$value"
  4051. shift
  4052. ;;
  4053. -p|--project-name)
  4054. read-0 value
  4055. export DEFAULT_PROJECT_NAME="$value"
  4056. compose_opts+=("--project-name $value")
  4057. shift
  4058. ;;
  4059. --color|-c)
  4060. if [ "$color" == "0" ]; then
  4061. err "Conflicting option --color with previous --no-ansi."
  4062. exit 1
  4063. fi
  4064. color=1
  4065. ansi_color yes
  4066. ;;
  4067. --no-ansi)
  4068. if [ "$color" == "1" ]; then
  4069. err "Conflicting option --no-ansi with previous --color."
  4070. exit 1
  4071. fi
  4072. color=0
  4073. ansi_color no
  4074. compose_opts+=("--no-ansi")
  4075. ;;
  4076. --no-relations)
  4077. export no_relations=true
  4078. ;;
  4079. --without-relation)
  4080. read-0 value
  4081. without_relations["$value"]=1
  4082. shift
  4083. ;;
  4084. --no-hooks)
  4085. export no_hooks=true
  4086. ;;
  4087. --no-init)
  4088. export no_init=true
  4089. ;;
  4090. --no-post-deploy)
  4091. export no_post_deploy=true
  4092. ;;
  4093. --no-pre-deploy)
  4094. export no_pre_deploy=true
  4095. ;;
  4096. --rebuild-relations-to-service|-R)
  4097. read-0 value
  4098. rebuild_relations_to_service+=("$value")
  4099. shift
  4100. ;;
  4101. --push-builds)
  4102. export COMPOSE_PUSH_TO_REGISTRY=1
  4103. ;;
  4104. --debug|-d)
  4105. export DEBUG=true
  4106. export VERBOSE=true
  4107. #compose_opts+=("--verbose" "--log-level" "DEBUG")
  4108. ;;
  4109. --add-compose-content|-Y)
  4110. read-0 value
  4111. compose_contents+=("$value")
  4112. shift
  4113. ;;
  4114. --dirs)
  4115. echo "CACHEDIR: $CACHEDIR"
  4116. echo "VARDIR: $VARDIR"
  4117. exit 0
  4118. ;;
  4119. --get-project-name)
  4120. project=$(get_default_project_name) || exit 1
  4121. echo "$project"
  4122. exit 0
  4123. ;;
  4124. --get-available-actions)
  4125. get_docker_charm_action
  4126. exit $?
  4127. ;;
  4128. --dry-compose-run)
  4129. export DRY_COMPOSE_RUN=true
  4130. ;;
  4131. --*|-*)
  4132. if str_pattern_matches "$arg" $DC_MATCH_MULTI; then
  4133. read-0 value
  4134. compose_opts+=("$arg" "$value")
  4135. shift;
  4136. elif str_pattern_matches "$arg" $DC_MATCH_SINGLE; then
  4137. compose_opts+=("$arg")
  4138. else
  4139. err "Unknown option '$arg'. Please check help:"
  4140. display_help >&2
  4141. exit 1
  4142. fi
  4143. ;;
  4144. *)
  4145. action="$arg"
  4146. stage="action"
  4147. if DC_USAGE=$(get_docker_compose_usage "$action"); then
  4148. is_docker_compose_action=true
  4149. DC_MATCH_MULTI=$(get_docker_compose_multi_opts_list "$action") &&
  4150. DC_MATCH_SINGLE="$(get_docker_compose_single_opts_list "$action")"
  4151. if [ "$DC_MATCH_MULTI" ]; then
  4152. DC_MATCH_SINGLE="$DC_MATCH_SINGLE $(echo "$DC_MATCH_MULTI" | sed -r 's/( |$)/=\* /g')"
  4153. fi
  4154. pos_args=($(echo "$DC_USAGE" | sed -r 's/\[-[^]]+\] ?//g;s/\[options\] ?//g'))
  4155. pos_args=("${pos_args[@]:1}")
  4156. # echo "USAGE: $DC_USAGE"
  4157. # echo "pos_args: ${pos_args[@]}"
  4158. # echo "MULTI: $DC_MATCH_MULTI"
  4159. # echo "SINGLE: $DC_MATCH_SINGLE"
  4160. # exit 1
  4161. else
  4162. stage="remainder"
  4163. fi
  4164. ;;
  4165. esac
  4166. ;;
  4167. "action") ## Only for docker-compose actions
  4168. case "$arg" in
  4169. --help|-h)
  4170. no_init=true ; no_hooks=true ; no_relations=true
  4171. action_opts+=("$arg")
  4172. ;;
  4173. --*|-*)
  4174. if [ "$is_docker_compose_action" ]; then
  4175. if str_pattern_matches "$arg" $DC_MATCH_MULTI; then
  4176. read-0 value
  4177. action_opts+=("$arg" "$value")
  4178. shift
  4179. elif str_pattern_matches "$arg" $DC_MATCH_SINGLE; then
  4180. action_opts+=("$arg")
  4181. else
  4182. err "Unknown option '$arg'. Please check '${DARKCYAN}$action${NORMAL}' help:"
  4183. docker-compose "$action" --help |
  4184. filter_docker_compose_help_message >&2
  4185. exit 1
  4186. fi
  4187. fi
  4188. ;;
  4189. *)
  4190. # echo "LOOP $1 : pos_arg: $pos_arg_ct // ${pos_args[$pos_arg_ct]}"
  4191. if [[ "${pos_args[$pos_arg_ct]}" == "[SERVICE...]" ]]; then
  4192. services_args+=("$arg")
  4193. elif [[ "${pos_args[$pos_arg_ct]}" == "SERVICE" ]]; then
  4194. services_args=("$arg") || exit 1
  4195. stage="remainder"
  4196. else
  4197. action_posargs+=("$arg")
  4198. ((pos_arg_ct++))
  4199. fi
  4200. ;;
  4201. esac
  4202. ;;
  4203. "remainder")
  4204. remainder_args+=("$arg")
  4205. while read-0 arg; do
  4206. remainder_args+=("$arg")
  4207. done
  4208. break 3
  4209. ;;
  4210. esac
  4211. shift
  4212. done < <(cla.normalize "$@")
  4213. [ -n "$CACHEDIR" ] || die "No cache directory defined."
  4214. [ -d "$CACHEDIR" ] || die "Cache directory '$CACHEDIR' doesn't exists."
  4215. case "$action" in
  4216. cache)
  4217. case "${remainder_args[0]}" in
  4218. clean)
  4219. clean_cache
  4220. exit 0
  4221. ;;
  4222. clear)
  4223. Wrap "${wrap_opts[@]}" -v -d "clear cache directory" -- rm -rf "$CACHEDIR/"*
  4224. ## clear all docker caches
  4225. ## image name are like '[$COMPOSE_DOCKER_REGISTRY]cache/charm/CHARM_NAME:HASH'
  4226. Wrap "${wrap_opts[@]}" -v -d "clear docker cache" <<EOF
  4227. docker images --format "{{.Repository}}:{{.Tag}}" |
  4228. egrep "^($COMPOSE_DOCKER_REGISTRY/)?cache/charm/[a-zA-Z0-9._-]+:[0-9a-f]{32,32}$" |
  4229. while read -r image; do
  4230. docker rmi "\$image" || true
  4231. done
  4232. EOF
  4233. exit 0
  4234. ;;
  4235. *)
  4236. err "Unknown cache command: ${DARKCYAN}${remainder_args[0]}${NORMAL}"
  4237. exit 1
  4238. ;;
  4239. esac
  4240. ;;
  4241. esac
  4242. export compose_contents
  4243. [ "${services_args[*]}" ] && debug " ${DARKWHITE}Services:$NORMAL ${DARKYELLOW}${services_args[*]}$NORMAL"
  4244. [ "${compose_opts[*]}" ] && debug " ${DARKWHITE}Main docker-compose opts:$NORMAL ${compose_opts[*]}"
  4245. [ "${action_posargs[*]}" ] && debug " ${DARKWHITE}Main docker-compose pos args:$NORMAL ${action_posargs[*]}"
  4246. [ "${action_opts[*]}" ] && debug " ${DARKWHITE}Action $DARKCYAN$action$NORMAL with opts:$NORMAL ${action_opts[*]}"
  4247. [ "${remainder_args[*]}" ] && debug " ${DARKWHITE}Remainder args:$NORMAL ${remainder_args[*]}"
  4248. aexport remainder_args
  4249. ##
  4250. ## Actual code
  4251. ##
  4252. COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1
  4253. COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1
  4254. export COMPOSE_YML_FILE COMPOSE_YML_CONTENT
  4255. charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct."
  4256. ##
  4257. ## Get services in command line.
  4258. ##
  4259. if [ -z "$is_docker_compose_action" -a "$action" ]; then
  4260. action_service=${remainder_args[0]}
  4261. if [ -z "$action_service" ]; then
  4262. err "No such command or action: ${DARKCYAN}$action${NORMAL}"
  4263. display_commands_help
  4264. exit 1
  4265. fi
  4266. ## Required by has_service_action
  4267. array_read-0 services_args < <(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys-0 2>/dev/null)
  4268. NO_CONSTRAINT_CHECK=1 get_all_relations "${services_args[@]}" >/dev/null || exit 1
  4269. remainder_args=("${remainder_args[@]:1}")
  4270. if has_service_action "$action_service" "$action" >/dev/null; then
  4271. is_service_action=true
  4272. services_args=("$action_service")
  4273. {
  4274. read-0 action_type
  4275. case "$action_type" in
  4276. "relation")
  4277. read-0 _ target_service _target_charm relation_name _ action_script_path
  4278. debug "Found action $DARKYELLOW${action_service}$NORMAL/$DARKBLUE$relation_name$NORMAL/$DARKCYAN$action$NORMAL (in $DARKYELLOW$target_service$NORMAL)"
  4279. services_args+=("$target_service")
  4280. ;;
  4281. "direct")
  4282. read-0 _ action_script_path
  4283. debug "Found action $DARKYELLOW${action_service}$NORMAL.$DARKCYAN$action$NORMAL"
  4284. ;;
  4285. esac
  4286. } < <(has_service_action "$action_service" "$action")
  4287. get_all_relations "${services_args[@]}" >/dev/null || {
  4288. echo " Hint: if this is unexpected, you can try to delete caches, and re-run the command." >&2
  4289. exit 1
  4290. }
  4291. ## Divert logging to stdout to stderr
  4292. log () { cat >&2; }
  4293. export -f log
  4294. else
  4295. die "Unknown action '${DARKCYAN}$action$NORMAL': It doesn't match any docker-compose commands nor inner charm actions."
  4296. fi
  4297. else
  4298. case "$action" in
  4299. ps|up)
  4300. if [ "${#services_args[@]}" == 0 ]; then
  4301. array_read-0 services_args < <(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys-0 2>/dev/null)
  4302. fi
  4303. ;;
  4304. config)
  4305. services_args=("${action_posargs[@]}")
  4306. ;;
  4307. esac
  4308. fi
  4309. export COMPOSE_ACTION="$action"
  4310. NO_CONSTRAINT_CHECK=True
  4311. case "$action" in
  4312. up)
  4313. NO_CONSTRAINT_CHECK=
  4314. ;;
  4315. esac
  4316. get_all_relations "${services_args[@]}" >/dev/null || exit 1
  4317. if [ -n "$is_docker_compose_action" ] && [ "${#services_args[@]}" -gt 0 ]; then
  4318. services=($(get_master_services "${services_args[@]}")) || exit 1
  4319. if [ "$action" == "up" ]; then
  4320. declare -A seen
  4321. services=($(get_ordered_service_dependencies "${services_args[@]}")) || exit 1
  4322. for service in "${services[@]}"; do
  4323. mservice=$(get_master_service_for_service "$service") || exit 1
  4324. [ "${seen[$mservice]}" ] && continue
  4325. type="$(get_service_type "$mservice")" || exit 1
  4326. ## remove run-once
  4327. [ "$type" == "run-once" ] && continue
  4328. [ "$type" == "stub" ] && continue
  4329. seen[$mservice]=1
  4330. action_posargs+=("$mservice")
  4331. done
  4332. else
  4333. action_posargs+=("${services[@]}")
  4334. fi
  4335. ## Get rid of subordinates
  4336. action_posargs=($(get_master_services "${action_posargs[@]}")) || exit 1
  4337. fi
  4338. get_docker_compose "${services_args[@]}" >/dev/null || { ## precalculate variable \$_current_docker_compose
  4339. err "Fails to compile base 'docker-compose.yml'"
  4340. exit 1
  4341. }
  4342. ##
  4343. ## Pre-action
  4344. ##
  4345. full_init=
  4346. case "$action" in
  4347. build)
  4348. full_init=true ## will actually stop after build
  4349. ;;
  4350. up|run)
  4351. full_init=true
  4352. post_hook=true
  4353. ;;
  4354. ""|down|restart|logs|config|ps)
  4355. full_init=
  4356. ;;
  4357. *)
  4358. if [ "$is_service_action" ]; then
  4359. full_init=true
  4360. keywords=($(egrep "^#*\s*compose:" "$action_script_path" | cut -f 2- -d:))
  4361. for keyword in "${keywords[@]}"; do
  4362. case "$keyword" in
  4363. no-hooks)
  4364. no_hooks=true
  4365. ;;
  4366. hooks)
  4367. full_init=true
  4368. ;;
  4369. esac
  4370. done
  4371. fi
  4372. ;;
  4373. esac
  4374. if [ -n "$full_init" ]; then
  4375. if [[ "$action" == "build" ]] || [[ -z "$no_init" && -z "$no_hooks" ]]; then
  4376. [[ "$action" == "build" ]] || Section "acquire charm's images"
  4377. run_service_acquire_images "${services_args[@]}" || exit 1
  4378. Feed
  4379. [ "$action" == "build" ] && {
  4380. exit 0
  4381. }
  4382. Section setup host resources
  4383. setup_host_resources "${services_args[@]}" || exit 1
  4384. ## init in order
  4385. Section initialisation
  4386. run_service_hook init "${services_args[@]}" || exit 1
  4387. fi
  4388. ## Get relations
  4389. if [[ -z "$no_relations" && -z "$no_hooks" ]]; then
  4390. if [ "${#rebuild_relations_to_service[@]}" != 0 ]; then
  4391. rebuild_relations_to_service=$(get_master_services "${rebuild_relations_to_service[@]}") || return 1
  4392. rebuild_relations_to_service=($rebuild_relations_to_service)
  4393. project=$(get_default_project_name) || return 1
  4394. for service in "${rebuild_relations_to_service[@]}"; do
  4395. for dir in "$VARDIR/relations/$project/"*"-${service}/"*; do
  4396. [ -d "$dir" ] && {
  4397. debug rm -rf "$dir"
  4398. rm -rf "$dir"
  4399. }
  4400. done
  4401. done
  4402. fi
  4403. run_service_relations "${services_args[@]}" || exit 1
  4404. fi
  4405. if [[ -z "$no_pre_deploy" && -z "$no_hooks" ]]; then
  4406. run_service_hook pre_deploy "${services_args[@]}" || exit 1
  4407. fi
  4408. fi | log
  4409. if [ "${PIPESTATUS[0]}" != 0 ]; then
  4410. exit 1
  4411. fi
  4412. [ "$action" == "build" ] && exit 0
  4413. if [ "$action" == "run" ] && [ "${#services_args}" != 0 ]; then
  4414. charm=$(get_service_charm "${services_args[0]}") || exit 1
  4415. metadata=$(charm.metadata "$charm") || exit 1
  4416. SERVICE_TYPE="$(printf "%s" "$metadata" | shyaml get-value type 2>/dev/null)" || true
  4417. if [ "$SERVICE_TYPE" == "run-once" ]; then
  4418. run_service_hook dc-pre-run "${services_args[@]}" || exit 1
  4419. fi
  4420. fi
  4421. export SERVICE_PACK="${services_args[*]}"
  4422. ##
  4423. ## Docker-compose
  4424. ##
  4425. errlvl="0"
  4426. case "$action" in
  4427. up|start|stop|build|run)
  4428. ## force daemon mode for up
  4429. if [[ "$action" == "up" ]]; then
  4430. if ! array_member action_opts -d; then
  4431. action_opts+=("-d")
  4432. fi
  4433. if ! array_member action_opts --remove-orphans; then
  4434. action_opts+=("--remove-orphans")
  4435. fi
  4436. fi
  4437. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  4438. ;;
  4439. logs)
  4440. if ! array_member action_opts --tail; then ## force daemon mode for up
  4441. action_opts+=("--tail" "10")
  4442. fi
  4443. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  4444. ;;
  4445. "")
  4446. launch_docker_compose "${compose_opts[@]}"
  4447. ;;
  4448. graph)
  4449. graph $SERVICE_PACK
  4450. ;;
  4451. config)
  4452. ## removing the services
  4453. services=($(get_master_services "${action_posargs[@]}")) || exit 1
  4454. ## forcing docker-compose config to output the config file to stdout and not stderr
  4455. out=$(launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" 2>&1) || {
  4456. echo "$out"
  4457. exit 1
  4458. }
  4459. echo "$out"
  4460. warn "Runtime configuration modification (from relations) are not included here."
  4461. ;;
  4462. down)
  4463. if ! array_member action_opts --remove-orphans; then ## force daemon mode for up
  4464. debug "Adding a default argument of '--remove-orphans'"
  4465. action_opts+=("--remove-orphans")
  4466. fi
  4467. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}"
  4468. ;;
  4469. *)
  4470. if [ "$is_service_action" ]; then
  4471. run_service_action "$action_service" "$action" "${remainder_args[@]}"
  4472. errlvl="$?"
  4473. errlvl "$errlvl"
  4474. else
  4475. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  4476. fi
  4477. ;;
  4478. esac || exit 1
  4479. if [ "$post_hook" -a "${#services_args[@]}" != 0 -a -z "$no_hooks" -a -z "$no_post_deploy" ]; then
  4480. run_service_hook post_deploy "${services_args[@]}" || exit 1
  4481. fi
  4482. if [ "$action" == "run" -a "${#services_args}" != 0 ]; then
  4483. if [ "$SERVICE_TYPE" == "run-once" ]; then
  4484. run_service_hook dc-post-run "${services_args[@]}" || exit 1
  4485. fi
  4486. fi
  4487. clean_unused_docker_compose || exit 1
  4488. exit "$errlvl"