You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

5899 lines
200 KiB

  1. #!/bin/bash
  2. ##
  3. ## TODO:
  4. ## - subordinate container should really be able to modify base image of their master
  5. ## - this could be done through docker-update
  6. ## - I'm not happy with the current build using 'build/' directory, this should be
  7. ## changed to:
  8. ## - always have a base image (specified in metadata), and always have hooks/install
  9. ## executed and merge in image (like docker-build-charm).
  10. ## - container base image is ALWAYS the image of the master container... this brings
  11. ## questions about a double way to express inheritage (through relations as it is
  12. ## implemented now, or through this base-image ?)
  13. ## - the name of the scripts for relation (aka relation_name-relation-joined) is bad as
  14. ## reading the name in a hooks/ dir, there are no way to know if we are the target or
  15. ## the base of the relation.
  16. ## - we could leverage a 'relations/' dir on the root of the charm, with both:
  17. ## 'relations/provide/relation_name' and 'relations/receive/relation_name'
  18. ## - a very bad point with the actual naming is that we can't have a providing AND
  19. ## receiving a relation with same name.
  20. ## - The cache system should keep md5 of docker-compose and other things between runs
  21. ## - The cache system should use underlying function that have only arguments inputs.
  22. ## This will allow to cache completely without issues function in time.
  23. ## - would probably need instrospection in charm custom action to know if these need
  24. ## init or relations to be set up.
  25. ## - Be clear about when the SERVICE name is used and the CHARM name is used.
  26. ## - in case of service contained in another container
  27. ## - in normal case
  28. ## - in docker-compose, can't use charm name: if we want 2 instances of the same charm
  29. ## we are stuck. What will be unique is the name of the service.
  30. ## - some relations are configured in compose.yml but should not trigger the loading
  31. ## of necessary component (for instance, apache --> log-rotate), if log-rotate is
  32. ## not there, this link should considered optional.
  33. ## - Could probably allow an unexistent charm to be populated with only "docker-image:"
  34. ## of the same name. Although this should trigger a visible warning.
  35. #:-
  36. [ -e /etc/shlib ] && . /etc/shlib || {
  37. echo "Unsatisfied dependency. Please install 'kal-shlib-core'."
  38. exit 1
  39. }
  40. #:-
  41. include common
  42. include pretty
  43. include parse
  44. include charm
  45. include array
  46. include cla
  47. include docker
  48. depends shyaml docker
  49. exname="compose"
  50. version=0.1
  51. usage="$exname [COMPOSE_OPTS] [ACTION [ACTION_OPTS]]"
  52. help="\
  53. $WHITE$exname$NORMAL jobs is to run various shell scripts to build
  54. a running orchestrated and configured docker containers. These shell
  55. scripts will have the opportunity to build a 'docker-compose.yml'.
  56. Once init script and relations scripts are executed, $WHITE$exname$NORMAL
  57. delegate the launching to ${WHITE}docker-compose${NORMAL} by providing it
  58. the final 'docker-compose.yml'.
  59. $WHITE$exname$NORMAL also leverage charms to offer some additional custom
  60. actions per charm, which are simply other scripts that can be
  61. run without launching ${WHITE}docker-compose${NORMAL}.
  62. In compose message, color coding is enforced as such:
  63. - ${DARKCYAN}action$NORMAL,
  64. - ${DARKBLUE}relation$NORMAL,
  65. - ${DARKPINK}charm${NORMAL},
  66. - ${DARKYELLOW}service${NORMAL},
  67. - ${WHITE}option-name${NORMAL}/${WHITE}command-name${NORMAL}/${WHITE}Section-Title${NORMAL}
  68. $WHITE$exname$NORMAL reads '/etc/compose.conf' for global variables, and
  69. '/etc/compose.local.conf' for local host adjustements.
  70. "
  71. time_now() { date +%s.%3N; }
  72. time_elapsed() { echo "scale=3; $2 - $1" | bc; }
  73. ## XXXvlab: this doesn't seem to work when 'compose' is called in
  74. ## a hook of a charm.
  75. #[[ "${BASH_SOURCE[0]}" == "" ]] && SOURCED=true
  76. $(return >/dev/null 2>&1) && SOURCED=true
  77. errlvl() { return "${1:-1}"; }
  78. export -f errlvl
  79. if [ "$UID" == 0 ]; then
  80. CACHEDIR=${CACHEDIR:-/var/cache/compose}
  81. VARDIR=${VARDIR:-/var/lib/compose}
  82. else
  83. [ "$XDG_CONFIG_HOME" ] && CACHEDIR=${CACHEDIR:-$XDG_CONFIG_HOME/compose}
  84. [ "$XDG_DATA_HOME" ] && VARDIR=${VARDIR:-$XDG_DATA_HOME/compose}
  85. CACHEDIR=${CACHEDIR:-$HOME/.cache/compose}
  86. VARDIR=${VARDIR:-$HOME/.local/share/compose}
  87. fi
  88. export VARDIR CACHEDIR
  89. export SERVICE_STATE_PATH=${SERVICE_STATE_PATH:-/var/lib/compose/state}
  90. md5_compat() { md5sum | cut -c -32; }
  91. quick_cat_file() { quick_cat_stdin < "$1"; }
  92. quick_cat_stdin() { local IFS=''; while read -r line; do echo "$line"; done ; }
  93. export -f quick_cat_file quick_cat_stdin md5_compat
  94. p-err() {
  95. "$@"
  96. echo "$?"
  97. }
  98. export -f p-err
  99. wyq() {
  100. local exp="$1"
  101. yq e -e -0 "$1"
  102. printf "%s" "$?"
  103. }
  104. wyq-r() {
  105. local exp="$1"
  106. yq e -e -0 -r=false "$1"
  107. printf "%s" "$?"
  108. }
  109. err-d () {
  110. local msg="$*"
  111. shift
  112. err "$msg"
  113. print:traceback 1
  114. }
  115. export -f err-d
  116. print:traceback() {
  117. local omit_level="${1:-0}"
  118. if [ -z "$DEBUG" ]; then
  119. echo " Note: traceback available if you provide {--debug|-d} option." >&2
  120. return 0
  121. fi
  122. echo "${WHITE}Traceback (most recent call last):${NORMAL}" >&2
  123. local i
  124. for ((i=${#FUNCNAME[@]} - 1; i > "$omit_level"; i--)); do
  125. local file="${BASH_SOURCE[$i]}"
  126. local line="${BASH_LINENO[$i - 1]}"
  127. local func="${FUNCNAME[$i]}"
  128. if [[ -f "$file" ]]; then
  129. # Get total number of lines in the file
  130. local total_lines
  131. total_lines=$(wc -l < "$file")
  132. # Calculate start and end lines for context
  133. local start_line=$((line - 2))
  134. local end_line=$((line + 2))
  135. [[ $start_line -lt 1 ]] && start_line=1
  136. [[ $end_line -gt $total_lines ]] && end_line=$total_lines
  137. # Extract context lines
  138. mapfile -s $((start_line - 1)) -n $((end_line - start_line + 1)) context_lines < "$file"
  139. # Calculate minimal indentation
  140. local min_indent=9999
  141. for line_text in "${context_lines[@]}"; do
  142. if [[ -n "$line_text" ]]; then
  143. # Get leading whitespace
  144. local leading_whitespace="${line_text%%[![:space:]]*}"
  145. local indent=${#leading_whitespace}
  146. if [[ $indent -lt $min_indent ]]; then
  147. min_indent=$indent
  148. fi
  149. fi
  150. done
  151. # Remove minimal indentation from each line
  152. for idx in "${!context_lines[@]}"; do
  153. context_lines[$idx]="${context_lines[$idx]:$min_indent}"
  154. done
  155. else
  156. context_lines=("<source unavailable>")
  157. min_indent=0
  158. start_line=1
  159. end_line=1
  160. fi
  161. # Print the traceback frame
  162. echo " File \"$file\", line $line, in ${WHITE}$func${NORMAL}:"
  163. # Print the context with line numbers
  164. local current_line=$start_line
  165. for context_line in "${context_lines[@]}"; do
  166. context_line="${context_line%$'\n'}"
  167. if [[ $current_line -eq $line ]]; then
  168. echo " ${DARKYELLOW}$current_line${NORMAL} ${context_line}"
  169. else
  170. echo " ${DARKGRAY}$current_line${NORMAL} ${context_line}"
  171. fi
  172. ((current_line++))
  173. done
  174. done >&2
  175. }
  176. export -f print:traceback
  177. clean_cache() {
  178. local i=0
  179. for f in $(ls -t "$CACHEDIR/"*.cache.* 2>/dev/null | tail -n +500); do
  180. ((i++))
  181. rm -f "$f"
  182. done
  183. if (( i > 0 )); then
  184. debug "${WHITE}Cleaned cache:${NORMAL} Removed $((i)) elements (current cache size is $(du -sh "$CACHEDIR" | cut -f 1))"
  185. fi
  186. }
  187. export DEFAULT_COMPOSE_FILE
  188. ##
  189. ## Merge YAML files
  190. ##
  191. export _merge_yaml_common_code="
  192. import sys
  193. import yaml
  194. try:
  195. from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
  196. except ImportError: ## pragma: no cover
  197. sys.stderr.write('YAML code in pure python\n')
  198. exit(1)
  199. from yaml import SafeLoader, SafeDumper
  200. class MySafeLoader(SafeLoader): pass
  201. class MySafeDumper(SafeDumper): pass
  202. try:
  203. # included in standard lib from Python 2.7
  204. from collections import OrderedDict
  205. except ImportError:
  206. # try importing the backported drop-in replacement
  207. # it's available on PyPI
  208. from ordereddict import OrderedDict
  209. ## Ensure that there are no collision with legacy OrderedDict
  210. ## that could be used for omap for instance.
  211. class MyOrderedDict(OrderedDict):
  212. pass
  213. MySafeDumper.add_representer(
  214. MyOrderedDict,
  215. lambda cls, data: cls.represent_dict(data.items()))
  216. def construct_omap(cls, node):
  217. cls.flatten_mapping(node)
  218. return MyOrderedDict(cls.construct_pairs(node))
  219. MySafeLoader.add_constructor(
  220. yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
  221. construct_omap)
  222. ##
  223. ## Support local and global objects
  224. ##
  225. class EncapsulatedNode(object): pass
  226. def mk_encapsulated_node(s, node):
  227. method = 'construct_%s' % (node.id, )
  228. data = getattr(s, method)(node)
  229. class _E(data.__class__, EncapsulatedNode):
  230. pass
  231. _E.__name__ = str(node.tag)
  232. _E._node = node
  233. return _E(data)
  234. def represent_encapsulated_node(s, o):
  235. value = s.represent_data(o.__class__.__bases__[0](o))
  236. value.tag = o.__class__.__name__
  237. return value
  238. MySafeDumper.add_multi_representer(EncapsulatedNode,
  239. represent_encapsulated_node)
  240. MySafeLoader.add_constructor(None, mk_encapsulated_node)
  241. def fc(filename):
  242. with open(filename) as f:
  243. return f.read()
  244. def merge(*args):
  245. # sys.stderr.write('%r\n' % (args, ))
  246. args = [arg for arg in args if arg is not None]
  247. if len(args) == 0:
  248. return None
  249. if len(args) == 1:
  250. return args[0]
  251. if all(isinstance(arg, (int, basestring, bool, float)) for arg in args):
  252. return args[-1]
  253. elif all(isinstance(arg, list) for arg in args):
  254. res = []
  255. for arg in args:
  256. for elt in arg:
  257. if elt in res:
  258. res.remove(elt)
  259. res.append(elt)
  260. return res
  261. elif all(isinstance(arg, dict) for arg in args):
  262. keys = set()
  263. for arg in args:
  264. keys |= set(arg.keys())
  265. dct = {}
  266. for key in keys:
  267. sub_args = []
  268. for arg in args:
  269. if key in arg:
  270. sub_args.append(arg)
  271. try:
  272. dct[key] = merge(*(a[key] for a in sub_args))
  273. except NotImplementedError as e:
  274. raise NotImplementedError(
  275. e.args[0],
  276. '%s.%s' % (key, e.args[1]) if e.args[1] else key,
  277. e.args[2])
  278. if dct[key] is None:
  279. del dct[key]
  280. return dct
  281. else:
  282. raise NotImplementedError(
  283. 'Unsupported types: %s'
  284. % (', '.join(list(set(arg.__class__.__name__ for arg in args)))), '', args)
  285. return None
  286. def merge_cli(*args):
  287. try:
  288. c = merge(*args)
  289. except NotImplementedError as e:
  290. sys.stderr.write('Merging Failed: %s.\n%s\n'
  291. ' Values are:\n %s\n'
  292. % (e.args[0],
  293. ' Conflicting key is %r.' % e.args[1] if e.args[1] else
  294. ' Conflict at base of structure.',
  295. '\\n '.join('v%d: %r' % (i, a)
  296. for i, a in enumerate(e.args[2]))))
  297. exit(1)
  298. if c is not None:
  299. print '%s' % yaml.dump(c, default_flow_style=False, Dumper=MySafeDumper)
  300. "
  301. merge_yaml() {
  302. if ! [ -r "$state_tmpdir/merge_yaml.py" ]; then
  303. cat <<EOF > "$state_tmpdir/merge_yaml.py"
  304. $_merge_yaml_common_code
  305. merge_cli(*(yaml.load(fc(f), Loader=MySafeLoader) for f in sys.argv[1:]))
  306. EOF
  307. fi
  308. python "$state_tmpdir/merge_yaml.py" "$@"
  309. }
  310. export -f merge_yaml
  311. merge_yaml_str() {
  312. local entries="$@"
  313. if ! [ -r "$state_tmpdir/merge_yaml_str.py" ]; then
  314. cat <<EOF > "$state_tmpdir/merge_yaml_str.py" || return 1
  315. $_merge_yaml_common_code
  316. merge_cli(*(yaml.load(f, Loader=MySafeLoader) for f in sys.argv[1:]))
  317. EOF
  318. fi
  319. if ! python "$state_tmpdir/merge_yaml_str.py" "$@"; then
  320. err "Failed to merge yaml strings:"
  321. local s
  322. for s in "$@"; do
  323. printf " - \n"
  324. printf "%s\n" "$s" | prefix " ${GRAY}|$NORMAL "
  325. done >&2
  326. return 1
  327. fi
  328. }
  329. export -f merge_yaml_str
  330. yaml_get_values() {
  331. local sep=${1:-$'\n'} value input type first elt
  332. input=$(cat -)
  333. if [ -z "$input" ] || [[ "$input" =~ ^None|null$ ]]; then
  334. return 0
  335. fi
  336. type=$(e "$input" | shyaml get-type)
  337. value=
  338. case "$type" in
  339. "sequence")
  340. first=1
  341. while read-0 elt; do
  342. elt="$(e "$elt" | yaml_get_interpret)" || return 1
  343. [ "$elt" ] || continue
  344. if [ "$first" ]; then
  345. first=
  346. else
  347. value+="$sep"
  348. fi
  349. first=
  350. value+="$elt"
  351. done < <(e "$input" | shyaml -y get-values-0)
  352. ;;
  353. "struct")
  354. while read-0 val; do
  355. value+=$'\n'"$(e "$val" | yaml_get_interpret)" || return 1
  356. done < <(e "$input" | shyaml -y values-0)
  357. ;;
  358. "NoneType")
  359. value=""
  360. ;;
  361. "str"|*)
  362. value+="$(e "$input" | yaml_get_interpret)"
  363. ;;
  364. esac
  365. e "$value"
  366. }
  367. export -f yaml_get_values
  368. yaml_key_val_str() {
  369. local entries="$@"
  370. if ! [ -r "$state_tmpdir/yaml_key_val_str.py" ]; then
  371. cat <<EOF > "$state_tmpdir/yaml_key_val_str.py"
  372. $_merge_yaml_common_code
  373. print '%s' % yaml.dump(
  374. {
  375. yaml.load(sys.argv[1], Loader=MySafeLoader):
  376. yaml.load(sys.argv[2], Loader=MySafeLoader)
  377. },
  378. default_flow_style=False,
  379. Dumper=MySafeDumper,
  380. )
  381. EOF
  382. fi
  383. python "$state_tmpdir/yaml_key_val_str.py" "$@"
  384. }
  385. export -f yaml_key_val_str
  386. ##
  387. ## Docker
  388. ##
  389. docker_has_image() {
  390. local image="$1"
  391. images=$(docker images -q "$image" 2>/dev/null) || {
  392. err "docker images call has failed unexpectedly."
  393. return 1
  394. }
  395. [ -n "$images" ]
  396. }
  397. export -f docker_has_image
  398. docker_image_id() {
  399. local image="$1"
  400. image_id=$(docker inspect "$image" --format='{{.Id}}') || return 1
  401. echo "$image_id" # | tee "$cache_file"
  402. }
  403. export -f docker_image_id
  404. cached_cmd_on_image() {
  405. local image="$1" cache_file
  406. image_id=$(docker_image_id "$image") || return 1
  407. cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  408. if [ -e "$cache_file" ]; then
  409. # debug "$FUNCNAME: cache hit ($*)"
  410. quick_cat_stdin < "$cache_file"
  411. return 0
  412. fi
  413. shift
  414. out=$(docker run -i --rm --entrypoint /bin/sh "$image_id" -c "$*") || return 1
  415. echo "$out" | tee "$cache_file"
  416. }
  417. export -f cached_cmd_on_image
  418. cmd_on_base_image() {
  419. local service="$1" base_image
  420. shift
  421. base_image=$(service_ensure_image_ready "$service") || return 1
  422. docker run -i --rm --entrypoint /bin/bash "$base_image" -c "$*"
  423. }
  424. export -f cmd_on_base_image
  425. cached_cmd_on_base_image() {
  426. local service="$1" base_image cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  427. shift
  428. if [ -e "$cache_file" ]; then
  429. # debug "$FUNCNAME: cache hit ($*)"
  430. quick_cat_stdin < "$cache_file"
  431. return 0
  432. fi
  433. base_image=$(service_ensure_image_ready "$service") || return 1
  434. result=$(cached_cmd_on_image "$base_image" "$@") || return 1
  435. echo "$result" | tee "$cache_file"
  436. }
  437. export -f cached_cmd_on_base_image
  438. docker_update() {
  439. ## YYY: warning, we a storing important information in cache, cache can
  440. ## be removed.
  441. ## We want here to cache the last script on given service whatever that script was
  442. local service="$1" script="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$1" \
  443. previous_base_image stored_image_id
  444. shift
  445. shift
  446. ## this will build it if necessary
  447. base_image=$(service_ensure_image_ready "$service") || return 1
  448. ## XXXvlab: there are probably ways to avoid rebuilding that each time
  449. image_id="$(docker_image_id "$base_image")" || return 1
  450. if [ -e "$cache_file" ]; then
  451. info "Cache file exists"
  452. read-0 previous_base_image stored_image_id < <(cat "$cache_file")
  453. info "previous: $previous_base_image"
  454. info "stored: $stored_image_id"
  455. else
  456. info "No cache file $cache_file"
  457. previous_base_image=""
  458. fi
  459. if [ "$previous_base_image" -a "$stored_image_id" == "$image_id" ]; then
  460. info "Resetting $base_image to $previous_base_image"
  461. docker tag "$previous_base_image" "$base_image" || return 1
  462. image_id="$(docker_image_id "$base_image")" || return 1
  463. else
  464. previous_base_image="$image_id"
  465. fi
  466. info "Updating base image: $base_image (hash: $image_id)"
  467. echo "$script" | dupd --debug -u "$base_image" -- "$@" || {
  468. err "Failed updating base image"
  469. return 1
  470. }
  471. new_image_id="$(docker_image_id "$base_image")"
  472. [ "$new_image_id" == "$previous_base_image" ] && {
  473. err "Image was not updated correctly (same id)."
  474. return 1
  475. }
  476. printf "%s\0" "$previous_base_image" "$new_image_id" > "$cache_file"
  477. info "Wrote cache file $cache_file"
  478. }
  479. export -f docker_update
  480. image_exposed_ports_0() {
  481. local image="$1"
  482. docker inspect --format='{{range $p, $conf := .Config.ExposedPorts}}{{$p}}{{"\x00"}}{{end}}' "$image"
  483. }
  484. export -f image_exposed_ports_0
  485. ## feature not yet included in docker: https://github.com/moby/moby/issues/16079
  486. docker_image_export_dir() {
  487. local image="$1" src="$2" dst="$3" container_id
  488. (
  489. container_id=$(docker create "$image") || exit 1
  490. trap_add EXIT,ERR "docker rm \"$container_id\" >/dev/null"
  491. docker cp "$container_id":"$src" "$dst"
  492. )
  493. }
  494. export -f docker_image_export_dir
  495. service_base_image_export_dir() {
  496. local service="$1" src="$2" dst="$3" base_image
  497. shift
  498. base_image=$(service_ensure_image_ready "$service") || return 1
  499. docker_image_export_dir "$base_image" "$src" "$dst"
  500. }
  501. export -f service_base_image_export_dir
  502. service_base_image_id() {
  503. local service="$1" src="$2" dst="$3" base_image
  504. shift
  505. base_image=$(service_ensure_image_ready "$service") || return 1
  506. docker inspect "$base_image" --format="{{ .Id }}"
  507. }
  508. export -f service_base_image_id
  509. ##
  510. ## Generic
  511. ##
  512. fn.exists() {
  513. declare -F "$1" >/dev/null
  514. }
  515. str_pattern_matches() {
  516. local str="$1"
  517. shift
  518. for pattern in "$@"; do
  519. eval "[[ \"$str\" == $pattern ]]" && return 0
  520. done
  521. return 1
  522. }
  523. str_matches() {
  524. local str="$1"
  525. shift
  526. for pattern in "$@"; do
  527. [[ "$str" == "$pattern" ]] && return 0
  528. done
  529. return 1
  530. }
  531. gen_password() {
  532. local l=( {a..z} {A..Z} {0..9} ) nl="${#l[@]}" size=${1:-16}
  533. while ((size--)); do
  534. echo -n "${l[$((RANDOM * nl / 32768))]}"
  535. done
  536. echo
  537. }
  538. export -f gen_password
  539. file_put() {
  540. local TARGET="$1"
  541. mkdir -p "$(dirname "$TARGET")" &&
  542. cat - > "$TARGET"
  543. }
  544. export -f file_put
  545. file_put_0() {
  546. local TARGET="$1"
  547. mkdir -p "$(dirname "$TARGET")" &&
  548. cat > "$TARGET"
  549. }
  550. export -f file_put_0
  551. fetch_file() {
  552. local src="$1"
  553. case "$src" in
  554. *"://"*)
  555. err "Unsupported target scheme."
  556. return 1
  557. ;;
  558. *)
  559. ## Try direct
  560. if ! [ -r "$src" ]; then
  561. err "File '$src' not found/readable."
  562. return 1
  563. fi
  564. cat "$src" || return 1
  565. ;;
  566. esac
  567. }
  568. export -f fetch_file
  569. ## receives stdin content to decompress on stdout
  570. ## stdout content should be tar format.
  571. uncompress_file() {
  572. local filename="$1"
  573. ## Warning, the content of the file is already as stdin, the filename
  574. ## is there to hint for correct decompression.
  575. case "$filename" in
  576. *".gz")
  577. gunzip
  578. ;;
  579. *".bz2")
  580. bunzip2
  581. ;;
  582. *)
  583. cat
  584. ;;
  585. esac
  586. }
  587. export -f uncompress_file
  588. get_file() {
  589. local src="$1"
  590. fetch_file "$src" | uncompress_file "$src"
  591. }
  592. export -f get_file
  593. ##
  594. ## Common database lib
  595. ##
  596. _clean_docker() {
  597. local _DB_NAME="$1" container_id="$2"
  598. (
  599. set +e
  600. debug "Removing container $_DB_NAME"
  601. docker stop "$container_id"
  602. docker rm "$_DB_NAME"
  603. docker network rm "${_DB_NAME}"
  604. rm -vf "$state_tmpdir/${_DB_NAME}.state"
  605. ) >&2
  606. }
  607. export -f _clean_docker
  608. get_service_base_image_dir_uid_gid() {
  609. local service="$1" dir="$2" uid_gid
  610. uid_gid=$(cached_cmd_on_base_image "$service" "stat -c '%u %g' '$dir'") || {
  611. debug "Failed to query '$dir' uid in ${DARKYELLOW}$service${NORMAL} base image."
  612. return 1
  613. }
  614. info "uid and gid from ${DARKYELLOW}$service${NORMAL}:$dir is '$uid_gid'"
  615. echo "$uid_gid"
  616. }
  617. export -f get_service_base_image_dir_uid_gid
  618. get_service_type() {
  619. if [ -z "$CHARM_STORE_HASH" ]; then
  620. err-d "Expected \$CHARM_STORE_HASH to be set."
  621. return 1
  622. fi
  623. local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH"
  624. if [ -z "$service" ]; then
  625. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  626. return 1
  627. fi
  628. if [ -e "$cache_file" ]; then
  629. # debug "$FUNCNAME: cache hit ($*)"
  630. cat "$cache_file"
  631. return 0
  632. fi
  633. master_target_service="$(get_top_master_service_for_service "$service")" || return 1
  634. charm=$(get_service_charm "$master_target_service") || return 1
  635. metadata=$(charm.metadata "$charm") || return 1
  636. printf "%s" "$metadata" | shyaml get-value type service 2>/dev/null |
  637. tee "$cache_file"
  638. }
  639. export -f get_service_type
  640. are_files_locked_in_dir() {
  641. local dir="$1" device hdev ldev
  642. device=$(stat -c %d "$dir") || {
  643. err "Can't stat '$dir'."
  644. return 1
  645. }
  646. device=$(printf "%04x" $device)
  647. hdev=${device:0:2}
  648. ldev=${device:2:2}
  649. inodes=$(find "$dir" -printf ':%i:\n')
  650. found=
  651. while read -r inode; do
  652. debug "try inode:$inode"
  653. if [[ "$inodes" == *":$inode:"* ]]; then
  654. found=1
  655. break
  656. fi
  657. done < <(cat /proc/locks | grep " $hdev:$ldev:" | sed -r "s/^.*$hdev:$ldev:([0-9]+).*$/\1/g")
  658. [ "$found" ]
  659. }
  660. export -f are_files_locked_in_dir
  661. set_db_params() {
  662. local docker_ip="$1" docker_network="$2"
  663. if [ -z "$DB_PARAMS_LOADED" ]; then
  664. DB_PARAMS_LOADED=1
  665. _set_db_params "$docker_ip" "$docker_network"
  666. fi
  667. }
  668. export -f set_db_params
  669. export _PID="$$"
  670. ensure_db_docker_running () {
  671. local _STATE_FILE errlvl project
  672. _DB_NAME="db_${DB_NAME}_${_PID}"
  673. _STATE_FILE="$state_tmpdir/${_DB_NAME}.state"
  674. if [ -e "$_STATE_FILE" ]; then
  675. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$(cat "$_STATE_FILE")"
  676. debug "Re-using previous docker/connection '$DOCKER_IP'."
  677. set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  678. return 0
  679. fi
  680. if [ -e "$state_tmpdir/${_DB_NAME}.working" ]; then
  681. ## avoid recursive calls.
  682. if [ -z "$DOCKER_IP" ]; then
  683. err "Currently figuring up DOCKER_IP, please set it yourself before this call if needed."
  684. return 1
  685. else
  686. debug "ignoring recursive call of 'ensure_db_docker_running'."
  687. fi
  688. return 0
  689. fi
  690. touch "$state_tmpdir/${_DB_NAME}.working"
  691. docker rm "$_DB_NAME" 2>/dev/null || true
  692. host_db_working_dir="$HOST_DATASTORE/${SERVICE_NAME}$DB_DATADIR"
  693. if is_db_locked; then
  694. info "Some process is using '$host_db_working_dir'. Trying to find a docker that would do this..."
  695. found=
  696. for docker_id in $(docker ps -q); do
  697. has_volume_mounted=$(
  698. docker inspect \
  699. --format "{{range .Mounts}}{{if eq .Destination \"$DB_DATADIR\"}}{{.Source}}{{end}}{{end}}" \
  700. "$docker_id")
  701. if [ "$has_volume_mounted" == "$host_db_working_dir" ]; then
  702. info "docker '$docker_id' uses '$has_volume_mounted'."
  703. project=$(docker inspect "$docker_id" \
  704. --format "{{index .Config.Labels \"compose.project\" }}") || continue
  705. info "docker '$docker_id' is from project '$project' (current project is '$PROJECT_NAME')."
  706. [ "$project" == "$PROJECT_NAME" ] || continue
  707. found="$docker_id"
  708. break
  709. fi
  710. done
  711. if [ -z "$found" ]; then
  712. err "Please shutdown any other docker using this directory."
  713. return 1
  714. fi
  715. export container_id="$found"
  716. info "Found docker $docker_id is already running."
  717. else
  718. verb "Database is not locked."
  719. if ! docker_has_image "$DOCKER_BASE_IMAGE"; then
  720. err "Unexpected missing docker image $DOCKER_BASE_IMAGE."
  721. return 1
  722. fi
  723. _set_server_db_params || return 1
  724. debug docker network create "$_DB_NAME"
  725. if ! network_id=$(docker network create "$_DB_NAME"); then
  726. err "'docker network create $_DB_NAME' failed !"
  727. _clean_docker "$_DB_NAME" "$container_id"
  728. rm "$state_tmpdir/${_DB_NAME}.working"
  729. return 1
  730. fi
  731. debug docker run -d \
  732. --name "$_DB_NAME" \
  733. "${server_docker_opts[@]}" \
  734. --network "$_DB_NAME" \
  735. -v "$host_db_working_dir:$DB_DATADIR" \
  736. "$DOCKER_BASE_IMAGE"
  737. if ! container_id=$(
  738. docker run -d \
  739. --name "$_DB_NAME" \
  740. "${server_docker_opts[@]}" \
  741. --network "$_DB_NAME" \
  742. -v "$host_db_working_dir:$DB_DATADIR" \
  743. "$DOCKER_BASE_IMAGE"
  744. ); then
  745. err "'docker run' failed !"
  746. _clean_docker "$_DB_NAME" "$container_id"
  747. rm "$state_tmpdir/${_DB_NAME}.working"
  748. return 1
  749. fi
  750. trap_add EXIT,ERR "_clean_docker \"$_DB_NAME\" \"$container_id\""
  751. fi
  752. if docker_ip=$(wait_for_docker_ip "$container_id"); then
  753. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip"
  754. echo "$docker_ip" > "$_STATE_FILE"
  755. debug "written '$_STATE_FILE'"
  756. rm "$state_tmpdir/${_DB_NAME}.working"
  757. set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  758. return 0
  759. else
  760. errlvl="$?"
  761. err "Db not found (errlvl: $errlvl). Tail of docker logs follows:"
  762. docker logs --tail=5 "$container_id" 2>&1 | prefix " | " >&2
  763. rm "$state_tmpdir/${_DB_NAME}.working"
  764. return "$errlvl"
  765. fi
  766. }
  767. export -f ensure_db_docker_running
  768. ## Require to set $db_docker_opts if needed, and $DB_PASSFILE
  769. ##
  770. _dcmd() {
  771. local docker_opts command="$1"
  772. shift
  773. debug "Db> $command $@"
  774. if [ "$HOST_DB_PASSFILE" -a -f "$LOCAL_DB_PASSFILE" -a "$CLIENT_DB_PASSFILE" ]; then
  775. verb "Found and using '$HOST_DB_PASSFILE' as '$CLIENT_DB_PASSFILE'."
  776. docker_opts=("${db_docker_opts[@]}" "-v" "$HOST_DB_PASSFILE:$CLIENT_DB_PASSFILE")
  777. else
  778. docker_opts=("${db_docker_opts[@]}")
  779. fi
  780. ## XXXX was here: actualy, we need only connection between this version and the client version
  781. debug docker run -i --rm \
  782. "${docker_opts[@]}" \
  783. --entrypoint "$command" "$DOCKER_BASE_IMAGE" "${db_cmd_opts[@]}" "$@"
  784. docker run -i --rm \
  785. "${docker_opts[@]}" \
  786. --entrypoint "$command" "$DOCKER_BASE_IMAGE" "${db_cmd_opts[@]}" "$@"
  787. }
  788. export -f _dcmd
  789. ## Executes code through db
  790. dcmd() {
  791. local fun
  792. [ "$DB_NAME" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_NAME."
  793. [ "$DB_DATADIR" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_DATADIR."
  794. # [ "$DB_PASSFILE" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_PASSFILE."
  795. [ "$_PID" ] || print_syntax_error "$FUNCNAME: You must provide \$_PID."
  796. for fun in is_db_locked _set_db_params ddb; do
  797. [ "$(type -t "$fun")" == "function" ] ||
  798. print_syntax_error "$FUNCNAME: You must provide function '$fun'."
  799. done
  800. ensure_db_docker_running </dev/null || return 1
  801. _dcmd "$@"
  802. }
  803. export -f dcmd
  804. get_docker_ips() {
  805. local name="$1" ip format network_id
  806. if ! docker inspect --format='{{ .NetworkSettings.Networks }}' "$name" >/dev/null 2>&1; then
  807. echo "default:$(docker inspect --format='{{ .NetworkSettings.IPAddress }}' "$name" 2>/dev/null)"
  808. else
  809. format='{{range $name, $conf := .NetworkSettings.Networks}}{{$name}}{{"\x00"}}{{$conf.IPAddress}}{{"\x00"}}{{end}}'
  810. while read-0 network_id ip; do
  811. printf "%s:%s\n" "$network_id" "$ip"
  812. done < <(docker inspect --format="$format" "$name")
  813. fi
  814. }
  815. export -f get_docker_ips
  816. get_docker_ip() {
  817. local name="$1"
  818. get_docker_ips "$name"
  819. }
  820. export -f get_docker_ip
  821. wait_docker_ip() {
  822. local name="$1" timeout="${2:-15}" timeout_count=0 docker_ip=
  823. start=$SECONDS
  824. while [ -z "$docker_ip" ]; do
  825. sleep 0.5
  826. docker_ip=$(get_docker_ip "$name") && break
  827. elapsed=$((SECONDS - start))
  828. if ((elapsed > timeout)); then
  829. err "${RED}timeout error${NORMAL}(${timeout}s):" \
  830. "Could not find '$name' docker container's IP."
  831. return 1
  832. fi
  833. [ "$elapsed" == "$old_elapsed" ] ||
  834. verb "Waiting for docker $name... ($elapsed/$timeout)"
  835. old_elapsed="$elapsed"
  836. done
  837. verb "Found docker $name network and IP: $docker_ip"
  838. echo "$docker_ip"
  839. }
  840. export -f wait_docker_ip
  841. wait_for_tcp_port() {
  842. local network=$1 host_port=$2 timeout=${3:-60}
  843. verb "Trying to connect to $host_port"
  844. bash_image=${DEFAULT_BASH_IMAGE:-docker.0k.io/bash}
  845. #echo docker run --rm -i --network "$network" "$bash_image" >&2
  846. docker run --rm -i --network "$network" "$bash_image" <<EOF
  847. start=\$SECONDS
  848. while true; do
  849. timeout 1 bash -c "</dev/tcp/${host_port/://}" >/dev/null 2>&1 && break
  850. sleep 0.2
  851. if [ "\$((SECONDS - start))" -gt "$timeout" ]; then
  852. exit 1
  853. fi
  854. done
  855. exit 0
  856. EOF
  857. if [ "$?" != 0 ]; then
  858. err "${RED}timeout error${NORMAL}(${timeout}s):"\
  859. "Could not connect to $host_port."
  860. return 1
  861. fi
  862. return 0
  863. }
  864. export -f wait_for_tcp_port
  865. ## Warning: requires a ``ddb`` matching current database to be checked
  866. wait_for_docker_ip() {
  867. local name=$1 DOCKER_IP= DOCKER_NETWORK= docker_ips= docker_ip= elapsed timeout=10
  868. docker_ip=$(wait_docker_ip "$name" 5) || return 1
  869. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip"
  870. if ! str_is_ipv4 "$DOCKER_IP"; then
  871. err "internal 'wait_docker_ip' did not return a valid IP. Returned IP is '$DOCKER_IP'."
  872. return 1
  873. fi
  874. set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  875. while read-0 port; do
  876. IFS="/" read port type <<<"$port"
  877. [ "$type" == "tcp" ] || continue
  878. wait_for_tcp_port "$DOCKER_NETWORK" "$DOCKER_IP:${port}" || return 17
  879. info "Host/Port $DOCKER_IP:${port} checked ${GREEN}open${NORMAL}."
  880. ## XXXvlab: what to do with more than one port ?
  881. break
  882. done < <(image_exposed_ports_0 "$container_id")
  883. ## Checking direct connection
  884. timeout=120
  885. start=$SECONDS
  886. while true; do
  887. if err=$(echo "$check_command" | ddb 2>&1 >/dev/null); then
  888. break
  889. fi
  890. if ! [[ "$err" == *"the database system is starting up" ]]; then
  891. err "${RED}db connection error${NORMAL}:" \
  892. "Could not connect to db on $DOCKER_IP container's IP."
  893. echo " Note: IP up, TCP ports is(are) open" >&2
  894. if [ "$err" ]; then
  895. echo " Error:" >&2
  896. printf "%s\n" "$err" | prefix " ${RED}!${NORMAL} " >&2
  897. fi
  898. return 18
  899. fi
  900. debug "Got 'database system is starting up' error."
  901. elapsed=$((SECONDS - start))
  902. if ((elapsed > timeout)); then
  903. err "${RED}db connection error${NORMAL}:"\
  904. "Could not connect to db on $DOCKER_IP" \
  905. "container's IP. (IP up, TCP ports is(are) open, sql answer after ${timeout}s)"
  906. return 1
  907. fi
  908. sleep 0.2
  909. done
  910. echo "${DOCKER_NETWORK}:${DOCKER_IP}"
  911. return 0
  912. }
  913. export -f wait_for_docker_ip
  914. docker_add_host_declaration() {
  915. local src_docker=$1 domain=$2 dst_docker=$3 dst_docker_ip= dst_docker_network
  916. dst_docker_ip=$(wait_docker_ip "$dst_docker") || exit 1
  917. IFS=: read dst_docker_ip dst_docker_network <<<"$dst_docker_ip"
  918. docker exec -i "$src_docker" bash <<EOF
  919. if cat /etc/hosts | grep -E "^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+$domain\$" > /dev/null 2>&1; then
  920. sed -ri "s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+$domain\$/$dst_docker_ip $domain/g" /etc/hosts
  921. else
  922. echo "$dst_docker_ip $domain" >> /etc/hosts
  923. fi
  924. EOF
  925. }
  926. export -f docker_add_host_declaration
  927. get_running_containers_for_service() {
  928. local service="$1" project="$2"
  929. project=${project:-$PROJECT_NAME}
  930. [ -n "$project" ] || {
  931. err "No project name was defined yet."
  932. return 1
  933. }
  934. docker ps \
  935. --filter label="compose.project=$project" \
  936. --filter label="compose.master-service=$service" \
  937. --format="{{.ID}}"
  938. }
  939. export -f get_running_containers_for_service
  940. get_container_network_ips() {
  941. local container="$1"
  942. docker inspect "$container" \
  943. --format='{{range $key, $val :=.NetworkSettings.Networks}}{{$key}}{{"\x00"}}{{$val.IPAddress}}{{"\x00"}}{{end}}'
  944. }
  945. export -f get_container_network_ips
  946. get_container_network_ip() {
  947. local container="$1"
  948. while read-0 network ip; do
  949. printf "%s\0" "$network" "$ip"
  950. break
  951. done < <(get_container_network_ips "$container")
  952. }
  953. export -f get_container_network_ip
  954. ##
  955. ## Internal Process
  956. ##
  957. get_docker_compose_links() {
  958. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  959. deps master_service master_target_service _relation_name \
  960. target_service _relation_config tech_dep
  961. if [ -z "$service" ]; then
  962. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  963. return 1
  964. fi
  965. if [ -e "$cache_file" ]; then
  966. # debug "$FUNCNAME: cache hit ($*)"
  967. cat "$cache_file"
  968. return 0
  969. fi
  970. master_service=$(get_top_master_service_for_service "$service") || return 1
  971. deps=()
  972. while read-0 _relation_name target_service _relation_config tech_dep; do
  973. master_target_service="$(get_top_master_service_for_service "$target_service")" || return 1
  974. [ "$master_service" == "$master_target_service" ] && continue
  975. type="$(get_service_type "$target_service")" || return 1
  976. [ "$type" == "stub" ] && continue
  977. [ "$type" == "run-once" ] && continue
  978. if [ "$tech_dep" == "reversed" ]; then
  979. deps+=("$(echo -en "$master_target_service:\n links:\n - $master_service")")
  980. elif [[ "$tech_dep" =~ ^(True|true)$ ]]; then
  981. deps+=("$(echo -en "$master_service:\n links:\n - $master_target_service")")
  982. fi
  983. ## XXXvlab: an attempt to add depends_on, but this doesn't work well actually
  984. ## as there's a circular dependency issue. We don't really want the full feature
  985. ## of depends_on, but just to add it as targets when doing an 'up'
  986. # deps+=("$(echo -en "$master_service:\n depends_on:\n - $master_target_service")")
  987. done < <(get_service_relations "$service")
  988. merge_yaml_str "${deps[@]}" | tee "$cache_file" || return 1
  989. if [ "${PIPESTATUS[0]}" != 0 ]; then
  990. rm "$cache_file"
  991. err "Failed to merge YAML from all ${WHITE}links${NORMAL} dependencies."
  992. return 1
  993. fi
  994. }
  995. _get_docker_compose_opts() {
  996. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  997. compose_def master_service docker_compose_opts
  998. if [ -z "$service" ]; then
  999. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  1000. return 1
  1001. fi
  1002. if [ -e "$cache_file" ]; then
  1003. # debug "$FUNCNAME: cache hit ($*)"
  1004. cat "$cache_file"
  1005. return 0
  1006. fi
  1007. compose_def="$(get_compose_service_def "$service")" || return 1
  1008. master_service="$(get_top_master_service_for_service "$service")"
  1009. if docker_compose_opts=$(echo "$compose_def" | shyaml get-value -y "docker-compose" 2>/dev/null); then
  1010. yaml_key_val_str "$master_service" "$docker_compose_opts"
  1011. fi | tee "$cache_file"
  1012. if [ "${PIPESTATUS[0]}" != 0 ]; then
  1013. rm "$cache_file"
  1014. return 1
  1015. fi
  1016. }
  1017. ##
  1018. ## By Reading the metadata.yml, we create a docker-compose.yml mixin.
  1019. ## Some metadata.yml (of subordinates) will indeed modify other
  1020. ## services than themselves.
  1021. _get_docker_compose_service_mixin() {
  1022. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1023. links_yaml base_mixin links_yaml docker_compose_options \
  1024. charm charm_part
  1025. if [ -z "$service" ]; then
  1026. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  1027. return 1
  1028. fi
  1029. if [ -e "$cache_file" ]; then
  1030. # debug "$FUNCNAME: cache hit ($*)"
  1031. cat "$cache_file"
  1032. return 0
  1033. fi
  1034. type=$(get_service_type "$service") || return 1
  1035. [ "$type" == "stub" ] && return 0
  1036. master_service=$(get_top_master_service_for_service "$service") || {
  1037. err "Failed to get top master service for service $DARKYELLOW$service$NORMAL"
  1038. return 1
  1039. }
  1040. ## The compose part
  1041. base_mixin="$master_service:
  1042. labels:
  1043. - compose.service=$service
  1044. - compose.master-service=${master_service}
  1045. - compose.project=$(get_default_project_name)"
  1046. links_yaml=$(get_docker_compose_links "$service") || return 1
  1047. docker_compose_options=$(_get_docker_compose_opts "$service") || return 1
  1048. ## the charm part
  1049. charm_part=$(get_docker_compose_mixin_from_metadata "$service") || return 1
  1050. ## Merge results
  1051. if [ "$charm_part" ]; then
  1052. charm_yaml="$(yaml_key_val_str "$master_service" "$charm_part")" || return 1
  1053. merge_yaml_str "$base_mixin" "$links_yaml" "$charm_yaml" "$docker_compose_options" || return 1
  1054. else
  1055. merge_yaml_str "$base_mixin" "$links_yaml" "$docker_compose_options" || return 1
  1056. fi | tee "$cache_file"
  1057. if [ "${PIPESTATUS[0]}" != 0 ]; then
  1058. err "Failed to constitute the base YAML for service '${DARKYELLOW}$service${NORMAL}'"
  1059. rm "$cache_file"
  1060. return 1
  1061. fi
  1062. }
  1063. export -f _get_docker_compose_service_mixin
  1064. ##
  1065. ## Get full `docker-compose.yml` format for all listed services (and
  1066. ## their deps)
  1067. ##
  1068. ## @export
  1069. ## @cache: !system !nofail +stdout
  1070. get_docker_compose () {
  1071. if [ -z "$CHARM_STORE_HASH" ]; then
  1072. err-d "Expected \$CHARM_STORE_HASH to be set."
  1073. return 1
  1074. fi
  1075. if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
  1076. err-d "Expected \$COMPOSE_YML_CONTENT_HASH to be set."
  1077. return 1
  1078. fi
  1079. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$CHARM_STORE_HASH" "$COMPOSE_YML_CONTENT_HASH")" \
  1080. entries services service start docker_compose_services
  1081. if [ -e "$cache_file" ]; then
  1082. # debug "$FUNCNAME: cache hit ($*) $cache_file"
  1083. touch "$cache_file" || return 1
  1084. cp "$cache_file"{,.wip} || return 1
  1085. export _CURRENT_DOCKER_COMPOSE="$cache_file.wip"
  1086. cat "$cache_file" || return 1
  1087. return 0
  1088. fi
  1089. ##
  1090. ## Adding sub services configurations
  1091. ##
  1092. declare -A entries
  1093. start_compilation=$SECONDS
  1094. debug "Compiling 'docker-compose.yml' base for ${DARKYELLOW}$*$NORMAL..."
  1095. for target_service in "$@"; do
  1096. start=$SECONDS
  1097. services=($(get_ordered_service_dependencies "$target_service")) || {
  1098. err "Failed to get dependencies for $DARKYELLOW$target_service$NORMAL"
  1099. return 1
  1100. }
  1101. if [ "$DEBUG" ]; then
  1102. debug " $DARKYELLOW$target_service$NORMAL deps:$DARKYELLOW" \
  1103. "${services[@]::$((${#services[@]} - 1))}" \
  1104. "$NORMAL$GRAY(in $((SECONDS - start))s)$NORMAL"
  1105. fi
  1106. for service in "${services[@]}"; do
  1107. if [ "${entries[$service]}" ]; then
  1108. ## Prevent double inclusion of same service if this
  1109. ## service is deps of two or more of your
  1110. ## requirements.
  1111. continue
  1112. fi
  1113. ## mark the service as "loaded" as well as it's containers
  1114. ## if this is a subordinate service
  1115. start_service=$SECONDS
  1116. entries[$service]=$(_get_docker_compose_service_mixin "$service") || {
  1117. err "Failed to get service mixin for $DARKYELLOW$service$NORMAL"
  1118. return 1
  1119. }
  1120. debug " Applied $DARKYELLOW$service$NORMAL charm metadata mixins $GRAY(in $((SECONDS - start_service))s)$NORMAL"
  1121. done
  1122. debug " ..finished all mixins for $DARKYELLOW$target_service$NORMAL $GRAY(in $((SECONDS - start))s)$NORMAL"
  1123. done
  1124. docker_compose_services=$(merge_yaml_str "${entries[@]}") || {
  1125. err "Failed to merge YAML services entries together."
  1126. return 1
  1127. }
  1128. base_v2="version: '2.1'"
  1129. merge_yaml_str "$(yaml_key_val_str "services" "$docker_compose_services")" \
  1130. "$base_v2" > "$cache_file" || return 1
  1131. cp "$cache_file"{,.wip} || return 1
  1132. export _CURRENT_DOCKER_COMPOSE="$cache_file.wip"
  1133. cat "$_CURRENT_DOCKER_COMPOSE" || return 1
  1134. debug " ..compilation of base 'docker-compose.yml' done $GRAY(in $((SECONDS - start_compilation))s)$NORMAL" || true
  1135. # debug " ** ${WHITE}docker-compose.yml${NORMAL}:"
  1136. # debug "$_current_docker_compose"
  1137. }
  1138. export -f get_docker_compose
  1139. _get_compose_service_def_cached () {
  1140. local service="$1" docker_compose="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  1141. if [ -e "$cache_file" ]; then
  1142. #debug "$FUNCNAME: STATIC cache hit"
  1143. cat "$cache_file" &&
  1144. touch "$cache_file" || return 1
  1145. return 0
  1146. fi
  1147. value=$(echo "$docker_compose" | shyaml get-value "${service//./\\.}" 2>/dev/null)
  1148. [ "$value" == None ] && value=""
  1149. if ! echo "$value" | shyaml get-value "charm" >/dev/null 2>&1; then
  1150. if charm.exists "$service"; then
  1151. value=$(merge_yaml <(echo "charm: $service") <(echo "$value")) || {
  1152. err "Can't merge YAML infered 'charm: $service' with base ${DARKYELLOW}$service${NORMAL} YAML definition."
  1153. return 1
  1154. }
  1155. else
  1156. err "No ${WHITE}charm${NORMAL} value for service $DARKYELLOW$service$NORMAL" \
  1157. "in compose, nor same name charm found."
  1158. return 1
  1159. fi
  1160. fi
  1161. echo "$value" | tee "$cache_file" || return 1
  1162. # if [ "${PIPESTATUS[0]}" != 0 ]; then
  1163. # rm "$cache_file"
  1164. # return 1
  1165. # fi
  1166. return 0
  1167. # if [ "${PIPESTATUS[0]}" != 0 -o \! -s "$cache_file" ]; then
  1168. # rm "$cache_file"
  1169. # err "PAS OK $service: $value"
  1170. # return 1
  1171. # fi
  1172. }
  1173. export -f _get_compose_service_def_cached
  1174. ## XXXvlab: a lot to be done to cache the results
  1175. get_compose_service_def () {
  1176. if [ -z "$COMBINED_HASH" ]; then
  1177. err-d "Expected \$COMBINED_HASH to be set."
  1178. return 1
  1179. fi
  1180. local service="$1" docker_compose cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \
  1181. result
  1182. if [ -e "$cache_file" ]; then
  1183. #debug "$FUNCNAME: SESSION cache hit"
  1184. cat "$cache_file" || return 1
  1185. return 0
  1186. fi
  1187. [ -z "$service" ] && print_syntax_error "Missing service as first argument."
  1188. docker_compose=$(get_compose_yml_content) || return 1
  1189. result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1
  1190. charm=$(e "$result" | shyaml get-value charm 2>/dev/null) || return 1
  1191. metadata=$(charm.metadata "$charm") || return 1
  1192. if default_options=$(printf "%s" "$metadata" | shyaml -y -q get-value default-options); then
  1193. default_options=$(yaml_key_val_str "options" "$default_options") || return 1
  1194. result=$(merge_yaml_str "$default_options" "$result") || return 1
  1195. fi
  1196. echo "$result" | tee "$cache_file" || return 1
  1197. }
  1198. export -f get_compose_service_def
  1199. _get_service_charm_cached () {
  1200. local service="$1" service_def="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  1201. if [ -e "$cache_file" ]; then
  1202. # debug "$FUNCNAME: cache hit $1"
  1203. cat "$cache_file" &&
  1204. touch "$cache_file" || return 1
  1205. return 0
  1206. fi
  1207. charm=$(echo "$service_def" | shyaml get-value charm 2>/dev/null)
  1208. if [ -z "$charm" ]; then
  1209. err "Missing ${WHITE}charm${NORMAL} value in service $DARKYELLOW$service$NORMAL definition."
  1210. return 1
  1211. fi
  1212. echo "$charm" | tee "$cache_file" || return 1
  1213. }
  1214. export -f _get_service_charm_cached
  1215. get_service_charm () {
  1216. local service="$1"
  1217. if [ -z "$service" ]; then
  1218. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  1219. return 1
  1220. fi
  1221. service_def=$(get_compose_service_def "$service") || return 1
  1222. _get_service_charm_cached "$service" "$service_def"
  1223. }
  1224. export -f get_service_charm
  1225. ## built above the docker-compose abstraction, so it relies on the
  1226. ## full docker-compose.yml to be already built.
  1227. get_service_def () {
  1228. local service="$1" def
  1229. if [ -z "$_CURRENT_DOCKER_COMPOSE" ]; then
  1230. err "${FUNCNAME[0]} is meant to be called after"\
  1231. "\$_CURRENT_DOCKER_COMPOSE has been calculated."
  1232. echo " Called by:" >&2
  1233. printf " - %s\n" "${FUNCNAME[@]:1}" >&2
  1234. return 1
  1235. fi
  1236. def=$(cat "$_CURRENT_DOCKER_COMPOSE" | shyaml get-value "services.${service//./\\.}" 2>/dev/null)
  1237. if [ -z "$def" ]; then
  1238. err "No definition for service $DARKYELLOW$service$NORMAL in compiled 'docker-compose.yml'."
  1239. return 1
  1240. fi
  1241. echo "$def"
  1242. }
  1243. export -f get_service_def
  1244. get_build_hash() {
  1245. local dir="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$1")" hash
  1246. if [ -e "$cache_file" ]; then
  1247. # debug "$FUNCNAME: cache hit ($*)"
  1248. cat "$cache_file"
  1249. return 0
  1250. fi
  1251. ## Check that there's a Dockerfile in this directory
  1252. if [ ! -e "$dir/Dockerfile" ]; then
  1253. err "No 'Dockerfile' found in '$dir'."
  1254. return 1
  1255. fi
  1256. ## use find to md5sum all files in the directory and make a final hash
  1257. hash=$(set -o pipefail; cd "$dir"; env -i find "." -type f -exec md5sum {} \; |
  1258. sort | md5sum | awk '{print $1}') || {
  1259. err "Failed to get hash for '$dir'."
  1260. return 1
  1261. }
  1262. printf "%s" "$hash" | tee "$cache_file"
  1263. return $?
  1264. }
  1265. export -f get_build_hash
  1266. ### Query/Get cached image from registry
  1267. ##
  1268. ## Returns on stdout the name of the image if found, or an empty string if not
  1269. cache:image:registry:get() {
  1270. local charm="$1" hash="$2" service="$3"
  1271. local charm_image_name="cache/charm/$charm"
  1272. local charm_image="$charm_image_name:$hash"
  1273. Elt "pulling ${DARKPINK}$charm${NORMAL} image from $COMPOSE_DOCKER_REGISTRY" >&2
  1274. if out=$(docker pull "$COMPOSE_DOCKER_REGISTRY/$charm_image" 2>&1); then
  1275. docker tag "$COMPOSE_DOCKER_REGISTRY/$charm_image" "$charm_image" || {
  1276. err "Failed set image '$COMPOSE_DOCKER_REGISTRY/$charm_image' as '$charm_image'" \
  1277. "for ${DARKYELLOW}$service${NORMAL}."
  1278. return 1
  1279. }
  1280. print_info "found" >&2
  1281. print_status success >&2
  1282. Feed >&2
  1283. printf "%s" "$charm_image" | tee "$cache_file"
  1284. return $?
  1285. fi
  1286. if [[ "$out" != *"manifest unknown"* ]] && [[ "$out" != *"not found"* ]]; then
  1287. print_status failure >&2
  1288. Feed >&2
  1289. err "Failed to pull image '$COMPOSE_DOCKER_REGISTRY/$charm_image'" \
  1290. "for ${DARKYELLOW}$service${NORMAL}:"
  1291. e "$out"$'\n' | prefix " ${GRAY}|${NORMAL} " >&2
  1292. return 1
  1293. fi
  1294. print_info "not found" >&2
  1295. if test "$type_method" = "long"; then
  1296. __status="[${NOOP}ABSENT${NORMAL}]"
  1297. else
  1298. echo -n "${NOOP}"
  1299. shift; shift;
  1300. echo -n "$*${NORMAL}"
  1301. fi >&2
  1302. Feed >&2
  1303. }
  1304. export -f cache:image:registry:get
  1305. ### Store cached image on registry
  1306. ##
  1307. ## Returns nothing
  1308. cache:image:registry:put() {
  1309. if [ -n "$COMPOSE_DOCKER_REGISTRY" ] && [ -n "$COMPOSE_PUSH_TO_REGISTRY" ]; then
  1310. local charm="$1" hash="$2" service="$3"
  1311. local charm_image_name="cache/charm/$charm"
  1312. local charm_image="$charm_image_name:$hash"
  1313. Wrap -d "pushing ${DARKPINK}$charm${NORMAL} image to $COMPOSE_DOCKER_REGISTRY" <<EOF || return 1
  1314. docker tag "$charm_image" "$COMPOSE_DOCKER_REGISTRY/$charm_image" &&
  1315. docker push "$COMPOSE_DOCKER_REGISTRY/$charm_image"
  1316. EOF
  1317. fi >&2
  1318. }
  1319. export -f cache:image:registry:put
  1320. ### Produce docker cached charm image 'cache/charm/$charm:$hash'
  1321. ##
  1322. ## Either by fetching it from a registry or by building it from a
  1323. ## Dockerfile.
  1324. cache:image:produce() {
  1325. local type="$1" src="$2" charm="$3" hash="$4" service="$5"
  1326. local charm_image_name="cache/charm/$charm"
  1327. local charm_image="$charm_image_name:$hash"
  1328. case "$type" in
  1329. fetch)
  1330. local specified_image="$src"
  1331. ## will not pull upstream image if already present locally
  1332. if ! docker_has_image "${specified_image}"; then
  1333. if ! out=$(docker pull "${specified_image}" 2>&1); then
  1334. err "Failed to pull image '$specified_image' for ${DARKYELLOW}$service${NORMAL}:"
  1335. echo "$out" | prefix " | " >&2
  1336. return 1
  1337. fi
  1338. fi
  1339. # specified_image_id=$(docker_image_id "$specified_image") || return 1
  1340. # charm_image_id=
  1341. # if docker_has_image "${image_dst}"; then
  1342. # charm_image_id=$(docker_image_id "${image_dst}") || return 1
  1343. # fi
  1344. # if [ "$specified_image_id" != "$charm_image_id" ]; then
  1345. docker tag "$specified_image" "${charm_image}" || return 1
  1346. # fi
  1347. ;;
  1348. build)
  1349. local service_build="$src"
  1350. build_opts=()
  1351. if [ "$COMPOSE_ACTION" == "build" ]; then
  1352. while read-0 arg; do
  1353. case "$arg" in
  1354. -t|--tag)
  1355. ## XXXvlab: doesn't seem to be actually a valid option
  1356. if [ -n "$COMPOSE_PUSH_TO_REGISTRY" ]; then
  1357. err "You can't use -t|--tag option when pushing to a registry."
  1358. exit 1
  1359. fi
  1360. has_named_image=true
  1361. read-0 val ## should always be okay because already checked
  1362. build_opts+=("$arg" "$val")
  1363. ;;
  1364. --help|-h)
  1365. docker-compose "$action" --help |
  1366. filter_docker_compose_help_message >&2
  1367. exit 0
  1368. ;;
  1369. --*|-*)
  1370. if str_pattern_matches "$arg" $DC_MATCH_MULTI; then
  1371. read-0 value
  1372. build_opts+=("$arg" "$value")
  1373. shift
  1374. elif str_pattern_matches "$arg" $DC_MATCH_SINGLE; then
  1375. build_opts+=("$arg")
  1376. else
  1377. err "Unexpected error while parsing a second time the build arguments."
  1378. fi
  1379. ;;
  1380. *)
  1381. ## Already parsed
  1382. build_opts+=("$arg")
  1383. ;;
  1384. esac
  1385. done < <(cla.normalize "${action_opts[@]}")
  1386. fi
  1387. if [ -z "$has_named_image" ]; then
  1388. build_opts+=(-t "${charm_image}")
  1389. fi
  1390. Wrap -v -d "Building ${DARKPINK}$charm${NORMAL}:$hash image" -- \
  1391. docker build "$service_build" -t "${charm_image}" "${build_opts[@]}" >&2 || {
  1392. err "Failed to build image '${charm_image}' for ${DARKYELLOW}$service${NORMAL}."
  1393. return 1
  1394. }
  1395. if [ -n "$has_named_image" ]; then
  1396. exit 0
  1397. fi
  1398. ;;
  1399. *)
  1400. err "Unknown type '$type'."
  1401. return 1
  1402. ;;
  1403. esac
  1404. }
  1405. export -f cache:image:produce
  1406. ## Will modify current $_CURRENT_DOCKER_COMPOSE file
  1407. service_ensure_image_ready() {
  1408. if [ -z "$COMBINED_HASH" ]; then
  1409. err "Expected \$COMBINED_HASH to be set."
  1410. return 1
  1411. fi
  1412. local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \
  1413. master_service service_def service_image service_build service_dockerfile image \
  1414. specified_image specified_image_id charm_image_name hash \
  1415. service_quoted
  1416. if [ -e "$cache_file" ]; then
  1417. #debug "$FUNCNAME: cache hit ($*)"
  1418. touch "$cache_file" || return 1
  1419. cat "$cache_file"
  1420. return 0
  1421. fi
  1422. if [ -z "$_CURRENT_DOCKER_COMPOSE" ]; then
  1423. err "${FUNCNAME[0]} is meant to be called after"\
  1424. "\$_CURRENT_DOCKER_COMPOSE has been calculated."
  1425. echo " Called by:" >&2
  1426. printf " - %s\n" "${FUNCNAME[@]:1}" >&2
  1427. return 1
  1428. fi
  1429. master_service="$(get_top_master_service_for_service "$service")" || {
  1430. err "Could not compute master service for service $DARKYELLOW$service$NORMAL."
  1431. return 1
  1432. }
  1433. if [ "$master_service" != "$service" ]; then
  1434. image=$(service_ensure_image_ready "$master_service") || return 1
  1435. printf "%s" "$image" | tee "$cache_file"
  1436. return $?
  1437. fi
  1438. ## check if \$_CURRENT_DOCKER_COMPOSE's service def is already correctly setup
  1439. local charm="$(get_service_charm "$service")" || return 1
  1440. local charm_image_name="cache/charm/$charm" || return 1
  1441. local service_def="$(get_service_def "$service")" || {
  1442. err "Could not get docker-compose service definition for $DARKYELLOW$service$NORMAL."
  1443. return 1
  1444. }
  1445. local service_quoted=${service//./\\.}
  1446. if specified_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null); then
  1447. if [ "$specified_image" == "$charm_image_name"* ]; then
  1448. ## Assume we already did the change
  1449. printf "%s" "$specified_image" | tee "$cache_file"
  1450. return 0
  1451. fi
  1452. if [[ "$specified_image" == "${COMPOSE_DOCKER_REGISTRY}/"* ]]; then
  1453. if ! docker_has_image "${specified_image}"; then
  1454. Wrap "${wrap_opts[@]}" \
  1455. -v -d "pulling ${DARKPINK}$charm${NORMAL}'s specified image from $COMPOSE_DOCKER_REGISTRY" -- \
  1456. docker pull "${specified_image}" >&2 || return 1
  1457. else
  1458. if [ -n "$DEBUG" ]; then
  1459. Elt "using local ${DARKPINK}$charm${NORMAL}'s specified image from $COMPOSE_DOCKER_REGISTRY" >&2
  1460. print_status noop >&2
  1461. Feed >&2
  1462. fi
  1463. fi
  1464. ## Already on the cache server
  1465. printf "%s" "$specified_image" | tee "$cache_file"
  1466. return 0
  1467. fi
  1468. src="$specified_image"
  1469. hash=$(echo "$specified_image" | md5sum | cut -f 1 -d " ") || return 1
  1470. type=fetch
  1471. ## replace image by charm image
  1472. yq -i ".services.[\"${service_quoted}\"].image = \"${charm_image_name}:${hash}\"" \
  1473. "$_CURRENT_DOCKER_COMPOSE" || return 1
  1474. else
  1475. if ! src=$(echo "$service_def" | shyaml get-value build 2>/dev/null); then
  1476. err "Service $DARKYELLOW$service$NORMAL has no ${WHITE}image${NORMAL} nor ${WHITE}build${NORMAL} parameter."
  1477. echo "$service_def" >&2
  1478. return 1
  1479. fi
  1480. ## According to https://stackoverflow.com/questions/32230577 , if there's a build,
  1481. ## then the built image will get name ${project}_${service}
  1482. hash=$(get_build_hash "$src") || return 1
  1483. type=build
  1484. ## delete build key from service_def and add image to charm_image_name
  1485. yq -i "del(.services.[\"${service_quoted}\"].build) |
  1486. .services.[\"${service_quoted}\"].image = \"${charm_image_name}:${hash}\"" \
  1487. "$_CURRENT_DOCKER_COMPOSE" || return 1
  1488. fi
  1489. if [ "$COMPOSE_ACTION" != "build" ] && docker_has_image "${charm_image_name}:${hash}"; then
  1490. if [ -n "$DEBUG" ]; then
  1491. Elt "using ${DARKPINK}$charm${NORMAL}'s image from local cache" >&2
  1492. print_status noop >&2
  1493. Feed >&2
  1494. fi
  1495. cache:image:registry:put "$charm" "$hash" "$service" || return 1
  1496. printf "%s" "${charm_image_name}:${hash}" | tee "$cache_file"
  1497. return $?
  1498. fi
  1499. ## Can we pull it ? Let's check on $COMPOSE_DOCKER_REGISTRY
  1500. if [ "$COMPOSE_ACTION" != "build" ] && [ -n "$COMPOSE_DOCKER_REGISTRY" ]; then
  1501. img=$(cache:image:registry:get "$charm" "$hash" "$service") || {
  1502. err "Failed to get image '$charm_image_name:$hash' from registry for ${DARKYELLOW}$service${NORMAL}."
  1503. return 1
  1504. }
  1505. [ -n "$img" ] && {
  1506. printf "%s" "$img" | tee "$cache_file"
  1507. return $?
  1508. }
  1509. fi
  1510. cache:image:produce "$type" "$src" "$charm" "$hash" "$service" || return 1
  1511. cache:image:registry:put "$charm" "$hash" "$service" || return 1
  1512. printf "%s" "${charm_image_name}:$hash" | tee "$cache_file"
  1513. return $?
  1514. }
  1515. export -f service_ensure_image_ready
  1516. get_charm_relation_def () {
  1517. local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1518. relation_def metadata
  1519. if [ -e "$cache_file" ]; then
  1520. # debug "$FUNCNAME: cache hit ($*)"
  1521. cat "$cache_file"
  1522. return 0
  1523. fi
  1524. metadata="$(charm.metadata "$charm")" || return 1
  1525. relation_def="$(echo "$metadata" | shyaml get-value "provides.${relation_name}" 2>/dev/null)"
  1526. echo "$relation_def" | tee "$cache_file"
  1527. }
  1528. export -f get_charm_relation_def
  1529. get_charm_tech_dep_orientation_for_relation() {
  1530. local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1531. relation_def value
  1532. if [ -e "$cache_file" ]; then
  1533. # debug "$FUNCNAME: cache hit ($*)"
  1534. cat "$cache_file"
  1535. return 0
  1536. fi
  1537. relation_def=$(get_charm_relation_def "$charm" "$relation_name" 2>/dev/null)
  1538. value=$(echo "$relation_def" | shyaml get-value 'tech-dep' 2>/dev/null)
  1539. value=${value:-True}
  1540. printf "%s" "$value" | tee "$cache_file"
  1541. }
  1542. export -f get_charm_tech_dep_orientation_for_relation
  1543. get_service_relation_tech_dep() {
  1544. local service="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1545. charm tech_dep
  1546. if [ -e "$cache_file" ]; then
  1547. # debug "$FUNCNAME: cache hit ($*)"
  1548. cat "$cache_file"
  1549. return 0
  1550. fi
  1551. charm=$(get_service_charm "$service") || return 1
  1552. tech_dep="$(get_charm_tech_dep_orientation_for_relation "$charm" "$relation_name")" || return 1
  1553. printf "%s" "$tech_dep" | tee "$cache_file"
  1554. }
  1555. export -f get_service_relation_tech_dep
  1556. ##
  1557. ## Use compose file to get deps, and relation definition in metadata.yml
  1558. ## for tech-dep attribute.
  1559. get_service_deps() {
  1560. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH")"
  1561. if [ -e "$cache_file" ]; then
  1562. # debug "$FUNCNAME: cache hit ($*)"
  1563. cat "$cache_file"
  1564. return 0
  1565. fi
  1566. (
  1567. set -o pipefail
  1568. get_service_relations "$service" | \
  1569. while read-0 relation_name target_service _relation_config tech_dep; do
  1570. echo "$target_service"
  1571. done | tee "$cache_file"
  1572. ) || return 1
  1573. }
  1574. export -f get_service_deps
  1575. ## XXXvlab: cache was disabled because improper. Indeed, this needs to cache
  1576. ## 'depths' full state. Second, it should be
  1577. _rec_get_depth() {
  1578. local elt=$1 dep deps max
  1579. [ "${depths[$elt]}" ] && return 0
  1580. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$(declare -pA depths)" "$GLOBAL_ALL_RELATIONS_HASH")"
  1581. if [ -e "$cache_file.depths" ]; then
  1582. #debug "$FUNCNAME: cache hit ($*) - $cache_file.depths"
  1583. while read-0 k v; do
  1584. depths["$k"]="$v"
  1585. done < "$cache_file.depths"
  1586. while read-0 k v; do
  1587. visited["$k"]="$v"
  1588. done < "$cache_file.visited"
  1589. return 0
  1590. fi
  1591. visited[$elt]=1
  1592. #debug "Setting visited[$elt]"
  1593. #debug "Asking for $DARKYELLOW$elt$NORMAL dependencies"
  1594. deps=$(get_service_deps "$elt") || {
  1595. debug "Failed get_service_deps $elt"
  1596. return 1
  1597. }
  1598. # debug "$elt deps are:" $deps
  1599. max=0
  1600. for dep in $deps; do
  1601. [ "${visited[$dep]}" ] && {
  1602. #debug "Already computing $dep"
  1603. continue
  1604. }
  1605. _rec_get_depth "$dep" || return 1
  1606. #debug "Requesting depth[$dep]"
  1607. if (( ${depths[$dep]} > max )); then
  1608. max="${depths[$dep]}"
  1609. fi
  1610. done
  1611. # debug "Setting depth[$elt] to $((max + 1))"
  1612. depths[$elt]=$((max + 1))
  1613. array_kv_to_stdin depths > "$cache_file.depths"
  1614. array_kv_to_stdin visited > "$cache_file.visited"
  1615. # debug "DEPTHS: $(declare -pA depths)"
  1616. # debug "$FUNCNAME: caching hit ($*) - $cache_file"
  1617. }
  1618. export -f _rec_get_depth
  1619. get_ordered_service_dependencies() {
  1620. local services=("$@") cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH")" \
  1621. i value key heads depths visited
  1622. if [ -e "$cache_file" ]; then
  1623. # debug "$FUNCNAME: cache hit ($*)"
  1624. cat "$cache_file"
  1625. return 0
  1626. fi
  1627. #debug "Figuring ordered deps of $DARKYELLOW${services[@]}$NORMAL"
  1628. if [ -z "${services[*]}" ]; then
  1629. return 0
  1630. # print_syntax_error "$FUNCNAME: no arguments"
  1631. # return 1
  1632. fi
  1633. declare -A depths
  1634. declare -A visited
  1635. heads=("${services[@]}")
  1636. while [ "${#heads[@]}" != 0 ]; do
  1637. array_pop heads head
  1638. _rec_get_depth "$head" || return 1
  1639. done
  1640. i=0
  1641. while [ "${#depths[@]}" != 0 ]; do
  1642. for key in "${!depths[@]}"; do
  1643. value="${depths[$key]}"
  1644. if [ "$value" == "$i" ]; then
  1645. echo "$key"
  1646. unset depths[$key]
  1647. fi
  1648. done
  1649. ((i++))
  1650. done | tee "$cache_file"
  1651. }
  1652. export -f get_ordered_service_dependencies
  1653. ## Modify $_CURRENT_DOCKER_COMPOSE file, and fills cache
  1654. run_service_acquire_images () {
  1655. local service subservice subservices loaded
  1656. _CURRENT_DOCKER_COMPOSE_HASH=$(hash_get < "$_CURRENT_DOCKER_COMPOSE")
  1657. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$_CURRENT_DOCKER_COMPOSE_HASH" "$COMBINED_HASH")"
  1658. if [ -e "$cache_file" ]; then
  1659. # debug "$FUNCNAME: cache hit ($*)"
  1660. touch "$cache_file" || return 1
  1661. cp "$cache_file" "$_CURRENT_DOCKER_COMPOSE" || return 1
  1662. return 0
  1663. fi
  1664. declare -A loaded
  1665. for service in "$@"; do
  1666. subservices=$(get_ordered_service_dependencies "$service") || return 1
  1667. for subservice in $subservices; do
  1668. if [ "${loaded[$subservice]}" ]; then
  1669. ## Prevent double inclusion of same service if this
  1670. ## service is deps of two or more of your
  1671. ## requirements.
  1672. continue
  1673. fi
  1674. type=$(get_service_type "$subservice") || return 1
  1675. MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
  1676. if [ "$type" != "stub" ]; then
  1677. DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$MASTER_BASE_SERVICE_NAME") || return 1
  1678. fi
  1679. loaded[$subservice]=1
  1680. done
  1681. done
  1682. cp "$_CURRENT_DOCKER_COMPOSE" "$cache_file" || return 1
  1683. return 0
  1684. }
  1685. run_service_hook () {
  1686. local action="$1" service subservice subservices loaded
  1687. shift
  1688. declare -A loaded
  1689. for service in "$@"; do
  1690. subservices=$(get_ordered_service_dependencies "$service") || return 1
  1691. for subservice in $subservices; do
  1692. if [ "${loaded[$subservice]}" ]; then
  1693. ## Prevent double inclusion of same service if this
  1694. ## service is deps of two or more of your
  1695. ## requirements.
  1696. continue
  1697. fi
  1698. charm=$(get_service_charm "$subservice") || return 1
  1699. charm.has_hook "$charm" "$action" >/dev/null || continue
  1700. type=$(get_service_type "$subservice") || return 1
  1701. PROJECT_NAME=$(get_default_project_name) || return 1
  1702. MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
  1703. MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1
  1704. if [ "$type" != "stub" ]; then
  1705. DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$MASTER_BASE_SERVICE_NAME") || return 1
  1706. fi
  1707. Wrap "${wrap_opts[@]}" -d "running $YELLOW$action$NORMAL hook of $DARKYELLOW$subservice$NORMAL in charm $DARKPINK$charm$NORMAL" <<EOF || return 1
  1708. export DOCKER_BASE_IMAGE="$DOCKER_BASE_IMAGE"
  1709. export SERVICE_NAME=$subservice
  1710. export IMAGE_NAME=$(echo "${PROJECT_NAME}" | tr -d "_-")_\${SERVICE_NAME}
  1711. export CONTAINER_NAME=\${IMAGE_NAME}_1
  1712. export CHARM_NAME="$charm"
  1713. export PROJECT_NAME="$PROJECT_NAME"
  1714. export SERVICE_DATASTORE="$DATASTORE/$subservice"
  1715. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$subservice"
  1716. export MASTER_BASE_SERVICE_NAME="$MASTER_BASE_SERVICE_NAME"
  1717. export MASTER_BASE_CHARM_NAME="$MASTER_BASE_CHARM_NAME"
  1718. charm.run_hook "local" "$charm" "$action"
  1719. EOF
  1720. loaded[$subservice]=1
  1721. done
  1722. done
  1723. return 0
  1724. }
  1725. host_resource_get() {
  1726. local location="$1" cfg="$2"
  1727. type=$(echo "$cfg" | shyaml get-value type 2>/dev/null) || {
  1728. err "Missing ${WHITE}type$NORMAL option in ${WHITE}get$NORMAL config for location '$location'"
  1729. return 1
  1730. }
  1731. if fn.exists host_resource_get_$type; then
  1732. host_resource_get_$type "$location" "$cfg"
  1733. else
  1734. err "Source ${WHITE}source$NORMAL type '$type' unknown for" \
  1735. "${WHITE}host-resource$NORMAL '$location' defined in" \
  1736. "$DARKYELLOW$subservice$NORMAL config."
  1737. return 1
  1738. fi
  1739. }
  1740. export -f host_resource_get
  1741. host_resource_get_git() {
  1742. local location="$1" cfg="$2" branch parent url
  1743. branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null)
  1744. branch=${branch:-master}
  1745. url=$(echo "$cfg" | shyaml get-value url 2>/dev/null)
  1746. parent="$(dirname "$location")"
  1747. (
  1748. mkdir -p "$parent" && cd "$parent" &&
  1749. git clone -b "$branch" "$url" "$(basename "$location")"
  1750. ) || return 1
  1751. }
  1752. export -f host_resource_get_git
  1753. host_resource_get_git-sub() {
  1754. local location="$1" cfg="$2" branch parent url
  1755. branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null)
  1756. branch=${branch:-master}
  1757. url=$(echo "$cfg" | shyaml get-value url 2>/dev/null)
  1758. parent="$(dirname "$location")"
  1759. (
  1760. mkdir -p "$parent" && cd "$parent" &&
  1761. git sub clone -b "$branch" "$url" "$(basename "$location")"
  1762. ) || return 1
  1763. }
  1764. export -f host_resource_get_git-sub
  1765. setup_host_resource () {
  1766. local subservice="$1" service_def location get cfg
  1767. service_def=$(get_compose_service_def "$subservice") || return 1
  1768. while read-0 location cfg; do
  1769. ## XXXvlab: will it be a git resources always ?
  1770. if [ -d "$location" -a ! -d "$location/.git" ]; then
  1771. err "Hum, location '$location' does not seem to be a git directory."
  1772. return 1
  1773. fi
  1774. if [ -d "$location" ]; then
  1775. info "host resource '$location' already set up."
  1776. continue
  1777. fi
  1778. get=$(echo "$cfg" | shyaml get-value get 2>/dev/null)
  1779. if [ -z "$get" ]; then
  1780. err "No host directory '$location' found, and no ${WHITE}source$NORMAL" \
  1781. "specified for $DARKYELLOW$subservice$NORMAL."
  1782. return 1
  1783. fi
  1784. host_resource_get "$location" "$get" || return 1
  1785. done < <(echo "$service_def" | shyaml key-values-0 host-resources 2>/dev/null)
  1786. }
  1787. export -f setup_host_resource
  1788. setup_host_resources () {
  1789. local service subservices subservice loaded
  1790. declare -A loaded
  1791. for service in "$@"; do
  1792. subservices=$(get_ordered_service_dependencies "$service") || return 1
  1793. for subservice in $subservices; do
  1794. if [ "${loaded[$subservice]}" ]; then
  1795. ## Prevent double inclusion of same service if this
  1796. ## service is deps of two or more of your
  1797. ## requirements.
  1798. continue
  1799. fi
  1800. setup_host_resource "$subservice" || return 1
  1801. loaded[$subservice]=1
  1802. done
  1803. done
  1804. return 0
  1805. }
  1806. export -f setup_host_resources
  1807. ## Works on stdin
  1808. cfg-get-value () {
  1809. local key="$1" out
  1810. if [ -z "$key" ]; then
  1811. yaml_get_interpret || return 1
  1812. return 0
  1813. fi
  1814. if ! out=$(shyaml -y get-value "$key" 2>/dev/null); then
  1815. err "The key $WHITE$key$NORMAL was not found in relation's data."
  1816. return 1
  1817. fi
  1818. printf "%s\n" "$out" | yaml_get_interpret
  1819. }
  1820. export -f cfg-get-value
  1821. relation-get () {
  1822. if [ -z "$RELATION_DATA_FILE" ]; then
  1823. err-d "$FUNCNAME: var \$RELATION_DATA_FILE is not set."
  1824. return 1
  1825. fi
  1826. cfg-get-value "$1" < "$RELATION_DATA_FILE"
  1827. }
  1828. export -f relation-get
  1829. expand_vars() {
  1830. local unlikely_prefix="UNLIKELY_PREFIX"
  1831. content=$(cat -)
  1832. ## find first identifier not in content
  1833. remaining_lines=$(echo "$content" | grep "^$unlikely_prefix")
  1834. while [ "$(echo "$remaining_lines" | grep "^$unlikely_prefix$")" ]; do
  1835. size_prefix="${#unlikely_prefix}"
  1836. first_matching=$(echo "$remaining_lines" |
  1837. grep -v "^$unlikely_prefix$" |
  1838. uniq -w "$((size_prefix + 1))" -c |
  1839. sort -rn |
  1840. head -n 1)
  1841. first_matching=${first_matching#"${x%%[![:space:]]*}"}
  1842. first_matching="${first_matching#* }"
  1843. next_char=${first_matching:$size_prefix:1}
  1844. if [ "$next_char" != "0" ]; then
  1845. unlikely_prefix+="0"
  1846. else
  1847. unlikely_prefix+="1"
  1848. fi
  1849. remaining_lines=$(echo "$remaining_lines" | grep "^$unlikely_prefix")
  1850. done
  1851. eval "cat <<$unlikely_prefix
  1852. $content
  1853. $unlikely_prefix"
  1854. }
  1855. export -f expand_vars
  1856. yaml_get_interpret() {
  1857. local content tag
  1858. content=$(cat -)
  1859. tag=$(echo "$content" | shyaml get-type) || return 1
  1860. content=$(echo "$content" | shyaml get-value) || return 1
  1861. if ! [ "${tag:0:1}" == "!" ]; then
  1862. echo "$content" || return 1
  1863. return 0
  1864. fi
  1865. case "$tag" in
  1866. "!bash-stdout")
  1867. echo "$content" | bash || {
  1868. err "shell code didn't end with errorlevel 0"
  1869. return 1
  1870. }
  1871. ;;
  1872. "!var-expand")
  1873. echo "$content" | expand_vars || {
  1874. err "shell expansion failed"
  1875. return 1
  1876. }
  1877. ;;
  1878. "!file-content")
  1879. source=$(echo "$content" | expand_vars) || {
  1880. err "shell expansion failed"
  1881. return 1
  1882. }
  1883. cat "$source" || return 1
  1884. ;;
  1885. *)
  1886. err "Invalid object tag ${WHITE}$tag${NORMAL}"
  1887. return 1
  1888. ;;
  1889. esac
  1890. }
  1891. export -f yaml_get_interpret
  1892. options-get () {
  1893. local key="$1" out
  1894. service_def=$(get_compose_service_def "$SERVICE_NAME") || return 1
  1895. if ! out=$(echo "$service_def" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1896. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  1897. return 1
  1898. fi
  1899. echo "$out" | yaml_get_interpret
  1900. }
  1901. export -f options-get
  1902. relation-base-compose-get () {
  1903. local key="$1" out
  1904. if ! out=$(echo "$RELATION_BASE_COMPOSE_DEF" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1905. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  1906. return 1
  1907. fi
  1908. echo "$out" | yaml_get_interpret
  1909. }
  1910. export -f relation-base-compose-get
  1911. relation-target-compose-get () {
  1912. local key="$1" out
  1913. if ! out=$(echo "$RELATION_TARGET_COMPOSE_DEF" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1914. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  1915. return 1
  1916. fi
  1917. echo "$out" | yaml_get_interpret
  1918. }
  1919. export -f relation-target-compose-get
  1920. relation-set () {
  1921. local key="$1" value="$2"
  1922. if [ -z "$RELATION_DATA_FILE" ]; then
  1923. err "$FUNCNAME: relation does not seems to be correctly setup."
  1924. return 1
  1925. fi
  1926. if ! [ -r "$RELATION_DATA_FILE" ]; then
  1927. err "$FUNCNAME: can't read relation's data." >&2
  1928. return 1
  1929. fi
  1930. _config_merge "$RELATION_DATA_FILE" <(yaml_key_val_str "$key" "$value")
  1931. }
  1932. export -f relation-set
  1933. _config_merge() {
  1934. local config_filename="$1" mixin="$2"
  1935. touch "$config_filename" &&
  1936. merge_yaml "$config_filename" "$mixin" > "$config_filename.tmp" || return 1
  1937. mv "$config_filename.tmp" "$config_filename"
  1938. }
  1939. export -f _config_merge
  1940. ## XXXvlab; this can be used only in relation, I'd like to use it in init.
  1941. config-add() {
  1942. local metadata="$1"
  1943. _config_merge "$RELATION_CONFIG" <(echo "$metadata")
  1944. }
  1945. export -f config-add
  1946. ## XXXvlab; this can be used only in relation, I'd like to use it in init.
  1947. init-config-add() {
  1948. local metadata="$1"
  1949. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" \
  1950. <(yaml_key_val_str "services" "$metadata")
  1951. }
  1952. export -f init-config-add
  1953. docker_get_uid() {
  1954. local service="$1" user="$2" uid
  1955. uid=$(cached_cmd_on_base_image "$service" "id -u \"$user\"") || {
  1956. debug "Failed to query for '$user' uid in ${DARKYELLOW}$service${NORMAL} base image."
  1957. return 1
  1958. }
  1959. info "uid from ${DARKYELLOW}$service${NORMAL} for user '$user' is '$uid'"
  1960. echo "$uid"
  1961. }
  1962. export -f docker_get_uid
  1963. docker_get_uid_gid() {
  1964. local service="$1" user="$2" group="$3" uid
  1965. uid_gid=$(cached_cmd_on_base_image "$service" "id -u \"$user\"; id -g \"$group\"") || {
  1966. debug "Failed to query for '$user' uid in ${DARKYELLOW}$service${NORMAL} base image."
  1967. return 1
  1968. }
  1969. info "uid from ${DARKYELLOW}$service${NORMAL} for user '$user' is '$uid_gid'"
  1970. echo "$uid_gid"
  1971. }
  1972. export -f docker_get_uid_gid
  1973. logstdout() {
  1974. local name="$1"
  1975. sed -r 's%^%'"${name}"'> %g'
  1976. }
  1977. export -f logstdout
  1978. logstderr() {
  1979. local name="$1"
  1980. sed -r 's%^(.*)$%'"${RED}${name}>${NORMAL} \1"'%g'
  1981. }
  1982. export -f logstderr
  1983. _run_service_relation () {
  1984. local relation_name="$1" service="$2" target_service="$3" relation_config="$4" relation_dir services
  1985. local errlvl
  1986. charm=$(get_service_charm "$service") || return 1
  1987. target_charm=$(get_service_charm "$target_service") || return 1
  1988. base_script_name=$(charm.has_relation_hook "$charm" "$relation_name" relation-joined) || true
  1989. target_script_name=$(charm.has_relation_hook "$target_charm" "$relation_name" relation-joined) || true
  1990. [ -n "$base_script_name" ] || [ -n "$target_script_name" ] || return 0
  1991. relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1
  1992. RELATION_DATA_FILE=$(get_relation_data_file "$service" "$target_service" "$relation_name" "$relation_config") || return 1
  1993. export BASE_SERVICE_NAME=$service
  1994. export BASE_CHARM_NAME=$charm
  1995. export BASE_CHARM_PATH=$(charm.get_dir "$charm")
  1996. export TARGET_SERVICE_NAME=$target_service
  1997. export TARGET_CHARM_NAME=$target_charm
  1998. export TARGET_CHARM_PATH=$(charm.get_dir "$target_charm")
  1999. export RELATION_DATA_FILE
  2000. target_errlvl=0
  2001. if [ -z "$target_script_name" ]; then
  2002. verb "No relation script $DARKBLUE$relation_name$NORMAL in target $DARKPINK$target_charm$NORMAL."
  2003. else
  2004. verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \
  2005. "for target $DARKYELLOW$target_service$NORMAL (charm $DARKPINK$target_charm$NORMAL)"
  2006. RELATION_CONFIG="$relation_dir/config_provider"
  2007. type=$(get_service_type "$target_service") || return 1
  2008. if [ "$type" != "stub" ]; then
  2009. DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$target_service") || return 1
  2010. fi
  2011. export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA
  2012. {
  2013. (
  2014. SERVICE_NAME=$target_service
  2015. SERVICE_DATASTORE="$DATASTORE/$target_service"
  2016. SERVICE_CONFIGSTORE="$CONFIGSTORE/$target_service"
  2017. export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE
  2018. charm.run_relation_hook local "$target_charm" "$relation_name" relation-joined
  2019. echo "$?" > "$relation_dir/target_errlvl"
  2020. ) | logstdout "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}"
  2021. } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@${NORMAL}" 3>&1 1>&2 2>&3
  2022. target_errlvl="$(cat "$relation_dir/target_errlvl")" || {
  2023. err "Relation script '$script_name' in $DARKPINK$target_charm$NORMAL" \
  2024. "failed before outputing an errorlevel."
  2025. ((target_errlvl |= "1" ))
  2026. }
  2027. if [ -e "$RELATION_CONFIG" ]; then
  2028. debug "Merging some new config info in $DARKYELLOW$target_service$NORMAL"
  2029. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" &&
  2030. rm "$RELATION_CONFIG"
  2031. ((target_errlvl |= "$?"))
  2032. fi
  2033. fi
  2034. if [ "$target_errlvl" == 0 ]; then
  2035. errlvl=0
  2036. if [ "$base_script_name" ]; then
  2037. verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \
  2038. "for $DARKYELLOW$service$NORMAL (charm $DARKPINK$charm$NORMAL)"
  2039. RELATION_CONFIG="$relation_dir/config_providee"
  2040. RELATION_DATA="$(cat "$RELATION_DATA_FILE")"
  2041. DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$service") || return 1
  2042. export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA
  2043. {
  2044. (
  2045. SERVICE_NAME=$service
  2046. SERVICE_DATASTORE="$DATASTORE/$service"
  2047. SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  2048. export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE
  2049. charm.run_relation_hook local "$charm" "$relation_name" relation-joined
  2050. echo "$?" > "$relation_dir/errlvl"
  2051. ) | logstdout "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}"
  2052. } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@$NORMAL" 3>&1 1>&2 2>&3
  2053. errlvl="$(cat "$relation_dir/errlvl")" || {
  2054. err "Relation script '$script_name' in $DARKPINK$charm$NORMAL" \
  2055. "failed before outputing an errorlevel."
  2056. ((errlvl |= "1" ))
  2057. }
  2058. if [ -e "$RELATION_CONFIG" ]; then
  2059. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" &&
  2060. rm "$RELATION_CONFIG"
  2061. ((errlvl |= "$?" ))
  2062. fi
  2063. if [ "$errlvl" != 0 ]; then
  2064. err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$service$NORMAL failed to run properly."
  2065. fi
  2066. else
  2067. verb "No relation script '$script_name' in charm $DARKPINK$charm$NORMAL. Ignoring."
  2068. fi
  2069. else
  2070. err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$target_service$NORMAL failed to run properly."
  2071. fi
  2072. if [ "$target_errlvl" == 0 -a "$errlvl" == 0 ]; then
  2073. debug "Relation $DARKBLUE$relation_name$NORMAL is established" \
  2074. "between $DARKYELLOW$service$NORMAL and $DARKYELLOW$target_service$NORMAL."
  2075. return 0
  2076. else
  2077. return 1
  2078. fi
  2079. }
  2080. export -f _run_service_relation
  2081. _get_compose_relations_cached () {
  2082. local compose_service_def="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  2083. relation_name relation_def target_service
  2084. if [ -e "$cache_file" ]; then
  2085. #debug "$FUNCNAME: STATIC cache hit $1"
  2086. cat "$cache_file" &&
  2087. touch "$cache_file" || return 1
  2088. return 0
  2089. fi
  2090. (
  2091. set -o pipefail
  2092. if [ "$compose_service_def" ]; then
  2093. while read-0 relation_name relation_def; do
  2094. ## XXXvlab: could we use braces here instead of parenthesis ?
  2095. (
  2096. case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in
  2097. "str")
  2098. target_service="$(echo "$relation_def" | shyaml get-value 2>/dev/null)" || return 1
  2099. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  2100. printf "%s\0" "$relation_name" "$target_service" "" "$tech_dep"
  2101. ;;
  2102. "sequence")
  2103. while read-0 target_service; do
  2104. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  2105. printf "%s\0" "$relation_name" "$target_service" "" "$tech_dep"
  2106. done < <(echo "$relation_def" | shyaml get-values-0 2>/dev/null)
  2107. ;;
  2108. "struct")
  2109. while read-0 target_service relation_config; do
  2110. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  2111. printf "%s\0" "$relation_name" "$target_service" "$relation_config" "$tech_dep"
  2112. done < <(echo "$relation_def" | shyaml key-values-0 2>/dev/null)
  2113. ;;
  2114. esac
  2115. ) </dev/null >> "$cache_file" || return 1
  2116. done < <(echo "$compose_service_def" | shyaml key-values-0 relations 2>/dev/null)
  2117. fi
  2118. )
  2119. if [ "$?" != 0 ]; then
  2120. err "Error while looking for compose relations."
  2121. rm -f "$cache_file" ## no cache
  2122. return 1
  2123. fi
  2124. [ -e "$cache_file" ] && cat "$cache_file"
  2125. return 0
  2126. }
  2127. export -f _get_compose_relations_cached
  2128. get_compose_relations () {
  2129. if [ -z "$COMBINED_HASH" ]; then
  2130. err-d "Expected \$COMBINED_HASH to be set."
  2131. return 1
  2132. fi
  2133. local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \
  2134. compose_def
  2135. if [ -e "$cache_file" ]; then
  2136. #debug "$FUNCNAME: SESSION cache hit $1"
  2137. cat "$cache_file"
  2138. return 0
  2139. fi
  2140. compose_def="$(get_compose_service_def "$service")" || return 1
  2141. _get_compose_relations_cached "$compose_def" > "$cache_file"
  2142. if [ "$?" != 0 ]; then
  2143. rm -f "$cache_file" ## no cache
  2144. return 1
  2145. fi
  2146. cat "$cache_file"
  2147. }
  2148. export -f get_compose_relations
  2149. get_all_services() {
  2150. local services compose_yml_services service
  2151. if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then
  2152. err-d "Can't access global \$GLOBAL_ALL_RELATIONS_HASH"
  2153. return 1
  2154. fi
  2155. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$GLOBAL_ALL_RELATIONS_HASH" "$(declare -f "$FUNCNAME")")" \
  2156. s rn ts rc td services service
  2157. if [ -e "$cache_file" ]; then
  2158. #debug "$FUNCNAME: cache hit $1"
  2159. cat "$cache_file"
  2160. return 0
  2161. fi
  2162. if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
  2163. err-d "Can't access global \$GLOBAL_ALL_RELATIONS"
  2164. return 1
  2165. fi
  2166. declare -A services
  2167. while read-0 s _ ts _ _; do
  2168. for service in "$s" "$ts"; do
  2169. [ "${services[$service]}" ] && continue
  2170. services["$service"]=1
  2171. echo "$service"
  2172. done
  2173. done < "$GLOBAL_ALL_RELATIONS" > "$cache_file.wip"
  2174. compose_yml_services=($(compose:yml:root:services)) || return 1
  2175. for service in "${compose_yml_services[@]}"; do
  2176. [ "${services[$service]}" ] && continue
  2177. services["$service"]=1
  2178. echo "$service"
  2179. done >> "$cache_file.wip"
  2180. mv "$cache_file"{.wip,} || return 1
  2181. cat "$cache_file"
  2182. }
  2183. export -f get_all_services
  2184. get_service_relations () {
  2185. if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
  2186. err-d "Can't access global \$GLOBAL_ALL_RELATIONS"
  2187. return 1
  2188. fi
  2189. local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$GLOBAL_ALL_RELATIONS_HASH" \
  2190. s rn ts rc td
  2191. if [ -e "$cache_file" ]; then
  2192. #debug "$FUNCNAME: SESSION cache hit $1"
  2193. cat "$cache_file"
  2194. return 0
  2195. fi
  2196. while read-0 s rn ts rc td; do
  2197. [[ "$s" == "$service" ]] || continue
  2198. printf "%s\0" "$rn" "$ts" "$rc" "$td"
  2199. done < <(cat "$GLOBAL_ALL_RELATIONS") > "$cache_file"
  2200. cat "$cache_file"
  2201. }
  2202. export -f get_service_relations
  2203. get_service_relation() {
  2204. local service="$1" relation="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  2205. rn ts rc td
  2206. if [ -e "$cache_file" ]; then
  2207. #debug "$FUNCNAME: SESSION cache hit $1"
  2208. cat "$cache_file"
  2209. return 0
  2210. fi
  2211. while read-0-err E rn ts rc td; do
  2212. [ "$relation" == "$rn" ] && {
  2213. printf "%s\0" "$ts" "$rc" "$td"
  2214. break
  2215. }
  2216. done < <(p-err get_service_relations "$service") > "${cache_file}.wip"
  2217. if [ "$?" != 0 ]; then
  2218. return 1
  2219. fi
  2220. if [ "$E" != 0 ]; then
  2221. return 1
  2222. fi
  2223. mv "${cache_file}"{.wip,} || return 1
  2224. cat "$cache_file"
  2225. }
  2226. export -f get_service_relation
  2227. ## From a service and a relation, get all relations targeting given
  2228. ## service with given relation.
  2229. ##
  2230. ## Returns a NUL separated list of couple of:
  2231. ## (base_service, relation_config)
  2232. ##
  2233. get_service_incoming_relations() {
  2234. if [ -z "$SUBSET_ALL_RELATIONS_HASH" ]; then
  2235. err-d "Expected \$SUBSET_ALL_RELATIONS_HASH to be set."
  2236. return 1
  2237. fi
  2238. local service="$1" relation="$2" \
  2239. cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$SUBSET_ALL_RELATIONS_HASH")" \
  2240. s rn ts rc td
  2241. if [ -e "$cache_file" ]; then
  2242. #debug "$FUNCNAME: SESSION cache hit $1"
  2243. cat "$cache_file"
  2244. return 0
  2245. fi
  2246. while read-0 s rn ts rc _td; do
  2247. [[ "$ts" == "$service" ]] || continue
  2248. [[ "$rn" == "$relation" ]] || continue
  2249. relation_data_file=$(get_relation_data_file "$s" "$ts" "$rn" "$rc") || return 1
  2250. printf "%s\0" "$s" "$(cat "$relation_data_file")" || return 1
  2251. debug "Found relation $rn from $s to $ts" >&2
  2252. done < "$SUBSET_ALL_RELATIONS" > "$cache_file.wip"
  2253. mv "$cache_file"{.wip,} || return 1
  2254. cat "$cache_file"
  2255. }
  2256. export -f get_service_incoming_relations
  2257. export TRAVERSE_SEPARATOR=:
  2258. ## Traverse on first service satisfying relation
  2259. service:traverse() {
  2260. local service_path="$1"
  2261. {
  2262. SEPARATOR=:
  2263. read -d "$TRAVERSE_SEPARATOR" service
  2264. while read -d "$TRAVERSE_SEPARATOR" relation; do
  2265. ## XXXvlab: Take only first service
  2266. if ! read-0 ts _ _ < <(get_service_relation "${service}" "${relation}"); then
  2267. err "Couldn't find relation ${DARKCYAN}${relation}${NORMAL}" \
  2268. "from ${DARKYELLOW}$service${NORMAL}."
  2269. return 1
  2270. fi
  2271. service="$ts"
  2272. done
  2273. echo "$service"
  2274. } < <(e "${service_path}${TRAVERSE_SEPARATOR}")
  2275. }
  2276. export -f service:traverse
  2277. service:relation-file() {
  2278. local service_path="$1" relation service relation_file
  2279. if ! [[ "$service_path" == *"$TRAVERSE_SEPARATOR"* ]]; then
  2280. err "Invalid argument '$service_path'." \
  2281. "Must provide a service path (no '${TRAVERSE_SEPARATOR}' found)."
  2282. return 1
  2283. fi
  2284. relation="${service_path##*${TRAVERSE_SEPARATOR}}"
  2285. service=$(service:traverse "${service_path%${TRAVERSE_SEPARATOR}*}") || return 1
  2286. if ! read-0 ts rc _ < <(get_service_relation "${service}" "${relation}"); then
  2287. err "Couldn't find relation ${DARKCYAN}${relation}${NORMAL}" \
  2288. "from ${DARKYELLOW}$service${NORMAL}."
  2289. return 1
  2290. fi
  2291. relation_dir=$(get_relation_data_dir "$service" "$ts" "$relation") || {
  2292. err "Failed to find relation file"
  2293. return 1
  2294. }
  2295. relation_file="$relation_dir/data"
  2296. if ! [ -e "$relation_file" ]; then
  2297. e "$rc" > "$relation_file"
  2298. chmod go-rwx "$relation_file" ## protecting this file
  2299. fi
  2300. echo "$relation_file"
  2301. }
  2302. export -f service:relation-file
  2303. service:relation-options() {
  2304. local service_path="$1" relation_file
  2305. relation_file=$(service:relation-file "$service_path") || {
  2306. err "Failed to find relation file"
  2307. return 1
  2308. }
  2309. cat "$relation_file"
  2310. }
  2311. export -f service:relation-options
  2312. relation:get() {
  2313. local service_path="$1" query="$2" relation_file
  2314. relation_file=$(service:relation-file "$service_path") || {
  2315. err "Failed to find relation file"
  2316. return 1
  2317. }
  2318. cfg-get-value "$query" < "$relation_file"
  2319. }
  2320. export -f relation:get
  2321. services:get:upable() {
  2322. if [ -z "$CHARM_STORE_HASH" ]; then
  2323. err-d "Expected \$CHARM_STORE_HASH to be set."
  2324. return 1
  2325. fi
  2326. local services_args=("$@") cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$CHARM_STORE_HASH" "$@")"
  2327. if [ -e "$cache_file" ]; then
  2328. touch "$cache_file" || return 1
  2329. cat "$cache_file"
  2330. return 0
  2331. fi
  2332. declare -A seen
  2333. services=($(get_ordered_service_dependencies "${services_args[@]}")) || exit 1
  2334. for service in "${services[@]}"; do
  2335. mservice=$(get_master_service_for_service "$service") || exit 1
  2336. [ "${seen[$mservice]}" ] && continue
  2337. type="$(get_service_type "$mservice")" || exit 1
  2338. ## remove run-once
  2339. [ "$type" == "run-once" ] && continue
  2340. [ "$type" == "stub" ] && continue
  2341. seen[$mservice]=1
  2342. echo "$mservice"
  2343. done > "$cache_file".wip
  2344. mv "$cache_file".wip "$cache_file"
  2345. cat "$cache_file"
  2346. }
  2347. export -f services:get:upable
  2348. service:state() {
  2349. local service="$1" states state
  2350. project_name=$(get_default_project_name) || return 1
  2351. states=()
  2352. for state in "$SERVICE_STATE_PATH"/"$project_name"/"$service"/*; do
  2353. [ -e "$state" ] || continue
  2354. state=${state##*/}
  2355. states+=("$state")
  2356. done
  2357. if [[ " ${states[*]} " == *" deploying "* ]]; then
  2358. echo "deploying"
  2359. elif [[ " ${states[*]} " == *" up "* ]]; then
  2360. echo "up"
  2361. else
  2362. echo "down"
  2363. fi
  2364. }
  2365. export -f service:state
  2366. _get_charm_metadata_uses() {
  2367. local metadata="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  2368. if [ -e "$cache_file" ]; then
  2369. #debug "$FUNCNAME: SESSION cache hit $1"
  2370. cat "$cache_file" || return 1
  2371. return 0
  2372. fi
  2373. printf "%s" "$metadata" | { shyaml key-values-0 uses 2>/dev/null || true; } | tee "$cache_file"
  2374. }
  2375. export -f _get_charm_metadata_uses
  2376. _get_service_metadata() {
  2377. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2378. charm
  2379. if [ -e "$cache_file" ]; then
  2380. #debug "$FUNCNAME: SESSION cache hit $1"
  2381. cat "$cache_file"
  2382. return 0
  2383. fi
  2384. charm="$(get_service_charm "$service")" || return 1
  2385. charm.metadata "$charm" > "$cache_file"
  2386. if [ "$?" != 0 ]; then
  2387. rm -f "$cache_file" ## no cache
  2388. return 1
  2389. fi
  2390. cat "$cache_file"
  2391. }
  2392. export -f _get_service_metadata
  2393. _get_service_uses() {
  2394. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2395. metadata
  2396. if [ -e "$cache_file" ]; then
  2397. #debug "$FUNCNAME: SESSION cache hit $1"
  2398. cat "$cache_file"
  2399. return 0
  2400. fi
  2401. metadata="$(_get_service_metadata "$service")" || return 1
  2402. _get_charm_metadata_uses "$metadata" > "$cache_file"
  2403. if [ "$?" != 0 ]; then
  2404. rm -f "$cache_file" ## no cache
  2405. return 1
  2406. fi
  2407. cat "$cache_file"
  2408. }
  2409. export -f _get_service_uses
  2410. _get_services_uses() {
  2411. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2412. service rn rd
  2413. if [ -e "$cache_file" ]; then
  2414. #debug "$FUNCNAME: SESSION cache hit $1"
  2415. cat "$cache_file"
  2416. return 0
  2417. fi
  2418. for service in "$@"; do
  2419. _get_service_uses "$service" | while read-0 rn rd; do
  2420. printf "%s\0" "$service" "$rn" "$rd"
  2421. done
  2422. [ "${PIPESTATUS[0]}" == 0 ] || {
  2423. return 1
  2424. }
  2425. done > "${cache_file}.wip"
  2426. mv "${cache_file}"{.wip,} &&
  2427. cat "$cache_file" || return 1
  2428. }
  2429. export -f _get_services_uses
  2430. _get_provides_provides() {
  2431. local provides="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2432. service rn rd
  2433. if [ -e "$cache_file" ]; then
  2434. # debug "$FUNCNAME: CACHEDIR cache hit $1"
  2435. cat "$cache_file"
  2436. return 0
  2437. fi
  2438. type=$(printf "%s" "$provides" | shyaml get-type)
  2439. case "$type" in
  2440. sequence)
  2441. while read-0 prov; do
  2442. printf "%s\0" "$prov" ""
  2443. done < <(echo "$provides" | shyaml get-values-0)
  2444. ;;
  2445. struct)
  2446. printf "%s" "$provides" | shyaml key-values-0
  2447. ;;
  2448. str)
  2449. printf "%s\0" "$(echo "$provides" | shyaml get-value)" ""
  2450. ;;
  2451. *)
  2452. err "Unexpected type '$type' for provider identifier in charm '$charm'."
  2453. return 1
  2454. esac | tee "$cache_file"
  2455. return "${PIPESTATUS[0]}"
  2456. }
  2457. _get_metadata_provides() {
  2458. local metadata="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2459. service rn rd
  2460. if [ -e "$cache_file" ]; then
  2461. #debug "$FUNCNAME: CACHEDIR cache hit"
  2462. cat "$cache_file"
  2463. return 0
  2464. fi
  2465. provides=$(printf "%s" "$metadata" | shyaml -q get-value -y provides "")
  2466. [ "$provides" -a "$provides" != "''" ] || { touch "$cache_file"; return 0; }
  2467. _get_provides_provides "$provides" | tee "$cache_file"
  2468. return "${PIPESTATUS[0]}"
  2469. }
  2470. _get_services_provides() {
  2471. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2472. service rn rd
  2473. if [ -e "$cache_file" ]; then
  2474. #debug "$FUNCNAME: SESSION cache hit $1"
  2475. cat "$cache_file"
  2476. return 0
  2477. fi
  2478. ## YYY: replace the inner loop by a cached function
  2479. for service in "$@"; do
  2480. metadata="$(_get_service_metadata "$service")" || return 1
  2481. while read-0 rn rd; do
  2482. printf "%s\0" "$service" "$rn" "$rd"
  2483. done < <(_get_metadata_provides "$metadata")
  2484. done > "$cache_file"
  2485. if [ "$?" != 0 ]; then
  2486. rm -f "$cache_file" ## no cache
  2487. return 1
  2488. fi
  2489. cat "$cache_file"
  2490. }
  2491. export -f _get_services_provides
  2492. _get_charm_provides() {
  2493. if [ -z "$CHARM_STORE_HASH" ]; then
  2494. err-d "Expected \$CHARM_STORE_HASH to be set."
  2495. return 1
  2496. fi
  2497. local cache_file="$CACHEDIR/$FUNCNAME.cache.$CHARM_STORE_HASH" errlvl
  2498. if [ -e "$cache_file" ]; then
  2499. #debug "$FUNCNAME: SESSION cache hit"
  2500. cat "$cache_file"
  2501. return 0
  2502. fi
  2503. start="$SECONDS"
  2504. debug "Getting charm provider list..."
  2505. while read-0 charm _ realpath metadata; do
  2506. metadata="$(charm.metadata "$charm")" || continue
  2507. # echo "reading $charm" >&2
  2508. while read-0 rn rd; do
  2509. printf "%s\0" "$charm" "$rn" "$rd"
  2510. done < <(_get_metadata_provides "$metadata")
  2511. done < <(charm.ls) | tee "$cache_file"
  2512. errlvl="${PIPESTATUS[0]}"
  2513. debug " ..charm provider list done $GRAY(in $((SECONDS - start))s)$NORMAL"
  2514. return "$errlvl"
  2515. }
  2516. _get_charm_providing() {
  2517. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2518. relation="$1"
  2519. if [ -e "$cache_file" ]; then
  2520. #debug "$FUNCNAME: SESSION cache hit $1"
  2521. cat "$cache_file"
  2522. return 0
  2523. fi
  2524. while read-0 charm relation_name relation_def; do
  2525. [ "$relation_name" == "$relation" ] || continue
  2526. printf "%s\0" "$charm" "$relation_def"
  2527. done < <(_get_charm_provides) > "$cache_file"
  2528. if [ "$?" != 0 ]; then
  2529. rm -f "$cache_file" ## no cache
  2530. return 1
  2531. fi
  2532. cat "$cache_file"
  2533. }
  2534. _get_services_providing() {
  2535. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2536. relation="$1"
  2537. shift ## services is "$@"
  2538. if [ -e "$cache_file" ]; then
  2539. #debug "$FUNCNAME: SESSION cache hit $1"
  2540. cat "$cache_file"
  2541. return 0
  2542. fi
  2543. while read-0 service relation_name relation_def; do
  2544. [ "$relation_name" == "$relation" ] || continue
  2545. printf "%s\0" "$service" "$relation_def"
  2546. done < <(_get_services_provides "$@") > "$cache_file"
  2547. if [ "$?" != 0 ]; then
  2548. rm -f "$cache_file" ## no cache
  2549. return 1
  2550. fi
  2551. cat "$cache_file"
  2552. }
  2553. export -f _get_services_provides
  2554. _out_new_relation_from_defs() {
  2555. local service="$1" rn="$2" ts="$3" prov_def="$4" rel_def="$5" rc td rc_prov
  2556. rc_prov=$(printf "%s" "$prov_def" | shyaml -y get-value "default-options" 2>/dev/null)
  2557. ## YYYvlab: should be seen even in no debug mode no ?
  2558. rc=$(printf "%s" "$rel_def" | shyaml -y get-value "default-options" 2>/dev/null)
  2559. td=$(echo "$prov_def" | shyaml get-value 'tech-dep' 2>/dev/null)
  2560. td=${td:-True}
  2561. rc=$(merge_yaml_str "$rc_prov" "$rc") || return 1
  2562. after=$(_out_after_value_from_def "$service" "$rn" "$rel_def") || return 1
  2563. printf "%s\0" "$after" "$service" "$relation_name" "$ts" "$rc" "$td"
  2564. }
  2565. _out_after_value_from_def() {
  2566. local service="$1" relation_name="$2" relation_def="$3" after_t after
  2567. if after_t=$(echo "$relation_def" | shyaml get-type after 2>/dev/null); then
  2568. case "$after_t" in
  2569. sequence)
  2570. after="$(echo "$relation_def" | shyaml get-values after 2>/dev/null)" || return 1
  2571. after=",$service:${after//$'\n'/,$service:},"
  2572. ;;
  2573. struct)
  2574. err "Invalid type for ${WHITE}after${NORMAL}'s value in ${DARKBLUE}$relation_name${NORMAL}'s definition."
  2575. return 1
  2576. ;;
  2577. str)
  2578. after=",$service:$(echo "$relation_def" | shyaml get-value after "" 2>/dev/null)," || return 1
  2579. ;;
  2580. esac
  2581. else
  2582. after=""
  2583. fi
  2584. e "$after"
  2585. }
  2586. get_all_compose_yml_service() {
  2587. if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
  2588. COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || {
  2589. err "Failed to get compose yml hash"
  2590. return 1
  2591. }
  2592. fi
  2593. local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH"
  2594. if [ -e "${cache_file}" ]; then
  2595. #debug "$FUNCNAME: cache hit: ${cache_file}"
  2596. cat "${cache_file}"
  2597. return 0
  2598. fi
  2599. compose_yml_content=$(get_compose_yml_content) || return 1
  2600. printf "%s" "${compose_yml_content}" | shyaml keys-0 2>/dev/null > "${cache_file}.wip" || {
  2601. err "Failed to get keys of compose content."
  2602. return 1
  2603. }
  2604. mv "${cache_file}"{.wip,} || return 1
  2605. cat "${cache_file}"
  2606. }
  2607. ## Outputs all relations array.
  2608. _service:all:relations_cached() {
  2609. local services service E
  2610. services=($(compose:yml:root:services)) || return 1
  2611. get_all_relations "${services[@]}" || return 1
  2612. }
  2613. ## Outputs all relations array.
  2614. service:all:relations() {
  2615. if [ -z "$COMBINED_HASH" ]; then
  2616. err-d "Expected \$COMBINED_HASH to be set."
  2617. return 1
  2618. fi
  2619. local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMBINED_HASH"
  2620. if [ -e "${cache_file}" ]; then
  2621. # debug "$FUNCNAME: SESSION cache hit $1"
  2622. cat "${cache_file}"
  2623. return 0
  2624. fi
  2625. _service:all:relations_cached > "${cache_file}.wip" || {
  2626. err-d "Failed to compute all relations."
  2627. return 1
  2628. }
  2629. mv "${cache_file}"{.wip,} || return 1
  2630. cat "${cache_file}"
  2631. }
  2632. _service:all:relations_hash_cached() {
  2633. if [ -z "$COMBINED_HASH" ]; then
  2634. err-d "Expected \$COMBINED_HASH to be set."
  2635. return 1
  2636. fi
  2637. local cache_file="$CACHEDIR/$FUNCNAME.cache.x${COMBINED_HASH}" \
  2638. hash
  2639. if [ -e "${cache_file}" ]; then
  2640. # debug "$FUNCNAME: SESSION cache hit $cache_file"
  2641. cat "${cache_file}"
  2642. return 0
  2643. fi
  2644. service:all:relations > "${cache_file}.pre" || {
  2645. err-d "Failed to get all relations."
  2646. return 1
  2647. }
  2648. {
  2649. p0 "$(hash_get < "${cache_file}.pre")" || return 1
  2650. cat "${cache_file}.pre"
  2651. rm "${cache_file}.pre"
  2652. } > "${cache_file}".wip || return 1
  2653. mv "${cache_file}"{.wip,} || return 1
  2654. cat "${cache_file}"
  2655. }
  2656. ## Get all relations from all services in the current compose file.
  2657. ## Sets GLOBAL_ALL_RELATIONS_HASH and returns all relations array.
  2658. service:all:set_relations_hash() {
  2659. if [ -n "$GLOBAL_ALL_RELATIONS" ]; then
  2660. if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then
  2661. err "Can't access global \$GLOBAL_ALL_RELATIONS_HASH"
  2662. echo " (despite \$GLOBAL_ALL_RELATIONS being set)" >&2
  2663. return 1
  2664. fi
  2665. return 0
  2666. fi
  2667. ## sets COMPOSE_YML_CONTENT_HASH
  2668. _service:all:relations_hash_cached >/dev/null || return 1
  2669. {
  2670. read-0 GLOBAL_ALL_RELATIONS_HASH || return 1
  2671. export GLOBAL_ALL_RELATIONS_HASH
  2672. ## transfer to statedir
  2673. export GLOBAL_ALL_RELATIONS="$CACHEDIR/$FUNCNAME.cache.$COMBINED_HASH"
  2674. cat > "$GLOBAL_ALL_RELATIONS"
  2675. } < <(_service:all:relations_hash_cached)
  2676. if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
  2677. err "Failed to set \$GLOBAL_ALL_RELATIONS."
  2678. return 1
  2679. fi
  2680. if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then
  2681. err "Failed to set \$GLOBAL_ALL_RELATIONS_HASH."
  2682. return 1
  2683. fi
  2684. }
  2685. get_subset_relations () {
  2686. local service all_services services start
  2687. if [ -n "$SUBSET_ALL_RELATIONS" ]; then
  2688. return 0
  2689. fi
  2690. if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then
  2691. err-d "Can't access global \$GLOBAL_ALL_RELATIONS_HASH"
  2692. return 1
  2693. fi
  2694. cache_hash=$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH" "$(declare -f "$FUNCNAME")")
  2695. local cache_file="$CACHEDIR/$FUNCNAME.cache.$cache_hash"
  2696. if [ -e "${cache_file}" ]; then
  2697. export SUBSET_ALL_RELATIONS="$cache_file"
  2698. hash=$(hash_get < "$cache_file") || return 1
  2699. export SUBSET_ALL_RELATIONS_HASH="$hash"
  2700. cat "${cache_file}"
  2701. return 0
  2702. fi
  2703. ## collect all connected services first
  2704. all_services=("$@")
  2705. declare -A services
  2706. while [ "${#all_services[@]}" != 0 ]; do
  2707. array_pop all_services service
  2708. # debug " Getting relations for $DARKYELLOW$service$NORMAL"
  2709. while read-0 s rn ts rc td; do
  2710. [[ "$s" == "$service" ]] || continue
  2711. # debug " adding relation $DARKBLUE$rn$NORMAL to $DARKYELLOW$ts$NORMAL"
  2712. p0 "$service" "$rn" "$ts" "$rc" "$td"
  2713. if [ -z "${services[$ts]}" ] && [[ " ${all_services[@]} " != *" $ts "* ]]; then
  2714. all_services+=("$ts")
  2715. fi
  2716. done < "$GLOBAL_ALL_RELATIONS"
  2717. services["$service"]=1
  2718. done > "$cache_file.wip"
  2719. mv "$cache_file"{.wip,} || return 1
  2720. export SUBSET_ALL_RELATIONS="$cache_file"
  2721. hash=$(hash_get < "$cache_file") || return 1
  2722. export SUBSET_ALL_RELATIONS_HASH="$hash"
  2723. cat "$cache_file"
  2724. }
  2725. export -f get_subset_relations
  2726. get_all_relations () {
  2727. if [ -z "$COMBINED_HASH" ]; then
  2728. err-d "Expected \$COMBINED_HASH to be set."
  2729. return 1
  2730. fi
  2731. if [ -n "$GLOBAL_ALL_RELATIONS" ]; then
  2732. cat "$GLOBAL_ALL_RELATIONS" || return 1
  2733. return 0
  2734. fi
  2735. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$COMBINED_HASH" "$(declare -p without_relations)")" \
  2736. services all_services service services_uses services_provides \
  2737. changed summon required recommended optional
  2738. if [ -e "${cache_file}" ]; then
  2739. #debug "$FUNCNAME: SESSION cache hit $1"
  2740. export GLOBAL_ALL_RELATIONS="$cache_file"
  2741. cat "${cache_file}"
  2742. return 0
  2743. fi
  2744. declare -A services
  2745. services_uses=()
  2746. ## XXXvlab: bwerk, leveraging cache to be able to get the errorlevel here.
  2747. _get_services_uses "$@" >/dev/null || return 1
  2748. array_read-0 services_uses < <(_get_services_uses "$@")
  2749. services_provides=()
  2750. ## XXXvlab: bwerk, leveraging cache to be able to get the errorlevel here.
  2751. _get_services_provides "$@" >/dev/null || return 1
  2752. array_read-0 services_provides < <(_get_services_provides "$@")
  2753. for service in "$@"; do
  2754. services[$service]=1
  2755. done
  2756. all_services=("$@")
  2757. while [ "${#all_services[@]}" != 0 ]; do
  2758. array_pop all_services service
  2759. while read-0-err E relation_name ts relation_config tech_dep; do
  2760. [ "${without_relations[$service:$relation_name]}" ] && {
  2761. debug "Ignoring compose $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> ${DARKYELLOW}$ts$NORMAL"
  2762. continue
  2763. }
  2764. ## First is priority, that can be adjusted in second step
  2765. printf "%s\0" "" "$service" "$relation_name" "$ts" "$relation_config" "$tech_dep"
  2766. ## adding target services ?
  2767. [ "${services[$ts]}" ] && continue
  2768. array_read-0 services_uses < <(_get_services_uses "$ts")
  2769. all_services+=("$ts")
  2770. services[$ts]=1
  2771. done < <(p-err get_compose_relations "$service")
  2772. if [ "$E" != 0 ]; then
  2773. err "Failed to get relations for $DARKYELLOW$service$NORMAL."
  2774. return 1
  2775. fi
  2776. done > "${cache_file}.wip"
  2777. while true; do
  2778. changed=
  2779. new_services_uses=()
  2780. summon=()
  2781. required=()
  2782. recommended=()
  2783. optional=()
  2784. while [ "${#services_uses[@]}" != 0 ]; do
  2785. service="${services_uses[0]}"
  2786. relation_name="${services_uses[1]}"
  2787. relation_def="${services_uses[2]}"
  2788. services_uses=("${services_uses[@]:3}")
  2789. [ "${without_relations[$service:$relation_name]}" ] && {
  2790. debug "Skipping $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW*$NORMAL"
  2791. continue
  2792. }
  2793. default_options=$(printf "%s" "$relation_def" | shyaml -y get-value "default-options" 2>/dev/null)
  2794. after=$(_out_after_value_from_def "$service" "$relation_name" "$relation_def") || return 1
  2795. ## is this "use" declaration satisfied ?
  2796. found=
  2797. while read-0 p s rn ts rc td; do
  2798. if [ -z "$found" -a "$service" == "$s" -a "$relation_name" == "$rn" ]; then
  2799. if [ "$default_options" ]; then
  2800. rc=$(merge_yaml_str "$default_options" "$rc") || return 1
  2801. fi
  2802. found="$ts"
  2803. p="$after"
  2804. fi
  2805. printf "%s\0" "$p" "$s" "$rn" "$ts" "$rc" "$td"
  2806. done < "${cache_file}.wip" > "${cache_file}.wip.new"
  2807. mv "${cache_file}.wip.new" "${cache_file}.wip"
  2808. if [ "$found" ]; then ## this "use" declaration was satisfied
  2809. debug "${DARKYELLOW}$service${NORMAL} use declaration for relation" \
  2810. "${DARKBLUE}$relation_name${NORMAL} is satisfied with ${DARKYELLOW}$found${NORMAL}"
  2811. continue
  2812. fi
  2813. auto=$(echo "$relation_def" | shyaml get-value auto pair 2>/dev/null)
  2814. auto=${auto:-pair}
  2815. case "$auto" in
  2816. "pair"|"summon")
  2817. service_list=()
  2818. array_read-0 service_list < <(array_keys_to_stdin services)
  2819. providers=()
  2820. providers_def=()
  2821. array_read-0 providers providers_def < <(_get_services_providing "$relation_name" "${service_list[@]}")
  2822. if [ "${#providers[@]}" == 1 ]; then
  2823. ts="${providers[0]}"
  2824. debug "Auto-pairs ${DARKYELLOW}$service${NORMAL}" \
  2825. "--${DARKBLUE}$relation_name${NORMAL}--> ${DARKYELLOW}$ts${NORMAL}"
  2826. _out_new_relation_from_defs "$service" "$relation_name" "$ts" \
  2827. "${providers_def[0]}" "$relation_def" \
  2828. >> "${cache_file}.wip" || return 1
  2829. ## Adding service
  2830. [ "${services[$ts]}" ] && continue
  2831. array_read-0 new_services_uses < <(_get_services_uses "$ts")
  2832. services[$ts]=1
  2833. changed=1
  2834. continue
  2835. fi
  2836. if [ "${#providers[@]}" -gt 1 ]; then
  2837. msg=""
  2838. warn "No auto-pairing ${DARKYELLOW}$service${NORMAL}" \
  2839. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  2840. "(> 1 provider)."
  2841. elif [ "$auto" == "summon" ]; then ## no provider
  2842. summon+=("$service" "$relation_name" "$relation_def")
  2843. fi
  2844. ;;
  2845. null|disable|disabled)
  2846. :
  2847. ;;
  2848. *)
  2849. err "Invalid ${WHITE}auto${NORMAL} value '$auto'."
  2850. return 1
  2851. ;;
  2852. esac
  2853. constraint=$(echo "$relation_def" | shyaml get-value constraint 2>/dev/null)
  2854. constraint=${constraint:-optional}
  2855. case "$constraint" in
  2856. "required")
  2857. required+=("$service" "$relation_name" "$relation_def")
  2858. ;;
  2859. "recommended")
  2860. recommended+=("$service" "$relation_name" "$relation_def")
  2861. ;;
  2862. "optional")
  2863. optional+=("$service" "$relation_name" "$relation_def")
  2864. ;;
  2865. *)
  2866. err "Invalid ${WHITE}constraint${NORMAL} value '$constraint'."
  2867. return 1
  2868. ;;
  2869. esac
  2870. new_services_uses+=("$service" "$relation_name" "$relation_def") ## re-queue it
  2871. done
  2872. services_uses=("${new_services_uses[@]}")
  2873. if [ "$changed" ]; then
  2874. continue
  2875. fi
  2876. ## situation is stable
  2877. if [ "${#summon[@]}" != 0 ]; then
  2878. declare -A summon_requeued=()
  2879. while [ "${#summon[@]}" != 0 ]; do
  2880. service="${summon[0]}"
  2881. relation_name="${summon[1]}"
  2882. relation_def="${summon[2]}"
  2883. summon=("${summon[@]:3}")
  2884. providers=()
  2885. providers_def=()
  2886. array_read-0 providers providers_def < <(_get_charm_providing "$relation_name" "${service_list[@]}")
  2887. ## select first provider that is not a stub
  2888. new_providers=()
  2889. new_providers_def=()
  2890. while [[ "${#providers[@]}" != 0 ]]; do
  2891. provider="${providers[0]}"
  2892. provider_def="${providers_def[0]}"
  2893. providers=("${providers[@]:1}")
  2894. providers_def=("${providers_def[@]:1}")
  2895. type="$(get_service_type "$provider")" || true
  2896. [ "$type" == "stub" ] && continue
  2897. new_providers+=("$provider")
  2898. new_providers_def+=("$provider_def")
  2899. done
  2900. providers=("${new_providers[@]}")
  2901. providers_def=("${new_providers_def[@]}")
  2902. if [ "${#providers[@]}" == 0 ]; then
  2903. err "Summoning a ${DARKBLUE}$relation_name${NORMAL} provider failed: none were found in charm store."
  2904. return 1
  2905. fi
  2906. if [ "${#providers[@]}" -gt 1 ]; then
  2907. ## if there are multiple providers (for instance
  2908. ## sql-database), there are some case where other
  2909. ## services will also summon a more specific
  2910. ## postgres-database, that will solve our
  2911. ## constraint. So we'd rather pass (and requeue)
  2912. if [ -z "${summon_requeued[$service/$relation_name]}" ]; then
  2913. debug "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  2914. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  2915. "(> 1 provider). Requeuing."
  2916. summon+=("$service" "$relation_name" "$relation_def") ## re-queue it
  2917. summon_requeued["$service/$relation_name"]=1
  2918. continue
  2919. else
  2920. warn "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  2921. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  2922. "(> 1 provider). Choosing first."
  2923. fi
  2924. fi
  2925. ts="${providers[0]}"
  2926. ## YYYvlab: should be seen even in no debug mode no ?
  2927. debug "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  2928. "--${DARKBLUE}$relation_name${NORMAL}--> ${DARKYELLOW}$ts${NORMAL}"
  2929. _out_new_relation_from_defs "$service" "$relation_name" "$ts" \
  2930. "${providers_def[0]}" "$relation_def" \
  2931. >> "${cache_file}.wip" || return 1
  2932. ## Adding service
  2933. [ "${services[$ts]}" ] && continue
  2934. array_read-0 services_uses < <(_get_services_uses "$ts")
  2935. services[$ts]=1
  2936. changed=1
  2937. continue 2
  2938. done
  2939. continue
  2940. fi
  2941. [ "$NO_CONSTRAINT_CHECK" ] && break
  2942. if [ "${#required[@]}" != 0 ]; then
  2943. echo "$(_display_solves required)" | sed -r "s/^/${RED}||${NORMAL} /g" >&2
  2944. err "Required relations not satisfied"
  2945. return 1
  2946. fi
  2947. if [ "${#recommended[@]}" != 0 ]; then
  2948. ## make recommendation
  2949. echo "$(_display_solves recommended)" | sed -r "s/^/${YELLOW}||${NORMAL} /g" >&2
  2950. fi
  2951. if [ -z "$QUIET" ]; then
  2952. if [ "${#optional[@]}" != 0 ]; then
  2953. ## inform about options
  2954. echo "$(_display_solves optional)" | sed -r "s/^/${BLUE}||${NORMAL} /g" >&2
  2955. fi
  2956. fi
  2957. # if [ "${#required[@]}" != 0 ]; then
  2958. # err "Required relations not satisfied"
  2959. # return 1
  2960. # fi
  2961. if [ "${#recommended[@]}" != 0 ]; then
  2962. warn "Recommended relations not satisfied"
  2963. fi
  2964. break
  2965. done
  2966. if [ "$?" != 0 ]; then
  2967. rm -f "${cache_file}"{,.wip,.wip.new} ## no cache
  2968. return 1
  2969. fi
  2970. ##
  2971. ## Sort relations thanks to uses =metadata.yml= relations.
  2972. ##
  2973. mv "${cache_file}.wip"{,.in} &&
  2974. rm -f "${cache_file}.wip.final" &&
  2975. touch "${cache_file}.wip.final" || {
  2976. err "Unexpected error when mangling cache files."
  2977. return 1
  2978. }
  2979. declare -A relation_done=()
  2980. while true; do
  2981. had_remaining_relation=
  2982. had_new_relation=
  2983. while read-0 p s rn ts rc td; do
  2984. if [ -z "$p" ] || [ "$p" == "," ]; then
  2985. relation_done["$s:$rn"]=1
  2986. # printf " .. %-30s %-30s %-30s\n" "$s" "$ts" "$rn" >&2
  2987. printf "%s\0" "$s" "$rn" "$ts" "$rc" "$td" >> "${cache_file}.wip.final"
  2988. had_new_relation=1
  2989. else
  2990. # printf " !! %-30s %-30s %-30s\n" "$p" "$s" "$rn" >&2
  2991. printf "%s\0" "$p" "$s" "$rn" "$ts" "$rc" "$td" >> "${cache_file}.wip.out"
  2992. had_remaining_relation=1
  2993. fi
  2994. done < "${cache_file}.wip.in"
  2995. [ -z "$had_remaining_relation" ] && break
  2996. mv "${cache_file}.wip."{out,in}
  2997. while read-0 p s rn ts rc td; do
  2998. for rel in "${!relation_done[@]}"; do
  2999. p="${p//,$rel,/,}"
  3000. done
  3001. # printf " CC %-30s %-30s %-30s\n" "$p" "$s" "$rn" >&2
  3002. if [ -z "$had_new_relation" ]; then
  3003. err "${DARKYELLOW}$s${NORMAL} --${DARKBLUE}$rn${NORMAL}--> ${DARKYELLOW}$ts${NORMAL} missing required ${WHITE}after${NORMAL} relations:"
  3004. for rel in ${p//,/ }; do
  3005. rel_s=${rel%%:*}
  3006. rel_r=${rel##*:}
  3007. echo " - ${DARKYELLOW}$rel_s${NORMAL} --${DARKBLUE}$rel_r${NORMAL}--> ${DARKGRAY}*${NORMAL}" >&2
  3008. done
  3009. else
  3010. printf "%s\0" "$p" "$s" "$rn" "$ts" "$rc" "$td" >> "${cache_file}.wip.out"
  3011. fi
  3012. done < "${cache_file}.wip.in"
  3013. if [ -z "$had_new_relation" ]; then
  3014. rm -f "${cache_file}"{,.wip{,new,in,out,final}} ## no cache
  3015. return 1
  3016. fi
  3017. mv "${cache_file}.wip."{out,in}
  3018. done
  3019. mv "${cache_file}"{.wip.final,} || return 1
  3020. export GLOBAL_ALL_RELATIONS="$cache_file"
  3021. GLOBAL_ALL_RELATIONS_HASH=$(hash_get < "$cache_file") || return 1
  3022. export GLOBAL_ALL_RELATIONS_HASH
  3023. cat "$cache_file"
  3024. }
  3025. export -f get_all_relations
  3026. _display_solves() {
  3027. local array_name="$1" by_relation msg
  3028. ## inform about options
  3029. msg=""
  3030. declare -A by_relation
  3031. while read-0 service relation_name relation_def; do
  3032. solves=$(printf "%s" "$relation_def" | shyaml -y get-value solves 2>/dev/null);
  3033. auto=$(printf "%s" "$relation_def" | shyaml get-value auto 2>/dev/null);
  3034. if [ -z "$solves" ]; then
  3035. continue
  3036. fi
  3037. by_relation[$relation_name]+=$(printf "\n %s" "${DARKYELLOW}$service$NORMAL for:")
  3038. if [ "$auto" == "pair" ]; then
  3039. requirement="add provider in cluster to auto-pair"
  3040. else
  3041. requirement="add explicit relation"
  3042. fi
  3043. while read-0 name def; do
  3044. by_relation[$relation_name]+=$(printf "\n - ${DARKCYAN}%-15s${NORMAL} %s (%s)" "$name" "$def" "$requirement")
  3045. done < <(printf "%s" "$solves" | shyaml key-values-0)
  3046. done < <(array_values_to_stdin "$array_name")
  3047. while read-0 relation_name message; do
  3048. msg+="$(printf "\n${DARKBLUE}%s$NORMAL provider is $array_name by%s" \
  3049. "$relation_name" "$message" )"
  3050. done < <(array_kv_to_stdin by_relation)
  3051. if [ "$msg" ]; then
  3052. printf "%s\n" "${msg:1}"
  3053. fi
  3054. }
  3055. get_compose_relation_def() {
  3056. local service="$1" relation="$2" relation_name target_service relation_config tech_dep
  3057. while read-0 relation_name target_service relation_config tech_dep; do
  3058. [ "$relation_name" == "$relation" ] || continue
  3059. printf "%s\0%s\0%s\0" "$target_service" "$relation_config" "$tech_dep"
  3060. done < <(get_compose_relations "$service") || return 1
  3061. }
  3062. export -f get_compose_relation_def
  3063. run_service_relations () {
  3064. local service services loaded subservices subservice
  3065. PROJECT_NAME=$(get_default_project_name) || return 1
  3066. export PROJECT_NAME
  3067. declare -A loaded
  3068. subservices=$(get_ordered_service_dependencies "$@") || return 1
  3069. for service in $subservices; do
  3070. # debug "Upping dep's relations of ${DARKYELLOW}$service${NORMAL}:"
  3071. for subservice in $(get_service_deps "$service") "$service"; do
  3072. [ "${loaded[$subservice]}" ] && continue
  3073. export BASE_SERVICE_NAME=$service
  3074. MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
  3075. MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1
  3076. RELATION_BASE_COMPOSE_DEF=$(get_compose_service_def "$subservice") || return 1
  3077. export RELATION_BASE_COMPOSE_DEF MASTER_BASE_{CHARM,SERVICE}_NAME
  3078. # debug " Relations of ${DARKYELLOW}$subservice${NORMAL}:"
  3079. while read-0 relation_name target_service relation_config tech_dep; do
  3080. [ "${without_relations[$service:$relation_name]}" ] && {
  3081. debug "Skipping $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW*$NORMAL"
  3082. continue
  3083. }
  3084. export relation_config
  3085. export TARGET_SERVICE_NAME=$target_service
  3086. MASTER_TARGET_SERVICE_NAME=$(get_top_master_service_for_service "$target_service") || return 1
  3087. MASTER_TARGET_CHARM_NAME=$(get_service_charm "$MASTER_TARGET_SERVICE_NAME") || return 1
  3088. RELATION_TARGET_COMPOSE_DEF=$(get_compose_service_def "$target_service") || return 1
  3089. export RELATION_TARGET_COMPOSE_DEF MASTER_TARGET_{CHARM,SERVICE}_NAME
  3090. Wrap "${wrap_opts[@]}" -d "building $DARKYELLOW$subservice$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW$target_service$NORMAL" <<EOF || return 1
  3091. _run_service_relation "$relation_name" "$subservice" "$target_service" "\$relation_config"
  3092. EOF
  3093. done < <(get_service_relations "$subservice") || return 1
  3094. loaded[$subservice]=1
  3095. done
  3096. done
  3097. }
  3098. export -f run_service_relations
  3099. _run_service_action_direct() {
  3100. local service="$1" action="$2" charm _dummy project_name
  3101. shift; shift
  3102. read-0 charm action_script_path || true ## against 'set -e' that could be setup in parent scripts
  3103. if read-0 _dummy || [ "$_dummy" ]; then
  3104. print_syntax_error "$FUNCNAME: too many arguments in action descriptor"
  3105. return 1
  3106. fi
  3107. project_name=$(get_default_project_name) || return 1
  3108. export PROJECT_NAME="$project_name"
  3109. export state_tmpdir
  3110. (
  3111. set +e ## Prevents unwanted leaks from parent shell
  3112. export COMPOSE_CONFIG=$(get_compose_yml_content)
  3113. export METADATA_CONFIG=$(charm.metadata "$charm")
  3114. export SERVICE_NAME=$service
  3115. export ACTION_NAME=$action
  3116. export ACTION_SCRIPT_PATH="$action_script_path"
  3117. export CONTAINER_NAME=$(get_top_master_service_for_service "$service")
  3118. export DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$CONTAINER_NAME")
  3119. export SERVICE_DATASTORE="$DATASTORE/$service"
  3120. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  3121. exname="$exname $ACTION_NAME $SERVICE_NAME" \
  3122. stdbuf -oL -eL bash -c 'charm.run_direct_action "$@"' -- "$charm" "$action" "$@"
  3123. ) 0<&6 ## inject general stdin
  3124. }
  3125. export -f _run_service_action_direct
  3126. _run_service_action_relation() {
  3127. local service="$1" action="$2" charm target_charm relation_name relation_config _dummy
  3128. shift; shift
  3129. read-0 charm target_service target_charm relation_name relation_config action_script_path || true
  3130. if read-0 _dummy || [ "$_dummy" ]; then
  3131. print_syntax_error "$FUNCNAME: too many arguments in action descriptor"
  3132. return 1
  3133. fi
  3134. RELATION_DATA_FILE=$(get_relation_data_file "$service" "$target_service" "$relation_name" "$relation_config") || return 1
  3135. export action_errlvl_file="$state_tmpdir/action-$service-$charm-$action-errlvl"
  3136. export state_tmpdir
  3137. (
  3138. set +e ## Prevents unwanted leaks from parent shell
  3139. export METADATA_CONFIG=$(charm.metadata "$charm")
  3140. export SERVICE_NAME=$service
  3141. export RELATION_TARGET_SERVICE="$target_service"
  3142. export RELATION_TARGET_CHARM="$target_charm"
  3143. export RELATION_BASE_SERVICE="$service"
  3144. export RELATION_BASE_CHARM="$charm"
  3145. export RELATION_DATA_FILE="$RELATION_DATA_FILE"
  3146. export ACTION_NAME=$action
  3147. export ACTION_SCRIPT_PATH="$action_script_path"
  3148. export CONTAINER_NAME=$(get_top_master_service_for_service "$service")
  3149. export DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$CONTAINER_NAME")
  3150. export SERVICE_DATASTORE="$DATASTORE/$service"
  3151. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  3152. exname="$exname $ACTION_NAME $SERVICE_NAME" \
  3153. stdbuf -oL -eL bash -c 'charm.run_relation_action "$@"' -- "$target_charm" "$relation_name" "$action" "$@"
  3154. ) 0<&6 ## inject general stdin
  3155. }
  3156. export -f _run_service_action_relation
  3157. get_relation_data_dir() {
  3158. local service="$1" target_service="$2" relation_name="$3" \
  3159. cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  3160. if [ -e "$cache_file" ]; then
  3161. # debug "$FUNCNAME: cache hit ($*)"
  3162. cat "$cache_file"
  3163. return 0
  3164. fi
  3165. local project relation_dir
  3166. project=${PROJECT_NAME}
  3167. if [ -z "$project" ]; then
  3168. project=$(get_default_project_name) || return 1
  3169. fi
  3170. relation_dir="$VARDIR/relations/$project/${service}-${target_service}/$relation_name"
  3171. if ! [ -d "$relation_dir" ]; then
  3172. mkdir -p "$relation_dir" || return 1
  3173. chmod go-rwx "$relation_dir" || return 1 ## protecting this directory
  3174. fi
  3175. echo "$relation_dir" | tee "$cache_file"
  3176. }
  3177. export -f get_relation_data_dir
  3178. get_relation_data_file() {
  3179. local service="$1" target_service="$2" relation_name="$3" relation_config="$4" \
  3180. new new_md5 relation_dir relation_data_file
  3181. relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1
  3182. relation_data_file="$relation_dir/data"
  3183. new=
  3184. if [ -e "$relation_data_file" ]; then
  3185. ## Has reference changed ?
  3186. new_md5=$(e "$relation_config" | md5_compat)
  3187. if [ "$new_md5" != "$(cat "$relation_data_file.md5_ref" 2>/dev/null)" ]; then
  3188. new=true
  3189. fi
  3190. else
  3191. new=true
  3192. fi
  3193. if [ -n "$new" ]; then
  3194. OLDUMASK=$(umask)
  3195. umask 0077
  3196. e "$relation_config" > "$relation_data_file"
  3197. umask "$OLDUMASK"
  3198. e "$relation_config" | md5_compat > "$relation_data_file.md5_ref"
  3199. fi
  3200. echo "$relation_data_file"
  3201. }
  3202. export -f get_relation_data_file
  3203. has_service_action () {
  3204. if [ -z "$CHARM_STORE_HASH" ]; then
  3205. err-d "Can't access global \$CHARM_STORE_HASH"
  3206. return 1
  3207. fi
  3208. local service="$1" action="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$2.$CHARM_STORE_HASH" \
  3209. charm target_charm relation_name target_service relation_config _tech_dep \
  3210. path
  3211. if [ -e "$cache_file" ]; then
  3212. # debug "$FUNCNAME: cache hit ($*)"
  3213. if [ -s "$cache_file" ]; then
  3214. cat "$cache_file"
  3215. return 0
  3216. else
  3217. return 1
  3218. fi
  3219. fi
  3220. charm=$(get_service_charm "$service") || return 1
  3221. ## Action directly provided ?
  3222. if path=$(charm.has_direct_action "$charm" "$action"); then
  3223. p0 "direct" "$charm" "$path" | tee "$cache_file"
  3224. return 0
  3225. fi
  3226. ## Action provided by relation ?
  3227. while read-0 relation_name target_service relation_config _tech_dep; do
  3228. target_charm=$(get_service_charm "$target_service") || return 1
  3229. if path=$(charm.has_relation_action "$target_charm" "$relation_name" "$action"); then
  3230. p0 "relation" "$charm" "$target_service" "$target_charm" "$relation_name" "$relation_config" "$path" | tee "$cache_file"
  3231. return 0
  3232. fi
  3233. done < <(get_service_relations "$service")
  3234. touch "$cache_file"
  3235. return 1
  3236. # master=$(get_top_master_service_for_service "$service")
  3237. # [ "$master" == "$charm" ] && return 1
  3238. # has_service_action "$master" "$action"
  3239. }
  3240. export -f has_service_action
  3241. run_service_action () {
  3242. local service="$1" action="$2" errlvl
  3243. shift ; shift
  3244. exec 6<&0 ## saving stdin
  3245. {
  3246. if ! read-0 action_type; then
  3247. info "Service $DARKYELLOW$service$NORMAL does not have any action $DARKCYAN$action$NORMAL defined."
  3248. info " Add an executable script to 'actions/$action' to implement action."
  3249. return 1
  3250. fi
  3251. "_run_service_action_${action_type}" "$service" "$action" "$@"
  3252. errlvl="$?"
  3253. } < <(has_service_action "$service" "$action")
  3254. exec 0<&6 6<&- ## restoring stdin
  3255. return "$errlvl"
  3256. }
  3257. export -f run_service_action
  3258. get_compose_relation_config() {
  3259. local service=$1 relation_config cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  3260. if [ -e "$cache_file" ]; then
  3261. # debug "$FUNCNAME: cache hit ($*)"
  3262. cat "$cache_file"
  3263. return 0
  3264. fi
  3265. compose_service_def=$(get_compose_service_def "$service") || return 1
  3266. echo "$compose_service_def" | shyaml get-value "relations" 2>/dev/null | tee "$cache_file"
  3267. }
  3268. export -f get_compose_relation_config
  3269. # ## Return key-values-0
  3270. # get_compose_relation_config_for_service() {
  3271. # local service=$1 relation_name=$2 relation_config
  3272. # compose_service_relations=$(get_compose_relation_config "$service") || return 1
  3273. # if ! relation_config=$(
  3274. # echo "$compose_service_relations" |
  3275. # shyaml get-value "${relation_name}" 2>/dev/null); then
  3276. # err "Couldn't find $DARKYELLOW${service}$NORMAL/${WHITE}${relation_name}$NORMAL" \
  3277. # "relation config in compose configuration."
  3278. # return 1
  3279. # fi
  3280. # if [ -z "$relation_config" ]; then
  3281. # err "Relation ${WHITE}mysql-database$NORMAL is empty in compose configuration."
  3282. # return 1
  3283. # fi
  3284. # if ! echo "$relation_config" | shyaml key-values-0 2>/dev/null; then
  3285. # err "No key/values in ${DARKBLUE}mysql-database$NORMAL of compose config."
  3286. # return 1
  3287. # fi
  3288. # }
  3289. # export -f get_compose_relation_config_for_service
  3290. _get_container_relation() {
  3291. local metadata=$1 found relation_name relation_def
  3292. found=
  3293. while read-0 relation_name relation_def; do
  3294. [ "$(echo "$relation_def" | shyaml get-value "scope" 2>/dev/null)" == "container" ] && {
  3295. found="$relation_name"
  3296. break
  3297. }
  3298. done < <(_get_charm_metadata_uses "$metadata")
  3299. if [ -z "$found" ]; then
  3300. err "Charm $DARKPINK$charm$NORMAL is a subordinate but does not have any required relation declaration with" \
  3301. "${WHITE}scope${NORMAL} set to 'container'."
  3302. return 1
  3303. fi
  3304. printf "%s" "$found"
  3305. }
  3306. _get_master_service_for_service_cached () {
  3307. local service="$1" charm="$2" metadata="$3" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  3308. charm requires master_charm target_charm target_service service_def found
  3309. if [ -e "$cache_file" ]; then
  3310. # debug "$FUNCNAME: STATIC cache hit ($1)"
  3311. cat "$cache_file" &&
  3312. touch "$cache_file" || return 1
  3313. return 0
  3314. fi
  3315. if ! [[ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" =~ ^True|true$ ]]; then
  3316. ## just return service name
  3317. echo "$service" | tee "$cache_file"
  3318. return 0
  3319. fi
  3320. ## Action provided by relation ?
  3321. container_relation=$(_get_container_relation "$metadata") || return 1
  3322. read-0 target_service _ _ < <(get_service_relation "$service" "$container_relation")
  3323. if [ -z "$target_service" ]; then
  3324. err "Couldn't find ${WHITE}relations.${container_relation}${NORMAL} in" \
  3325. "${DARKYELLOW}$service$NORMAL compose definition."
  3326. err ${FUNCNAME[@]}
  3327. return 1
  3328. fi
  3329. echo "$target_service" | tee "$cache_file"
  3330. }
  3331. export -f _get_master_service_for_service_cached
  3332. get_master_service_for_service() {
  3333. if [ -z "$CHARM_STORE_HASH" ]; then
  3334. err-d "Expected \$CHARM_STORE_HASH to be set."
  3335. return 1
  3336. fi
  3337. local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH" \
  3338. charm metadata result
  3339. if [ -e "$cache_file" ]; then
  3340. # debug "$FUNCNAME: SESSION cache hit ($*)"
  3341. cat "$cache_file" || return 1
  3342. return 0
  3343. fi
  3344. charm=$(get_service_charm "$service") || return 1
  3345. metadata=$(charm.metadata "$charm" 2>/dev/null) || {
  3346. metadata=""
  3347. warn "No charm $DARKPINK$charm$NORMAL found."
  3348. }
  3349. result=$(_get_master_service_for_service_cached "$service" "$charm" "$metadata") || return 1
  3350. echo "$result" | tee "$cache_file" || return 1
  3351. }
  3352. export -f get_master_service_for_service
  3353. get_top_master_service_for_service() {
  3354. if [ -z "$CHARM_STORE_HASH" ]; then
  3355. err-d "Expected \$CHARM_STORE_HASH to be set."
  3356. return 1
  3357. fi
  3358. local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH" \
  3359. current_service
  3360. if [ -e "$cache_file" ]; then
  3361. # debug "$FUNCNAME: cache hit ($*)"
  3362. touch "$cache_file" || return 1
  3363. cat "$cache_file"
  3364. return 0
  3365. fi
  3366. current_service="$service"
  3367. while true; do
  3368. master_service=$(get_master_service_for_service "$current_service") || return 1
  3369. [ "$master_service" == "$current_service" ] && break
  3370. current_service="$master_service"
  3371. done
  3372. echo "$current_service" | tee "$cache_file"
  3373. return 0
  3374. }
  3375. export -f get_top_master_service_for_service
  3376. ##
  3377. ## The result is a mixin that is not always a complete valid
  3378. ## docker-compose entry (thinking of subordinates). The result
  3379. ## will be merge with master charms.
  3380. _get_docker_compose_mixin_from_metadata_cached() {
  3381. local service="$1" charm="$2" metadata="$3" \
  3382. has_build_dir="$4" \
  3383. cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  3384. metadata_file metadata volumes docker_compose subordinate image \
  3385. mixin mixins tmemory memory limit docker_memory
  3386. if [ -e "$cache_file" ]; then
  3387. #debug "$FUNCNAME: STATIC cache hit $1"
  3388. cat "$cache_file" &&
  3389. touch "$cache_file" || return 1
  3390. return 0
  3391. fi
  3392. mixins=("$(echo -en "labels:\n- compose.charm=$charm")")
  3393. if [ "$metadata" ]; then
  3394. ## resources to volumes
  3395. volumes=$(
  3396. for resource_type in data config; do
  3397. while read-0 resource; do
  3398. eval "echo \" - \$HOST_${resource_type^^}STORE/\$service\$resource:\$resource:rw\""
  3399. done < <(echo "$metadata" | shyaml get-values-0 "${resource_type}-resources" 2>/dev/null)
  3400. done
  3401. while read-0 resource; do
  3402. if [[ "$resource" == /*:/*:* ]]; then
  3403. echo " - $resource"
  3404. elif [[ "$resource" == /*:/* ]]; then
  3405. echo " - $resource:rw"
  3406. elif [[ "$resource" == /*:* ]]; then
  3407. echo " - ${resource%%:*}:$resource"
  3408. elif [[ "$resource" =~ ^/[^:]+$ ]]; then
  3409. echo " - $resource:$resource:rw"
  3410. else
  3411. die "Invalid host-resource specified in 'metadata.yml'."
  3412. fi
  3413. done < <(printf "%s" "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null)
  3414. while read-0 resource; do
  3415. dest="$(charm.get_dir "$charm")/resources$resource"
  3416. if ! [ -e "$dest" ]; then
  3417. die "charm-resource: '$resource' does not exist (file: '$dest')."
  3418. fi
  3419. echo " - $dest:$resource:ro"
  3420. done < <(echo "$metadata" | shyaml get-values-0 "charm-resources" 2>/dev/null)
  3421. ) || return 1
  3422. if [ "$volumes" ]; then
  3423. mixins+=("volumes:"$'\n'"$volumes")
  3424. fi
  3425. type="$(printf "%s" "$metadata" | shyaml get-value type 2>/dev/null)" || true
  3426. if [ "$type" != "run-once" ]; then
  3427. mixins+=("restart: unless-stopped")
  3428. fi
  3429. docker_compose=$(printf "%s" "$metadata" | shyaml get-value -y "docker-compose" 2>/dev/null) || true
  3430. if [ "$docker_compose" ]; then
  3431. mixins+=("$docker_compose")
  3432. fi
  3433. if [[ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" =~ ^True|true$ ]]; then
  3434. subordinate=true
  3435. fi
  3436. fi
  3437. image=$(echo "$metadata" | shyaml get-value "docker-image" 2>/dev/null)
  3438. [ "$image" == "None" ] && image=""
  3439. if [ -n "$image" ]; then
  3440. if [ -n "$subordinate" ]; then
  3441. err "Subordinate charm can not have a ${WHITE}docker-image${NORMAL} value."
  3442. return 1
  3443. fi
  3444. mixins+=("image: $image")
  3445. elif [ "$has_build_dir" ]; then
  3446. if [ "$subordinate" ]; then
  3447. err "Subordinate charm can not have a 'build' sub directory."
  3448. return 1
  3449. fi
  3450. mixins+=("build: $(charm.get_dir "$charm")/build")
  3451. fi
  3452. limit=$(e "$metadata" | yq '.limit' 2>/dev/null) || return 1
  3453. [ "$limit" == "null" ] && limit=""
  3454. if [ -n "$limit" ]; then
  3455. if ! read-0-err E tmemory memory < <(e "$limit" | wyq ".memory | type, .memory") ||
  3456. [ "$E" != 0 ]; then
  3457. err "Unexpected error in ${DARKPINK}$charm${NORMAL}'s metadata when parsing ${WHITE}.limit${NORMAL}"
  3458. return 1
  3459. fi
  3460. case "$tmemory" in
  3461. '!!str'|'!!int')
  3462. docker_memory="$(e "$memory" | numfmt --from iec)" || {
  3463. err "Invalid format specified for .limit.memory: '$memory'."
  3464. return 1
  3465. }
  3466. ;;
  3467. '!!float')
  3468. err "Unexpected value in ${DARKPINK}$charm${NORMAL}'s metadata for ${WHITE}.limit.memory${NORMAL}."
  3469. echo " You need to specify a unit (like 'K', 'M', 'G' ...)." >&2
  3470. return 1
  3471. ;;
  3472. '!!null')
  3473. :
  3474. ;;
  3475. *)
  3476. err "Unexpected type '${tmemory#!!}' in ${DARKPINK}$charm${NORMAL}'s metadata" \
  3477. "for ${WHITE}.limit.memory${NORMAL}."
  3478. echo " You need to check ${DARKPINK}$charm${NORMAL}'s metadata " \
  3479. "for ${WHITE}.limit.memory${NORMAL} and provide a valid value" >&2
  3480. echo " Example values: '1.5G', '252M', ..." >&2
  3481. return 1
  3482. ;;
  3483. esac
  3484. if [ -n "$docker_memory" ]; then
  3485. if [[ "$docker_memory" -lt 6291456 ]]; then
  3486. err "Can't limit service to lower than 6M."
  3487. echo " Specified limit of $memory (=$docker_memory) is lower than docker's min limit of 6M (=6291456)." >&2
  3488. echo " The provided limit to memory is lower than minimum memory for a container." >&2
  3489. echo " Please remove memory limit in ${DARKPINK}$charm${NORMAL}'s metadata or raise it." >&2
  3490. return 1
  3491. fi
  3492. mixins+=(
  3493. "mem_limit: $docker_memory"
  3494. "memswap_limit: $docker_memory"
  3495. )
  3496. fi
  3497. fi
  3498. ## Final merging
  3499. mixin=$(merge_yaml_str "${mixins[@]}") || {
  3500. err "Failed to merge mixins from ${DARKPINK}${charm}${NORMAL} metadata."
  3501. return 1
  3502. }
  3503. echo "$mixin" | tee "$cache_file"
  3504. }
  3505. export -f _get_docker_compose_mixin_from_metadata_cached
  3506. get_docker_compose_mixin_from_metadata() {
  3507. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  3508. if [ -e "$cache_file" ]; then
  3509. #debug "$FUNCNAME: SESSION cache hit ($*)"
  3510. cat "$cache_file"
  3511. return 0
  3512. fi
  3513. charm=$(get_service_charm "$service") || return 1
  3514. metadata="$(charm.metadata "$charm" 2>/dev/null)" || return 1
  3515. has_build_dir=
  3516. [ -d "$(charm.get_dir "$charm")/build" ] && has_build_dir=true
  3517. mixin=$(_get_docker_compose_mixin_from_metadata_cached "$service" "$charm" "$metadata" "$has_build_dir") || return 1
  3518. echo "$mixin" | tee "$cache_file"
  3519. }
  3520. export -f get_docker_compose_mixin_from_metadata
  3521. _save() {
  3522. local name="$1"
  3523. cat - | tee -a "$docker_compose_dir/.data/$name"
  3524. }
  3525. export -f _save
  3526. get_default_project_name() {
  3527. if [ -n "$DEFAULT_PROJECT_NAME" ]; then
  3528. echo "$DEFAULT_PROJECT_NAME"
  3529. return 0
  3530. fi
  3531. local normalized_path compose_yml_location name
  3532. compose_yml_location="$(get_compose_yml_location)" || return 1
  3533. if [ -n "$compose_yml_location" ]; then
  3534. if normalized_path=$(readlink -f "$compose_yml_location"); then
  3535. name="${normalized_path%/*}" ## dirname
  3536. name="${name##*/}" ## basename
  3537. name="${name%%-deploy}" ## remove any '-deploy'
  3538. name="${name,,}" ## lowercase
  3539. e "$name"
  3540. return 0
  3541. fi
  3542. fi
  3543. echo "orphan"
  3544. return 0
  3545. }
  3546. export -f get_default_project_name
  3547. get_running_compose_containers() {
  3548. ## XXXvlab: docker bug: there will be a final newline anyway
  3549. docker ps --filter label="compose.service" --format='{{.ID}}'
  3550. }
  3551. export -f get_running_compose_containers
  3552. get_healthy_container_ip_for_service () {
  3553. local service="$1" port="$2" timeout=${3:-60}
  3554. local containers container container_network container_ip
  3555. containers="$(get_running_containers_for_service "$service")"
  3556. if [ -z "$containers" ]; then
  3557. err "No containers running for service $DARKYELLOW$service$NORMAL."
  3558. return 1
  3559. fi
  3560. ## XXXvlab: taking first container is probably not a good idea
  3561. container="$(echo "$containers" | head -n 1)"
  3562. ## XXXvlab: taking first ip is probably not a good idea
  3563. read-0 container_network container_ip < <(get_container_network_ip "$container")
  3564. if [ -z "$container_ip" ]; then
  3565. err "Can't get container's IP. You should check health of" \
  3566. "${DARKYELLOW}$service${NORMAL}'s container."
  3567. return 1
  3568. fi
  3569. wait_for_tcp_port "$container_network" "$container_ip:$port" "$timeout" || {
  3570. err "TCP port of ${DARKYELLOW}$service${NORMAL}'s container doesn't seem open"
  3571. echo " Please check that container is healthy. Here are last logs:" >&2
  3572. docker logs "$container" --tail=10 | prefix " ${GRAY}|${NORMAL} " >&2
  3573. return 1
  3574. }
  3575. info "Host/Port ${container_ip}:${port} checked ${GREEN}open${NORMAL}."
  3576. echo "$container_network:$container_ip"
  3577. }
  3578. export -f get_healthy_container_ip_for_service
  3579. switch_to_relation_service() {
  3580. local relation="$1"
  3581. ## XXXvlab: can't get real config here
  3582. if ! read-0 ts _ _ < <(get_service_relation "$SERVICE_NAME" "$relation"); then
  3583. err "Couldn't find relation ${DARKCYAN}$relation${NORMAL}."
  3584. return 1
  3585. fi
  3586. export SERVICE_NAME="$ts"
  3587. export SERVICE_DATASTORE="$DATASTORE/$SERVICE_NAME"
  3588. DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$SERVICE_NAME")
  3589. export DOCKER_BASE_IMAGE
  3590. target_charm=$(get_service_charm "$ts") || return 1
  3591. target_charm_path=$(charm.get_dir "$target_charm") || return 1
  3592. cd "$target_charm_path"
  3593. }
  3594. export -f switch_to_relation_service
  3595. get_volumes_for_container() {
  3596. local container="$1"
  3597. docker inspect \
  3598. --format '{{range $mount := .Mounts}}{{$mount.Source}}{{"\x00"}}{{$mount.Destination}}{{"\x00"}}{{end}}' \
  3599. "$container"
  3600. }
  3601. export -f get_volumes_for_container
  3602. is_volume_used() {
  3603. local volume="$1" container_id src dst
  3604. while read -r container_id; do
  3605. while read-0 src dst; do
  3606. [[ "$src/" == "$volume"/* ]] && return 0
  3607. done < <(get_volumes_for_container "$container_id")
  3608. done < <(get_running_compose_containers)
  3609. return 1
  3610. }
  3611. export -f is_volume_used
  3612. clean_unused_docker_compose() {
  3613. for f in /var/lib/compose/docker-compose/*; do
  3614. [ -e "$f" ] || continue
  3615. is_volume_used "$f" && continue
  3616. debug "Cleaning unused docker-compose ${f##*/}"
  3617. rm -rf "$f" || return 1
  3618. done
  3619. return 0
  3620. }
  3621. export -f clean_unused_docker_compose
  3622. docker_compose_store() {
  3623. local file="$1" sha
  3624. sha=$(hash_get 64 < "$file") || return 1
  3625. project=$(get_default_project_name) || return 1
  3626. dst="/var/lib/compose/docker-compose/$sha/$project"
  3627. mkdir -p "$dst" || return 1
  3628. cat <<EOF > "$dst/.env" || return 1
  3629. DOCKER_COMPOSE_PATH=$dst
  3630. COMPOSE_HTTP_TIMEOUT=7200
  3631. EOF
  3632. cp "$file" "$dst/docker-compose.yml" || return 1
  3633. mkdir -p "$dst/bin" || return 1
  3634. cat <<EOF > "$dst/bin/dc" || return 1
  3635. #!/bin/bash
  3636. $(declare -f read-0)
  3637. docker_run_opts=()
  3638. while read-0 opt; do
  3639. if [[ "\$opt" == "!env:"* ]]; then
  3640. opt="\${opt##!env:}"
  3641. var="\${opt%%=*}"
  3642. value="\${opt#*=}"
  3643. export "\$var"="\$value"
  3644. else
  3645. docker_run_opts+=("\$opt")
  3646. fi
  3647. done < <(cat "$COMPOSE_LAUNCHER_OPTS")
  3648. docker_run_opts+=(
  3649. "-w" "$dst"
  3650. "--entrypoint" "/usr/local/bin/docker-compose"
  3651. )
  3652. [ -t 1 ] && {
  3653. docker_run_opts+=("-ti")
  3654. }
  3655. exec docker run --rm "\${docker_run_opts[@]}" "\${COMPOSE_DOCKER_IMAGE:-docker.0k.io/compose}" "\$@"
  3656. EOF
  3657. chmod +x "$dst/bin/dc" || return 1
  3658. printf "%s" "$sha"
  3659. }
  3660. export -f docker_compose_store
  3661. launch_docker_compose() {
  3662. local charm docker_compose_tmpdir docker_compose_dir
  3663. docker_compose_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX)
  3664. #debug "Creating temporary docker-compose directory in '$docker_compose_tmpdir'."
  3665. trap_add EXIT "rm -rf \"$docker_compose_tmpdir\""
  3666. ## docker-compose will name network from the parent dir name
  3667. project=$(get_default_project_name)
  3668. mkdir -p "$docker_compose_tmpdir/$project"
  3669. docker_compose_dir="$docker_compose_tmpdir/$project"
  3670. if [ -z "$_CURRENT_DOCKER_COMPOSE" ]; then
  3671. err "${FUNCNAME[0]} is meant to be called after"\
  3672. "\$_CURRENT_DOCKER_COMPOSE has been calculated."
  3673. echo " Called by:" >&2
  3674. printf " - %s\n" "${FUNCNAME[@]:1}" >&2
  3675. return 1
  3676. fi
  3677. cat "$_CURRENT_DOCKER_COMPOSE" > "$docker_compose_dir/docker-compose.yml" || return 1
  3678. if [ -e "$state_tmpdir/to-merge-in-docker-compose.yml" ]; then
  3679. # debug "Merging some config data in docker-compose.yml:"
  3680. # debug "$(cat $state_tmpdir/to-merge-in-docker-compose.yml)"
  3681. _config_merge "$docker_compose_dir/docker-compose.yml" "$state_tmpdir/to-merge-in-docker-compose.yml" || return 1
  3682. fi
  3683. if [ -z "$(echo $(cat "$docker_compose_dir/docker-compose.yml"))" ]; then
  3684. die "Generated 'docker-compose.yml' is unexpectedly empty."
  3685. fi
  3686. ## XXXvlab: could be more specific and only link the needed charms
  3687. ## XXXvlab: why do we need these links ? If this is for the build command, then it is not useful anymore.
  3688. # for charm in $(shyaml keys services < "$docker_compose_dir/docker-compose.yml"); do
  3689. # if charm.exists "$charm"; then
  3690. # ln -sf "$(charm.get_dir "$charm")" "$docker_compose_dir/$charm" || exit 1
  3691. # fi
  3692. # done
  3693. mkdir "$docker_compose_dir/.data"
  3694. if [ -z "$COMPOSE_DISABLE_DOCKER_COMPOSE_STORE" ]; then
  3695. sha=$(docker_compose_store "$docker_compose_dir/docker-compose.yml") || return 1
  3696. fi
  3697. {
  3698. {
  3699. {
  3700. if [ -z "$COMPOSE_DISABLE_DOCKER_COMPOSE_STORE" ]; then
  3701. cd "/var/lib/compose/docker-compose/$sha/$project" || return 1
  3702. else
  3703. cd "$docker_compose_dir" || return 1
  3704. fi
  3705. if [ -f ".env" ]; then
  3706. debug "${WHITE}.env$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:"
  3707. debug "$(cat ".env" | prefix " $GRAY|$NORMAL ")"
  3708. fi
  3709. debug "${WHITE}docker-compose.yml$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:"
  3710. debug "$(cat "docker-compose.yml" | prefix " $GRAY|$NORMAL ")"
  3711. debug "${WHITE}Launching$NORMAL: docker-compose $@"
  3712. if [ "$DRY_COMPOSE_RUN" ]; then
  3713. echo docker-compose "$@"
  3714. else
  3715. docker-compose "$@"
  3716. fi
  3717. echo "$?" > "$docker_compose_dir/.data/errlvl"
  3718. } | _save stdout
  3719. } 3>&1 1>&2 2>&3 | _save stderr
  3720. } 3>&1 1>&2 2>&3
  3721. if tail -n 1 "$docker_compose_dir/.data/stderr" | grep -E "Service .+ failed to build: Error getting container [0-9a-f]+ from driver devicemapper: (open|Error mounting) /dev/mapper/docker-.*: no such file or directory$" >/dev/null 2>&1; then
  3722. err "Detected bug https://github.com/docker/docker/issues/4036 ... "
  3723. err "Please re-launch your command, or switch from 'devicemapper' driver to 'overlayfs' or 'aufs'."
  3724. fi
  3725. docker_compose_errlvl="$(cat "$docker_compose_dir/.data/errlvl" 2>/dev/null)"
  3726. if [ -z "$docker_compose_errlvl" ]; then
  3727. err "Something went wrong before you could gather docker-compose errorlevel."
  3728. return 1
  3729. fi
  3730. return "$docker_compose_errlvl"
  3731. }
  3732. export -f launch_docker_compose
  3733. get_compose_yml_location() {
  3734. if ! [ -z ${COMPOSE_YML_FILE+x} ]; then ## if set, even if empty
  3735. echo "$COMPOSE_YML_FILE"
  3736. return 0
  3737. fi
  3738. parent=$(while ! [ -f "./compose.yml" ]; do
  3739. [ "$PWD" == "/" ] && exit 0
  3740. cd ..
  3741. done; echo "$PWD"
  3742. )
  3743. if [ "$parent" ]; then
  3744. echo "$parent/compose.yml"
  3745. return 0
  3746. fi
  3747. ## XXXvlab: do we need this additional environment variable,
  3748. ## COMPOSE_YML_FILE is not sufficient ?
  3749. if [ "$DEFAULT_COMPOSE_FILE" ]; then
  3750. if ! [ -e "$DEFAULT_COMPOSE_FILE" ]; then
  3751. warn "No 'compose.yml' was found in current or parent dirs," \
  3752. "and \$DEFAULT_COMPOSE_FILE points to an unexistent file." \
  3753. "(${DEFAULT_COMPOSE_FILE})"
  3754. return 0
  3755. fi
  3756. echo "$DEFAULT_COMPOSE_FILE"
  3757. return 0
  3758. fi
  3759. warn "No 'compose.yml' was found in current or parent dirs, and no \$DEFAULT_COMPOSE_FILE was set."
  3760. return 0
  3761. }
  3762. export -f get_compose_yml_location
  3763. get_compose_yml_content() {
  3764. local cache_file="$state_tmpdir/$FUNCNAME.cache"
  3765. if [ -e "$cache_file" ]; then
  3766. cat "$cache_file" &&
  3767. touch "$cache_file" || return 1
  3768. return 0
  3769. fi
  3770. if [ -z "$COMPOSE_YML_FILE" ]; then
  3771. COMPOSE_YML_FILE=$(get_compose_yml_location) || return 1
  3772. fi
  3773. if [ -e "$COMPOSE_YML_FILE" ]; then
  3774. # debug "Found $WHITE$exname$NORMAL YAML file in '$COMPOSE_YML_FILE'."
  3775. COMPOSE_YML_CONTENT=$(cat "$COMPOSE_YML_FILE") || {
  3776. err "Could not read '$COMPOSE_YML_FILE'."
  3777. return 1
  3778. }
  3779. else
  3780. debug "No compose file found. Using an empty one."
  3781. COMPOSE_YML_CONTENT=""
  3782. fi
  3783. COMPOSE_YML_CONTENT=$(merge_yaml_str "$COMPOSE_YML_CONTENT" "${compose_contents[@]}") || return 1
  3784. output=$(echo "$COMPOSE_YML_CONTENT"| shyaml get-value 2>&1)
  3785. if [ "$?" != 0 ]; then
  3786. outputed_something=
  3787. while IFS='' read -r line1 && IFS='' read -r line2; do
  3788. [ "$outputed_something" ] || err "Invalid YAML in '$COMPOSE_YML_FILE':"
  3789. outputed_something=true
  3790. echo "$line1 $GRAY($line2)$NORMAL"
  3791. done < <(echo "$output" | grep ^yaml.scanner -A 100 |
  3792. sed -r 's/^ in "<stdin>", //g' | sed -r 's/^yaml.scanner.[a-zA-Z]+: //g') |
  3793. prefix " $GRAY|$NORMAL "
  3794. [ "$outputed_something" ] || {
  3795. err "Unexpected error while running 'shyaml get-value' on '$COMPOSE_YML_FILE':"
  3796. echo "$output" | prefix " $GRAY|$NORMAL "
  3797. }
  3798. return 1
  3799. fi
  3800. echo "$COMPOSE_YML_CONTENT" | tee "$cache_file" || return 1
  3801. }
  3802. export -f get_compose_yml_content
  3803. compose:yml:hash() {
  3804. local cache_file="$state_tmpdir/$FUNCNAME.cache"
  3805. if [ -e "$cache_file" ]; then
  3806. cat "$cache_file" &&
  3807. touch "$cache_file" || return 1
  3808. return 0
  3809. fi
  3810. compose_yml_content=$(get_compose_yml_content) || return 1
  3811. compose_yml_hash=$(echo "$compose_yml_content" | hash_get) || return 1
  3812. e "$compose_yml_hash" | tee "$cache_file" || return 1
  3813. }
  3814. export -f compose:yml:hash
  3815. compose:yml:root:services() {
  3816. local cache_file="$state_tmpdir/$FUNCNAME.cache" services compose_yml_content
  3817. if [ -e "$cache_file" ]; then
  3818. cat "$cache_file" &&
  3819. touch "$cache_file" || return 1
  3820. return 0
  3821. fi
  3822. compose_yml_content=$(get_compose_yml_content) || return 1
  3823. services=($(e "$compose_yml_content" | shyaml keys)) || return 1
  3824. e "${services[*]}" | tee "$cache_file" || return 1
  3825. }
  3826. export -f compose:yml:root:services
  3827. get_default_target_services() {
  3828. local services=("$@")
  3829. if [ -z "${services[*]}" ]; then
  3830. if [ "$DEFAULT_SERVICES" ]; then
  3831. debug "No service provided, using $WHITE\$DEFAULT_SERVICES$NORMAL variable." \
  3832. "Target services: $DARKYELLOW$DEFAULT_SERVICES$NORMAL"
  3833. services="$DEFAULT_SERVICES"
  3834. else
  3835. err "No service provided."
  3836. return 1
  3837. fi
  3838. fi
  3839. echo "${services[*]}"
  3840. }
  3841. export -f get_default_target_services
  3842. get_master_services() {
  3843. local loaded master_service service
  3844. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" )"
  3845. if [ -e "$cache_file" ]; then
  3846. cat "$cache_file" &&
  3847. touch "$cache_file" || return 1
  3848. return 0
  3849. fi
  3850. declare -A loaded
  3851. for service in "$@"; do
  3852. master_service=$(get_top_master_service_for_service "$service") || return 1
  3853. if [ "${loaded[$master_service]}" ]; then
  3854. continue
  3855. fi
  3856. echo "$master_service"
  3857. loaded["$master_service"]=1
  3858. done > "$cache_file".wip || return 1
  3859. mv "$cache_file"{.wip,} || return 1
  3860. cat "$cache_file" || return 1
  3861. }
  3862. export -f get_master_services
  3863. get_current_docker_container_id() {
  3864. local line
  3865. line=$(cat "/proc/self/cpuset") || return 1
  3866. [[ "$line" == *docker* ]] || return 1
  3867. echo "${line##*/}"
  3868. }
  3869. export -f get_current_docker_container_id
  3870. ## if we are in a docker compose, we might want to know what is the
  3871. ## real host path of some local paths.
  3872. get_host_path() {
  3873. local path="$1"
  3874. path=$(realpath "$path") || return 1
  3875. container_id=$(get_current_docker_container_id) || {
  3876. print "%s" "$path"
  3877. return 0
  3878. }
  3879. biggest_dst=
  3880. current_src=
  3881. while read-0 src dst; do
  3882. [[ "$path" == "$dst"* ]] || continue
  3883. if [[ "${#biggest_dst}" < "${#dst}" ]]; then
  3884. biggest_dst="$dst"
  3885. current_src="$src"
  3886. fi
  3887. done < <(get_volumes_for_container "$container_id")
  3888. if [ "$current_src" ]; then
  3889. printf "%s" "$current_src"
  3890. else
  3891. return 1
  3892. fi
  3893. }
  3894. export -f get_host_path
  3895. _setup_state_dir() {
  3896. export state_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX)
  3897. #debug "Creating temporary state directory in '$state_tmpdir'."
  3898. # trap_add EXIT "debug \"Removing temporary state directory in $state_tmpdir.\";\
  3899. # rm -rf \"$state_tmpdir\""
  3900. trap_add EXIT "rm -rf \"$state_tmpdir\""
  3901. }
  3902. get_docker_compose_help_msg() {
  3903. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  3904. docker_compose_help_msg
  3905. if [ -e "$cache_file" ]; then
  3906. cat "$cache_file" &&
  3907. touch "$cache_file" || return 1
  3908. return 0
  3909. fi
  3910. docker_compose_help_msg=$(docker-compose $action --help 2>/dev/null) || return 1
  3911. echo "$docker_compose_help_msg" |
  3912. tee "$cache_file" || return 1
  3913. }
  3914. get_docker_compose_usage() {
  3915. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  3916. docker_compose_help_msg
  3917. if [ -e "$cache_file" ]; then
  3918. cat "$cache_file" &&
  3919. touch "$cache_file" || return 1
  3920. return 0
  3921. fi
  3922. docker_compose_help_msg=$(get_docker_compose_help_msg $action) || return 1
  3923. echo "$docker_compose_help_msg" |
  3924. grep -m 1 "^Usage:" -A 10000 |
  3925. egrep -m 1 "^\$" -B 10000 |
  3926. nspc |
  3927. sed -r 's/^Usage: //g' |
  3928. tee "$cache_file" || return 1
  3929. }
  3930. get_docker_compose_opts_help() {
  3931. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  3932. docker_compose_help_msg
  3933. if [ -e "$cache_file" ]; then
  3934. cat "$cache_file" &&
  3935. touch "$cache_file" || return 1
  3936. return 0
  3937. fi
  3938. docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1
  3939. echo "$docker_compose_opts_help" |
  3940. grep '^Options:' -A 20000 |
  3941. tail -n +2 |
  3942. { cat ; echo; } |
  3943. egrep -m 1 "^\S*\$" -B 10000 |
  3944. head -n -1 |
  3945. tee "$cache_file" || return 1
  3946. }
  3947. get_docker_compose_commands_help() {
  3948. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  3949. docker_compose_help_msg
  3950. if [ -e "$cache_file" ]; then
  3951. cat "$cache_file" &&
  3952. touch "$cache_file" || return 1
  3953. return 0
  3954. fi
  3955. docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1
  3956. echo "$docker_compose_opts_help" |
  3957. grep '^Commands:' -A 20000 |
  3958. tail -n +2 |
  3959. { cat ; echo; } |
  3960. egrep -m 1 "^\S*\$" -B 10000 |
  3961. head -n -1 |
  3962. tee "$cache_file" || return 1
  3963. }
  3964. get_docker_compose_opts_list() {
  3965. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  3966. docker_compose_help_msg
  3967. if [ -e "$cache_file" ]; then
  3968. cat "$cache_file" &&
  3969. touch "$cache_file" || return 1
  3970. return 0
  3971. fi
  3972. docker_compose_opts_help=$(get_docker_compose_opts_help $action) || return 1
  3973. echo "$docker_compose_opts_help" |
  3974. egrep "^\s+-" |
  3975. sed -r 's/\s+((((-[a-zA-Z]|--[a-zA-Z0-9-]+)( [A-Z=]+|=[^ ]+)?)(, )?)+)\s+.*$/\1/g' |
  3976. tee "$cache_file" || return 1
  3977. }
  3978. options_parser() {
  3979. sed -r 's/^(\s+(((-[a-zA-Z]|--[a-zA-Z0-9-]+)([ =]([a-zA-Z_=\"\[]|\])+)?(, | )?)+)\s+)[^ ].*$/\x0\2\x0\0/g'
  3980. printf "\0"
  3981. }
  3982. remove_options_in_option_help_msg() {
  3983. {
  3984. read-0 null
  3985. if [ "$null" ]; then
  3986. err "options parsing error, should start with an option line."
  3987. return 1
  3988. fi
  3989. while read-0 opt full_txt;do
  3990. multi_opts="$(printf "%s " $opt | multi_opts_filter)"
  3991. single_opts="$(printf "%s " $opt | single_opts_filter)"
  3992. for to_remove in "$@"; do
  3993. str_matches "$to_remove" $multi_opts $single_opts && {
  3994. continue 2
  3995. }
  3996. done
  3997. echo -n "$full_txt"
  3998. done
  3999. } < <(options_parser)
  4000. }
  4001. _MULTIOPTION_REGEX='^((-[a-zA-Z]|--[a-zA-Z0-9-]+)(, )?)+'
  4002. _MULTIOPTION_REGEX_LINE_FILTER=$_MULTIOPTION_REGEX'(\s|=)'
  4003. multi_opts_filter() {
  4004. egrep "$_MULTIOPTION_REGEX_LINE_FILTER" |
  4005. sed -r "s/^($_MULTIOPTION_REGEX)(\s|=).*$/\1/g" |
  4006. tr ',' "\n" | nspc
  4007. }
  4008. single_opts_filter() {
  4009. egrep -v "$_MULTIOPTION_REGEX_LINE_FILTER" |
  4010. tr ',' "\n" | nspc
  4011. }
  4012. get_docker_compose_multi_opts_list() {
  4013. local action="$1" opts_list
  4014. opts_list=$(get_docker_compose_opts_list "$action") || return 1
  4015. echo "$opts_list" | multi_opts_filter
  4016. }
  4017. get_docker_compose_single_opts_list() {
  4018. local action="$1" opts_list
  4019. opts_list=$(get_docker_compose_opts_list "$action") || return 1
  4020. echo "$opts_list" | single_opts_filter
  4021. }
  4022. display_commands_help() {
  4023. local charm_actions
  4024. echo
  4025. echo "${WHITE}Commands${NORMAL} (added by compose):"
  4026. echo " ${DARKCYAN}cache${NORMAL} Control compose's cache"
  4027. echo " ${DARKCYAN}status${NORMAL} Display statuses of services"
  4028. echo
  4029. echo "${WHITE}Commands${NORMAL} (thanks to docker-compose):"
  4030. get_docker_compose_commands_help | sed -r "s/ ([a-z]+)(\s+)/ ${DARKCYAN}\1${NORMAL}\2/g"
  4031. charm_actions_help=$(get_docker_charm_action_help) || return 1
  4032. if [ "$charm_actions_help" ]; then
  4033. echo
  4034. echo "${WHITE}Charm actions${NORMAL}:"
  4035. printf "%s\n" "$charm_actions_help" | \
  4036. sed -r "s/^ ([a-z0-9-]+)(\s+)([a-z0-9-]+)(\s+)/ ${DARKCYAN}\1${NORMAL}\2${DARKYELLOW}\3${NORMAL}\4/g"
  4037. fi
  4038. }
  4039. get_docker_charm_action() {
  4040. local services service charm relation_name target_service relation_config \
  4041. target_charm services
  4042. ## XXXvlab: this is for get_service_relations
  4043. NO_CONSTRAINT_CHECK=True service:all:set_relations_hash || {
  4044. err-d "Failed to set relations hash."
  4045. return 1
  4046. }
  4047. services=($(get_all_services)) || return 1
  4048. for service in "${services[@]}"; do
  4049. printf "%s:\n" "$service"
  4050. charm=$(get_service_charm "$service") || return 1
  4051. for action in $(charm.ls_direct_actions "$charm"); do
  4052. printf " %s:\n" "$action"
  4053. printf " type: %s\n" "direct"
  4054. done
  4055. while read-0 relation_name target_service _relation_config _tech_dep; do
  4056. target_charm=$(get_service_charm "$target_service") || return 1
  4057. for action in $(charm.ls_relation_actions "$target_charm" "$relation_name"); do
  4058. printf " %s:\n" "$action"
  4059. printf " type: %s\n" "indirect"
  4060. printf " inherited: %s\n" "$target_charm"
  4061. done
  4062. done < <(get_service_relations "$service")
  4063. done
  4064. }
  4065. export -f get_docker_charm_action
  4066. get_docker_charm_action_help() {
  4067. local services service charm relation_name target_service relation_config \
  4068. target_charm
  4069. ## XXXvlab: this is for get_service_relations
  4070. NO_CONSTRAINT_CHECK=True service:all:set_relations_hash || {
  4071. err-d "Failed to set relations hash."
  4072. return 1
  4073. }
  4074. services=($(get_all_services)) || return 1
  4075. for service in "${services[@]}"; do
  4076. out=$(
  4077. charm=$(get_service_charm "$service") || return 1
  4078. for action in $(charm.ls_direct_actions "$charm"); do
  4079. printf " %-28s %s\n" "$action $service" "Direct action from ${DARKPINK}$charm${NORMAL}"
  4080. done
  4081. while read-0 relation_name target_service _relation_config _tech_dep; do
  4082. target_charm=$(get_service_charm "$target_service") || return 1
  4083. for action in $(charm.ls_relation_actions "$target_charm" "$relation_name"); do
  4084. printf " %-28s %s\n" "$action $service" "Indirect action from ${DARKPINK}$target_charm${NORMAL}"
  4085. done
  4086. done < <(get_service_relations "$service")
  4087. )
  4088. if [ "$out" ]; then
  4089. echo " for ${DARKYELLOW}$service${NORMAL}:"
  4090. printf "%s\n" "$out"
  4091. fi
  4092. done
  4093. }
  4094. display_help() {
  4095. print_help
  4096. echo "${WHITE}Usage${NORMAL}:"
  4097. echo " $usage"
  4098. echo " $usage cache {clean|clear}"
  4099. echo "${WHITE}Options${NORMAL}:"
  4100. echo " -h, --help Print this message and quit"
  4101. echo " (ignoring any other options)"
  4102. echo " -V, --version Print current version and quit"
  4103. echo " (ignoring any other options)"
  4104. echo " --dirs Display data dirs and quit"
  4105. echo " (ignoring any other options)"
  4106. echo " --get-project-name Display project name and quit"
  4107. echo " (ignoring any other options)"
  4108. echo " --get-available-actions Display all available actions and quit"
  4109. echo " (ignoring any other options)"
  4110. echo " -v, --verbose Be more verbose"
  4111. echo " -q, --quiet Be quiet"
  4112. echo " -d, --debug Print full debugging information (sets also verbose)"
  4113. echo " --dry-compose-run If docker-compose will be run, only print out what"
  4114. echo " command line will be used."
  4115. echo " --no-relations Do not run any relation script"
  4116. echo " --no-hooks Do not run any hook script"
  4117. echo " --no-init Do not run any init script"
  4118. echo " --no-post-deploy Do not run any post-deploy script"
  4119. echo " --no-pre-deploy Do not run any pre-deploy script"
  4120. echo " --without-relation RELATION "
  4121. echo " Do not run given relation"
  4122. echo " -R, --rebuild-relations-to-service SERVICE"
  4123. echo " Will rebuild all relations to given service"
  4124. echo " --add-compose-content, -Y YAML"
  4125. echo " Will merge some direct YAML with the current compose"
  4126. echo " -c, --color Force color mode (default is to detect if in tty mode)"
  4127. echo " --push-builds Will push cached docker images to docker cache registry"
  4128. get_docker_compose_opts_help | remove_options_in_option_help_msg --version --help --verbose |
  4129. filter_docker_compose_help_message
  4130. display_commands_help
  4131. }
  4132. _graph_service() {
  4133. local service="$1" base="$1"
  4134. charm=$(get_service_charm "$service") || return 1
  4135. metadata=$(charm.metadata "$charm") || return 1
  4136. subordinate=$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)
  4137. if [[ "$subordinate" =~ ^True|true$ ]]; then
  4138. requires="$(echo "$metadata" | shyaml get-value "requires" 2>/dev/null)"
  4139. master_charm=
  4140. while read-0 relation_name relation; do
  4141. [ "$(echo "$relation" | shyaml get-value "scope" 2>/dev/null)" == "container" ] || continue
  4142. interface="$(echo "$relation" | shyaml get-value "interface" 2>/dev/null)"
  4143. if [ -z "$interface" ]; then
  4144. err "No ${WHITE}$interface${NORMAL} set for relation $relation_name."
  4145. return 1
  4146. fi
  4147. ## Action provided by relation ?
  4148. target_service=
  4149. while read-0 relation_name candidate_target_service _relation_config _tech_dep; do
  4150. [ "$interface" == "$relation_name" ] && {
  4151. target_service="$candidate_target_service"
  4152. break
  4153. }
  4154. done < <(get_service_relations "$service")
  4155. if [ -z "$target_service" ]; then
  4156. err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \
  4157. "${DARKYELLOW}$service$NORMAL compose definition."
  4158. return 1
  4159. fi
  4160. master_service="$target_service"
  4161. master_charm=$(get_service_charm "$target_service") || return 1
  4162. break
  4163. done < <(echo "$requires" | shyaml key-values-0 2>/dev/null)
  4164. fi
  4165. _graph_node_service "$service" "$base" "$charm"
  4166. _graph_edge_service "$service" "$subordinate" "$master_service"
  4167. }
  4168. _graph_node_service() {
  4169. local service="$1" base="$2" charm="$3"
  4170. cat <<EOF
  4171. "$(_graph_node_service_label ${service})" [
  4172. style = "filled, $([[ "$subordinate" =~ ^True|true$ ]] && echo "dashed" || echo "bold")"
  4173. penwidth = $([[ "$subordinate" =~ ^True|true$ ]] && echo "3" || echo "5")
  4174. color = $([ "$base" ] && echo "blue" || echo "black")
  4175. fillcolor = "white"
  4176. fontname = "Courier New"
  4177. shape = "Mrecord"
  4178. label =<$(_graph_node_service_content "$service")>
  4179. ];
  4180. EOF
  4181. }
  4182. _graph_edge_service() {
  4183. local service="$1" subordinate="$2" master_service="$3"
  4184. while read-0 relation_name target_service relation_config tech_dep; do
  4185. cat <<EOF
  4186. "$(_graph_node_service_label ${service})" -> "$(_graph_node_service_label ${target_service})" [
  4187. penwidth = $([ "$master_service" == "$target_service" ] && echo 3 || echo 2)
  4188. fontsize = 16
  4189. fontcolor = "black"
  4190. style = $([ "$master_service" == "$target_service" ] && echo dashed || echo "\"\"")
  4191. weight = $([ "$master_service" == "$target_service" ] && echo 2.0 || echo 1.0)
  4192. dir = $([ "$master_service" == "$target_service" ] && echo none || echo both)
  4193. arrowtail = odot
  4194. # arrowhead = dotlicurve
  4195. taillabel = "$relation_name" ];
  4196. EOF
  4197. done < <(get_service_relations "$service") || return 1
  4198. }
  4199. _graph_node_service_label() {
  4200. local service="$1"
  4201. echo "service_$service"
  4202. }
  4203. _graph_node_service_content() {
  4204. local service="$1"
  4205. charm=$(get_service_charm "$service") || return 1
  4206. cat <<EOF
  4207. <table border="0" cellborder="0" cellpadding="3" bgcolor="white">
  4208. <tr>
  4209. <td bgcolor="black" align="center" colspan="2">
  4210. <font color="white">$service</font>
  4211. </td>
  4212. </tr>
  4213. $(if [ "$charm" != "$service" ]; then
  4214. cat <<EOF2
  4215. <tr>
  4216. <td align="left" port="r0">charm: $charm</td>
  4217. </tr>
  4218. EOF2
  4219. fi)
  4220. </table>
  4221. EOF
  4222. }
  4223. cla_contains () {
  4224. local e
  4225. for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done
  4226. return 1
  4227. }
  4228. filter_docker_compose_help_message() {
  4229. cat - |
  4230. sed -r "s/docker-compose run/${DARKWHITE}compose${NORMAL} ${DARKCYAN}$action${NORMAL}/g;
  4231. s/docker-compose.yml/compose.yml/g;
  4232. s/SERVICES?/${DARKYELLOW}\0${NORMAL}/g;
  4233. s/^(\s+)\\$/\1${WHITE}\$${NORMAL}/g;
  4234. s/^(\s+)run/\1${DARKCYAN}$action${NORMAL}/g;
  4235. s/docker-compose/${DARKWHITE}compose${NORMAL}/g"
  4236. }
  4237. graph() {
  4238. local services=("$@")
  4239. declare -A entries
  4240. cat <<EOF
  4241. digraph g {
  4242. graph [
  4243. fontsize=30
  4244. labelloc="t"
  4245. label=""
  4246. splines=true
  4247. overlap=false
  4248. #rankdir = "LR"
  4249. ];
  4250. ratio = auto;
  4251. EOF
  4252. for target_service in "$@"; do
  4253. services=$(get_ordered_service_dependencies "$target_service") || return 1
  4254. for service in $services; do
  4255. [ "${entries[$service]}" ] && continue || entries[$service]=1
  4256. if cla_contains "$service" "${services[@]}"; then
  4257. base=true
  4258. else
  4259. base=
  4260. fi
  4261. _graph_service "$service" "$base"
  4262. done
  4263. done
  4264. echo "}"
  4265. }
  4266. cached_wget() {
  4267. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(p0 "$@" | md5_compat)" \
  4268. url="$1"
  4269. if [ -e "$cache_file" ]; then
  4270. cat "$cache_file"
  4271. touch "$cache_file"
  4272. return 0
  4273. fi
  4274. wget -O- "${url}" |
  4275. tee "$cache_file"
  4276. if [ "${PIPESTATUS[0]}" != 0 ]; then
  4277. rm "$cache_file"
  4278. die "Unable to fetch '$url'."
  4279. return 1
  4280. fi
  4281. }
  4282. export -f cached_wget
  4283. [ "$SOURCED" ] && return 0
  4284. trap_add "EXIT" clean_cache
  4285. export COMPOSE_DOCKER_REGISTRY="${COMPOSE_DOCKER_REGISTRY:-docker.0k.io}"
  4286. if [ -z "$DISABLE_SYSTEM_CONFIG_FILE" ]; then
  4287. if [ -r /etc/default/charm ]; then
  4288. . "/etc/default/charm"
  4289. fi
  4290. if [ -r "/etc/default/$exname" ]; then
  4291. . "/etc/default/$exname"
  4292. fi
  4293. ## XXXvlab: should provide YML config opportunities in possible parent dirs ?
  4294. ## userdir ? and global /etc/compose.yml ?
  4295. for cfgfile in /etc/compose.conf /etc/compose.local.conf \
  4296. /etc/default/compose /etc/compose/local.conf; do
  4297. [ -e "$cfgfile" ] || continue
  4298. . "$cfgfile" || die "Loading config file '$cfgfile' failed."
  4299. done
  4300. fi
  4301. _setup_state_dir
  4302. mkdir -p "$CACHEDIR" || exit 1
  4303. log () { cat; }
  4304. export -f log
  4305. ##
  4306. ## Argument parsing
  4307. ##
  4308. wrap_opts=()
  4309. services=()
  4310. remainder_args=()
  4311. compose_opts=()
  4312. compose_contents=()
  4313. action_opts=()
  4314. services_args=()
  4315. pos_arg_ct=0
  4316. no_hooks=
  4317. no_init=
  4318. action=
  4319. stage="main" ## switches from 'main', to 'action', 'remainder'
  4320. is_docker_compose_action=
  4321. is_docker_compose_action_multi_service=
  4322. rebuild_relations_to_service=()
  4323. color=
  4324. declare -A without_relations
  4325. DC_MATCH_MULTI=$(get_docker_compose_multi_opts_list) &&
  4326. DC_MATCH_SINGLE=$(get_docker_compose_single_opts_list) || exit 1
  4327. while read-0 arg; do
  4328. case "$stage" in
  4329. "main")
  4330. case "$arg" in
  4331. --help|-h)
  4332. no_init=true ; no_hooks=true ; no_relations=true
  4333. display_help
  4334. exit 0
  4335. ;;
  4336. --verbose|-v)
  4337. export VERBOSE=true
  4338. compose_opts+=("--verbose")
  4339. ;;
  4340. --quiet|-q)
  4341. export QUIET=true
  4342. export wrap_opts+=("-q")
  4343. log () { cat >&2; }
  4344. export -f log
  4345. ;;
  4346. --version|-V)
  4347. print_version
  4348. docker-compose --version
  4349. docker --version
  4350. exit 0
  4351. ;;
  4352. -f|--file)
  4353. read-0 value
  4354. [ -e "$value" ] || die "File $value doesn't exists"
  4355. export COMPOSE_YML_FILE="$value"
  4356. shift
  4357. ;;
  4358. -p|--project-name)
  4359. read-0 value
  4360. export DEFAULT_PROJECT_NAME="$value"
  4361. compose_opts+=("--project-name $value")
  4362. shift
  4363. ;;
  4364. --color|-c)
  4365. if [ "$color" == "0" ]; then
  4366. err "Conflicting option --color with previous --no-ansi."
  4367. exit 1
  4368. fi
  4369. color=1
  4370. ansi_color yes
  4371. ;;
  4372. --no-ansi)
  4373. if [ "$color" == "1" ]; then
  4374. err "Conflicting option --no-ansi with previous --color."
  4375. exit 1
  4376. fi
  4377. color=0
  4378. ansi_color no
  4379. compose_opts+=("--no-ansi")
  4380. ;;
  4381. --no-relations)
  4382. export no_relations=true
  4383. ;;
  4384. --without-relation)
  4385. read-0 value
  4386. without_relations["$value"]=1
  4387. shift
  4388. ;;
  4389. --no-hooks)
  4390. export no_hooks=true
  4391. ;;
  4392. --no-init)
  4393. export no_init=true
  4394. ;;
  4395. --no-post-deploy)
  4396. export no_post_deploy=true
  4397. ;;
  4398. --no-pre-deploy)
  4399. export no_pre_deploy=true
  4400. ;;
  4401. --rebuild-relations-to-service|-R)
  4402. read-0 value
  4403. rebuild_relations_to_service+=("$value")
  4404. shift
  4405. ;;
  4406. --push-builds)
  4407. export COMPOSE_PUSH_TO_REGISTRY=1
  4408. ;;
  4409. --debug|-d)
  4410. export DEBUG=true
  4411. export VERBOSE=true
  4412. #compose_opts+=("--verbose" "--log-level" "DEBUG")
  4413. ;;
  4414. --add-compose-content|-Y)
  4415. read-0 value
  4416. compose_contents+=("$value")
  4417. shift
  4418. ;;
  4419. --dirs)
  4420. echo "CACHEDIR: $CACHEDIR"
  4421. echo "VARDIR: $VARDIR"
  4422. exit 0
  4423. ;;
  4424. --get-project-name)
  4425. project=$(get_default_project_name) || exit 1
  4426. echo "$project"
  4427. exit 0
  4428. ;;
  4429. --get-available-actions)
  4430. get_docker_charm_action
  4431. exit $?
  4432. ;;
  4433. --dry-compose-run)
  4434. export DRY_COMPOSE_RUN=true
  4435. ;;
  4436. --*|-*)
  4437. if str_pattern_matches "$arg" $DC_MATCH_MULTI; then
  4438. read-0 value
  4439. compose_opts+=("$arg" "$value")
  4440. shift;
  4441. elif str_pattern_matches "$arg" $DC_MATCH_SINGLE; then
  4442. compose_opts+=("$arg")
  4443. else
  4444. err "Unknown option '$arg'. Please check help:"
  4445. display_help >&2
  4446. exit 1
  4447. fi
  4448. ;;
  4449. *)
  4450. action="$arg"
  4451. stage="action"
  4452. if DC_USAGE=$(get_docker_compose_usage "$action"); then
  4453. is_docker_compose_action=true
  4454. DC_MATCH_MULTI=$(get_docker_compose_multi_opts_list "$action") &&
  4455. DC_MATCH_SINGLE="$(get_docker_compose_single_opts_list "$action")"
  4456. if [ "$DC_MATCH_MULTI" ]; then
  4457. DC_MATCH_SINGLE="$DC_MATCH_SINGLE $(echo "$DC_MATCH_MULTI" | sed -r 's/( |$)/=\* /g')"
  4458. fi
  4459. pos_args=($(echo "$DC_USAGE" | sed -r 's/\[-[^]]+\] ?//g;s/\[options\] ?//g'))
  4460. pos_args=("${pos_args[@]:1}")
  4461. if [[ "${pos_args[0]}" == "[SERVICE...]" ]]; then
  4462. is_docker_compose_action_multi_service=1
  4463. elif [[ "${pos_args[0]}" == "SERVICE" ]]; then
  4464. is_docker_compose_action_multi_service=0
  4465. fi
  4466. # echo "USAGE: $DC_USAGE"
  4467. # echo "pos_args: ${pos_args[@]}"
  4468. # echo "MULTI: $DC_MATCH_MULTI"
  4469. # echo "SINGLE: $DC_MATCH_SINGLE"
  4470. # exit 1
  4471. else
  4472. stage="remainder"
  4473. fi
  4474. ;;
  4475. esac
  4476. ;;
  4477. "action") ## Only for docker-compose actions
  4478. case "$arg" in
  4479. --help|-h)
  4480. no_init=true ; no_hooks=true ; no_relations=true
  4481. action_opts+=("$arg")
  4482. ;;
  4483. --*|-*)
  4484. if [ "$is_docker_compose_action" ]; then
  4485. if str_pattern_matches "$arg" $DC_MATCH_MULTI; then
  4486. read-0 value
  4487. action_opts+=("$arg" "$value")
  4488. shift
  4489. elif str_pattern_matches "$arg" $DC_MATCH_SINGLE; then
  4490. action_opts+=("$arg")
  4491. else
  4492. err "Unknown option '$arg'. Please check '${DARKCYAN}$action${NORMAL}' help:"
  4493. docker-compose "$action" --help |
  4494. filter_docker_compose_help_message >&2
  4495. exit 1
  4496. fi
  4497. fi
  4498. ;;
  4499. *)
  4500. # echo "LOOP $1 : pos_arg: $pos_arg_ct // ${pos_args[$pos_arg_ct]}"
  4501. if [[ "${pos_args[$pos_arg_ct]}" == "[SERVICE...]" ]]; then
  4502. services_args+=("$arg")
  4503. elif [[ "${pos_args[$pos_arg_ct]}" == "SERVICE" ]]; then
  4504. services_args=("$arg") || exit 1
  4505. stage="remainder"
  4506. else
  4507. action_posargs+=("$arg")
  4508. ((pos_arg_ct++))
  4509. fi
  4510. ;;
  4511. esac
  4512. ;;
  4513. "remainder")
  4514. remainder_args+=("$arg")
  4515. while read-0 arg; do
  4516. remainder_args+=("$arg")
  4517. done
  4518. break 3
  4519. ;;
  4520. esac
  4521. shift
  4522. done < <(cla.normalize "$@")
  4523. ## These actions are additions to docker-compose actions and charm
  4524. ## actions
  4525. more_actions=(status)
  4526. if [[ "$action" == *" "* ]]; then
  4527. err "Invalid action name containing spaces: ${DARKCYAN}$action${NORMAL}"
  4528. exit 1
  4529. fi
  4530. is_more_action=
  4531. [[ " ${more_actions[*]} " == *" $action "* ]] && is_more_action=true
  4532. [ -n "$CACHEDIR" ] || die "No cache directory defined."
  4533. [ -d "$CACHEDIR" ] || die "Cache directory '$CACHEDIR' doesn't exists."
  4534. case "$action" in
  4535. cache)
  4536. case "${remainder_args[0]}" in
  4537. clean)
  4538. clean_cache
  4539. exit 0
  4540. ;;
  4541. clear)
  4542. Wrap "${wrap_opts[@]}" -v -d "clear cache directory" -- rm -rf "$CACHEDIR/"*
  4543. ## clear all docker caches
  4544. ## image name are like '[$COMPOSE_DOCKER_REGISTRY]cache/charm/CHARM_NAME:HASH'
  4545. Wrap "${wrap_opts[@]}" -v -d "clear docker cache" <<EOF
  4546. docker images --format "{{.Repository}}:{{.Tag}}" |
  4547. egrep "^($COMPOSE_DOCKER_REGISTRY/)?cache/charm/[a-zA-Z0-9._-]+:[0-9a-f]{32,32}$" |
  4548. while read -r image; do
  4549. docker rmi "\$image" || true
  4550. done
  4551. EOF
  4552. exit 0
  4553. ;;
  4554. *)
  4555. err "Unknown cache command: ${DARKCYAN}${remainder_args[0]}${NORMAL}"
  4556. exit 1
  4557. ;;
  4558. esac
  4559. ;;
  4560. status)
  4561. state_inner_cols=(name charm type state root)
  4562. state_all_services=
  4563. state_services=()
  4564. state_columns=()
  4565. state_columns_default=(name charm type state version)
  4566. state_filters=()
  4567. state_columns_default_msg=""
  4568. for col in "${state_columns_default[@]}"; do
  4569. if [ -n "$state_columns_default_msg" ]; then
  4570. state_columns_default_msg+=","
  4571. fi
  4572. state_columns_default_msg+="$col"
  4573. done
  4574. help="\
  4575. Display status information on services.
  4576. If no services are provided, all services in the root compose file
  4577. will be displayed. Use the --all option to display status of all
  4578. services (including dependencies).
  4579. $exname offers a few possible columns that can be complete on a charm
  4580. level by implementing an \`actions/get-COLNAME\` script.
  4581. These are the compose's columns: ${state_inner_cols[@]}.
  4582. Usage: status [options] [SERVICE...]
  4583. Options:
  4584. -h, --help Print this message and quit
  4585. -a, --all Display status of all services
  4586. (removes all filter, and will add a
  4587. 'root' first column by default)
  4588. -c, --column Column to display, can provide several
  4589. separated by commas, or option can be repeated.
  4590. (default: ${state_columns_default_msg})
  4591. -f, --filter Filter services by a key=value pair,
  4592. separated by commas or can be repeated.
  4593. (default: --filter root=yes)
  4594. "
  4595. while read-0 arg; do
  4596. case "$arg" in
  4597. --help|-h)
  4598. echo "$help"
  4599. exit 0
  4600. ;;
  4601. --all|-a)
  4602. if [ "${#state_services[@]}" -gt 0 ]; then
  4603. err "Cannot use --all and provide services at the same time."
  4604. exit 1
  4605. fi
  4606. if [[ "${#state_filters[@]}" -gt 0 ]]; then
  4607. err "Cannot use --all and provide filters at the same time."
  4608. exit 1
  4609. fi
  4610. state_all_services=1
  4611. ;;
  4612. --column|-c)
  4613. read-0 value
  4614. if [[ "$value" == *,* ]]; then
  4615. state_columns+=(${value//,/ })
  4616. else
  4617. state_columns+=("$value")
  4618. fi
  4619. ;;
  4620. --filter|-f)
  4621. if [ "${#state_services[@]}" -gt 0 ]; then
  4622. err "Cannot use --filter and provide services at the same time."
  4623. exit 1
  4624. fi
  4625. if [ -n "$state_all_services" ]; then
  4626. err "Cannot use --all and provide filters at the same time."
  4627. exit 1
  4628. fi
  4629. read-0 value
  4630. if [[ "$value" == *,* ]]; then
  4631. state_filters+=(${value//,/ })
  4632. else
  4633. state_filters+=("$value")
  4634. fi
  4635. ;;
  4636. --*|-*)
  4637. err "Unknown option '$arg'. Please check help:"
  4638. echo "$help" >&2
  4639. ;;
  4640. *)
  4641. if [ -n "$state_all_services" ]; then
  4642. err "Cannot use --all and provide services at the same time."
  4643. exit 1
  4644. fi
  4645. if [[ "${#state_filters[@]}" -gt 0 ]]; then
  4646. err "Cannot use --filter and provide filters at the same time."
  4647. exit 1
  4648. fi
  4649. state_services+=("$arg")
  4650. ;;
  4651. esac
  4652. done < <(cla.normalize "${remainder_args[@]}")
  4653. if [ "${#state_columns[@]}" == 0 ]; then
  4654. state_columns=("${state_columns_default[@]}")
  4655. fi
  4656. ;;
  4657. esac
  4658. export compose_contents
  4659. [ "${services_args[*]}" ] && debug " ${DARKWHITE}Services:$NORMAL ${DARKYELLOW}${services_args[*]}$NORMAL"
  4660. [ "${compose_opts[*]}" ] && debug " ${DARKWHITE}Main docker-compose opts:$NORMAL ${compose_opts[*]}"
  4661. [ "${action_posargs[*]}" ] && debug " ${DARKWHITE}Main docker-compose pos args:$NORMAL ${action_posargs[*]}"
  4662. [ "${action_opts[*]}" ] && debug " ${DARKWHITE}Action $DARKCYAN$action$NORMAL with opts:$NORMAL ${action_opts[*]}"
  4663. [ "${remainder_args[*]}" ] && debug " ${DARKWHITE}Remainder args:$NORMAL ${remainder_args[*]}"
  4664. aexport remainder_args
  4665. ##
  4666. ## Actual code
  4667. ##
  4668. COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1
  4669. COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1
  4670. COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || exit 1
  4671. CHARM_STORE_HASH=$(charm.store_metadata_hash) || exit 1
  4672. COMBINED_HASH=$(H "$COMPOSE_YML_CONTENT_HASH" "$CHARM_STORE_HASH") || exit 1
  4673. export COMPOSE_YML_FILE COMPOSE_YML_CONTENT COMPOSE_YML_CONTENT_HASH CHARM_STORE_HASH COMBINED_HASH
  4674. charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct."
  4675. ##
  4676. ## Get services in command line.
  4677. ##
  4678. if [ -z "$is_docker_compose_action" ] && [ -z "$is_more_action" ] && [ -n "$action" ]; then
  4679. action_service=${remainder_args[0]}
  4680. if [ -z "$action_service" ]; then
  4681. err "No such command or action: ${DARKCYAN}$action${NORMAL}"
  4682. display_commands_help
  4683. exit 1
  4684. fi
  4685. services_args=($(compose:yml:root:services)) || return 1
  4686. ## Required by has_service_action
  4687. service:all:set_relations_hash
  4688. remainder_args=("${remainder_args[@]:1}")
  4689. if has_service_action "$action_service" "$action" >/dev/null; then
  4690. is_service_action=true
  4691. services_args=("$action_service")
  4692. {
  4693. read-0 action_type
  4694. case "$action_type" in
  4695. "relation")
  4696. read-0 _ target_service _target_charm relation_name _ action_script_path
  4697. debug "Found action $DARKYELLOW${action_service}$NORMAL/$DARKBLUE$relation_name$NORMAL/$DARKCYAN$action$NORMAL (in $DARKYELLOW$target_service$NORMAL)"
  4698. services_args+=("$target_service")
  4699. ;;
  4700. "direct")
  4701. read-0 _ action_script_path
  4702. debug "Found action $DARKYELLOW${action_service}$NORMAL.$DARKCYAN$action$NORMAL"
  4703. ;;
  4704. esac
  4705. } < <(has_service_action "$action_service" "$action")
  4706. get_all_relations "${services_args[@]}" >/dev/null || {
  4707. echo " Hint: if this is unexpected, you can try to delete caches, and re-run the command." >&2
  4708. exit 1
  4709. }
  4710. ## Divert logging to stdout to stderr
  4711. log () { cat >&2; }
  4712. export -f log
  4713. else
  4714. die "Unknown action '${DARKCYAN}$action$NORMAL': It doesn't match any docker-compose commands nor inner charm actions."
  4715. fi
  4716. else
  4717. case "$action" in
  4718. ps|up)
  4719. if [ "${#services_args[@]}" == 0 ]; then
  4720. services_args=($(compose:yml:root:services)) || return 1
  4721. fi
  4722. ;;
  4723. status)
  4724. services_args=("${state_services[@]}")
  4725. if [ "${#services_args[@]}" == 0 ] && [ -z "$state_all_services" ]; then
  4726. services_args=($(compose:yml:root:services)) || return 1
  4727. fi
  4728. ;;
  4729. config)
  4730. services_args=("${action_posargs[@]}")
  4731. ;;
  4732. esac
  4733. fi
  4734. export COMPOSE_ACTION="$action"
  4735. NO_CONSTRAINT_CHECK=True
  4736. case "$action" in
  4737. up|status|run)
  4738. NO_CONSTRAINT_CHECK=
  4739. if [ -n "$DEBUG" ]; then
  4740. Elt "solve all relations"
  4741. start=$(time_now)
  4742. fi
  4743. service:all:set_relations_hash || exit 1
  4744. if [ -n "$DEBUG" ]; then
  4745. elapsed="$(time_elapsed $start "$(time_now)")" || exit 1
  4746. print_info "$(printf "%.3fs" "$elapsed")"
  4747. Feedback
  4748. fi
  4749. all_services=($(get_all_services)) || exit 1
  4750. ## check that services_args is a subset of all_services
  4751. for service in "${services_args[@]}"; do
  4752. [[ " ${all_services[*]} " == *" $service "* ]] || {
  4753. err "Service ${DARKYELLOW}$service${NORMAL} is not defined in the current compose file."
  4754. echo " Neither is is a dependency of a service in the compose file." >&2
  4755. echo " These are the services directly or indirectly available from current compose file:" >&2
  4756. for service in "${all_services[@]}"; do
  4757. echo " - ${DARKYELLOW}$service${NORMAL}" >&2
  4758. done
  4759. exit 1
  4760. }
  4761. done
  4762. ;;
  4763. esac
  4764. case "$action" in
  4765. up)
  4766. PROJECT_NAME=$(get_default_project_name) || exit 1
  4767. ## Remove all intents (*ing states)
  4768. rm -f "$SERVICE_STATE_PATH/$PROJECT_NAME"/*/*ing || true
  4769. ## Notify that we have the intent to bring up all these
  4770. ## This will be use in inner or concurrent 'run' to include the
  4771. ## services that are supposed to be up.
  4772. mkdir -p "$SERVICE_STATE_PATH/$PROJECT_NAME" || exit 1
  4773. services_args_deps=($(get_ordered_service_dependencies "${services_args[@]}")) || exit 1
  4774. for service in "${services_args_deps[@]}"; do
  4775. mkdir -p "$SERVICE_STATE_PATH/$PROJECT_NAME"/"$service" || exit 1
  4776. [ -e "$SERVICE_STATE_PATH/$PROJECT_NAME"/"$service"/up ] || {
  4777. touch "$SERVICE_STATE_PATH/$PROJECT_NAME"/"$service"/deploying || exit 1
  4778. }
  4779. done
  4780. ## remove services not included in compose.yml anymore
  4781. all_services_deps=($(get_ordered_service_dependencies "${all_services[@]}")) || exit 1
  4782. for service in "$SERVICE_STATE_PATH/$PROJECT_NAME"/*/up; do
  4783. [ -e "$service" ] || continue
  4784. state=${service##*/}
  4785. service=${service%/$state}
  4786. service=${service##*/}
  4787. if [[ " ${all_services_deps[*]} " != *" ${service} "* ]]; then
  4788. touch "$SERVICE_STATE_PATH/$PROJECT_NAME"/"${service}"/orphaning || exit 1
  4789. fi
  4790. done
  4791. ;;
  4792. run)
  4793. PROJECT_NAME=$(get_default_project_name) || return 1
  4794. if [ -d "$SERVICE_STATE_PATH/$PROJECT_NAME" ]; then
  4795. ## Notify that we have the intent to bring up all these
  4796. ## This will be use in inner or concurrent 'run' to include the
  4797. ## services that are supposed to be up.
  4798. for service in "$SERVICE_STATE_PATH/$PROJECT_NAME"/*/{up,deploying}; do
  4799. [ -e "$service" ] || continue
  4800. state=${service##*/}
  4801. service=${service%/$state}
  4802. service=${service##*/}
  4803. ## don't add if orphaning
  4804. [ -e "$SERVICE_STATE_PATH/$PROJECT_NAME"/"${service}"/orphaning ] && continue
  4805. done
  4806. fi
  4807. ;;
  4808. status)
  4809. if [ -n "${state_all_services}" ] || [[ "${#state_filters[@]}" -gt 0 ]]; then
  4810. services_args=("${all_services[@]}")
  4811. fi
  4812. ;;
  4813. esac
  4814. if [ -n "$DEBUG" ]; then
  4815. Elt "get relation subset"
  4816. start=$(time_now)
  4817. fi
  4818. get_subset_relations "${services_args[@]}" >/dev/null || exit 1
  4819. if [ -n "$DEBUG" ]; then
  4820. elapsed="$(time_elapsed $start "$(time_now)")" || exit 1
  4821. print_info "$(printf "%.3fs" "$elapsed")"
  4822. Feedback
  4823. fi
  4824. if [ -n "$is_docker_compose_action" ] && [ "${#services_args[@]}" -gt 0 ]; then
  4825. services=($(get_master_services "${services_args[@]}")) || exit 1
  4826. if [ "$action" == "up" ]; then
  4827. action_posargs+=($(services:get:upable "${services_args[@]}")) || exit 1
  4828. elif [ "$is_docker_compose_action_multi_service" == "1" ]; then
  4829. action_posargs+=("${services[@]}")
  4830. elif [ "$is_docker_compose_action_multi_service" == "0" ]; then
  4831. action_posargs+=("${services[0]}") ## only the first service is the legit one
  4832. fi
  4833. ## Get rid of subordinates
  4834. action_posargs=($(get_master_services "${action_posargs[@]}")) || exit 1
  4835. fi
  4836. get_docker_compose "${services_args[@]}" >/dev/null || { ## precalculate variable \$_current_docker_compose
  4837. err "Fails to compile base 'docker-compose.yml'"
  4838. exit 1
  4839. }
  4840. ##
  4841. ## Pre-action
  4842. ##
  4843. full_init=
  4844. case "$action" in
  4845. build)
  4846. full_init=true ## will actually stop after build
  4847. ;;
  4848. up|run)
  4849. full_init=true
  4850. post_hook=true
  4851. ;;
  4852. ""|down|restart|logs|config|ps|status)
  4853. full_init=
  4854. ;;
  4855. *)
  4856. if [ "$is_service_action" ]; then
  4857. full_init=true
  4858. keywords=($(egrep "^#*\s*compose:" "$action_script_path" | cut -f 2- -d:))
  4859. for keyword in "${keywords[@]}"; do
  4860. case "$keyword" in
  4861. no-hooks)
  4862. no_hooks=true
  4863. ;;
  4864. hooks)
  4865. full_init=true
  4866. ;;
  4867. esac
  4868. done
  4869. fi
  4870. ;;
  4871. esac
  4872. if [ -n "$full_init" ]; then
  4873. if [[ "$action" == "build" ]] || [[ -z "$no_init" && -z "$no_hooks" ]]; then
  4874. [[ "$action" == "build" ]] || Section "acquire charm's images"
  4875. run_service_acquire_images "${services_args[@]}" || exit 1
  4876. Feed
  4877. [ "$action" == "build" ] && {
  4878. exit 0
  4879. }
  4880. Section setup host resources
  4881. setup_host_resources "${services_args[@]}" || exit 1
  4882. ## init in order
  4883. Section initialisation
  4884. run_service_hook init "${services_args[@]}" || exit 1
  4885. fi
  4886. ## Get relations
  4887. if [[ -z "$no_relations" && -z "$no_hooks" ]]; then
  4888. if [ "${#rebuild_relations_to_service[@]}" != 0 ]; then
  4889. rebuild_relations_to_service=$(get_master_services "${rebuild_relations_to_service[@]}") || return 1
  4890. rebuild_relations_to_service=($rebuild_relations_to_service)
  4891. project=$(get_default_project_name) || return 1
  4892. for service in "${rebuild_relations_to_service[@]}"; do
  4893. for dir in "$VARDIR/relations/$project/"*"-${service}/"*; do
  4894. [ -d "$dir" ] && {
  4895. debug rm -rf "$dir"
  4896. rm -rf "$dir"
  4897. }
  4898. done
  4899. done
  4900. fi
  4901. run_service_relations "${services_args[@]}" || exit 1
  4902. fi
  4903. if [[ -z "$no_pre_deploy" && -z "$no_hooks" ]]; then
  4904. run_service_hook pre_deploy "${services_args[@]}" || exit 1
  4905. fi
  4906. fi | log
  4907. if [ "${PIPESTATUS[0]}" != 0 ]; then
  4908. exit 1
  4909. fi
  4910. [ "$action" == "build" ] && exit 0
  4911. if [ "$action" == "status" ]; then
  4912. if [[ -n "${state_all_services}" ]] || [[ "${#state_filters[@]}" -gt 0 ]]; then
  4913. compose_yml_services=($(compose:yml:root:services)) || exit 1
  4914. fi
  4915. if [[ -n "${state_all_services}" ]]; then
  4916. state_columns=("root" ${state_columns[@]})
  4917. fi
  4918. state_columns_raw=()
  4919. for col in "${state_columns[@]}"; do
  4920. if [[ "$col" == "-"* ]]; then
  4921. col=${col#-}
  4922. fi
  4923. state_columns_raw+=("$col")
  4924. done
  4925. state_columns_align=""
  4926. for col in "${state_columns[@]}"; do
  4927. if [[ "$col" == "-"* ]]; then
  4928. state_columns_align+="+"
  4929. else
  4930. state_columns_align+="-"
  4931. fi
  4932. done
  4933. while read-0-err E "${state_columns_raw[@]}"; do
  4934. values=()
  4935. for col in "${state_columns_raw[@]}"; do
  4936. color=
  4937. value="${!col}"
  4938. read -r -- value_trim <<<"${!col}"
  4939. case "$col" in
  4940. root)
  4941. case "$value_trim" in
  4942. 0) value=" ";;
  4943. 1) value="*";;
  4944. esac
  4945. ;;
  4946. name) color=darkyellow;;
  4947. charm) color=darkpink;;
  4948. state)
  4949. case "$value_trim" in
  4950. up) color=green;;
  4951. down) color=gray;;
  4952. deploying) color=yellow;;
  4953. *) color=red;;
  4954. esac
  4955. ;;
  4956. type)
  4957. case "$value_trim" in
  4958. run-once) color=gray;;
  4959. stub) color=gray;;
  4960. *) color=darkcyan;;
  4961. esac
  4962. ;;
  4963. *)
  4964. if [[ "${value_trim}" == "N/A" ]]; then
  4965. color=gray
  4966. fi
  4967. ;;
  4968. esac
  4969. color="${color^^}"
  4970. if [ -n "$color" ]; then
  4971. values+=("${!color}$value${NORMAL}")
  4972. else
  4973. values+=("$value")
  4974. fi
  4975. done
  4976. first=1
  4977. for value in "${values[@]}"; do
  4978. if [ -n "$first" ]; then
  4979. first=
  4980. else
  4981. printf " "
  4982. fi
  4983. printf "%s" "$value"
  4984. done
  4985. printf "\n"
  4986. done < <(
  4987. set -o pipefail
  4988. filter_cols=()
  4989. for filter in "${state_filters[@]}"; do
  4990. IFS="=" read -r key value <<<"$filter"
  4991. ## if not already in state_columns_raw
  4992. [[ " ${state_columns_raw[*]} " == *" $key "* ]] ||
  4993. filter_cols+=("$key")
  4994. done
  4995. for service in "${services_args[@]}"; do
  4996. declare -A values=()
  4997. for col in "${state_columns_raw[@]}" "${filter_cols[@]}"; do
  4998. case "$col" in
  4999. root)
  5000. if [[ " ${compose_yml_services[*]} " == *" ${service} "* ]]; then
  5001. value="1"
  5002. else
  5003. value="0"
  5004. fi
  5005. ;;
  5006. name) value="$service" ;;
  5007. charm)
  5008. value=$(get_service_charm "$service") || { echo 1; exit 1; }
  5009. ;;
  5010. state)
  5011. value=$(service:state "$service") || { echo 1; exit 1; }
  5012. ;;
  5013. type)
  5014. value=$(get_service_type "$service") || { echo 1; exit 1; }
  5015. ;;
  5016. *)
  5017. if has_service_action "$service" "get-$col" >/dev/null; then
  5018. state_msg=$(run_service_action "$service" "get-$col") || { echo 1; exit 1 ; }
  5019. if [[ "$state_msg" == *$'\n'* ]]; then
  5020. value="${state_msg%%$'\n'*}"
  5021. ## XXXvlab: For now, these are not used, but we could
  5022. ## display them in additional lines (in same "cell")
  5023. msgs="${state_msg#*$'\n'}"
  5024. else
  5025. value=${state_msg}
  5026. fi
  5027. else
  5028. value="N/A"
  5029. fi
  5030. ;;
  5031. esac
  5032. values["$col"]="$value"
  5033. done
  5034. for filter in "${state_filters[@]}"; do
  5035. IFS="=" read -r key value <<<"$filter"
  5036. [[ "${values[$key]}" != "$value" ]] &&
  5037. continue 2
  5038. done
  5039. for col in "${state_columns_raw[@]}"; do
  5040. p0 "${values[$col]}"
  5041. done
  5042. done | col-0:normalize:size "${state_columns_align}"
  5043. echo 0
  5044. )
  5045. if [ "$E" != 0 ]; then
  5046. echo "E: '$E'" >&2
  5047. exit 1
  5048. fi
  5049. exit 0
  5050. fi
  5051. if [ "$action" == "run" ] && [ "${#services_args}" != 0 ]; then
  5052. charm=$(get_service_charm "${services_args[0]}") || exit 1
  5053. metadata=$(charm.metadata "$charm") || exit 1
  5054. SERVICE_TYPE="$(printf "%s" "$metadata" | shyaml get-value type 2>/dev/null)" || true
  5055. if [ "$SERVICE_TYPE" == "run-once" ]; then
  5056. run_service_hook dc-pre-run "${services_args[@]}" || exit 1
  5057. fi
  5058. fi
  5059. export SERVICE_PACK="${services_args[*]}"
  5060. ##
  5061. ## Docker-compose
  5062. ##
  5063. errlvl="0"
  5064. case "$action" in
  5065. up|start|stop|build|run)
  5066. ## force daemon mode for up
  5067. if [[ "$action" == "up" ]]; then
  5068. if ! array_member action_opts -d; then
  5069. action_opts+=("-d")
  5070. fi
  5071. if ! array_member action_opts --remove-orphans; then
  5072. action_opts+=("--remove-orphans")
  5073. fi
  5074. fi
  5075. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  5076. ;;
  5077. logs)
  5078. if ! array_member action_opts --tail; then ## force daemon mode for up
  5079. action_opts+=("--tail" "10")
  5080. fi
  5081. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  5082. ;;
  5083. "")
  5084. launch_docker_compose "${compose_opts[@]}"
  5085. ;;
  5086. graph)
  5087. graph $SERVICE_PACK
  5088. ;;
  5089. config)
  5090. ## removing the services
  5091. services=($(get_master_services "${action_posargs[@]}")) || exit 1
  5092. ## forcing docker-compose config to output the config file to stdout and not stderr
  5093. out=$(launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" 2>&1) || {
  5094. echo "$out"
  5095. exit 1
  5096. }
  5097. echo "$out"
  5098. warn "Runtime configuration modification (from relations) are not included here."
  5099. ;;
  5100. down)
  5101. if ! array_member action_opts --remove-orphans; then ## force daemon mode for up
  5102. debug "Adding a default argument of '--remove-orphans'"
  5103. action_opts+=("--remove-orphans")
  5104. fi
  5105. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}"
  5106. ;;
  5107. *)
  5108. if [ "$is_service_action" ]; then
  5109. run_service_action "$action_service" "$action" "${remainder_args[@]}"
  5110. errlvl="$?"
  5111. errlvl "$errlvl"
  5112. else
  5113. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  5114. fi
  5115. ;;
  5116. esac || exit 1
  5117. if [ "$post_hook" -a "${#services_args[@]}" != 0 -a -z "$no_hooks" -a -z "$no_post_deploy" ]; then
  5118. run_service_hook post_deploy "${services_args[@]}" || exit 1
  5119. fi
  5120. if [ "$action" == "run" -a "${#services_args}" != 0 ]; then
  5121. if [ "$SERVICE_TYPE" == "run-once" ]; then
  5122. run_service_hook dc-post-run "${services_args[@]}" || exit 1
  5123. fi
  5124. fi
  5125. case "$action" in
  5126. up)
  5127. ## Notify that services in 'deploying' states have been deployed
  5128. for service in "$SERVICE_STATE_PATH/$PROJECT_NAME"/*/deploying; do
  5129. [ -e "$service" ] || continue
  5130. state=${service##*/}
  5131. service=${service%/$state}
  5132. service=${service##*/}
  5133. mv "$SERVICE_STATE_PATH/$PROJECT_NAME"/"${service}"/{deploying,up} || exit 1
  5134. done
  5135. ## Notify that services in 'orphaning' states have been removed
  5136. for service in "$SERVICE_STATE_PATH/$PROJECT_NAME"/*/orphaning; do
  5137. [ -e "$service" ] || continue
  5138. state=${service##*/}
  5139. service=${service%/$state}
  5140. service=${service##*/}
  5141. rm "$SERVICE_STATE_PATH/$PROJECT_NAME"/"${service}"/orphaning || exit 1
  5142. done
  5143. ;;
  5144. down)
  5145. PROJECT_NAME=$(get_default_project_name) || return 1
  5146. if [ -d "$SERVICE_STATE_PATH/$PROJECT_NAME" ]; then
  5147. if ! dir_is_empty "$SERVICE_STATE_PATH/$PROJECT_NAME"; then
  5148. rm -f "$SERVICE_STATE_PATH/$PROJECT_NAME"/*/*
  5149. fi
  5150. rmdir "$SERVICE_STATE_PATH/$PROJECT_NAME"/{*,}
  5151. fi
  5152. ;;
  5153. esac
  5154. clean_unused_docker_compose || exit 1
  5155. exit "$errlvl"