You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

6397 lines
216 KiB

  1. #!/bin/bash
  2. ##
  3. ## TODO:
  4. ## - subordinate container should really be able to modify base image of their master
  5. ## - this could be done through docker-update
  6. ## - I'm not happy with the current build using 'build/' directory, this should be
  7. ## changed to:
  8. ## - always have a base image (specified in metadata), and always have hooks/install
  9. ## executed and merge in image (like docker-build-charm).
  10. ## - container base image is ALWAYS the image of the master container... this brings
  11. ## questions about a double way to express inheritage (through relations as it is
  12. ## implemented now, or through this base-image ?)
  13. ## - the name of the scripts for relation (aka relation_name-relation-joined) is bad as
  14. ## reading the name in a hooks/ dir, there are no way to know if we are the target or
  15. ## the base of the relation.
  16. ## - we could leverage a 'relations/' dir on the root of the charm, with both:
  17. ## 'relations/provide/relation_name' and 'relations/receive/relation_name'
  18. ## - a very bad point with the actual naming is that we can't have a providing AND
  19. ## receiving a relation with same name.
  20. ## - The cache system should keep md5 of docker-compose and other things between runs
  21. ## - The cache system should use underlying function that have only arguments inputs.
  22. ## This will allow to cache completely without issues function in time.
  23. ## - would probably need instrospection in charm custom action to know if these need
  24. ## init or relations to be set up.
  25. ## - Be clear about when the SERVICE name is used and the CHARM name is used.
  26. ## - in case of service contained in another container
  27. ## - in normal case
  28. ## - in docker-compose, can't use charm name: if we want 2 instances of the same charm
  29. ## we are stuck. What will be unique is the name of the service.
  30. ## - some relations are configured in compose.yml but should not trigger the loading
  31. ## of necessary component (for instance, apache --> log-rotate), if log-rotate is
  32. ## not there, this link should considered optional.
  33. ## - Could probably allow an unexistent charm to be populated with only "docker-image:"
  34. ## of the same name. Although this should trigger a visible warning.
  35. #:-
  36. [ -e /etc/shlib ] && . /etc/shlib || {
  37. echo "Unsatisfied dependency. Please install 'kal-shlib-core'."
  38. exit 1
  39. }
  40. #:-
  41. include common
  42. include pretty
  43. include parse
  44. include charm
  45. include array
  46. include cla
  47. include docker
  48. depends shyaml docker
  49. exname="compose"
  50. version=0.1
  51. usage="$exname [COMPOSE_OPTS] [ACTION [ACTION_OPTS]]"
  52. help="\
  53. $WHITE$exname$NORMAL jobs is to run various shell scripts to build
  54. a running orchestrated and configured docker containers. These shell
  55. scripts will have the opportunity to build a 'docker-compose.yml'.
  56. Once init script and relations scripts are executed, $WHITE$exname$NORMAL
  57. delegate the launching to ${WHITE}docker-compose${NORMAL} by providing it
  58. the final 'docker-compose.yml'.
  59. $WHITE$exname$NORMAL also leverage charms to offer some additional custom
  60. actions per charm, which are simply other scripts that can be
  61. run without launching ${WHITE}docker-compose${NORMAL}.
  62. In compose message, color coding is enforced as such:
  63. - ${DARKCYAN}action$NORMAL,
  64. - ${DARKBLUE}relation$NORMAL,
  65. - ${DARKPINK}charm${NORMAL},
  66. - ${DARKYELLOW}service${NORMAL},
  67. - ${WHITE}option-name${NORMAL}/${WHITE}command-name${NORMAL}/${WHITE}Section-Title${NORMAL}
  68. $WHITE$exname$NORMAL reads '/etc/compose.conf' for global variables, and
  69. '/etc/compose.local.conf' for local host adjustements.
  70. "
  71. time_now() { date +%s.%3N; }
  72. time_elapsed() { echo "scale=3; $2 - $1" | bc; }
  73. ## XXXvlab: this doesn't seem to work when 'compose' is called in
  74. ## a hook of a charm.
  75. #[[ "${BASH_SOURCE[0]}" == "" ]] && SOURCED=true
  76. $(return >/dev/null 2>&1) && SOURCED=true
  77. errlvl() { return "${1:-1}"; }
  78. export -f errlvl
  79. if [ "$UID" == 0 ]; then
  80. CACHEDIR=${CACHEDIR:-/var/cache/compose}
  81. VARDIR=${VARDIR:-/var/lib/compose}
  82. else
  83. [ "$XDG_CONFIG_HOME" ] && CACHEDIR=${CACHEDIR:-$XDG_CONFIG_HOME/compose}
  84. [ "$XDG_DATA_HOME" ] && VARDIR=${VARDIR:-$XDG_DATA_HOME/compose}
  85. CACHEDIR=${CACHEDIR:-$HOME/.cache/compose}
  86. VARDIR=${VARDIR:-$HOME/.local/share/compose}
  87. fi
  88. export VARDIR CACHEDIR
  89. export SERVICE_STATE_PATH=${SERVICE_STATE_PATH:-/var/lib/compose/state}
  90. md5_compat() { md5sum | cut -c -32; }
  91. quick_cat_file() { quick_cat_stdin < "$1"; }
  92. quick_cat_stdin() { local IFS=''; while read -r line; do echo "$line"; done ; }
  93. export -f quick_cat_file quick_cat_stdin md5_compat
  94. p-err() {
  95. "$@"
  96. echo "$?"
  97. }
  98. export -f p-err
  99. wyq() {
  100. local exp="$1"
  101. yq e -e -0 "$1"
  102. printf "%s" "$?"
  103. }
  104. wyq-r() {
  105. local exp="$1"
  106. yq e -e -0 -r=false "$1"
  107. printf "%s" "$?"
  108. }
  109. err-d () {
  110. local msg="$*"
  111. shift
  112. err "$msg"
  113. print:traceback 1
  114. }
  115. export -f err-d
  116. print:traceback() {
  117. local omit_level="${1:-0}"
  118. if [ -z "$DEBUG" ]; then
  119. echo " Note: traceback available if you provide {--debug|-d} option." >&2
  120. return 0
  121. fi
  122. echo "${WHITE}Traceback (most recent call last):${NORMAL}" >&2
  123. local i
  124. for ((i=${#FUNCNAME[@]} - 1; i > "$omit_level"; i--)); do
  125. local file="${BASH_SOURCE[$i]}"
  126. local line="${BASH_LINENO[$i - 1]}"
  127. local func="${FUNCNAME[$i]}"
  128. if [[ -f "$file" ]]; then
  129. # Get total number of lines in the file
  130. local total_lines
  131. total_lines=$(wc -l < "$file")
  132. # Calculate start and end lines for context
  133. local start_line=$((line - 2))
  134. local end_line=$((line + 2))
  135. [[ $start_line -lt 1 ]] && start_line=1
  136. [[ $end_line -gt $total_lines ]] && end_line=$total_lines
  137. # Extract context lines
  138. mapfile -s $((start_line - 1)) -n $((end_line - start_line + 1)) context_lines < "$file"
  139. # Calculate minimal indentation
  140. local min_indent=9999
  141. for line_text in "${context_lines[@]}"; do
  142. if [[ -n "$line_text" ]]; then
  143. # Get leading whitespace
  144. local leading_whitespace="${line_text%%[![:space:]]*}"
  145. local indent=${#leading_whitespace}
  146. if [[ $indent -lt $min_indent ]]; then
  147. min_indent=$indent
  148. fi
  149. fi
  150. done
  151. # Remove minimal indentation from each line
  152. for idx in "${!context_lines[@]}"; do
  153. context_lines[$idx]="${context_lines[$idx]:$min_indent}"
  154. done
  155. else
  156. context_lines=("<source unavailable>")
  157. min_indent=0
  158. start_line=1
  159. end_line=1
  160. fi
  161. # Print the traceback frame
  162. echo " File \"$file\", line $line, in ${WHITE}$func${NORMAL}:"
  163. # Print the context with line numbers
  164. local current_line=$start_line
  165. for context_line in "${context_lines[@]}"; do
  166. context_line="${context_line%$'\n'}"
  167. if [[ $current_line -eq $line ]]; then
  168. echo " ${DARKYELLOW}$current_line${NORMAL} ${context_line}"
  169. else
  170. echo " ${DARKGRAY}$current_line${NORMAL} ${context_line}"
  171. fi
  172. ((current_line++))
  173. done
  174. done >&2
  175. }
  176. export -f print:traceback
  177. clean_cache() {
  178. local i=0
  179. for f in $(ls -t "$CACHEDIR/"*.cache.* 2>/dev/null | tail -n +500); do
  180. ((i++))
  181. rm -f "$f"
  182. done
  183. if (( i > 0 )); then
  184. debug "${WHITE}Cleaned cache:${NORMAL} Removed $((i)) elements (current cache size is $(du -sh "$CACHEDIR" | cut -f 1))"
  185. fi
  186. }
  187. export DEFAULT_COMPOSE_FILE
  188. ##
  189. ## Merge YAML files
  190. ##
  191. export _merge_yaml_common_code="
  192. import sys
  193. import yaml
  194. try:
  195. from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
  196. except ImportError: ## pragma: no cover
  197. sys.stderr.write('YAML code in pure python\n')
  198. exit(1)
  199. from yaml import SafeLoader, SafeDumper
  200. class MySafeLoader(SafeLoader): pass
  201. class MySafeDumper(SafeDumper): pass
  202. try:
  203. # included in standard lib from Python 2.7
  204. from collections import OrderedDict
  205. except ImportError:
  206. # try importing the backported drop-in replacement
  207. # it's available on PyPI
  208. from ordereddict import OrderedDict
  209. ## Ensure that there are no collision with legacy OrderedDict
  210. ## that could be used for omap for instance.
  211. class MyOrderedDict(OrderedDict):
  212. pass
  213. MySafeDumper.add_representer(
  214. MyOrderedDict,
  215. lambda cls, data: cls.represent_dict(data.items()))
  216. def construct_omap(cls, node):
  217. cls.flatten_mapping(node)
  218. return MyOrderedDict(cls.construct_pairs(node))
  219. MySafeLoader.add_constructor(
  220. yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
  221. construct_omap)
  222. ##
  223. ## Support local and global objects
  224. ##
  225. class EncapsulatedNode(object): pass
  226. def mk_encapsulated_node(s, node):
  227. method = 'construct_%s' % (node.id, )
  228. data = getattr(s, method)(node)
  229. class _E(data.__class__, EncapsulatedNode):
  230. pass
  231. _E.__name__ = str(node.tag)
  232. _E._node = node
  233. return _E(data)
  234. def represent_encapsulated_node(s, o):
  235. value = s.represent_data(o.__class__.__bases__[0](o))
  236. value.tag = o.__class__.__name__
  237. return value
  238. MySafeDumper.add_multi_representer(EncapsulatedNode,
  239. represent_encapsulated_node)
  240. MySafeLoader.add_constructor(None, mk_encapsulated_node)
  241. def fc(filename):
  242. with open(filename) as f:
  243. return f.read()
  244. def merge(*args):
  245. # sys.stderr.write('%r\n' % (args, ))
  246. args = [arg for arg in args if arg is not None]
  247. if len(args) == 0:
  248. return None
  249. if len(args) == 1:
  250. return args[0]
  251. if all(isinstance(arg, (int, basestring, bool, float)) for arg in args):
  252. return args[-1]
  253. elif all(isinstance(arg, list) for arg in args):
  254. res = []
  255. for arg in args:
  256. for elt in arg:
  257. if elt in res:
  258. res.remove(elt)
  259. res.append(elt)
  260. return res
  261. elif all(isinstance(arg, dict) for arg in args):
  262. keys = set()
  263. for arg in args:
  264. keys |= set(arg.keys())
  265. dct = {}
  266. for key in keys:
  267. sub_args = []
  268. for arg in args:
  269. if key in arg:
  270. sub_args.append(arg)
  271. try:
  272. dct[key] = merge(*(a[key] for a in sub_args))
  273. except NotImplementedError as e:
  274. raise NotImplementedError(
  275. e.args[0],
  276. '%s.%s' % (key, e.args[1]) if e.args[1] else key,
  277. e.args[2])
  278. if dct[key] is None:
  279. del dct[key]
  280. return dct
  281. else:
  282. raise NotImplementedError(
  283. 'Unsupported types: %s'
  284. % (', '.join(list(set(arg.__class__.__name__ for arg in args)))), '', args)
  285. return None
  286. def merge_cli(*args):
  287. try:
  288. c = merge(*args)
  289. except NotImplementedError as e:
  290. sys.stderr.write('Merging Failed: %s.\n%s\n'
  291. ' Values are:\n %s\n'
  292. % (e.args[0],
  293. ' Conflicting key is %r.' % e.args[1] if e.args[1] else
  294. ' Conflict at base of structure.',
  295. '\\n '.join('v%d: %r' % (i, a)
  296. for i, a in enumerate(e.args[2]))))
  297. exit(1)
  298. if c is not None:
  299. print '%s' % yaml.dump(c, default_flow_style=False, Dumper=MySafeDumper)
  300. "
  301. merge_yaml() {
  302. if ! [ -r "$state_tmpdir/merge_yaml.py" ]; then
  303. cat <<EOF > "$state_tmpdir/merge_yaml.py"
  304. $_merge_yaml_common_code
  305. merge_cli(*(yaml.load(fc(f), Loader=MySafeLoader) for f in sys.argv[1:]))
  306. EOF
  307. fi
  308. python "$state_tmpdir/merge_yaml.py" "$@"
  309. }
  310. export -f merge_yaml
  311. merge_yaml_str() {
  312. local entries="$@"
  313. if ! [ -r "$state_tmpdir/merge_yaml_str.py" ]; then
  314. cat <<EOF > "$state_tmpdir/merge_yaml_str.py" || return 1
  315. $_merge_yaml_common_code
  316. merge_cli(*(yaml.load(f, Loader=MySafeLoader) for f in sys.argv[1:]))
  317. EOF
  318. fi
  319. if ! python "$state_tmpdir/merge_yaml_str.py" "$@"; then
  320. err "Failed to merge yaml strings:"
  321. local s
  322. for s in "$@"; do
  323. printf " - \n"
  324. printf "%s\n" "$s" | prefix " ${GRAY}|$NORMAL "
  325. done >&2
  326. return 1
  327. fi
  328. }
  329. export -f merge_yaml_str
  330. yaml_get_values() {
  331. local sep=${1:-$'\n'} value input type first elt
  332. input=$(cat -)
  333. if [ -z "$input" ] || [[ "$input" =~ ^None|null$ ]]; then
  334. return 0
  335. fi
  336. type=$(e "$input" | shyaml get-type)
  337. value=
  338. case "$type" in
  339. "sequence")
  340. first=1
  341. while read-0 elt; do
  342. elt="$(e "$elt" | yaml_get_interpret)" || return 1
  343. [ "$elt" ] || continue
  344. if [ "$first" ]; then
  345. first=
  346. else
  347. value+="$sep"
  348. fi
  349. first=
  350. value+="$elt"
  351. done < <(e "$input" | shyaml -y get-values-0)
  352. ;;
  353. "struct")
  354. while read-0 val; do
  355. value+=$'\n'"$(e "$val" | yaml_get_interpret)" || return 1
  356. done < <(e "$input" | shyaml -y values-0)
  357. ;;
  358. "NoneType")
  359. value=""
  360. ;;
  361. "str"|*)
  362. value+="$(e "$input" | yaml_get_interpret)"
  363. ;;
  364. esac
  365. e "$value"
  366. }
  367. export -f yaml_get_values
  368. yaml_key_val_str() {
  369. local entries="$@"
  370. if ! [ -r "$state_tmpdir/yaml_key_val_str.py" ]; then
  371. cat <<EOF > "$state_tmpdir/yaml_key_val_str.py"
  372. $_merge_yaml_common_code
  373. print '%s' % yaml.dump(
  374. {
  375. yaml.load(sys.argv[1], Loader=MySafeLoader):
  376. yaml.load(sys.argv[2], Loader=MySafeLoader)
  377. },
  378. default_flow_style=False,
  379. Dumper=MySafeDumper,
  380. )
  381. EOF
  382. fi
  383. python "$state_tmpdir/yaml_key_val_str.py" "$@"
  384. }
  385. export -f yaml_key_val_str
  386. ##
  387. ## Intended for charm
  388. ##
  389. password:get() {
  390. local service="$SERVICE_NAME" label="$1" type="$2" size="${2:-32}"
  391. local password_file
  392. password_file="$(password:get-location "$label" "$type" "$size")" || return 1
  393. cat "$password_file"
  394. }
  395. export -f password:get
  396. password:get-location() {
  397. local service="$SERVICE_NAME" label="$1" type="$2" size="${2:-32}"
  398. if [ -z "$service" ]; then
  399. err "${FUNCNAME[0]}: should be called from a charm hook"
  400. return 1
  401. fi
  402. charm=$(get_service_charm "$service") || return 1
  403. case "$type" in
  404. "internal")
  405. :
  406. ;;
  407. "user")
  408. if ! fn.exists "$charm:password:check"; then
  409. err "Charm '$charm' doesn't provide a '$charm:password:check' function."
  410. echo " This function is required to check if a user password was changed." >&2
  411. return 1
  412. fi
  413. ;;
  414. *)
  415. err "${FUNCNAME[0]}: unknown type '$type'"
  416. echo " Should be 'internal' or 'user'." >&2
  417. return 1
  418. ;;
  419. esac
  420. local password_file="$SERVICE_DATASTORE"/.compose/passwords/"$type"/"$label"
  421. if ! [ -f "$password_file" ]; then
  422. info "Generating password" >&2
  423. mkdir -p "${password_file%/*}" &&
  424. umask 077 &&
  425. openssl rand -base64 32 | tr -d '\n' > "$password_file"
  426. e "$password_file"
  427. return 0
  428. fi
  429. if [ "$type" != "user" ]; then
  430. info "Using existing secret password" >&2
  431. e "$password_file"
  432. return 0
  433. fi
  434. if [ -s "$password_file" ]; then
  435. ## password was already set by user through the service
  436. e "$password_file"
  437. return 0
  438. fi
  439. if ! out=$("$charm:password:check" "$label" "$password_file"); then
  440. err "Failed to check password for '$label' in '$charm'."
  441. return 1
  442. fi
  443. case "$out" in
  444. "changed")
  445. info "Password for '$label' has changed, forgetting it." >&2
  446. e "" > "$password_file" ## empty file
  447. e "$password_file"
  448. return 0
  449. ;;
  450. "ok")
  451. e "$password_file"
  452. return 0
  453. ;;
  454. *)
  455. err "Unknown return value from '$charm:password:check': $out"
  456. return 1
  457. ;;
  458. esac
  459. }
  460. export -f password:get-location
  461. passwords:advertise() {
  462. local services file label
  463. services=("$@")
  464. local service
  465. complete_msg=""
  466. for service in "${services[@]}"; do
  467. dir="$DATASTORE"/"$service"/.compose/passwords/user
  468. [ -d "$dir" ] || continue
  469. msg=""
  470. for file in "$dir"/*; do
  471. [ -s "$file" ] || continue
  472. ls -al "$file" >&2
  473. label=${file##*/}
  474. msg+=" - $label is $(cat "$file")"$'\n'
  475. done
  476. if [ -n "$msg" ]; then
  477. msg=" ${DARKYELLOW}$service${NORMAL}:"$'\n'"$msg"
  478. fi
  479. complete_msg+="$msg"
  480. done
  481. if [ -n "$complete_msg" ]; then
  482. echo
  483. echo "${WHITE}Compose generated passwords${NORMAL}:"
  484. echo " (Consider changing these password in their respective services)"
  485. echo
  486. echo "$complete_msg"
  487. fi >&2
  488. }
  489. ##
  490. ## Docker
  491. ##
  492. docker_has_image() {
  493. local image="$1"
  494. images=$(docker images -q "$image" 2>/dev/null) || {
  495. err "docker images call has failed unexpectedly."
  496. return 1
  497. }
  498. [ -n "$images" ]
  499. }
  500. export -f docker_has_image
  501. docker_image_id() {
  502. local image="$1"
  503. image_id=$(docker inspect "$image" --format='{{.Id}}') || return 1
  504. echo "$image_id" # | tee "$cache_file"
  505. }
  506. export -f docker_image_id
  507. cached_cmd_on_image() {
  508. local image="$1" cache_file
  509. image_id=$(docker_image_id "$image") || return 1
  510. cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  511. if [ -e "$cache_file" ]; then
  512. # debug "$FUNCNAME: cache hit ($*)"
  513. quick_cat_stdin < "$cache_file"
  514. return 0
  515. fi
  516. shift
  517. out=$(docker run -i --rm --entrypoint /bin/sh "$image_id" -c "$*") || return 1
  518. echo "$out" | tee "$cache_file"
  519. }
  520. export -f cached_cmd_on_image
  521. cmd_on_base_image() {
  522. local service="$1" base_image
  523. shift
  524. base_image=$(service_ensure_image_ready "$service") || return 1
  525. docker run -i --rm --entrypoint /bin/bash "$base_image" -c "$*"
  526. }
  527. export -f cmd_on_base_image
  528. cached_cmd_on_base_image() {
  529. local service="$1" base_image cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  530. shift
  531. if [ -e "$cache_file" ]; then
  532. # debug "$FUNCNAME: cache hit ($*)"
  533. quick_cat_stdin < "$cache_file"
  534. return 0
  535. fi
  536. base_image=$(service_ensure_image_ready "$service") || return 1
  537. result=$(cached_cmd_on_image "$base_image" "$@") || return 1
  538. echo "$result" | tee "$cache_file"
  539. }
  540. export -f cached_cmd_on_base_image
  541. docker_update() {
  542. ## YYY: warning, we a storing important information in cache, cache can
  543. ## be removed.
  544. ## We want here to cache the last script on given service whatever that script was
  545. local service="$1" script="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$1" \
  546. previous_base_image stored_image_id
  547. shift
  548. shift
  549. ## this will build it if necessary
  550. base_image=$(service_ensure_image_ready "$service") || return 1
  551. ## XXXvlab: there are probably ways to avoid rebuilding that each time
  552. image_id="$(docker_image_id "$base_image")" || return 1
  553. if [ -e "$cache_file" ]; then
  554. info "Cache file exists"
  555. read-0 previous_base_image stored_image_id < <(cat "$cache_file")
  556. info "previous: $previous_base_image"
  557. info "stored: $stored_image_id"
  558. else
  559. info "No cache file $cache_file"
  560. previous_base_image=""
  561. fi
  562. if [ "$previous_base_image" -a "$stored_image_id" == "$image_id" ]; then
  563. info "Resetting $base_image to $previous_base_image"
  564. docker tag "$previous_base_image" "$base_image" || return 1
  565. image_id="$(docker_image_id "$base_image")" || return 1
  566. else
  567. previous_base_image="$image_id"
  568. fi
  569. info "Updating base image: $base_image (hash: $image_id)"
  570. echo "$script" | dupd --debug -u "$base_image" -- "$@" || {
  571. err "Failed updating base image"
  572. return 1
  573. }
  574. new_image_id="$(docker_image_id "$base_image")"
  575. [ "$new_image_id" == "$previous_base_image" ] && {
  576. err "Image was not updated correctly (same id)."
  577. return 1
  578. }
  579. printf "%s\0" "$previous_base_image" "$new_image_id" > "$cache_file"
  580. info "Wrote cache file $cache_file"
  581. }
  582. export -f docker_update
  583. image_exposed_ports_0() {
  584. local image="$1"
  585. docker inspect --format='{{range $p, $conf := .Config.ExposedPorts}}{{$p}}{{"\x00"}}{{end}}' "$image"
  586. }
  587. export -f image_exposed_ports_0
  588. ## feature not yet included in docker: https://github.com/moby/moby/issues/16079
  589. docker_image_export_dir() {
  590. local image="$1" src="$2" dst="$3" container_id
  591. (
  592. container_id=$(docker create "$image") || exit 1
  593. trap_add EXIT,ERR "docker rm \"$container_id\" >/dev/null"
  594. docker cp "$container_id":"$src" "$dst"
  595. )
  596. }
  597. export -f docker_image_export_dir
  598. service_base_image_export_dir() {
  599. local service="$1" src="$2" dst="$3" base_image
  600. shift
  601. base_image=$(service_ensure_image_ready "$service") || return 1
  602. docker_image_export_dir "$base_image" "$src" "$dst"
  603. }
  604. export -f service_base_image_export_dir
  605. service_base_image_id() {
  606. local service="$1" src="$2" dst="$3" base_image
  607. shift
  608. base_image=$(service_ensure_image_ready "$service") || return 1
  609. docker inspect "$base_image" --format="{{ .Id }}"
  610. }
  611. export -f service_base_image_id
  612. ##
  613. ## Generic
  614. ##
  615. fn.exists() {
  616. declare -F "$1" >/dev/null
  617. }
  618. str_pattern_matches() {
  619. local str="$1"
  620. shift
  621. for pattern in "$@"; do
  622. eval "[[ \"$str\" == $pattern ]]" && return 0
  623. done
  624. return 1
  625. }
  626. str_matches() {
  627. local str="$1"
  628. shift
  629. for pattern in "$@"; do
  630. [[ "$str" == "$pattern" ]] && return 0
  631. done
  632. return 1
  633. }
  634. gen_password() {
  635. local l=( {a..z} {A..Z} {0..9} ) nl="${#l[@]}" size=${1:-16}
  636. while ((size--)); do
  637. echo -n "${l[$((RANDOM * nl / 32768))]}"
  638. done
  639. echo
  640. }
  641. export -f gen_password
  642. file_put() {
  643. local TARGET="$1"
  644. mkdir -p "$(dirname "$TARGET")" &&
  645. cat - > "$TARGET"
  646. }
  647. export -f file_put
  648. file_put_0() {
  649. local TARGET="$1"
  650. mkdir -p "$(dirname "$TARGET")" &&
  651. cat > "$TARGET"
  652. }
  653. export -f file_put_0
  654. fetch_file() {
  655. local src="$1"
  656. case "$src" in
  657. *"://"*)
  658. err "Unsupported target scheme."
  659. return 1
  660. ;;
  661. *)
  662. ## Try direct
  663. if ! [ -r "$src" ]; then
  664. err "File '$src' not found/readable."
  665. return 1
  666. fi
  667. cat "$src" || return 1
  668. ;;
  669. esac
  670. }
  671. export -f fetch_file
  672. ## receives stdin content to decompress on stdout
  673. ## stdout content should be tar format.
  674. uncompress_file() {
  675. local filename="$1"
  676. ## Warning, the content of the file is already as stdin, the filename
  677. ## is there to hint for correct decompression.
  678. case "$filename" in
  679. *".gz")
  680. gunzip
  681. ;;
  682. *".bz2")
  683. bunzip2
  684. ;;
  685. *)
  686. cat
  687. ;;
  688. esac
  689. }
  690. export -f uncompress_file
  691. get_file() {
  692. local src="$1"
  693. fetch_file "$src" | uncompress_file "$src"
  694. }
  695. export -f get_file
  696. ##
  697. ## Common database lib
  698. ##
  699. _clean_docker() {
  700. local _DB_NAME="$1" container_id="$2"
  701. (
  702. set +e
  703. debug "Removing container $_DB_NAME"
  704. docker stop "$container_id"
  705. docker rm "$_DB_NAME"
  706. docker network rm "${_DB_NAME}"
  707. rm -vf "$state_tmpdir/${_DB_NAME}.state"
  708. ) >&2
  709. }
  710. export -f _clean_docker
  711. get_service_base_image_dir_uid_gid() {
  712. local service="$1" dir="$2" uid_gid
  713. uid_gid=$(cached_cmd_on_base_image "$service" "stat -c '%u %g' '$dir'") || {
  714. debug "Failed to query '$dir' uid in ${DARKYELLOW}$service${NORMAL} base image."
  715. return 1
  716. }
  717. info "uid and gid from ${DARKYELLOW}$service${NORMAL}:$dir is '$uid_gid'"
  718. echo "$uid_gid"
  719. }
  720. export -f get_service_base_image_dir_uid_gid
  721. get_service_type() {
  722. if [ -z "$CHARM_STORE_HASH" ]; then
  723. err-d "Expected \$CHARM_STORE_HASH to be set."
  724. return 1
  725. fi
  726. local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH"
  727. if [ -z "$service" ]; then
  728. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  729. return 1
  730. fi
  731. if [ -e "$cache_file" ]; then
  732. # debug "$FUNCNAME: cache hit ($*)"
  733. cat "$cache_file"
  734. return 0
  735. fi
  736. master_target_service="$(get_top_master_service_for_service "$service")" || return 1
  737. charm=$(get_service_charm "$master_target_service") || return 1
  738. metadata=$(charm.metadata "$charm") || return 1
  739. printf "%s" "$metadata" | shyaml get-value type service 2>/dev/null |
  740. tee "$cache_file"
  741. }
  742. export -f get_service_type
  743. are_files_locked_in_dir() {
  744. local dir="$1" device hdev ldev
  745. device=$(stat -c %d "$dir") || {
  746. err "Can't stat '$dir'."
  747. return 1
  748. }
  749. device=$(printf "%04x" $device)
  750. hdev=${device:0:2}
  751. ldev=${device:2:2}
  752. inodes=$(find "$dir" -printf ':%i:\n')
  753. found=
  754. while read -r inode; do
  755. debug "try inode:$inode"
  756. if [[ "$inodes" == *":$inode:"* ]]; then
  757. found=1
  758. break
  759. fi
  760. done < <(cat /proc/locks | grep " $hdev:$ldev:" | sed -r "s/^.*$hdev:$ldev:([0-9]+).*$/\1/g")
  761. [ "$found" ]
  762. }
  763. export -f are_files_locked_in_dir
  764. set_db_params() {
  765. local docker_ip="$1" docker_network="$2"
  766. if [ -z "$DB_PARAMS_LOADED" ]; then
  767. DB_PARAMS_LOADED=1
  768. _set_db_params "$docker_ip" "$docker_network"
  769. fi
  770. }
  771. export -f set_db_params
  772. export _PID="$$"
  773. ensure_db_docker_running () {
  774. local _STATE_FILE errlvl project
  775. _DB_NAME="db_${DB_NAME}_${_PID}"
  776. _STATE_FILE="$state_tmpdir/${_DB_NAME}.state"
  777. if [ -e "$_STATE_FILE" ]; then
  778. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$(cat "$_STATE_FILE")"
  779. debug "Re-using previous docker/connection '$DOCKER_IP'."
  780. set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  781. return 0
  782. fi
  783. if [ -e "$state_tmpdir/${_DB_NAME}.working" ]; then
  784. ## avoid recursive calls.
  785. if [ -z "$DOCKER_IP" ]; then
  786. err "Currently figuring up DOCKER_IP, please set it yourself before this call if needed."
  787. return 1
  788. else
  789. debug "ignoring recursive call of 'ensure_db_docker_running'."
  790. fi
  791. return 0
  792. fi
  793. touch "$state_tmpdir/${_DB_NAME}.working"
  794. docker rm "$_DB_NAME" 2>/dev/null || true
  795. host_db_working_dir="$HOST_DATASTORE/${SERVICE_NAME}$DB_DATADIR"
  796. if is_db_locked; then
  797. info "Some process is using '$host_db_working_dir'. Trying to find a docker that would do this..."
  798. found=
  799. for docker_id in $(docker ps -q); do
  800. has_volume_mounted=$(
  801. docker inspect \
  802. --format "{{range .Mounts}}{{if eq .Destination \"$DB_DATADIR\"}}{{.Source}}{{end}}{{end}}" \
  803. "$docker_id")
  804. if [ "$has_volume_mounted" == "$host_db_working_dir" ]; then
  805. info "docker '$docker_id' uses '$has_volume_mounted'."
  806. project=$(docker inspect "$docker_id" \
  807. --format "{{index .Config.Labels \"compose.project\" }}") || continue
  808. info "docker '$docker_id' is from project '$project' (current project is '$PROJECT_NAME')."
  809. [ "$project" == "$PROJECT_NAME" ] || continue
  810. found="$docker_id"
  811. break
  812. fi
  813. done
  814. if [ -z "$found" ]; then
  815. err "Please shutdown any other docker using this directory."
  816. return 1
  817. fi
  818. export container_id="$found"
  819. info "Found docker $docker_id is already running."
  820. else
  821. verb "Database is not locked."
  822. if ! docker_has_image "$DOCKER_BASE_IMAGE"; then
  823. err "Unexpected missing docker image $DOCKER_BASE_IMAGE."
  824. return 1
  825. fi
  826. _set_server_db_params || return 1
  827. debug docker network create "$_DB_NAME"
  828. if ! network_id=$(docker network create "$_DB_NAME"); then
  829. err "'docker network create $_DB_NAME' failed !"
  830. _clean_docker "$_DB_NAME" "$container_id"
  831. rm "$state_tmpdir/${_DB_NAME}.working"
  832. return 1
  833. fi
  834. debug docker run -d \
  835. --name "$_DB_NAME" \
  836. "${server_docker_opts[@]}" \
  837. --network "$_DB_NAME" \
  838. -v "$host_db_working_dir:$DB_DATADIR" \
  839. "$DOCKER_BASE_IMAGE"
  840. if ! container_id=$(
  841. docker run -d \
  842. --name "$_DB_NAME" \
  843. "${server_docker_opts[@]}" \
  844. --network "$_DB_NAME" \
  845. -v "$host_db_working_dir:$DB_DATADIR" \
  846. "$DOCKER_BASE_IMAGE"
  847. ); then
  848. err "'docker run' failed !"
  849. _clean_docker "$_DB_NAME" "$container_id"
  850. rm "$state_tmpdir/${_DB_NAME}.working"
  851. return 1
  852. fi
  853. trap_add EXIT,ERR "_clean_docker \"$_DB_NAME\" \"$container_id\""
  854. fi
  855. if docker_ip=$(wait_for_docker_ip "$container_id"); then
  856. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip"
  857. echo "$docker_ip" > "$_STATE_FILE"
  858. debug "written '$_STATE_FILE'"
  859. rm "$state_tmpdir/${_DB_NAME}.working"
  860. set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  861. return 0
  862. else
  863. errlvl="$?"
  864. err "Db not found (errlvl: $errlvl). Tail of docker logs follows:"
  865. docker logs --tail=5 "$container_id" 2>&1 | prefix " | " >&2
  866. rm "$state_tmpdir/${_DB_NAME}.working"
  867. return "$errlvl"
  868. fi
  869. }
  870. export -f ensure_db_docker_running
  871. ## Require to set $db_docker_opts if needed, and $DB_PASSFILE
  872. ##
  873. _dcmd() {
  874. local docker_opts command="$1"
  875. shift
  876. debug "Db> $command $@"
  877. if [ "$HOST_DB_PASSFILE" -a -f "$LOCAL_DB_PASSFILE" -a "$CLIENT_DB_PASSFILE" ]; then
  878. verb "Found and using '$HOST_DB_PASSFILE' as '$CLIENT_DB_PASSFILE'."
  879. docker_opts=("${db_docker_opts[@]}" "-v" "$HOST_DB_PASSFILE:$CLIENT_DB_PASSFILE")
  880. else
  881. docker_opts=("${db_docker_opts[@]}")
  882. fi
  883. ## XXXX was here: actualy, we need only connection between this version and the client version
  884. debug docker run -i --rm \
  885. "${docker_opts[@]}" \
  886. --entrypoint "$command" "$DOCKER_BASE_IMAGE" "${db_cmd_opts[@]}" "$@"
  887. docker run -i --rm \
  888. "${docker_opts[@]}" \
  889. --entrypoint "$command" "$DOCKER_BASE_IMAGE" "${db_cmd_opts[@]}" "$@"
  890. }
  891. export -f _dcmd
  892. ## Executes code through db
  893. dcmd() {
  894. local fun
  895. [ "$DB_NAME" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_NAME."
  896. [ "$DB_DATADIR" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_DATADIR."
  897. # [ "$DB_PASSFILE" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_PASSFILE."
  898. [ "$_PID" ] || print_syntax_error "$FUNCNAME: You must provide \$_PID."
  899. for fun in is_db_locked _set_db_params ddb; do
  900. [ "$(type -t "$fun")" == "function" ] ||
  901. print_syntax_error "$FUNCNAME: You must provide function '$fun'."
  902. done
  903. ensure_db_docker_running </dev/null || return 1
  904. _dcmd "$@"
  905. }
  906. export -f dcmd
  907. get_docker_ips() {
  908. local name="$1" ip format network_id
  909. if ! docker inspect --format='{{ .NetworkSettings.Networks }}' "$name" >/dev/null 2>&1; then
  910. echo "default:$(docker inspect --format='{{ .NetworkSettings.IPAddress }}' "$name" 2>/dev/null)"
  911. else
  912. format='{{range $name, $conf := .NetworkSettings.Networks}}{{$name}}{{"\x00"}}{{$conf.IPAddress}}{{"\x00"}}{{end}}'
  913. while read-0 network_id ip; do
  914. printf "%s:%s\n" "$network_id" "$ip"
  915. done < <(docker inspect --format="$format" "$name")
  916. fi
  917. }
  918. export -f get_docker_ips
  919. get_docker_ip() {
  920. local name="$1"
  921. get_docker_ips "$name"
  922. }
  923. export -f get_docker_ip
  924. wait_docker_ip() {
  925. local name="$1" timeout="${2:-15}" timeout_count=0 docker_ip=
  926. start=$SECONDS
  927. while [ -z "$docker_ip" ]; do
  928. sleep 0.5
  929. docker_ip=$(get_docker_ip "$name") && break
  930. elapsed=$((SECONDS - start))
  931. if ((elapsed > timeout)); then
  932. err "${RED}timeout error${NORMAL}(${timeout}s):" \
  933. "Could not find '$name' docker container's IP."
  934. return 1
  935. fi
  936. [ "$elapsed" == "$old_elapsed" ] ||
  937. verb "Waiting for docker $name... ($elapsed/$timeout)"
  938. old_elapsed="$elapsed"
  939. done
  940. verb "Found docker $name network and IP: $docker_ip"
  941. echo "$docker_ip"
  942. }
  943. export -f wait_docker_ip
  944. wait_for_tcp_port() {
  945. local network=$1 host_port=$2 timeout=${3:-60}
  946. verb "Trying to connect to $host_port"
  947. bash_image=${DEFAULT_BASH_IMAGE:-docker.0k.io/bash}
  948. #echo docker run --rm -i --network "$network" "$bash_image" >&2
  949. docker run --rm -i --network "$network" "$bash_image" <<EOF
  950. start=\$SECONDS
  951. while true; do
  952. timeout 1 bash -c "</dev/tcp/${host_port/://}" >/dev/null 2>&1 && break
  953. sleep 0.2
  954. if [ "\$((SECONDS - start))" -gt "$timeout" ]; then
  955. exit 1
  956. fi
  957. done
  958. exit 0
  959. EOF
  960. if [ "$?" != 0 ]; then
  961. err "${RED}timeout error${NORMAL}(${timeout}s):"\
  962. "Could not connect to $host_port."
  963. return 1
  964. fi
  965. return 0
  966. }
  967. export -f wait_for_tcp_port
  968. ## Warning: requires a ``ddb`` matching current database to be checked
  969. wait_for_docker_ip() {
  970. local name=$1 DOCKER_IP= DOCKER_NETWORK= docker_ips= docker_ip= elapsed timeout=10
  971. docker_ip=$(wait_docker_ip "$name" 5) || return 1
  972. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip"
  973. if ! str_is_ipv4 "$DOCKER_IP"; then
  974. err "internal 'wait_docker_ip' did not return a valid IP. Returned IP is '$DOCKER_IP'."
  975. return 1
  976. fi
  977. set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  978. while read-0 port; do
  979. IFS="/" read port type <<<"$port"
  980. [ "$type" == "tcp" ] || continue
  981. wait_for_tcp_port "$DOCKER_NETWORK" "$DOCKER_IP:${port}" || return 17
  982. info "Host/Port $DOCKER_IP:${port} checked ${GREEN}open${NORMAL}."
  983. ## XXXvlab: what to do with more than one port ?
  984. break
  985. done < <(image_exposed_ports_0 "$container_id")
  986. ## Checking direct connection
  987. timeout=120
  988. start=$SECONDS
  989. while true; do
  990. if err=$(echo "$check_command" | ddb 2>&1 >/dev/null); then
  991. break
  992. fi
  993. if ! [[ "$err" == *"the database system is starting up" ]]; then
  994. err "${RED}db connection error${NORMAL}:" \
  995. "Could not connect to db on $DOCKER_IP container's IP."
  996. echo " Note: IP up, TCP ports is(are) open" >&2
  997. if [ "$err" ]; then
  998. echo " Error:" >&2
  999. printf "%s\n" "$err" | prefix " ${RED}!${NORMAL} " >&2
  1000. fi
  1001. return 18
  1002. fi
  1003. debug "Got 'database system is starting up' error."
  1004. elapsed=$((SECONDS - start))
  1005. if ((elapsed > timeout)); then
  1006. err "${RED}db connection error${NORMAL}:"\
  1007. "Could not connect to db on $DOCKER_IP" \
  1008. "container's IP. (IP up, TCP ports is(are) open, sql answer after ${timeout}s)"
  1009. return 1
  1010. fi
  1011. sleep 0.2
  1012. done
  1013. echo "${DOCKER_NETWORK}:${DOCKER_IP}"
  1014. return 0
  1015. }
  1016. export -f wait_for_docker_ip
  1017. docker_add_host_declaration() {
  1018. local src_docker=$1 domain=$2 dst_docker=$3 dst_docker_ip= dst_docker_network
  1019. dst_docker_ip=$(wait_docker_ip "$dst_docker") || exit 1
  1020. IFS=: read dst_docker_ip dst_docker_network <<<"$dst_docker_ip"
  1021. docker exec -i "$src_docker" bash <<EOF
  1022. if cat /etc/hosts | grep -E "^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+$domain\$" > /dev/null 2>&1; then
  1023. sed -ri "s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+$domain\$/$dst_docker_ip $domain/g" /etc/hosts
  1024. else
  1025. echo "$dst_docker_ip $domain" >> /etc/hosts
  1026. fi
  1027. EOF
  1028. }
  1029. export -f docker_add_host_declaration
  1030. get_running_containers_for_service() {
  1031. local service="$1" project="$2"
  1032. project=${project:-$PROJECT_NAME}
  1033. [ -n "$project" ] || {
  1034. err "No project name was defined yet."
  1035. return 1
  1036. }
  1037. docker ps \
  1038. --filter label="compose.project=$project" \
  1039. --filter label="compose.master-service=$service" \
  1040. --format="{{.ID}}"
  1041. }
  1042. export -f get_running_containers_for_service
  1043. get_container_network_ips() {
  1044. local container="$1"
  1045. docker inspect "$container" \
  1046. --format='{{range $key, $val :=.NetworkSettings.Networks}}{{$key}}{{"\x00"}}{{$val.IPAddress}}{{"\x00"}}{{end}}'
  1047. }
  1048. export -f get_container_network_ips
  1049. get_container_network_ip() {
  1050. local container="$1"
  1051. while read-0 network ip; do
  1052. printf "%s\0" "$network" "$ip"
  1053. break
  1054. done < <(get_container_network_ips "$container")
  1055. }
  1056. export -f get_container_network_ip
  1057. ##
  1058. ## Internal Process
  1059. ##
  1060. get_docker_compose_links() {
  1061. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1062. deps master_service master_target_service _relation_name \
  1063. target_service _relation_config tech_dep
  1064. if [ -z "$service" ]; then
  1065. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  1066. return 1
  1067. fi
  1068. if [ -e "$cache_file" ]; then
  1069. # debug "$FUNCNAME: cache hit ($*)"
  1070. cat "$cache_file"
  1071. return 0
  1072. fi
  1073. master_service=$(get_top_master_service_for_service "$service") || return 1
  1074. deps=()
  1075. while read-0 _relation_name target_service _relation_config tech_dep; do
  1076. master_target_service="$(get_top_master_service_for_service "$target_service")" || return 1
  1077. [ "$master_service" == "$master_target_service" ] && continue
  1078. type="$(get_service_type "$target_service")" || return 1
  1079. [ "$type" == "stub" ] && continue
  1080. [ "$type" == "run-once" ] && continue
  1081. if [ "$tech_dep" == "reversed" ]; then
  1082. deps+=("$(echo -en "$master_target_service:\n links:\n - $master_service")")
  1083. elif [[ "$tech_dep" =~ ^(True|true)$ ]]; then
  1084. deps+=("$(echo -en "$master_service:\n links:\n - $master_target_service")")
  1085. fi
  1086. ## XXXvlab: an attempt to add depends_on, but this doesn't work well actually
  1087. ## as there's a circular dependency issue. We don't really want the full feature
  1088. ## of depends_on, but just to add it as targets when doing an 'up'
  1089. # deps+=("$(echo -en "$master_service:\n depends_on:\n - $master_target_service")")
  1090. done < <(get_service_relations "$service")
  1091. merge_yaml_str "${deps[@]}" | tee "$cache_file" || return 1
  1092. if [ "${PIPESTATUS[0]}" != 0 ]; then
  1093. rm "$cache_file"
  1094. err "Failed to merge YAML from all ${WHITE}links${NORMAL} dependencies."
  1095. return 1
  1096. fi
  1097. }
  1098. _get_docker_compose_opts() {
  1099. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1100. compose_def master_service docker_compose_opts
  1101. if [ -z "$service" ]; then
  1102. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  1103. return 1
  1104. fi
  1105. if [ -e "$cache_file" ]; then
  1106. # debug "$FUNCNAME: cache hit ($*)"
  1107. cat "$cache_file"
  1108. return 0
  1109. fi
  1110. compose_def="$(get_compose_service_def "$service")" || return 1
  1111. master_service="$(get_top_master_service_for_service "$service")"
  1112. if docker_compose_opts=$(echo "$compose_def" | shyaml get-value -y "docker-compose" 2>/dev/null); then
  1113. yaml_key_val_str "$master_service" "$docker_compose_opts"
  1114. fi | tee "$cache_file"
  1115. if [ "${PIPESTATUS[0]}" != 0 ]; then
  1116. rm "$cache_file"
  1117. return 1
  1118. fi
  1119. }
  1120. ##
  1121. ## By Reading the metadata.yml, we create a docker-compose.yml mixin.
  1122. ## Some metadata.yml (of subordinates) will indeed modify other
  1123. ## services than themselves.
  1124. _get_docker_compose_service_mixin() {
  1125. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1126. links_yaml base_mixin links_yaml docker_compose_options \
  1127. charm charm_part
  1128. if [ -z "$service" ]; then
  1129. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  1130. return 1
  1131. fi
  1132. if [ -e "$cache_file" ]; then
  1133. # debug "$FUNCNAME: cache hit ($*)"
  1134. cat "$cache_file"
  1135. return 0
  1136. fi
  1137. type=$(get_service_type "$service") || return 1
  1138. [ "$type" == "stub" ] && return 0
  1139. master_service=$(get_top_master_service_for_service "$service") || {
  1140. err "Failed to get top master service for service $DARKYELLOW$service$NORMAL"
  1141. return 1
  1142. }
  1143. ## The compose part
  1144. base_mixin="$master_service:
  1145. labels:
  1146. - compose.service=$service
  1147. - compose.master-service=${master_service}
  1148. - compose.project=$(get_default_project_name)"
  1149. links_yaml=$(get_docker_compose_links "$service") || return 1
  1150. docker_compose_options=$(_get_docker_compose_opts "$service") || return 1
  1151. ## the charm part
  1152. charm_part=$(get_docker_compose_mixin_from_metadata "$service") || return 1
  1153. ## Merge results
  1154. if [ "$charm_part" ]; then
  1155. charm_yaml="$(yaml_key_val_str "$master_service" "$charm_part")" || return 1
  1156. merge_yaml_str "$base_mixin" "$links_yaml" "$charm_yaml" "$docker_compose_options" || return 1
  1157. else
  1158. merge_yaml_str "$base_mixin" "$links_yaml" "$docker_compose_options" || return 1
  1159. fi | tee "$cache_file"
  1160. if [ "${PIPESTATUS[0]}" != 0 ]; then
  1161. err "Failed to constitute the base YAML for service '${DARKYELLOW}$service${NORMAL}'"
  1162. rm "$cache_file"
  1163. return 1
  1164. fi
  1165. }
  1166. export -f _get_docker_compose_service_mixin
  1167. ##
  1168. ## Get full `docker-compose.yml` format for all listed services (and
  1169. ## their deps)
  1170. ##
  1171. ## @export
  1172. ## @cache: !system !nofail +stdout
  1173. get_docker_compose () {
  1174. if [ -z "$CHARM_STORE_HASH" ]; then
  1175. err-d "Expected \$CHARM_STORE_HASH to be set."
  1176. return 1
  1177. fi
  1178. if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
  1179. err-d "Expected \$COMPOSE_YML_CONTENT_HASH to be set."
  1180. return 1
  1181. fi
  1182. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$CHARM_STORE_HASH" "$COMPOSE_YML_CONTENT_HASH")" \
  1183. entries services service start docker_compose_services
  1184. if [ -e "$cache_file" ]; then
  1185. # debug "$FUNCNAME: cache hit ($*) $cache_file"
  1186. touch "$cache_file" || return 1
  1187. cp "$cache_file"{,.wip} || return 1
  1188. export _CURRENT_DOCKER_COMPOSE="$cache_file.wip"
  1189. cat "$cache_file" || return 1
  1190. return 0
  1191. fi
  1192. ##
  1193. ## Adding sub services configurations
  1194. ##
  1195. declare -A entries
  1196. start_compilation=$SECONDS
  1197. debug "Compiling 'docker-compose.yml' base for ${DARKYELLOW}$*$NORMAL..."
  1198. for target_service in "$@"; do
  1199. start=$SECONDS
  1200. services=($(get_ordered_service_dependencies "$target_service")) || {
  1201. err "Failed to get dependencies for $DARKYELLOW$target_service$NORMAL"
  1202. return 1
  1203. }
  1204. if [ "$DEBUG" ]; then
  1205. debug " $DARKYELLOW$target_service$NORMAL deps:$DARKYELLOW" \
  1206. "${services[@]::$((${#services[@]} - 1))}" \
  1207. "$NORMAL$GRAY(in $((SECONDS - start))s)$NORMAL"
  1208. fi
  1209. for service in "${services[@]}"; do
  1210. if [ "${entries[$service]}" ]; then
  1211. ## Prevent double inclusion of same service if this
  1212. ## service is deps of two or more of your
  1213. ## requirements.
  1214. continue
  1215. fi
  1216. ## mark the service as "loaded" as well as it's containers
  1217. ## if this is a subordinate service
  1218. start_service=$SECONDS
  1219. entries[$service]=$(_get_docker_compose_service_mixin "$service") || {
  1220. err "Failed to get service mixin for $DARKYELLOW$service$NORMAL"
  1221. return 1
  1222. }
  1223. debug " Applied $DARKYELLOW$service$NORMAL charm metadata mixins $GRAY(in $((SECONDS - start_service))s)$NORMAL"
  1224. done
  1225. debug " ..finished all mixins for $DARKYELLOW$target_service$NORMAL $GRAY(in $((SECONDS - start))s)$NORMAL"
  1226. done
  1227. docker_compose_services=$(merge_yaml_str "${entries[@]}") || {
  1228. err "Failed to merge YAML services entries together."
  1229. return 1
  1230. }
  1231. base_v2="version: '2.1'"
  1232. merge_yaml_str "$(yaml_key_val_str "services" "$docker_compose_services")" \
  1233. "$base_v2" > "$cache_file" || return 1
  1234. cp "$cache_file"{,.wip} || return 1
  1235. export _CURRENT_DOCKER_COMPOSE="$cache_file.wip"
  1236. cat "$_CURRENT_DOCKER_COMPOSE" || return 1
  1237. debug " ..compilation of base 'docker-compose.yml' done $GRAY(in $((SECONDS - start_compilation))s)$NORMAL" || true
  1238. # debug " ** ${WHITE}docker-compose.yml${NORMAL}:"
  1239. # debug "$_current_docker_compose"
  1240. }
  1241. export -f get_docker_compose
  1242. _get_compose_service_def_cached () {
  1243. local service="$1" docker_compose="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  1244. if [ -e "$cache_file" ]; then
  1245. #debug "$FUNCNAME: STATIC cache hit"
  1246. cat "$cache_file" &&
  1247. touch "$cache_file" || return 1
  1248. return 0
  1249. fi
  1250. value=$(echo "$docker_compose" | shyaml get-value "${service//./\\.}" 2>/dev/null)
  1251. [ "$value" == None ] && value=""
  1252. if ! echo "$value" | shyaml get-value "charm" >/dev/null 2>&1; then
  1253. if charm.exists "$service"; then
  1254. value=$(merge_yaml <(echo "charm: $service") <(echo "$value")) || {
  1255. err "Can't merge YAML infered 'charm: $service' with base ${DARKYELLOW}$service${NORMAL} YAML definition."
  1256. return 1
  1257. }
  1258. else
  1259. err "No ${WHITE}charm${NORMAL} value for service $DARKYELLOW$service$NORMAL" \
  1260. "in compose, nor same name charm found."
  1261. return 1
  1262. fi
  1263. fi
  1264. echo "$value" | tee "$cache_file" || return 1
  1265. # if [ "${PIPESTATUS[0]}" != 0 ]; then
  1266. # rm "$cache_file"
  1267. # return 1
  1268. # fi
  1269. return 0
  1270. # if [ "${PIPESTATUS[0]}" != 0 -o \! -s "$cache_file" ]; then
  1271. # rm "$cache_file"
  1272. # err "PAS OK $service: $value"
  1273. # return 1
  1274. # fi
  1275. }
  1276. export -f _get_compose_service_def_cached
  1277. ## XXXvlab: a lot to be done to cache the results
  1278. get_compose_service_def () {
  1279. if [ -z "$COMBINED_HASH" ]; then
  1280. err-d "Expected \$COMBINED_HASH to be set."
  1281. return 1
  1282. fi
  1283. local service="$1" docker_compose cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \
  1284. result
  1285. if [ -e "$cache_file" ]; then
  1286. #debug "$FUNCNAME: SESSION cache hit"
  1287. cat "$cache_file" || return 1
  1288. return 0
  1289. fi
  1290. [ -z "$service" ] && print_syntax_error "Missing service as first argument."
  1291. docker_compose=$(get_compose_yml_content) || return 1
  1292. result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1
  1293. charm=$(e "$result" | shyaml get-value charm 2>/dev/null) || return 1
  1294. metadata=$(charm.metadata "$charm") || return 1
  1295. if default_options=$(printf "%s" "$metadata" | shyaml -y -q get-value default-options); then
  1296. default_options=$(yaml_key_val_str "options" "$default_options") || return 1
  1297. result=$(merge_yaml_str "$default_options" "$result") || return 1
  1298. fi
  1299. echo "$result" | tee "$cache_file" || return 1
  1300. }
  1301. export -f get_compose_service_def
  1302. _get_service_charm_cached () {
  1303. local service="$1" service_def="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  1304. if [ -e "$cache_file" ]; then
  1305. # debug "$FUNCNAME: cache hit $1"
  1306. cat "$cache_file" &&
  1307. touch "$cache_file" || return 1
  1308. return 0
  1309. fi
  1310. charm=$(echo "$service_def" | shyaml get-value charm 2>/dev/null)
  1311. if [ -z "$charm" ]; then
  1312. err "Missing ${WHITE}charm${NORMAL} value in service $DARKYELLOW$service$NORMAL definition."
  1313. return 1
  1314. fi
  1315. echo "$charm" | tee "$cache_file" || return 1
  1316. }
  1317. export -f _get_service_charm_cached
  1318. get_service_charm () {
  1319. local service="$1"
  1320. if [ -z "$service" ]; then
  1321. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  1322. return 1
  1323. fi
  1324. service_def=$(get_compose_service_def "$service") || return 1
  1325. _get_service_charm_cached "$service" "$service_def"
  1326. }
  1327. export -f get_service_charm
  1328. ## built above the docker-compose abstraction, so it relies on the
  1329. ## full docker-compose.yml to be already built.
  1330. get_service_def () {
  1331. local service="$1" def
  1332. if [ -z "$_CURRENT_DOCKER_COMPOSE" ]; then
  1333. err "${FUNCNAME[0]} is meant to be called after"\
  1334. "\$_CURRENT_DOCKER_COMPOSE has been calculated."
  1335. echo " Called by:" >&2
  1336. printf " - %s\n" "${FUNCNAME[@]:1}" >&2
  1337. return 1
  1338. fi
  1339. def=$(cat "$_CURRENT_DOCKER_COMPOSE" | shyaml get-value "services.${service//./\\.}" 2>/dev/null)
  1340. if [ -z "$def" ]; then
  1341. err "No definition for service $DARKYELLOW$service$NORMAL in compiled 'docker-compose.yml'."
  1342. return 1
  1343. fi
  1344. echo "$def"
  1345. }
  1346. export -f get_service_def
  1347. get_build_hash() {
  1348. local dir="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$1")" hash
  1349. if [ -e "$cache_file" ]; then
  1350. # debug "$FUNCNAME: cache hit ($*)"
  1351. cat "$cache_file"
  1352. return 0
  1353. fi
  1354. ## Check that there's a Dockerfile in this directory
  1355. if [ ! -e "$dir/Dockerfile" ]; then
  1356. err "No 'Dockerfile' found in '$dir'."
  1357. return 1
  1358. fi
  1359. ## use find to md5sum all files in the directory and make a final hash
  1360. hash=$(set -o pipefail; cd "$dir"; env -i find "." -type f -exec md5sum {} \; |
  1361. sort | md5sum | awk '{print $1}') || {
  1362. err "Failed to get hash for '$dir'."
  1363. return 1
  1364. }
  1365. printf "%s" "$hash" | tee "$cache_file"
  1366. return $?
  1367. }
  1368. export -f get_build_hash
  1369. ### Query/Get cached image from registry
  1370. ##
  1371. ## Returns on stdout the name of the image if found, or an empty string if not
  1372. cache:image:registry:get() {
  1373. local charm="$1" hash="$2" service="$3"
  1374. local charm_image_name="cache/charm/$charm"
  1375. local charm_image="$charm_image_name:$hash"
  1376. Elt "pulling ${DARKPINK}$charm${NORMAL} image from $COMPOSE_DOCKER_REGISTRY" >&2
  1377. if out=$(docker pull "$COMPOSE_DOCKER_REGISTRY/$charm_image" 2>&1); then
  1378. docker tag "$COMPOSE_DOCKER_REGISTRY/$charm_image" "$charm_image" || {
  1379. err "Failed set image '$COMPOSE_DOCKER_REGISTRY/$charm_image' as '$charm_image'" \
  1380. "for ${DARKYELLOW}$service${NORMAL}."
  1381. return 1
  1382. }
  1383. print_info "found" >&2
  1384. print_status success >&2
  1385. Feed >&2
  1386. printf "%s" "$charm_image" | tee "$cache_file"
  1387. return $?
  1388. fi
  1389. if [[ "$out" != *"manifest unknown"* ]] && [[ "$out" != *"not found"* ]]; then
  1390. print_status failure >&2
  1391. Feed >&2
  1392. err-d "Failed to pull image '$COMPOSE_DOCKER_REGISTRY/$charm_image'" \
  1393. "for ${DARKYELLOW}$service${NORMAL}:"
  1394. e "$out"$'\n' | prefix " ${GRAY}|${NORMAL} " >&2
  1395. return 1
  1396. fi
  1397. print_info "not found" >&2
  1398. if test "$type_method" = "long"; then
  1399. __status="[${NOOP}ABSENT${NORMAL}]"
  1400. else
  1401. echo -n "${NOOP}"
  1402. shift; shift;
  1403. echo -n "$*${NORMAL}"
  1404. fi >&2
  1405. Feed >&2
  1406. }
  1407. export -f cache:image:registry:get
  1408. ### Store cached image on registry
  1409. ##
  1410. ## Returns nothing
  1411. cache:image:registry:put() {
  1412. if [ -n "$COMPOSE_DOCKER_REGISTRY" ] && [ -n "$COMPOSE_PUSH_TO_REGISTRY" ]; then
  1413. local charm="$1" hash="$2" service="$3"
  1414. local charm_image_name="cache/charm/$charm"
  1415. local charm_image="$charm_image_name:$hash"
  1416. Wrap -d "pushing ${DARKPINK}$charm${NORMAL} image to $COMPOSE_DOCKER_REGISTRY" <<EOF || return 1
  1417. docker tag "$charm_image" "$COMPOSE_DOCKER_REGISTRY/$charm_image" &&
  1418. docker push "$COMPOSE_DOCKER_REGISTRY/$charm_image"
  1419. EOF
  1420. fi >&2
  1421. }
  1422. export -f cache:image:registry:put
  1423. ### Produce docker cached charm image 'cache/charm/$charm:$hash'
  1424. ##
  1425. ## Either by fetching it from a registry or by building it from a
  1426. ## Dockerfile.
  1427. cache:image:produce() {
  1428. local type="$1" src="$2" charm="$3" hash="$4" service="$5"
  1429. local charm_image_name="cache/charm/$charm"
  1430. local charm_image="$charm_image_name:$hash"
  1431. case "$type" in
  1432. fetch)
  1433. local specified_image="$src"
  1434. ## will not pull upstream image if already present locally
  1435. if ! docker_has_image "${specified_image}"; then
  1436. if ! out=$(docker pull "${specified_image}" 2>&1); then
  1437. err "Failed to pull image '$specified_image' for ${DARKYELLOW}$service${NORMAL}:"
  1438. echo "$out" | prefix " | " >&2
  1439. return 1
  1440. fi
  1441. fi
  1442. # specified_image_id=$(docker_image_id "$specified_image") || return 1
  1443. # charm_image_id=
  1444. # if docker_has_image "${image_dst}"; then
  1445. # charm_image_id=$(docker_image_id "${image_dst}") || return 1
  1446. # fi
  1447. # if [ "$specified_image_id" != "$charm_image_id" ]; then
  1448. docker tag "$specified_image" "${charm_image}" || return 1
  1449. # fi
  1450. ;;
  1451. build)
  1452. local service_build="$src"
  1453. build_opts=()
  1454. if [ "$COMPOSE_ACTION" == "build" ]; then
  1455. while read-0 arg; do
  1456. case "$arg" in
  1457. -t|--tag)
  1458. ## XXXvlab: doesn't seem to be actually a valid option
  1459. if [ -n "$COMPOSE_PUSH_TO_REGISTRY" ]; then
  1460. err "You can't use -t|--tag option when pushing to a registry."
  1461. exit 1
  1462. fi
  1463. has_named_image=true
  1464. read-0 val ## should always be okay because already checked
  1465. build_opts+=("$arg" "$val")
  1466. ;;
  1467. --help|-h)
  1468. docker-compose "$action" --help |
  1469. filter_docker_compose_help_message >&2
  1470. exit 0
  1471. ;;
  1472. --*|-*)
  1473. if str_pattern_matches "$arg" $DC_MATCH_MULTI; then
  1474. read-0 value
  1475. build_opts+=("$arg" "$value")
  1476. shift
  1477. elif str_pattern_matches "$arg" $DC_MATCH_SINGLE; then
  1478. build_opts+=("$arg")
  1479. else
  1480. err "Unexpected error while parsing a second time the build arguments."
  1481. fi
  1482. ;;
  1483. *)
  1484. ## Already parsed
  1485. build_opts+=("$arg")
  1486. ;;
  1487. esac
  1488. done < <(cla.normalize "${action_opts[@]}")
  1489. fi
  1490. if [ -z "$has_named_image" ]; then
  1491. build_opts+=(-t "${charm_image}")
  1492. fi
  1493. Wrap -v -d "Building ${DARKPINK}$charm${NORMAL}:$hash image" -- \
  1494. docker build "$service_build" -t "${charm_image}" "${build_opts[@]}" >&2 || {
  1495. err "Failed to build image '${charm_image}' for ${DARKYELLOW}$service${NORMAL}."
  1496. return 1
  1497. }
  1498. if [ -n "$has_named_image" ]; then
  1499. exit 0
  1500. fi
  1501. ;;
  1502. *)
  1503. err "Unknown type '$type'."
  1504. return 1
  1505. ;;
  1506. esac
  1507. }
  1508. export -f cache:image:produce
  1509. ## Will modify current $_CURRENT_DOCKER_COMPOSE file
  1510. service_ensure_image_ready() {
  1511. if [ -z "$COMBINED_HASH" ]; then
  1512. err "Expected \$COMBINED_HASH to be set."
  1513. return 1
  1514. fi
  1515. local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \
  1516. master_service service_def service_image service_build service_dockerfile image \
  1517. specified_image specified_image_id charm_image_name hash \
  1518. service_quoted
  1519. if [ -e "$cache_file" ]; then
  1520. #debug "$FUNCNAME: cache hit ($*)"
  1521. touch "$cache_file" || return 1
  1522. cat "$cache_file"
  1523. return 0
  1524. fi
  1525. if [ -z "$_CURRENT_DOCKER_COMPOSE" ]; then
  1526. err "${FUNCNAME[0]} is meant to be called after"\
  1527. "\$_CURRENT_DOCKER_COMPOSE has been calculated."
  1528. echo " Called by:" >&2
  1529. printf " - %s\n" "${FUNCNAME[@]:1}" >&2
  1530. return 1
  1531. fi
  1532. master_service="$(get_top_master_service_for_service "$service")" || {
  1533. err "Could not compute master service for service $DARKYELLOW$service$NORMAL."
  1534. return 1
  1535. }
  1536. if [ "$master_service" != "$service" ]; then
  1537. image=$(service_ensure_image_ready "$master_service") || return 1
  1538. printf "%s" "$image" | tee "$cache_file"
  1539. return $?
  1540. fi
  1541. ## check if \$_CURRENT_DOCKER_COMPOSE's service def is already correctly setup
  1542. local charm="$(get_service_charm "$service")" || return 1
  1543. local charm_image_name="cache/charm/$charm" || return 1
  1544. local service_def="$(get_service_def "$service")" || {
  1545. err "Could not get docker-compose service definition for $DARKYELLOW$service$NORMAL."
  1546. return 1
  1547. }
  1548. local service_quoted=${service//./\\.}
  1549. if specified_image=$(e "$service_def" | shyaml get-value image 2>/dev/null); then
  1550. if [[ "$specified_image" == "$charm_image_name:"* ]]; then
  1551. ## Assume we already did the change
  1552. printf "%s" "$specified_image" | tee "$cache_file"
  1553. return 0
  1554. fi
  1555. if [[ "$specified_image" == "${COMPOSE_DOCKER_REGISTRY}/"* ]]; then
  1556. if ! docker_has_image "${specified_image}"; then
  1557. Wrap "${wrap_opts[@]}" \
  1558. -v -d "pulling ${DARKPINK}$charm${NORMAL}'s specified image from $COMPOSE_DOCKER_REGISTRY" -- \
  1559. docker pull "${specified_image}" >&2 || return 1
  1560. else
  1561. if [ -n "$DEBUG" ]; then
  1562. Elt "using local ${DARKPINK}$charm${NORMAL}'s specified image from $COMPOSE_DOCKER_REGISTRY" >&2
  1563. print_status noop >&2
  1564. Feed >&2
  1565. fi
  1566. fi
  1567. ## Already on the cache server
  1568. printf "%s" "$specified_image" | tee "$cache_file"
  1569. return 0
  1570. fi
  1571. src="$specified_image"
  1572. hash=$(e "$specified_image"$'\n' | md5sum | cut -f 1 -d " ") || return 1
  1573. type=fetch
  1574. ## replace image by charm image
  1575. yq -i ".services.[\"${service_quoted}\"].image = \"${charm_image_name}:${hash}\"" \
  1576. "$_CURRENT_DOCKER_COMPOSE" || return 1
  1577. else
  1578. if ! src=$(echo "$service_def" | shyaml get-value build 2>/dev/null); then
  1579. err "Service $DARKYELLOW$service$NORMAL has no ${WHITE}image${NORMAL} nor ${WHITE}build${NORMAL} parameter."
  1580. echo "$service_def" >&2
  1581. return 1
  1582. fi
  1583. ## According to https://stackoverflow.com/questions/32230577 , if there's a build,
  1584. ## then the built image will get name ${project}_${service}
  1585. hash=$(get_build_hash "$src") || return 1
  1586. type=build
  1587. ## delete build key from service_def and add image to charm_image_name
  1588. yq -i "del(.services.[\"${service_quoted}\"].build) |
  1589. .services.[\"${service_quoted}\"].image = \"${charm_image_name}:${hash}\"" \
  1590. "$_CURRENT_DOCKER_COMPOSE" || return 1
  1591. fi
  1592. if [ "$COMPOSE_ACTION" != "build" ] && docker_has_image "${charm_image_name}:${hash}"; then
  1593. if [ -n "$DEBUG" ]; then
  1594. Elt "using ${DARKPINK}$charm${NORMAL}'s image from local cache" >&2
  1595. print_status noop >&2
  1596. Feed >&2
  1597. fi
  1598. cache:image:registry:put "$charm" "$hash" "$service" || return 1
  1599. printf "%s" "${charm_image_name}:${hash}" | tee "$cache_file"
  1600. return $?
  1601. fi
  1602. ## Can we pull it ? Let's check on $COMPOSE_DOCKER_REGISTRY
  1603. if [ "$COMPOSE_ACTION" != "build" ] && [ -n "$COMPOSE_DOCKER_REGISTRY" ]; then
  1604. img=$(cache:image:registry:get "$charm" "$hash" "$service") || {
  1605. err "Failed to get image '$charm_image_name:$hash' from registry for ${DARKYELLOW}$service${NORMAL}."
  1606. return 1
  1607. }
  1608. [ -n "$img" ] && {
  1609. printf "%s" "$img" | tee "$cache_file"
  1610. return $?
  1611. }
  1612. fi
  1613. cache:image:produce "$type" "$src" "$charm" "$hash" "$service" || return 1
  1614. cache:image:registry:put "$charm" "$hash" "$service" || return 1
  1615. printf "%s" "${charm_image_name}:$hash" | tee "$cache_file"
  1616. return $?
  1617. }
  1618. export -f service_ensure_image_ready
  1619. get_charm_relation_def () {
  1620. local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1621. relation_def metadata
  1622. if [ -e "$cache_file" ]; then
  1623. # debug "$FUNCNAME: cache hit ($*)"
  1624. cat "$cache_file"
  1625. return 0
  1626. fi
  1627. metadata="$(charm.metadata "$charm")" || return 1
  1628. relation_def="$(echo "$metadata" | shyaml get-value "provides.${relation_name}" 2>/dev/null)"
  1629. echo "$relation_def" | tee "$cache_file"
  1630. }
  1631. export -f get_charm_relation_def
  1632. get_charm_tech_dep_orientation_for_relation() {
  1633. local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1634. relation_def value
  1635. if [ -e "$cache_file" ]; then
  1636. # debug "$FUNCNAME: cache hit ($*)"
  1637. cat "$cache_file"
  1638. return 0
  1639. fi
  1640. relation_def=$(get_charm_relation_def "$charm" "$relation_name" 2>/dev/null)
  1641. value=$(echo "$relation_def" | shyaml get-value 'tech-dep' 2>/dev/null)
  1642. value=${value:-True}
  1643. printf "%s" "$value" | tee "$cache_file"
  1644. }
  1645. export -f get_charm_tech_dep_orientation_for_relation
  1646. get_service_relation_tech_dep() {
  1647. local service="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1648. charm tech_dep
  1649. if [ -e "$cache_file" ]; then
  1650. # debug "$FUNCNAME: cache hit ($*)"
  1651. cat "$cache_file"
  1652. return 0
  1653. fi
  1654. charm=$(get_service_charm "$service") || return 1
  1655. tech_dep="$(get_charm_tech_dep_orientation_for_relation "$charm" "$relation_name")" || return 1
  1656. printf "%s" "$tech_dep" | tee "$cache_file"
  1657. }
  1658. export -f get_service_relation_tech_dep
  1659. ##
  1660. ## Use compose file to get deps, and relation definition in metadata.yml
  1661. ## for tech-dep attribute.
  1662. get_service_deps() {
  1663. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH")"
  1664. if [ -e "$cache_file" ]; then
  1665. # debug "$FUNCNAME: cache hit ($*)"
  1666. cat "$cache_file"
  1667. return 0
  1668. fi
  1669. (
  1670. set -o pipefail
  1671. get_service_relations "$service" | \
  1672. while read-0 relation_name target_service _relation_config tech_dep; do
  1673. echo "$target_service"
  1674. done | tee "$cache_file"
  1675. ) || return 1
  1676. }
  1677. export -f get_service_deps
  1678. ## XXXvlab: cache was disabled because improper. Indeed, this needs to cache
  1679. ## 'depths' full state. Second, it should be
  1680. _rec_get_depth() {
  1681. local elt=$1 dep deps max
  1682. [ "${depths[$elt]}" ] && return 0
  1683. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$(declare -pA depths)" "$GLOBAL_ALL_RELATIONS_HASH")"
  1684. if [ -e "$cache_file.depths" ]; then
  1685. #debug "$FUNCNAME: cache hit ($*) - $cache_file.depths"
  1686. while read-0 k v; do
  1687. depths["$k"]="$v"
  1688. done < "$cache_file.depths"
  1689. while read-0 k v; do
  1690. visited["$k"]="$v"
  1691. done < "$cache_file.visited"
  1692. return 0
  1693. fi
  1694. visited[$elt]=1
  1695. #debug "Setting visited[$elt]"
  1696. #debug "Asking for $DARKYELLOW$elt$NORMAL dependencies"
  1697. deps=$(get_service_deps "$elt") || {
  1698. debug "Failed get_service_deps $elt"
  1699. return 1
  1700. }
  1701. # debug "$elt deps are:" $deps
  1702. max=0
  1703. for dep in $deps; do
  1704. [ "${visited[$dep]}" ] && {
  1705. #debug "Already computing $dep"
  1706. continue
  1707. }
  1708. _rec_get_depth "$dep" || return 1
  1709. #debug "Requesting depth[$dep]"
  1710. if (( ${depths[$dep]} > max )); then
  1711. max="${depths[$dep]}"
  1712. fi
  1713. done
  1714. # debug "Setting depth[$elt] to $((max + 1))"
  1715. depths[$elt]=$((max + 1))
  1716. array_kv_to_stdin depths > "$cache_file.depths"
  1717. array_kv_to_stdin visited > "$cache_file.visited"
  1718. # debug "DEPTHS: $(declare -pA depths)"
  1719. # debug "$FUNCNAME: caching hit ($*) - $cache_file"
  1720. }
  1721. export -f _rec_get_depth
  1722. get_ordered_service_dependencies() {
  1723. local services=("$@") cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH")" \
  1724. i value key heads depths visited
  1725. if [ -e "$cache_file" ]; then
  1726. # debug "$FUNCNAME: cache hit ($*)"
  1727. cat "$cache_file"
  1728. return 0
  1729. fi
  1730. #debug "Figuring ordered deps of $DARKYELLOW${services[@]}$NORMAL"
  1731. if [ -z "${services[*]}" ]; then
  1732. return 0
  1733. # print_syntax_error "$FUNCNAME: no arguments"
  1734. # return 1
  1735. fi
  1736. declare -A depths
  1737. declare -A visited
  1738. heads=("${services[@]}")
  1739. while [ "${#heads[@]}" != 0 ]; do
  1740. array_pop heads head
  1741. _rec_get_depth "$head" || return 1
  1742. done
  1743. i=0
  1744. while [ "${#depths[@]}" != 0 ]; do
  1745. for key in "${!depths[@]}"; do
  1746. value="${depths[$key]}"
  1747. if [ "$value" == "$i" ]; then
  1748. echo "$key"
  1749. unset depths[$key]
  1750. fi
  1751. done
  1752. ((i++))
  1753. done | tee "$cache_file"
  1754. }
  1755. export -f get_ordered_service_dependencies
  1756. ## Modify $_CURRENT_DOCKER_COMPOSE file, and fills cache
  1757. run_service_acquire_images () {
  1758. local service subservice subservices loaded
  1759. _CURRENT_DOCKER_COMPOSE_HASH=$(hash_get < "$_CURRENT_DOCKER_COMPOSE")
  1760. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$_CURRENT_DOCKER_COMPOSE_HASH" "$COMBINED_HASH")"
  1761. if [ -e "$cache_file" ]; then
  1762. # debug "$FUNCNAME: cache hit ($*)"
  1763. touch "$cache_file" || return 1
  1764. cp "$cache_file" "$_CURRENT_DOCKER_COMPOSE" || return 1
  1765. return 0
  1766. fi
  1767. declare -A loaded
  1768. for service in "$@"; do
  1769. subservices=$(get_ordered_service_dependencies "$service") || return 1
  1770. for subservice in $subservices; do
  1771. if [ "${loaded[$subservice]}" ]; then
  1772. ## Prevent double inclusion of same service if this
  1773. ## service is deps of two or more of your
  1774. ## requirements.
  1775. continue
  1776. fi
  1777. type=$(get_service_type "$subservice") || return 1
  1778. MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
  1779. if [ "$type" != "stub" ]; then
  1780. DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$MASTER_BASE_SERVICE_NAME") || return 1
  1781. fi
  1782. loaded[$subservice]=1
  1783. done
  1784. done
  1785. cp "$_CURRENT_DOCKER_COMPOSE" "$cache_file" || return 1
  1786. return 0
  1787. }
  1788. run_service_hook () {
  1789. local action="$1" service subservice subservices loaded
  1790. shift
  1791. declare -A loaded
  1792. for service in "$@"; do
  1793. subservices=$(get_ordered_service_dependencies "$service") || return 1
  1794. for subservice in $subservices; do
  1795. if [ "${loaded[$subservice]}" ]; then
  1796. ## Prevent double inclusion of same service if this
  1797. ## service is deps of two or more of your
  1798. ## requirements.
  1799. continue
  1800. fi
  1801. charm=$(get_service_charm "$subservice") || return 1
  1802. charm.has_hook "$charm" "$action" >/dev/null || continue
  1803. type=$(get_service_type "$subservice") || return 1
  1804. PROJECT_NAME=$(get_default_project_name) || return 1
  1805. MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
  1806. MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1
  1807. if [ "$type" != "stub" ]; then
  1808. DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$MASTER_BASE_SERVICE_NAME") || return 1
  1809. fi
  1810. Wrap "${wrap_opts[@]}" -d "running $YELLOW$action$NORMAL hook of $DARKYELLOW$subservice$NORMAL in charm $DARKPINK$charm$NORMAL" <<EOF || return 1
  1811. export DOCKER_BASE_IMAGE="$DOCKER_BASE_IMAGE"
  1812. export SERVICE_NAME=$subservice
  1813. export IMAGE_NAME=$(echo "${PROJECT_NAME}" | tr -d "_-")_\${SERVICE_NAME}
  1814. export CONTAINER_NAME=\${IMAGE_NAME}_1
  1815. export CHARM_NAME="$charm"
  1816. export PROJECT_NAME="$PROJECT_NAME"
  1817. export SERVICE_DATASTORE="$DATASTORE/$subservice"
  1818. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$subservice"
  1819. export MASTER_BASE_SERVICE_NAME="$MASTER_BASE_SERVICE_NAME"
  1820. export MASTER_BASE_CHARM_NAME="$MASTER_BASE_CHARM_NAME"
  1821. charm.run_hook "local" "$charm" "$action"
  1822. EOF
  1823. loaded[$subservice]=1
  1824. done
  1825. done
  1826. return 0
  1827. }
  1828. host_resource_get() {
  1829. local location="$1" cfg="$2"
  1830. type=$(echo "$cfg" | shyaml get-value type 2>/dev/null) || {
  1831. err "Missing ${WHITE}type$NORMAL option in ${WHITE}get$NORMAL config for location '$location'"
  1832. return 1
  1833. }
  1834. if fn.exists host_resource_get_$type; then
  1835. host_resource_get_$type "$location" "$cfg"
  1836. else
  1837. err "Source ${WHITE}source$NORMAL type '$type' unknown for" \
  1838. "${WHITE}host-resource$NORMAL '$location' defined in" \
  1839. "$DARKYELLOW$subservice$NORMAL config."
  1840. return 1
  1841. fi
  1842. }
  1843. export -f host_resource_get
  1844. host_resource_get_git() {
  1845. local location="$1" cfg="$2" branch parent url
  1846. branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null)
  1847. branch=${branch:-master}
  1848. url=$(echo "$cfg" | shyaml get-value url 2>/dev/null)
  1849. parent="$(dirname "$location")"
  1850. (
  1851. mkdir -p "$parent" && cd "$parent" &&
  1852. git clone -b "$branch" "$url" "$(basename "$location")"
  1853. ) || return 1
  1854. }
  1855. export -f host_resource_get_git
  1856. host_resource_get_git-sub() {
  1857. local location="$1" cfg="$2" branch parent url
  1858. branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null)
  1859. branch=${branch:-master}
  1860. url=$(echo "$cfg" | shyaml get-value url 2>/dev/null)
  1861. parent="$(dirname "$location")"
  1862. (
  1863. mkdir -p "$parent" && cd "$parent" &&
  1864. git sub clone -b "$branch" "$url" "$(basename "$location")"
  1865. ) || return 1
  1866. }
  1867. export -f host_resource_get_git-sub
  1868. setup_host_resource () {
  1869. local subservice="$1" service_def location get cfg
  1870. service_def=$(get_compose_service_def "$subservice") || return 1
  1871. while read-0 location cfg; do
  1872. ## XXXvlab: will it be a git resources always ?
  1873. if [ -d "$location" -a ! -d "$location/.git" ]; then
  1874. err "Hum, location '$location' does not seem to be a git directory."
  1875. return 1
  1876. fi
  1877. if [ -d "$location" ]; then
  1878. info "host resource '$location' already set up."
  1879. continue
  1880. fi
  1881. get=$(echo "$cfg" | shyaml get-value get 2>/dev/null)
  1882. if [ -z "$get" ]; then
  1883. err "No host directory '$location' found, and no ${WHITE}source$NORMAL" \
  1884. "specified for $DARKYELLOW$subservice$NORMAL."
  1885. return 1
  1886. fi
  1887. host_resource_get "$location" "$get" || return 1
  1888. done < <(echo "$service_def" | shyaml key-values-0 host-resources 2>/dev/null)
  1889. }
  1890. export -f setup_host_resource
  1891. setup_host_resources () {
  1892. local service subservices subservice loaded
  1893. declare -A loaded
  1894. for service in "$@"; do
  1895. subservices=$(get_ordered_service_dependencies "$service") || return 1
  1896. for subservice in $subservices; do
  1897. if [ "${loaded[$subservice]}" ]; then
  1898. ## Prevent double inclusion of same service if this
  1899. ## service is deps of two or more of your
  1900. ## requirements.
  1901. continue
  1902. fi
  1903. setup_host_resource "$subservice" || return 1
  1904. loaded[$subservice]=1
  1905. done
  1906. done
  1907. return 0
  1908. }
  1909. export -f setup_host_resources
  1910. ## Works on stdin
  1911. cfg-get-value () {
  1912. local key="$1" out
  1913. if [ -z "$key" ]; then
  1914. yaml_get_interpret || return 1
  1915. return 0
  1916. fi
  1917. if ! out=$(shyaml -y get-value "$key" 2>/dev/null); then
  1918. err "The key $WHITE$key$NORMAL was not found in relation's data."
  1919. return 1
  1920. fi
  1921. printf "%s\n" "$out" | yaml_get_interpret
  1922. }
  1923. export -f cfg-get-value
  1924. relation-get () {
  1925. if [ -z "$RELATION_DATA_FILE" ]; then
  1926. err-d "$FUNCNAME: var \$RELATION_DATA_FILE is not set."
  1927. return 1
  1928. fi
  1929. cfg-get-value "$1" < "$RELATION_DATA_FILE"
  1930. }
  1931. export -f relation-get
  1932. expand_vars() {
  1933. local unlikely_prefix="UNLIKELY_PREFIX"
  1934. content=$(cat -)
  1935. ## find first identifier not in content
  1936. remaining_lines=$(echo "$content" | grep "^$unlikely_prefix")
  1937. while [ "$(echo "$remaining_lines" | grep "^$unlikely_prefix$")" ]; do
  1938. size_prefix="${#unlikely_prefix}"
  1939. first_matching=$(echo "$remaining_lines" |
  1940. grep -v "^$unlikely_prefix$" |
  1941. uniq -w "$((size_prefix + 1))" -c |
  1942. sort -rn |
  1943. head -n 1)
  1944. first_matching=${first_matching#"${x%%[![:space:]]*}"}
  1945. first_matching="${first_matching#* }"
  1946. next_char=${first_matching:$size_prefix:1}
  1947. if [ "$next_char" != "0" ]; then
  1948. unlikely_prefix+="0"
  1949. else
  1950. unlikely_prefix+="1"
  1951. fi
  1952. remaining_lines=$(echo "$remaining_lines" | grep "^$unlikely_prefix")
  1953. done
  1954. eval "cat <<$unlikely_prefix
  1955. $content
  1956. $unlikely_prefix"
  1957. }
  1958. export -f expand_vars
  1959. yaml_get_interpret() {
  1960. local content tag
  1961. content=$(cat -)
  1962. tag=$(echo "$content" | shyaml get-type) || return 1
  1963. content=$(echo "$content" | shyaml get-value) || return 1
  1964. if ! [ "${tag:0:1}" == "!" ]; then
  1965. echo "$content" || return 1
  1966. return 0
  1967. fi
  1968. case "$tag" in
  1969. "!bash-stdout")
  1970. echo "$content" | bash || {
  1971. err "shell code didn't end with errorlevel 0"
  1972. return 1
  1973. }
  1974. ;;
  1975. "!var-expand")
  1976. echo "$content" | expand_vars || {
  1977. err "shell expansion failed"
  1978. return 1
  1979. }
  1980. ;;
  1981. "!file-content")
  1982. source=$(echo "$content" | expand_vars) || {
  1983. err "shell expansion failed"
  1984. return 1
  1985. }
  1986. cat "$source" || return 1
  1987. ;;
  1988. *)
  1989. err "Invalid object tag ${WHITE}$tag${NORMAL}"
  1990. return 1
  1991. ;;
  1992. esac
  1993. }
  1994. export -f yaml_get_interpret
  1995. options-get () {
  1996. local key="$1" out
  1997. service_def=$(get_compose_service_def "$SERVICE_NAME") || return 1
  1998. if ! out=$(echo "$service_def" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1999. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  2000. return 1
  2001. fi
  2002. echo "$out" | yaml_get_interpret
  2003. }
  2004. export -f options-get
  2005. relation-base-compose-get () {
  2006. local key="$1" out
  2007. if ! out=$(echo "$RELATION_BASE_COMPOSE_DEF" | shyaml -y get-value "options.$key" 2>/dev/null); then
  2008. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  2009. return 1
  2010. fi
  2011. echo "$out" | yaml_get_interpret
  2012. }
  2013. export -f relation-base-compose-get
  2014. relation-target-compose-get () {
  2015. local key="$1" out
  2016. if ! out=$(echo "$RELATION_TARGET_COMPOSE_DEF" | shyaml -y get-value "options.$key" 2>/dev/null); then
  2017. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  2018. return 1
  2019. fi
  2020. echo "$out" | yaml_get_interpret
  2021. }
  2022. export -f relation-target-compose-get
  2023. relation-set () {
  2024. local key="$1" value="$2"
  2025. if [ -z "$RELATION_DATA_FILE" ]; then
  2026. err "$FUNCNAME: relation does not seems to be correctly setup."
  2027. return 1
  2028. fi
  2029. if ! [ -r "$RELATION_DATA_FILE" ]; then
  2030. err "$FUNCNAME: can't read relation's data." >&2
  2031. return 1
  2032. fi
  2033. _config_merge "$RELATION_DATA_FILE" <(yaml_key_val_str "$key" "$value")
  2034. }
  2035. export -f relation-set
  2036. _config_merge() {
  2037. local config_filename="$1" mixin="$2"
  2038. touch "$config_filename" &&
  2039. merge_yaml "$config_filename" "$mixin" > "$config_filename.tmp" || return 1
  2040. mv "$config_filename.tmp" "$config_filename"
  2041. }
  2042. export -f _config_merge
  2043. ## XXXvlab; this can be used only in relation, I'd like to use it in init.
  2044. config-add() {
  2045. local metadata="$1"
  2046. _config_merge "$RELATION_CONFIG" <(echo "$metadata")
  2047. }
  2048. export -f config-add
  2049. ## XXXvlab; this can be used only in relation, I'd like to use it in init.
  2050. init-config-add() {
  2051. local metadata="$1"
  2052. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" \
  2053. <(yaml_key_val_str "services" "$metadata")
  2054. }
  2055. export -f init-config-add
  2056. docker_get_uid() {
  2057. local service="$1" user="$2" uid
  2058. uid=$(cached_cmd_on_base_image "$service" "id -u \"$user\"") || {
  2059. debug "Failed to query for '$user' uid in ${DARKYELLOW}$service${NORMAL} base image."
  2060. return 1
  2061. }
  2062. info "uid from ${DARKYELLOW}$service${NORMAL} for user '$user' is '$uid'"
  2063. echo "$uid"
  2064. }
  2065. export -f docker_get_uid
  2066. docker_get_uid_gid() {
  2067. local service="$1" user="$2" group="$3" uid
  2068. uid_gid=$(cached_cmd_on_base_image "$service" "id -u \"$user\"; id -g \"$group\"") || {
  2069. debug "Failed to query for '$user' uid in ${DARKYELLOW}$service${NORMAL} base image."
  2070. return 1
  2071. }
  2072. info "uid from ${DARKYELLOW}$service${NORMAL} for user '$user' is '$uid_gid'"
  2073. echo "$uid_gid"
  2074. }
  2075. export -f docker_get_uid_gid
  2076. logstdout() {
  2077. local name="$1"
  2078. sed -r 's%^%'"${name}"'> %g'
  2079. }
  2080. export -f logstdout
  2081. logstderr() {
  2082. local name="$1"
  2083. sed -r 's%^(.*)$%'"${RED}${name}>${NORMAL} \1"'%g'
  2084. }
  2085. export -f logstderr
  2086. _run_service_relation () {
  2087. local relation_name="$1" service="$2" target_service="$3" relation_config="$4" relation_dir services
  2088. local errlvl
  2089. charm=$(get_service_charm "$service") || return 1
  2090. target_charm=$(get_service_charm "$target_service") || return 1
  2091. base_script_name=$(charm.has_relation_hook "$charm" "$relation_name" relation-joined) || true
  2092. target_script_name=$(charm.has_relation_hook "$target_charm" "$relation_name" relation-joined) || true
  2093. [ -n "$base_script_name" ] || [ -n "$target_script_name" ] || return 0
  2094. relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1
  2095. RELATION_DATA_FILE=$(get_relation_data_file "$service" "$target_service" "$relation_name" "$relation_config") || return 1
  2096. export BASE_SERVICE_NAME=$service
  2097. export BASE_CHARM_NAME=$charm
  2098. export BASE_CHARM_PATH=$(charm.get_dir "$charm")
  2099. export TARGET_SERVICE_NAME=$target_service
  2100. export TARGET_CHARM_NAME=$target_charm
  2101. export TARGET_CHARM_PATH=$(charm.get_dir "$target_charm")
  2102. export RELATION_DATA_FILE
  2103. target_errlvl=0
  2104. if [ -z "$target_script_name" ]; then
  2105. verb "No relation script $DARKBLUE$relation_name$NORMAL in target $DARKPINK$target_charm$NORMAL."
  2106. else
  2107. verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \
  2108. "for target $DARKYELLOW$target_service$NORMAL (charm $DARKPINK$target_charm$NORMAL)"
  2109. RELATION_CONFIG="$relation_dir/config_provider"
  2110. type=$(get_service_type "$target_service") || return 1
  2111. if [ "$type" != "stub" ]; then
  2112. DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$target_service") || return 1
  2113. fi
  2114. export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA
  2115. {
  2116. (
  2117. SERVICE_NAME=$target_service
  2118. SERVICE_DATASTORE="$DATASTORE/$target_service"
  2119. SERVICE_CONFIGSTORE="$CONFIGSTORE/$target_service"
  2120. export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE
  2121. charm.run_relation_hook local "$target_charm" "$relation_name" relation-joined
  2122. echo "$?" > "$relation_dir/target_errlvl"
  2123. ) | logstdout "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}"
  2124. } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@${NORMAL}" 3>&1 1>&2 2>&3
  2125. target_errlvl="$(cat "$relation_dir/target_errlvl")" || {
  2126. err "Relation script '$script_name' in $DARKPINK$target_charm$NORMAL" \
  2127. "failed before outputing an errorlevel."
  2128. ((target_errlvl |= "1" ))
  2129. }
  2130. if [ -e "$RELATION_CONFIG" ]; then
  2131. debug "Merging some new config info in $DARKYELLOW$target_service$NORMAL"
  2132. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" &&
  2133. rm "$RELATION_CONFIG"
  2134. ((target_errlvl |= "$?"))
  2135. fi
  2136. fi
  2137. if [ "$target_errlvl" == 0 ]; then
  2138. errlvl=0
  2139. if [ "$base_script_name" ]; then
  2140. verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \
  2141. "for $DARKYELLOW$service$NORMAL (charm $DARKPINK$charm$NORMAL)"
  2142. RELATION_CONFIG="$relation_dir/config_providee"
  2143. RELATION_DATA="$(cat "$RELATION_DATA_FILE")"
  2144. DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$service") || return 1
  2145. export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA
  2146. {
  2147. (
  2148. SERVICE_NAME=$service
  2149. SERVICE_DATASTORE="$DATASTORE/$service"
  2150. SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  2151. export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE
  2152. charm.run_relation_hook local "$charm" "$relation_name" relation-joined
  2153. echo "$?" > "$relation_dir/errlvl"
  2154. ) | logstdout "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}"
  2155. } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@$NORMAL" 3>&1 1>&2 2>&3
  2156. errlvl="$(cat "$relation_dir/errlvl")" || {
  2157. err "Relation script '$script_name' in $DARKPINK$charm$NORMAL" \
  2158. "failed before outputing an errorlevel."
  2159. ((errlvl |= "1" ))
  2160. }
  2161. if [ -e "$RELATION_CONFIG" ]; then
  2162. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" &&
  2163. rm "$RELATION_CONFIG"
  2164. ((errlvl |= "$?" ))
  2165. fi
  2166. if [ "$errlvl" != 0 ]; then
  2167. err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$service$NORMAL failed to run properly."
  2168. fi
  2169. else
  2170. verb "No relation script '$script_name' in charm $DARKPINK$charm$NORMAL. Ignoring."
  2171. fi
  2172. else
  2173. err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$target_service$NORMAL failed to run properly."
  2174. fi
  2175. if [ "$target_errlvl" == 0 -a "$errlvl" == 0 ]; then
  2176. debug "Relation $DARKBLUE$relation_name$NORMAL is established" \
  2177. "between $DARKYELLOW$service$NORMAL and $DARKYELLOW$target_service$NORMAL."
  2178. return 0
  2179. else
  2180. return 1
  2181. fi
  2182. }
  2183. export -f _run_service_relation
  2184. _get_compose_relations_cached () {
  2185. local compose_service_def="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  2186. relation_name relation_def target_service
  2187. if [ -e "$cache_file" ]; then
  2188. #debug "$FUNCNAME: STATIC cache hit $1"
  2189. cat "$cache_file" &&
  2190. touch "$cache_file" || return 1
  2191. return 0
  2192. fi
  2193. (
  2194. set -o pipefail
  2195. if [ "$compose_service_def" ]; then
  2196. while read-0 relation_name relation_def; do
  2197. ## XXXvlab: could we use braces here instead of parenthesis ?
  2198. (
  2199. case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in
  2200. "str")
  2201. target_service="$(echo "$relation_def" | shyaml get-value 2>/dev/null)" || return 1
  2202. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  2203. printf "%s\0" "$relation_name" "$target_service" "" "$tech_dep"
  2204. ;;
  2205. "sequence")
  2206. while read-0 target_service; do
  2207. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  2208. printf "%s\0" "$relation_name" "$target_service" "" "$tech_dep"
  2209. done < <(echo "$relation_def" | shyaml get-values-0 2>/dev/null)
  2210. ;;
  2211. "struct")
  2212. while read-0 target_service relation_config; do
  2213. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  2214. printf "%s\0" "$relation_name" "$target_service" "$relation_config" "$tech_dep"
  2215. done < <(echo "$relation_def" | shyaml key-values-0 2>/dev/null)
  2216. ;;
  2217. esac
  2218. ) </dev/null >> "$cache_file" || return 1
  2219. done < <(echo "$compose_service_def" | shyaml key-values-0 relations 2>/dev/null)
  2220. fi
  2221. )
  2222. if [ "$?" != 0 ]; then
  2223. err "Error while looking for compose relations."
  2224. rm -f "$cache_file" ## no cache
  2225. return 1
  2226. fi
  2227. [ -e "$cache_file" ] && cat "$cache_file"
  2228. return 0
  2229. }
  2230. export -f _get_compose_relations_cached
  2231. get_compose_relations () {
  2232. if [ -z "$COMBINED_HASH" ]; then
  2233. err-d "Expected \$COMBINED_HASH to be set."
  2234. return 1
  2235. fi
  2236. local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$COMBINED_HASH" \
  2237. compose_def
  2238. if [ -e "$cache_file" ]; then
  2239. #debug "$FUNCNAME: SESSION cache hit $1"
  2240. cat "$cache_file"
  2241. return 0
  2242. fi
  2243. compose_def="$(get_compose_service_def "$service")" || return 1
  2244. _get_compose_relations_cached "$compose_def" > "$cache_file"
  2245. if [ "$?" != 0 ]; then
  2246. rm -f "$cache_file" ## no cache
  2247. return 1
  2248. fi
  2249. cat "$cache_file"
  2250. }
  2251. export -f get_compose_relations
  2252. get_all_services() {
  2253. local services compose_yml_services service
  2254. if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then
  2255. err-d "Can't access global \$GLOBAL_ALL_RELATIONS_HASH"
  2256. return 1
  2257. fi
  2258. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$GLOBAL_ALL_RELATIONS_HASH" "$(declare -f "$FUNCNAME")")" \
  2259. s rn ts rc td services service
  2260. if [ -e "$cache_file" ]; then
  2261. #debug "$FUNCNAME: cache hit $1"
  2262. cat "$cache_file"
  2263. return 0
  2264. fi
  2265. if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
  2266. err-d "Can't access global \$GLOBAL_ALL_RELATIONS"
  2267. return 1
  2268. fi
  2269. declare -A services
  2270. while read-0 s _ ts _ _; do
  2271. for service in "$s" "$ts"; do
  2272. [ "${services[$service]}" ] && continue
  2273. services["$service"]=1
  2274. echo "$service"
  2275. done
  2276. done < "$GLOBAL_ALL_RELATIONS" > "$cache_file.wip"
  2277. compose_yml_services=($(compose:yml:root:services)) || return 1
  2278. for service in "${compose_yml_services[@]}"; do
  2279. [ "${services[$service]}" ] && continue
  2280. services["$service"]=1
  2281. echo "$service"
  2282. done >> "$cache_file.wip"
  2283. mv "$cache_file"{.wip,} || return 1
  2284. cat "$cache_file"
  2285. }
  2286. export -f get_all_services
  2287. get_service_relations () {
  2288. if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
  2289. err-d "Can't access global \$GLOBAL_ALL_RELATIONS"
  2290. return 1
  2291. fi
  2292. local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$GLOBAL_ALL_RELATIONS_HASH" \
  2293. s rn ts rc td
  2294. if [ -e "$cache_file" ]; then
  2295. #debug "$FUNCNAME: SESSION cache hit $1"
  2296. cat "$cache_file"
  2297. return 0
  2298. fi
  2299. while read-0 s rn ts rc td; do
  2300. [[ "$s" == "$service" ]] || continue
  2301. printf "%s\0" "$rn" "$ts" "$rc" "$td"
  2302. done < <(cat "$GLOBAL_ALL_RELATIONS") > "$cache_file"
  2303. cat "$cache_file"
  2304. }
  2305. export -f get_service_relations
  2306. get_service_relation() {
  2307. local service="$1" relation="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  2308. rn ts rc td
  2309. if [ -e "$cache_file" ]; then
  2310. #debug "$FUNCNAME: SESSION cache hit $1"
  2311. cat "$cache_file"
  2312. return 0
  2313. fi
  2314. while read-0-err E rn ts rc td; do
  2315. [ "$relation" == "$rn" ] && {
  2316. printf "%s\0" "$ts" "$rc" "$td"
  2317. break
  2318. }
  2319. done < <(p-err get_service_relations "$service") > "${cache_file}.wip"
  2320. if [ "$?" != 0 ]; then
  2321. return 1
  2322. fi
  2323. if [ "$E" != 0 ]; then
  2324. return 1
  2325. fi
  2326. mv "${cache_file}"{.wip,} || return 1
  2327. cat "$cache_file"
  2328. }
  2329. export -f get_service_relation
  2330. ## From a service and a relation, get all relations targeting given
  2331. ## service with given relation.
  2332. ##
  2333. ## Returns a NUL separated list of couple of:
  2334. ## (base_service, relation_config)
  2335. ##
  2336. get_service_incoming_relations() {
  2337. if [ -z "$SUBSET_ALL_RELATIONS_HASH" ]; then
  2338. err-d "Expected \$SUBSET_ALL_RELATIONS_HASH to be set."
  2339. return 1
  2340. fi
  2341. local service="$1" relation="$2" \
  2342. cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$@" "$SUBSET_ALL_RELATIONS_HASH")" \
  2343. s rn ts rc td
  2344. if [ -e "$cache_file" ]; then
  2345. #debug "$FUNCNAME: SESSION cache hit $1"
  2346. cat "$cache_file"
  2347. return 0
  2348. fi
  2349. while read-0 s rn ts rc _td; do
  2350. [[ "$ts" == "$service" ]] || continue
  2351. [[ "$rn" == "$relation" ]] || continue
  2352. relation_data_file=$(get_relation_data_file "$s" "$ts" "$rn" "$rc") || return 1
  2353. printf "%s\0" "$s" "$(cat "$relation_data_file")" || return 1
  2354. debug "Found relation $rn from $s to $ts" >&2
  2355. done < "$SUBSET_ALL_RELATIONS" > "$cache_file.wip"
  2356. mv "$cache_file"{.wip,} || return 1
  2357. cat "$cache_file"
  2358. }
  2359. export -f get_service_incoming_relations
  2360. export TRAVERSE_SEPARATOR=:
  2361. ## Traverse on first service satisfying relation
  2362. service:traverse() {
  2363. local service_path="$1"
  2364. {
  2365. SEPARATOR=:
  2366. read -d "$TRAVERSE_SEPARATOR" service
  2367. while read -d "$TRAVERSE_SEPARATOR" relation; do
  2368. ## XXXvlab: Take only first service
  2369. if ! read-0 ts _ _ < <(get_service_relation "${service}" "${relation}"); then
  2370. err "Couldn't find relation ${DARKCYAN}${relation}${NORMAL}" \
  2371. "from ${DARKYELLOW}$service${NORMAL}."
  2372. return 1
  2373. fi
  2374. service="$ts"
  2375. done
  2376. echo "$service"
  2377. } < <(e "${service_path}${TRAVERSE_SEPARATOR}")
  2378. }
  2379. export -f service:traverse
  2380. service:relation-file() {
  2381. local service_path="$1" relation service relation_file
  2382. if ! [[ "$service_path" == *"$TRAVERSE_SEPARATOR"* ]]; then
  2383. err "Invalid argument '$service_path'." \
  2384. "Must provide a service path (no '${TRAVERSE_SEPARATOR}' found)."
  2385. return 1
  2386. fi
  2387. relation="${service_path##*${TRAVERSE_SEPARATOR}}"
  2388. service=$(service:traverse "${service_path%${TRAVERSE_SEPARATOR}*}") || return 1
  2389. if ! read-0 ts rc _ < <(get_service_relation "${service}" "${relation}"); then
  2390. err "Couldn't find relation ${DARKCYAN}${relation}${NORMAL}" \
  2391. "from ${DARKYELLOW}$service${NORMAL}."
  2392. return 1
  2393. fi
  2394. relation_dir=$(get_relation_data_dir "$service" "$ts" "$relation") || {
  2395. err "Failed to find relation file"
  2396. return 1
  2397. }
  2398. relation_file="$relation_dir/data"
  2399. if ! [ -e "$relation_file" ]; then
  2400. e "$rc" > "$relation_file"
  2401. chmod go-rwx "$relation_file" ## protecting this file
  2402. fi
  2403. echo "$relation_file"
  2404. }
  2405. export -f service:relation-file
  2406. service:relation-options() {
  2407. local service_path="$1" relation_file
  2408. relation_file=$(service:relation-file "$service_path") || {
  2409. err "Failed to find relation file"
  2410. return 1
  2411. }
  2412. cat "$relation_file"
  2413. }
  2414. export -f service:relation-options
  2415. relation:get() {
  2416. local service_path="$1" query="$2" relation_file
  2417. relation_file=$(service:relation-file "$service_path") || {
  2418. err "Failed to find relation file"
  2419. return 1
  2420. }
  2421. cfg-get-value "$query" < "$relation_file"
  2422. }
  2423. export -f relation:get
  2424. services:get:upable() {
  2425. if [ -z "$CHARM_STORE_HASH" ]; then
  2426. err-d "Expected \$CHARM_STORE_HASH to be set."
  2427. return 1
  2428. fi
  2429. local services_args=("$@") cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$CHARM_STORE_HASH" "$@")"
  2430. if [ -e "$cache_file" ]; then
  2431. touch "$cache_file" || return 1
  2432. cat "$cache_file"
  2433. return 0
  2434. fi
  2435. declare -A seen
  2436. services=($(get_ordered_service_dependencies "${services_args[@]}")) || exit 1
  2437. for service in "${services[@]}"; do
  2438. mservice=$(get_master_service_for_service "$service") || exit 1
  2439. [ "${seen[$mservice]}" ] && continue
  2440. type="$(get_service_type "$mservice")" || exit 1
  2441. ## remove run-once
  2442. [ "$type" == "run-once" ] && continue
  2443. [ "$type" == "stub" ] && continue
  2444. seen[$mservice]=1
  2445. echo "$mservice"
  2446. done > "$cache_file".wip
  2447. mv "$cache_file".wip "$cache_file"
  2448. cat "$cache_file"
  2449. }
  2450. export -f services:get:upable
  2451. service:state() {
  2452. local service="$1" states state
  2453. project_name=$(get_default_project_name) || return 1
  2454. states=()
  2455. for state in "$SERVICE_STATE_PATH"/"$project_name"/"$service"/*; do
  2456. [ -e "$state" ] || continue
  2457. state=${state##*/}
  2458. states+=("$state")
  2459. done
  2460. if [[ " ${states[*]} " == *" deploying "* ]]; then
  2461. echo "deploying"
  2462. elif [[ " ${states[*]} " == *" up "* ]]; then
  2463. echo "up"
  2464. else
  2465. echo "down"
  2466. fi
  2467. }
  2468. export -f service:state
  2469. charm:upstream-version() {
  2470. local charm="$1" version cache_file="$state_tmpdir/$FUNCNAME.cache.$1" path
  2471. if [ -e "$cache_file" ]; then
  2472. {
  2473. read-0 errlvl
  2474. cat
  2475. } <"$cache_file"
  2476. return $errlvl
  2477. fi
  2478. (
  2479. if ! mkdir "$cache_file.lock" 2>/dev/null; then
  2480. while true; do
  2481. sleep 0.1
  2482. [ -d "${cache_file}.lock" ] || break
  2483. done
  2484. if [ -e "$cache_file" ]; then
  2485. {
  2486. read-0 errlvl
  2487. if [ "$errlvl" == 0 ]; then
  2488. cat
  2489. else
  2490. cat >&2
  2491. fi
  2492. } <"$cache_file"
  2493. return $errlvl
  2494. fi
  2495. return 1
  2496. fi
  2497. trap_add EXIT,ERR "rmdir \"${cache_file}\".lock"
  2498. if ! path=$(charm.has_direct_action "$charm" "upstream-versions"); then
  2499. touch "$cache_file"
  2500. return 0
  2501. fi
  2502. rm -f "${cache_file}.wip"
  2503. touch "${cache_file}.wip"
  2504. (
  2505. version=$("$path" -l 1)
  2506. errlvl=$?
  2507. if [ "$errlvl" != 0 ]; then
  2508. err "Action ${WHITE}upstream-versions${NORMAL} failed for ${DARKPINK}$charm${NORMAL}."
  2509. return $errlvl
  2510. fi
  2511. if path=$(charm.has_direct_action "$charm" "upstream-version-normalize"); then
  2512. version=$("$path" "$version")
  2513. errlvl=$?
  2514. if [ "$errlvl" != 0 ]; then
  2515. err "Failed to normalize upstream version for ${DARKPINK}$charm${NORMAL}."
  2516. return $errlvl
  2517. fi
  2518. fi
  2519. echo "$version"
  2520. ) > "${cache_file}.wip" 2>&1
  2521. errlvl=$?
  2522. p0 "$errlvl" > "${cache_file}"
  2523. if [ "$errlvl" != 0 ]; then
  2524. cat "${cache_file}.wip" | tee -a "${cache_file}" >&2
  2525. rm "${cache_file}.wip"
  2526. return $errlvl
  2527. fi
  2528. cat "${cache_file}.wip" | tee -a "${cache_file}"
  2529. rm "${cache_file}.wip"
  2530. )
  2531. }
  2532. export -f charm:upstream-version
  2533. service:upstream-version() {
  2534. local service="$1" version
  2535. charm=$(get_service_charm "$service") || return $?
  2536. version=$(charm:upstream-version "$charm") || return $?
  2537. e "$version"
  2538. }
  2539. export -f service:upstream-version
  2540. _get_charm_metadata_uses() {
  2541. local metadata="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  2542. if [ -e "$cache_file" ]; then
  2543. #debug "$FUNCNAME: SESSION cache hit $1"
  2544. cat "$cache_file" || return 1
  2545. return 0
  2546. fi
  2547. printf "%s" "$metadata" | { shyaml key-values-0 uses 2>/dev/null || true; } | tee "$cache_file"
  2548. }
  2549. export -f _get_charm_metadata_uses
  2550. _get_service_metadata() {
  2551. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2552. charm
  2553. if [ -e "$cache_file" ]; then
  2554. #debug "$FUNCNAME: SESSION cache hit $1"
  2555. cat "$cache_file"
  2556. return 0
  2557. fi
  2558. charm="$(get_service_charm "$service")" || return 1
  2559. charm.metadata "$charm" > "$cache_file"
  2560. if [ "$?" != 0 ]; then
  2561. rm -f "$cache_file" ## no cache
  2562. return 1
  2563. fi
  2564. cat "$cache_file"
  2565. }
  2566. export -f _get_service_metadata
  2567. _get_service_uses() {
  2568. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2569. metadata
  2570. if [ -e "$cache_file" ]; then
  2571. #debug "$FUNCNAME: SESSION cache hit $1"
  2572. cat "$cache_file"
  2573. return 0
  2574. fi
  2575. metadata="$(_get_service_metadata "$service")" || return 1
  2576. _get_charm_metadata_uses "$metadata" > "$cache_file"
  2577. if [ "$?" != 0 ]; then
  2578. rm -f "$cache_file" ## no cache
  2579. return 1
  2580. fi
  2581. cat "$cache_file"
  2582. }
  2583. export -f _get_service_uses
  2584. _get_services_uses() {
  2585. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2586. service rn rd
  2587. if [ -e "$cache_file" ]; then
  2588. #debug "$FUNCNAME: SESSION cache hit $1"
  2589. cat "$cache_file"
  2590. return 0
  2591. fi
  2592. for service in "$@"; do
  2593. _get_service_uses "$service" | while read-0 rn rd; do
  2594. printf "%s\0" "$service" "$rn" "$rd"
  2595. done
  2596. [ "${PIPESTATUS[0]}" == 0 ] || {
  2597. return 1
  2598. }
  2599. done > "${cache_file}.wip"
  2600. mv "${cache_file}"{.wip,} &&
  2601. cat "$cache_file" || return 1
  2602. }
  2603. export -f _get_services_uses
  2604. _get_provides_provides() {
  2605. local provides="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2606. service rn rd
  2607. if [ -e "$cache_file" ]; then
  2608. # debug "$FUNCNAME: CACHEDIR cache hit $1"
  2609. cat "$cache_file"
  2610. return 0
  2611. fi
  2612. type=$(printf "%s" "$provides" | shyaml get-type)
  2613. case "$type" in
  2614. sequence)
  2615. while read-0 prov; do
  2616. printf "%s\0" "$prov" ""
  2617. done < <(echo "$provides" | shyaml get-values-0)
  2618. ;;
  2619. struct)
  2620. printf "%s" "$provides" | shyaml key-values-0
  2621. ;;
  2622. str)
  2623. printf "%s\0" "$(echo "$provides" | shyaml get-value)" ""
  2624. ;;
  2625. *)
  2626. err "Unexpected type '$type' for provider identifier in charm '$charm'."
  2627. return 1
  2628. esac | tee "$cache_file"
  2629. return "${PIPESTATUS[0]}"
  2630. }
  2631. _get_metadata_provides() {
  2632. local metadata="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2633. service rn rd
  2634. if [ -e "$cache_file" ]; then
  2635. #debug "$FUNCNAME: CACHEDIR cache hit"
  2636. cat "$cache_file"
  2637. return 0
  2638. fi
  2639. provides=$(printf "%s" "$metadata" | shyaml -q get-value -y provides "")
  2640. [ "$provides" -a "$provides" != "''" ] || { touch "$cache_file"; return 0; }
  2641. _get_provides_provides "$provides" | tee "$cache_file"
  2642. return "${PIPESTATUS[0]}"
  2643. }
  2644. _get_services_provides() {
  2645. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2646. service rn rd
  2647. if [ -e "$cache_file" ]; then
  2648. #debug "$FUNCNAME: SESSION cache hit $1"
  2649. cat "$cache_file"
  2650. return 0
  2651. fi
  2652. ## YYY: replace the inner loop by a cached function
  2653. for service in "$@"; do
  2654. metadata="$(_get_service_metadata "$service")" || return 1
  2655. while read-0 rn rd; do
  2656. printf "%s\0" "$service" "$rn" "$rd"
  2657. done < <(_get_metadata_provides "$metadata")
  2658. done > "$cache_file"
  2659. if [ "$?" != 0 ]; then
  2660. rm -f "$cache_file" ## no cache
  2661. return 1
  2662. fi
  2663. cat "$cache_file"
  2664. }
  2665. export -f _get_services_provides
  2666. _get_charm_provides() {
  2667. if [ -z "$CHARM_STORE_HASH" ]; then
  2668. err-d "Expected \$CHARM_STORE_HASH to be set."
  2669. return 1
  2670. fi
  2671. local cache_file="$CACHEDIR/$FUNCNAME.cache.$CHARM_STORE_HASH" errlvl
  2672. if [ -e "$cache_file" ]; then
  2673. #debug "$FUNCNAME: SESSION cache hit"
  2674. cat "$cache_file"
  2675. return 0
  2676. fi
  2677. start="$SECONDS"
  2678. debug "Getting charm provider list..."
  2679. while read-0 charm _ realpath metadata; do
  2680. metadata="$(charm.metadata "$charm")" || continue
  2681. # echo "reading $charm" >&2
  2682. while read-0 rn rd; do
  2683. printf "%s\0" "$charm" "$rn" "$rd"
  2684. done < <(_get_metadata_provides "$metadata")
  2685. done < <(charm.ls) | tee "$cache_file"
  2686. errlvl="${PIPESTATUS[0]}"
  2687. debug " ..charm provider list done $GRAY(in $((SECONDS - start))s)$NORMAL"
  2688. return "$errlvl"
  2689. }
  2690. _get_charm_providing() {
  2691. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2692. relation="$1"
  2693. if [ -e "$cache_file" ]; then
  2694. #debug "$FUNCNAME: SESSION cache hit $1"
  2695. cat "$cache_file"
  2696. return 0
  2697. fi
  2698. while read-0 charm relation_name relation_def; do
  2699. [ "$relation_name" == "$relation" ] || continue
  2700. printf "%s\0" "$charm" "$relation_def"
  2701. done < <(_get_charm_provides) > "$cache_file"
  2702. if [ "$?" != 0 ]; then
  2703. rm -f "$cache_file" ## no cache
  2704. return 1
  2705. fi
  2706. cat "$cache_file"
  2707. }
  2708. _get_services_providing() {
  2709. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  2710. relation="$1"
  2711. shift ## services is "$@"
  2712. if [ -e "$cache_file" ]; then
  2713. #debug "$FUNCNAME: SESSION cache hit $1"
  2714. cat "$cache_file"
  2715. return 0
  2716. fi
  2717. while read-0 service relation_name relation_def; do
  2718. [ "$relation_name" == "$relation" ] || continue
  2719. printf "%s\0" "$service" "$relation_def"
  2720. done < <(_get_services_provides "$@") > "$cache_file"
  2721. if [ "$?" != 0 ]; then
  2722. rm -f "$cache_file" ## no cache
  2723. return 1
  2724. fi
  2725. cat "$cache_file"
  2726. }
  2727. export -f _get_services_provides
  2728. _out_new_relation_from_defs() {
  2729. local service="$1" rn="$2" ts="$3" prov_def="$4" rel_def="$5" rc td rc_prov
  2730. rc_prov=$(printf "%s" "$prov_def" | shyaml -y get-value "default-options" 2>/dev/null)
  2731. ## YYYvlab: should be seen even in no debug mode no ?
  2732. rc=$(printf "%s" "$rel_def" | shyaml -y get-value "default-options" 2>/dev/null)
  2733. td=$(echo "$prov_def" | shyaml get-value 'tech-dep' 2>/dev/null)
  2734. td=${td:-True}
  2735. rc=$(merge_yaml_str "$rc_prov" "$rc") || return 1
  2736. after=$(_out_after_value_from_def "$service" "$rn" "$rel_def") || return 1
  2737. printf "%s\0" "$after" "$service" "$relation_name" "$ts" "$rc" "$td"
  2738. }
  2739. _out_after_value_from_def() {
  2740. local service="$1" relation_name="$2" relation_def="$3" after_t after
  2741. if after_t=$(echo "$relation_def" | shyaml get-type after 2>/dev/null); then
  2742. case "$after_t" in
  2743. sequence)
  2744. after="$(echo "$relation_def" | shyaml get-values after 2>/dev/null)" || return 1
  2745. after=",$service:${after//$'\n'/,$service:},"
  2746. ;;
  2747. struct)
  2748. err "Invalid type for ${WHITE}after${NORMAL}'s value in ${DARKBLUE}$relation_name${NORMAL}'s definition."
  2749. return 1
  2750. ;;
  2751. str)
  2752. after=",$service:$(echo "$relation_def" | shyaml get-value after "" 2>/dev/null)," || return 1
  2753. ;;
  2754. esac
  2755. else
  2756. after=""
  2757. fi
  2758. e "$after"
  2759. }
  2760. get_all_compose_yml_service() {
  2761. if [ -z "$COMPOSE_YML_CONTENT_HASH" ]; then
  2762. COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || {
  2763. err "Failed to get compose yml hash"
  2764. return 1
  2765. }
  2766. fi
  2767. local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMPOSE_YML_CONTENT_HASH"
  2768. if [ -e "${cache_file}" ]; then
  2769. #debug "$FUNCNAME: cache hit: ${cache_file}"
  2770. cat "${cache_file}"
  2771. return 0
  2772. fi
  2773. compose_yml_content=$(get_compose_yml_content) || return 1
  2774. printf "%s" "${compose_yml_content}" | shyaml keys-0 2>/dev/null > "${cache_file}.wip" || {
  2775. err "Failed to get keys of compose content."
  2776. return 1
  2777. }
  2778. mv "${cache_file}"{.wip,} || return 1
  2779. cat "${cache_file}"
  2780. }
  2781. ## Outputs all relations array.
  2782. _service:all:relations_cached() {
  2783. local services service E
  2784. services=($(compose:yml:root:services)) || {
  2785. err-d "Failed to get services from compose yml."
  2786. return 1
  2787. }
  2788. get_all_relations "${services[@]}" || return 1
  2789. }
  2790. ## Outputs all relations array.
  2791. service:all:relations() {
  2792. if [ -z "$COMBINED_HASH" ]; then
  2793. err-d "Expected \$COMBINED_HASH to be set."
  2794. return 1
  2795. fi
  2796. local cache_file="$CACHEDIR/$FUNCNAME.cache.$COMBINED_HASH"
  2797. if [ -e "${cache_file}" ]; then
  2798. # debug "$FUNCNAME: SESSION cache hit $1"
  2799. cat "${cache_file}"
  2800. return 0
  2801. fi
  2802. _service:all:relations_cached > "${cache_file}.wip" || {
  2803. err-d "Failed to compute all relations."
  2804. return 1
  2805. }
  2806. mv "${cache_file}"{.wip,} || return 1
  2807. cat "${cache_file}"
  2808. }
  2809. _service:all:relations_hash_cached() {
  2810. if [ -z "$COMBINED_HASH" ]; then
  2811. err-d "Expected \$COMBINED_HASH to be set."
  2812. return 1
  2813. fi
  2814. local cache_file="$CACHEDIR/$FUNCNAME.cache.x${COMBINED_HASH}" \
  2815. hash
  2816. if [ -e "${cache_file}" ]; then
  2817. # debug "$FUNCNAME: SESSION cache hit $cache_file"
  2818. cat "${cache_file}"
  2819. return 0
  2820. fi
  2821. service:all:relations > "${cache_file}.pre" || {
  2822. err-d "Failed to get all relations."
  2823. return 1
  2824. }
  2825. {
  2826. p0 "$(hash_get < "${cache_file}.pre")" || return 1
  2827. cat "${cache_file}.pre"
  2828. rm "${cache_file}.pre"
  2829. } > "${cache_file}".wip || return 1
  2830. mv "${cache_file}"{.wip,} || return 1
  2831. cat "${cache_file}"
  2832. }
  2833. ## Get all relations from all services in the current compose file.
  2834. ## Sets GLOBAL_ALL_RELATIONS_HASH and returns all relations array.
  2835. service:all:set_relations_hash() {
  2836. if [ -n "$GLOBAL_ALL_RELATIONS" ]; then
  2837. if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then
  2838. err "Can't access global \$GLOBAL_ALL_RELATIONS_HASH"
  2839. echo " (despite \$GLOBAL_ALL_RELATIONS being set)" >&2
  2840. return 1
  2841. fi
  2842. return 0
  2843. fi
  2844. ## sets COMPOSE_YML_CONTENT_HASH
  2845. _service:all:relations_hash_cached >/dev/null || return 1
  2846. {
  2847. read-0 GLOBAL_ALL_RELATIONS_HASH || return 1
  2848. export GLOBAL_ALL_RELATIONS_HASH
  2849. ## transfer to statedir
  2850. export GLOBAL_ALL_RELATIONS="$CACHEDIR/$FUNCNAME.cache.$COMBINED_HASH"
  2851. cat > "$GLOBAL_ALL_RELATIONS"
  2852. } < <(_service:all:relations_hash_cached)
  2853. if [ -z "$GLOBAL_ALL_RELATIONS" ]; then
  2854. err "Failed to set \$GLOBAL_ALL_RELATIONS."
  2855. return 1
  2856. fi
  2857. if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then
  2858. err "Failed to set \$GLOBAL_ALL_RELATIONS_HASH."
  2859. return 1
  2860. fi
  2861. }
  2862. get_subset_relations () {
  2863. local service all_services services start
  2864. if [ -n "$SUBSET_ALL_RELATIONS" ]; then
  2865. return 0
  2866. fi
  2867. if [ -z "$GLOBAL_ALL_RELATIONS_HASH" ]; then
  2868. err-d "Can't access global \$GLOBAL_ALL_RELATIONS_HASH"
  2869. return 1
  2870. fi
  2871. cache_hash=$(H "$@" "$GLOBAL_ALL_RELATIONS_HASH" "$(declare -f "$FUNCNAME")")
  2872. local cache_file="$CACHEDIR/$FUNCNAME.cache.$cache_hash"
  2873. if [ -e "${cache_file}" ]; then
  2874. export SUBSET_ALL_RELATIONS="$cache_file"
  2875. hash=$(hash_get < "$cache_file") || return 1
  2876. export SUBSET_ALL_RELATIONS_HASH="$hash"
  2877. cat "${cache_file}"
  2878. return 0
  2879. fi
  2880. ## collect all connected services first
  2881. all_services=("$@")
  2882. declare -A services
  2883. while [ "${#all_services[@]}" != 0 ]; do
  2884. array_pop all_services service
  2885. # debug " Getting relations for $DARKYELLOW$service$NORMAL"
  2886. while read-0 s rn ts rc td; do
  2887. [[ "$s" == "$service" ]] || continue
  2888. # debug " adding relation $DARKBLUE$rn$NORMAL to $DARKYELLOW$ts$NORMAL"
  2889. p0 "$service" "$rn" "$ts" "$rc" "$td"
  2890. if [ -z "${services[$ts]}" ] && [[ " ${all_services[@]} " != *" $ts "* ]]; then
  2891. all_services+=("$ts")
  2892. fi
  2893. done < "$GLOBAL_ALL_RELATIONS"
  2894. services["$service"]=1
  2895. done > "$cache_file.wip"
  2896. mv "$cache_file"{.wip,} || return 1
  2897. export SUBSET_ALL_RELATIONS="$cache_file"
  2898. hash=$(hash_get < "$cache_file") || return 1
  2899. export SUBSET_ALL_RELATIONS_HASH="$hash"
  2900. cat "$cache_file"
  2901. }
  2902. export -f get_subset_relations
  2903. get_all_relations () {
  2904. if [ -z "$COMBINED_HASH" ]; then
  2905. err-d "Expected \$COMBINED_HASH to be set."
  2906. return 1
  2907. fi
  2908. if [ -n "$GLOBAL_ALL_RELATIONS" ]; then
  2909. cat "$GLOBAL_ALL_RELATIONS" || return 1
  2910. return 0
  2911. fi
  2912. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" "$COMBINED_HASH" "$(declare -p without_relations)")" \
  2913. services all_services service services_uses services_provides \
  2914. changed summon required recommended optional
  2915. if [ -e "${cache_file}" ]; then
  2916. #debug "$FUNCNAME: SESSION cache hit $1"
  2917. export GLOBAL_ALL_RELATIONS="$cache_file"
  2918. cat "${cache_file}"
  2919. return 0
  2920. fi
  2921. declare -A services
  2922. services_uses=()
  2923. ## XXXvlab: bwerk, leveraging cache to be able to get the errorlevel here.
  2924. _get_services_uses "$@" >/dev/null || return 1
  2925. array_read-0 services_uses < <(_get_services_uses "$@")
  2926. services_provides=()
  2927. ## XXXvlab: bwerk, leveraging cache to be able to get the errorlevel here.
  2928. _get_services_provides "$@" >/dev/null || return 1
  2929. array_read-0 services_provides < <(_get_services_provides "$@")
  2930. for service in "$@"; do
  2931. services[$service]=1
  2932. done
  2933. all_services=("$@")
  2934. while [ "${#all_services[@]}" != 0 ]; do
  2935. array_pop all_services service
  2936. while read-0-err E relation_name ts relation_config tech_dep; do
  2937. [ "${without_relations[$service:$relation_name]}" ] && {
  2938. debug "Ignoring compose $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> ${DARKYELLOW}$ts$NORMAL"
  2939. continue
  2940. }
  2941. ## First is priority, that can be adjusted in second step
  2942. printf "%s\0" "" "$service" "$relation_name" "$ts" "$relation_config" "$tech_dep"
  2943. ## adding target services ?
  2944. [ "${services[$ts]}" ] && continue
  2945. array_read-0 services_uses < <(_get_services_uses "$ts")
  2946. all_services+=("$ts")
  2947. services[$ts]=1
  2948. done < <(p-err get_compose_relations "$service")
  2949. if [ "$E" != 0 ]; then
  2950. err "Failed to get relations for $DARKYELLOW$service$NORMAL."
  2951. return 1
  2952. fi
  2953. done > "${cache_file}.wip"
  2954. while true; do
  2955. changed=
  2956. new_services_uses=()
  2957. summon=()
  2958. required=()
  2959. recommended=()
  2960. optional=()
  2961. while [ "${#services_uses[@]}" != 0 ]; do
  2962. service="${services_uses[0]}"
  2963. relation_name="${services_uses[1]}"
  2964. relation_def="${services_uses[2]}"
  2965. services_uses=("${services_uses[@]:3}")
  2966. [ "${without_relations[$service:$relation_name]}" ] && {
  2967. debug "Skipping $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW*$NORMAL"
  2968. continue
  2969. }
  2970. default_options=$(printf "%s" "$relation_def" | shyaml -y get-value "default-options" 2>/dev/null)
  2971. after=$(_out_after_value_from_def "$service" "$relation_name" "$relation_def") || return 1
  2972. ## is this "use" declaration satisfied ?
  2973. found=
  2974. while read-0 p s rn ts rc td; do
  2975. if [ -z "$found" -a "$service" == "$s" -a "$relation_name" == "$rn" ]; then
  2976. if [ "$default_options" ]; then
  2977. rc=$(merge_yaml_str "$default_options" "$rc") || return 1
  2978. fi
  2979. found="$ts"
  2980. p="$after"
  2981. fi
  2982. printf "%s\0" "$p" "$s" "$rn" "$ts" "$rc" "$td"
  2983. done < "${cache_file}.wip" > "${cache_file}.wip.new"
  2984. mv "${cache_file}.wip.new" "${cache_file}.wip"
  2985. if [ "$found" ]; then ## this "use" declaration was satisfied
  2986. debug "${DARKYELLOW}$service${NORMAL} use declaration for relation" \
  2987. "${DARKBLUE}$relation_name${NORMAL} is satisfied with ${DARKYELLOW}$found${NORMAL}"
  2988. continue
  2989. fi
  2990. auto=$(echo "$relation_def" | shyaml get-value auto pair 2>/dev/null)
  2991. auto=${auto:-pair}
  2992. case "$auto" in
  2993. "pair"|"summon")
  2994. service_list=()
  2995. array_read-0 service_list < <(array_keys_to_stdin services)
  2996. providers=()
  2997. providers_def=()
  2998. array_read-0 providers providers_def < <(_get_services_providing "$relation_name" "${service_list[@]}")
  2999. if [ "${#providers[@]}" == 1 ]; then
  3000. ts="${providers[0]}"
  3001. debug "Auto-pairs ${DARKYELLOW}$service${NORMAL}" \
  3002. "--${DARKBLUE}$relation_name${NORMAL}--> ${DARKYELLOW}$ts${NORMAL}"
  3003. _out_new_relation_from_defs "$service" "$relation_name" "$ts" \
  3004. "${providers_def[0]}" "$relation_def" \
  3005. >> "${cache_file}.wip" || return 1
  3006. ## Adding service
  3007. [ "${services[$ts]}" ] && continue
  3008. array_read-0 new_services_uses < <(_get_services_uses "$ts")
  3009. services[$ts]=1
  3010. changed=1
  3011. continue
  3012. fi
  3013. if [ "${#providers[@]}" -gt 1 ]; then
  3014. msg=""
  3015. warn "No auto-pairing ${DARKYELLOW}$service${NORMAL}" \
  3016. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  3017. "(> 1 provider)."
  3018. elif [ "$auto" == "summon" ]; then ## no provider
  3019. summon+=("$service" "$relation_name" "$relation_def")
  3020. fi
  3021. ;;
  3022. null|disable|disabled)
  3023. :
  3024. ;;
  3025. *)
  3026. err "Invalid ${WHITE}auto${NORMAL} value '$auto'."
  3027. return 1
  3028. ;;
  3029. esac
  3030. constraint=$(echo "$relation_def" | shyaml get-value constraint 2>/dev/null)
  3031. constraint=${constraint:-optional}
  3032. case "$constraint" in
  3033. "required")
  3034. required+=("$service" "$relation_name" "$relation_def")
  3035. ;;
  3036. "recommended")
  3037. recommended+=("$service" "$relation_name" "$relation_def")
  3038. ;;
  3039. "optional")
  3040. optional+=("$service" "$relation_name" "$relation_def")
  3041. ;;
  3042. *)
  3043. err "Invalid ${WHITE}constraint${NORMAL} value '$constraint'."
  3044. return 1
  3045. ;;
  3046. esac
  3047. new_services_uses+=("$service" "$relation_name" "$relation_def") ## re-queue it
  3048. done
  3049. services_uses=("${new_services_uses[@]}")
  3050. if [ "$changed" ]; then
  3051. continue
  3052. fi
  3053. ## situation is stable
  3054. if [ "${#summon[@]}" != 0 ]; then
  3055. declare -A summon_requeued=()
  3056. while [ "${#summon[@]}" != 0 ]; do
  3057. service="${summon[0]}"
  3058. relation_name="${summon[1]}"
  3059. relation_def="${summon[2]}"
  3060. summon=("${summon[@]:3}")
  3061. providers=()
  3062. providers_def=()
  3063. array_read-0 providers providers_def < <(_get_charm_providing "$relation_name" "${service_list[@]}")
  3064. ## select first provider that is not a stub
  3065. new_providers=()
  3066. new_providers_def=()
  3067. while [[ "${#providers[@]}" != 0 ]]; do
  3068. provider="${providers[0]}"
  3069. provider_def="${providers_def[0]}"
  3070. providers=("${providers[@]:1}")
  3071. providers_def=("${providers_def[@]:1}")
  3072. type="$(get_service_type "$provider")" || true
  3073. [ "$type" == "stub" ] && continue
  3074. new_providers+=("$provider")
  3075. new_providers_def+=("$provider_def")
  3076. done
  3077. providers=("${new_providers[@]}")
  3078. providers_def=("${new_providers_def[@]}")
  3079. if [ "${#providers[@]}" == 0 ]; then
  3080. err "Summoning a ${DARKBLUE}$relation_name${NORMAL} provider failed: none were found in charm store."
  3081. return 1
  3082. fi
  3083. if [ "${#providers[@]}" -gt 1 ]; then
  3084. ## if there are multiple providers (for instance
  3085. ## sql-database), there are some case where other
  3086. ## services will also summon a more specific
  3087. ## postgres-database, that will solve our
  3088. ## constraint. So we'd rather pass (and requeue)
  3089. if [ -z "${summon_requeued[$service/$relation_name]}" ]; then
  3090. debug "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  3091. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  3092. "(> 1 provider). Requeuing."
  3093. summon+=("$service" "$relation_name" "$relation_def") ## re-queue it
  3094. summon_requeued["$service/$relation_name"]=1
  3095. continue
  3096. else
  3097. warn "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  3098. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  3099. "(> 1 provider). Choosing first."
  3100. fi
  3101. fi
  3102. ts="${providers[0]}"
  3103. ## YYYvlab: should be seen even in no debug mode no ?
  3104. debug "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  3105. "--${DARKBLUE}$relation_name${NORMAL}--> ${DARKYELLOW}$ts${NORMAL}"
  3106. _out_new_relation_from_defs "$service" "$relation_name" "$ts" \
  3107. "${providers_def[0]}" "$relation_def" \
  3108. >> "${cache_file}.wip" || return 1
  3109. ## Adding service
  3110. [ "${services[$ts]}" ] && continue
  3111. array_read-0 services_uses < <(_get_services_uses "$ts")
  3112. services[$ts]=1
  3113. changed=1
  3114. continue 2
  3115. done
  3116. continue
  3117. fi
  3118. [ "$NO_CONSTRAINT_CHECK" ] && break
  3119. if [ "${#required[@]}" != 0 ]; then
  3120. echo "$(_display_solves required)" | sed -r "s/^/${RED}||${NORMAL} /g" >&2
  3121. err "Required relations not satisfied"
  3122. return 1
  3123. fi
  3124. if [ "${#recommended[@]}" != 0 ]; then
  3125. ## make recommendation
  3126. echo "$(_display_solves recommended)" | sed -r "s/^/${YELLOW}||${NORMAL} /g" >&2
  3127. fi
  3128. if [ -z "$QUIET" ]; then
  3129. if [ "${#optional[@]}" != 0 ]; then
  3130. ## inform about options
  3131. echo "$(_display_solves optional)" | sed -r "s/^/${BLUE}||${NORMAL} /g" >&2
  3132. fi
  3133. fi
  3134. # if [ "${#required[@]}" != 0 ]; then
  3135. # err "Required relations not satisfied"
  3136. # return 1
  3137. # fi
  3138. if [ "${#recommended[@]}" != 0 ]; then
  3139. warn "Recommended relations not satisfied"
  3140. fi
  3141. break
  3142. done
  3143. if [ "$?" != 0 ]; then
  3144. rm -f "${cache_file}"{,.wip,.wip.new} ## no cache
  3145. return 1
  3146. fi
  3147. ##
  3148. ## Sort relations thanks to uses =metadata.yml= relations.
  3149. ##
  3150. mv "${cache_file}.wip"{,.in} &&
  3151. rm -f "${cache_file}.wip.final" &&
  3152. touch "${cache_file}.wip.final" || {
  3153. err "Unexpected error when mangling cache files."
  3154. return 1
  3155. }
  3156. declare -A relation_done=()
  3157. while true; do
  3158. had_remaining_relation=
  3159. had_new_relation=
  3160. while read-0 p s rn ts rc td; do
  3161. if [ -z "$p" ] || [ "$p" == "," ]; then
  3162. relation_done["$s:$rn"]=1
  3163. # printf " .. %-30s %-30s %-30s\n" "$s" "$ts" "$rn" >&2
  3164. printf "%s\0" "$s" "$rn" "$ts" "$rc" "$td" >> "${cache_file}.wip.final"
  3165. had_new_relation=1
  3166. else
  3167. # printf " !! %-30s %-30s %-30s\n" "$p" "$s" "$rn" >&2
  3168. printf "%s\0" "$p" "$s" "$rn" "$ts" "$rc" "$td" >> "${cache_file}.wip.out"
  3169. had_remaining_relation=1
  3170. fi
  3171. done < "${cache_file}.wip.in"
  3172. [ -z "$had_remaining_relation" ] && break
  3173. mv "${cache_file}.wip."{out,in}
  3174. while read-0 p s rn ts rc td; do
  3175. for rel in "${!relation_done[@]}"; do
  3176. p="${p//,$rel,/,}"
  3177. done
  3178. # printf " CC %-30s %-30s %-30s\n" "$p" "$s" "$rn" >&2
  3179. if [ -z "$had_new_relation" ]; then
  3180. err "${DARKYELLOW}$s${NORMAL} --${DARKBLUE}$rn${NORMAL}--> ${DARKYELLOW}$ts${NORMAL} missing required ${WHITE}after${NORMAL} relations:"
  3181. for rel in ${p//,/ }; do
  3182. rel_s=${rel%%:*}
  3183. rel_r=${rel##*:}
  3184. echo " - ${DARKYELLOW}$rel_s${NORMAL} --${DARKBLUE}$rel_r${NORMAL}--> ${DARKGRAY}*${NORMAL}" >&2
  3185. done
  3186. else
  3187. printf "%s\0" "$p" "$s" "$rn" "$ts" "$rc" "$td" >> "${cache_file}.wip.out"
  3188. fi
  3189. done < "${cache_file}.wip.in"
  3190. if [ -z "$had_new_relation" ]; then
  3191. rm -f "${cache_file}"{,.wip{,new,in,out,final}} ## no cache
  3192. return 1
  3193. fi
  3194. mv "${cache_file}.wip."{out,in}
  3195. done
  3196. mv "${cache_file}"{.wip.final,} || return 1
  3197. export GLOBAL_ALL_RELATIONS="$cache_file"
  3198. GLOBAL_ALL_RELATIONS_HASH=$(hash_get < "$cache_file") || return 1
  3199. export GLOBAL_ALL_RELATIONS_HASH
  3200. cat "$cache_file"
  3201. }
  3202. export -f get_all_relations
  3203. _display_solves() {
  3204. local array_name="$1" by_relation msg
  3205. ## inform about options
  3206. msg=""
  3207. declare -A by_relation
  3208. while read-0 service relation_name relation_def; do
  3209. solves=$(printf "%s" "$relation_def" | shyaml -y get-value solves 2>/dev/null);
  3210. auto=$(printf "%s" "$relation_def" | shyaml get-value auto 2>/dev/null);
  3211. if [ -z "$solves" ]; then
  3212. continue
  3213. fi
  3214. by_relation[$relation_name]+=$(printf "\n %s" "${DARKYELLOW}$service$NORMAL for:")
  3215. if [ "$auto" == "pair" ]; then
  3216. requirement="add provider in cluster to auto-pair"
  3217. else
  3218. requirement="add explicit relation"
  3219. fi
  3220. while read-0 name def; do
  3221. by_relation[$relation_name]+=$(printf "\n - ${DARKCYAN}%-15s${NORMAL} %s (%s)" "$name" "$def" "$requirement")
  3222. done < <(printf "%s" "$solves" | shyaml key-values-0)
  3223. done < <(array_values_to_stdin "$array_name")
  3224. while read-0 relation_name message; do
  3225. msg+="$(printf "\n${DARKBLUE}%s$NORMAL provider is $array_name by%s" \
  3226. "$relation_name" "$message" )"
  3227. done < <(array_kv_to_stdin by_relation)
  3228. if [ "$msg" ]; then
  3229. printf "%s\n" "${msg:1}"
  3230. fi
  3231. }
  3232. get_compose_relation_def() {
  3233. local service="$1" relation="$2" relation_name target_service relation_config tech_dep
  3234. while read-0 relation_name target_service relation_config tech_dep; do
  3235. [ "$relation_name" == "$relation" ] || continue
  3236. printf "%s\0%s\0%s\0" "$target_service" "$relation_config" "$tech_dep"
  3237. done < <(get_compose_relations "$service") || return 1
  3238. }
  3239. export -f get_compose_relation_def
  3240. run_service_relations () {
  3241. local service services loaded subservices subservice
  3242. PROJECT_NAME=$(get_default_project_name) || return 1
  3243. export PROJECT_NAME
  3244. declare -A loaded
  3245. subservices=$(get_ordered_service_dependencies "$@") || return 1
  3246. for service in $subservices; do
  3247. # debug "Upping dep's relations of ${DARKYELLOW}$service${NORMAL}:"
  3248. for subservice in $(get_service_deps "$service") "$service"; do
  3249. [ "${loaded[$subservice]}" ] && continue
  3250. export BASE_SERVICE_NAME=$service
  3251. MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
  3252. MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1
  3253. RELATION_BASE_COMPOSE_DEF=$(get_compose_service_def "$subservice") || return 1
  3254. export RELATION_BASE_COMPOSE_DEF MASTER_BASE_{CHARM,SERVICE}_NAME
  3255. # debug " Relations of ${DARKYELLOW}$subservice${NORMAL}:"
  3256. while read-0 relation_name target_service relation_config tech_dep; do
  3257. [ "${without_relations[$service:$relation_name]}" ] && {
  3258. debug "Skipping $DARKYELLOW$service$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW*$NORMAL"
  3259. continue
  3260. }
  3261. export relation_config
  3262. export TARGET_SERVICE_NAME=$target_service
  3263. MASTER_TARGET_SERVICE_NAME=$(get_top_master_service_for_service "$target_service") || return 1
  3264. MASTER_TARGET_CHARM_NAME=$(get_service_charm "$MASTER_TARGET_SERVICE_NAME") || return 1
  3265. RELATION_TARGET_COMPOSE_DEF=$(get_compose_service_def "$target_service") || return 1
  3266. export RELATION_TARGET_COMPOSE_DEF MASTER_TARGET_{CHARM,SERVICE}_NAME
  3267. Wrap "${wrap_opts[@]}" -d "building $DARKYELLOW$subservice$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW$target_service$NORMAL" <<EOF || return 1
  3268. _run_service_relation "$relation_name" "$subservice" "$target_service" "\$relation_config"
  3269. EOF
  3270. done < <(get_service_relations "$subservice") || return 1
  3271. loaded[$subservice]=1
  3272. done
  3273. done
  3274. }
  3275. export -f run_service_relations
  3276. _run_service_action_direct() {
  3277. local service="$1" action="$2" charm _dummy project_name
  3278. shift; shift
  3279. read-0 charm action_script_path || true ## against 'set -e' that could be setup in parent scripts
  3280. if read-0 _dummy || [ "$_dummy" ]; then
  3281. print_syntax_error "$FUNCNAME: too many arguments in action descriptor"
  3282. return 1
  3283. fi
  3284. project_name=$(get_default_project_name) || return 1
  3285. export PROJECT_NAME="$project_name"
  3286. export state_tmpdir
  3287. (
  3288. set +e ## Prevents unwanted leaks from parent shell
  3289. export COMPOSE_CONFIG=$(get_compose_yml_content)
  3290. export METADATA_CONFIG=$(charm.metadata "$charm")
  3291. export SERVICE_NAME=$service
  3292. export ACTION_NAME=$action
  3293. export ACTION_SCRIPT_PATH="$action_script_path"
  3294. export CONTAINER_NAME=$(get_top_master_service_for_service "$service")
  3295. export DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$CONTAINER_NAME")
  3296. export SERVICE_DATASTORE="$DATASTORE/$service"
  3297. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  3298. exname="$exname $ACTION_NAME $SERVICE_NAME" \
  3299. stdbuf -oL -eL bash -c 'charm.run_direct_action "$@"' -- "$charm" "$action" "$@"
  3300. ) 0<&6 ## inject general stdin
  3301. }
  3302. export -f _run_service_action_direct
  3303. _run_service_action_relation() {
  3304. local service="$1" action="$2" charm target_charm relation_name relation_config _dummy
  3305. shift; shift
  3306. read-0 charm target_service target_charm relation_name relation_config action_script_path || true
  3307. if read-0 _dummy || [ "$_dummy" ]; then
  3308. print_syntax_error "$FUNCNAME: too many arguments in action descriptor"
  3309. return 1
  3310. fi
  3311. RELATION_DATA_FILE=$(get_relation_data_file "$service" "$target_service" "$relation_name" "$relation_config") || return 1
  3312. export action_errlvl_file="$state_tmpdir/action-$service-$charm-$action-errlvl"
  3313. export state_tmpdir
  3314. (
  3315. set +e ## Prevents unwanted leaks from parent shell
  3316. export METADATA_CONFIG=$(charm.metadata "$charm")
  3317. export SERVICE_NAME=$service
  3318. export RELATION_TARGET_SERVICE="$target_service"
  3319. export RELATION_TARGET_CHARM="$target_charm"
  3320. export RELATION_BASE_SERVICE="$service"
  3321. export RELATION_BASE_CHARM="$charm"
  3322. export RELATION_DATA_FILE="$RELATION_DATA_FILE"
  3323. export ACTION_NAME=$action
  3324. export ACTION_SCRIPT_PATH="$action_script_path"
  3325. export CONTAINER_NAME=$(get_top_master_service_for_service "$service")
  3326. export DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$CONTAINER_NAME")
  3327. export SERVICE_DATASTORE="$DATASTORE/$service"
  3328. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  3329. exname="$exname $ACTION_NAME $SERVICE_NAME" \
  3330. stdbuf -oL -eL bash -c 'charm.run_relation_action "$@"' -- "$target_charm" "$relation_name" "$action" "$@"
  3331. ) 0<&6 ## inject general stdin
  3332. }
  3333. export -f _run_service_action_relation
  3334. get_relation_data_dir() {
  3335. local service="$1" target_service="$2" relation_name="$3" \
  3336. cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  3337. if [ -e "$cache_file" ]; then
  3338. # debug "$FUNCNAME: cache hit ($*)"
  3339. cat "$cache_file"
  3340. return 0
  3341. fi
  3342. local project relation_dir
  3343. project=${PROJECT_NAME}
  3344. if [ -z "$project" ]; then
  3345. project=$(get_default_project_name) || return 1
  3346. fi
  3347. relation_dir="$VARDIR/relations/$project/${service}-${target_service}/$relation_name"
  3348. if ! [ -d "$relation_dir" ]; then
  3349. mkdir -p "$relation_dir" || return 1
  3350. chmod go-rwx "$relation_dir" || return 1 ## protecting this directory
  3351. fi
  3352. echo "$relation_dir" | tee "$cache_file"
  3353. }
  3354. export -f get_relation_data_dir
  3355. get_relation_data_file() {
  3356. local service="$1" target_service="$2" relation_name="$3" relation_config="$4" \
  3357. new new_md5 relation_dir relation_data_file
  3358. relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1
  3359. relation_data_file="$relation_dir/data"
  3360. new=
  3361. if [ -e "$relation_data_file" ]; then
  3362. ## Has reference changed ?
  3363. new_md5=$(e "$relation_config" | md5_compat)
  3364. if [ "$new_md5" != "$(cat "$relation_data_file.md5_ref" 2>/dev/null)" ]; then
  3365. new=true
  3366. fi
  3367. else
  3368. new=true
  3369. fi
  3370. if [ -n "$new" ]; then
  3371. OLDUMASK=$(umask)
  3372. umask 0077
  3373. e "$relation_config" > "$relation_data_file"
  3374. umask "$OLDUMASK"
  3375. e "$relation_config" | md5_compat > "$relation_data_file.md5_ref"
  3376. fi
  3377. echo "$relation_data_file"
  3378. }
  3379. export -f get_relation_data_file
  3380. has_service_action () {
  3381. if [ -z "$CHARM_STORE_HASH" ]; then
  3382. err-d "Can't access global \$CHARM_STORE_HASH"
  3383. return 1
  3384. fi
  3385. local service="$1" action="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$2.$CHARM_STORE_HASH" \
  3386. charm target_charm relation_name target_service relation_config _tech_dep \
  3387. path
  3388. if [ -e "$cache_file" ]; then
  3389. # debug "$FUNCNAME: cache hit ($*)"
  3390. if [ -s "$cache_file" ]; then
  3391. cat "$cache_file"
  3392. return 0
  3393. else
  3394. return 1
  3395. fi
  3396. fi
  3397. charm=$(get_service_charm "$service") || return 1
  3398. ## Action directly provided ?
  3399. if path=$(charm.has_direct_action "$charm" "$action"); then
  3400. p0 "direct" "$charm" "$path" | tee "$cache_file"
  3401. return 0
  3402. fi
  3403. ## Action provided by relation ?
  3404. while read-0 relation_name target_service relation_config _tech_dep; do
  3405. target_charm=$(get_service_charm "$target_service") || return 1
  3406. if path=$(charm.has_relation_action "$target_charm" "$relation_name" "$action"); then
  3407. p0 "relation" "$charm" "$target_service" "$target_charm" "$relation_name" "$relation_config" "$path" | tee "$cache_file"
  3408. return 0
  3409. fi
  3410. done < <(get_service_relations "$service")
  3411. touch "$cache_file"
  3412. return 1
  3413. # master=$(get_top_master_service_for_service "$service")
  3414. # [ "$master" == "$charm" ] && return 1
  3415. # has_service_action "$master" "$action"
  3416. }
  3417. export -f has_service_action
  3418. run_service_action () {
  3419. local service="$1" action="$2" errlvl
  3420. shift ; shift
  3421. exec 6<&0 ## saving stdin
  3422. {
  3423. if ! read-0 action_type; then
  3424. info "Service $DARKYELLOW$service$NORMAL does not have any action $DARKCYAN$action$NORMAL defined."
  3425. info " Add an executable script to 'actions/$action' to implement action."
  3426. return 1
  3427. fi
  3428. "_run_service_action_${action_type}" "$service" "$action" "$@"
  3429. errlvl="$?"
  3430. } < <(has_service_action "$service" "$action")
  3431. exec 0<&6 6<&- ## restoring stdin
  3432. return "$errlvl"
  3433. }
  3434. export -f run_service_action
  3435. get_compose_relation_config() {
  3436. local service=$1 relation_config cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  3437. if [ -e "$cache_file" ]; then
  3438. # debug "$FUNCNAME: cache hit ($*)"
  3439. cat "$cache_file"
  3440. return 0
  3441. fi
  3442. compose_service_def=$(get_compose_service_def "$service") || return 1
  3443. echo "$compose_service_def" | shyaml get-value "relations" 2>/dev/null | tee "$cache_file"
  3444. }
  3445. export -f get_compose_relation_config
  3446. # ## Return key-values-0
  3447. # get_compose_relation_config_for_service() {
  3448. # local service=$1 relation_name=$2 relation_config
  3449. # compose_service_relations=$(get_compose_relation_config "$service") || return 1
  3450. # if ! relation_config=$(
  3451. # echo "$compose_service_relations" |
  3452. # shyaml get-value "${relation_name}" 2>/dev/null); then
  3453. # err "Couldn't find $DARKYELLOW${service}$NORMAL/${WHITE}${relation_name}$NORMAL" \
  3454. # "relation config in compose configuration."
  3455. # return 1
  3456. # fi
  3457. # if [ -z "$relation_config" ]; then
  3458. # err "Relation ${WHITE}mysql-database$NORMAL is empty in compose configuration."
  3459. # return 1
  3460. # fi
  3461. # if ! echo "$relation_config" | shyaml key-values-0 2>/dev/null; then
  3462. # err "No key/values in ${DARKBLUE}mysql-database$NORMAL of compose config."
  3463. # return 1
  3464. # fi
  3465. # }
  3466. # export -f get_compose_relation_config_for_service
  3467. _get_container_relation() {
  3468. local metadata=$1 found relation_name relation_def
  3469. found=
  3470. while read-0 relation_name relation_def; do
  3471. [ "$(echo "$relation_def" | shyaml get-value "scope" 2>/dev/null)" == "container" ] && {
  3472. found="$relation_name"
  3473. break
  3474. }
  3475. done < <(_get_charm_metadata_uses "$metadata")
  3476. if [ -z "$found" ]; then
  3477. err "Charm $DARKPINK$charm$NORMAL is a subordinate but does not have any required relation declaration with" \
  3478. "${WHITE}scope${NORMAL} set to 'container'."
  3479. return 1
  3480. fi
  3481. printf "%s" "$found"
  3482. }
  3483. _get_master_service_for_service_cached () {
  3484. local service="$1" charm="$2" metadata="$3" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  3485. charm requires master_charm target_charm target_service service_def found
  3486. if [ -e "$cache_file" ]; then
  3487. # debug "$FUNCNAME: STATIC cache hit ($1)"
  3488. cat "$cache_file" &&
  3489. touch "$cache_file" || return 1
  3490. return 0
  3491. fi
  3492. if ! [[ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" =~ ^True|true$ ]]; then
  3493. ## just return service name
  3494. echo "$service" | tee "$cache_file"
  3495. return 0
  3496. fi
  3497. ## Action provided by relation ?
  3498. container_relation=$(_get_container_relation "$metadata") || return 1
  3499. read-0 target_service _ _ < <(get_service_relation "$service" "$container_relation")
  3500. if [ -z "$target_service" ]; then
  3501. err "Couldn't find ${WHITE}relations.${container_relation}${NORMAL} in" \
  3502. "${DARKYELLOW}$service$NORMAL compose definition."
  3503. err ${FUNCNAME[@]}
  3504. return 1
  3505. fi
  3506. echo "$target_service" | tee "$cache_file"
  3507. }
  3508. export -f _get_master_service_for_service_cached
  3509. get_master_service_for_service() {
  3510. if [ -z "$CHARM_STORE_HASH" ]; then
  3511. err-d "Expected \$CHARM_STORE_HASH to be set."
  3512. return 1
  3513. fi
  3514. local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH" \
  3515. charm metadata result
  3516. if [ -e "$cache_file" ]; then
  3517. # debug "$FUNCNAME: SESSION cache hit ($*)"
  3518. cat "$cache_file" || return 1
  3519. return 0
  3520. fi
  3521. charm=$(get_service_charm "$service") || return 1
  3522. metadata=$(charm.metadata "$charm" 2>/dev/null) || {
  3523. metadata=""
  3524. warn "No charm $DARKPINK$charm$NORMAL found."
  3525. }
  3526. result=$(_get_master_service_for_service_cached "$service" "$charm" "$metadata") || return 1
  3527. echo "$result" | tee "$cache_file" || return 1
  3528. }
  3529. export -f get_master_service_for_service
  3530. get_top_master_service_for_service() {
  3531. if [ -z "$CHARM_STORE_HASH" ]; then
  3532. err-d "Expected \$CHARM_STORE_HASH to be set."
  3533. return 1
  3534. fi
  3535. local service="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$1.$CHARM_STORE_HASH" \
  3536. current_service
  3537. if [ -e "$cache_file" ]; then
  3538. # debug "$FUNCNAME: cache hit ($*)"
  3539. touch "$cache_file" || return 1
  3540. cat "$cache_file"
  3541. return 0
  3542. fi
  3543. current_service="$service"
  3544. while true; do
  3545. master_service=$(get_master_service_for_service "$current_service") || return 1
  3546. [ "$master_service" == "$current_service" ] && break
  3547. current_service="$master_service"
  3548. done
  3549. echo "$current_service" | tee "$cache_file"
  3550. return 0
  3551. }
  3552. export -f get_top_master_service_for_service
  3553. ##
  3554. ## The result is a mixin that is not always a complete valid
  3555. ## docker-compose entry (thinking of subordinates). The result
  3556. ## will be merge with master charms.
  3557. _get_docker_compose_mixin_from_metadata_cached() {
  3558. local service="$1" charm="$2" metadata="$3" \
  3559. has_build_dir="$4" \
  3560. cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  3561. metadata_file metadata volumes docker_compose subordinate image \
  3562. mixin mixins tmemory memory limit docker_memory
  3563. if [ -e "$cache_file" ]; then
  3564. #debug "$FUNCNAME: STATIC cache hit $1"
  3565. cat "$cache_file" &&
  3566. touch "$cache_file" || return 1
  3567. return 0
  3568. fi
  3569. mixins=("$(echo -en "labels:\n- compose.charm=$charm")")
  3570. if [ "$metadata" ]; then
  3571. ## resources to volumes
  3572. volumes=$(
  3573. for resource_type in data config; do
  3574. while read-0 resource; do
  3575. eval "echo \" - \$HOST_${resource_type^^}STORE/\$service\$resource:\$resource:rw\""
  3576. done < <(echo "$metadata" | shyaml get-values-0 "${resource_type}-resources" 2>/dev/null)
  3577. done
  3578. while read-0 resource; do
  3579. if [[ "$resource" == /*:/*:* ]]; then
  3580. echo " - $resource"
  3581. elif [[ "$resource" == /*:/* ]]; then
  3582. echo " - $resource:rw"
  3583. elif [[ "$resource" == /*:* ]]; then
  3584. echo " - ${resource%%:*}:$resource"
  3585. elif [[ "$resource" =~ ^/[^:]+$ ]]; then
  3586. echo " - $resource:$resource:rw"
  3587. else
  3588. die "Invalid host-resource specified in 'metadata.yml'."
  3589. fi
  3590. done < <(printf "%s" "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null)
  3591. while read-0 resource; do
  3592. dest="$(charm.get_dir "$charm")/resources$resource"
  3593. if ! [ -e "$dest" ]; then
  3594. die "charm-resource: '$resource' does not exist (file: '$dest')."
  3595. fi
  3596. echo " - $dest:$resource:ro"
  3597. done < <(echo "$metadata" | shyaml get-values-0 "charm-resources" 2>/dev/null)
  3598. ) || return 1
  3599. if [ "$volumes" ]; then
  3600. mixins+=("volumes:"$'\n'"$volumes")
  3601. fi
  3602. type="$(printf "%s" "$metadata" | shyaml get-value type 2>/dev/null)" || true
  3603. if [ "$type" != "run-once" ]; then
  3604. mixins+=("restart: unless-stopped")
  3605. fi
  3606. docker_compose=$(printf "%s" "$metadata" | shyaml get-value -y "docker-compose" 2>/dev/null) || true
  3607. if [ "$docker_compose" ]; then
  3608. mixins+=("$docker_compose")
  3609. fi
  3610. if [[ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" =~ ^True|true$ ]]; then
  3611. subordinate=true
  3612. fi
  3613. fi
  3614. image=$(echo "$metadata" | shyaml get-value "docker-image" 2>/dev/null)
  3615. [ "$image" == "None" ] && image=""
  3616. if [ -n "$image" ]; then
  3617. if [ -n "$subordinate" ]; then
  3618. err "Subordinate charm can not have a ${WHITE}docker-image${NORMAL} value."
  3619. return 1
  3620. fi
  3621. mixins+=("image: $image")
  3622. elif [ "$has_build_dir" ]; then
  3623. if [ "$subordinate" ]; then
  3624. err "Subordinate charm can not have a 'build' sub directory."
  3625. return 1
  3626. fi
  3627. mixins+=("build: $(charm.get_dir "$charm")/build")
  3628. fi
  3629. limit=$(e "$metadata" | yq '.limit' 2>/dev/null) || return 1
  3630. [ "$limit" == "null" ] && limit=""
  3631. if [ -n "$limit" ]; then
  3632. if ! read-0-err E tmemory memory < <(e "$limit" | wyq ".memory | type, .memory") ||
  3633. [ "$E" != 0 ]; then
  3634. err "Unexpected error in ${DARKPINK}$charm${NORMAL}'s metadata when parsing ${WHITE}.limit${NORMAL}"
  3635. return 1
  3636. fi
  3637. case "$tmemory" in
  3638. '!!str'|'!!int')
  3639. docker_memory="$(e "$memory" | numfmt --from iec)" || {
  3640. err "Invalid format specified for .limit.memory: '$memory'."
  3641. return 1
  3642. }
  3643. ;;
  3644. '!!float')
  3645. err "Unexpected value in ${DARKPINK}$charm${NORMAL}'s metadata for ${WHITE}.limit.memory${NORMAL}."
  3646. echo " You need to specify a unit (like 'K', 'M', 'G' ...)." >&2
  3647. return 1
  3648. ;;
  3649. '!!null')
  3650. :
  3651. ;;
  3652. *)
  3653. err "Unexpected type '${tmemory#!!}' in ${DARKPINK}$charm${NORMAL}'s metadata" \
  3654. "for ${WHITE}.limit.memory${NORMAL}."
  3655. echo " You need to check ${DARKPINK}$charm${NORMAL}'s metadata " \
  3656. "for ${WHITE}.limit.memory${NORMAL} and provide a valid value" >&2
  3657. echo " Example values: '1.5G', '252M', ..." >&2
  3658. return 1
  3659. ;;
  3660. esac
  3661. if [ -n "$docker_memory" ]; then
  3662. if [[ "$docker_memory" -lt 6291456 ]]; then
  3663. err "Can't limit service to lower than 6M."
  3664. echo " Specified limit of $memory (=$docker_memory) is lower than docker's min limit of 6M (=6291456)." >&2
  3665. echo " The provided limit to memory is lower than minimum memory for a container." >&2
  3666. echo " Please remove memory limit in ${DARKPINK}$charm${NORMAL}'s metadata or raise it." >&2
  3667. return 1
  3668. fi
  3669. mixins+=(
  3670. "mem_limit: $docker_memory"
  3671. "memswap_limit: $docker_memory"
  3672. )
  3673. fi
  3674. fi
  3675. ## Final merging
  3676. mixin=$(merge_yaml_str "${mixins[@]}") || {
  3677. err "Failed to merge mixins from ${DARKPINK}${charm}${NORMAL} metadata."
  3678. return 1
  3679. }
  3680. echo "$mixin" | tee "$cache_file"
  3681. }
  3682. export -f _get_docker_compose_mixin_from_metadata_cached
  3683. get_docker_compose_mixin_from_metadata() {
  3684. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  3685. if [ -e "$cache_file" ]; then
  3686. #debug "$FUNCNAME: SESSION cache hit ($*)"
  3687. cat "$cache_file"
  3688. return 0
  3689. fi
  3690. charm=$(get_service_charm "$service") || return 1
  3691. metadata="$(charm.metadata "$charm" 2>/dev/null)" || return 1
  3692. has_build_dir=
  3693. [ -d "$(charm.get_dir "$charm")/build" ] && has_build_dir=true
  3694. mixin=$(_get_docker_compose_mixin_from_metadata_cached "$service" "$charm" "$metadata" "$has_build_dir") || return 1
  3695. echo "$mixin" | tee "$cache_file"
  3696. }
  3697. export -f get_docker_compose_mixin_from_metadata
  3698. _save() {
  3699. local name="$1"
  3700. cat - | tee -a "$docker_compose_dir/.data/$name"
  3701. }
  3702. export -f _save
  3703. get_default_project_name() {
  3704. if [ -n "$DEFAULT_PROJECT_NAME" ]; then
  3705. echo "$DEFAULT_PROJECT_NAME"
  3706. return 0
  3707. fi
  3708. local normalized_path compose_yml_location name
  3709. compose_yml_location="$(get_compose_yml_location)" || return 1
  3710. if [ -n "$compose_yml_location" ]; then
  3711. if normalized_path=$(readlink -f "$compose_yml_location"); then
  3712. name="${normalized_path%/*}" ## dirname
  3713. name="${name##*/}" ## basename
  3714. name="${name%%-deploy}" ## remove any '-deploy'
  3715. name="${name,,}" ## lowercase
  3716. e "$name"
  3717. return 0
  3718. fi
  3719. fi
  3720. echo "orphan"
  3721. return 0
  3722. }
  3723. export -f get_default_project_name
  3724. get_running_compose_containers() {
  3725. ## XXXvlab: docker bug: there will be a final newline anyway
  3726. docker ps --filter label="compose.service" --format='{{.ID}}'
  3727. }
  3728. export -f get_running_compose_containers
  3729. get_healthy_container_ip_for_service () {
  3730. local service="$1" port="$2" timeout=${3:-60}
  3731. local containers container container_network container_ip
  3732. containers="$(get_running_containers_for_service "$service")"
  3733. if [ -z "$containers" ]; then
  3734. err "No containers running for service $DARKYELLOW$service$NORMAL."
  3735. return 1
  3736. fi
  3737. if [ "$(echo "$containers" | wc -l)" -gt 1 ]; then
  3738. err "More than 1 container running for service $DARKYELLOW$SERVICE_NAME$NORMAL."
  3739. echo " Please contact administrator to fix this issue." >&2
  3740. return 1
  3741. fi
  3742. ## XXXvlab: taking first container is probably not a good idea
  3743. container="$(echo "$containers" | head -n 1)"
  3744. ## XXXvlab: taking first ip is probably not a good idea
  3745. read-0 container_network container_ip < <(get_container_network_ip "$container")
  3746. if [ -z "$container_ip" ]; then
  3747. err "Can't get container's IP. You should check health of" \
  3748. "${DARKYELLOW}$service${NORMAL}'s container."
  3749. return 1
  3750. fi
  3751. wait_for_tcp_port "$container_network" "$container_ip:$port" "$timeout" || {
  3752. err "TCP port of ${DARKYELLOW}$service${NORMAL}'s container doesn't seem open"
  3753. echo " Please check that container is healthy. Here are last logs:" >&2
  3754. docker logs "$container" --tail=10 | prefix " ${GRAY}|${NORMAL} " >&2
  3755. return 1
  3756. }
  3757. info "Host/Port ${container_ip}:${port} checked ${GREEN}open${NORMAL}."
  3758. echo "$container_network:$container_ip"
  3759. }
  3760. export -f get_healthy_container_ip_for_service
  3761. switch_to_relation_service() {
  3762. local relation="$1"
  3763. ## XXXvlab: can't get real config here
  3764. if ! read-0 ts _ _ < <(get_service_relation "$SERVICE_NAME" "$relation"); then
  3765. err "Couldn't find relation ${DARKCYAN}$relation${NORMAL}."
  3766. return 1
  3767. fi
  3768. export SERVICE_NAME="$ts"
  3769. export SERVICE_DATASTORE="$DATASTORE/$SERVICE_NAME"
  3770. DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$SERVICE_NAME")
  3771. export DOCKER_BASE_IMAGE
  3772. target_charm=$(get_service_charm "$ts") || return 1
  3773. target_charm_path=$(charm.get_dir "$target_charm") || return 1
  3774. cd "$target_charm_path"
  3775. }
  3776. export -f switch_to_relation_service
  3777. get_volumes_for_container() {
  3778. local container="$1"
  3779. docker inspect \
  3780. --format '{{range $mount := .Mounts}}{{$mount.Source}}{{"\x00"}}{{$mount.Destination}}{{"\x00"}}{{end}}' \
  3781. "$container"
  3782. }
  3783. export -f get_volumes_for_container
  3784. is_volume_used() {
  3785. local volume="$1" container_id src dst
  3786. while read -r container_id; do
  3787. while read-0 src dst; do
  3788. [[ "$src/" == "$volume"/* ]] && return 0
  3789. done < <(get_volumes_for_container "$container_id")
  3790. done < <(get_running_compose_containers)
  3791. return 1
  3792. }
  3793. export -f is_volume_used
  3794. clean_unused_docker_compose() {
  3795. for f in /var/lib/compose/docker-compose/*; do
  3796. [ -e "$f" ] || continue
  3797. is_volume_used "$f" && continue
  3798. debug "Cleaning unused docker-compose ${f##*/}"
  3799. rm -rf "$f" || return 1
  3800. done
  3801. return 0
  3802. }
  3803. export -f clean_unused_docker_compose
  3804. docker_compose_store() {
  3805. local file="$1" sha
  3806. sha=$(hash_get 64 < "$file") || return 1
  3807. project=$(get_default_project_name) || return 1
  3808. dst="/var/lib/compose/docker-compose/$sha/$project"
  3809. mkdir -p "$dst" || return 1
  3810. cat <<EOF > "$dst/.env" || return 1
  3811. DOCKER_COMPOSE_PATH=$dst
  3812. COMPOSE_HTTP_TIMEOUT=7200
  3813. EOF
  3814. cp "$file" "$dst/docker-compose.yml" || return 1
  3815. mkdir -p "$dst/bin" || return 1
  3816. cat <<EOF > "$dst/bin/dc" || return 1
  3817. #!/bin/bash
  3818. $(declare -f read-0)
  3819. docker_run_opts=()
  3820. while read-0 opt; do
  3821. if [[ "\$opt" == "!env:"* ]]; then
  3822. opt="\${opt##!env:}"
  3823. var="\${opt%%=*}"
  3824. value="\${opt#*=}"
  3825. export "\$var"="\$value"
  3826. else
  3827. docker_run_opts+=("\$opt")
  3828. fi
  3829. done < <(cat "$COMPOSE_LAUNCHER_OPTS")
  3830. docker_run_opts+=(
  3831. "-w" "$dst"
  3832. "--entrypoint" "/usr/local/bin/docker-compose"
  3833. )
  3834. [ -t 1 ] && {
  3835. docker_run_opts+=("-ti")
  3836. }
  3837. exec docker run --rm "\${docker_run_opts[@]}" "\${COMPOSE_DOCKER_IMAGE:-docker.0k.io/compose}" "\$@"
  3838. EOF
  3839. chmod +x "$dst/bin/dc" || return 1
  3840. printf "%s" "$sha"
  3841. }
  3842. export -f docker_compose_store
  3843. launch_docker_compose() {
  3844. local charm docker_compose_tmpdir docker_compose_dir
  3845. docker_compose_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX)
  3846. #debug "Creating temporary docker-compose directory in '$docker_compose_tmpdir'."
  3847. trap_add EXIT "rm -rf \"$docker_compose_tmpdir\""
  3848. ## docker-compose will name network from the parent dir name
  3849. project=$(get_default_project_name)
  3850. mkdir -p "$docker_compose_tmpdir/$project"
  3851. docker_compose_dir="$docker_compose_tmpdir/$project"
  3852. if [ -z "$_CURRENT_DOCKER_COMPOSE" ]; then
  3853. err "${FUNCNAME[0]} is meant to be called after"\
  3854. "\$_CURRENT_DOCKER_COMPOSE has been calculated."
  3855. echo " Called by:" >&2
  3856. printf " - %s\n" "${FUNCNAME[@]:1}" >&2
  3857. return 1
  3858. fi
  3859. cat "$_CURRENT_DOCKER_COMPOSE" > "$docker_compose_dir/docker-compose.yml" || return 1
  3860. if [ -e "$state_tmpdir/to-merge-in-docker-compose.yml" ]; then
  3861. # debug "Merging some config data in docker-compose.yml:"
  3862. # debug "$(cat $state_tmpdir/to-merge-in-docker-compose.yml)"
  3863. _config_merge "$docker_compose_dir/docker-compose.yml" "$state_tmpdir/to-merge-in-docker-compose.yml" || return 1
  3864. fi
  3865. if [ -z "$(echo $(cat "$docker_compose_dir/docker-compose.yml"))" ]; then
  3866. die "Generated 'docker-compose.yml' is unexpectedly empty."
  3867. fi
  3868. ## XXXvlab: could be more specific and only link the needed charms
  3869. ## XXXvlab: why do we need these links ? If this is for the build command, then it is not useful anymore.
  3870. # for charm in $(shyaml keys services < "$docker_compose_dir/docker-compose.yml"); do
  3871. # if charm.exists "$charm"; then
  3872. # ln -sf "$(charm.get_dir "$charm")" "$docker_compose_dir/$charm" || exit 1
  3873. # fi
  3874. # done
  3875. mkdir "$docker_compose_dir/.data"
  3876. if [ -z "$COMPOSE_DISABLE_DOCKER_COMPOSE_STORE" ]; then
  3877. sha=$(docker_compose_store "$docker_compose_dir/docker-compose.yml") || return 1
  3878. fi
  3879. {
  3880. {
  3881. {
  3882. if [ -z "$COMPOSE_DISABLE_DOCKER_COMPOSE_STORE" ]; then
  3883. cd "/var/lib/compose/docker-compose/$sha/$project" || return 1
  3884. else
  3885. cd "$docker_compose_dir" || return 1
  3886. fi
  3887. if [ -f ".env" ]; then
  3888. debug "${WHITE}.env$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:"
  3889. debug "$(cat ".env" | prefix " $GRAY|$NORMAL ")"
  3890. fi
  3891. debug "${WHITE}docker-compose.yml$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:"
  3892. debug "$(cat "docker-compose.yml" | prefix " $GRAY|$NORMAL ")"
  3893. debug "${WHITE}Launching$NORMAL: docker-compose $@"
  3894. if [ "$DRY_COMPOSE_RUN" ]; then
  3895. echo docker-compose "$@"
  3896. else
  3897. docker-compose "$@"
  3898. fi
  3899. echo "$?" > "$docker_compose_dir/.data/errlvl"
  3900. } | _save stdout
  3901. } 3>&1 1>&2 2>&3 | _save stderr
  3902. } 3>&1 1>&2 2>&3
  3903. if tail -n 1 "$docker_compose_dir/.data/stderr" | grep -E "Service .+ failed to build: Error getting container [0-9a-f]+ from driver devicemapper: (open|Error mounting) /dev/mapper/docker-.*: no such file or directory$" >/dev/null 2>&1; then
  3904. err "Detected bug https://github.com/docker/docker/issues/4036 ... "
  3905. err "Please re-launch your command, or switch from 'devicemapper' driver to 'overlayfs' or 'aufs'."
  3906. fi
  3907. docker_compose_errlvl="$(cat "$docker_compose_dir/.data/errlvl" 2>/dev/null)"
  3908. if [ -z "$docker_compose_errlvl" ]; then
  3909. err "Something went wrong before you could gather docker-compose errorlevel."
  3910. return 1
  3911. fi
  3912. return "$docker_compose_errlvl"
  3913. }
  3914. export -f launch_docker_compose
  3915. get_compose_yml_location() {
  3916. if ! [ -z ${COMPOSE_YML_FILE+x} ]; then ## if set, even if empty
  3917. echo "$COMPOSE_YML_FILE"
  3918. return 0
  3919. fi
  3920. parent=$(while ! [ -f "./compose.yml" ]; do
  3921. [ "$PWD" == "/" ] && exit 0
  3922. cd ..
  3923. done; echo "$PWD"
  3924. )
  3925. if [ "$parent" ]; then
  3926. echo "$parent/compose.yml"
  3927. return 0
  3928. fi
  3929. ## XXXvlab: do we need this additional environment variable,
  3930. ## COMPOSE_YML_FILE is not sufficient ?
  3931. if [ "$DEFAULT_COMPOSE_FILE" ]; then
  3932. if ! [ -e "$DEFAULT_COMPOSE_FILE" ]; then
  3933. warn "No 'compose.yml' was found in current or parent dirs," \
  3934. "and \$DEFAULT_COMPOSE_FILE points to an unexistent file." \
  3935. "(${DEFAULT_COMPOSE_FILE})"
  3936. return 0
  3937. fi
  3938. echo "$DEFAULT_COMPOSE_FILE"
  3939. return 0
  3940. fi
  3941. warn "No 'compose.yml' was found in current or parent dirs, and no \$DEFAULT_COMPOSE_FILE was set."
  3942. return 0
  3943. }
  3944. export -f get_compose_yml_location
  3945. get_compose_yml_content() {
  3946. local cache_file="$state_tmpdir/$FUNCNAME.cache"
  3947. if [ -e "$cache_file" ]; then
  3948. cat "$cache_file" &&
  3949. touch "$cache_file" || return 1
  3950. return 0
  3951. fi
  3952. if [ -z "$COMPOSE_YML_FILE" ]; then
  3953. COMPOSE_YML_FILE=$(get_compose_yml_location) || return 1
  3954. fi
  3955. if [ -e "$COMPOSE_YML_FILE" ]; then
  3956. # debug "Found $WHITE$exname$NORMAL YAML file in '$COMPOSE_YML_FILE'."
  3957. COMPOSE_YML_CONTENT=$(cat "$COMPOSE_YML_FILE") || {
  3958. err "Could not read '$COMPOSE_YML_FILE'."
  3959. return 1
  3960. }
  3961. else
  3962. debug "No compose file found. Using an empty one."
  3963. COMPOSE_YML_CONTENT=""
  3964. fi
  3965. COMPOSE_YML_CONTENT=$(merge_yaml_str "$COMPOSE_YML_CONTENT" "${compose_contents[@]}") || return 1
  3966. output=$(echo "$COMPOSE_YML_CONTENT"| shyaml get-value 2>&1)
  3967. if [ "$?" != 0 ]; then
  3968. outputed_something=
  3969. while IFS='' read -r line1 && IFS='' read -r line2; do
  3970. [ "$outputed_something" ] || err "Invalid YAML in '$COMPOSE_YML_FILE':"
  3971. outputed_something=true
  3972. echo "$line1 $GRAY($line2)$NORMAL"
  3973. done < <(echo "$output" | grep ^yaml.scanner -A 100 |
  3974. sed -r 's/^ in "<stdin>", //g' | sed -r 's/^yaml.scanner.[a-zA-Z]+: //g') |
  3975. prefix " $GRAY|$NORMAL "
  3976. [ "$outputed_something" ] || {
  3977. err "Unexpected error while running 'shyaml get-value' on '$COMPOSE_YML_FILE':"
  3978. echo "$output" | prefix " $GRAY|$NORMAL "
  3979. }
  3980. return 1
  3981. fi
  3982. echo "$COMPOSE_YML_CONTENT" | tee "$cache_file" || return 1
  3983. }
  3984. export -f get_compose_yml_content
  3985. compose:yml:hash() {
  3986. local cache_file="$state_tmpdir/$FUNCNAME.cache"
  3987. if [ -e "$cache_file" ]; then
  3988. cat "$cache_file" &&
  3989. touch "$cache_file" || return 1
  3990. return 0
  3991. fi
  3992. compose_yml_content=$(get_compose_yml_content) || return 1
  3993. compose_yml_hash=$(echo "$compose_yml_content" | hash_get) || return 1
  3994. e "$compose_yml_hash" | tee "$cache_file" || return 1
  3995. }
  3996. export -f compose:yml:hash
  3997. compose:yml:root:services() {
  3998. local cache_file="$state_tmpdir/$FUNCNAME.cache" services compose_yml_content
  3999. if [ -e "$cache_file" ]; then
  4000. cat "$cache_file" &&
  4001. touch "$cache_file" || return 1
  4002. return 0
  4003. fi
  4004. compose_yml_content=$(get_compose_yml_content) || return 1
  4005. services=($(e "$compose_yml_content" | shyaml keys)) || return 1
  4006. e "${services[*]}" | tee "$cache_file" || return 1
  4007. }
  4008. export -f compose:yml:root:services
  4009. get_default_target_services() {
  4010. local services=("$@")
  4011. if [ -z "${services[*]}" ]; then
  4012. if [ "$DEFAULT_SERVICES" ]; then
  4013. debug "No service provided, using $WHITE\$DEFAULT_SERVICES$NORMAL variable." \
  4014. "Target services: $DARKYELLOW$DEFAULT_SERVICES$NORMAL"
  4015. services="$DEFAULT_SERVICES"
  4016. else
  4017. err "No service provided."
  4018. return 1
  4019. fi
  4020. fi
  4021. echo "${services[*]}"
  4022. }
  4023. export -f get_default_target_services
  4024. get_master_services() {
  4025. local loaded master_service service
  4026. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(H "$@" )"
  4027. if [ -e "$cache_file" ]; then
  4028. cat "$cache_file" &&
  4029. touch "$cache_file" || return 1
  4030. return 0
  4031. fi
  4032. declare -A loaded
  4033. for service in "$@"; do
  4034. master_service=$(get_top_master_service_for_service "$service") || return 1
  4035. if [ "${loaded[$master_service]}" ]; then
  4036. continue
  4037. fi
  4038. echo "$master_service"
  4039. loaded["$master_service"]=1
  4040. done > "$cache_file".wip || return 1
  4041. mv "$cache_file"{.wip,} || return 1
  4042. cat "$cache_file" || return 1
  4043. }
  4044. export -f get_master_services
  4045. get_current_docker_container_id() {
  4046. local line
  4047. line=$(cat "/proc/self/cpuset") || return 1
  4048. [[ "$line" == *docker* ]] || return 1
  4049. echo "${line##*/}"
  4050. }
  4051. export -f get_current_docker_container_id
  4052. ## if we are in a docker compose, we might want to know what is the
  4053. ## real host path of some local paths.
  4054. get_host_path() {
  4055. local path="$1"
  4056. path=$(realpath "$path") || return 1
  4057. container_id=$(get_current_docker_container_id) || {
  4058. print "%s" "$path"
  4059. return 0
  4060. }
  4061. biggest_dst=
  4062. current_src=
  4063. while read-0 src dst; do
  4064. [[ "$path" == "$dst"* ]] || continue
  4065. if [[ "${#biggest_dst}" < "${#dst}" ]]; then
  4066. biggest_dst="$dst"
  4067. current_src="$src"
  4068. fi
  4069. done < <(get_volumes_for_container "$container_id")
  4070. if [ "$current_src" ]; then
  4071. printf "%s" "$current_src"
  4072. else
  4073. return 1
  4074. fi
  4075. }
  4076. export -f get_host_path
  4077. _setup_state_dir() {
  4078. export state_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX)
  4079. #debug "Creating temporary state directory in '$state_tmpdir'."
  4080. # trap_add EXIT "debug \"Removing temporary state directory in $state_tmpdir.\";\
  4081. # rm -rf \"$state_tmpdir\""
  4082. trap_add EXIT "rm -rf \"$state_tmpdir\""
  4083. }
  4084. get_docker_compose_help_msg() {
  4085. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  4086. docker_compose_help_msg
  4087. if [ -e "$cache_file" ]; then
  4088. cat "$cache_file" &&
  4089. touch "$cache_file" || return 1
  4090. return 0
  4091. fi
  4092. docker_compose_help_msg=$(docker-compose $action --help 2>/dev/null) || return 1
  4093. echo "$docker_compose_help_msg" |
  4094. tee "$cache_file" || return 1
  4095. }
  4096. get_docker_compose_usage() {
  4097. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  4098. docker_compose_help_msg
  4099. if [ -e "$cache_file" ]; then
  4100. cat "$cache_file" &&
  4101. touch "$cache_file" || return 1
  4102. return 0
  4103. fi
  4104. docker_compose_help_msg=$(get_docker_compose_help_msg $action) || return 1
  4105. echo "$docker_compose_help_msg" |
  4106. grep -m 1 "^Usage:" -A 10000 |
  4107. egrep -m 1 "^\$" -B 10000 |
  4108. nspc |
  4109. sed -r 's/^Usage: //g' |
  4110. tee "$cache_file" || return 1
  4111. }
  4112. get_docker_compose_opts_help() {
  4113. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  4114. docker_compose_help_msg
  4115. if [ -e "$cache_file" ]; then
  4116. cat "$cache_file" &&
  4117. touch "$cache_file" || return 1
  4118. return 0
  4119. fi
  4120. docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1
  4121. echo "$docker_compose_opts_help" |
  4122. grep '^Options:' -A 20000 |
  4123. tail -n +2 |
  4124. { cat ; echo; } |
  4125. egrep -m 1 "^\S*\$" -B 10000 |
  4126. head -n -1 |
  4127. tee "$cache_file" || return 1
  4128. }
  4129. get_docker_compose_commands_help() {
  4130. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  4131. docker_compose_help_msg
  4132. if [ -e "$cache_file" ]; then
  4133. cat "$cache_file" &&
  4134. touch "$cache_file" || return 1
  4135. return 0
  4136. fi
  4137. docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1
  4138. echo "$docker_compose_opts_help" |
  4139. grep '^Commands:' -A 20000 |
  4140. tail -n +2 |
  4141. { cat ; echo; } |
  4142. egrep -m 1 "^\S*\$" -B 10000 |
  4143. head -n -1 |
  4144. tee "$cache_file" || return 1
  4145. }
  4146. get_docker_compose_opts_list() {
  4147. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$({ p0 "$1"; cat "$(which docker-compose)"; } | md5_compat)" \
  4148. docker_compose_help_msg
  4149. if [ -e "$cache_file" ]; then
  4150. cat "$cache_file" &&
  4151. touch "$cache_file" || return 1
  4152. return 0
  4153. fi
  4154. docker_compose_opts_help=$(get_docker_compose_opts_help $action) || return 1
  4155. echo "$docker_compose_opts_help" |
  4156. egrep "^\s+-" |
  4157. sed -r 's/\s+((((-[a-zA-Z]|--[a-zA-Z0-9-]+)( [A-Z=]+|=[^ ]+)?)(, )?)+)\s+.*$/\1/g' |
  4158. tee "$cache_file" || return 1
  4159. }
  4160. options_parser() {
  4161. sed -r 's/^(\s+(((-[a-zA-Z]|--[a-zA-Z0-9-]+)([ =]([a-zA-Z_=\"\[]|\])+)?(, | )?)+)\s+)[^ ].*$/\x0\2\x0\0/g'
  4162. printf "\0"
  4163. }
  4164. remove_options_in_option_help_msg() {
  4165. {
  4166. read-0 null
  4167. if [ "$null" ]; then
  4168. err "options parsing error, should start with an option line."
  4169. return 1
  4170. fi
  4171. while read-0 opt full_txt;do
  4172. multi_opts="$(printf "%s " $opt | multi_opts_filter)"
  4173. single_opts="$(printf "%s " $opt | single_opts_filter)"
  4174. for to_remove in "$@"; do
  4175. str_matches "$to_remove" $multi_opts $single_opts && {
  4176. continue 2
  4177. }
  4178. done
  4179. echo -n "$full_txt"
  4180. done
  4181. } < <(options_parser)
  4182. }
  4183. _MULTIOPTION_REGEX='^((-[a-zA-Z]|--[a-zA-Z0-9-]+)(, )?)+'
  4184. _MULTIOPTION_REGEX_LINE_FILTER=$_MULTIOPTION_REGEX'(\s|=)'
  4185. multi_opts_filter() {
  4186. egrep "$_MULTIOPTION_REGEX_LINE_FILTER" |
  4187. sed -r "s/^($_MULTIOPTION_REGEX)(\s|=).*$/\1/g" |
  4188. tr ',' "\n" | nspc
  4189. }
  4190. single_opts_filter() {
  4191. egrep -v "$_MULTIOPTION_REGEX_LINE_FILTER" |
  4192. tr ',' "\n" | nspc
  4193. }
  4194. get_docker_compose_multi_opts_list() {
  4195. local action="$1" opts_list
  4196. opts_list=$(get_docker_compose_opts_list "$action") || return 1
  4197. echo "$opts_list" | multi_opts_filter
  4198. }
  4199. get_docker_compose_single_opts_list() {
  4200. local action="$1" opts_list
  4201. opts_list=$(get_docker_compose_opts_list "$action") || return 1
  4202. echo "$opts_list" | single_opts_filter
  4203. }
  4204. display_commands_help() {
  4205. local charm_actions
  4206. echo
  4207. echo "${WHITE}Commands${NORMAL} (added by compose):"
  4208. echo " ${DARKCYAN}cache${NORMAL} Control compose's cache"
  4209. echo " ${DARKCYAN}status${NORMAL} Display statuses of services"
  4210. echo
  4211. echo "${WHITE}Commands${NORMAL} (thanks to docker-compose):"
  4212. get_docker_compose_commands_help | sed -r "s/ ([a-z]+)(\s+)/ ${DARKCYAN}\1${NORMAL}\2/g"
  4213. charm_actions_help=$(get_docker_charm_action_help) || return 1
  4214. if [ "$charm_actions_help" ]; then
  4215. echo
  4216. echo "${WHITE}Charm actions${NORMAL}:"
  4217. printf "%s\n" "$charm_actions_help" | \
  4218. sed -r "s/^ ([a-z0-9-]+)(\s+)([a-z0-9-]+)(\s+)/ ${DARKCYAN}\1${NORMAL}\2${DARKYELLOW}\3${NORMAL}\4/g"
  4219. fi
  4220. }
  4221. get_docker_charm_action() {
  4222. local services service charm relation_name target_service relation_config \
  4223. target_charm services
  4224. ## XXXvlab: this is for get_service_relations
  4225. NO_CONSTRAINT_CHECK=True service:all:set_relations_hash || {
  4226. err-d "Failed to set relations hash."
  4227. return 1
  4228. }
  4229. services=($(get_all_services)) || return 1
  4230. for service in "${services[@]}"; do
  4231. printf "%s:\n" "$service"
  4232. charm=$(get_service_charm "$service") || return 1
  4233. for action in $(charm.ls_direct_actions "$charm"); do
  4234. printf " %s:\n" "$action"
  4235. printf " type: %s\n" "direct"
  4236. done
  4237. while read-0 relation_name target_service _relation_config _tech_dep; do
  4238. target_charm=$(get_service_charm "$target_service") || return 1
  4239. for action in $(charm.ls_relation_actions "$target_charm" "$relation_name"); do
  4240. printf " %s:\n" "$action"
  4241. printf " type: %s\n" "indirect"
  4242. printf " inherited: %s\n" "$target_charm"
  4243. done
  4244. done < <(get_service_relations "$service")
  4245. done
  4246. }
  4247. export -f get_docker_charm_action
  4248. get_docker_charm_action_help() {
  4249. local services service charm relation_name target_service relation_config \
  4250. target_charm
  4251. ## XXXvlab: this is for get_service_relations
  4252. NO_CONSTRAINT_CHECK=True service:all:set_relations_hash || {
  4253. err-d "Failed to set relations hash."
  4254. return 1
  4255. }
  4256. services=($(get_all_services)) || return 1
  4257. for service in "${services[@]}"; do
  4258. out=$(
  4259. charm=$(get_service_charm "$service") || return 1
  4260. for action in $(charm.ls_direct_actions "$charm"); do
  4261. printf " %-28s %s\n" "$action $service" "Direct action from ${DARKPINK}$charm${NORMAL}"
  4262. done
  4263. while read-0 relation_name target_service _relation_config _tech_dep; do
  4264. target_charm=$(get_service_charm "$target_service") || return 1
  4265. for action in $(charm.ls_relation_actions "$target_charm" "$relation_name"); do
  4266. printf " %-28s %s\n" "$action $service" "Indirect action from ${DARKPINK}$target_charm${NORMAL}"
  4267. done
  4268. done < <(get_service_relations "$service")
  4269. )
  4270. if [ "$out" ]; then
  4271. echo " for ${DARKYELLOW}$service${NORMAL}:"
  4272. printf "%s\n" "$out"
  4273. fi
  4274. done
  4275. }
  4276. display_help() {
  4277. print_help
  4278. echo "${WHITE}Usage${NORMAL}:"
  4279. echo " $usage"
  4280. echo " $usage cache {clean|clear}"
  4281. echo "${WHITE}Options${NORMAL}:"
  4282. echo " -h, --help Print this message and quit"
  4283. echo " (ignoring any other options)"
  4284. echo " -V, --version Print current version and quit"
  4285. echo " (ignoring any other options)"
  4286. echo " --dirs Display data dirs and quit"
  4287. echo " (ignoring any other options)"
  4288. echo " --get-project-name Display project name and quit"
  4289. echo " (ignoring any other options)"
  4290. echo " --get-available-actions Display all available actions and quit"
  4291. echo " (ignoring any other options)"
  4292. echo " -v, --verbose Be more verbose"
  4293. echo " -q, --quiet Be quiet"
  4294. echo " -d, --debug Print full debugging information (sets also verbose)"
  4295. echo " --dry-compose-run If docker-compose will be run, only print out what"
  4296. echo " command line will be used."
  4297. echo " --no-relations Do not run any relation script"
  4298. echo " --no-hooks Do not run any hook script"
  4299. echo " --no-init Do not run any init script"
  4300. echo " --no-post-deploy Do not run any post-deploy script"
  4301. echo " --no-pre-deploy Do not run any pre-deploy script"
  4302. echo " --without-relation RELATION "
  4303. echo " Do not run given relation"
  4304. echo " -R, --rebuild-relations-to-service SERVICE"
  4305. echo " Will rebuild all relations to given service"
  4306. echo " --add-compose-content, -Y YAML"
  4307. echo " Will merge some direct YAML with the current compose"
  4308. echo " -c, --color Force color mode (default is to detect if in tty mode)"
  4309. echo " --push-builds Will push cached docker images to docker cache registry"
  4310. get_docker_compose_opts_help | remove_options_in_option_help_msg --version --help --verbose |
  4311. filter_docker_compose_help_message
  4312. display_commands_help
  4313. }
  4314. _graph_service() {
  4315. local service="$1" base="$1"
  4316. charm=$(get_service_charm "$service") || return 1
  4317. metadata=$(charm.metadata "$charm") || return 1
  4318. subordinate=$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)
  4319. if [[ "$subordinate" =~ ^True|true$ ]]; then
  4320. requires="$(echo "$metadata" | shyaml get-value "requires" 2>/dev/null)"
  4321. master_charm=
  4322. while read-0 relation_name relation; do
  4323. [ "$(echo "$relation" | shyaml get-value "scope" 2>/dev/null)" == "container" ] || continue
  4324. interface="$(echo "$relation" | shyaml get-value "interface" 2>/dev/null)"
  4325. if [ -z "$interface" ]; then
  4326. err "No ${WHITE}$interface${NORMAL} set for relation $relation_name."
  4327. return 1
  4328. fi
  4329. ## Action provided by relation ?
  4330. target_service=
  4331. while read-0 relation_name candidate_target_service _relation_config _tech_dep; do
  4332. [ "$interface" == "$relation_name" ] && {
  4333. target_service="$candidate_target_service"
  4334. break
  4335. }
  4336. done < <(get_service_relations "$service")
  4337. if [ -z "$target_service" ]; then
  4338. err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \
  4339. "${DARKYELLOW}$service$NORMAL compose definition."
  4340. return 1
  4341. fi
  4342. master_service="$target_service"
  4343. master_charm=$(get_service_charm "$target_service") || return 1
  4344. break
  4345. done < <(echo "$requires" | shyaml key-values-0 2>/dev/null)
  4346. fi
  4347. _graph_node_service "$service" "$base" "$charm"
  4348. _graph_edge_service "$service" "$subordinate" "$master_service"
  4349. }
  4350. _graph_node_service() {
  4351. local service="$1" base="$2" charm="$3"
  4352. cat <<EOF
  4353. "$(_graph_node_service_label ${service})" [
  4354. style = "filled, $([[ "$subordinate" =~ ^True|true$ ]] && echo "dashed" || echo "bold")"
  4355. penwidth = $([[ "$subordinate" =~ ^True|true$ ]] && echo "3" || echo "5")
  4356. color = $([ "$base" ] && echo "blue" || echo "black")
  4357. fillcolor = "white"
  4358. fontname = "Courier New"
  4359. shape = "Mrecord"
  4360. label =<$(_graph_node_service_content "$service")>
  4361. ];
  4362. EOF
  4363. }
  4364. _graph_edge_service() {
  4365. local service="$1" subordinate="$2" master_service="$3"
  4366. while read-0 relation_name target_service relation_config tech_dep; do
  4367. cat <<EOF
  4368. "$(_graph_node_service_label ${service})" -> "$(_graph_node_service_label ${target_service})" [
  4369. penwidth = $([ "$master_service" == "$target_service" ] && echo 3 || echo 2)
  4370. fontsize = 16
  4371. fontcolor = "black"
  4372. style = $([ "$master_service" == "$target_service" ] && echo dashed || echo "\"\"")
  4373. weight = $([ "$master_service" == "$target_service" ] && echo 2.0 || echo 1.0)
  4374. dir = $([ "$master_service" == "$target_service" ] && echo none || echo both)
  4375. arrowtail = odot
  4376. # arrowhead = dotlicurve
  4377. taillabel = "$relation_name" ];
  4378. EOF
  4379. done < <(get_service_relations "$service") || return 1
  4380. }
  4381. _graph_node_service_label() {
  4382. local service="$1"
  4383. echo "service_$service"
  4384. }
  4385. _graph_node_service_content() {
  4386. local service="$1"
  4387. charm=$(get_service_charm "$service") || return 1
  4388. cat <<EOF
  4389. <table border="0" cellborder="0" cellpadding="3" bgcolor="white">
  4390. <tr>
  4391. <td bgcolor="black" align="center" colspan="2">
  4392. <font color="white">$service</font>
  4393. </td>
  4394. </tr>
  4395. $(if [ "$charm" != "$service" ]; then
  4396. cat <<EOF2
  4397. <tr>
  4398. <td align="left" port="r0">charm: $charm</td>
  4399. </tr>
  4400. EOF2
  4401. fi)
  4402. </table>
  4403. EOF
  4404. }
  4405. cla_contains () {
  4406. local e
  4407. for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done
  4408. return 1
  4409. }
  4410. filter_docker_compose_help_message() {
  4411. cat - |
  4412. sed -r "s/docker-compose run/${DARKWHITE}compose${NORMAL} ${DARKCYAN}$action${NORMAL}/g;
  4413. s/docker-compose.yml/compose.yml/g;
  4414. s/SERVICES?/${DARKYELLOW}\0${NORMAL}/g;
  4415. s/^(\s+)\\$/\1${WHITE}\$${NORMAL}/g;
  4416. s/^(\s+)run/\1${DARKCYAN}$action${NORMAL}/g;
  4417. s/docker-compose/${DARKWHITE}compose${NORMAL}/g"
  4418. }
  4419. graph() {
  4420. local services=("$@")
  4421. declare -A entries
  4422. cat <<EOF
  4423. digraph g {
  4424. graph [
  4425. fontsize=30
  4426. labelloc="t"
  4427. label=""
  4428. splines=true
  4429. overlap=false
  4430. #rankdir = "LR"
  4431. ];
  4432. ratio = auto;
  4433. EOF
  4434. for target_service in "$@"; do
  4435. services=$(get_ordered_service_dependencies "$target_service") || return 1
  4436. for service in $services; do
  4437. [ "${entries[$service]}" ] && continue || entries[$service]=1
  4438. if cla_contains "$service" "${services[@]}"; then
  4439. base=true
  4440. else
  4441. base=
  4442. fi
  4443. _graph_service "$service" "$base"
  4444. done
  4445. done
  4446. echo "}"
  4447. }
  4448. cached_wget() {
  4449. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(p0 "$@" | md5_compat)" \
  4450. url="$1"
  4451. if [ -e "$cache_file" ]; then
  4452. cat "$cache_file"
  4453. touch "$cache_file"
  4454. return 0
  4455. fi
  4456. wget -O- "${url}" |
  4457. tee "$cache_file"
  4458. if [ "${PIPESTATUS[0]}" != 0 ]; then
  4459. rm "$cache_file"
  4460. die "Unable to fetch '$url'."
  4461. return 1
  4462. fi
  4463. }
  4464. export -f cached_wget
  4465. [ "$SOURCED" ] && return 0
  4466. trap_add "EXIT" clean_cache
  4467. export COMPOSE_DOCKER_REGISTRY="${COMPOSE_DOCKER_REGISTRY:-docker.0k.io}"
  4468. if [ -z "$DISABLE_SYSTEM_CONFIG_FILE" ]; then
  4469. if [ -r /etc/default/charm ]; then
  4470. . "/etc/default/charm"
  4471. fi
  4472. if [ -r "/etc/default/$exname" ]; then
  4473. . "/etc/default/$exname"
  4474. fi
  4475. ## XXXvlab: should provide YML config opportunities in possible parent dirs ?
  4476. ## userdir ? and global /etc/compose.yml ?
  4477. for cfgfile in /etc/compose.conf /etc/compose.local.conf \
  4478. /etc/default/compose /etc/compose/local.conf; do
  4479. [ -e "$cfgfile" ] || continue
  4480. . "$cfgfile" || die "Loading config file '$cfgfile' failed."
  4481. done
  4482. fi
  4483. _setup_state_dir
  4484. mkdir -p "$CACHEDIR" || exit 1
  4485. log () { cat; }
  4486. export -f log
  4487. ##
  4488. ## Argument parsing
  4489. ##
  4490. wrap_opts=()
  4491. services=()
  4492. remainder_args=()
  4493. compose_opts=()
  4494. compose_contents=()
  4495. action_opts=()
  4496. services_args=()
  4497. pos_arg_ct=0
  4498. no_hooks=
  4499. no_init=
  4500. action=
  4501. stage="main" ## switches from 'main', to 'action', 'remainder'
  4502. is_docker_compose_action=
  4503. is_docker_compose_action_multi_service=
  4504. rebuild_relations_to_service=()
  4505. color=
  4506. declare -A without_relations
  4507. DC_MATCH_MULTI=$(get_docker_compose_multi_opts_list) &&
  4508. DC_MATCH_SINGLE=$(get_docker_compose_single_opts_list) || exit 1
  4509. while read-0 arg; do
  4510. case "$stage" in
  4511. "main")
  4512. case "$arg" in
  4513. --help|-h)
  4514. no_init=true ; no_hooks=true ; no_relations=true
  4515. display_help
  4516. exit 0
  4517. ;;
  4518. --verbose|-v)
  4519. export VERBOSE=true
  4520. compose_opts+=("--verbose")
  4521. ;;
  4522. --quiet|-q)
  4523. export QUIET=true
  4524. export wrap_opts+=("-q")
  4525. log () { cat >&2; }
  4526. export -f log
  4527. ;;
  4528. --version|-V)
  4529. print_version
  4530. docker-compose --version
  4531. docker --version
  4532. exit 0
  4533. ;;
  4534. -f|--file)
  4535. read-0 value
  4536. [ -e "$value" ] || die "File $value doesn't exists"
  4537. export COMPOSE_YML_FILE="$value"
  4538. shift
  4539. ;;
  4540. -p|--project-name)
  4541. read-0 value
  4542. export DEFAULT_PROJECT_NAME="$value"
  4543. compose_opts+=("--project-name $value")
  4544. shift
  4545. ;;
  4546. --color|-c)
  4547. if [ "$color" == "0" ]; then
  4548. err "Conflicting option --color with previous --no-ansi."
  4549. exit 1
  4550. fi
  4551. color=1
  4552. ansi_color yes
  4553. ;;
  4554. --no-ansi)
  4555. if [ "$color" == "1" ]; then
  4556. err "Conflicting option --no-ansi with previous --color."
  4557. exit 1
  4558. fi
  4559. color=0
  4560. ansi_color no
  4561. compose_opts+=("--no-ansi")
  4562. ;;
  4563. --no-relations)
  4564. export no_relations=true
  4565. ;;
  4566. --without-relation)
  4567. read-0 value
  4568. without_relations["$value"]=1
  4569. shift
  4570. ;;
  4571. --no-hooks)
  4572. export no_hooks=true
  4573. ;;
  4574. --no-init)
  4575. export no_init=true
  4576. ;;
  4577. --no-post-deploy)
  4578. export no_post_deploy=true
  4579. ;;
  4580. --no-pre-deploy)
  4581. export no_pre_deploy=true
  4582. ;;
  4583. --rebuild-relations-to-service|-R)
  4584. read-0 value
  4585. rebuild_relations_to_service+=("$value")
  4586. shift
  4587. ;;
  4588. --push-builds)
  4589. export COMPOSE_PUSH_TO_REGISTRY=1
  4590. ;;
  4591. --debug|-d)
  4592. export DEBUG=true
  4593. export VERBOSE=true
  4594. #compose_opts+=("--verbose" "--log-level" "DEBUG")
  4595. ;;
  4596. --add-compose-content|-Y)
  4597. read-0 value
  4598. compose_contents+=("$value")
  4599. shift
  4600. ;;
  4601. --dirs)
  4602. echo "CACHEDIR: $CACHEDIR"
  4603. echo "VARDIR: $VARDIR"
  4604. exit 0
  4605. ;;
  4606. --get-project-name)
  4607. project=$(get_default_project_name) || exit 1
  4608. echo "$project"
  4609. exit 0
  4610. ;;
  4611. --get-available-actions)
  4612. get_docker_charm_action
  4613. exit $?
  4614. ;;
  4615. --dry-compose-run)
  4616. export DRY_COMPOSE_RUN=true
  4617. ;;
  4618. --*|-*)
  4619. if str_pattern_matches "$arg" $DC_MATCH_MULTI; then
  4620. read-0 value
  4621. compose_opts+=("$arg" "$value")
  4622. shift;
  4623. elif str_pattern_matches "$arg" $DC_MATCH_SINGLE; then
  4624. compose_opts+=("$arg")
  4625. else
  4626. err "Unknown option '$arg'. Please check help:"
  4627. display_help >&2
  4628. exit 1
  4629. fi
  4630. ;;
  4631. *)
  4632. action="$arg"
  4633. stage="action"
  4634. if DC_USAGE=$(get_docker_compose_usage "$action"); then
  4635. is_docker_compose_action=true
  4636. DC_MATCH_MULTI=$(get_docker_compose_multi_opts_list "$action") &&
  4637. DC_MATCH_SINGLE="$(get_docker_compose_single_opts_list "$action")"
  4638. if [ "$DC_MATCH_MULTI" ]; then
  4639. DC_MATCH_SINGLE="$DC_MATCH_SINGLE $(echo "$DC_MATCH_MULTI" | sed -r 's/( |$)/=\* /g')"
  4640. fi
  4641. pos_args=($(echo "$DC_USAGE" | sed -r 's/\[-[^]]+\] ?//g;s/\[options\] ?//g'))
  4642. pos_args=("${pos_args[@]:1}")
  4643. if [[ "${pos_args[0]}" == "[SERVICE...]" ]]; then
  4644. is_docker_compose_action_multi_service=1
  4645. elif [[ "${pos_args[0]}" == "SERVICE" ]]; then
  4646. is_docker_compose_action_multi_service=0
  4647. fi
  4648. # echo "USAGE: $DC_USAGE"
  4649. # echo "pos_args: ${pos_args[@]}"
  4650. # echo "MULTI: $DC_MATCH_MULTI"
  4651. # echo "SINGLE: $DC_MATCH_SINGLE"
  4652. # exit 1
  4653. else
  4654. stage="remainder"
  4655. fi
  4656. ;;
  4657. esac
  4658. ;;
  4659. "action") ## Only for docker-compose actions
  4660. case "$arg" in
  4661. --help|-h)
  4662. no_init=true ; no_hooks=true ; no_relations=true
  4663. action_opts+=("$arg")
  4664. ;;
  4665. --*|-*)
  4666. if [ "$is_docker_compose_action" ]; then
  4667. if str_pattern_matches "$arg" $DC_MATCH_MULTI; then
  4668. read-0 value
  4669. action_opts+=("$arg" "$value")
  4670. shift
  4671. elif str_pattern_matches "$arg" $DC_MATCH_SINGLE; then
  4672. action_opts+=("$arg")
  4673. else
  4674. err "Unknown option '$arg'. Please check '${DARKCYAN}$action${NORMAL}' help:"
  4675. docker-compose "$action" --help |
  4676. filter_docker_compose_help_message >&2
  4677. exit 1
  4678. fi
  4679. fi
  4680. ;;
  4681. *)
  4682. # echo "LOOP $1 : pos_arg: $pos_arg_ct // ${pos_args[$pos_arg_ct]}"
  4683. if [[ "${pos_args[$pos_arg_ct]}" == "[SERVICE...]" ]]; then
  4684. services_args+=("$arg")
  4685. elif [[ "${pos_args[$pos_arg_ct]}" == "SERVICE" ]]; then
  4686. services_args=("$arg") || exit 1
  4687. stage="remainder"
  4688. else
  4689. action_posargs+=("$arg")
  4690. ((pos_arg_ct++))
  4691. fi
  4692. ;;
  4693. esac
  4694. ;;
  4695. "remainder")
  4696. remainder_args+=("$arg")
  4697. while read-0 arg; do
  4698. remainder_args+=("$arg")
  4699. done
  4700. break 3
  4701. ;;
  4702. esac
  4703. shift
  4704. done < <(cla.normalize "$@")
  4705. ## These actions are additions to docker-compose actions and charm
  4706. ## actions
  4707. more_actions=(status)
  4708. if [[ "$action" == *" "* ]]; then
  4709. err "Invalid action name containing spaces: ${DARKCYAN}$action${NORMAL}"
  4710. exit 1
  4711. fi
  4712. is_more_action=
  4713. [[ " ${more_actions[*]} " == *" $action "* ]] && is_more_action=true
  4714. [ -n "$CACHEDIR" ] || die "No cache directory defined."
  4715. [ -d "$CACHEDIR" ] || die "Cache directory '$CACHEDIR' doesn't exists."
  4716. case "$action" in
  4717. cache)
  4718. case "${remainder_args[0]}" in
  4719. clean)
  4720. clean_cache
  4721. exit 0
  4722. ;;
  4723. clear)
  4724. Wrap "${wrap_opts[@]}" -v -d "clear cache directory" -- rm -rf "$CACHEDIR/"*
  4725. ## clear all docker caches
  4726. ## image name are like '[$COMPOSE_DOCKER_REGISTRY]cache/charm/CHARM_NAME:HASH'
  4727. Wrap "${wrap_opts[@]}" -v -d "clear docker cache" <<EOF
  4728. docker images --format "{{.Repository}}:{{.Tag}}" |
  4729. egrep "^($COMPOSE_DOCKER_REGISTRY/)?cache/charm/[a-zA-Z0-9._-]+:[0-9a-f]{32,32}$" |
  4730. while read -r image; do
  4731. docker rmi "\$image" || true
  4732. done
  4733. EOF
  4734. exit 0
  4735. ;;
  4736. *)
  4737. err "Unknown cache command: ${DARKCYAN}${remainder_args[0]}${NORMAL}"
  4738. exit 1
  4739. ;;
  4740. esac
  4741. ;;
  4742. status)
  4743. state_inner_cols=(name charm type state root)
  4744. state_all_services=
  4745. state_services=()
  4746. state_columns=()
  4747. state_columns_default=(name charm type state version)
  4748. state_filters=()
  4749. state_columns_default_msg=""
  4750. for col in "${state_columns_default[@]}"; do
  4751. if [ -n "$state_columns_default_msg" ]; then
  4752. state_columns_default_msg+=","
  4753. fi
  4754. state_columns_default_msg+="$col"
  4755. done
  4756. help="\
  4757. Display status information on services.
  4758. If no services are provided, all services in the root compose file
  4759. will be displayed. Use the --all option to display status of all
  4760. services (including dependencies).
  4761. $exname offers a few possible columns that can be complete on a charm
  4762. level by implementing an \`actions/get-COLNAME\` script.
  4763. These are the compose's columns: ${state_inner_cols[@]}.
  4764. Usage: status [options] [SERVICE...]
  4765. Options:
  4766. -h, --help Print this message and quit
  4767. -a, --all Display status of all services (removes all
  4768. filter, and will add a 'root' first column by
  4769. default)
  4770. -c, --column Columns to display, can provide several separated
  4771. by commas, or option can be repeated. You can add
  4772. a sign prefix to the name of the column to force
  4773. the alignment of the column (+: right, -: left),
  4774. (default: ${state_columns_default_msg})
  4775. -f, --filter Filter services by a key=value pair,
  4776. separated by commas or can be repeated.
  4777. (default: --filter root=yes)
  4778. -r, --raw Raw data output (no colors nor alignment)
  4779. -0 Separate field with NUL char. Implies raw data
  4780. output.
  4781. "
  4782. while read-0 arg; do
  4783. case "$arg" in
  4784. --help|-h)
  4785. echo "$help"
  4786. exit 0
  4787. ;;
  4788. --raw|-r|-0)
  4789. state_raw_output="$arg";
  4790. ## check if any state_columns have alignements specs
  4791. for col in "${state_columns[@]}"; do
  4792. if [[ "$col" == [-+]* ]]; then
  4793. err "Cannot use $arg and provide columns with alignment specs."
  4794. exit 1
  4795. fi
  4796. done
  4797. if [[ "$arg" == "-0" ]]; then
  4798. state_raw_output_nul=1
  4799. fi
  4800. ;;
  4801. --all|-a)
  4802. if [ "${#state_services[@]}" -gt 0 ]; then
  4803. err "Cannot use --all and provide services at the same time."
  4804. exit 1
  4805. fi
  4806. if [[ "${#state_filters[@]}" -gt 0 ]]; then
  4807. err "Cannot use --all and provide filters at the same time."
  4808. exit 1
  4809. fi
  4810. state_all_services=1
  4811. ;;
  4812. --column|-c)
  4813. read-0 value
  4814. if [[ "$value" == *,* ]]; then
  4815. state_columns_candidate=(${value//,/ })
  4816. else
  4817. state_columns_candidate=("$value")
  4818. fi
  4819. if [[ -n "$state_raw_output" ]]; then
  4820. for col in "${state_columns_candidate[@]}"; do
  4821. if [[ "$col" == [-+]* ]]; then
  4822. err "Cannot use ${state_raw_output} and provide columns with alignment specs."
  4823. exit 1
  4824. fi
  4825. done
  4826. fi
  4827. state_columns+=("${state_columns_candidate[@]}")
  4828. ;;
  4829. --filter|-f)
  4830. if [ "${#state_services[@]}" -gt 0 ]; then
  4831. err "Cannot use --filter and provide services at the same time."
  4832. exit 1
  4833. fi
  4834. if [ -n "$state_all_services" ]; then
  4835. err "Cannot use --all and provide filters at the same time."
  4836. exit 1
  4837. fi
  4838. read-0 value
  4839. if [[ "$value" == *,* ]]; then
  4840. state_filters+=(${value//,/ })
  4841. else
  4842. state_filters+=("$value")
  4843. fi
  4844. ;;
  4845. --*|-*)
  4846. err "Unknown option '$arg'. Please check help:"
  4847. echo "$help" >&2
  4848. ;;
  4849. *)
  4850. if [ -n "$state_all_services" ]; then
  4851. err "Cannot use --all and provide services at the same time."
  4852. exit 1
  4853. fi
  4854. if [[ "${#state_filters[@]}" -gt 0 ]]; then
  4855. err "Cannot use --filter and provide filters at the same time."
  4856. exit 1
  4857. fi
  4858. state_services+=("$arg")
  4859. ;;
  4860. esac
  4861. done < <(cla.normalize "${remainder_args[@]}")
  4862. if [ "${#state_columns[@]}" == 0 ]; then
  4863. state_columns=("${state_columns_default[@]}")
  4864. fi
  4865. ;;
  4866. esac
  4867. export compose_contents
  4868. [ "${services_args[*]}" ] && debug " ${DARKWHITE}Services:$NORMAL ${DARKYELLOW}${services_args[*]}$NORMAL"
  4869. [ "${compose_opts[*]}" ] && debug " ${DARKWHITE}Main docker-compose opts:$NORMAL ${compose_opts[*]}"
  4870. [ "${action_posargs[*]}" ] && debug " ${DARKWHITE}Main docker-compose pos args:$NORMAL ${action_posargs[*]}"
  4871. [ "${action_opts[*]}" ] && debug " ${DARKWHITE}Action $DARKCYAN$action$NORMAL with opts:$NORMAL ${action_opts[*]}"
  4872. [ "${remainder_args[*]}" ] && debug " ${DARKWHITE}Remainder args:$NORMAL ${remainder_args[*]}"
  4873. aexport remainder_args
  4874. ##
  4875. ## Actual code
  4876. ##
  4877. if [ -n "$DEBUG" ]; then
  4878. Elt "compute hashes"
  4879. start=$(time_now)
  4880. fi
  4881. COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1
  4882. COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1
  4883. COMPOSE_YML_CONTENT_HASH=$(compose:yml:hash) || exit 1
  4884. CHARM_STORE_HASH=$(charm.store_metadata_hash) || exit 1
  4885. COMBINED_HASH=$(H "$COMPOSE_YML_CONTENT_HASH" "$CHARM_STORE_HASH") || exit 1
  4886. export COMPOSE_YML_FILE COMPOSE_YML_CONTENT COMPOSE_YML_CONTENT_HASH CHARM_STORE_HASH COMBINED_HASH
  4887. if [ -n "$DEBUG" ]; then
  4888. elapsed="$(time_elapsed $start "$(time_now)")" || exit 1
  4889. print_info "$(printf "%.3fs" "$elapsed")"
  4890. Feedback
  4891. fi
  4892. charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct."
  4893. ##
  4894. ## Get services in command line.
  4895. ##
  4896. if [ -z "$is_docker_compose_action" ] && [ -z "$is_more_action" ] && [ -n "$action" ]; then
  4897. action_service=${remainder_args[0]}
  4898. if [ -z "$action_service" ]; then
  4899. err "No such command or action: ${DARKCYAN}$action${NORMAL}"
  4900. display_commands_help
  4901. exit 1
  4902. fi
  4903. services_args=($(compose:yml:root:services)) || return 1
  4904. ## Required by has_service_action
  4905. service:all:set_relations_hash
  4906. remainder_args=("${remainder_args[@]:1}")
  4907. if has_service_action "$action_service" "$action" >/dev/null; then
  4908. is_service_action=true
  4909. services_args=("$action_service")
  4910. {
  4911. read-0 action_type
  4912. case "$action_type" in
  4913. "relation")
  4914. read-0 _ target_service _target_charm relation_name _ action_script_path
  4915. debug "Found action $DARKYELLOW${action_service}$NORMAL/$DARKBLUE$relation_name$NORMAL/$DARKCYAN$action$NORMAL (in $DARKYELLOW$target_service$NORMAL)"
  4916. services_args+=("$target_service")
  4917. ;;
  4918. "direct")
  4919. read-0 _ action_script_path
  4920. debug "Found action $DARKYELLOW${action_service}$NORMAL.$DARKCYAN$action$NORMAL"
  4921. ;;
  4922. esac
  4923. } < <(has_service_action "$action_service" "$action")
  4924. get_all_relations "${services_args[@]}" >/dev/null || {
  4925. echo " Hint: if this is unexpected, you can try to delete caches, and re-run the command." >&2
  4926. exit 1
  4927. }
  4928. ## Divert logging to stdout to stderr
  4929. log () { cat >&2; }
  4930. export -f log
  4931. else
  4932. die "Unknown action '${DARKCYAN}$action$NORMAL': It doesn't match any docker-compose commands nor inner charm actions."
  4933. fi
  4934. else
  4935. case "$action" in
  4936. ps|up)
  4937. if [ "${#services_args[@]}" == 0 ]; then
  4938. services_args=($(compose:yml:root:services)) || exit 1
  4939. fi
  4940. ;;
  4941. status)
  4942. services_args=("${state_services[@]}")
  4943. if [ "${#services_args[@]}" == 0 ] && [ -z "$state_all_services" ]; then
  4944. services_args=($(compose:yml:root:services)) || exit 1
  4945. fi
  4946. ;;
  4947. config)
  4948. services_args=("${action_posargs[@]}")
  4949. ;;
  4950. esac
  4951. fi
  4952. export COMPOSE_ACTION="$action"
  4953. NO_CONSTRAINT_CHECK=True
  4954. case "$action" in
  4955. up|status|run)
  4956. NO_CONSTRAINT_CHECK=
  4957. if [ -n "$DEBUG" ]; then
  4958. Elt "solve all relations"
  4959. start=$(time_now)
  4960. fi
  4961. service:all:set_relations_hash || exit 1
  4962. if [ -n "$DEBUG" ]; then
  4963. elapsed="$(time_elapsed $start "$(time_now)")" || exit 1
  4964. print_info "$(printf "%.3fs" "$elapsed")"
  4965. Feedback
  4966. fi
  4967. all_services=($(get_all_services)) || exit 1
  4968. ## check that services_args is a subset of all_services
  4969. for service in "${services_args[@]}"; do
  4970. [[ " ${all_services[*]} " == *" $service "* ]] || {
  4971. err "Service ${DARKYELLOW}$service${NORMAL} is not defined in the current compose file."
  4972. echo " Neither is is a dependency of a service in the compose file." >&2
  4973. echo " These are the services directly or indirectly available from current compose file:" >&2
  4974. for service in "${all_services[@]}"; do
  4975. echo " - ${DARKYELLOW}$service${NORMAL}" >&2
  4976. done
  4977. exit 1
  4978. }
  4979. done
  4980. ;;
  4981. esac
  4982. case "$action" in
  4983. up)
  4984. PROJECT_NAME=$(get_default_project_name) || exit 1
  4985. ## Remove all intents (*ing states)
  4986. rm -f "$SERVICE_STATE_PATH/$PROJECT_NAME"/*/*ing || true
  4987. ## Notify that we have the intent to bring up all these
  4988. ## This will be use in inner or concurrent 'run' to include the
  4989. ## services that are supposed to be up.
  4990. mkdir -p "$SERVICE_STATE_PATH/$PROJECT_NAME" || exit 1
  4991. services_args_deps=($(get_ordered_service_dependencies "${services_args[@]}")) || exit 1
  4992. for service in "${services_args_deps[@]}"; do
  4993. mkdir -p "$SERVICE_STATE_PATH/$PROJECT_NAME"/"$service" || exit 1
  4994. [ -e "$SERVICE_STATE_PATH/$PROJECT_NAME"/"$service"/up ] || {
  4995. touch "$SERVICE_STATE_PATH/$PROJECT_NAME"/"$service"/deploying || exit 1
  4996. }
  4997. done
  4998. ## remove services not included in compose.yml anymore
  4999. all_services_deps=($(get_ordered_service_dependencies "${all_services[@]}")) || exit 1
  5000. for service in "$SERVICE_STATE_PATH/$PROJECT_NAME"/*/up; do
  5001. [ -e "$service" ] || continue
  5002. state=${service##*/}
  5003. service=${service%/$state}
  5004. service=${service##*/}
  5005. if [[ " ${all_services_deps[*]} " != *" ${service} "* ]]; then
  5006. touch "$SERVICE_STATE_PATH/$PROJECT_NAME"/"${service}"/orphaning || exit 1
  5007. fi
  5008. done
  5009. ;;
  5010. run)
  5011. PROJECT_NAME=$(get_default_project_name) || return 1
  5012. if [ -d "$SERVICE_STATE_PATH/$PROJECT_NAME" ]; then
  5013. ## Notify that we have the intent to bring up all these
  5014. ## This will be use in inner or concurrent 'run' to include the
  5015. ## services that are supposed to be up.
  5016. for service in "$SERVICE_STATE_PATH/$PROJECT_NAME"/*/{up,deploying}; do
  5017. [ -e "$service" ] || continue
  5018. state=${service##*/}
  5019. service=${service%/$state}
  5020. service=${service##*/}
  5021. ## don't add if orphaning
  5022. [ -e "$SERVICE_STATE_PATH/$PROJECT_NAME"/"${service}"/orphaning ] && continue
  5023. done
  5024. fi
  5025. ;;
  5026. status)
  5027. if [ -n "${state_all_services}" ] || [[ "${#state_filters[@]}" -gt 0 ]]; then
  5028. services_args=("${all_services[@]}")
  5029. fi
  5030. ;;
  5031. esac
  5032. if [ "$action" != "down" ]; then
  5033. if [ -n "$DEBUG" ]; then
  5034. Elt "get relation subset"
  5035. start=$(time_now)
  5036. fi
  5037. get_subset_relations "${services_args[@]}" >/dev/null || exit 1
  5038. if [ -n "$DEBUG" ]; then
  5039. elapsed="$(time_elapsed $start "$(time_now)")" || exit 1
  5040. print_info "$(printf "%.3fs" "$elapsed")"
  5041. Feedback
  5042. fi
  5043. fi
  5044. if [ -n "$is_docker_compose_action" ] && [ "${#services_args[@]}" -gt 0 ]; then
  5045. services=($(get_master_services "${services_args[@]}")) || exit 1
  5046. if [ "$action" == "up" ]; then
  5047. action_posargs+=($(services:get:upable "${services_args[@]}")) || exit 1
  5048. elif [ "$is_docker_compose_action_multi_service" == "1" ]; then
  5049. action_posargs+=("${services[@]}")
  5050. elif [ "$is_docker_compose_action_multi_service" == "0" ]; then
  5051. action_posargs+=("${services[0]}") ## only the first service is the legit one
  5052. fi
  5053. ## Get rid of subordinates
  5054. action_posargs=($(get_master_services "${action_posargs[@]}")) || exit 1
  5055. fi
  5056. get_docker_compose "${services_args[@]}" >/dev/null || { ## precalculate variable \$_current_docker_compose
  5057. err "Fails to compile base 'docker-compose.yml'"
  5058. exit 1
  5059. }
  5060. ##
  5061. ## Pre-action
  5062. ##
  5063. full_init=
  5064. case "$action" in
  5065. build)
  5066. full_init=true ## will actually stop after build
  5067. ;;
  5068. up|run)
  5069. full_init=true
  5070. post_hook=true
  5071. ;;
  5072. ""|down|restart|logs|config|ps|status)
  5073. full_init=
  5074. ;;
  5075. *)
  5076. if [ "$is_service_action" ]; then
  5077. full_init=true
  5078. keywords=($(egrep "^#*\s*compose:" "$action_script_path" | cut -f 2- -d:))
  5079. for keyword in "${keywords[@]}"; do
  5080. case "$keyword" in
  5081. no-hooks)
  5082. no_hooks=true
  5083. ;;
  5084. hooks)
  5085. full_init=true
  5086. ;;
  5087. esac
  5088. done
  5089. fi
  5090. ;;
  5091. esac
  5092. if [ -n "$full_init" ]; then
  5093. if [[ "$action" == "build" ]] || [[ -z "$no_init" && -z "$no_hooks" ]]; then
  5094. [[ "$action" == "build" ]] || Section "acquire charm's images"
  5095. run_service_acquire_images "${services_args[@]}" || exit 1
  5096. Feed
  5097. [ "$action" == "build" ] && {
  5098. exit 0
  5099. }
  5100. Section setup host resources
  5101. setup_host_resources "${services_args[@]}" || exit 1
  5102. ## init in order
  5103. Section initialisation
  5104. run_service_hook init "${services_args[@]}" || exit 1
  5105. fi
  5106. ## Get relations
  5107. if [[ -z "$no_relations" && -z "$no_hooks" ]]; then
  5108. if [ "${#rebuild_relations_to_service[@]}" != 0 ]; then
  5109. rebuild_relations_to_service=$(get_master_services "${rebuild_relations_to_service[@]}") || return 1
  5110. rebuild_relations_to_service=($rebuild_relations_to_service)
  5111. project=$(get_default_project_name) || return 1
  5112. for service in "${rebuild_relations_to_service[@]}"; do
  5113. for dir in "$VARDIR/relations/$project/"*"-${service}/"*; do
  5114. [ -d "$dir" ] && {
  5115. debug rm -rf "$dir"
  5116. rm -rf "$dir"
  5117. }
  5118. done
  5119. done
  5120. fi
  5121. run_service_relations "${services_args[@]}" || exit 1
  5122. fi
  5123. if [[ -z "$no_pre_deploy" && -z "$no_hooks" ]]; then
  5124. run_service_hook pre_deploy "${services_args[@]}" || exit 1
  5125. fi
  5126. fi | log
  5127. if [ "${PIPESTATUS[0]}" != 0 ]; then
  5128. exit 1
  5129. fi
  5130. [ "$action" == "build" ] && exit 0
  5131. state:fields:resolve-parallel() {
  5132. local cols rowsservice jobs state_msg out errlvl col
  5133. first_job=1
  5134. tick_pid=
  5135. concurrent_jobs=0
  5136. MAX_CONCURRENT_JOBS=$((3 + $(nproc)))
  5137. rows=()
  5138. cols=()
  5139. while [ "$#" -gt 0 ]; do
  5140. case "$1" in
  5141. --) shift; rows=("$@"); break;;
  5142. *) cols+=("$1") ;;
  5143. esac
  5144. shift
  5145. done
  5146. for col in "${cols[@]}"; do
  5147. for service in "${rows[@]}"; do
  5148. if [ "$concurrent_jobs" -ge "$MAX_CONCURRENT_JOBS" ]; then
  5149. wait -n # -p job_id ## not supported in this version of bash
  5150. ## job list is not accurate, but the number of elt is
  5151. ((concurrent_jobs--))
  5152. fi
  5153. (
  5154. p0 "$service" "$col" "-1" "" ## started
  5155. out=$(
  5156. case "${col//_/-}" in
  5157. root)
  5158. if [[ " ${compose_yml_services[*]} " == *" ${service} "* ]]; then
  5159. echo "1"
  5160. else
  5161. echo "0"
  5162. fi
  5163. ;;
  5164. name) e "$service" ;;
  5165. charm) get_service_charm "$service" ;;
  5166. state) service:state "$service" ;;
  5167. type) get_service_type "$service" ;;
  5168. upstream-version) service:upstream-version "$service" ;;
  5169. *)
  5170. if has_service_action "$service" "get-$col" >/dev/null; then
  5171. state_msg=$(run_service_action "$service" "get-$col") || exit 1
  5172. if [[ "$state_msg" == *$'\n'* ]]; then
  5173. e "${state_msg%%$'\n'*}"
  5174. else
  5175. e "${state_msg}"
  5176. fi
  5177. fi
  5178. ;;
  5179. esac 2>&1
  5180. )
  5181. errlvl="$?"
  5182. p0 "$service" "$col" "$errlvl" "$out"
  5183. ) &
  5184. jobs=("${jobs[@]}" $!)
  5185. ((concurrent_jobs++))
  5186. if [ -n "$first_job" ]; then
  5187. ## launch tick
  5188. (
  5189. while true; do
  5190. sleep 0.1
  5191. p0 "" "" "" ""
  5192. done
  5193. ) &
  5194. tick_pid=$!
  5195. first_job=
  5196. fi
  5197. done
  5198. done
  5199. wait "${jobs[@]}"
  5200. if [ -n "$tick_pid" ]; then
  5201. kill "$tick_pid"
  5202. fi
  5203. }
  5204. export -f state:fields:resolve-parallel
  5205. if [ "$action" == "status" ]; then
  5206. if [ -n "$DEBUG" ]; then
  5207. start=$(time_now)
  5208. fi
  5209. if ! [ -t 1 ]; then
  5210. state_raw_output=1
  5211. fi
  5212. if [[ -n "${state_all_services}" ]] || [[ "${#state_filters[@]}" -gt 0 ]]; then
  5213. compose_yml_services=($(compose:yml:root:services)) || exit 1
  5214. fi
  5215. if [[ -n "${state_all_services}" ]]; then
  5216. state_columns=("root" ${state_columns[@]})
  5217. fi
  5218. state_columns_raw=()
  5219. for col in "${state_columns[@]}"; do
  5220. if [[ "$col" =~ ^[+-] ]]; then
  5221. col=${col:1}
  5222. fi
  5223. state_columns_raw+=("${col//-/_}")
  5224. done
  5225. state_columns_align=""
  5226. for col in "${state_columns[@]}"; do
  5227. if [[ "$col" == "-"* ]]; then
  5228. state_columns_align+="-"
  5229. elif [[ "$col" == "+"* ]]; then
  5230. state_columns_align+="+"
  5231. else
  5232. case "${col//_/-}" in
  5233. version|upstream-version) state_columns_align+="+";;
  5234. *) state_columns_align+="-";;
  5235. esac
  5236. fi
  5237. done
  5238. declare -A state_columns_idx=()
  5239. declare -A filter_idx=()
  5240. filter_cols=()
  5241. non_filter_cols=("${state_columns_raw[@]}")
  5242. for filter in "${state_filters[@]}"; do
  5243. IFS="=" read -r key value <<<"$filter"
  5244. if [[ " ${non_filter_cols[*]} " == *" $key "* ]]; then
  5245. ## remove from non_filter_cols
  5246. non_filter_cols=(${non_filter_cols[*]/$key})
  5247. fi
  5248. state_columns_idx["$col"]="${#filter_cols[@]}"
  5249. filter_cols+=("${key}")
  5250. done
  5251. tot_nb_cols=$(( ${#non_filter_cols[@]} + ${#filter_cols[@]} ))
  5252. ## make services_idx
  5253. declare -A services_idx=()
  5254. idx=0
  5255. for service in "${services_args[@]}"; do
  5256. services_idx["$service"]=$((idx++))
  5257. done
  5258. ## make state_columns_idx
  5259. idx=0
  5260. for col in "${non_filter_cols[@]}"; do
  5261. state_columns_idx["$col"]=$((${#filter_cols[@]} + idx++))
  5262. done
  5263. values=() ## all values
  5264. new_service_args=("${services_args[@]}") ## will remove service not satisfying filters
  5265. while read-0 service col E out; do
  5266. if [[ " ${new_service_args[*]} " != *" $service "* ]]; then
  5267. continue
  5268. fi
  5269. col_index="${state_columns_idx[$col]}"
  5270. service_index="${services_idx[$service]}"
  5271. values[service_index * tot_nb_cols + col_index]="$out"
  5272. ## check if all filter are valuated and satisfied
  5273. for filter in "${state_filters[@]}"; do
  5274. IFS="=" read -r key value <<<"$filter"
  5275. col_index="${state_columns_idx[$key]}"
  5276. if [ -z "${values[$((service_index * tot_nb_cols + col_index))]}" ]; then
  5277. break
  5278. fi
  5279. if [ "${values[$((service_index * tot_nb_cols + col_index))]}" != "$value" ]; then
  5280. new_service_args=(${new_service_args[*]/"$service"})
  5281. break
  5282. fi
  5283. done
  5284. done < <(state:fields:resolve-parallel "${filter_cols[@]}" -- "${services_args[@]}")
  5285. services_args=("${new_service_args[@]}")
  5286. if [ "${#services_args[@]}" == 0 ]; then
  5287. warn "No services found matching the filters." >&2
  5288. exit 0
  5289. fi
  5290. spinner_chars="⠋⠙⠸⠴⠤⠦⠇"
  5291. spinner_idx=0
  5292. SPINNERGRAY=$'\e[38;5;16;48;5;234m'
  5293. SPINNERRUNNING=$'\e[38;5;28;48;5;234m'
  5294. first_draw=1
  5295. last_draw=
  5296. if [ -z "$state_raw_output" ]; then
  5297. echo -en "\e[?25l"; stty -echo 2>/dev/null
  5298. trap_add EXIT,ERR "echo -en '\e[?25h'; stty echo 2>/dev/null"
  5299. fi
  5300. errors=()
  5301. declare -A errors_hash_idx=()
  5302. error_idx=0
  5303. values_valued=0
  5304. values_total=$(( ${#services_args[@]} * ${#state_columns_raw[@]} ))
  5305. values_threshold=$(( values_total / 2 ))
  5306. while read-0 service col E out; do
  5307. if [ -n "$service" ]; then
  5308. col_index="${state_columns_idx[$col]}"
  5309. service_index="${services_idx[$service]}"
  5310. if [[ "$E" -gt 0 ]]; then
  5311. error_hash=$(H "$col" "$E" "$out")
  5312. matching_error_idx="${errors_hash_idx[$error_hash]}"
  5313. if [[ -z "${matching_error_idx}" ]]; then
  5314. errors+=("$error_idx:$service:$col:$E:$out")
  5315. out="!Err[$((error_idx))]"
  5316. errors_hash_idx["$error_hash"]="$error_idx"
  5317. error_idx=$((error_idx + 1))
  5318. else
  5319. ## find the error to add the service
  5320. error="${errors[$matching_error_idx]}"
  5321. error="${error#*:}"
  5322. error_service="${error%%:*}"
  5323. error_tail="${error#*:}"
  5324. errors[matching_error_idx]="$matching_error_idx:$error_service,$service:$error_tail"
  5325. out="!Err[$((matching_error_idx))]"
  5326. fi
  5327. elif [[ "$E" == -1 ]]; then
  5328. values[service_index * tot_nb_cols + col_index]=$'\t'
  5329. continue
  5330. fi
  5331. values[service_index * tot_nb_cols + col_index]="$out"
  5332. values_valued=$((values_valued + 1))
  5333. if [[ "$values_valued" == "$values_total" ]]; then
  5334. last_draw=1
  5335. else
  5336. continue
  5337. fi
  5338. fi
  5339. [ -n "$state_raw_output" ] && continue
  5340. [[ $((values_valued)) -lt $((values_threshold)) ]] && continue
  5341. ## Draw table
  5342. if [ -n "$first_draw" ]; then
  5343. first_draw=
  5344. full_table=""
  5345. else
  5346. ## move up one line per service
  5347. full_table=$'\e'"[${#services_args[@]}A"
  5348. fi
  5349. spinner_idx=$(( (spinner_idx + 1) % ${#spinner_chars} ))
  5350. while read-0-err E "${state_columns_raw[@]}"; do
  5351. line_values=()
  5352. for col in "${state_columns_raw[@]}"; do
  5353. color=
  5354. value="${!col}"
  5355. read -r -- value_trim <<<"${!col}"
  5356. case "${value_trim}" in
  5357. "N/A") color=gray ;;
  5358. "!Err"*) color=darkred ;;
  5359. "⠿") color=spinnergray ;;
  5360. *)
  5361. if [[ "$spinner_chars" == *"$value_trim"* ]]; then
  5362. color=spinnerrunning
  5363. else
  5364. case "${col//_/-}" in
  5365. root)
  5366. case "$value_trim" in
  5367. 0) value=" ";;
  5368. 1) value="*";;
  5369. esac
  5370. ;;
  5371. name) color=darkyellow;;
  5372. charm) color=darkpink;;
  5373. state)
  5374. case "$value_trim" in
  5375. up) color=green;;
  5376. down) color=gray;;
  5377. deploying) color=yellow;;
  5378. *) color=red;;
  5379. esac
  5380. ;;
  5381. type)
  5382. case "$value_trim" in
  5383. run-once) color=gray;;
  5384. stub) color=gray;;
  5385. *) color=darkcyan;;
  5386. esac
  5387. ;;
  5388. *)
  5389. ;;
  5390. esac
  5391. fi
  5392. ;;
  5393. esac
  5394. color="${color^^}"
  5395. if [ -n "$color" ]; then
  5396. line_values+=("${!color}$value${NORMAL}")
  5397. else
  5398. line_values+=("$value")
  5399. fi
  5400. done
  5401. first=1
  5402. full_line=""
  5403. for value in "${line_values[@]}"; do
  5404. if [ -n "$first" ]; then
  5405. first=
  5406. else
  5407. full_line+=" "
  5408. fi
  5409. full_line+="$value"
  5410. done
  5411. full_table+="$full_line"$'\e[K\n'
  5412. done < <(
  5413. set -o pipefail
  5414. for service in "${services_args[@]}"; do
  5415. for col in "${state_columns_raw[@]}"; do
  5416. col_index="${state_columns_idx[$col]}"
  5417. service_index="${services_idx[$service]}"
  5418. value_idx="$((service_index * tot_nb_cols + col_index))"
  5419. if ! [[ -v "values[value_idx]" ]]; then
  5420. p0 " ⠿ "
  5421. continue
  5422. fi
  5423. value="${values[value_idx]}"
  5424. if [[ "$value" == $'\t' ]]; then
  5425. p0 " ${spinner_chars:$spinner_idx:1} "
  5426. elif [ -z "$value" ]; then
  5427. p0 "N/A"
  5428. else
  5429. p0 "$value"
  5430. fi
  5431. done
  5432. done | {
  5433. if [ -z "$state_raw_output" ]; then
  5434. col-0:normalize:size "${state_columns_align}"
  5435. else
  5436. cat
  5437. fi
  5438. }
  5439. echo 0
  5440. )
  5441. printf "%s" "$full_table"
  5442. if [ "$E" != 0 ]; then
  5443. err "Unexpected failure while drawing table"
  5444. exit $E
  5445. fi
  5446. done < <(state:fields:resolve-parallel "${non_filter_cols[@]}" -- "${services_args[@]}")
  5447. for error in "${errors[@]}"; do
  5448. echo "" >&2
  5449. idx=${error%%:*}; error=${error#*:}
  5450. service=${error%%:*}; error=${error#*:}
  5451. col=${error%%:*}; error=${error#*:}
  5452. E=${error%%:*}; error=${error#*:}
  5453. service_list_str=""
  5454. services=(${service//,/ })
  5455. first=1
  5456. for service in "${services[@]}"; do
  5457. if [ -n "$first" ]; then
  5458. first=
  5459. else
  5460. service_list_str+=", "
  5461. fi
  5462. service_list_str+="${DARKYELLOW}$service${NORMAL}"
  5463. done
  5464. echo "${RED}Error${DARKRED}[$idx]:${NORMAL} while computing" \
  5465. "${WHITE}$col${NORMAL} for $service_list_str" >&2
  5466. echo "$error" | prefix " ${GRAY}|${NORMAL} " >&2
  5467. echo " ${GRAY}..${NORMAL} ${WHITE}Exited${NORMAL} with errorlevel ${DARKRED}$E${NORMAL}" >&2
  5468. done
  5469. if [[ "${#errors[@]}" -gt 0 ]]; then
  5470. if [ -n "$DEBUG" ]; then
  5471. Elt "table computation ${DARKRED}failed${NORMAL}"
  5472. elapsed="$(time_elapsed $start "$(time_now)")" || exit 1
  5473. print_info "${elapsed}s"
  5474. Feedback
  5475. fi
  5476. exit 1
  5477. fi
  5478. if [ -n "$state_raw_output" ]; then
  5479. for service in "${services_args[@]}"; do
  5480. first=1
  5481. for col in "${state_columns_raw[@]}"; do
  5482. col_index="${state_columns_idx[$col]}"
  5483. service_index="${services_idx[$service]}"
  5484. value_idx="$((service_index * tot_nb_cols + col_index))"
  5485. value="${values[$value_idx]}"
  5486. if [ -n "$first" ]; then
  5487. first=
  5488. else
  5489. if [ -n "$state_raw_output_nul" ]; then
  5490. printf "\0"
  5491. else
  5492. printf " "
  5493. fi
  5494. fi
  5495. printf "%s" "$value"
  5496. done
  5497. if [ -n "$state_raw_output_nul" ]; then
  5498. printf "\0"
  5499. else
  5500. printf "\n"
  5501. fi
  5502. done
  5503. fi
  5504. if [ -n "$DEBUG" ]; then
  5505. Elt "table computation ${GREEN}successful${NORMAL}"
  5506. elapsed="$(time_elapsed $start "$(time_now)")" || exit 1
  5507. print_info "${elapsed}s"
  5508. Feedback
  5509. fi
  5510. exit 0
  5511. fi
  5512. if [ "$action" == "run" ] && [ "${#services_args}" != 0 ]; then
  5513. charm=$(get_service_charm "${services_args[0]}") || exit 1
  5514. metadata=$(charm.metadata "$charm") || exit 1
  5515. SERVICE_TYPE="$(printf "%s" "$metadata" | shyaml get-value type 2>/dev/null)" || true
  5516. if [ "$SERVICE_TYPE" == "run-once" ]; then
  5517. run_service_hook dc-pre-run "${services_args[@]}" || exit 1
  5518. fi
  5519. fi
  5520. export SERVICE_PACK="${services_args[*]}"
  5521. ##
  5522. ## Docker-compose
  5523. ##
  5524. errlvl="0"
  5525. case "$action" in
  5526. up|start|stop|build|run)
  5527. ## force daemon mode for up
  5528. if [[ "$action" == "up" ]]; then
  5529. if ! array_member action_opts -d; then
  5530. action_opts+=("-d")
  5531. fi
  5532. if ! array_member action_opts --remove-orphans; then
  5533. action_opts+=("--remove-orphans")
  5534. fi
  5535. fi
  5536. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  5537. ;;
  5538. logs)
  5539. if ! array_member action_opts --tail; then ## force daemon mode for up
  5540. action_opts+=("--tail" "10")
  5541. fi
  5542. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  5543. ;;
  5544. "")
  5545. launch_docker_compose "${compose_opts[@]}"
  5546. ;;
  5547. graph)
  5548. graph $SERVICE_PACK
  5549. ;;
  5550. config)
  5551. ## removing the services
  5552. services=($(get_master_services "${action_posargs[@]}")) || exit 1
  5553. ## forcing docker-compose config to output the config file to stdout and not stderr
  5554. out=$(launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" 2>&1) || {
  5555. echo "$out"
  5556. exit 1
  5557. }
  5558. echo "$out"
  5559. warn "Runtime configuration modification (from relations) are not included here."
  5560. ;;
  5561. down)
  5562. if ! array_member action_opts --remove-orphans; then ## force daemon mode for up
  5563. debug "Adding a default argument of '--remove-orphans'"
  5564. action_opts+=("--remove-orphans")
  5565. fi
  5566. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}"
  5567. ;;
  5568. *)
  5569. if [ "$is_service_action" ]; then
  5570. run_service_action "$action_service" "$action" "${remainder_args[@]}"
  5571. errlvl="$?"
  5572. errlvl "$errlvl"
  5573. else
  5574. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  5575. fi
  5576. ;;
  5577. esac || exit 1
  5578. if [ "$post_hook" -a "${#services_args[@]}" != 0 -a -z "$no_hooks" -a -z "$no_post_deploy" ]; then
  5579. run_service_hook post_deploy "${services_args[@]}" || exit 1
  5580. fi
  5581. if [ "$action" == "run" -a "${#services_args}" != 0 ]; then
  5582. if [ "$SERVICE_TYPE" == "run-once" ]; then
  5583. run_service_hook dc-post-run "${services_args[@]}" || exit 1
  5584. fi
  5585. fi
  5586. case "$action" in
  5587. up)
  5588. ## Notify that services in 'deploying' states have been deployed
  5589. for service in "$SERVICE_STATE_PATH/$PROJECT_NAME"/*/deploying; do
  5590. [ -e "$service" ] || continue
  5591. state=${service##*/}
  5592. service=${service%/$state}
  5593. service=${service##*/}
  5594. mv "$SERVICE_STATE_PATH/$PROJECT_NAME"/"${service}"/{deploying,up} || exit 1
  5595. done
  5596. ## Notify that services in 'orphaning' states have been removed
  5597. for service in "$SERVICE_STATE_PATH/$PROJECT_NAME"/*/orphaning; do
  5598. [ -e "$service" ] || continue
  5599. state=${service##*/}
  5600. service=${service%/$state}
  5601. service=${service##*/}
  5602. rm "$SERVICE_STATE_PATH/$PROJECT_NAME"/"${service}"/orphaning || exit 1
  5603. done
  5604. ;;
  5605. down)
  5606. PROJECT_NAME=$(get_default_project_name) || return 1
  5607. if [ -d "$SERVICE_STATE_PATH/$PROJECT_NAME" ]; then
  5608. if ! dir_is_empty "$SERVICE_STATE_PATH/$PROJECT_NAME"; then
  5609. rm -f "$SERVICE_STATE_PATH/$PROJECT_NAME"/*/*
  5610. fi
  5611. rmdir "$SERVICE_STATE_PATH/$PROJECT_NAME"/{*,}
  5612. fi
  5613. ;;
  5614. esac
  5615. clean_unused_docker_compose || exit 1
  5616. exit "$errlvl"