You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

4052 lines
133 KiB

  1. #!/bin/bash
  2. ##
  3. ## TODO:
  4. ## - subordinate container should really be able to modify base image of their master
  5. ## - this could be done through docker-update
  6. ## - I'm not happy with the current build using 'build/' directory, this should be
  7. ## changed to:
  8. ## - always have a base image (specified in metadata), and always have hooks/install
  9. ## executed and merge in image (like docker-build-charm).
  10. ## - container base image is ALWAYS the image of the master container... this brings
  11. ## questions about a double way to express inheritage (through relations as it is
  12. ## implemented now, or through this base-image ?)
  13. ## - the name of the scripts for relation (aka relation_name-relation-joined) is bad as
  14. ## reading the name in a hooks/ dir, there are no way to know if we are the target or
  15. ## the base of the relation.
  16. ## - we could leverage a 'relations/' dir on the root of the charm, with both:
  17. ## 'relations/provide/relation_name' and 'relations/receive/relation_name'
  18. ## - a very bad point with the actual naming is that we can't have a providing AND
  19. ## receiving a relation with same name.
  20. ## - The cache system should keep md5 of docker-compose and other things between runs
  21. ## - The cache system should use underlying function that have only arguments inputs.
  22. ## This will allow to cache completely without issues function in time.
  23. ## - would probably need instrospection in charm custom action to know if these need
  24. ## init or relations to be set up.
  25. ## - Be clear about when the SERVICE name is used and the CHARM name is used.
  26. ## - in case of service contained in another container
  27. ## - in normal case
  28. ## - in docker-compose, can't use charm name: if we want 2 instances of the same charm
  29. ## we are stuck. What will be unique is the name of the service.
  30. ## - some relations are configured in compose.yml but should not trigger the loading
  31. ## of necessary component (for instance, apache --> log-rotate), if log-rotate is
  32. ## not there, this link should considered optional.
  33. ## - Could probably allow an unexistent charm to be populated with only "docker-image:"
  34. ## of the same name. Although this should trigger a visible warning.
  35. #:-
  36. [ -e /etc/shlib ] && . /etc/shlib || {
  37. echo "Unsatisfied dependency. Please install 'kal-shlib-core'."
  38. exit 1
  39. }
  40. #:-
  41. include common
  42. include pretty
  43. include parse
  44. include charm
  45. include array
  46. include cla
  47. depends shyaml docker
  48. exname="compose"
  49. version=0.1
  50. usage="$exname [COMPOSE_OPTS] [ACTION [ACTION_OPTS]]"
  51. help="\
  52. $WHITE$exname$NORMAL jobs is to run various shell scripts to build
  53. a running orchestrated and configured docker containers. These shell
  54. scripts will have the opportunity to build a 'docker-compose.yml'.
  55. Once init script and relations scripts are executed, $WHITE$exname$NORMAL
  56. delegate the launching to ${WHITE}docker-compose${NORMAL} by providing it
  57. the final 'docker-compose.yml'.
  58. $WHITE$exname$NORMAL also leverage charms to offer some additional custom
  59. actions per charm, which are simply other scripts that can be
  60. run without launching ${WHITE}docker-compose${NORMAL}.
  61. In compose message, color coding is enforced as such:
  62. - ${DARKCYAN}action$NORMAL,
  63. - ${DARKBLUE}relation$NORMAL,
  64. - ${DARKPINK}charm${NORMAL},
  65. - ${DARKYELLOW}service${NORMAL},
  66. - ${WHITE}option-name${NORMAL}/${WHITE}command-name${NORMAL}/${WHITE}Section-Title${NORMAL}
  67. $WHITE$exname$NORMAL reads '/etc/compose.conf' for global variables, and
  68. '/etc/compose.local.conf' for local host adjustements.
  69. "
  70. ## XXXvlab: this doesn't seem to work when 'compose' is called in
  71. ## a hook of a charm.
  72. #[[ "${BASH_SOURCE[0]}" == "" ]] && SOURCED=true
  73. $(return >/dev/null 2>&1) && SOURCED=true
  74. if [ "$UID" == 0 ]; then
  75. CACHEDIR=${CACHEDIR:-/var/cache/compose}
  76. VARDIR=${VARDIR:-/var/lib/compose}
  77. else
  78. [ "$XDG_CONFIG_HOME" ] && CACHEDIR=${CACHEDIR:-$XDG_CONFIG_HOME/compose}
  79. [ "$XDG_DATA_HOME" ] && VARDIR=${VARDIR:-$XDG_DATA_HOME/compose}
  80. CACHEDIR=${CACHEDIR:-$HOME/.cache/compose}
  81. VARDIR=${VARDIR:-$HOME/.local/share/compose}
  82. fi
  83. export VARDIR CACHEDIR
  84. md5_compat() { md5sum | cut -c -32; }
  85. quick_cat_file() { quick_cat_stdin < "$1"; }
  86. quick_cat_stdin() { local IFS=''; while read -r line; do echo "$line"; done ; }
  87. export -f quick_cat_file quick_cat_stdin md5_compat
  88. clean_cache() {
  89. local i=0
  90. for f in $(ls -t "$CACHEDIR/"*.cache.* 2>/dev/null | tail -n +500); do
  91. ((i++))
  92. rm -f "$f"
  93. done
  94. if (( i > 0 )); then
  95. debug "${WHITE}Cleaned cache:${NORMAL} Removed $((i)) elements (current cache size is $(du -sh "$CACHEDIR" | cut -f 1))"
  96. fi
  97. }
  98. usage="$exname SERVICE"'
  99. Deploy and manage a swarm of containers to provide services based on
  100. a ``compose.yml`` definition and charms from a ``charm-store``.
  101. '
  102. export DEFAULT_COMPOSE_FILE
  103. ##
  104. ## Merge YAML files
  105. ##
  106. export _merge_yaml_common_code="
  107. import sys
  108. import yaml
  109. try:
  110. from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
  111. except ImportError: ## pragma: no cover
  112. sys.stderr.write('YAML code in pure python\n')
  113. exit(1)
  114. from yaml import SafeLoader, SafeDumper
  115. class MySafeLoader(SafeLoader): pass
  116. class MySafeDumper(SafeDumper): pass
  117. try:
  118. # included in standard lib from Python 2.7
  119. from collections import OrderedDict
  120. except ImportError:
  121. # try importing the backported drop-in replacement
  122. # it's available on PyPI
  123. from ordereddict import OrderedDict
  124. ## Ensure that there are no collision with legacy OrderedDict
  125. ## that could be used for omap for instance.
  126. class MyOrderedDict(OrderedDict):
  127. pass
  128. MySafeDumper.add_representer(
  129. MyOrderedDict,
  130. lambda cls, data: cls.represent_dict(data.items()))
  131. def construct_omap(cls, node):
  132. cls.flatten_mapping(node)
  133. return MyOrderedDict(cls.construct_pairs(node))
  134. MySafeLoader.add_constructor(
  135. yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
  136. construct_omap)
  137. ##
  138. ## Support local and global objects
  139. ##
  140. class EncapsulatedNode(object): pass
  141. def mk_encapsulated_node(s, node):
  142. method = 'construct_%s' % (node.id, )
  143. data = getattr(s, method)(node)
  144. class _E(data.__class__, EncapsulatedNode):
  145. pass
  146. _E.__name__ = str(node.tag)
  147. _E._node = node
  148. return _E(data)
  149. def represent_encapsulated_node(s, o):
  150. value = s.represent_data(o.__class__.__bases__[0](o))
  151. value.tag = o.__class__.__name__
  152. return value
  153. MySafeDumper.add_multi_representer(EncapsulatedNode,
  154. represent_encapsulated_node)
  155. MySafeLoader.add_constructor(None, mk_encapsulated_node)
  156. def fc(filename):
  157. with open(filename) as f:
  158. return f.read()
  159. def merge(*args):
  160. # sys.stderr.write('%r\n' % (args, ))
  161. args = [arg for arg in args if arg is not None]
  162. if len(args) == 0:
  163. return None
  164. if len(args) == 1:
  165. return args[0]
  166. if all(isinstance(arg, (int, basestring, bool)) for arg in args):
  167. return args[-1]
  168. elif all(isinstance(arg, list) for arg in args):
  169. res = []
  170. for arg in args:
  171. for elt in arg:
  172. if elt in res:
  173. res.remove(elt)
  174. res.append(elt)
  175. return res
  176. elif all(isinstance(arg, dict) for arg in args):
  177. keys = set()
  178. for arg in args:
  179. keys |= set(arg.keys())
  180. dct = {}
  181. for key in keys:
  182. sub_args = []
  183. for arg in args:
  184. if key in arg:
  185. sub_args.append(arg)
  186. try:
  187. dct[key] = merge(*(a[key] for a in sub_args))
  188. except NotImplementedError as e:
  189. raise NotImplementedError(
  190. e.args[0],
  191. '%s.%s' % (key, e.args[1]) if e.args[1] else key,
  192. e.args[2])
  193. if dct[key] is None:
  194. del dct[key]
  195. return dct
  196. else:
  197. raise NotImplementedError(
  198. 'Unsupported types: %s'
  199. % (', '.join(list(set(arg.__class__.__name__ for arg in args)))), '', args)
  200. return None
  201. def merge_cli(*args):
  202. try:
  203. c = merge(*args)
  204. except NotImplementedError as e:
  205. sys.stderr.write('Merging Failed: %s.\n%s\n'
  206. ' Values are:\n %s\n'
  207. % (e.args[0],
  208. ' Conflicting key is %r.' % e.args[1] if e.args[1] else
  209. ' Conflict at base of structure.',
  210. '\\n '.join('v%d: %r' % (i, a)
  211. for i, a in enumerate(e.args[2]))))
  212. exit(1)
  213. if c is not None:
  214. print '%s' % yaml.dump(c, default_flow_style=False, Dumper=MySafeDumper)
  215. "
  216. merge_yaml() {
  217. if ! [ -r "$state_tmpdir/merge_yaml.py" ]; then
  218. cat <<EOF > "$state_tmpdir/merge_yaml.py"
  219. $_merge_yaml_common_code
  220. merge_cli(*(yaml.load(fc(f), Loader=MySafeLoader) for f in sys.argv[1:]))
  221. EOF
  222. fi
  223. python "$state_tmpdir/merge_yaml.py" "$@"
  224. }
  225. export -f merge_yaml
  226. merge_yaml_str() {
  227. local entries="$@"
  228. if ! [ -r "$state_tmpdir/merge_yaml_str.py" ]; then
  229. cat <<EOF > "$state_tmpdir/merge_yaml_str.py" || return 1
  230. $_merge_yaml_common_code
  231. merge_cli(*(yaml.load(f, Loader=MySafeLoader) for f in sys.argv[1:]))
  232. EOF
  233. fi
  234. if ! python "$state_tmpdir/merge_yaml_str.py" "$@"; then
  235. err "Failed to merge yaml strings:"
  236. local s
  237. for s in "$@"; do
  238. printf " - \n"
  239. printf "%s\n" "$s" | prefix " ${GRAY}|$NORMAL "
  240. done >&2
  241. return 1
  242. fi
  243. }
  244. export -f merge_yaml_str
  245. yaml_key_val_str() {
  246. local entries="$@"
  247. if ! [ -r "$state_tmpdir/yaml_key_val_str.py" ]; then
  248. cat <<EOF > "$state_tmpdir/yaml_key_val_str.py"
  249. $_merge_yaml_common_code
  250. print '%s' % yaml.dump(
  251. {
  252. yaml.load(sys.argv[1], Loader=MySafeLoader):
  253. yaml.load(sys.argv[2], Loader=MySafeLoader)
  254. },
  255. default_flow_style=False,
  256. Dumper=MySafeDumper,
  257. )
  258. EOF
  259. fi
  260. python "$state_tmpdir/yaml_key_val_str.py" "$@"
  261. }
  262. export -f yaml_key_val_str
  263. ##
  264. ## Docker
  265. ##
  266. docker_has_image() {
  267. local image="$1"
  268. images=$(docker images -q "$image" 2>/dev/null) || {
  269. err "docker images call has failed unexpectedly."
  270. return 1
  271. }
  272. [ "$images" ]
  273. }
  274. export -f docker_has_image
  275. docker_image_id() {
  276. local image="$1" image_id cache_file="$state_dir/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  277. if [ -e "$cache_file" ]; then
  278. # debug "$FUNCNAME: cache hit ($*)"
  279. quick_cat_stdin < "$cache_file"
  280. return 0
  281. fi
  282. image_id=$(docker inspect "$image" --format='{{.Id}}') || return 1
  283. echo "$image_id" | tee "$cache_file"
  284. }
  285. export -f docker_image_id
  286. cached_cmd_on_image() {
  287. local image="$1" cache_file
  288. image_id=$(docker_image_id "$image") || return 1
  289. cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  290. if [ -e "$cache_file" ]; then
  291. # debug "$FUNCNAME: cache hit ($*)"
  292. quick_cat_stdin < "$cache_file"
  293. return 0
  294. fi
  295. shift
  296. out=$(docker run -i --rm --entrypoint /bin/sh "$image_id" -c "$*") || return 1
  297. echo "$out" | tee "$cache_file"
  298. }
  299. export -f cached_cmd_on_image
  300. cmd_on_base_image() {
  301. local service="$1" base_image
  302. shift
  303. base_image=$(service_base_docker_image "$service") || return 1
  304. docker run -i --rm --entrypoint /bin/bash "$base_image" -c "$*"
  305. }
  306. export -f cmd_on_base_image
  307. cached_cmd_on_base_image() {
  308. local service="$1" base_image cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  309. shift
  310. if [ -e "$cache_file" ]; then
  311. # debug "$FUNCNAME: cache hit ($*)"
  312. quick_cat_stdin < "$cache_file"
  313. return 0
  314. fi
  315. base_image=$(service_base_docker_image "$service") || return 1
  316. if ! docker_has_image "$base_image"; then
  317. docker pull "$base_image"
  318. fi
  319. result=$(cached_cmd_on_image "$base_image" "$@") || return 1
  320. echo "$result" | tee "$cache_file"
  321. }
  322. export -f cached_cmd_on_base_image
  323. image_exposed_ports_0() {
  324. local image="$1"
  325. docker inspect --format='{{range $p, $conf := .Config.ExposedPorts}}{{$p}}{{"\x00"}}{{end}}' "$image"
  326. }
  327. export -f image_exposed_ports_0
  328. ##
  329. ## Generic
  330. ##
  331. fn.exists() {
  332. declare -F "$1" >/dev/null
  333. }
  334. str_pattern_matches() {
  335. local str="$1"
  336. shift
  337. for pattern in "$@"; do
  338. eval "[[ \"$str\" == $pattern ]]" && return 0
  339. done
  340. return 1
  341. }
  342. str_matches() {
  343. local str="$1"
  344. shift
  345. for pattern in "$@"; do
  346. [[ "$str" == "$pattern" ]] && return 0
  347. done
  348. return 1
  349. }
  350. gen_password() {
  351. local l=( {a..z} {A..Z} {0..9} ) nl="${#l[@]}" size=${1:-16}
  352. while ((size--)); do
  353. echo -n "${l[$((RANDOM * nl / 32768))]}"
  354. done
  355. echo
  356. }
  357. export -f gen_password
  358. file_put() {
  359. local TARGET="$1"
  360. mkdir -p "$(dirname "$TARGET")" &&
  361. cat - > "$TARGET"
  362. }
  363. export -f file_put
  364. file_put_0() {
  365. local TARGET="$1"
  366. mkdir -p "$(dirname "$TARGET")" &&
  367. cat > "$TARGET"
  368. }
  369. export -f file_put_0
  370. fetch_file() {
  371. local src="$1"
  372. case "$src" in
  373. *"://"*)
  374. err "Unsupported target scheme."
  375. return 1
  376. ;;
  377. *)
  378. ## Try direct
  379. if ! [ -r "$src" ]; then
  380. err "File '$src' not found/readable."
  381. return 1
  382. fi
  383. cat "$src" || return 1
  384. ;;
  385. esac
  386. }
  387. export -f fetch_file
  388. ## receives stdin content to decompress on stdout
  389. ## stdout content should be tar format.
  390. uncompress_file() {
  391. local filename="$1"
  392. ## Warning, the content of the file is already as stdin, the filename
  393. ## is there to hint for correct decompression.
  394. case "$filename" in
  395. *".gz")
  396. gunzip
  397. ;;
  398. *".bz2")
  399. bunzip2
  400. ;;
  401. *)
  402. cat
  403. ;;
  404. esac
  405. }
  406. export -f uncompress_file
  407. get_file() {
  408. local src="$1"
  409. fetch_file "$src" | uncompress_file "$src"
  410. }
  411. export -f get_file
  412. ##
  413. ## Common database lib
  414. ##
  415. _clean_docker() {
  416. local _DB_NAME="$1" container_id="$2"
  417. (
  418. set +e
  419. debug "Removing container $_DB_NAME"
  420. docker stop "$container_id"
  421. docker rm "$_DB_NAME"
  422. docker network rm "${_DB_NAME}"
  423. rm -vf "$state_tmpdir/${_DB_NAME}.state"
  424. )
  425. }
  426. export -f _clean_docker
  427. get_service_base_image_dir_uid_gid() {
  428. local service="$1" dir="$2" uid_gid
  429. uid_gid=$(cached_cmd_on_base_image "$service" "stat -c '%u %g' '$dir'") || {
  430. debug "Failed to query '$dir' uid in ${DARKYELLOW}$service${NORMAL} base image."
  431. return 1
  432. }
  433. info "uid and gid from ${DARKYELLOW}$service${NORMAL}:$dir is '$uid_gid'"
  434. echo "$uid_gid"
  435. }
  436. export -f get_service_base_image_dir_uid_gid
  437. get_service_type() {
  438. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  439. if [ -z "$service" ]; then
  440. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  441. return 1
  442. fi
  443. if [ -e "$cache_file" ]; then
  444. # debug "$FUNCNAME: cache hit ($*)"
  445. cat "$cache_file"
  446. return 0
  447. fi
  448. charm=$(get_service_charm "$service") || return 1
  449. metadata=$(charm.metadata "$charm") || return 1
  450. printf "%s" "$metadata" | shyaml get-value type service 2>/dev/null |
  451. tee "$cache_file"
  452. }
  453. are_files_locked_in_dir() {
  454. local dir="$1" device hdev ldev
  455. device=$(stat -c %d "$dir") || {
  456. err "Can't stat %d."
  457. return 1
  458. }
  459. device=$(printf "%04x" $device)
  460. hdev=${device:0:2}
  461. ldev=${device:2:2}
  462. inodes=$(find "$dir" -printf ':%i:\n')
  463. found=
  464. while read -r inode; do
  465. debug "try inode:$inode"
  466. if [[ "$inodes" == *":$inode:"* ]]; then
  467. found=1
  468. break
  469. fi
  470. done < <(cat /proc/locks | grep " $hdev:$ldev:" | sed -r "s/^.*$hdev:$ldev:([0-9]+).*$/\1/g")
  471. [ "$found" ]
  472. }
  473. export -f are_files_locked_in_dir
  474. export _PID="$$"
  475. ensure_db_docker_running () {
  476. local _STATE_FILE
  477. _DB_NAME="db_${DB_NAME}_${_PID}"
  478. _STATE_FILE="$state_tmpdir/${_DB_NAME}.state"
  479. if [ -e "$_STATE_FILE" ]; then
  480. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$(cat "$_STATE_FILE")"
  481. debug "Re-using previous docker/connection '$DOCKER_IP'."
  482. _set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  483. return 0
  484. fi
  485. if [ -e "$state_tmpdir/${_DB_NAME}.working" ]; then
  486. ## avoid recursive calls.
  487. if [ -z "$DOCKER_IP" ]; then
  488. err "Currently figuring up DOCKER_IP, please set it yourself before this call if needed."
  489. return 1
  490. else
  491. debug "ignoring recursive call of 'ensure_db_docker_running'."
  492. fi
  493. return 0
  494. fi
  495. touch "$state_tmpdir/${_DB_NAME}.working"
  496. docker rm "$_DB_NAME" 2>/dev/null || true
  497. host_db_working_dir="$DATASTORE/${SERVICE_NAME}$DB_DATADIR"
  498. if is_db_locked; then
  499. info "Some process is using '$host_db_working_dir'. Trying to find a docker that would do this..."
  500. found=
  501. for docker_id in $(docker ps -q); do
  502. has_volume_mounted=$(
  503. docker inspect \
  504. --format "{{range .Mounts}}{{if eq .Destination \"$DB_DATADIR\"}}{{.Source}}{{end}}{{end}}" \
  505. "$docker_id")
  506. if [ "$has_volume_mounted" == "$host_db_working_dir" ]; then
  507. found="$docker_id"
  508. break
  509. fi
  510. done
  511. if [ -z "$found" ]; then
  512. err "Please shutdown any other docker using this directory."
  513. return 1
  514. fi
  515. export container_id="$found"
  516. info "Found docker $docker_id is already running."
  517. else
  518. verb "Database is not locked."
  519. if ! docker_has_image "$DOCKER_BASE_IMAGE"; then
  520. docker pull "$DOCKER_BASE_IMAGE"
  521. fi
  522. docker_opts=
  523. debug docker network create "$_DB_NAME"
  524. if ! network_id=$(docker network create "$_DB_NAME"); then
  525. err "'docker network create $_DB_NAME' failed !"
  526. _clean_docker "$_DB_NAME" "$container_id"
  527. rm "$state_tmpdir/${_DB_NAME}.working"
  528. return 1
  529. fi
  530. debug docker run -d \
  531. --name "$_DB_NAME" \
  532. $docker_opts \
  533. --network "$_DB_NAME" \
  534. -v "$host_db_working_dir:$DB_DATADIR" \
  535. "$DOCKER_BASE_IMAGE"
  536. if ! container_id=$(
  537. docker run -d \
  538. --name "$_DB_NAME" \
  539. $docker_opts \
  540. --network "$_DB_NAME" \
  541. -v "$host_db_working_dir:$DB_DATADIR" \
  542. "$DOCKER_BASE_IMAGE"
  543. ); then
  544. err "'docker run' failed !"
  545. _clean_docker "$_DB_NAME" "$container_id"
  546. rm "$state_tmpdir/${_DB_NAME}.working"
  547. return 1
  548. fi
  549. trap_add EXIT,ERR "_clean_docker \"$_DB_NAME\" \"$container_id\""
  550. fi
  551. if docker_ip=$(wait_for_docker_ip "$container_id"); then
  552. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip"
  553. echo "$docker_ip" > "$_STATE_FILE"
  554. debug "written '$_STATE_FILE'"
  555. rm "$state_tmpdir/${_DB_NAME}.working"
  556. _set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  557. return 0
  558. else
  559. errlvl="$?"
  560. err "Db not found (errlvl: $errlvl). Tail of docker logs follows:"
  561. docker logs --tail=5 "$container_id" 2>&1 | prefix " | " >&2
  562. rm "$state_tmpdir/${_DB_NAME}.working"
  563. return "$errlvl"
  564. fi
  565. }
  566. export -f ensure_db_docker_running
  567. ## Require to set $db_docker_opts if needed, and $DB_PASSFILE
  568. ##
  569. _dcmd() {
  570. local docker_opts command="$1"
  571. shift
  572. debug "Db> $command $@"
  573. if [ -f "$HOST_DB_PASSFILE" -a "$CLIENT_DB_PASSFILE" ]; then
  574. verb "Found and using '$HOST_DB_PASSFILE' as '$CLIENT_DB_PASSFILE'."
  575. docker_opts=("${db_docker_opts[@]}" "-v" "$HOST_DB_PASSFILE:$CLIENT_DB_PASSFILE")
  576. else
  577. docker_opts=("${db_docker_opts[@]}")
  578. fi
  579. ## XXXX was here: actualy, we need only connection between this version and the client version
  580. debug docker run -i --rm \
  581. "${docker_opts[@]}" \
  582. --entrypoint "$command" "$DOCKER_BASE_IMAGE" "${db_cmd_opts[@]}" "$@"
  583. docker run -i --rm \
  584. "${docker_opts[@]}" \
  585. --entrypoint "$command" "$DOCKER_BASE_IMAGE" "${db_cmd_opts[@]}" "$@"
  586. }
  587. export -f _dcmd
  588. ## Executes code through db
  589. dcmd() {
  590. local fun
  591. [ "$DB_NAME" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_NAME."
  592. [ "$DB_DATADIR" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_DATADIR."
  593. # [ "$DB_PASSFILE" ] || print_syntax_error "$FUNCNAME: You must provide \$DB_PASSFILE."
  594. [ "$_PID" ] || print_syntax_error "$FUNCNAME: You must provide \$_PID."
  595. for fun in is_db_locked _set_db_params ddb; do
  596. [ "$(type -t "$fun")" == "function" ] ||
  597. print_syntax_error "$FUNCNAME: You must provide function '$fun'."
  598. done
  599. ensure_db_docker_running </dev/null || return 1
  600. _dcmd "$@"
  601. }
  602. export -f dcmd
  603. get_docker_ips() {
  604. local name="$1" ip format network_id
  605. if ! docker inspect --format='{{ .NetworkSettings.Networks }}' "$name" >/dev/null 2>&1; then
  606. echo "default:$(docker inspect --format='{{ .NetworkSettings.IPAdress }}' "$name" 2>/dev/null)"
  607. else
  608. format='{{range $name, $conf := .NetworkSettings.Networks}}{{$name}}{{"\x00"}}{{$conf.IPAddress}}{{"\x00"}}{{end}}'
  609. while read-0 network_id ip; do
  610. printf "%s:%s\n" "$network_id" "$ip"
  611. done < <(docker inspect --format="$format" "$name")
  612. fi
  613. }
  614. export -f get_docker_ips
  615. get_docker_ip() {
  616. local name="$1"
  617. get_docker_ips "$name"
  618. }
  619. export -f get_docker_ip
  620. wait_docker_ip() {
  621. local name="$1" timeout="${2:-15}" timeout_count=0 docker_ip=
  622. start=$SECONDS
  623. while [ -z "$docker_ip" ]; do
  624. sleep 0.5
  625. docker_ip=$(get_docker_ip "$name") && break
  626. elapsed=$((SECONDS - start))
  627. if ((elapsed > timeout)); then
  628. err "${RED}timeout error${NORMAL}(${timeout}s):" \
  629. "Could not find '$name' docker container's IP."
  630. return 1
  631. fi
  632. [ "$elapsed" == "$old_elapsed" ] ||
  633. verb "Waiting for docker $name... ($elapsed/$timeout)"
  634. old_elapsed="$elapsed"
  635. done
  636. verb "Found docker $name network and IP: $docker_ip"
  637. echo "$docker_ip"
  638. }
  639. export -f wait_docker_ip
  640. wait_for_tcp_port() {
  641. local network=$1 host_port=$2 timeout=20
  642. verb "Trying to connect to $host_port"
  643. bash_image=${DEFAULT_BASH_IMAGE:-docker.0k.io/bash}
  644. echo docker run --rm -i --network "$network" "$bash_image" >&2
  645. docker run --rm -i --network "$network" "$bash_image" <<EOF
  646. start=\$SECONDS
  647. while true; do
  648. timeout 1 bash -c "</dev/tcp/${host_port/://}" >/dev/null 2>&1 && break
  649. sleep 0.2
  650. if [ "\$((SECONDS - start))" -gt "$timeout" ]; then
  651. exit 1
  652. fi
  653. done
  654. exit 0
  655. EOF
  656. if [ "$?" != 0 ]; then
  657. err "${RED}timeout error${NORMAL}(${timeout}s):"\
  658. "Could not connect to $host_port."
  659. return 1
  660. fi
  661. return 0
  662. }
  663. export -f wait_for_tcp_port
  664. ## Warning: requires a ``ddb`` matching current database to be checked
  665. wait_for_docker_ip() {
  666. local name=$1 DOCKER_IP= DOCKER_NETWORK= docker_ips= docker_ip= elapsed timeout=10
  667. docker_ip=$(wait_docker_ip "$name" 5) || return 1
  668. IFS=: read DOCKER_NETWORK DOCKER_IP <<<"$docker_ip"
  669. if ! str_is_ipv4 "$DOCKER_IP"; then
  670. err "internal 'wait_docker_ip' did not return a valid IP. Returned IP is '$DOCKER_IP'."
  671. return 1
  672. fi
  673. _set_db_params "$DOCKER_IP" "$DOCKER_NETWORK"
  674. while read-0 port; do
  675. IFS="/" read port type <<<"$port"
  676. [ "$type" == "tcp" ] || continue
  677. wait_for_tcp_port "$DOCKER_NETWORK" "$DOCKER_IP:${port}" || return 17
  678. info "Host/Port $DOCKER_IP:${port} checked ${GREEN}open${NORMAL}."
  679. done < <(image_exposed_ports_0 "$container_id")
  680. ## Checking direct connection
  681. timeout=10
  682. start=$SECONDS
  683. while true; do
  684. if err=$(echo "SELECT 1;" | ddb 2>&1 >/dev/null); then
  685. break
  686. fi
  687. if ! [[ "$err" == *"the database system is starting up" ]]; then
  688. err "${RED}db connection error${NORMAL}:" \
  689. "Could not connect to db on $DOCKER_IP container's IP."
  690. echo " Note: IP up, TCP ports is(are) open" >&2
  691. if [ "$err" ]; then
  692. echo " Error:" >&2
  693. printf "%s\n" "$err" | prefix " ${RED}!${NORMAL} " >&2
  694. fi
  695. return 18
  696. fi
  697. debug "Got starting up error '$err'."
  698. elapsed=$((SECONDS - start))
  699. if ((elapsed > timeout)); then
  700. err "${RED}db connection error${NORMAL}:"\
  701. "Could not connect to db on $DOCKER_IP" \
  702. "container's IP. (IP up, TCP ports is(are) open, sql answer after ${timeout}s)"
  703. return 1
  704. fi
  705. sleep 0.2
  706. done
  707. echo "${DOCKER_NETWORK}:${DOCKER_IP}"
  708. return 0
  709. }
  710. export -f wait_for_docker_ip
  711. docker_add_host_declaration() {
  712. local src_docker=$1 domain=$2 dst_docker=$3 dst_docker_ip= dst_docker_network
  713. dst_docker_ip=$(wait_docker_ip "$dst_docker") || exit 1
  714. IFS=: read dst_docker_ip dst_docker_network <<<"$dst_docker_ip"
  715. docker exec -i "$src_docker" bash <<EOF
  716. if cat /etc/hosts | grep -E "^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+$domain\$" > /dev/null 2>&1; then
  717. sed -ri "s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+$domain\$/$dst_docker_ip $domain/g" /etc/hosts
  718. else
  719. echo "$dst_docker_ip $domain" >> /etc/hosts
  720. fi
  721. EOF
  722. }
  723. export -f docker_add_host_declaration
  724. get_running_containers_for_service() {
  725. local service="$1"
  726. docker ps --filter label="compose.service=$service" --format="{{.ID}}"
  727. }
  728. export -f get_running_containers_for_service
  729. get_container_network_ips() {
  730. local container="$1"
  731. docker inspect "$container" \
  732. --format='{{range $key, $val :=.NetworkSettings.Networks}}{{$key}}{{"\x00"}}{{$val.IPAddress}}{{"\x00"}}{{end}}'
  733. }
  734. export -f get_container_network_ips
  735. get_container_network_ip() {
  736. local container="$1"
  737. while read-0 network ip; do
  738. printf "%s\0" "$network" "$ip"
  739. break
  740. done < <(get_container_network_ips "$container")
  741. }
  742. export -f get_container_network_ip
  743. ##
  744. ## Internal Process
  745. ##
  746. get_docker_compose_links() {
  747. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  748. deps master_service master_target_service _relation_name \
  749. target_service _relation_config tech_dep
  750. if [ -z "$service" ]; then
  751. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  752. return 1
  753. fi
  754. if [ -e "$cache_file" ]; then
  755. # debug "$FUNCNAME: cache hit ($*)"
  756. cat "$cache_file"
  757. return 0
  758. fi
  759. master_service=$(get_top_master_service_for_service "$service") || return 1
  760. deps=()
  761. while read-0 _relation_name target_service _relation_config tech_dep; do
  762. master_target_service="$(get_top_master_service_for_service "$target_service")" || return 1
  763. [ "$master_service" == "$master_target_service" ] && continue
  764. type="$(get_service_type "$target_service")" || return 1
  765. [ "$type" == "run-once" ] && continue
  766. if [ "$tech_dep" == "reversed" ]; then
  767. deps+=("$(echo -en "$master_target_service:\n links:\n - $master_service")")
  768. elif [ "$tech_dep" == "True" ]; then
  769. deps+=("$(echo -en "$master_service:\n links:\n - $master_target_service")")
  770. fi
  771. ## XXXvlab: an attempt to add depends_on, but this doesn't work well actually
  772. ## as there's a circular dependency issue. We don't really want the full feature
  773. ## of depends_on, but just to add it as targets when doing an 'up'
  774. # deps+=("$(echo -en "$master_service:\n depends_on:\n - $master_target_service")")
  775. done < <(get_service_relations "$service")
  776. merge_yaml_str "${deps[@]}" | tee "$cache_file" || return 1
  777. if [ "${PIPESTATUS[0]}" != 0 ]; then
  778. rm "$cache_file"
  779. err "Failed to merge YAML from all ${WHITE}links${NORMAL} dependencies."
  780. return 1
  781. fi
  782. }
  783. _get_docker_compose_opts() {
  784. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  785. compose_def master_service docker_compose_opts
  786. if [ -z "$service" ]; then
  787. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  788. return 1
  789. fi
  790. if [ -e "$cache_file" ]; then
  791. # debug "$FUNCNAME: cache hit ($*)"
  792. cat "$cache_file"
  793. return 0
  794. fi
  795. compose_def="$(get_compose_service_def "$service")" || return 1
  796. master_service="$(get_top_master_service_for_service "$service")"
  797. if docker_compose_opts=$(echo "$compose_def" | shyaml get-value -y "docker-compose" 2>/dev/null); then
  798. yaml_key_val_str "$master_service" "$docker_compose_opts"
  799. fi | tee "$cache_file"
  800. if [ "${PIPESTATUS[0]}" != 0 ]; then
  801. rm "$cache_file"
  802. return 1
  803. fi
  804. }
  805. ##
  806. ## By Reading the metadata.yml, we create a docker-compose.yml mixin.
  807. ## Some metadata.yml (of subordinates) will indeed modify other
  808. ## services than themselves.
  809. _get_docker_compose_service_mixin() {
  810. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  811. links_yaml base_mixin links_yaml docker_compose_options \
  812. charm charm_part
  813. if [ -z "$service" ]; then
  814. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  815. return 1
  816. fi
  817. if [ -e "$cache_file" ]; then
  818. # debug "$FUNCNAME: cache hit ($*)"
  819. cat "$cache_file"
  820. return 0
  821. fi
  822. master_service=$(get_top_master_service_for_service "$service") || {
  823. err "Failed to get top master service for service $DARKYELLOW$service$NORMAL"
  824. return 1
  825. }
  826. ## The compose part
  827. base_mixin="$master_service:
  828. labels:
  829. - compose.service=$service
  830. - compose.master-service=${master_service}
  831. - compose.project=$(get_default_project_name)"
  832. links_yaml=$(get_docker_compose_links "$service") || return 1
  833. docker_compose_options=$(_get_docker_compose_opts "$service") || return 1
  834. ## the charm part
  835. charm_part=$(get_docker_compose_mixin_from_metadata "$service") || return 1
  836. ## Merge results
  837. if [ "$charm_part" ]; then
  838. charm_yaml="$(yaml_key_val_str "$master_service" "$charm_part")" || return 1
  839. merge_yaml_str "$base_mixin" "$links_yaml" "$charm_yaml" "$docker_compose_options" || return 1
  840. else
  841. merge_yaml_str "$base_mixin" "$links_yaml" "$docker_compose_options" || return 1
  842. fi | tee "$cache_file"
  843. if [ "${PIPESTATUS[0]}" != 0 ]; then
  844. err "Failed to constitute the base YAML for service '${DARKYELLOW}$service${NORMAL}'"
  845. rm "$cache_file"
  846. return 1
  847. fi
  848. }
  849. export -f _get_docker_compose_service_mixin
  850. ##
  851. ## Get full `docker-compose.yml` format for all listed services (and
  852. ## their deps)
  853. ##
  854. ## @export
  855. ## @cache: !system !nofail +stdout
  856. get_docker_compose () {
  857. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  858. entries services service start docker_compose_services
  859. if [ -e "$cache_file" ]; then
  860. # debug "$FUNCNAME: cache hit ($*)"
  861. cat "$cache_file"
  862. return 0
  863. fi
  864. ##
  865. ## Adding sub services configurations
  866. ##
  867. declare -A entries
  868. start_compilation=$SECONDS
  869. debug "Compiling 'docker-compose.yml' base for ${DARKYELLOW}$*$NORMAL..."
  870. for target_service in "$@"; do
  871. start=$SECONDS
  872. services=$(get_ordered_service_dependencies "$target_service") || {
  873. err "Failed to get dependencies for $DARKYELLOW$target_service$NORMAL"
  874. return 1
  875. }
  876. debug " $DARKYELLOW$target_service$NORMAL deps:$DARKYELLOW" $services "$NORMAL$GRAY(in $((SECONDS - start))s)$NORMAL"
  877. for service in $services; do
  878. if [ "${entries[$service]}" ]; then
  879. ## Prevent double inclusion of same service if this
  880. ## service is deps of two or more of your
  881. ## requirements.
  882. continue
  883. fi
  884. ## mark the service as "loaded" as well as it's containers
  885. ## if this is a subordinate service
  886. start_service=$SECONDS
  887. entries[$service]=$(_get_docker_compose_service_mixin "$service") || {
  888. err "Failed to get service mixin for $DARKYELLOW$service$NORMAL"
  889. return 1
  890. }
  891. debug " Applied $DARKYELLOW$service$NORMAL charm metadata mixins $GRAY(in $((SECONDS - start_service))s)$NORMAL"
  892. done
  893. debug " ..finished all mixins for $DARKYELLOW$target_service$NORMAL $GRAY(in $((SECONDS - start))s)$NORMAL"
  894. done
  895. docker_compose_services=$(merge_yaml_str "${entries[@]}") || {
  896. err "Failed to merge YAML services entries together."
  897. return 1
  898. }
  899. base_v2="version: '2.0'"
  900. merge_yaml_str "$(yaml_key_val_str "services" "$docker_compose_services")" \
  901. "$base_v2" > "$cache_file" || return 1
  902. export _current_docker_compose="$(cat "$cache_file")"
  903. echo "$_current_docker_compose"
  904. debug " ..compilation of base 'docker-compose.yml' done $GRAY(in $((SECONDS - start_compilation))s)$NORMAL" || true
  905. # debug " ** ${WHITE}docker-compose.yml${NORMAL}:"
  906. # debug "$_current_docker_compose"
  907. }
  908. export -f get_docker_compose
  909. _get_compose_service_def_cached () {
  910. local service="$1" docker_compose="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  911. if [ -e "$cache_file" ]; then
  912. #debug "$FUNCNAME: STATIC cache hit"
  913. cat "$cache_file" &&
  914. touch "$cache_file" || return 1
  915. return 0
  916. fi
  917. value=$(echo "$docker_compose" | shyaml get-value "${service//./\\.}" 2>/dev/null)
  918. [ "$value" == None ] && value=""
  919. if ! echo "$value" | shyaml get-value "charm" >/dev/null 2>&1; then
  920. if charm.exists "$service"; then
  921. value=$(merge_yaml <(echo "charm: $service") <(echo "$value")) || {
  922. err "Can't merge YAML infered 'charm: $service' with base ${DARKYELLOW}$service${NORMAL} YAML definition."
  923. return 1
  924. }
  925. else
  926. err "No ${WHITE}charm${NORMAL} value for service $DARKYELLOW$service$NORMAL" \
  927. "in compose, nor same name charm found."
  928. return 1
  929. fi
  930. fi
  931. echo "$value" | tee "$cache_file" || return 1
  932. # if [ "${PIPESTATUS[0]}" != 0 ]; then
  933. # rm "$cache_file"
  934. # return 1
  935. # fi
  936. return 0
  937. # if [ "${PIPESTATUS[0]}" != 0 -o \! -s "$cache_file" ]; then
  938. # rm "$cache_file"
  939. # err "PAS OK $service: $value"
  940. # return 1
  941. # fi
  942. }
  943. export -f _get_compose_service_def_cached
  944. ## XXXvlab: a lot to be done to cache the results
  945. get_compose_service_def () {
  946. local service="$1" docker_compose cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  947. result
  948. if [ -e "$cache_file" ]; then
  949. #debug "$FUNCNAME: SESSION cache hit"
  950. cat "$cache_file" || return 1
  951. return 0
  952. fi
  953. [ -z "$service" ] && print_syntax_error "Missing service as first argument."
  954. docker_compose=$(get_compose_yml_content) || return 1
  955. result=$(_get_compose_service_def_cached "$service" "$docker_compose") || return 1
  956. charm=$(echo "$result" |shyaml get-value charm 2>/dev/null) || return 1
  957. metadata=$(charm.metadata "$charm") || return 1
  958. if default_options=$(printf "%s" "$metadata" | shyaml -y -q get-value default-options); then
  959. default_options=$(yaml_key_val_str "options" "$default_options") || return 1
  960. result=$(merge_yaml_str "$default_options" "$result") || return 1
  961. fi
  962. echo "$result" | tee "$cache_file" || return 1
  963. }
  964. export -f get_compose_service_def
  965. _get_service_charm_cached () {
  966. local service="$1" service_def="$2" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  967. if [ -e "$cache_file" ]; then
  968. # debug "$FUNCNAME: cache hit $1"
  969. cat "$cache_file" &&
  970. touch "$cache_file" || return 1
  971. return 0
  972. fi
  973. charm=$(echo "$service_def" | shyaml get-value charm 2>/dev/null)
  974. if [ -z "$charm" ]; then
  975. err "Missing ${WHITE}charm${NORMAL} value in service $DARKYELLOW$service$NORMAL definition."
  976. return 1
  977. fi
  978. echo "$charm" | tee "$cache_file" || return 1
  979. }
  980. export -f _get_service_charm_cached
  981. get_service_charm () {
  982. local service="$1"
  983. if [ -z "$service" ]; then
  984. echo ${FUNCNAME[@]} >&2
  985. print_syntax_error "$FUNCNAME: Please specify a service as first argument."
  986. return 1
  987. fi
  988. service_def=$(get_compose_service_def "$service") || return 1
  989. _get_service_charm_cached "$service" "$service_def"
  990. }
  991. export -f get_service_charm
  992. ## built above the docker-compose abstraction, so it relies on the
  993. ## full docker-compose.yml to be already built.
  994. get_service_def () {
  995. local service="$1" def
  996. if [ -z "$_current_docker_compose" ]; then
  997. print_syntax_error "$FUNCNAME is meant to be called after"\
  998. "\$_current_docker_compose has been calculated."
  999. fi
  1000. def=$(echo "$_current_docker_compose" | shyaml get-value "services.${service//./\\.}" 2>/dev/null)
  1001. if [ -z "$def" ]; then
  1002. err "No definition for service $DARKYELLOW$service$NORMAL in compiled 'docker-compose.yml'."
  1003. return 1
  1004. fi
  1005. echo "$def"
  1006. }
  1007. export -f get_service_def
  1008. ## Return the base docker image name of a service
  1009. service_base_docker_image() {
  1010. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1011. master_service service_def service_image service_build service_dockerfile
  1012. if [ -e "$cache_file" ]; then
  1013. # debug "$FUNCNAME: cache hit ($*)"
  1014. cat "$cache_file"
  1015. return 0
  1016. fi
  1017. master_service="$(get_top_master_service_for_service "$service")" || {
  1018. err "Could not compute master service for service $DARKYELLOW$service$NORMAL."
  1019. return 1
  1020. }
  1021. service_def="$(get_service_def "$master_service")" || {
  1022. err "Could not get docker-compose service definition for $DARKYELLOW$master_service$NORMAL."
  1023. return 1
  1024. }
  1025. service_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null)
  1026. if [ "$?" != 0 ]; then
  1027. ## According to https://stackoverflow.com/questions/32230577 , if there's a build,
  1028. ## then the builded image will get name ${project}_${service}
  1029. project=$(get_default_project_name) || return 1
  1030. image_name="${project}_${service}"
  1031. if ! docker_has_image "$image_name"; then
  1032. service_build=$(echo "$service_def" | shyaml get-value build 2>/dev/null)
  1033. if [ "$?" != 0 ]; then
  1034. err "Service $DARKYELLOW$service$NORMAL has no ${WHITE}image${NORMAL} nor ${WHITE}build${NORMAL} parameter."
  1035. echo "$service_def" >&2
  1036. return 1
  1037. fi
  1038. docker build "$service_build" -t "${project}_${service}" >&2 || {
  1039. err "Failed to build image for ${DARKYELLOW}$service${NORMAL}."
  1040. return 1
  1041. }
  1042. fi
  1043. printf "%s" "${project}_${service}"
  1044. else
  1045. printf "%s" "${service_image}"
  1046. fi | tee "$cache_file"
  1047. if [ "${PIPESTATUS[0]}" != 0 ]; then
  1048. rm "$cache_file"
  1049. return 1
  1050. fi
  1051. }
  1052. export -f service_base_docker_image
  1053. get_charm_relation_def () {
  1054. local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1055. relation_def metadata
  1056. if [ -e "$cache_file" ]; then
  1057. # debug "$FUNCNAME: cache hit ($*)"
  1058. cat "$cache_file"
  1059. return 0
  1060. fi
  1061. metadata="$(charm.metadata "$charm")" || return 1
  1062. relation_def="$(echo "$metadata" | shyaml get-value "provides.${relation_name}" 2>/dev/null)"
  1063. echo "$relation_def" | tee "$cache_file"
  1064. }
  1065. export -f get_charm_relation_def
  1066. get_charm_tech_dep_orientation_for_relation() {
  1067. local charm="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1068. relation_def value
  1069. if [ -e "$cache_file" ]; then
  1070. # debug "$FUNCNAME: cache hit ($*)"
  1071. cat "$cache_file"
  1072. return 0
  1073. fi
  1074. relation_def=$(get_charm_relation_def "$charm" "$relation_name" 2>/dev/null)
  1075. value=$(echo "$relation_def" | shyaml get-value 'tech-dep' 2>/dev/null)
  1076. value=${value:-True}
  1077. printf "%s" "$value" | tee "$cache_file"
  1078. }
  1079. export -f get_charm_tech_dep_orientation_for_relation
  1080. get_service_relation_tech_dep() {
  1081. local service="$1" relation_name="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1.$2" \
  1082. charm tech_dep
  1083. if [ -e "$cache_file" ]; then
  1084. # debug "$FUNCNAME: cache hit ($*)"
  1085. cat "$cache_file"
  1086. return 0
  1087. fi
  1088. charm=$(get_service_charm "$service") || return 1
  1089. tech_dep="$(get_charm_tech_dep_orientation_for_relation "$charm" "$relation_name")" || return 1
  1090. printf "%s" "$tech_dep" | tee "$cache_file"
  1091. }
  1092. export -f get_service_relation_tech_dep
  1093. ##
  1094. ## Use compose file to get deps, and relation definition in metadata.yml
  1095. ## for tech-dep attribute.
  1096. get_service_deps() {
  1097. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  1098. if [ -e "$cache_file" ]; then
  1099. # debug "$FUNCNAME: cache hit ($*)"
  1100. cat "$cache_file"
  1101. return 0
  1102. fi
  1103. (
  1104. set -o pipefail
  1105. get_service_relations "$service" | \
  1106. while read-0 relation_name target_service _relation_config tech_dep; do
  1107. echo "$target_service"
  1108. done | tee "$cache_file"
  1109. ) || return 1
  1110. }
  1111. export -f get_service_deps
  1112. _rec_get_depth() {
  1113. local elt=$1 dep deps max cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  1114. [ "${depths[$elt]}" ] && return 0
  1115. if [ -e "$cache_file" ]; then
  1116. # debug "$FUNCNAME: cache hit ($*)"
  1117. depths[$elt]=$(cat "$cache_file")
  1118. return 0
  1119. fi
  1120. visited[$elt]=1
  1121. #debug "Setting visited[$elt]"
  1122. #debug "Asking for $DARKYELLOW$elt$NORMAL dependencies"
  1123. deps=$(get_service_deps "$elt") || {
  1124. debug "Failed get_service_deps $elt"
  1125. return 1
  1126. }
  1127. # debug "$elt deps are:" $deps
  1128. max=0
  1129. for dep in $deps; do
  1130. [ "${visited[$dep]}" ] && {
  1131. #debug "Already computing $dep"
  1132. continue
  1133. }
  1134. _rec_get_depth "$dep" || return 1
  1135. #debug "Requesting depth[$dep]"
  1136. if (( ${depths[$dep]} > max )); then
  1137. max="${depths[$dep]}"
  1138. fi
  1139. done
  1140. # debug "Setting depth[$elt] to $((max + 1))"
  1141. depths[$elt]=$((max + 1))
  1142. echo "${depths[$elt]}" > $cache_file
  1143. }
  1144. export -f _rec_get_depth
  1145. get_ordered_service_dependencies() {
  1146. local services=("$@") cache_file="$state_tmpdir/$FUNCNAME.cache.$(echo "$*" | md5_compat)"
  1147. if [ -e "$cache_file" ]; then
  1148. # debug "$FUNCNAME: cache hit ($*)"
  1149. cat "$cache_file"
  1150. return 0
  1151. fi
  1152. #debug "Figuring ordered deps of $DARKYELLOW$services$NORMAL"
  1153. if [ -z "${services[*]}" ]; then
  1154. return 0
  1155. # print_syntax_error "$FUNCNAME: no arguments"
  1156. # return 1
  1157. fi
  1158. declare -A depths
  1159. declare -A visited
  1160. heads=("${services[@]}")
  1161. while [ "${#heads[@]}" != 0 ]; do
  1162. array_pop heads head
  1163. _rec_get_depth "$head" || return 1
  1164. done
  1165. i=0
  1166. while [ "${#depths[@]}" != 0 ]; do
  1167. for key in "${!depths[@]}"; do
  1168. value="${depths[$key]}"
  1169. if [ "$value" == "$i" ]; then
  1170. echo "$key"
  1171. unset depths[$key]
  1172. fi
  1173. done
  1174. i=$((i + 1))
  1175. done | tee "$cache_file"
  1176. }
  1177. export -f get_ordered_service_dependencies
  1178. run_service_hook () {
  1179. local action="$1" service subservice subservices loaded
  1180. shift
  1181. declare -A loaded
  1182. for service in "$@"; do
  1183. subservices=$(get_ordered_service_dependencies "$service") || return 1
  1184. for subservice in $subservices; do
  1185. if [ "${loaded[$subservice]}" ]; then
  1186. ## Prevent double inclusion of same service if this
  1187. ## service is deps of two or more of your
  1188. ## requirements.
  1189. continue
  1190. fi
  1191. charm=$(get_service_charm "$subservice") || return 1
  1192. charm.has_hook "$charm" "$action" >/dev/null || continue
  1193. PROJECT_NAME=$(get_default_project_name) || return 1
  1194. MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
  1195. MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1
  1196. DOCKER_BASE_IMAGE=$(service_base_docker_image "$MASTER_BASE_SERVICE_NAME") || return 1
  1197. Wrap -d "running $YELLOW$action$NORMAL hook of $DARKYELLOW$subservice$NORMAL in charm $DARKPINK$charm$NORMAL" <<EOF || return 1
  1198. export DOCKER_BASE_IMAGE="$DOCKER_BASE_IMAGE"
  1199. export SERVICE_NAME=$subservice
  1200. export IMAGE_NAME=$(echo "${PROJECT_NAME}" | tr -d "_-")_\${SERVICE_NAME}
  1201. export CONTAINER_NAME=\${IMAGE_NAME}_1
  1202. export CHARM_NAME="$charm"
  1203. export PROJECT_NAME="$PROJECT_NAME"
  1204. export SERVICE_DATASTORE="$DATASTORE/$subservice"
  1205. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$subservice"
  1206. export MASTER_BASE_SERVICE_NAME="$MASTER_BASE_SERVICE_NAME"
  1207. export MASTER_BASE_CHARM_NAME="$MASTER_BASE_CHARM_NAME"
  1208. charm.run_hook "$charm" "$action"
  1209. EOF
  1210. loaded[$subservice]=1
  1211. done
  1212. done
  1213. return 0
  1214. }
  1215. host_resource_get() {
  1216. local location="$1" cfg="$2"
  1217. type=$(echo "$cfg" | shyaml get-value type 2>/dev/null) || {
  1218. err "Missing ${WHITE}type$NORMAL option in ${WHITE}get$NORMAL config for location '$location'"
  1219. return 1
  1220. }
  1221. if fn.exists host_resource_get_$type; then
  1222. host_resource_get_$type "$location" "$cfg"
  1223. else
  1224. err "Source ${WHITE}source$NORMAL type '$type' unknown for" \
  1225. "${WHITE}host-resource$NORMAL '$location' defined in" \
  1226. "$DARKYELLOW$subservice$NORMAL config."
  1227. return 1
  1228. fi
  1229. }
  1230. export -f host_resource_get
  1231. host_resource_get_git() {
  1232. local location="$1" cfg="$2" branch parent url
  1233. branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null)
  1234. branch=${branch:-master}
  1235. url=$(echo "$cfg" | shyaml get-value url 2>/dev/null)
  1236. parent="$(dirname "$location")"
  1237. (
  1238. mkdir -p "$parent" && cd "$parent" &&
  1239. git clone -b "$branch" "$url" "$(basename "$location")"
  1240. ) || return 1
  1241. }
  1242. export -f host_resource_get_git
  1243. host_resource_get_git-sub() {
  1244. local location="$1" cfg="$2" branch parent url
  1245. branch=$(echo "$cfg" | shyaml get-value branch 2>/dev/null)
  1246. branch=${branch:-master}
  1247. url=$(echo "$cfg" | shyaml get-value url 2>/dev/null)
  1248. parent="$(dirname "$location")"
  1249. (
  1250. mkdir -p "$parent" && cd "$parent" &&
  1251. git sub clone -b "$branch" "$url" "$(basename "$location")"
  1252. ) || return 1
  1253. }
  1254. export -f host_resource_get_git-sub
  1255. setup_host_resource () {
  1256. local subservice="$1" service_def location get cfg
  1257. service_def=$(get_compose_service_def "$subservice") || return 1
  1258. while read-0 location cfg; do
  1259. ## XXXvlab: will it be a git resources always ?
  1260. if [ -d "$location" -a ! -d "$location/.git" ]; then
  1261. err "Hum, location '$location' does not seem to be a git directory."
  1262. return 1
  1263. fi
  1264. if [ -d "$location" ]; then
  1265. info "host resource '$location' already set up."
  1266. continue
  1267. fi
  1268. get=$(echo "$cfg" | shyaml get-value get 2>/dev/null)
  1269. if [ -z "$get" ]; then
  1270. err "No host directory '$location' found, and no ${WHITE}source$NORMAL" \
  1271. "specified for $DARKYELLOW$subservice$NORMAL."
  1272. return 1
  1273. fi
  1274. host_resource_get "$location" "$get" || return 1
  1275. done < <(echo "$service_def" | shyaml key-values-0 host-resources 2>/dev/null)
  1276. }
  1277. export -f setup_host_resource
  1278. setup_host_resources () {
  1279. local service subservices subservice loaded
  1280. declare -A loaded
  1281. for service in "$@"; do
  1282. subservices=$(get_ordered_service_dependencies "$service") || return 1
  1283. for subservice in $subservices; do
  1284. if [ "${loaded[$subservice]}" ]; then
  1285. ## Prevent double inclusion of same service if this
  1286. ## service is deps of two or more of your
  1287. ## requirements.
  1288. continue
  1289. fi
  1290. setup_host_resource "$service"
  1291. loaded[$subservice]=1
  1292. done
  1293. done
  1294. return 0
  1295. }
  1296. export -f setup_host_resources
  1297. relation-get () {
  1298. local key="$1" out
  1299. if ! out=$(cat "$RELATION_DATA_FILE" | shyaml -y get-value "$key" 2>/dev/null); then
  1300. err "The key $WHITE$key$NORMAL was not found in relation's data."
  1301. return 1
  1302. fi
  1303. echo "$out" | yaml_get_interpret
  1304. }
  1305. export -f relation-get
  1306. expand_vars() {
  1307. local unlikely_prefix="UNLIKELY_PREFIX"
  1308. content=$(cat -)
  1309. ## find first identifier not in content
  1310. remaining_lines=$(echo "$content" | grep "^$unlikely_prefix")
  1311. while [ "$(echo "$remaining_lines" | grep "^$unlikely_prefix$")" ]; do
  1312. size_prefix="${#unlikely_prefix}"
  1313. first_matching=$(echo "$remaining_lines" |
  1314. grep -v "^$unlikely_prefix$" |
  1315. uniq -w "$((size_prefix + 1))" -c |
  1316. sort -rn |
  1317. head -n 1)
  1318. first_matching=${first_matching#"${x%%[![:space:]]*}"}
  1319. first_matching="${first_matching#* }"
  1320. next_char=${first_matching:$size_prefix:1}
  1321. if [ "$next_char" != "0" ]; then
  1322. unlikely_prefix+="0"
  1323. else
  1324. unlikely_prefix+="1"
  1325. fi
  1326. remaining_lines=$(echo "$remaining_lines" | grep "^$unlikely_prefix")
  1327. done
  1328. eval "cat <<$unlikely_prefix
  1329. $content
  1330. $unlikely_prefix"
  1331. }
  1332. export -f expand_vars
  1333. yaml_get_interpret() {
  1334. local content tag
  1335. content=$(cat -)
  1336. tag=$(echo "$content" | shyaml -y get-value) || return 1
  1337. tag="${tag%% *}"
  1338. content=$(echo "$content" | shyaml get-value) || return 1
  1339. if ! [ "${tag:0:1}" == "!" ]; then
  1340. echo "$content" || return 1
  1341. return 0
  1342. fi
  1343. case "$tag" in
  1344. "!bash-stdout")
  1345. echo "$content" | bash || {
  1346. err "shell code didn't end with errorlevel 0"
  1347. return 1
  1348. }
  1349. ;;
  1350. "!var-expand")
  1351. echo "$content" | expand_vars || {
  1352. err "shell expansion failed"
  1353. return 1
  1354. }
  1355. ;;
  1356. *)
  1357. err "Invalid object tag ${WHITE}$tag${NORMAL}"
  1358. return 1
  1359. ;;
  1360. esac
  1361. }
  1362. export -f yaml_get_interpret
  1363. options-get () {
  1364. local key="$1" out
  1365. service_def=$(get_compose_service_def "$SERVICE_NAME") || return 1
  1366. if ! out=$(echo "$service_def" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1367. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  1368. return 1
  1369. fi
  1370. echo "$out" | yaml_get_interpret
  1371. }
  1372. export -f options-get
  1373. relation-base-compose-get () {
  1374. local key="$1" out
  1375. if ! out=$(echo "$RELATION_BASE_COMPOSE_DEF" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1376. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  1377. return 1
  1378. fi
  1379. echo "$out" | yaml_get_interpret
  1380. }
  1381. export -f relation-base-compose-get
  1382. relation-target-compose-get () {
  1383. local key="$1" out
  1384. if ! out=$(echo "$RELATION_TARGET_COMPOSE_DEF" | shyaml -y get-value "options.$key" 2>/dev/null); then
  1385. err "The key $WHITE$key$NORMAL was not found in base service compose definition.."
  1386. return 1
  1387. fi
  1388. echo "$out" | yaml_get_interpret
  1389. }
  1390. export -f relation-target-compose-get
  1391. relation-set () {
  1392. local key="$1" value="$2"
  1393. if [ -z "$RELATION_DATA_FILE" ]; then
  1394. err "$FUNCNAME: relation does not seems to be correctly setup."
  1395. return 1
  1396. fi
  1397. if ! [ -r "$RELATION_DATA_FILE" ]; then
  1398. err "$FUNCNAME: can't read relation's data." >&2
  1399. return 1
  1400. fi
  1401. _config_merge "$RELATION_DATA_FILE" <(yaml_key_val_str "$key" "$value")
  1402. }
  1403. export -f relation-set
  1404. _config_merge() {
  1405. local config_filename="$1" merge_to_file="$2"
  1406. touch "$config_filename" &&
  1407. merge_yaml "$config_filename" "$merge_to_file" > "$config_filename.tmp" || return 1
  1408. mv "$config_filename.tmp" "$config_filename"
  1409. }
  1410. export -f _config_merge
  1411. ## XXXvlab; this can be used only in relation, I'd like to use it in init.
  1412. config-add() {
  1413. local metadata="$1"
  1414. _config_merge "$RELATION_CONFIG" <(echo "$metadata")
  1415. }
  1416. export -f config-add
  1417. ## XXXvlab; this can be used only in relation, I'd like to use it in init.
  1418. init-config-add() {
  1419. local metadata="$1"
  1420. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" \
  1421. <(yaml_key_val_str "services" "$metadata")
  1422. }
  1423. export -f init-config-add
  1424. docker_get_uid() {
  1425. local service="$1" user="$2" uid
  1426. uid=$(cached_cmd_on_base_image "$service" "id -u \"$user\"") || {
  1427. debug "Failed to query for '$user' uid in ${DARKYELLOW}$service${NORMAL} base image."
  1428. return 1
  1429. }
  1430. info "uid from ${DARKYELLOW}$service${NORMAL} for user '$user' is '$uid'"
  1431. echo "$uid"
  1432. }
  1433. export -f docker_get_uid
  1434. logstdout() {
  1435. local name="$1"
  1436. sed -r 's%^%'"${name}"'> %g'
  1437. }
  1438. export -f logstdout
  1439. logstderr() {
  1440. local name="$1"
  1441. sed -r 's%^(.*)$%'"${RED}${name}>${NORMAL} \1"'%g'
  1442. }
  1443. export -f logstderr
  1444. _run_service_relation () {
  1445. local relation_name="$1" service="$2" target_service="$3" relation_config="$4" relation_dir services
  1446. charm=$(get_service_charm "$service") || return 1
  1447. target_charm=$(get_service_charm "$target_service") || return 1
  1448. base_script_name=$(charm.has_relation_hook "$charm" "$relation_name" relation-joined) || true
  1449. target_script_name=$(charm.has_relation_hook "$target_charm" "$relation_name" relation-joined) || true
  1450. [ "$base_script_name" -o "$target_script_name" ] || return 0
  1451. relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1
  1452. RELATION_DATA_FILE=$(get_relation_data_file "$service" "$target_service" "$relation_name" "$relation_config") || return 1
  1453. export BASE_SERVICE_NAME=$service
  1454. export BASE_CHARM_NAME=$charm
  1455. export BASE_CHARM_PATH=$(charm.get_dir "$charm")
  1456. export TARGET_SERVICE_NAME=$target_service
  1457. export TARGET_CHARM_NAME=$target_charm
  1458. export TARGET_CHARM_PATH=$(charm.get_dir "$target_charm")
  1459. export RELATION_DATA_FILE
  1460. target_errlvl=0
  1461. if [ -z "$target_script_name" ]; then
  1462. verb "No relation script $DARKBLUE$relation_name$NORMAL in target $DARKPINK$target_charm$NORMAL."
  1463. else
  1464. verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \
  1465. "for target $DARKYELLOW$target_service$NORMAL (charm $DARKPINK$target_charm$NORMAL)"
  1466. RELATION_CONFIG="$relation_dir/config_provider"
  1467. DOCKER_BASE_IMAGE=$(service_base_docker_image "$target_service") || return 1
  1468. export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA
  1469. {
  1470. (
  1471. SERVICE_NAME=$target_service
  1472. SERVICE_DATASTORE="$DATASTORE/$target_service"
  1473. SERVICE_CONFIGSTORE="$CONFIGSTORE/$target_service"
  1474. export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE
  1475. charm.run_relation_hook "$target_charm" "$relation_name" relation-joined
  1476. echo "$?" > "$relation_dir/target_errlvl"
  1477. ) | logstdout "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}"
  1478. } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$target_service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@${NORMAL}" 3>&1 1>&2 2>&3
  1479. target_errlvl="$(cat "$relation_dir/target_errlvl")" || {
  1480. err "Relation script '$script_name' in $DARKPINK$target_charm$NORMAL" \
  1481. "failed before outputing an errorlevel."
  1482. ((target_errlvl |= "1" ))
  1483. }
  1484. if [ -e "$RELATION_CONFIG" ]; then
  1485. debug "Merging some new config info in $DARKYELLOW$target_service$NORMAL"
  1486. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" &&
  1487. rm "$RELATION_CONFIG"
  1488. ((target_errlvl |= "$?"))
  1489. fi
  1490. fi
  1491. if [ "$target_errlvl" == 0 ]; then
  1492. errlvl=0
  1493. if [ "$base_script_name" ]; then
  1494. verb "Running ${DARKBLUE}$relation_name${NORMAL} relation-joined script" \
  1495. "for $DARKYELLOW$service$NORMAL (charm $DARKPINK$charm$NORMAL)"
  1496. RELATION_CONFIG="$relation_dir/config_providee"
  1497. RELATION_DATA="$(cat "$RELATION_DATA_FILE")"
  1498. DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") || return 1
  1499. export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA
  1500. {
  1501. (
  1502. SERVICE_NAME=$service
  1503. SERVICE_DATASTORE="$DATASTORE/$service"
  1504. SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  1505. export SERVICE_NAME DOCKER_BASE_IMAGE SERVICE_DATASTORE SERVICE_CONFIGSTORE
  1506. charm.run_relation_hook "$charm" "$relation_name" relation-joined
  1507. echo "$?" > "$relation_dir/errlvl"
  1508. ) | logstdout "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${GREEN}@${NORMAL}"
  1509. } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$service$NORMAL/$DARKBLUE$relation_name$NORMAL (joined) ${RED}@$NORMAL" 3>&1 1>&2 2>&3
  1510. errlvl="$(cat "$relation_dir/errlvl")" || {
  1511. err "Relation script '$script_name' in $DARKPINK$charm$NORMAL" \
  1512. "failed before outputing an errorlevel."
  1513. ((errlvl |= "1" ))
  1514. }
  1515. if [ -e "$RELATION_CONFIG" ]; then
  1516. _config_merge "$state_tmpdir/to-merge-in-docker-compose.yml" "$RELATION_CONFIG" &&
  1517. rm "$RELATION_CONFIG"
  1518. ((errlvl |= "$?" ))
  1519. fi
  1520. if [ "$errlvl" != 0 ]; then
  1521. err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$service$NORMAL failed to run properly."
  1522. fi
  1523. else
  1524. verb "No relation script '$script_name' in charm $DARKPINK$charm$NORMAL. Ignoring."
  1525. fi
  1526. else
  1527. err "Relation $DARKBLUE$relation_name$NORMAL on $DARKYELLOW$target_service$NORMAL failed to run properly."
  1528. fi
  1529. if [ "$target_errlvl" == 0 -a "$errlvl" == 0 ]; then
  1530. debug "Relation $DARKBLUE$relation_name$NORMAL is established" \
  1531. "between $DARKYELLOW$service$NORMAL and $DARKYELLOW$target_service$NORMAL."
  1532. return 0
  1533. else
  1534. return 1
  1535. fi
  1536. }
  1537. export -f _run_service_relation
  1538. _get_compose_relations_cached () {
  1539. local compose_service_def="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  1540. relation_name relation_def target_service
  1541. if [ -e "$cache_file" ]; then
  1542. #debug "$FUNCNAME: STATIC cache hit $1"
  1543. cat "$cache_file" &&
  1544. touch "$cache_file" || return 1
  1545. return 0
  1546. fi
  1547. (
  1548. set -o pipefail
  1549. if [ "$compose_service_def" ]; then
  1550. while read-0 relation_name relation_def; do
  1551. ## XXXvlab: could we use braces here instead of parenthesis ?
  1552. (
  1553. case "$(echo "$relation_def" | shyaml get-type 2>/dev/null)" in
  1554. "str")
  1555. target_service="$(echo "$relation_def" | shyaml get-value 2>/dev/null)" || return 1
  1556. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  1557. printf "%s\0" "$relation_name" "$target_service" "" "$tech_dep"
  1558. ;;
  1559. "sequence")
  1560. while read-0 target_service; do
  1561. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  1562. printf "%s\0" "$relation_name" "$target_service" "" "$tech_dep"
  1563. done < <(echo "$relation_def" | shyaml get-values-0 2>/dev/null)
  1564. ;;
  1565. "struct")
  1566. while read-0 target_service relation_config; do
  1567. tech_dep="$(get_service_relation_tech_dep "$target_service" "$relation_name")" || return 1
  1568. printf "%s\0" "$relation_name" "$target_service" "$relation_config" "$tech_dep"
  1569. done < <(echo "$relation_def" | shyaml key-values-0 2>/dev/null)
  1570. ;;
  1571. esac
  1572. ) </dev/null >> "$cache_file" || return 1
  1573. done < <(echo "$compose_service_def" | shyaml key-values-0 relations 2>/dev/null)
  1574. fi
  1575. )
  1576. if [ "$?" != 0 ]; then
  1577. err "Error while looking for compose relations."
  1578. rm -f "$cache_file" ## no cache
  1579. return 1
  1580. fi
  1581. [ -e "$cache_file" ] && cat "$cache_file"
  1582. return 0
  1583. }
  1584. export -f _get_compose_relations_cached
  1585. get_compose_relations () {
  1586. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1587. compose_def
  1588. if [ -e "$cache_file" ]; then
  1589. #debug "$FUNCNAME: SESSION cache hit $1"
  1590. cat "$cache_file"
  1591. return 0
  1592. fi
  1593. compose_def="$(get_compose_service_def "$service")" || return 1
  1594. _get_compose_relations_cached "$compose_def" > "$cache_file"
  1595. if [ "$?" != 0 ]; then
  1596. rm -f "$cache_file" ## no cache
  1597. return 1
  1598. fi
  1599. cat "$cache_file"
  1600. }
  1601. export -f get_compose_relations
  1602. get_service_relations () {
  1603. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1604. s rn ts rc td
  1605. if [ -e "$cache_file" ]; then
  1606. #debug "$FUNCNAME: SESSION cache hit $1"
  1607. cat "$cache_file"
  1608. return 0
  1609. fi
  1610. if [ -z "$ALL_RELATIONS" ]; then
  1611. err "Can't access global \$ALL_RELATIONS"
  1612. return 1
  1613. fi
  1614. while read-0 s rn ts rc td; do
  1615. [[ "$s" == "$service" ]] || continue
  1616. printf "%s\0" "$rn" "$ts" "$rc" "$td"
  1617. done < <(cat "$ALL_RELATIONS") > "$cache_file"
  1618. if [ "$?" != 0 ]; then
  1619. rm -f "$cache_file" ## no cache
  1620. return 1
  1621. fi
  1622. cat "$cache_file"
  1623. }
  1624. export -f get_service_relations
  1625. get_service_relation() {
  1626. local service="$1" relation="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1627. rn ts rc td
  1628. if [ -e "$cache_file" ]; then
  1629. #debug "$FUNCNAME: SESSION cache hit $1"
  1630. cat "$cache_file"
  1631. return 0
  1632. fi
  1633. while read-0 rn ts rc td; do
  1634. [ "$relation" == "$rn" ] && {
  1635. printf "%s\0" "$ts" "$rc" "$td"
  1636. break
  1637. }
  1638. done < <(get_service_relations "$service") > "$cache_file"
  1639. if [ "$?" != 0 ]; then
  1640. rm -f "$cache_file" ## no cache
  1641. return 1
  1642. fi
  1643. cat "$cache_file"
  1644. }
  1645. export -f get_service_relation
  1646. _get_charm_metadata_uses() {
  1647. local metadata="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  1648. if [ -e "$cache_file" ]; then
  1649. #debug "$FUNCNAME: SESSION cache hit $1"
  1650. cat "$cache_file" || return 1
  1651. return 0
  1652. fi
  1653. printf "%s" "$metadata" | { shyaml key-values-0 uses 2>/dev/null || true; } | tee "$cache_file"
  1654. }
  1655. export -f _get_charm_metadata_uses
  1656. _get_service_metadata() {
  1657. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1658. charm
  1659. if [ -e "$cache_file" ]; then
  1660. #debug "$FUNCNAME: SESSION cache hit $1"
  1661. cat "$cache_file"
  1662. return 0
  1663. fi
  1664. charm="$(get_service_charm "$service")" || return 1
  1665. charm.metadata "$charm" > "$cache_file"
  1666. if [ "$?" != 0 ]; then
  1667. rm -f "$cache_file" ## no cache
  1668. return 1
  1669. fi
  1670. cat "$cache_file"
  1671. }
  1672. export -f _get_service_metadata
  1673. _get_service_uses() {
  1674. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  1675. metadata
  1676. if [ -e "$cache_file" ]; then
  1677. #debug "$FUNCNAME: SESSION cache hit $1"
  1678. cat "$cache_file"
  1679. return 0
  1680. fi
  1681. metadata="$(_get_service_metadata "$service")" || return 1
  1682. _get_charm_metadata_uses "$metadata" > "$cache_file"
  1683. if [ "$?" != 0 ]; then
  1684. rm -f "$cache_file" ## no cache
  1685. return 1
  1686. fi
  1687. cat "$cache_file"
  1688. }
  1689. export -f _get_service_uses
  1690. _get_services_uses() {
  1691. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  1692. service rn rd
  1693. if [ -e "$cache_file" ]; then
  1694. #debug "$FUNCNAME: SESSION cache hit $1"
  1695. cat "$cache_file"
  1696. return 0
  1697. fi
  1698. for service in "$@"; do
  1699. _get_service_uses "$service" | while read-0 rn rd; do
  1700. printf "%s\0" "$service" "$rn" "$rd"
  1701. done
  1702. [ "${PIPESTATUS[0]}" == 0 ] || {
  1703. return 1
  1704. }
  1705. done > "${cache_file}.wip"
  1706. mv "${cache_file}"{.wip,} &&
  1707. cat "$cache_file" || return 1
  1708. }
  1709. export -f _get_services_uses
  1710. _get_provides_provides() {
  1711. local provides="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  1712. service rn rd
  1713. if [ -e "$cache_file" ]; then
  1714. # debug "$FUNCNAME: CACHEDIR cache hit $1"
  1715. cat "$cache_file"
  1716. return 0
  1717. fi
  1718. type=$(printf "%s" "$provides" | shyaml get-type)
  1719. case "$type" in
  1720. sequence)
  1721. while read-0 prov; do
  1722. printf "%s\0" "$prov" ""
  1723. done < <(echo "$provides" | shyaml get-values-0)
  1724. ;;
  1725. struct)
  1726. printf "%s" "$provides" | shyaml key-values-0
  1727. ;;
  1728. str)
  1729. printf "%s\0" "$(echo "$provides" | shyaml get-value)" ""
  1730. ;;
  1731. *)
  1732. err "Unexpected type '$type' for provider identifier in charm '$charm'."
  1733. return 1
  1734. esac | tee "$cache_file"
  1735. return "${PIPESTATUS[0]}"
  1736. }
  1737. _get_metadata_provides() {
  1738. local metadata="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  1739. service rn rd
  1740. if [ -e "$cache_file" ]; then
  1741. #debug "$FUNCNAME: CACHEDIR cache hit"
  1742. cat "$cache_file"
  1743. return 0
  1744. fi
  1745. provides=$(printf "%s" "$metadata" | shyaml get-value -y -q provides "")
  1746. [ "$provides" -a "$provides" != "''" ] || { touch "$cache_file"; return 0; }
  1747. _get_provides_provides "$provides" | tee "$cache_file"
  1748. return "${PIPESTATUS[0]}"
  1749. }
  1750. _get_services_provides() {
  1751. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  1752. service rn rd
  1753. if [ -e "$cache_file" ]; then
  1754. #debug "$FUNCNAME: SESSION cache hit $1"
  1755. cat "$cache_file"
  1756. return 0
  1757. fi
  1758. ## YYY: replace the inner loop by a cached function
  1759. for service in "$@"; do
  1760. metadata="$(_get_service_metadata "$service")" || return 1
  1761. while read-0 rn rd; do
  1762. printf "%s\0" "$service" "$rn" "$rd"
  1763. done < <(_get_metadata_provides "$metadata")
  1764. done > "$cache_file"
  1765. if [ "$?" != 0 ]; then
  1766. rm -f "$cache_file" ## no cache
  1767. return 1
  1768. fi
  1769. cat "$cache_file"
  1770. }
  1771. export -f _get_services_provides
  1772. _get_charm_provides() {
  1773. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(charm.store_metadata_hash)"
  1774. if [ -e "$cache_file" ]; then
  1775. #debug "$FUNCNAME: SESSION cache hit"
  1776. cat "$cache_file"
  1777. return 0
  1778. fi
  1779. start="$SECONDS"
  1780. debug "Getting charm provider list..."
  1781. while read-0 charm _ realpath metadata; do
  1782. metadata="$(charm.metadata "$charm")" || continue
  1783. # echo "reading $charm" >&2
  1784. while read-0 rn rd; do
  1785. printf "%s\0" "$charm" "$rn" "$rd"
  1786. done < <(_get_metadata_provides "$metadata")
  1787. done < <(charm.ls) | tee "$cache_file"
  1788. errlvl="${PIPESTATUS[0]}"
  1789. debug " ..charm provider list done $GRAY(in $((SECONDS - start))s)$NORMAL"
  1790. return "$errlvl"
  1791. }
  1792. _get_charm_providing() {
  1793. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  1794. relation="$1"
  1795. if [ -e "$cache_file" ]; then
  1796. #debug "$FUNCNAME: SESSION cache hit $1"
  1797. cat "$cache_file"
  1798. return 0
  1799. fi
  1800. while read-0 charm relation_name relation_def; do
  1801. [ "$relation_name" == "$relation" ] || continue
  1802. printf "%s\0" "$charm" "$relation_def"
  1803. done < <(_get_charm_provides) > "$cache_file"
  1804. if [ "$?" != 0 ]; then
  1805. rm -f "$cache_file" ## no cache
  1806. return 1
  1807. fi
  1808. cat "$cache_file"
  1809. }
  1810. _get_services_providing() {
  1811. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  1812. relation="$1"
  1813. shift ## services is "$@"
  1814. if [ -e "$cache_file" ]; then
  1815. #debug "$FUNCNAME: SESSION cache hit $1"
  1816. cat "$cache_file"
  1817. return 0
  1818. fi
  1819. while read-0 service relation_name relation_def; do
  1820. [ "$relation_name" == "$relation" ] || continue
  1821. printf "%s\0" "$service" "$relation_def"
  1822. done < <(_get_services_provides "$@") > "$cache_file"
  1823. if [ "$?" != 0 ]; then
  1824. rm -f "$cache_file" ## no cache
  1825. return 1
  1826. fi
  1827. cat "$cache_file"
  1828. }
  1829. export -f _get_services_provides
  1830. _out_new_relation_from_defs() {
  1831. local service="$1" rn="$2" ts="$3" prov_def="$4" rel_def="$5" rc td rc_prov
  1832. rc_prov=$(printf "%s" "$prov_def" | shyaml -y get-value "default-options" 2>/dev/null)
  1833. ## YYYvlab: should be seen even in no debug mode no ?
  1834. rc=$(printf "%s" "$rel_def" | shyaml -y get-value "default-options" 2>/dev/null)
  1835. td=$(echo "$prov_def" | shyaml get-value 'tech-dep' 2>/dev/null)
  1836. td=${td:-True}
  1837. rc=$(merge_yaml_str "$rc_prov" "$rc") || return 1
  1838. printf "%s\0" "$service" "$relation_name" "$ts" "$rc" "$td"
  1839. }
  1840. get_all_relations () {
  1841. local cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)" \
  1842. services
  1843. if [ -e "${cache_file}" ]; then
  1844. #debug "$FUNCNAME: SESSION cache hit $1"
  1845. cat "${cache_file}"
  1846. return 0
  1847. fi
  1848. declare -A services
  1849. services_uses=()
  1850. ## XXXvlab: bwerk, leveraging cache to be able to get the errorlevel here.
  1851. _get_services_uses "$@" || return 1
  1852. array_read-0 services_uses < <(_get_services_uses "$@")
  1853. services_provides=()
  1854. ## XXXvlab: bwerk, leveraging cache to be able to get the errorlevel here.
  1855. _get_services_provides "$@" || return 1
  1856. array_read-0 services_provides < <(_get_services_provides "$@")
  1857. for service in "$@"; do
  1858. services[$service]=1
  1859. done
  1860. all_services=("$@")
  1861. while [ "${#all_services[@]}" != 0 ]; do
  1862. array_pop all_services service
  1863. while read-0 relation_name ts relation_config tech_dep; do
  1864. printf "%s\0" "$service" "$relation_name" "$ts" "$relation_config" "$tech_dep"
  1865. ## adding target services ?
  1866. [ "${services[$ts]}" ] && continue
  1867. array_read-0 services_uses < <(_get_services_uses "$ts")
  1868. all_services+=("$ts")
  1869. services[$ts]=1
  1870. done < <(get_compose_relations "$service")
  1871. done > "${cache_file}.wip"
  1872. while true; do
  1873. changed=
  1874. new_services_uses=()
  1875. summon=()
  1876. required=()
  1877. recommended=()
  1878. optional=()
  1879. while [ "${#services_uses[@]}" != 0 ]; do
  1880. service="${services_uses[0]}"
  1881. relation_name="${services_uses[1]}"
  1882. relation_def="${services_uses[2]}"
  1883. services_uses=("${services_uses[@]:3}")
  1884. default_options=$(printf "%s" "$relation_def" | shyaml -y get-value "default-options" 2>/dev/null)
  1885. ## is this "use" declaration satisfied ?
  1886. found=
  1887. while read-0 s rn ts rc td; do
  1888. if [ -z "$found" -a "$service" == "$s" -a "$relation_name" == "$rn" ]; then
  1889. if [ "$default_options" ]; then
  1890. rc=$(merge_yaml_str "$default_options" "$rc") || return 1
  1891. fi
  1892. found="$ts"
  1893. fi
  1894. printf "%s\0" "$s" "$rn" "$ts" "$rc" "$td"
  1895. done < "${cache_file}.wip" > "${cache_file}.wip.new"
  1896. mv "${cache_file}.wip.new" "${cache_file}.wip"
  1897. if [ "$found" ]; then ## this "use" declaration was satisfied
  1898. debug "${DARKYELLOW}$service${NORMAL} use declaration for relation " \
  1899. "${DARKBLUE}$relation_name${NORMAL} is satisfied with ${DARKYELLOW}$found${NORMAL}"
  1900. continue
  1901. fi
  1902. auto=$(echo "$relation_def" | shyaml get-value auto pair 2>/dev/null)
  1903. case "$auto" in
  1904. "pair")
  1905. service_list=()
  1906. array_read-0 service_list < <(array_keys_to_stdin services)
  1907. providers=()
  1908. array_read-0 providers providers_def < <(_get_services_providing "$relation_name" "${service_list[@]}")
  1909. if [ "${#providers[@]}" == 1 ]; then
  1910. ts="${providers[0]}"
  1911. debug "Auto-pairs ${DARKYELLOW}$service${NORMAL}" \
  1912. "--${DARKBLUE}$relation_name${NORMAL}--> ${DARKYELLOW}$ts${NORMAL}"
  1913. _out_new_relation_from_defs "$service" "$relation_name" "$ts" \
  1914. "${providers_def[0]}" "$relation_def" \
  1915. >> "${cache_file}.wip"
  1916. ## Adding service
  1917. [ "${services[$ts]}" ] && continue
  1918. array_read-0 new_services_uses < <(_get_services_uses "$ts")
  1919. services[$ts]=1
  1920. changed=1
  1921. continue
  1922. elif [ "${#providers[@]}" -gt 1 ]; then
  1923. msg=""
  1924. warn "No auto-pairing ${DARKYELLOW}$service${NORMAL}" \
  1925. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  1926. "(> 1 provider)."
  1927. continue
  1928. else
  1929. : ## Do nothing
  1930. fi
  1931. ;;
  1932. "summon")
  1933. summon+=("$service" "$relation_name" "$relation_def")
  1934. ;;
  1935. ""|null|disable|disabled)
  1936. :
  1937. ;;
  1938. *)
  1939. err "Invalid ${WHITE}auto${NORMAL} value '$auto'."
  1940. return 1
  1941. ;;
  1942. esac
  1943. constraint=$(echo "$relation_def" | shyaml get-value constraint auto-pair 2>/dev/null)
  1944. case "$constraint" in
  1945. "required")
  1946. required+=("$service" "$relation_name" "$relation_def")
  1947. ;;
  1948. "recommended")
  1949. recommended+=("$service" "$relation_name" "$relation_def")
  1950. ;;
  1951. "optional")
  1952. optional+=("$service" "$relation_name" "$relation_def")
  1953. ;;
  1954. *)
  1955. err "Invalid ${WHITE}constraint${NORMAL} value '$contraint'."
  1956. return 1
  1957. ;;
  1958. esac
  1959. new_services_uses+=("$service" "$relation_name" "$relation_def") ## re-queue it
  1960. done
  1961. services_uses=("${new_services_uses[@]}")
  1962. if [ "$changed" ]; then
  1963. continue
  1964. fi
  1965. ## situation is stable
  1966. if [ "${#summon[@]}" != 0 ]; then
  1967. while [ "${#summon[@]}" != 0 ]; do
  1968. service="${summon[0]}"
  1969. relation_name="${summon[1]}"
  1970. relation_def="${summon[2]}"
  1971. summon=("${summon[@]:3}")
  1972. providers=()
  1973. providers_def=()
  1974. array_read-0 providers providers_def < <(_get_charm_providing "$relation_name" "${service_list[@]}")
  1975. if [ "${#providers[@]}" == 0 ]; then
  1976. die "Summoning a ${DARKBLUE}$relation_name${NORMAL} provider failed: none were found in charm store."
  1977. fi
  1978. if [ "${#providers[@]}" -gt 1 ]; then
  1979. warn "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  1980. "--${DARKBLUE}$relation_name${NORMAL}--> ($DARKYELLOW""${providers[@]}""$NORMAL)"\
  1981. "(> 1 provider). Choosing first."
  1982. fi
  1983. ts="${providers[0]}"
  1984. ## YYYvlab: should be seen even in no debug mode no ?
  1985. debug "Auto-summon ${DARKYELLOW}$service${NORMAL}" \
  1986. "--${DARKBLUE}$relation_name${NORMAL}--> ${DARKYELLOW}$ts${NORMAL}"
  1987. _out_new_relation_from_defs "$service" "$relation_name" "$ts" \
  1988. "${providers_def[0]}" "$relation_def" \
  1989. >> "${cache_file}.wip"
  1990. ## Adding service
  1991. [ "${services[$ts]}" ] && continue
  1992. array_read-0 new_services_uses < <(_get_services_uses "$ts")
  1993. services[$ts]=1
  1994. changed=1
  1995. done
  1996. continue
  1997. fi
  1998. [ "$NO_CONSTRAINT_CHECK" ] && break
  1999. if [ "${#required[@]}" != 0 ]; then
  2000. echo "$(_display_solves required)" | sed -r "s/^/${RED}||${NORMAL} /g" >&2
  2001. err "Required relations not satisfied"
  2002. return 1
  2003. fi
  2004. if [ "${#recommended[@]}" != 0 ]; then
  2005. ## make recommendation
  2006. echo "$(_display_solves recommended)" | sed -r "s/^/${YELLOW}||${NORMAL} /g" >&2
  2007. fi
  2008. if [ "${#optional[@]}" != 0 ]; then
  2009. ## inform about options
  2010. echo "$(_display_solves optional)" | sed -r "s/^/${BLUE}||${NORMAL} /g" >&2
  2011. fi
  2012. # if [ "${#required[@]}" != 0 ]; then
  2013. # err "Required relations not satisfied"
  2014. # return 1
  2015. # fi
  2016. if [ "${#recommended[@]}" != 0 ]; then
  2017. warn "Recommended relations not satisfied"
  2018. fi
  2019. break
  2020. done
  2021. if [ "$?" != 0 ]; then
  2022. rm -f "${cache_file}"{,.wip,.wip.new} ## no cache
  2023. return 1
  2024. fi
  2025. export ALL_RELATIONS="$cache_file"
  2026. mv "${cache_file}"{.wip,}
  2027. cat "$cache_file"
  2028. }
  2029. export -f get_all_relations
  2030. _display_solves() {
  2031. local array_name="$1" by_relation msg
  2032. ## inform about options
  2033. msg=""
  2034. declare -A by_relation
  2035. while read-0 service relation_name relation_def; do
  2036. solves=$(printf "%s" "$relation_def" | shyaml -y get-value solves 2>/dev/null);
  2037. auto=$(printf "%s" "$relation_def" | shyaml get-value auto 2>/dev/null);
  2038. if [ -z "$solves" ]; then
  2039. continue
  2040. fi
  2041. by_relation[$relation_name]+=$(printf "\n %s" "${DARKYELLOW}$service$NORMAL for:")
  2042. if [ "$auto" == "pair" ]; then
  2043. requirement="add provider in cluster to auto-pair"
  2044. else
  2045. requirement="add explicit relation"
  2046. fi
  2047. while read-0 name def; do
  2048. by_relation[$relation_name]+=$(printf "\n - ${DARKCYAN}%-15s${NORMAL} %s (%s)" "$name" "$def" "$requirement")
  2049. done < <(printf "%s" "$solves" | shyaml key-values-0)
  2050. done < <(array_values_to_stdin "$array_name")
  2051. while read-0 relation_name message; do
  2052. msg+="$(printf "\n${DARKBLUE}%s$NORMAL provider is $array_name by%s" \
  2053. "$relation_name" "$message" )"
  2054. done < <(array_kv_to_stdin by_relation)
  2055. if [ "$msg" ]; then
  2056. printf "%s\n" "${msg:1}"
  2057. fi
  2058. }
  2059. get_compose_relation_def() {
  2060. local service="$1" relation="$2" relation_name target_service relation_config tech_dep
  2061. while read-0 relation_name target_service relation_config tech_dep; do
  2062. [ "$relation_name" == "$relation" ] || continue
  2063. printf "%s\0%s\0%s\0" "$target_service" "$relation_config" "$tech_dep"
  2064. done < <(get_compose_relations "$service") || return 1
  2065. }
  2066. export -f get_compose_relation_def
  2067. run_service_relations () {
  2068. local service services loaded subservices subservice
  2069. PROJECT_NAME=$(get_default_project_name) || return 1
  2070. export PROJECT_NAME
  2071. declare -A loaded
  2072. subservices=$(get_ordered_service_dependencies "$@") || return 1
  2073. for service in $subservices; do
  2074. # debug "Upping dep's relations of ${DARKYELLOW}$service${NORMAL}:"
  2075. for subservice in $(get_service_deps "$service") "$service"; do
  2076. [ "${loaded[$subservice]}" ] && continue
  2077. export BASE_SERVICE_NAME=$service
  2078. MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
  2079. MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1
  2080. RELATION_BASE_COMPOSE_DEF=$(get_compose_service_def "$subservice") || return 1
  2081. export RELATION_BASE_COMPOSE_DEF MASTER_BASE_{CHARM,SERVICE}_NAME
  2082. # debug " Relations of ${DARKYELLOW}$subservice${NORMAL}:"
  2083. while read-0 relation_name target_service relation_config tech_dep; do
  2084. export relation_config
  2085. export TARGET_SERVICE_NAME=$target_service
  2086. MASTER_TARGET_SERVICE_NAME=$(get_top_master_service_for_service "$target_service") || return 1
  2087. MASTER_TARGET_CHARM_NAME=$(get_service_charm "$MASTER_TARGET_SERVICE_NAME") || return 1
  2088. RELATION_TARGET_COMPOSE_DEF=$(get_compose_service_def "$target_service") || return 1
  2089. export RELATION_TARGET_COMPOSE_DEF MASTER_TARGET_{CHARM,SERVICE}_NAME
  2090. Wrap -d "Building $DARKYELLOW$subservice$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW$target_service$NORMAL" <<EOF || return 1
  2091. _run_service_relation "$relation_name" "$subservice" "$target_service" "\$relation_config"
  2092. EOF
  2093. done < <(get_service_relations "$subservice") || return 1
  2094. loaded[$subservice]=1
  2095. done
  2096. done
  2097. }
  2098. export -f run_service_relations
  2099. _run_service_action_direct() {
  2100. local service="$1" action="$2" charm _dummy
  2101. shift; shift
  2102. read-0 charm || true ## against 'set -e' that could be setup in parent scripts
  2103. if read-0 _dummy || [ "$_dummy" ]; then
  2104. print_syntax_error "$FUNCNAME: too many arguments in action descriptor"
  2105. return 1
  2106. fi
  2107. export action_errlvl_file="$state_tmpdir/action-$service-$charm-$action-errlvl"
  2108. export state_tmpdir
  2109. {
  2110. (
  2111. set +e ## Prevents unwanted leaks from parent shell
  2112. export COMPOSE_CONFIG=$(get_compose_yml_content)
  2113. export METADATA_CONFIG=$(charm.metadata "$charm")
  2114. export SERVICE_NAME=$service
  2115. export ACTION_NAME=$action
  2116. export CONTAINER_NAME=$(get_top_master_service_for_service "$service")
  2117. export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME")
  2118. export SERVICE_DATASTORE="$DATASTORE/$service"
  2119. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  2120. exname="$exname $ACTION_NAME $SERVICE_NAME" \
  2121. stdbuf -oL -eL bash -c 'charm.run_direct_action "$@"' -- "$charm" "$action" "$@"
  2122. echo "$?" > "$action_errlvl_file"
  2123. ) | logstdout "$DARKYELLOW$service$NORMAL/${DARKCYAN}$action${NORMAL} ${GREEN}@${NORMAL}"
  2124. } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$service$NORMAL/${DARKCYAN}$action${NORMAL} ${RED}@$NORMAL" 3>&1 1>&2 2>&3
  2125. if ! [ -e "$action_errlvl_file" ]; then
  2126. err "Action $DARKYELLOW$service$NORMAL:$DARKCYAN$action$NORMAL has failed without having time" \
  2127. "to output an errlvl"
  2128. return 1
  2129. fi
  2130. return "$(cat "$action_errlvl_file")"
  2131. }
  2132. export -f _run_service_action_direct
  2133. _run_service_action_relation() {
  2134. local service="$1" action="$2" charm target_charm relation_name relation_config _dummy
  2135. shift; shift
  2136. read-0 charm target_service target_charm relation_name relation_config || true
  2137. if read-0 _dummy || [ "$_dummy" ]; then
  2138. print_syntax_error "$FUNCNAME: too many arguments in action descriptor"
  2139. return 1
  2140. fi
  2141. export RELATION_DATA_FILE=$(get_relation_data_file "$service" "$target_service" "$relation_name" "$relation_config")
  2142. export action_errlvl_file="$state_tmpdir/action-$service-$charm-$action-errlvl"
  2143. export state_tmpdir
  2144. {
  2145. (
  2146. set +e ## Prevents unwanted leaks from parent shell
  2147. export METADATA_CONFIG=$(charm.metadata "$charm")
  2148. export SERVICE_NAME=$service
  2149. export RELATION_TARGET_SERVICE="$target_service"
  2150. export RELATION_TARGET_CHARM="$target_charm"
  2151. export RELATION_BASE_SERVICE="$service"
  2152. export RELATION_BASE_CHARM="$charm"
  2153. export ACTION_NAME=$action
  2154. export CONTAINER_NAME=$(get_top_master_service_for_service "$service")
  2155. export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME")
  2156. export SERVICE_DATASTORE="$DATASTORE/$service"
  2157. export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
  2158. exname="$exname $ACTION_NAME $SERVICE_NAME" \
  2159. stdbuf -oL -eL bash -c 'charm.run_relation_action "$@"' -- "$target_charm" "$relation_name" "$action" "$@"
  2160. echo "$?" > "$action_errlvl_file"
  2161. ) | logstdout "$DARKYELLOW$service$NORMAL/${DARKCYAN}$action${NORMAL} ${GREEN}@${NORMAL}"
  2162. } 3>&1 1>&2 2>&3 | logstderr "$DARKYELLOW$service$NORMAL/${DARKCYAN}$action${NORMAL} ${RED}@$NORMAL" 3>&1 1>&2 2>&3
  2163. if ! [ -e "$action_errlvl_file" ]; then
  2164. err "Action $DARKYELLOW$service$NORMAL:$DARKCYAN$action$NORMAL has failed without having time" \
  2165. "to output an errlvl"
  2166. return 1
  2167. fi
  2168. return "$(cat "$action_errlvl_file")"
  2169. }
  2170. export -f _run_service_action_relation
  2171. get_relation_data_dir() {
  2172. local service="$1" target_service="$2" relation_name="$3" \
  2173. cache_file="$state_tmpdir/$FUNCNAME.cache.$(printf "%s\0" "$@" | md5_compat)"
  2174. if [ -e "$cache_file" ]; then
  2175. # debug "$FUNCNAME: cache hit ($*)"
  2176. cat "$cache_file"
  2177. return 0
  2178. fi
  2179. project=$(get_default_project_name) || return 1
  2180. relation_dir="$VARDIR/relations/$project/${service}-${target_service}/$relation_name"
  2181. if ! [ -d "$relation_dir" ]; then
  2182. mkdir -p "$relation_dir" || return 1
  2183. chmod go-rwx "$relation_dir" || return 1 ## protecting this directory
  2184. fi
  2185. echo "$relation_dir" | tee "$cache_file"
  2186. }
  2187. export -f get_relation_data_dir
  2188. get_relation_data_file() {
  2189. local service="$1" target_service="$2" relation_name="$3" relation_config="$4"
  2190. relation_dir=$(get_relation_data_dir "$service" "$target_service" "$relation_name") || return 1
  2191. relation_data_file="$relation_dir/data"
  2192. new=
  2193. if [ -e "$relation_data_file" ]; then
  2194. ## Has reference changed ?
  2195. new_md5=$(echo "$relation_config" | md5_compat)
  2196. if [ "$new_md5" != "$(cat "$relation_data_file.md5_ref" 2>/dev/null)" ]; then
  2197. new=true
  2198. fi
  2199. else
  2200. new=true
  2201. fi
  2202. if [ "$new" ]; then
  2203. echo "$relation_config" > "$relation_data_file"
  2204. chmod go-rwx "$relation_data_file" ## protecting this file
  2205. echo "$relation_config" | md5_compat > "$relation_data_file.md5_ref"
  2206. fi
  2207. echo "$relation_data_file"
  2208. }
  2209. export -f get_relation_data_file
  2210. has_service_action () {
  2211. local service="$1" action="$2" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2212. charm target_charm relation_name target_service relation_config _tech_dep
  2213. if [ -e "$cache_file" ]; then
  2214. # debug "$FUNCNAME: cache hit ($*)"
  2215. cat "$cache_file"
  2216. return 0
  2217. fi
  2218. charm=$(get_service_charm "$service") || return 1
  2219. ## Action directly provided ?
  2220. if charm.has_direct_action "$charm" "$action" >/dev/null; then
  2221. echo -en "direct\0$charm" | tee "$cache_file"
  2222. return 0
  2223. fi
  2224. ## Action provided by relation ?
  2225. while read-0 relation_name target_service relation_config _tech_dep; do
  2226. target_charm=$(get_service_charm "$target_service") || return 1
  2227. if charm.has_relation_action "$target_charm" "$relation_name" "$action" >/dev/null; then
  2228. echo -en "relation\0$charm\0$target_service\0$target_charm\0$relation_name\0$relation_config" | tee "$cache_file"
  2229. return 0
  2230. fi
  2231. done < <(get_service_relations "$service")
  2232. return 1
  2233. # master=$(get_top_master_service_for_service "$service")
  2234. # [ "$master" == "$charm" ] && return 1
  2235. # has_service_action "$master" "$action"
  2236. }
  2237. export -f has_service_action
  2238. run_service_action () {
  2239. local service="$1" action="$2"
  2240. shift ; shift
  2241. {
  2242. if ! read-0 action_type; then
  2243. info "Service $DARKYELLOW$service$NORMAL does not have any action $DARKCYAN$action$NORMAL defined."
  2244. info " Add an executable script to 'actions/$action' to implement action."
  2245. return 1
  2246. fi
  2247. Section "running $DARKYELLOW$service$NORMAL/$DARKCYAN$action$NORMAL ($action_type)"; Feed
  2248. "_run_service_action_${action_type}" "$service" "$action" "$@"
  2249. } < <(has_service_action "$service" "$action")
  2250. }
  2251. export -f run_service_action
  2252. get_compose_relation_config() {
  2253. local service=$1 relation_config cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  2254. if [ -e "$cache_file" ]; then
  2255. # debug "$FUNCNAME: cache hit ($*)"
  2256. cat "$cache_file"
  2257. return 0
  2258. fi
  2259. compose_service_def=$(get_compose_service_def "$service") || return 1
  2260. echo "$compose_service_def" | shyaml get-value "relations" 2>/dev/null | tee "$cache_file"
  2261. }
  2262. export -f get_compose_relation_config
  2263. # ## Return key-values-0
  2264. # get_compose_relation_config_for_service() {
  2265. # local service=$1 relation_name=$2 relation_config
  2266. # compose_service_relations=$(get_compose_relation_config "$service") || return 1
  2267. # if ! relation_config=$(
  2268. # echo "$compose_service_relations" |
  2269. # shyaml get-value "${relation_name}" 2>/dev/null); then
  2270. # err "Couldn't find $DARKYELLOW${service}$NORMAL/${WHITE}${relation_name}$NORMAL" \
  2271. # "relation config in compose configuration."
  2272. # return 1
  2273. # fi
  2274. # if [ -z "$relation_config" ]; then
  2275. # err "Relation ${WHITE}mysql-database$NORMAL is empty in compose configuration."
  2276. # return 1
  2277. # fi
  2278. # if ! echo "$relation_config" | shyaml key-values-0 2>/dev/null; then
  2279. # err "No key/values in ${DARKBLUE}mysql-database$NORMAL of compose config."
  2280. # return 1
  2281. # fi
  2282. # }
  2283. # export -f get_compose_relation_config_for_service
  2284. _get_container_relation() {
  2285. local metadata=$1 found relation_name relation_def
  2286. found=
  2287. while read-0 relation_name relation_def; do
  2288. [ "$(echo "$relation_def" | shyaml get-value "scope" 2>/dev/null)" == "container" ] && {
  2289. found="$relation_name"
  2290. break
  2291. }
  2292. done < <(_get_charm_metadata_uses "$metadata")
  2293. if [ -z "$found" ]; then
  2294. die "Charm $DARKPINK$charm$NORMAL is a subordinate but does not have any required relation declaration with" \
  2295. "${WHITE}scope${NORMAL} set to 'container'."
  2296. fi
  2297. printf "%s" "$found"
  2298. }
  2299. _get_master_service_for_service_cached () {
  2300. local service="$1" charm="$2" metadata="$3" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  2301. charm requires master_charm target_charm target_service service_def found
  2302. if [ -e "$cache_file" ]; then
  2303. # debug "$FUNCNAME: STATIC cache hit ($1)"
  2304. cat "$cache_file" &&
  2305. touch "$cache_file" || return 1
  2306. return 0
  2307. fi
  2308. if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" != "True" ]; then
  2309. ## just return service name
  2310. echo "$service" | tee "$cache_file"
  2311. return 0
  2312. fi
  2313. ## Action provided by relation ?
  2314. container_relation=$(_get_container_relation "$metadata")
  2315. read-0 target_service _ _ < <(get_service_relation "$service" "$container_relation")
  2316. if [ -z "$target_service" ]; then
  2317. err "Couldn't find ${WHITE}relations.${container_relation}${NORMAL} in" \
  2318. "${DARKYELLOW}$service$NORMAL compose definition."
  2319. err ${FUNCNAME[@]}
  2320. return 1
  2321. fi
  2322. echo "$target_service" | tee "$cache_file"
  2323. }
  2324. export -f _get_master_service_for_service_cached
  2325. get_master_service_for_service() {
  2326. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2327. charm metadata result
  2328. if [ -e "$cache_file" ]; then
  2329. # debug "$FUNCNAME: SESSION cache hit ($*)"
  2330. cat "$cache_file" || return 1
  2331. return 0
  2332. fi
  2333. charm=$(get_service_charm "$service") || return 1
  2334. metadata=$(charm.metadata "$charm" 2>/dev/null) || {
  2335. metadata=""
  2336. warn "No charm $DARKPINK$charm$NORMAL found."
  2337. }
  2338. result=$(_get_master_service_for_service_cached "$service" "$charm" "$metadata") || return 1
  2339. echo "$result" | tee "$cache_file" || return 1
  2340. }
  2341. export -f get_master_service_for_service
  2342. get_top_master_service_for_service() {
  2343. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
  2344. current_service
  2345. if [ -e "$cache_file" ]; then
  2346. # debug "$FUNCNAME: cache hit ($*)"
  2347. cat "$cache_file"
  2348. return 0
  2349. fi
  2350. current_service="$service"
  2351. while true; do
  2352. master_service=$(get_master_service_for_service "$current_service") || return 1
  2353. [ "$master_service" == "$current_service" ] && break
  2354. current_service="$master_service"
  2355. done
  2356. echo "$current_service" | tee "$cache_file"
  2357. return 0
  2358. }
  2359. export -f get_top_master_service_for_service
  2360. ##
  2361. ## The result is a mixin that is not always a complete valid
  2362. ## docker-compose entry (thinking of subordinates). The result
  2363. ## will be merge with master charms.
  2364. _get_docker_compose_mixin_from_metadata_cached() {
  2365. local service="$1" charm="$2" metadata="$3" \
  2366. has_build_dir="$4" \
  2367. cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  2368. metadata_file metadata volumes docker_compose subordinate image mixin mixins
  2369. if [ -e "$cache_file" ]; then
  2370. #debug "$FUNCNAME: STATIC cache hit $1"
  2371. cat "$cache_file" &&
  2372. touch "$cache_file" || return 1
  2373. return 0
  2374. fi
  2375. mixins=("$(echo -en "labels:\n- compose.charm=$charm")")
  2376. if [ "$metadata" ]; then
  2377. ## resources to volumes
  2378. volumes=$(
  2379. for resource_type in data config; do
  2380. while read-0 resource; do
  2381. eval "echo \" - \$${resource_type^^}STORE/\$service\$resource:\$resource:rw\""
  2382. done < <(echo "$metadata" | shyaml get-values-0 "${resource_type}-resources" 2>/dev/null)
  2383. done
  2384. while read-0 resource; do
  2385. if [[ "$resource" == /*:/*:* ]]; then
  2386. echo " - $resource"
  2387. elif [[ "$resource" == /*:/* ]]; then
  2388. echo " - $resource:rw"
  2389. elif [[ "$resource" == /*:* ]]; then
  2390. echo " - ${resource%%:*}:$resource"
  2391. elif [[ "$resource" =~ ^/[^:]+$ ]]; then
  2392. echo " - $resource:$resource:rw"
  2393. else
  2394. die "Invalid host-resource specified in 'metadata.yml'."
  2395. fi
  2396. done < <(printf "%s" "$metadata" | shyaml get-values-0 "host-resources" 2>/dev/null)
  2397. while read-0 resource; do
  2398. dest="$(charm.get_dir "$charm")/resources$resource"
  2399. if ! [ -e "$dest" ]; then
  2400. die "charm-resource: '$resource' does not exist (file: '$dest')."
  2401. fi
  2402. echo " - $dest:$resource:ro"
  2403. done < <(echo "$metadata" | shyaml get-values-0 "charm-resources" 2>/dev/null)
  2404. ) || return 1
  2405. if [ "$volumes" ]; then
  2406. mixins+=("volumes:"$'\n'"$volumes")
  2407. fi
  2408. type="$(printf "%s" "$metadata" | shyaml get-value type 2>/dev/null)" || true
  2409. if [ "$type" != "run-once" ]; then
  2410. mixins+=("restart: unless-stopped")
  2411. fi
  2412. docker_compose=$(printf "%s" "$metadata" | shyaml get-value -y "docker-compose" 2>/dev/null) || true
  2413. if [ "$docker_compose" ]; then
  2414. mixins+=("$docker_compose")
  2415. fi
  2416. if [ "$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)" == "True" ]; then
  2417. subordinate=true
  2418. fi
  2419. fi
  2420. image=$(echo "$metadata" | shyaml get-value "docker-image" 2>/dev/null)
  2421. [ "$image" == "None" ] && image=""
  2422. if [ "$image" ]; then
  2423. if [ "$subordinate" ]; then
  2424. err "Subordinate charm can not have a ${WHITE}docker-image${NORMAL} value."
  2425. return 1
  2426. fi
  2427. mixins+=("image: $image")
  2428. elif [ "$has_build_dir" ]; then
  2429. if [ "$subordinate" ]; then
  2430. err "Subordinate charm can not have a 'build' sub directory."
  2431. return 1
  2432. fi
  2433. mixins+=("build: $(charm.get_dir "$charm")/build")
  2434. fi
  2435. mixin=$(merge_yaml_str "${mixins[@]}") || {
  2436. err "Failed to merge mixins from ${DARKPINK}${charm}${NORMAL} metadata."
  2437. return 1
  2438. }
  2439. echo "$mixin" | tee "$cache_file"
  2440. }
  2441. export -f _get_docker_compose_mixin_from_metadata_cached
  2442. get_docker_compose_mixin_from_metadata() {
  2443. local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1"
  2444. if [ -e "$cache_file" ]; then
  2445. #debug "$FUNCNAME: SESSION cache hit ($*)"
  2446. cat "$cache_file"
  2447. return 0
  2448. fi
  2449. charm=$(get_service_charm "$service") || return 1
  2450. metadata="$(charm.metadata "$charm" 2>/dev/null)" || return 1
  2451. has_build_dir=
  2452. [ -d "$(charm.get_dir "$charm")/build" ] && has_build_dir=true
  2453. mixin=$(_get_docker_compose_mixin_from_metadata_cached "$service" "$charm" "$metadata" "$has_build_dir") || return 1
  2454. echo "$mixin" | tee "$cache_file"
  2455. }
  2456. export -f get_docker_compose_mixin_from_metadata
  2457. _save() {
  2458. local name="$1"
  2459. cat - | tee -a "$docker_compose_dir/.data/$name"
  2460. }
  2461. export -f _save
  2462. get_default_project_name() {
  2463. if [ "$DEFAULT_PROJECT_NAME" ]; then
  2464. echo "$DEFAULT_PROJECT_NAME"
  2465. return 0
  2466. fi
  2467. compose_yml_location="$(get_compose_yml_location)" || return 1
  2468. if [ "$compose_yml_location" ]; then
  2469. if normalized_path=$(readlink -f "$compose_yml_location"); then
  2470. name="$(basename "$(dirname "$normalized_path")")"
  2471. echo "${name%%-deploy}"
  2472. return 0
  2473. fi
  2474. fi
  2475. echo "orphan"
  2476. return 0
  2477. }
  2478. export -f get_default_project_name
  2479. get_running_compose_containers() {
  2480. ## XXXvlab: docker bug: there will be a final newline anyway
  2481. docker ps --filter label="compose.service" --format='{{.ID}}'
  2482. }
  2483. export -f get_running_compose_containers
  2484. get_volumes_for_container() {
  2485. local container="$1"
  2486. docker inspect \
  2487. --format '{{range $mount := .Mounts}}{{$mount.Source}}{{"\x00"}}{{$mount.Destination}}{{"\x00"}}{{end}}' \
  2488. "$container"
  2489. }
  2490. export -f get_volumes_for_container
  2491. is_volume_used() {
  2492. local volume="$1" container_id src dst
  2493. while read container_id; do
  2494. while read-0 src dst; do
  2495. [[ "$src" == "$volume"/* ]] && return 0
  2496. done < <(get_volumes_for_container "$container_id")
  2497. done < <(get_running_compose_containers)
  2498. return 1
  2499. }
  2500. export -f is_volume_used
  2501. clean_unused_docker_compose() {
  2502. for f in /var/lib/compose/docker-compose/*; do
  2503. [ -e "$f" ] || continue
  2504. is_volume_used "$f" && continue
  2505. debug "Cleaning unused docker-compose ${f##*/}"
  2506. rm -rf "$f" || return 1
  2507. done
  2508. }
  2509. export -f clean_unused_docker_compose
  2510. stdin_get_hash() {
  2511. local sha
  2512. sha=$(sha256sum) || return 1
  2513. sha=${sha:0:64}
  2514. echo "$sha"
  2515. }
  2516. export -f stdin_get_hash
  2517. file_get_hash() {
  2518. stdin_get_hash < "$1" || return 1
  2519. }
  2520. export -f file_get_hash
  2521. docker_compose_store() {
  2522. local file="$1" sha
  2523. sha=$(file_get_hash "$file") || return 1
  2524. project=$(get_default_project_name) || return 1
  2525. dst="/var/lib/compose/docker-compose/$sha/$project"
  2526. mkdir -p "$dst" || return 1
  2527. cat <<EOF > "$dst/.env" || return 1
  2528. DOCKER_COMPOSE_PATH=$dst
  2529. EOF
  2530. cp "$file" "$dst/docker-compose.yml" || return 1
  2531. mkdir -p "$dst/bin" || return 1
  2532. cat <<EOF > "$dst/bin/dc" || return 1
  2533. #!/bin/bash
  2534. $(declare -f read-0)
  2535. docker_run_opts=()
  2536. while read-0 opt; do
  2537. docker_run_opts+=("\$opt")
  2538. done < <(cat "$COMPOSE_LAUNCHER_OPTS")
  2539. docker_run_opts+=(
  2540. "-w" "$dst"
  2541. "--entrypoint" "/usr/local/bin/docker-compose"
  2542. )
  2543. [ -t 1 ] && {
  2544. docker_run_opts+=("-ti")
  2545. }
  2546. exec docker run --rm "\${docker_run_opts[@]}" "${COMPOSE_DOCKER_IMAGE:-docker.0k.io/compose}" "\$@"
  2547. EOF
  2548. chmod +x "$dst/bin/dc" || return 1
  2549. printf "%s" "$sha"
  2550. }
  2551. launch_docker_compose() {
  2552. local charm docker_compose_tmpdir docker_compose_dir
  2553. docker_compose_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX)
  2554. #debug "Creating temporary docker-compose directory in '$docker_compose_tmpdir'."
  2555. trap_add EXIT "rm -rf \"$docker_compose_tmpdir\""
  2556. ## docker-compose will name network from the parent dir name
  2557. project=$(get_default_project_name)
  2558. mkdir -p "$docker_compose_tmpdir/$project"
  2559. docker_compose_dir="$docker_compose_tmpdir/$project"
  2560. get_docker_compose $SERVICE_PACK > "$docker_compose_dir/docker-compose.yml" || return 1
  2561. if [ -e "$state_tmpdir/to-merge-in-docker-compose.yml" ]; then
  2562. # debug "Merging some config data in docker-compose.yml:"
  2563. # debug "$(cat $state_tmpdir/to-merge-in-docker-compose.yml)"
  2564. _config_merge "$docker_compose_dir/docker-compose.yml" "$state_tmpdir/to-merge-in-docker-compose.yml" || return 1
  2565. fi
  2566. if [ -z "$(echo $(cat "$docker_compose_dir/docker-compose.yml"))" ]; then
  2567. die "Generated 'docker-compose.yml' is unexpectedly empty."
  2568. fi
  2569. ## XXXvlab: could be more specific and only link the needed charms
  2570. ## XXXvlab: why do we need these links ? If this is for the build command, then it is not useful anymore.
  2571. # for charm in $(shyaml keys services < "$docker_compose_dir/docker-compose.yml"); do
  2572. # if charm.exists "$charm"; then
  2573. # ln -sf "$(charm.get_dir "$charm")" "$docker_compose_dir/$charm" || exit 1
  2574. # fi
  2575. # done
  2576. mkdir "$docker_compose_dir/.data"
  2577. if [ -z "$COMPOSE_DISABLE_DOCKER_COMPOSE_STORE" ]; then
  2578. sha=$(docker_compose_store "$docker_compose_dir/docker-compose.yml") || return 1
  2579. fi
  2580. {
  2581. {
  2582. if [ -z "$COMPOSE_DISABLE_DOCKER_COMPOSE_STORE" ]; then
  2583. cd "/var/lib/compose/docker-compose/$sha/$project"
  2584. else
  2585. cd "$docker_compose_dir"
  2586. fi
  2587. if [ -f ".env" ]; then
  2588. debug "${WHITE}.env$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:"
  2589. debug "$(cat ".env" | prefix " $GRAY|$NORMAL ")"
  2590. fi
  2591. debug "${WHITE}docker-compose.yml$NORMAL for $DARKYELLOW$SERVICE_PACK$NORMAL:"
  2592. debug "$(cat "docker-compose.yml" | prefix " $GRAY|$NORMAL ")"
  2593. debug "${WHITE}Launching$NORMAL: docker-compose $@"
  2594. if [ "$DRY_COMPOSE_RUN" ]; then
  2595. echo docker-compose "$@"
  2596. else
  2597. docker-compose "$@"
  2598. fi
  2599. echo "$?" > "$docker_compose_dir/.data/errlvl"
  2600. } | _save stdout
  2601. } 3>&1 1>&2 2>&3 | _save stderr 3>&1 1>&2 2>&3
  2602. if tail -n 1 "$docker_compose_dir/.data/stderr" | egrep "Service .+ failed to build: Error getting container [0-9a-f]+ from driver devicemapper: (open|Error mounting) /dev/mapper/docker-.*: no such file or directory$" >/dev/null 2>&1; then
  2603. err "Detected bug https://github.com/docker/docker/issues/4036 ... "
  2604. err "Please re-launch your command, or switch from 'devicemapper' driver to 'overlayfs' or 'aufs'."
  2605. fi
  2606. docker_compose_errlvl="$(cat "$docker_compose_dir/.data/errlvl" 2>/dev/null)"
  2607. if [ -z "$docker_compose_errlvl" ]; then
  2608. err "Something went wrong before you could gather docker-compose errorlevel."
  2609. return 1
  2610. fi
  2611. return "$docker_compose_errlvl"
  2612. }
  2613. export -f launch_docker_compose
  2614. get_compose_yml_location() {
  2615. if ! [ -z ${COMPOSE_YML_FILE+x} ]; then ## if set, even if empty
  2616. echo "$COMPOSE_YML_FILE"
  2617. return 0
  2618. fi
  2619. parent=$(while ! [ -f "./compose.yml" ]; do
  2620. [ "$PWD" == "/" ] && exit 0
  2621. cd ..
  2622. done; echo "$PWD"
  2623. )
  2624. if [ "$parent" ]; then
  2625. echo "$parent/compose.yml"
  2626. return 0
  2627. fi
  2628. ## XXXvlab: do we need this additional environment variable,
  2629. ## COMPOSE_YML_FILE is not sufficient ?
  2630. if [ "$DEFAULT_COMPOSE_FILE" ]; then
  2631. if ! [ -e "$DEFAULT_COMPOSE_FILE" ]; then
  2632. warn "No 'compose.yml' was found in current or parent dirs," \
  2633. "and \$DEFAULT_COMPOSE_FILE points to an unexistent file." \
  2634. "(${DEFAULT_COMPOSE_FILE})"
  2635. return 0
  2636. fi
  2637. echo "$DEFAULT_COMPOSE_FILE"
  2638. return 0
  2639. fi
  2640. warn "No 'compose.yml' was found in current or parent dirs, and no \$DEFAULT_COMPOSE_FILE was set."
  2641. return 0
  2642. }
  2643. export -f get_compose_yml_location
  2644. get_compose_yml_content() {
  2645. local cache_file="$state_tmpdir/$FUNCNAME.cache"
  2646. if [ -e "$cache_file" ]; then
  2647. cat "$cache_file" &&
  2648. touch "$cache_file" || return 1
  2649. return 0
  2650. fi
  2651. if [ -z "$COMPOSE_YML_FILE" ]; then
  2652. COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1
  2653. fi
  2654. if [ -e "$COMPOSE_YML_FILE" ]; then
  2655. debug "Found $WHITE$exname$NORMAL YAML file in '$COMPOSE_YML_FILE'."
  2656. COMPOSE_YML_CONTENT=$(cat "$COMPOSE_YML_FILE") || {
  2657. err "Could not read '$COMPOSE_YML_FILE'."
  2658. return 1
  2659. }
  2660. else
  2661. debug "No compose file found. Using an empty one."
  2662. COMPOSE_YML_CONTENT=""
  2663. fi
  2664. COMPOSE_YML_CONTENT=$(merge_yaml_str "$COMPOSE_YML_CONTENT" "${compose_contents[@]}") || return 1
  2665. output=$(echo "$COMPOSE_YML_CONTENT"| shyaml get-value 2>&1)
  2666. if [ "$?" != 0 ]; then
  2667. outputed_something=
  2668. while IFS='' read -r line1 && IFS='' read -r line2; do
  2669. [ "$outputed_something" ] || err "Invalid YAML in '$COMPOSE_YML_FILE':"
  2670. outputed_something=true
  2671. echo "$line1 $GRAY($line2)$NORMAL"
  2672. done < <(echo "$output" | grep ^yaml.scanner -A 100 |
  2673. sed -r 's/^ in "<stdin>", //g' | sed -r 's/^yaml.scanner.[a-zA-Z]+: //g') |
  2674. prefix " $GRAY|$NORMAL "
  2675. [ "$outputed_something" ] || {
  2676. err "Unexpected error while running 'shyaml get-value' on '$COMPOSE_YML_FILE':"
  2677. echo "$output" | prefix " $GRAY|$NORMAL "
  2678. }
  2679. return 1
  2680. fi
  2681. echo "$COMPOSE_YML_CONTENT" | tee "$cache_file" || return 1
  2682. }
  2683. export -f get_compose_yml_content
  2684. get_default_target_services() {
  2685. local services=("$@")
  2686. if [ -z "${services[*]}" ]; then
  2687. if [ "$DEFAULT_SERVICES" ]; then
  2688. debug "No service provided, using $WHITE\$DEFAULT_SERVICES$NORMAL variable." \
  2689. "Target services: $DARKYELLOW$DEFAULT_SERVICES$NORMAL"
  2690. services="$DEFAULT_SERVICES"
  2691. else
  2692. err "No service provided."
  2693. return 1
  2694. fi
  2695. fi
  2696. echo "${services[*]}"
  2697. }
  2698. export -f get_default_target_services
  2699. get_master_services() {
  2700. local loaded master_service service
  2701. declare -A loaded
  2702. for service in "$@"; do
  2703. master_service=$(get_top_master_service_for_service "$service") || return 1
  2704. if [ "${loaded[$master_service]}" ]; then
  2705. continue
  2706. fi
  2707. echo "$master_service"
  2708. loaded["$master_service"]=1
  2709. done | xargs printf "%s "
  2710. return "${PIPESTATUS[0]}"
  2711. }
  2712. export -f get_master_services
  2713. get_current_docker_container_id() {
  2714. local line
  2715. line=$(cat "/proc/self/cpuset") || return 1
  2716. [[ "$line" == *docker* ]] || return 1
  2717. echo "${line##*/}"
  2718. }
  2719. export -f get_current_docker_container_id
  2720. ## if we are in a docker compose, we might want to know what is the
  2721. ## real host path of some local paths.
  2722. get_host_path() {
  2723. local path="$1"
  2724. path=$(realpath "$path") || return 1
  2725. container_id=$(get_current_docker_container_id) || {
  2726. print "%s" "$path"
  2727. return 0
  2728. }
  2729. biggest_dst=
  2730. current_src=
  2731. while read-0 src dst; do
  2732. [[ "$path" == "$dst"* ]] || continue
  2733. if [[ "${#biggest_dst}" < "${#dst}" ]]; then
  2734. biggest_dst="$dst"
  2735. current_src="$src"
  2736. fi
  2737. done < <(get_volumes_for_container "$container_id")
  2738. if [ "$current_src" ]; then
  2739. printf "%s" "$current_src"
  2740. else
  2741. return 1
  2742. fi
  2743. }
  2744. export -f get_host_path
  2745. _setup_state_dir() {
  2746. export state_tmpdir=$(mktemp -d -t tmp.XXXXXXXXXX)
  2747. #debug "Creating temporary state directory in '$state_tmpdir'."
  2748. # trap_add EXIT "debug \"Removing temporary state directory in $state_tmpdir.\";\
  2749. # rm -rf \"$state_tmpdir\""
  2750. trap_add EXIT "rm -rf \"$state_tmpdir\""
  2751. }
  2752. get_docker_compose_help_msg() {
  2753. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \
  2754. docker_compose_help_msg
  2755. if [ -e "$cache_file" ]; then
  2756. cat "$cache_file" &&
  2757. touch "$cache_file" || return 1
  2758. return 0
  2759. fi
  2760. docker_compose_help_msg=$(docker-compose $action --help 2>/dev/null) || return 1
  2761. echo "$docker_compose_help_msg" |
  2762. tee "$cache_file" || return 1
  2763. }
  2764. get_docker_compose_usage() {
  2765. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \
  2766. docker_compose_help_msg
  2767. if [ -e "$cache_file" ]; then
  2768. cat "$cache_file" &&
  2769. touch "$cache_file" || return 1
  2770. return 0
  2771. fi
  2772. docker_compose_help_msg=$(get_docker_compose_help_msg $action) || return 1
  2773. echo "$docker_compose_help_msg" |
  2774. grep -m 1 "^Usage:" -A 10000 |
  2775. egrep -m 1 "^\$" -B 10000 |
  2776. xargs printf "%s " |
  2777. sed -r 's/^Usage: //g' |
  2778. tee "$cache_file" || return 1
  2779. }
  2780. get_docker_compose_opts_help() {
  2781. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \
  2782. docker_compose_help_msg
  2783. if [ -e "$cache_file" ]; then
  2784. cat "$cache_file" &&
  2785. touch "$cache_file" || return 1
  2786. return 0
  2787. fi
  2788. docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1
  2789. echo "$docker_compose_opts_help" |
  2790. grep '^Options:' -A 20000 |
  2791. tail -n +2 |
  2792. { cat ; echo; } |
  2793. egrep -m 1 "^\S*\$" -B 10000 |
  2794. head -n -1 |
  2795. tee "$cache_file" || return 1
  2796. }
  2797. get_docker_compose_commands_help() {
  2798. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \
  2799. docker_compose_help_msg
  2800. if [ -e "$cache_file" ]; then
  2801. cat "$cache_file" &&
  2802. touch "$cache_file" || return 1
  2803. return 0
  2804. fi
  2805. docker_compose_opts_help=$(get_docker_compose_help_msg $action) || return 1
  2806. echo "$docker_compose_opts_help" |
  2807. grep '^Commands:' -A 20000 |
  2808. tail -n +2 |
  2809. { cat ; echo; } |
  2810. egrep -m 1 "^\S*\$" -B 10000 |
  2811. head -n -1 |
  2812. tee "$cache_file" || return 1
  2813. }
  2814. get_docker_compose_opts_list() {
  2815. local action="$1" cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$1"; cat "$(which docker-compose)" | md5_compat)" \
  2816. docker_compose_help_msg
  2817. if [ -e "$cache_file" ]; then
  2818. cat "$cache_file" &&
  2819. touch "$cache_file" || return 1
  2820. return 0
  2821. fi
  2822. docker_compose_opts_help=$(get_docker_compose_opts_help $action) || return 1
  2823. echo "$docker_compose_opts_help" |
  2824. egrep "^\s+-" |
  2825. sed -r 's/\s+((((-[a-zA-Z]|--[a-zA-Z0-9-]+)( [A-Z=]+|=[^ ]+)?)(, )?)+)\s+.*$/\1/g' |
  2826. tee "$cache_file" || return 1
  2827. }
  2828. options_parser() {
  2829. sed -r 's/^(\s+(((-[a-zA-Z]|--[a-zA-Z0-9-]+)([ =]([a-zA-Z_=\"\[]|\])+)?(, | )?)+)\s+)[^ ].*$/\x0\2\x0\0/g'
  2830. printf "\0"
  2831. }
  2832. remove_options_in_option_help_msg() {
  2833. {
  2834. read-0 null
  2835. if [ "$null" ]; then
  2836. err "options parsing error, should start with an option line."
  2837. return 1
  2838. fi
  2839. while read-0 opt full_txt;do
  2840. multi_opts="$(printf "%s " $opt | multi_opts_filter)"
  2841. single_opts="$(printf "%s " $opt | single_opts_filter)"
  2842. for to_remove in "$@"; do
  2843. str_matches "$to_remove" $multi_opts $single_opts && {
  2844. continue 2
  2845. }
  2846. done
  2847. echo -n "$full_txt"
  2848. done
  2849. } < <(options_parser)
  2850. }
  2851. _MULTIOPTION_REGEX='^((-[a-zA-Z]|--[a-zA-Z0-9-]+)(, )?)+'
  2852. _MULTIOPTION_REGEX_LINE_FILTER=$_MULTIOPTION_REGEX'(\s|=)'
  2853. multi_opts_filter() {
  2854. egrep "$_MULTIOPTION_REGEX_LINE_FILTER" |
  2855. sed -r "s/^($_MULTIOPTION_REGEX)(\s|=).*$/\1/g" |
  2856. tr ',' "\n" | xargs printf "%s "
  2857. }
  2858. single_opts_filter() {
  2859. egrep -v "$_MULTIOPTION_REGEX_LINE_FILTER" |
  2860. tr ',' "\n" | xargs printf "%s "
  2861. }
  2862. get_docker_compose_multi_opts_list() {
  2863. local action="$1" opts_list
  2864. opts_list=$(get_docker_compose_opts_list "$action") || return 1
  2865. echo "$opts_list" | multi_opts_filter
  2866. }
  2867. get_docker_compose_single_opts_list() {
  2868. local action="$1" opts_list
  2869. opts_list=$(get_docker_compose_opts_list "$action") || return 1
  2870. echo "$opts_list" | single_opts_filter
  2871. }
  2872. display_commands_help() {
  2873. local charm_actions
  2874. echo
  2875. echo "${WHITE}Commands${NORMAL} (thanks to docker-compose):"
  2876. get_docker_compose_commands_help | sed -r "s/ ([a-z]+)(\s+)/ ${DARKCYAN}\1${NORMAL}\2/g"
  2877. charm_actions_help=$(get_docker_charm_action_help) || return 1
  2878. if [ "$charm_actions_help" ]; then
  2879. echo
  2880. echo "${WHITE}Charm actions${NORMAL}:"
  2881. printf "%s\n" "$charm_actions_help" | \
  2882. sed -r "s/^ ([a-z0-9-]+)(\s+)([a-z0-9-]+)(\s+)/ ${DARKCYAN}\1${NORMAL}\2${DARKYELLOW}\3${NORMAL}\4/g"
  2883. fi
  2884. }
  2885. get_docker_charm_action_help() {
  2886. local services service charm relation_name target_service relation_config \
  2887. target_charm
  2888. services=($(get_compose_yml_content | shyaml keys 2>/dev/null))
  2889. for service in "${services[@]}"; do
  2890. out=$(
  2891. charm=$(get_service_charm "$service") || return 1
  2892. for action in $(charm.ls_direct_actions "$charm"); do
  2893. printf " %-28s %s\n" "$action $service" "Direct action from ${DARKPINK}$charm${NORMAL}"
  2894. done
  2895. while read-0 relation_name target_service _relation_config _tech_dep; do
  2896. target_charm=$(get_service_charm "$target_service") || return 1
  2897. for action in $(charm.ls_relation_actions "$target_charm" "$relation_name"); do
  2898. printf " %-28s %s\n" "$action $service" "Indirect action from ${DARKPINK}$target_charm${NORMAL}"
  2899. done
  2900. done < <(get_compose_relations "$service")
  2901. )
  2902. if [ "$out" ]; then
  2903. echo " for ${DARKYELLOW}$service${NORMAL}:"
  2904. printf "%s\n" "$out"
  2905. fi
  2906. done
  2907. }
  2908. display_help() {
  2909. print_help
  2910. echo "${WHITE}Options${NORMAL}:"
  2911. echo " -h, --help Print this message and quit"
  2912. echo " (ignoring any other options)"
  2913. echo " -V, --version Print current version and quit"
  2914. echo " (ignoring any other options)"
  2915. echo " --dirs Display data dirs and quit"
  2916. echo " (ignoring any other options)"
  2917. echo " -v, --verbose Be more verbose"
  2918. echo " -d, --debug Print full debugging information (sets also verbose)"
  2919. echo " --dry-compose-run If docker-compose will be run, only print out what"
  2920. echo " command line will be used."
  2921. echo " --rebuild-relations-to-service, -R SERVICE"
  2922. echo " Will rebuild all relations to given service"
  2923. echo " --add-compose-content, -Y YAML"
  2924. echo " Will merge some direct YAML with the current compose"
  2925. get_docker_compose_opts_help | remove_options_in_option_help_msg --version --help --verbose |
  2926. filter_docker_compose_help_message
  2927. display_commands_help
  2928. }
  2929. _graph_service() {
  2930. local service="$1" base="$1"
  2931. charm=$(get_service_charm "$service") || return 1
  2932. metadata=$(charm.metadata "$charm") || return 1
  2933. subordinate=$(echo "$metadata" | shyaml get-value "subordinate" 2>/dev/null)
  2934. if [ "$subordinate" == "True" ]; then
  2935. requires="$(echo "$metadata" | shyaml get-value "requires" 2>/dev/null)"
  2936. master_charm=
  2937. while read-0 relation_name relation; do
  2938. [ "$(echo "$relation" | shyaml get-value "scope" 2>/dev/null)" == "container" ] || continue
  2939. interface="$(echo "$relation" | shyaml get-value "interface" 2>/dev/null)"
  2940. if [ -z "$interface" ]; then
  2941. err "No ${WHITE}$interface${NORMAL} set for relation $relation_name."
  2942. return 1
  2943. fi
  2944. ## Action provided by relation ?
  2945. target_service=
  2946. while read-0 relation_name candidate_target_service _relation_config _tech_dep; do
  2947. [ "$interface" == "$relation_name" ] && {
  2948. target_service="$candidate_target_service"
  2949. break
  2950. }
  2951. done < <(get_service_relations "$service")
  2952. if [ -z "$target_service" ]; then
  2953. err "Couldn't find ${WHITE}relations.$interface${NORMAL} in" \
  2954. "${DARKYELLOW}$service$NORMAL compose definition."
  2955. return 1
  2956. fi
  2957. master_service="$target_service"
  2958. master_charm=$(get_service_charm "$target_service") || return 1
  2959. break
  2960. done < <(echo "$requires" | shyaml key-values-0 2>/dev/null)
  2961. fi
  2962. _graph_node_service "$service" "$base" "$charm"
  2963. _graph_edge_service "$service" "$subordinate" "$master_service"
  2964. }
  2965. _graph_node_service() {
  2966. local service="$1" base="$2" charm="$3"
  2967. cat <<EOF
  2968. "$(_graph_node_service_label ${service})" [
  2969. style = "filled, $([ "$subordinate" == "True" ] && echo "dashed" || echo "bold")"
  2970. penwidth = $([ "$subordinate" == "True" ] && echo "3" || echo "5")
  2971. color = $([ "$base" ] && echo "blue" || echo "black")
  2972. fillcolor = "white"
  2973. fontname = "Courier New"
  2974. shape = "Mrecord"
  2975. label =<$(_graph_node_service_content "$service")>
  2976. ];
  2977. EOF
  2978. }
  2979. _graph_edge_service() {
  2980. local service="$1" subordinate="$2" master_service="$3"
  2981. while read-0 relation_name target_service relation_config tech_dep; do
  2982. cat <<EOF
  2983. "$(_graph_node_service_label ${service})" -> "$(_graph_node_service_label ${target_service})" [
  2984. penwidth = $([ "$master_service" == "$target_service" ] && echo 3 || echo 2)
  2985. fontsize = 16
  2986. fontcolor = "black"
  2987. style = $([ "$master_service" == "$target_service" ] && echo dashed || echo "\"\"")
  2988. weight = $([ "$master_service" == "$target_service" ] && echo 2.0 || echo 1.0)
  2989. dir = $([ "$master_service" == "$target_service" ] && echo none || echo both)
  2990. arrowtail = odot
  2991. # arrowhead = dotlicurve
  2992. taillabel = "$relation_name" ];
  2993. EOF
  2994. done < <(get_service_relations "$service") || return 1
  2995. }
  2996. _graph_node_service_label() {
  2997. local service="$1"
  2998. echo "service_$service"
  2999. }
  3000. _graph_node_service_content() {
  3001. local service="$1"
  3002. charm=$(get_service_charm "$service") || return 1
  3003. cat <<EOF
  3004. <table border="0" cellborder="0" cellpadding="3" bgcolor="white">
  3005. <tr>
  3006. <td bgcolor="black" align="center" colspan="2">
  3007. <font color="white">$service</font>
  3008. </td>
  3009. </tr>
  3010. $(if [ "$charm" != "$service" ]; then
  3011. cat <<EOF2
  3012. <tr>
  3013. <td align="left" port="r0">charm: $charm</td>
  3014. </tr>
  3015. EOF2
  3016. fi)
  3017. </table>
  3018. EOF
  3019. }
  3020. cla_contains () {
  3021. local e
  3022. for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done
  3023. return 1
  3024. }
  3025. filter_docker_compose_help_message() {
  3026. cat - |
  3027. sed -r "s/docker-compose run/${DARKWHITE}compose${NORMAL} ${DARKCYAN}$action${NORMAL}/g;
  3028. s/docker-compose.yml/compose.yml/g;
  3029. s/SERVICES?/${DARKYELLOW}\0${NORMAL}/g;
  3030. s/^(\s+)\\$/\1${WHITE}\$${NORMAL}/g;
  3031. s/^(\s+)run/\1${DARKCYAN}$action${NORMAL}/g;
  3032. s/docker-compose/${DARKWHITE}compose${NORMAL}/g"
  3033. }
  3034. graph() {
  3035. local services=("$@")
  3036. declare -A entries
  3037. cat <<EOF
  3038. digraph g {
  3039. graph [
  3040. fontsize=30
  3041. labelloc="t"
  3042. label=""
  3043. splines=true
  3044. overlap=false
  3045. #rankdir = "LR"
  3046. ];
  3047. ratio = auto;
  3048. EOF
  3049. for target_service in "$@"; do
  3050. services=$(get_ordered_service_dependencies "$target_service") || return 1
  3051. for service in $services; do
  3052. [ "${entries[$service]}" ] && continue || entries[$service]=1
  3053. if cla_contains "$service" "${services[@]}"; then
  3054. base=true
  3055. else
  3056. base=
  3057. fi
  3058. _graph_service "$service" "$base"
  3059. done
  3060. done
  3061. echo "}"
  3062. }
  3063. cached_wget() {
  3064. local cache_file="$CACHEDIR/$FUNCNAME.cache.$(echo "$*" | md5_compat)" \
  3065. url="$1"
  3066. if [ -e "$cache_file" ]; then
  3067. cat "$cache_file"
  3068. touch "$cache_file"
  3069. return 0
  3070. fi
  3071. wget -O- "${url}" |
  3072. tee "$cache_file"
  3073. if [ "${PIPESTATUS[0]}" != 0 ]; then
  3074. rm "$cache_file"
  3075. die "Unable to fetch '$url'."
  3076. return 1
  3077. fi
  3078. }
  3079. export -f cached_wget
  3080. [ "$SOURCED" ] && return 0
  3081. trap_add "EXIT" clean_cache
  3082. if [ -z "$DISABLE_SYSTEM_CONFIG_FILE" ]; then
  3083. if [ -r /etc/default/charm ]; then
  3084. . /etc/default/charm
  3085. fi
  3086. if [ -r "/etc/default/$exname" ]; then
  3087. . "/etc/default/$exname"
  3088. fi
  3089. ## XXXvlab: should provide YML config opportunities in possible parent dirs ?
  3090. ## userdir ? and global /etc/compose.yml ?
  3091. for cfgfile in /etc/compose.conf /etc/compose.local.conf \
  3092. /etc/default/compose /etc/compose/local.conf; do
  3093. [ -e "$cfgfile" ] || continue
  3094. . "$cfgfile" || die "Loading config file '$cfgfile' failed."
  3095. done
  3096. fi
  3097. _setup_state_dir
  3098. mkdir -p "$CACHEDIR" || exit 1
  3099. ##
  3100. ## Argument parsing
  3101. ##
  3102. services=()
  3103. remainder_args=()
  3104. compose_opts=()
  3105. compose_contents=()
  3106. action_opts=()
  3107. services_args=()
  3108. pos_arg_ct=0
  3109. no_hooks=
  3110. no_init=
  3111. action=
  3112. stage="main" ## switches from 'main', to 'action', 'remainder'
  3113. is_docker_compose_action=
  3114. rebuild_relations_to_service=()
  3115. DC_MATCH_MULTI=$(get_docker_compose_multi_opts_list) &&
  3116. DC_MATCH_SINGLE=$(get_docker_compose_single_opts_list) || return 1
  3117. while read-0 arg; do
  3118. case "$stage" in
  3119. "main")
  3120. case "$arg" in
  3121. --help|-h)
  3122. no_init=true ; no_hooks=true ; no_relations=true
  3123. display_help
  3124. exit 0
  3125. ;;
  3126. --verbose|-v)
  3127. export VERBOSE=true
  3128. compose_opts+=("--verbose")
  3129. ;;
  3130. --version|-V)
  3131. print_version
  3132. docker-compose --version
  3133. docker --version
  3134. exit 0
  3135. ;;
  3136. -f|--file)
  3137. read-0 value
  3138. [ -e "$value" ] || die "File $value doesn't exists"
  3139. export DEFAULT_COMPOSE_FILE="$value"
  3140. compose_opts+=("--file $value")
  3141. shift
  3142. ;;
  3143. -p|--project-name)
  3144. read-0 value
  3145. export DEFAULT_PROJECT_NAME="$value"
  3146. compose_opts+=("--project-name $value")
  3147. shift
  3148. ;;
  3149. --no-relations)
  3150. export no_relations=true
  3151. ;;
  3152. --no-hooks)
  3153. export no_hooks=true
  3154. ;;
  3155. --no-init)
  3156. export no_init=true
  3157. ;;
  3158. --rebuild-relations-to-service|-R)
  3159. read-0 value
  3160. rebuild_relations_to_service+=("$value")
  3161. shift
  3162. ;;
  3163. --debug)
  3164. export DEBUG=true
  3165. export VERBOSE=true
  3166. #compose_opts+=("--verbose" "--log-level" "DEBUG")
  3167. ;;
  3168. --add-compose-content|-Y)
  3169. read-0 value
  3170. compose_contents+=("$value")
  3171. shift
  3172. ;;
  3173. --dirs)
  3174. echo "CACHEDIR: $CACHEDIR"
  3175. echo "VARDIR: $VARDIR"
  3176. exit 0
  3177. ;;
  3178. --dry-compose-run)
  3179. export DRY_COMPOSE_RUN=true
  3180. ;;
  3181. --*|-*)
  3182. if str_pattern_matches "$arg" $DC_MATCH_MULTI; then
  3183. read-0 value
  3184. compose_opts+=("$arg" "$value")
  3185. shift;
  3186. elif str_pattern_matches "$arg" $DC_MATCH_SINGLE; then
  3187. compose_opts+=("$arg")
  3188. else
  3189. err "Unknown option '$arg'. Please check help:"
  3190. display_help
  3191. exit 1
  3192. fi
  3193. ;;
  3194. *)
  3195. action="$arg"
  3196. stage="action"
  3197. if DC_USAGE=$(get_docker_compose_usage "$action"); then
  3198. is_docker_compose_action=true
  3199. DC_MATCH_MULTI=$(get_docker_compose_multi_opts_list "$action") &&
  3200. DC_MATCH_SINGLE="$(get_docker_compose_single_opts_list "$action")"
  3201. if [ "$DC_MATCH_MULTI" ]; then
  3202. DC_MATCH_SINGLE="$DC_MATCH_SINGLE $(echo "$DC_MATCH_MULTI" | sed -r 's/( |$)/=\* /g')"
  3203. fi
  3204. pos_args=($(echo "$DC_USAGE" | sed -r 's/\[-[^]]+\] ?//g;s/\[options\] ?//g'))
  3205. pos_args=("${pos_args[@]:1}")
  3206. # echo "USAGE: $DC_USAGE"
  3207. # echo "pos_args: ${pos_args[@]}"
  3208. # echo "MULTI: $DC_MATCH_MULTI"
  3209. # echo "SINGLE: $DC_MATCH_SINGLE"
  3210. # exit 1
  3211. else
  3212. stage="remainder"
  3213. fi
  3214. ;;
  3215. esac
  3216. ;;
  3217. "action") ## Only for docker-compose actions
  3218. case "$arg" in
  3219. --help|-h)
  3220. no_init=true ; no_hooks=true ; no_relations=true
  3221. action_opts+=("$arg")
  3222. ;;
  3223. --*|-*)
  3224. if [ "$is_docker_compose_action" ]; then
  3225. if str_pattern_matches "$arg" $DC_MATCH_MULTI; then
  3226. read-0 value
  3227. action_opts+=("$arg" "$value")
  3228. shift
  3229. elif str_pattern_matches "$arg" $DC_MATCH_SINGLE; then
  3230. action_opts+=("$arg")
  3231. else
  3232. err "Unknown option '$arg'. Please check '${DARKCYAN}$action${NORMAL}' help:"
  3233. docker-compose "$action" --help |
  3234. filter_docker_compose_help_message >&2
  3235. exit 1
  3236. fi
  3237. fi
  3238. ;;
  3239. *)
  3240. # echo "LOOP $1 : pos_arg: $pos_arg_ct // ${pos_args[$pos_arg_ct]}"
  3241. if [[ "${pos_args[$pos_arg_ct]}" == "[SERVICE...]" ]]; then
  3242. services_args+=("$arg")
  3243. elif [[ "${pos_args[$pos_arg_ct]}" == "SERVICE" ]]; then
  3244. services_args=("$arg") || exit 1
  3245. stage="remainder"
  3246. else
  3247. action_posargs+=("$arg")
  3248. ((pos_arg_ct++))
  3249. fi
  3250. ;;
  3251. esac
  3252. ;;
  3253. "remainder")
  3254. remainder_args+=("$arg")
  3255. while read-0 arg; do
  3256. remainder_args+=("$arg")
  3257. done
  3258. break 3
  3259. ;;
  3260. esac
  3261. shift
  3262. done < <(cla.normalize "$@")
  3263. export compose_contents
  3264. [ "${services_args[*]}" ] && debug " ${DARKWHITE}Services:$NORMAL ${DARKYELLOW}${services_args[*]}$NORMAL"
  3265. [ "${compose_opts[*]}" ] && debug " ${DARKWHITE}Main docker-compose opts:$NORMAL ${compose_opts[*]}"
  3266. [ "${action_posargs[*]}" ] && debug " ${DARKWHITE}Main docker-compose pos args:$NORMAL ${action_posargs[*]}"
  3267. [ "${action_opts[*]}" ] && debug " ${DARKWHITE}Action $DARKCYAN$action$NORMAL with opts:$NORMAL ${action_opts[*]}"
  3268. [ "${remainder_args[*]}" ] && debug " ${DARKWHITE}Remainder args:$NORMAL ${remainder_args[*]}"
  3269. aexport remainder_args
  3270. ##
  3271. ## Actual code
  3272. ##
  3273. COMPOSE_YML_FILE=$(get_compose_yml_location) || exit 1
  3274. COMPOSE_YML_CONTENT=$(get_compose_yml_content) || exit 1
  3275. export COMPOSE_YML_FILE COMPOSE_YML_CONTENT
  3276. charm.sanity_checks || die "Sanity checks about charm-store failed. Please correct."
  3277. ##
  3278. ## Get services in command line.
  3279. ##
  3280. if [ -z "$is_docker_compose_action" -a "$action" ]; then
  3281. action_service=${remainder_args[0]}
  3282. if [ -z "$action_service" ]; then
  3283. err "No such command or action: ${DARKCYAN}$action${NORMAL}"
  3284. display_commands_help
  3285. exit 1
  3286. fi
  3287. remainder_args=("${remainder_args[@]:1}")
  3288. if has_service_action "$action_service" "$action" >/dev/null; then
  3289. is_service_action=true
  3290. {
  3291. read-0 action_type
  3292. case "$action_type" in
  3293. "relation")
  3294. read-0 _ target_service _target_charm relation_name
  3295. debug "Found action $DARKYELLOW${action_service}$NORMAL/$DARKBLUE$relation_name$NORMAL/$DARKCYAN$action$NORMAL (in $DARKYELLOW$target_service$NORMAL)"
  3296. ;;
  3297. "direct")
  3298. debug "Found action $DARKYELLOW${action_service}$NORMAL.$DARKCYAN$action$NORMAL"
  3299. ;;
  3300. esac
  3301. } < <(has_service_action "$action_service" "$action")
  3302. services_args=("$action_service")
  3303. else
  3304. die "Unknown action '${DARKCYAN}$action$NORMAL': It doesn't match any docker-compose commands nor inner charm actions."
  3305. fi
  3306. else
  3307. case "$action" in
  3308. ps|up)
  3309. if [ "${#services_args[@]}" == 0 ]; then
  3310. array_read-0 services_args < <(printf "%s" "$COMPOSE_YML_CONTENT" | shyaml keys-0 2>/dev/null)
  3311. fi
  3312. ;;
  3313. config)
  3314. services_args=("${action_posargs[@]}")
  3315. ;;
  3316. esac
  3317. fi
  3318. NO_CONSTRAINT_CHECK=True
  3319. case "$action" in
  3320. up)
  3321. NO_CONSTRAINT_CHECK=
  3322. ;;
  3323. esac
  3324. get_all_relations "${services_args[@]}" >/dev/null || exit 1
  3325. if [ "$is_docker_compose_action" -a "${#services_args[@]}" -gt 0 ]; then
  3326. services=($(get_master_services "${services_args[@]}")) || exit 1
  3327. if [ "$action" == "up" ]; then
  3328. ## remove run-once
  3329. for service in "${services_args[@]}"; do
  3330. type="$(get_service_type "$service")" || exit 1
  3331. if [ "$type" != "run-once" ]; then
  3332. action_posargs+=("$service")
  3333. fi
  3334. done
  3335. else
  3336. action_posargs+=("${services[@]}")
  3337. fi
  3338. fi
  3339. get_docker_compose "${services_args[@]}" >/dev/null || { ## precalculate variable \$_current_docker_compose
  3340. err "Fails to compile base 'docker-compose.yml'"
  3341. exit 1
  3342. }
  3343. ##
  3344. ## Pre-action
  3345. ##
  3346. full_init=
  3347. case "$action" in
  3348. up|run)
  3349. full_init=true
  3350. post_hook=true
  3351. ;;
  3352. ""|down|restart|logs|config|ps)
  3353. full_init=
  3354. ;;
  3355. *)
  3356. if [ "$is_service_action" ]; then
  3357. full_init=true
  3358. fi
  3359. ;;
  3360. esac
  3361. if [ "$full_init" ]; then
  3362. ## init in order
  3363. if [ -z "$no_init" ]; then
  3364. Section setup host resources
  3365. setup_host_resources "${services_args[@]}" || exit 1
  3366. Section initialisation
  3367. run_service_hook init "${services_args[@]}" || exit 1
  3368. fi
  3369. ## Get relations
  3370. if [ -z "$no_relations" ]; then
  3371. if [ "${#rebuild_relations_to_service[@]}" != 0 ]; then
  3372. rebuild_relations_to_service=$(get_master_services "${rebuild_relations_to_service[@]}") || return 1
  3373. rebuild_relations_to_service=($rebuild_relations_to_service)
  3374. project=$(get_default_project_name) || return 1
  3375. for service in "${rebuild_relations_to_service[@]}"; do
  3376. for dir in "$VARDIR/relations/$project/"*"-${service}/"*; do
  3377. [ -d "$dir" ] && {
  3378. debug rm -rf "$dir"
  3379. rm -rf "$dir"
  3380. }
  3381. done
  3382. done
  3383. fi
  3384. run_service_relations "${services_args[@]}" || exit 1
  3385. fi
  3386. run_service_hook pre_deploy "${services_args[@]}" || exit 1
  3387. fi
  3388. if [ "$action" == "run" -a "${#services_args}" != 0 ]; then
  3389. charm=$(get_service_charm "${services_args[0]}") || exit 1
  3390. metadata=$(charm.metadata "$charm") || exit 1
  3391. type="$(printf "%s" "$metadata" | shyaml get-value type 2>/dev/null)" || true
  3392. if [ "$type" == "run-once" ]; then
  3393. run_service_hook dc-pre-run "${services_args[@]}" || exit 1
  3394. fi
  3395. fi
  3396. export SERVICE_PACK="${services_args[*]}"
  3397. ##
  3398. ## Docker-compose
  3399. ##
  3400. case "$action" in
  3401. up|start|stop|build|run)
  3402. ## force daemon mode for up
  3403. if [[ "$action" == "up" ]]; then
  3404. if ! array_member action_opts -d; then
  3405. action_opts+=("-d")
  3406. fi
  3407. if ! array_member action_opts --remove-orphans; then
  3408. action_opts+=("--remove-orphans")
  3409. fi
  3410. fi
  3411. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  3412. ;;
  3413. logs)
  3414. if ! array_member action_opts --tail; then ## force daemon mode for up
  3415. action_opts+=("--tail" "10")
  3416. fi
  3417. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  3418. ;;
  3419. "")
  3420. launch_docker_compose "${compose_opts[@]}"
  3421. ;;
  3422. graph)
  3423. graph $SERVICE_PACK
  3424. ;;
  3425. config)
  3426. ## removing the services
  3427. services=($(get_master_services "${action_posargs[@]}")) || exit 1
  3428. ## forcing docker-compose config to output the config file to stdout and not stderr
  3429. out=$(launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}" 2>&1) || {
  3430. echo "$out"
  3431. exit 1
  3432. }
  3433. echo "$out"
  3434. warn "Runtime configuration modification (from relations) are not included here."
  3435. ;;
  3436. down)
  3437. if ! array_member action_opts --remove-orphans; then ## force daemon mode for up
  3438. debug "Adding a default argument of '--remove-orphans'"
  3439. action_opts+=("--remove-orphans")
  3440. fi
  3441. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${remainder_args[@]}"
  3442. ;;
  3443. *)
  3444. if [ "$is_service_action" ]; then
  3445. run_service_action "$SERVICE_PACK" "$action" "${remainder_args[@]}"
  3446. else
  3447. launch_docker_compose "${compose_opts[@]}" "$action" "${action_opts[@]}" "${action_posargs[@]}" "${remainder_args[@]}"
  3448. fi
  3449. ;;
  3450. esac || exit 1
  3451. if [ "$post_hook" -a "${#services_args[@]}" != 0 ]; then
  3452. run_service_hook post_deploy "${services_args[@]}" || exit 1
  3453. fi
  3454. clean_unused_docker_compose || return 1