You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

376 lines
13 KiB

  1. #!/bin/bash
  2. ## compose: no-hooks
  3. if [ -z "$SERVICE_DATASTORE" ]; then
  4. echo "This script is meant to be run through 'compose' to work properly." >&2
  5. exit 1
  6. fi
  7. version=0.1
  8. usage="$exname [-h|--help] [--force|-f] [TARGET_VERSION]"
  9. help="
  10. USAGE:
  11. $usage
  12. DESCRIPTION:
  13. Migrate the current $SERVICE_NAME service to given target version. Don't
  14. forget to change your =compose.yml= accordingly afterwards.
  15. EXAMPLES:
  16. $exname 21.0.0
  17. "
  18. no_hint=
  19. force=
  20. target=
  21. while [ "$1" ]; do
  22. case "$1" in
  23. "--help"|"-h")
  24. print_help >&2
  25. exit 0
  26. ;;
  27. "--force"|"-f")
  28. force=yes
  29. ;;
  30. "--color"|"-c")
  31. ansi_color yes
  32. ;;
  33. "--no-hint")
  34. no_hint=yes
  35. ;;
  36. --*|-*)
  37. err "Unexpected optional argument '$1'"
  38. print_usage >&2
  39. exit 1
  40. ;;
  41. *)
  42. [ -z "$target" ] && { target=$1 ; shift ; continue ; }
  43. err "Unexpected positional argument '$1'"
  44. print_usage >&2
  45. exit 1
  46. ;;
  47. esac
  48. shift
  49. done
  50. mongo:image:version() {
  51. local image="$1" image_version
  52. ## XXXvlab: why not always use the pragmatic second method ?
  53. if [[ "${image}" =~ ^.*:[0-9]+.[0-9]+.[0-9]+$ ]]; then
  54. debug "Infering mongod's version from image's name $image"
  55. image_version="${image#*:}"
  56. image_version="${image_version%-myc}"
  57. else
  58. debug "Looking for mongod's version in image $image"
  59. if ! out=$(docker run --rm -i --entrypoint mongod "$image" --version); then
  60. err "Couldn't infer image '$image' mongod's version."
  61. exit 1
  62. fi
  63. out=${out%%$'\n'*}
  64. out=${out%%$'\r'*}
  65. image_version=${out#db version v}
  66. fi
  67. echo "$image_version"
  68. }
  69. mongo:container:version() {
  70. local container="$1"
  71. if ! out=$(docker exec -i "$container" mongo --version); then
  72. err "Couldn't infer container's '$container' mongod's version."
  73. exit 1
  74. fi
  75. out=${out%%$'\n'*}
  76. out=${out%%$'\r'*}
  77. image_version=${out#db version v}
  78. echo "$image_version"
  79. }
  80. mongo:container:mongo () {
  81. local container="$1"
  82. docker exec -i "$container" mongo 2>&1
  83. }
  84. mongo:container:fix_compat() {
  85. local container="$1" image_version="$2"
  86. out=$(echo "db.adminCommand( { setFeatureCompatibilityVersion: \"${image_version%.*}\" } )" |
  87. mongo:container:mongo "$container" 2>&1)
  88. if [[ "$out" == *"\"ok\" : 1"* ]]; then
  89. debug "Feature Compatibility set to ${WHITE}${image_version%.*}${NORMAL}"
  90. else
  91. err "Failed to set feature compatibility version to ${WHITE}${image_version%.*}${NORMAL}:"
  92. echo "$out" | prefix " | " >&2
  93. return 1
  94. fi
  95. }
  96. mongo:container:enable_read() {
  97. local container="$1"
  98. ## Enable read if db version >= 4.2
  99. if ! version=$(mongo:container:version "$container"); then
  100. err "Couldn't get container's mongod version"
  101. exit 1
  102. fi
  103. if version_gt "$version" 4.1; then
  104. cmd="db.getMongo().setSecondaryOk()"
  105. out=$(mongo:container:mongo "$container" <<<"$cmd") || {
  106. err "Failed database command. Output:"
  107. echo "$out" | prefix " | " >&2
  108. exit 1
  109. }
  110. fi
  111. }
  112. current_image_version=$(mongo:image:version "${DOCKER_BASE_IMAGE}") || exit 1
  113. if ! [[ "$current_image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then
  114. err "Current mongo version ${WHITE}$current_image_version${NORMAL} is unsupported yet."
  115. exit 1
  116. fi
  117. starting_image_version=$current_image_version
  118. ## This next line has to be on stdout, it is used by `vps` to get information
  119. info "Current mongo version: ${WHITE}$current_image_version${NORMAL}" 2>&1
  120. last_available_versions=(
  121. $(DEBUG= docker:tags:fetch docker.0k.io/mongo 30 '[0-9]+\.[0-9+]\.[0-9]+-myc$' |
  122. sed -r 's/-myc$//g' |
  123. sort -rV)
  124. )
  125. debug "Last available versions: ${WHITE}${last_available_versions[*]}${NORMAL}"
  126. if [ -z "$target" ]; then
  127. info "Latest version available: ${WHITE}${last_available_versions[0]}${NORMAL}"
  128. fi
  129. ## XXXvlab: put this in kal-shlib-common
  130. version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; }
  131. last_upgradable_versions=()
  132. for v in "${last_available_versions[@]}"; do
  133. minor="${v#*.}"
  134. minor="${minor%.**}"
  135. ## In mongo versioning system, only even minor number are stable
  136. (( "$minor" % 2 )) && continue
  137. if version_gt "$v" "$current_image_version"; then
  138. last_upgradable_versions+=("$v")
  139. fi
  140. done
  141. debug "Last upgradable versions: ${WHITE}${last_upgradable_versions[*]}${NORMAL}"
  142. if [ "${#last_upgradable_versions[@]}" == 0 ]; then
  143. if [ -n "$target" ] && [ "$target" != "$current_image_version" ]; then
  144. warn "Provided target version ${WHITE}$target${NORMAL} is likely incorrect."
  145. echo " It is either non-existent and/or inferior to current version." >&2
  146. fi
  147. ## This next line has to be on stdout, it is used by `vps` to get information
  148. info "${DARKYELLOW}mongo${NORMAL} is already ${GREEN}up-to-date${NORMAL}." 2>&1
  149. exit 0
  150. fi
  151. if [ -z "$target" ]; then
  152. info "Target latest version: ${WHITE}${last_upgradable_versions[0]}${NORMAL}"
  153. target=${last_upgradable_versions[0]}
  154. else
  155. if [[ " ${last_available_versions[*]} " != *" $target "* ]]; then
  156. err "Invalid version $target selected, please specify one of:"
  157. for v in "${last_upgradable_versions[@]}"; do
  158. echo " - $v"
  159. done >&2
  160. exit 1
  161. fi
  162. last_upgradable_versions_filtered=()
  163. info "Target version ${WHITE}$target${NORMAL}"
  164. for v in "${last_upgradable_versions[@]}"; do
  165. if [ "$target" == "$v" ] || version_gt "$target" "$v"; then
  166. last_upgradable_versions_filtered+=("$v")
  167. fi
  168. done
  169. last_upgradable_versions=("${last_upgradable_versions_filtered[@]}")
  170. fi
  171. upgrade_path=($(printf "%s\n" "${last_upgradable_versions[@]}" | sort -rV | awk -F '.' '!seen[$1 "." $2]++ {print}' | sort -V))
  172. containers="$(get_running_containers_for_service "$SERVICE_NAME")"
  173. debug "Running containers for service $SERVICE_NAME: $containers"
  174. debug "Upgrade path: ${WHITE}${upgrade_path[@]}${NORMAL}"
  175. container_stopped=()
  176. if [ -n "$containers" ]; then
  177. #err "Running container(s) for $DARKYELLOW$SERVICE_NAME$NORMAL are still running:"
  178. for container in $containers; do
  179. debug "Stopping container $container"
  180. docker stop "$container" >/dev/null || {
  181. err "Failed to stop container '$container'."
  182. exit 1
  183. }
  184. docker rm "$container" > /dev/null || {
  185. err "Couldn't delete container '$container'."
  186. }
  187. container_stopped+=("$container")
  188. done
  189. fi
  190. ## XXXvlab: taking first container is probably not a good idea
  191. container="$(echo "$containers" | head -n 1)"
  192. settmpdir MIGRATION_TMPDIR
  193. set -o pipefail
  194. failed=
  195. while [[ "${#upgrade_path[@]}" != 0 ]]; do
  196. image_version="${upgrade_path[0]}"
  197. upgrade_path=("${upgrade_path[@]:1}")
  198. ## Prevent build of image instead of pulling it
  199. next_image="docker.0k.io/mongo:${image_version}-myc"
  200. if ! docker_has_image "$next_image"; then
  201. out=$(docker pull "$next_image" 2>&1) || {
  202. err "Couldn't pull image $next_image:"
  203. printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " >&2
  204. unexpected_stop=1
  205. break 1
  206. }
  207. fi
  208. while true; do
  209. info "Upgrading step ${WHITE}$current_image_version${NORMAL} => ${WHITE}$image_version${NORMAL}"
  210. rm -f "$MIGRATION_TMPDIR/migration.log" && touch "$MIGRATION_TMPDIR/migration.log"
  211. uuid=$(openssl rand -hex 16)
  212. (
  213. {
  214. cmd=(
  215. compose --no-hooks
  216. --add-compose-content="$SERVICE_NAME:
  217. docker-compose:
  218. image: docker.0k.io/mongo:${image_version}-myc
  219. hostname: $SERVICE_NAME
  220. "
  221. run --rm --label "migration-uuid=$uuid" "$SERVICE_NAME"
  222. )
  223. echo COMMAND: "${cmd[@]}"
  224. "${cmd[@]}" 2>&1
  225. echo "ERRORLEVEL: $?"
  226. } | tee -a "$MIGRATION_TMPDIR/migration.log" >/dev/null
  227. ) &
  228. pid="$!"
  229. debug "Migration launched with PID ${YELLOW}$pid${NORMAL}"
  230. expected_stop=""
  231. unexpected_stop=""
  232. errlvl=""
  233. while read -r line; do
  234. case "$line" in
  235. *" NETWORK "*"[initandlisten] waiting for connections on port"*|\
  236. *" NETWORK "*"[listener] waiting for connections on port"*|\
  237. *"\"NETWORK\""*"\"Waiting for connections\""*)
  238. migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
  239. if [ "${image_version%.*}" != "${current_image_version%.*}" ]; then
  240. mongo:container:enable_read "$migration_container" &&
  241. mongo:container:fix_compat "$migration_container" "$image_version" || {
  242. failed=1
  243. break 3
  244. }
  245. fi
  246. current_image="$next_image"
  247. current_image_version="$image_version"
  248. out=$(docker stop "$migration_container" 2>&1) || {
  249. err "Failed to stop the migration container $migration_container for $image_version:"
  250. printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
  251. break 3
  252. }
  253. expected_stop="1"
  254. debug "Interrupting server process after migration (${YELLOW}$pid${NORMAL})" >&2
  255. kill "$pid"
  256. errlvl="0"
  257. break
  258. ;;
  259. *"getaddrinfo(\"$SERVICE_NAME\") failed: Name or service not known"*)
  260. err "Mongo issue with resolving '$SERVICE_NAME'"
  261. failed=1
  262. break 3
  263. ;;
  264. *"Replication has not yet been configured"*)
  265. warn "Mongo was not setup right away, could lead to issues"
  266. ;;
  267. "ERRORLEVEL: "*)
  268. unexpected_stop="1"
  269. errlvl="${line#ERRORLEVEL: }"
  270. err "Unexpected stop of mongod migration container with errorlevel $errlvl"
  271. break
  272. ;;
  273. esac
  274. done < <(tail -f "$MIGRATION_TMPDIR/migration.log")
  275. [ "$expected_stop" == "1" ] && {
  276. docker rmi "$current_image" >/dev/null 2>&1 || true
  277. continue 2
  278. }
  279. ##
  280. ## Damage control, there are some things we can solve
  281. ##
  282. if grep "UPGRADE PROBLEM: Found an invalid featureCompatibilityVersion document" "$MIGRATION_TMPDIR/migration.log" >/dev/null 2>&1; then
  283. if [ -z "$feature_comp" ]; then
  284. feature_comp=1
  285. info "Detected featureCompatibilityVersion issue, forcing upgrade from base version"
  286. echo " This will have for effect to reload the current version and set the featureCompatibilityVersion" >&2
  287. upgrade_path=("$current_image_version" "$image_version" "${upgrade_path[@]}")
  288. break
  289. else
  290. err "Already tried to mitigate featureCompatibilityVersion without success..."
  291. break 2
  292. fi
  293. fi
  294. err "Upgrade to ${WHITE}$image_version${NORMAL} ${DARKRED}failed${NORMAL}"
  295. failed=1
  296. break 2
  297. done
  298. done
  299. if [ -z "$unexpected_stop" -a -z "$expected_stop" ]; then
  300. migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
  301. err "Stopping migration docker container '$migration_container' after failure"
  302. docker stop "$migration_container"
  303. fi
  304. if [ -n "$failed" ]; then
  305. echo "${WHITE}Failing migration logs:${NORMAL}" >&2
  306. cat "$MIGRATION_TMPDIR/migration.log" | prefix " ${GRAY}|${NORMAL} " >&2
  307. fi
  308. if [ "$starting_image_version" == "$current_image_version" ]; then
  309. warn "Database not migrated. Current version is still: ${WHITE}$current_image_version${NORMAL}" >&2
  310. exit 1
  311. fi
  312. ## This next line has to be on stdout, it is used by `vps` to get information
  313. info "${GREEN}Successfully${NORMAL} upgraded from ${WHITE}$starting_image_version${NORMAL} to ${WHITE}$current_image_version${NORMAL}" 2>&1
  314. if [ -z "$no_hint" ]; then
  315. cat <<EOF >&2
  316. Don't forget to force the version in your \`\`compose.yml\`\`. For instance:
  317. ${DARKYELLOW}$SERVICE_NAME${NORMAL}:
  318. ${DARKGRAY}# ...${NORMAL}
  319. ${WHITE}docker-compose${NORMAL}:
  320. ${WHITE}image${NORMAL}: docker.0k.io/mongo:${current_image_version}-myc
  321. ${DARKGRAY}# ...${NORMAL}
  322. You could do this automatically with:
  323. yq eval -i ".$SERVICE_NAME.docker-compose.image = \"docker.0k.io/mongo:${current_image_version}-myc\" // \"\"" compose.yml
  324. And don't forget to run \`compose up\` afterwards.
  325. EOF
  326. fi
  327. exit 0