fork 0k-charms
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

569 lines
20 KiB

  1. #!/bin/bash
  2. ## compose: no-hooks
  3. if [ -z "$SERVICE_DATASTORE" ]; then
  4. echo "This script is meant to be run through 'compose' to work properly." >&2
  5. exit 1
  6. fi
  7. version=0.1
  8. usage="$exname [-h|--help] [--force|-f] [TARGET_VERSION]"
  9. help="
  10. USAGE:
  11. $usage
  12. DESCRIPTION:
  13. Migrate the current $SERVICE_NAME service to given target version. Don't
  14. forget to change your \`\`compose.yml\`\` accordingly afterwards.
  15. EXAMPLES:
  16. $exname 21.0.0
  17. "
  18. no_hint=
  19. force=
  20. target=
  21. while [ "$1" ]; do
  22. case "$1" in
  23. "--help"|"-h")
  24. print_help >&2
  25. exit 0
  26. ;;
  27. "--force"|"-f")
  28. force=yes
  29. ;;
  30. "--color"|"-c")
  31. ansi_color yes
  32. ;;
  33. "--no-hint")
  34. no_hint=yes
  35. ;;
  36. --*|-*)
  37. err "Unexpected optional argument '$1'"
  38. print_usage >&2
  39. exit 1
  40. ;;
  41. *)
  42. [ -z "$target" ] && { target=$1 ; shift ; continue ; }
  43. err "Unexpected positional argument '$1'"
  44. print_usage >&2
  45. exit 1
  46. ;;
  47. esac
  48. shift
  49. done
  50. postgres:image:version() {
  51. local image="$1" image_version
  52. ## XXXvlab: why not always use the pragmatic second method ?
  53. if [[ "${image}" =~ ^.*:[0-9]+.[0-9]+.[0-9]+$ ]]; then
  54. debug "Infering postgres's version from image's name $image"
  55. image_version="${image#*:}"
  56. image_version="${image_version%-myc}"
  57. else
  58. debug "Looking for postgres's version in image $image"
  59. if ! out=$(docker run --rm -i --entrypoint postgres "$image" --version); then
  60. err "Couldn't infer image '$image' postgres's version."
  61. exit 1
  62. fi
  63. ## Expected `$out` content of the form of: 'postgres (PostgreSQL) 10.14'
  64. out=${out%%$'\n'*}
  65. out=${out%%$'\r'*}
  66. image_version=${out#"postgres (PostgreSQL) "}.0
  67. ## check if this is a version
  68. if ! [[ "$image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then
  69. err "Couldn't infer image '$image' postgres's version: invalid version."
  70. exit 1
  71. fi
  72. fi
  73. echo "$image_version"
  74. }
  75. postgres:container:version() {
  76. local container="$1"
  77. if ! out=$(docker exec -i "$container" postgres --version); then
  78. err "Couldn't infer container's '$container' postgres's version."
  79. exit 1
  80. fi
  81. out=${out%%$'\n'*}
  82. out=${out%%$'\r'*}
  83. image_version=${out#"postgres (PostgreSQL) "}.0
  84. ## check if this is a version
  85. if ! [[ "$image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then
  86. err "Couldn't infer image '$image' postgres's version: invalid version '$image_version'."
  87. exit 1
  88. fi
  89. echo "$image_version"
  90. }
  91. ## XXXvlab: there's an issue with using ``docker stop ..``, then
  92. ## ``docker rmi ..`` on the stopped container's image. It can fail on
  93. ## some VPS (and on dev platform) saying the image is still in use.
  94. docker:container:stop() {
  95. local container=$1 timeout=${2:-60}
  96. ## Stop the container
  97. if ! out=$(docker stop "$container"); then
  98. err "Failed to stop container $container"
  99. echo "$out" | prefix " | " >&2
  100. return 1
  101. fi
  102. local start=$SECONDS
  103. # Wait for the container to stop
  104. while [ -n "$(
  105. curl -s --unix-socket /var/run/docker.sock \
  106. http://localhost/containers/json?all=1 |
  107. jq -r '.[] | .Id[:12] , .Id , ( .Names | .[] | ltrimstr("/"))' | grep ^"$container"$)" ]; do
  108. if [ $(($SECONDS - start)) -ge $timeout ]; then
  109. err "Timeout waiting for container '$container' to stop." >&2
  110. return 1
  111. fi
  112. sleep 0.1
  113. done
  114. return 0
  115. }
  116. postgres:container:psql () {
  117. local container="$1"
  118. docker exec -i "$container" psql 2>&1
  119. }
  120. current_image_version=$(postgres:image:version "${DOCKER_BASE_IMAGE}") || exit 1
  121. if ! [[ "$current_image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then
  122. err "Current postgres version ${WHITE}$current_image_version${NORMAL} is unsupported yet."
  123. exit 1
  124. fi
  125. starting_image_version=$current_image_version
  126. ## This next line has to be on stdout, it is used by `vps` to get information
  127. info "Current postgres version: ${WHITE}$current_image_version${NORMAL}" 2>&1
  128. last_available_versions=(
  129. $(DEBUG= docker:tags:fetch docker.0k.io/postgres 30 '[0-9]+\.[0-9]+\.[0-9]+-myc$' |
  130. sed -r 's/-myc$//g' |
  131. sort -rV)
  132. )
  133. debug "Last available versions: ${WHITE}${last_available_versions[*]}${NORMAL}"
  134. if [ -z "$target" ]; then
  135. info "Latest version available: ${WHITE}${last_available_versions[0]}${NORMAL}"
  136. fi
  137. ## XXXvlab: put this in kal-shlib-common
  138. version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; }
  139. last_upgradable_versions=()
  140. for v in "${last_available_versions[@]}"; do
  141. if version_gt "$v" "$current_image_version"; then
  142. last_upgradable_versions+=("$v")
  143. fi
  144. done
  145. if [ "${#last_upgradable_versions[@]}" == 0 ]; then
  146. if [ -n "$target" ] && [ "$target" != "$current_image_version" ]; then
  147. warn "Provided target version ${WHITE}$target${NORMAL} is likely incorrect."
  148. echo " It is either non-existent and/or inferior to current version." >&2
  149. fi
  150. ## This next line has to be on stdout, it is used by `vps` to get information
  151. info "${DARKYELLOW}postgres${NORMAL} is already ${GREEN}up-to-date${NORMAL}." 2>&1
  152. exit 0
  153. fi
  154. if [ -z "$target" ]; then
  155. info "Target latest version: ${WHITE}${last_upgradable_versions[0]}${NORMAL}"
  156. target=${last_upgradable_versions[0]}
  157. else
  158. if [[ " ${last_available_versions[*]} " != *" $target "* ]]; then
  159. err "Invalid version $target selected, please specify one of:"
  160. for v in "${last_upgradable_versions[@]}"; do
  161. echo " - $v"
  162. done >&2
  163. exit 1
  164. fi
  165. last_upgradable_versions_filtered=()
  166. info "Target version ${WHITE}$target${NORMAL}"
  167. for v in "${last_upgradable_versions[@]}"; do
  168. if [ "$target" == "$v" ] || version_gt "$target" "$v"; then
  169. last_upgradable_versions_filtered+=("$v")
  170. fi
  171. done
  172. last_upgradable_versions=("${last_upgradable_versions_filtered[@]}")
  173. fi
  174. debug "Last upgradable versions: ${WHITE}${last_upgradable_versions[*]}${NORMAL}"
  175. upgrade_path=($(printf "%s\n" "${last_upgradable_versions[@]}" | sort -V | tail -n 1))
  176. containers="$(get_running_containers_for_service "$SERVICE_NAME")"
  177. debug "Running containers for service $SERVICE_NAME: $containers"
  178. debug "Upgrade path: ${WHITE}${upgrade_path[@]}${NORMAL}"
  179. container_stopped=()
  180. if [ -n "$containers" ]; then
  181. #err "Running container(s) for $DARKYELLOW$SERVICE_NAME$NORMAL are still running:"
  182. for container in $containers; do
  183. debug "Stopping container $container"
  184. docker stop "$container" >/dev/null || {
  185. err "Failed to stop container '$container'."
  186. exit 1
  187. }
  188. ## We need to delete it as otherwise, on final ``compose --debug up``
  189. ## it won't be recreated, and it will not be the correct version.
  190. docker rm "$container" > /dev/null || {
  191. err "Couldn't delete container '$container'."
  192. }
  193. container_stopped+=("$container")
  194. done
  195. fi
  196. ## XXXvlab: taking first container is probably not a good idea
  197. container="$(echo "$containers" | head -n 1)"
  198. settmpdir MIGRATION_TMPDIR
  199. debug "Migration temporary directory: $MIGRATION_TMPDIR"
  200. set -o pipefail
  201. (
  202. . "$CHARM_PATH/lib/common"
  203. cp "$LOCAL_DB_PASSFILE" "$MIGRATION_TMPDIR/pgpass"
  204. )
  205. current_image="docker.0k.io/postgres:${current_image_version}-myc"
  206. if ! docker_has_image "$current_image"; then
  207. out=$(docker pull "$current_image" 2>&1) || {
  208. err "Couldn't pull image $current_image:"
  209. printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " >&2
  210. exit 1
  211. }
  212. fi
  213. migration_backup_dirs=()
  214. ## In postgres, it seems it is not useful to have a path. So path is
  215. ## only one step here.
  216. failed=
  217. while [[ "${#upgrade_path[@]}" != 0 ]]; do
  218. image_version="${upgrade_path[0]}"
  219. if version_gt "$image_version" 12; then
  220. docker_version=$(docker info --format '{{.ServerVersion}}')
  221. if ! version_gt "$docker_version" 20.10.0; then
  222. err "Sorry, ${DARKYELLOW}$SERVICE_NAME${NORMAL} ${WHITE}$image_version${NORMAL}" \
  223. "require docker version >= 20.10 (current: $docker_version)"
  224. break
  225. fi
  226. fi
  227. upgrade_path=("${upgrade_path[@]:1}")
  228. ## Prevent build of image instead of pulling it
  229. while true; do
  230. info "Upgrading step ${WHITE}$current_image_version${NORMAL} => ${WHITE}$image_version${NORMAL}"
  231. rm -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log" &&
  232. touch "$MIGRATION_TMPDIR/dump-${current_image_version}.log"
  233. rm -f "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" &&
  234. touch "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log"
  235. uuid=$(openssl rand -hex 16)
  236. ## Generate dump with pg_dumpall of current version:
  237. (
  238. {
  239. cmd=(
  240. compose --no-hooks
  241. --add-compose-content="$SERVICE_NAME:
  242. docker-compose:
  243. image: docker.0k.io/postgres:${current_image_version}-myc
  244. hostname: $SERVICE_NAME
  245. "
  246. run --rm --label "migration-uuid=$uuid" "$SERVICE_NAME"
  247. )
  248. echo COMMAND: "${cmd[@]}"
  249. "${cmd[@]}" 2>&1
  250. echo "ERRORLEVEL: $?"
  251. } | tee "$MIGRATION_TMPDIR/dump-${current_image_version}.log" >/dev/null
  252. ) &
  253. pid="$!"
  254. debug "Dumping container of $SERVICE_NAME in ${WHITE}$current_image_version${NORMAL}" \
  255. "launched with PID ${YELLOW}$pid${NORMAL}"
  256. expected_stop=""
  257. unexpected_stop=""
  258. errlvl=""
  259. while read -r line; do
  260. case "$line" in
  261. *" database system is ready to accept connections"*)
  262. migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
  263. if [ -z "$migration_container" ]; then
  264. err "Couldn't find migration container for $current_image_version"
  265. echo " despite catched line stating postgres is listening." >&2
  266. break 3
  267. fi
  268. ## dump all
  269. debug "Start dumping postgres database of ${WHITE}$current_image_version${NORMAL}"
  270. if ! out=$(
  271. {
  272. docker exec -u 70 "${migration_container}" pg_dumpall |
  273. gzip > "$MIGRATION_TMPDIR/dump.sql.gz"
  274. } 2>&1 ); then
  275. err "Failed to dump postgres database."
  276. printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
  277. unexpected_stop="1"
  278. break
  279. fi
  280. docker:container:stop "$migration_container" || {
  281. err "Failed to stop the dump container $migration_container for $current_image_version"
  282. break 3
  283. }
  284. expected_stop="1"
  285. debug "Interrupting server process after dump (${YELLOW}$pid${NORMAL})" >&2
  286. kill "$pid"
  287. errlvl="0"
  288. break
  289. ;;
  290. *"getaddrinfo(\"$SERVICE_NAME\") failed: Name or service not known"*)
  291. err "Postgres issue with resolving '$SERVICE_NAME'"
  292. failed=1
  293. break 3
  294. ;;
  295. "ERRORLEVEL: "*)
  296. unexpected_stop="1"
  297. errlvl="${line#ERRORLEVEL: }"
  298. err "Unexpected stop of postgres dump container with errorlevel $errlvl"
  299. break
  300. ;;
  301. esac
  302. done < <(tail -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log")
  303. [ "$unexpected_stop" == "1" ] && {
  304. migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
  305. if [ -n "$migration_container" ]; then
  306. err "Stopping dumping docker container '$migration_container' after failure"
  307. docker stop "$migration_container" >/dev/null 2>&1 || true
  308. fi
  309. failed=1
  310. break
  311. }
  312. [ "$expected_stop" == "1" ] && {
  313. out=$(docker rmi "$current_image" 2>&1) || {
  314. err "Failed to remove image $current_image"
  315. printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
  316. sleep 2
  317. docker start "$migration_container" >&2
  318. break 2
  319. }
  320. }
  321. next_image="docker.0k.io/postgres:${image_version}-myc"
  322. if ! docker_has_image "$next_image"; then
  323. out=$(docker pull "$next_image" 2>&1) || {
  324. err "Couldn't pull image $next_image:"
  325. printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " >&2
  326. unexpected_stop=1
  327. break 2
  328. }
  329. fi
  330. rm -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log"
  331. uuid=$(openssl rand -hex 16)
  332. mv "$SERVICE_DATASTORE"{,".migration-${current_image_version}-${uuid}"} 2>&1 || {
  333. err "Couldn't move datastore to backup location for $SERVICE_NAME"
  334. unexpected_stop=1
  335. break 2
  336. }
  337. migration_backup_dirs+=("$SERVICE_DATASTORE.migration-${current_image_version}-${uuid}")
  338. (
  339. {
  340. cmd=(
  341. compose --debug
  342. --add-compose-content="$SERVICE_NAME:
  343. docker-compose:
  344. image: docker.0k.io/postgres:${image_version}-myc
  345. hostname: $SERVICE_NAME
  346. "
  347. run --rm --label "migration-uuid=$uuid" "$SERVICE_NAME"
  348. )
  349. echo COMMAND: "${cmd[@]}"
  350. "${cmd[@]}" 2>&1
  351. echo "ERRORLEVEL: $?"
  352. } | tee "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" >/dev/null
  353. ) &
  354. pid="$!"
  355. debug "Migration launched with PID ${YELLOW}$pid${NORMAL}"
  356. expected_stop=""
  357. unexpected_stop=""
  358. errlvl=""
  359. while read -r line; do
  360. case "$line" in
  361. *" database system is ready to accept connections"*)
  362. migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
  363. ## dump all
  364. debug "Start restoring in postgres database ${WHITE}$image_version${NORMAL}"
  365. if ! out=$(
  366. {
  367. gunzip -c "$MIGRATION_TMPDIR/dump.sql.gz" |
  368. docker exec -i -u 70 "${migration_container}" psql
  369. } 2>&1 ); then
  370. err "Failed to restore postgres database."
  371. printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
  372. unexpected_stop="1"
  373. break
  374. fi
  375. current_image="$next_image"
  376. current_image_version="$image_version"
  377. docker:container:stop "$migration_container" || {
  378. err "Failed to stop the migration container $migration_container for $image_version:"
  379. break 3
  380. }
  381. expected_stop="1"
  382. debug "Interrupting server process after migration (${YELLOW}$pid${NORMAL})" >&2
  383. kill "$pid"
  384. errlvl="0"
  385. break
  386. ;;
  387. *"getaddrinfo(\"$SERVICE_NAME\") failed: Name or service not known"*)
  388. err "Postgres issue with resolving '$SERVICE_NAME'"
  389. failed=1
  390. break 3
  391. ;;
  392. *"Replication has not yet been configured"*)
  393. warn "Postgres was not setup right away, could lead to issues"
  394. ;;
  395. "ERRORLEVEL: "*)
  396. unexpected_stop="1"
  397. errlvl="${line#ERRORLEVEL: }"
  398. err "Unexpected stop of postgres migration container with errorlevel $errlvl"
  399. failed=1
  400. break
  401. ;;
  402. esac
  403. done < <(tail -f "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log")
  404. [ "$unexpected_stop" == "1" ] && {
  405. migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
  406. if [ -n "$migration_container" ]; then
  407. err "Stopping dumping docker container '$migration_container' after failure"
  408. docker stop "$migration_container" >/dev/null 2>&1 || true
  409. fi
  410. failed=1
  411. break
  412. }
  413. [ "$expected_stop" == "1" ] && {
  414. continue 2
  415. }
  416. ##
  417. ## Damage control, there are some things we can solve
  418. ##
  419. if grep "UPGRADE PROBLEM: Found an invalid featureCompatibilityVersion document" "$MIGRATION_TMPDIR/migration.log" >/dev/null 2>&1; then
  420. if [ -z "$feature_comp" ]; then
  421. feature_comp=1
  422. info "Detected featureCompatibilityVersion issue, forcing upgrade from base version"
  423. echo " This will have for effect to reload the current version and set the featureCompatibilityVersion" >&2
  424. upgrade_path=("$current_image_version" "$image_version" "${upgrade_path[@]}")
  425. break
  426. else
  427. err "Already tried to mitigate featureCompatibilityVersion without success..."
  428. break 2
  429. fi
  430. fi
  431. err "Upgrade to ${WHITE}$image_version${NORMAL} ${DARKRED}failed${NORMAL}"
  432. failed=1
  433. break 2
  434. done
  435. done
  436. if [ -z "$unexpected_stop" ] && [ -z "$expected_stop" ]; then
  437. migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
  438. if [ -n "$migration_container" ]; then
  439. err "Stopping migration docker container '$migration_container' after failure"
  440. docker stop "$migration_container"
  441. fi
  442. fi
  443. if [ -n "$failed" ]; then
  444. if [ -e "$MIGRATION_TMPDIR/dump-${current_image_version}.log" ]; then
  445. echo "${WHITE}Failing dump logs:${NORMAL}" >&2
  446. cat "$MIGRATION_TMPDIR/dump-${current_image_version}.log" | prefix " ${GRAY}|${NORMAL} " >&2
  447. fi
  448. if [ -e "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" ]; then
  449. echo "${WHITE}Failing migration logs:${NORMAL}" >&2
  450. cat "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" | prefix " ${GRAY}|${NORMAL} " >&2
  451. fi
  452. fi
  453. if [ -n "$unexpected_stop" ]; then
  454. if [ -e "$SERVICE_DATASTORE.migration-${current_image_version}-${uuid}" ]; then
  455. if [ -e "$SERVICE_DATASTORE" ]; then
  456. mv -v "$SERVICE_DATASTORE"{,".migration-${image_version}-${uuid}-failed"} || {
  457. err "Couldn't move back datastore to backup location for $SERVICE_NAME"
  458. }
  459. fi
  460. mv -v "$SERVICE_DATASTORE"{".migration-${current_image_version}-${uuid}",} || {
  461. err "Couldn't move back datastore to backup location for $SERVICE_NAME"
  462. }
  463. fi
  464. err "Migration failed unexpectedly."
  465. exit 1
  466. fi
  467. if [ "$starting_image_version" == "$current_image_version" ]; then
  468. warn "Database not migrated. Current version is still: ${WHITE}$current_image_version${NORMAL}" >&2
  469. exit 1
  470. fi
  471. (
  472. . "$CHARM_PATH/lib/common"
  473. cp "$MIGRATION_TMPDIR/pgpass" "$LOCAL_DB_PASSFILE"
  474. ) || {
  475. err "Couldn't restore postgres password file."
  476. exit 1
  477. }
  478. for migration_backup_dir in "${migration_backup_dirs[@]}"; do
  479. out=$(rm -rf "$migration_backup_dir") || {
  480. warn "Couldn't remove backup directory $migration_backup_dir"
  481. printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
  482. }
  483. done
  484. ## This next line has to be on stdout, it is used by `vps` to get information
  485. info "${GREEN}Successfully${NORMAL} upgraded from ${WHITE}$starting_image_version${NORMAL} to ${WHITE}$current_image_version${NORMAL}" 2>&1
  486. if [ -z "$no_hint" ]; then
  487. cat <<EOF >&2
  488. Don't forget to force the version in your \`\`compose.yml\`\`. For instance:
  489. ${DARKYELLOW}$SERVICE_NAME${NORMAL}:
  490. ${DARKGRAY}# ...${NORMAL}
  491. ${WHITE}docker-compose${NORMAL}:
  492. ${WHITE}image${NORMAL}: docker.0k.io/postgres:${current_image_version}-myc
  493. ${DARKGRAY}# ...${NORMAL}
  494. You could do this automatically with:
  495. yq eval -i ".$SERVICE_NAME.docker-compose.image = \"docker.0k.io/postgres:${current_image_version}-myc\" // \"\"" compose.yml
  496. And don't forget to run \`compose up\` afterwards.
  497. EOF
  498. fi
  499. exit 0