From 4ab63b8a744aac6393e3be15145117886eb890c5 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Mon, 20 Nov 2023 11:02:30 +0100 Subject: [PATCH] new: [postgres] add ``upgrade`` action --- postgres/actions/upgrade | 540 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 540 insertions(+) create mode 100755 postgres/actions/upgrade diff --git a/postgres/actions/upgrade b/postgres/actions/upgrade new file mode 100755 index 0000000..06bfbc0 --- /dev/null +++ b/postgres/actions/upgrade @@ -0,0 +1,540 @@ +#!/bin/bash +## compose: no-hooks + + +if [ -z "$SERVICE_DATASTORE" ]; then + echo "This script is meant to be run through 'compose' to work properly." >&2 + exit 1 +fi + +version=0.1 +usage="$exname [-h|--help] [--force|-f] [TARGET_VERSION]" +help=" +USAGE: + +$usage + +DESCRIPTION: + +Migrate the current $SERVICE_NAME service to given target version. Don't +forget to change your \`\`compose.yml\`\` accordingly afterwards. + +EXAMPLES: + + $exname 21.0.0 + +" + +no_hint= +force= +target= +while [ "$1" ]; do + case "$1" in + "--help"|"-h") + print_help >&2 + exit 0 + ;; + "--force"|"-f") + force=yes + ;; + "--color"|"-c") + ansi_color yes + ;; + "--no-hint") + no_hint=yes + ;; + --*|-*) + err "Unexpected optional argument '$1'" + print_usage >&2 + exit 1 + ;; + *) + [ -z "$target" ] && { target=$1 ; shift ; continue ; } + err "Unexpected positional argument '$1'" + print_usage >&2 + exit 1 + ;; + esac + shift +done + +postgres:image:version() { + local image="$1" image_version + ## XXXvlab: why not always use the pragmatic second method ? + if [[ "${image}" =~ ^.*:[0-9]+.[0-9]+.[0-9]+$ ]]; then + debug "Infering postgres's version from image's name $image" + image_version="${image#*:}" + image_version="${image_version%-myc}" + else + debug "Looking for postgres's version in image $image" + if ! out=$(docker run --rm -i --entrypoint postgres "$image" --version); then + err "Couldn't infer image '$image' postgres's version." + exit 1 + fi + ## Expected `$out` content of the form of: 'postgres (PostgreSQL) 10.14' + out=${out%%$'\n'*} + out=${out%%$'\r'*} + image_version=${out#"postgres (PostgreSQL) "}.0 + ## check if this is a version + if ! [[ "$image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then + err "Couldn't infer image '$image' postgres's version: invalid version." + exit 1 + fi + fi + echo "$image_version" +} + +postgres:container:version() { + local container="$1" + if ! out=$(docker exec -i "$container" postgres --version); then + err "Couldn't infer container's '$container' postgres's version." + exit 1 + fi + out=${out%%$'\n'*} + out=${out%%$'\r'*} + image_version=${out#"postgres (PostgreSQL) "}.0 + ## check if this is a version + if ! [[ "$image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then + err "Couldn't infer image '$image' postgres's version: invalid version '$image_version'." + exit 1 + fi + echo "$image_version" +} + +postgres:container:psql () { + local container="$1" + docker exec -i "$container" psql 2>&1 +} + +current_image_version=$(postgres:image:version "${DOCKER_BASE_IMAGE}") || exit 1 +if ! [[ "$current_image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then + err "Current postgres version ${WHITE}$current_image_version${NORMAL} is unsupported yet." + exit 1 +fi +starting_image_version=$current_image_version +## This next line has to be on stdout, it is used by `vps` to get information +info "Current postgres version: ${WHITE}$current_image_version${NORMAL}" 2>&1 + +last_available_versions=( + $(DEBUG= docker:tags:fetch docker.0k.io/postgres 30 '[0-9]+\.[0-9]+\.[0-9]+-myc$' | + sed -r 's/-myc$//g' | + sort -rV) +) + +debug "Last available versions: ${WHITE}${last_available_versions[*]}${NORMAL}" +if [ -z "$target" ]; then + info "Latest version available: ${WHITE}${last_available_versions[0]}${NORMAL}" +fi +## XXXvlab: put this in kal-shlib-common +version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; } + +last_upgradable_versions=() +for v in "${last_available_versions[@]}"; do + if version_gt "$v" "$current_image_version"; then + last_upgradable_versions+=("$v") + fi +done + +if [ "${#last_upgradable_versions[@]}" == 0 ]; then + if [ -n "$target" ] && [ "$target" != "$current_image_version" ]; then + warn "Provided target version ${WHITE}$target${NORMAL} is likely incorrect." + echo " It is either non-existent and/or inferior to current version." >&2 + fi + ## This next line has to be on stdout, it is used by `vps` to get information + info "${DARKYELLOW}postgres${NORMAL} is already ${GREEN}up-to-date${NORMAL}." 2>&1 + exit 0 +fi + +if [ -z "$target" ]; then + info "Target latest version: ${WHITE}${last_upgradable_versions[0]}${NORMAL}" + target=${last_upgradable_versions[0]} +else + if [[ " ${last_available_versions[*]} " != *" $target "* ]]; then + err "Invalid version $target selected, please specify one of:" + for v in "${last_upgradable_versions[@]}"; do + echo " - $v" + done >&2 + exit 1 + fi + last_upgradable_versions_filtered=() + info "Target version ${WHITE}$target${NORMAL}" + for v in "${last_upgradable_versions[@]}"; do + if [ "$target" == "$v" ] || version_gt "$target" "$v"; then + last_upgradable_versions_filtered+=("$v") + fi + done + last_upgradable_versions=("${last_upgradable_versions_filtered[@]}") +fi + +debug "Last upgradable versions: ${WHITE}${last_upgradable_versions[*]}${NORMAL}" + +upgrade_path=($(printf "%s\n" "${last_upgradable_versions[@]}" | sort -V | tail -n 1)) + +containers="$(get_running_containers_for_service "$SERVICE_NAME")" +debug "Running containers for service $SERVICE_NAME: $containers" +debug "Upgrade path: ${WHITE}${upgrade_path[@]}${NORMAL}" +container_stopped=() +if [ -n "$containers" ]; then + #err "Running container(s) for $DARKYELLOW$SERVICE_NAME$NORMAL are still running:" + for container in $containers; do + debug "Stopping container $container" + docker stop "$container" >/dev/null || { + err "Failed to stop container '$container'." + exit 1 + } + ## We need to delete it as otherwise, on final ``compose --debug up`` + ## it won't be recreated, and it will not be the correct version. + docker rm "$container" > /dev/null || { + err "Couldn't delete container '$container'." + } + container_stopped+=("$container") + done +fi + +## XXXvlab: taking first container is probably not a good idea +container="$(echo "$containers" | head -n 1)" + +settmpdir MIGRATION_TMPDIR +debug "Migration temporary directory: $MIGRATION_TMPDIR" +set -o pipefail + +( + . "$CHARM_PATH/lib/common" + cp "$LOCAL_DB_PASSFILE" "$MIGRATION_TMPDIR/pgpass" +) + + +current_image="docker.0k.io/postgres:${current_image_version}-myc" +if ! docker_has_image "$current_image"; then + out=$(docker pull "$current_image" 2>&1) || { + err "Couldn't pull image $current_image:" + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " >&2 + exit 1 + } +fi + +migration_backup_dirs=() + +## In postgres, it seems it is not useful to have a path. So path is +## only one step here. + +failed= +while [[ "${#upgrade_path[@]}" != 0 ]]; do + image_version="${upgrade_path[0]}" + + if version_gt "$image_version" 12; then + docker_version=$(docker info --format '{{.ServerVersion}}') + if ! version_gt "$docker_version" 20.10.0; then + err "Sorry, ${DARKYELLOW}$SERVICE_NAME${NORMAL} ${WHITE}$image_version${NORMAL}" \ + "require docker version >= 20.10 (current: $docker_version)" + break + fi + fi + upgrade_path=("${upgrade_path[@]:1}") + + ## Prevent build of image instead of pulling it + + while true; do + + info "Upgrading step ${WHITE}$current_image_version${NORMAL} => ${WHITE}$image_version${NORMAL}" + rm -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log" && + touch "$MIGRATION_TMPDIR/dump-${current_image_version}.log" + rm -f "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" && + touch "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" + uuid=$(openssl rand -hex 16) + + ## Generate dump with pg_dumpall of current version: + + ( + { + cmd=( + compose --no-hooks + --add-compose-content="$SERVICE_NAME: + docker-compose: + image: docker.0k.io/postgres:${current_image_version}-myc + hostname: $SERVICE_NAME +" + run --rm --label "migration-uuid=$uuid" "$SERVICE_NAME" + ) + echo COMMAND: "${cmd[@]}" + "${cmd[@]}" 2>&1 + echo "ERRORLEVEL: $?" + } | tee "$MIGRATION_TMPDIR/dump-${current_image_version}.log" >/dev/null + ) & + pid="$!" + debug "Dumping container of $SERVICE_NAME in ${WHITE}$current_image_version${NORMAL}" \ + "launched with PID ${YELLOW}$pid${NORMAL}" + expected_stop="" + unexpected_stop="" + errlvl="" + while read -r line; do + case "$line" in + *" database system is ready to accept connections"*) + + migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}') + if [ -z "$migration_container" ]; then + err "Couldn't find migration container for $current_image_version" + echo " despite catched line stating postgres is listening." >&2 + break 3 + fi + ## dump all + debug "Start dumping postgres database of ${WHITE}$current_image_version${NORMAL}" + if ! out=$( + { + docker exec -u 70 "${migration_container}" pg_dumpall | + gzip > "$MIGRATION_TMPDIR/dump.sql.gz" + } 2>&1 ); then + err "Failed to dump postgres database." + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " + unexpected_stop="1" + break + fi + + out=$(docker stop "$migration_container" 2>&1) || { + err "Failed to stop the dump container $migration_container for $current_image_version:" + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " + break 3 + } + expected_stop="1" + debug "Interrupting server process after dump (${YELLOW}$pid${NORMAL})" >&2 + kill "$pid" + errlvl="0" + break + ;; + *"getaddrinfo(\"$SERVICE_NAME\") failed: Name or service not known"*) + err "Postgres issue with resolving '$SERVICE_NAME'" + failed=1 + break 3 + ;; + "ERRORLEVEL: "*) + unexpected_stop="1" + errlvl="${line#ERRORLEVEL: }" + err "Unexpected stop of postgres dump container with errorlevel $errlvl" + break + ;; + esac + done < <(tail -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log") + [ "$unexpected_stop" == "1" ] && { + migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}') + if [ -n "$migration_container" ]; then + err "Stopping dumping docker container '$migration_container' after failure" + docker stop "$migration_container" >/dev/null 2>&1 || true + fi + failed=1 + break + } + + [ "$expected_stop" == "1" ] && { + out=$(docker rmi "$current_image" 2>&1) || { + err "Failed to remove image $current_image" + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " + break 2 + } + } + next_image="docker.0k.io/postgres:${image_version}-myc" + if ! docker_has_image "$next_image"; then + out=$(docker pull "$next_image" 2>&1) || { + err "Couldn't pull image $next_image:" + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " >&2 + unexpected_stop=1 + break 2 + } + fi + rm -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log" + uuid=$(openssl rand -hex 16) + mv "$SERVICE_DATASTORE"{,".migration-${current_image_version}-${uuid}"} 2>&1 || { + err "Couldn't move datastore to backup location for $SERVICE_NAME" + unexpected_stop=1 + break 2 + } + migration_backup_dirs+=("$SERVICE_DATASTORE.migration-${current_image_version}-${uuid}") + ( + { + cmd=( + compose --debug + --add-compose-content="$SERVICE_NAME: + docker-compose: + image: docker.0k.io/postgres:${image_version}-myc + hostname: $SERVICE_NAME +" + run --rm --label "migration-uuid=$uuid" "$SERVICE_NAME" + ) + echo COMMAND: "${cmd[@]}" + "${cmd[@]}" 2>&1 + echo "ERRORLEVEL: $?" + } | tee "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" >/dev/null + ) & + pid="$!" + debug "Migration launched with PID ${YELLOW}$pid${NORMAL}" + expected_stop="" + unexpected_stop="" + errlvl="" + while read -r line; do + case "$line" in + *" database system is ready to accept connections"*) + + migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}') + + ## dump all + debug "Start restoring in postgres database ${WHITE}$image_version${NORMAL}" + if ! out=$( + { + gunzip -c "$MIGRATION_TMPDIR/dump.sql.gz" | + docker exec -i -u 70 "${migration_container}" psql + } 2>&1 ); then + err "Failed to restore postgres database." + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " + unexpected_stop="1" + break + fi + + current_image="$next_image" + current_image_version="$image_version" + out=$(docker stop "$migration_container" 2>&1) || { + err "Failed to stop the migration container $migration_container for $image_version:" + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " + break 3 + } + expected_stop="1" + debug "Interrupting server process after migration (${YELLOW}$pid${NORMAL})" >&2 + kill "$pid" + errlvl="0" + break + ;; + *"getaddrinfo(\"$SERVICE_NAME\") failed: Name or service not known"*) + err "Postgres issue with resolving '$SERVICE_NAME'" + failed=1 + break 3 + ;; + *"Replication has not yet been configured"*) + warn "Postgres was not setup right away, could lead to issues" + ;; + "ERRORLEVEL: "*) + unexpected_stop="1" + errlvl="${line#ERRORLEVEL: }" + err "Unexpected stop of postgres migration container with errorlevel $errlvl" + failed=1 + break + ;; + esac + done < <(tail -f "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log") + + [ "$unexpected_stop" == "1" ] && { + migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}') + if [ -n "$migration_container" ]; then + err "Stopping dumping docker container '$migration_container' after failure" + docker stop "$migration_container" >/dev/null 2>&1 || true + fi + failed=1 + break + } + + [ "$expected_stop" == "1" ] && { + continue 2 + } + + ## + ## Damage control, there are some things we can solve + ## + + if grep "UPGRADE PROBLEM: Found an invalid featureCompatibilityVersion document" "$MIGRATION_TMPDIR/migration.log" >/dev/null 2>&1; then + if [ -z "$feature_comp" ]; then + feature_comp=1 + info "Detected featureCompatibilityVersion issue, forcing upgrade from base version" + echo " This will have for effect to reload the current version and set the featureCompatibilityVersion" >&2 + upgrade_path=("$current_image_version" "$image_version" "${upgrade_path[@]}") + break + else + err "Already tried to mitigate featureCompatibilityVersion without success..." + break 2 + fi + fi + err "Upgrade to ${WHITE}$image_version${NORMAL} ${DARKRED}failed${NORMAL}" + failed=1 + break 2 + done + +done + +if [ -z "$unexpected_stop" ] && [ -z "$expected_stop" ]; then + migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}') + if [ -n "$migration_container" ]; then + err "Stopping migration docker container '$migration_container' after failure" + docker stop "$migration_container" + fi +fi + +if [ -n "$failed" ]; then + if [ -e "$MIGRATION_TMPDIR/dump-${current_image_version}.log" ]; then + echo "${WHITE}Failing dump logs:${NORMAL}" >&2 + cat "$MIGRATION_TMPDIR/dump-${current_image_version}.log" | prefix " ${GRAY}|${NORMAL} " >&2 + fi + if [ -e "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" ]; then + echo "${WHITE}Failing migration logs:${NORMAL}" >&2 + cat "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" | prefix " ${GRAY}|${NORMAL} " >&2 + fi +fi + +if [ -n "$unexpected_stop" ]; then + if [ -e "$SERVICE_DATASTORE.migration-${current_image_version}-${uuid}" ]; then + if [ -e "$SERVICE_DATASTORE" ]; then + mv -v "$SERVICE_DATASTORE"{,".migration-${image_version}-${uuid}-failed"} || { + err "Couldn't move back datastore to backup location for $SERVICE_NAME" + } + fi + mv -v "$SERVICE_DATASTORE"{".migration-${current_image_version}-${uuid}",} || { + err "Couldn't move back datastore to backup location for $SERVICE_NAME" + } + fi + err "Migration failed unexpectedly." + exit 1 +fi + +if [ "$starting_image_version" == "$current_image_version" ]; then + warn "Database not migrated. Current version is still: ${WHITE}$current_image_version${NORMAL}" >&2 + exit 1 +fi + + +( + . "$CHARM_PATH/lib/common" + cp "$MIGRATION_TMPDIR/pgpass" "$LOCAL_DB_PASSFILE" +) || { + err "Couldn't restore postgres password file." + exit 1 +} + +for migration_backup_dir in "${migration_backup_dirs[@]}"; do + out=$(rm -rf "$migration_backup_dir") || { + warn "Couldn't remove backup directory $migration_backup_dir" + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " + } +done +## This next line has to be on stdout, it is used by `vps` to get information +info "${GREEN}Successfully${NORMAL} upgraded from ${WHITE}$starting_image_version${NORMAL} to ${WHITE}$current_image_version${NORMAL}" 2>&1 + + + +if [ -z "$no_hint" ]; then + cat <&2 +Don't forget to force the version in your \`\`compose.yml\`\`. For instance: + + ${DARKYELLOW}$SERVICE_NAME${NORMAL}: + + ${DARKGRAY}# ...${NORMAL} + + ${WHITE}docker-compose${NORMAL}: + ${WHITE}image${NORMAL}: docker.0k.io/postgres:${current_image_version}-myc + + ${DARKGRAY}# ...${NORMAL} + +You could do this automatically with: + + yq eval -i ".$SERVICE_NAME.docker-compose.image = \"docker.0k.io/postgres:${current_image_version}-myc\" // \"\"" compose.yml + +And don't forget to run \`compose up\` afterwards. + +EOF +fi + +exit 0 \ No newline at end of file