0k-charms/postgres/actions/upgrade

#!/bin/bash
## compose: no-hooks


if [ -z "$SERVICE_DATASTORE" ]; then
    echo "This script is meant to be run through 'compose' to work properly." >&2
    exit 1
fi

version=0.1
usage="$exname [-h|--help] [--force|-f] [TARGET_VERSION]"
help="
USAGE:

$usage

DESCRIPTION:

Migrate the current $SERVICE_NAME service to given target version. Don't
forget to change your \`\`compose.yml\`\` accordingly afterwards.

EXAMPLES:

  $exname 21.0.0

"

no_hint=
force=
target=
while [ "$1" ]; do
    case "$1" in
        "--help"|"-h")
            print_help >&2
            exit 0
            ;;
        "--force"|"-f")
            force=yes
            ;;
        "--color"|"-c")
            ansi_color yes
            ;;
        "--no-hint")
            no_hint=yes
            ;;
        --*|-*)
            err "Unexpected optional argument '$1'"
            print_usage >&2
            exit 1
            ;;
        *)
            [ -z "$target" ] && { target=$1 ; shift ; continue ; }
            err "Unexpected positional argument '$1'"
            print_usage >&2
            exit 1
        ;;
    esac
    shift
done

postgres:image:version() {
    local image="$1" image_version
    ## XXXvlab: why not always use the pragmatic second method ?
    if [[ "${image}" =~ ^.*:[0-9]+.[0-9]+.[0-9]+$ ]]; then
        debug "Infering postgres's version from image's name $image"
        image_version="${image#*:}"
        image_version="${image_version%-myc}"
    else
        debug "Looking for postgres's version in image $image"
        if ! out=$(docker run --rm -i --entrypoint postgres "$image" --version); then
            err "Couldn't infer image '$image' postgres's version."
            exit 1
        fi
        ## Expected `$out` content of the form of: 'postgres (PostgreSQL) 10.14'
        out=${out%%$'\n'*}
        out=${out%%$'\r'*}
        image_version=${out#"postgres (PostgreSQL) "}.0
        ## check if this is a version
        if ! [[ "$image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then
            err "Couldn't infer image '$image' postgres's version: invalid version."
            exit 1
        fi
    fi
    echo "$image_version"
}

postgres:container:version() {
    local container="$1"
    if ! out=$(docker exec -i "$container" postgres --version); then
        err "Couldn't infer container's '$container' postgres's version."
        exit 1
    fi
    out=${out%%$'\n'*}
    out=${out%%$'\r'*}
    image_version=${out#"postgres (PostgreSQL) "}.0
    ## check if this is a version
    if ! [[ "$image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then
        err "Couldn't infer image '$image' postgres's version: invalid version '$image_version'."
        exit 1
    fi
    echo "$image_version"
}

## XXXvlab: there's an issue with using ``docker stop ..``, then
## ``docker rmi ..`` on the stopped container's image. It can fail on
## some VPS (and on dev platform) saying the image is still in use.
docker:container:stop() {
    local container=$1 timeout=${2:-60}

    ## Stop the container
    if ! out=$(docker stop "$container"); then
        err "Failed to stop container $container"
        echo "$out" | prefix "  | " >&2
        return 1
    fi

    local start=$SECONDS

    # Wait for the container to stop
    while [ -n "$(
        curl -s --unix-socket /var/run/docker.sock \
		http://localhost/containers/json?all=1 |
		jq -r '.[] | .Id[:12] , .Id , ( .Names | .[] | ltrimstr("/"))' | grep ^"$container"$)" ]; do
        if [ $(($SECONDS - start)) -ge $timeout ]; then
            err "Timeout waiting for container '$container' to stop." >&2
            return 1
        fi
        sleep 0.1
    done
    return 0
}


postgres:container:psql () {
    local container="$1"
    docker exec -i "$container" psql 2>&1
}

current_image_version=$(postgres:image:version "${DOCKER_BASE_IMAGE}") || exit 1
if ! [[ "$current_image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then
    err "Current postgres version ${WHITE}$current_image_version${NORMAL} is unsupported yet."
    exit 1
fi
starting_image_version=$current_image_version
## This next line has to be on stdout, it is used by `vps` to get information
info "Current postgres version: ${WHITE}$current_image_version${NORMAL}" 2>&1

last_available_versions=(
    $(DEBUG= docker:tags:fetch docker.0k.io/postgres 30 '[0-9]+\.[0-9]+\.[0-9]+-myc$' |
          sed -r 's/-myc$//g' |
          sort -rV)
)

debug "Last available versions: ${WHITE}${last_available_versions[*]}${NORMAL}"
if [ -z "$target" ]; then
    info "Latest version available: ${WHITE}${last_available_versions[0]}${NORMAL}"
fi
## XXXvlab: put this in kal-shlib-common
version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; }

last_upgradable_versions=()
for v in "${last_available_versions[@]}"; do
    if version_gt "$v" "$current_image_version"; then
        last_upgradable_versions+=("$v")
    fi
done

if [ "${#last_upgradable_versions[@]}" == 0 ]; then
    if [ -n "$target" ] && [ "$target" != "$current_image_version" ]; then
        warn "Provided target version ${WHITE}$target${NORMAL} is likely incorrect."
        echo "   It is either non-existent and/or inferior to current version." >&2
    fi
    ## This next line has to be on stdout, it is used by `vps` to get information
    info "${DARKYELLOW}postgres${NORMAL} is already ${GREEN}up-to-date${NORMAL}." 2>&1
    exit 0
fi

if [ -z "$target" ]; then
    info "Target latest version: ${WHITE}${last_upgradable_versions[0]}${NORMAL}"
    target=${last_upgradable_versions[0]}
else
    if [[ " ${last_available_versions[*]} " != *" $target "* ]]; then
        err "Invalid version $target selected, please specify one of:"
        for v in "${last_upgradable_versions[@]}"; do
            echo " - $v"
        done >&2
        exit 1
    fi
    last_upgradable_versions_filtered=()
    info "Target version ${WHITE}$target${NORMAL}"
    for v in "${last_upgradable_versions[@]}"; do
        if [ "$target" == "$v" ] || version_gt "$target" "$v"; then
            last_upgradable_versions_filtered+=("$v")
        fi
    done
    last_upgradable_versions=("${last_upgradable_versions_filtered[@]}")
fi

debug "Last upgradable versions: ${WHITE}${last_upgradable_versions[*]}${NORMAL}"

upgrade_path=($(printf "%s\n" "${last_upgradable_versions[@]}" | sort -V | tail -n 1))

containers="$(get_running_containers_for_service "$SERVICE_NAME")"
debug "Running containers for service $SERVICE_NAME: $containers"
debug "Upgrade path: ${WHITE}${upgrade_path[@]}${NORMAL}"
container_stopped=()
if [ -n "$containers" ]; then
    #err "Running container(s) for $DARKYELLOW$SERVICE_NAME$NORMAL are still running:"
    for container in $containers; do
        debug "Stopping container $container"
        docker stop "$container" >/dev/null || {
            err "Failed to stop container '$container'."
            exit 1
        }
        ## We need to delete it as otherwise, on final ``compose --debug up``
        ## it won't be recreated, and it will not be the correct version.
        docker rm "$container" > /dev/null || {
            err "Couldn't delete container '$container'."
        }
        container_stopped+=("$container")
    done
fi

## XXXvlab: taking first container is probably not a good idea
container="$(echo "$containers" | head -n 1)"

settmpdir MIGRATION_TMPDIR
debug "Migration temporary directory: $MIGRATION_TMPDIR"
set -o pipefail

(
    . "$CHARM_PATH/lib/common"
    cp "$LOCAL_DB_PASSFILE" "$MIGRATION_TMPDIR/pgpass"
)


current_image="docker.0k.io/postgres:${current_image_version}-myc"
if ! docker_has_image "$current_image"; then
    out=$(docker pull "$current_image" 2>&1) || {
        err "Couldn't pull image $current_image:"
        printf "%s\n" "$out" | prefix "  ${GRAY}|${NORMAL} " >&2
        exit 1
    }
fi

migration_backup_dirs=()

## In postgres, it seems it is not useful to have a path. So path is
## only one step here.

failed=
while [[ "${#upgrade_path[@]}" != 0 ]]; do
    image_version="${upgrade_path[0]}"

    if version_gt "$image_version" 12; then
        docker_version=$(docker info --format '{{.ServerVersion}}')
        if ! version_gt "$docker_version" 20.10.0; then
            err "Sorry, ${DARKYELLOW}$SERVICE_NAME${NORMAL} ${WHITE}$image_version${NORMAL}" \
                "require docker version >= 20.10 (current: $docker_version)"
            break
        fi
    fi
    upgrade_path=("${upgrade_path[@]:1}")

    ## Prevent build of image instead of pulling it

    while true; do

        info "Upgrading step ${WHITE}$current_image_version${NORMAL} => ${WHITE}$image_version${NORMAL}"
        rm -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log" &&
            touch "$MIGRATION_TMPDIR/dump-${current_image_version}.log"
        rm -f "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" &&
            touch "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log"
        uuid=$(openssl rand -hex 16)

        ## Generate dump with pg_dumpall of current version:

        (
            {
                cmd=(
                    compose --no-hooks
                    --add-compose-content="$SERVICE_NAME:
  docker-compose:
    image: docker.0k.io/postgres:${current_image_version}-myc
    hostname: $SERVICE_NAME
"
                    run --rm --label "migration-uuid=$uuid" "$SERVICE_NAME"
                )
                echo COMMAND: "${cmd[@]}"
                "${cmd[@]}" 2>&1
                echo "ERRORLEVEL: $?"
            } | tee "$MIGRATION_TMPDIR/dump-${current_image_version}.log" >/dev/null
        ) &
        pid="$!"
        debug "Dumping container of $SERVICE_NAME in ${WHITE}$current_image_version${NORMAL}" \
              "launched with PID ${YELLOW}$pid${NORMAL}"
        expected_stop=""
        unexpected_stop=""
        errlvl=""
        while read -r line; do
            case "$line" in
                *" database system is ready to accept connections"*)

                    migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
                    if [ -z "$migration_container" ]; then
                        err "Couldn't find migration container for $current_image_version"
                        echo "  despite catched line stating postgres is listening." >&2
                        break 3
                    fi
                    ## dump all
                    debug "Start dumping postgres database of ${WHITE}$current_image_version${NORMAL}"
                    if ! out=$(
                        {
                            docker exec -u 70 "${migration_container}" pg_dumpall |
                                gzip > "$MIGRATION_TMPDIR/dump.sql.gz"
                        } 2>&1 ); then
                        err "Failed to dump postgres database."
                        printf "%s\n" "$out" | prefix "  ${GRAY}|${NORMAL} "
                        unexpected_stop="1"
                        break
                    fi

                    docker:container:stop "$migration_container" || {
                        err "Failed to stop the dump container $migration_container for $current_image_version"
                        break 3
                    }
                    expected_stop="1"
                    debug "Interrupting server process after dump (${YELLOW}$pid${NORMAL})" >&2
                    kill "$pid"
                    errlvl="0"
                    break
                    ;;
                *"getaddrinfo(\"$SERVICE_NAME\") failed: Name or service not known"*)
                    err "Postgres issue with resolving '$SERVICE_NAME'"
                    failed=1
                    break 3
                    ;;
                "ERRORLEVEL: "*)
                    unexpected_stop="1"
                    errlvl="${line#ERRORLEVEL: }"
                    err "Unexpected stop of postgres dump container with errorlevel $errlvl"
                    break
                    ;;
            esac
        done < <(tail -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log")
        [ "$unexpected_stop" == "1" ] && {
            migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
            if [ -n "$migration_container" ]; then
                err "Stopping dumping docker container '$migration_container' after failure"
                docker stop "$migration_container" >/dev/null 2>&1 || true
            fi
            failed=1
            break
        }

        [ "$expected_stop" == "1" ] && {
            out=$(docker rmi "$current_image" 2>&1) || {
                err "Failed to remove image $current_image"
                printf "%s\n" "$out" | prefix "  ${GRAY}|${NORMAL} "
		        sleep 2
	            docker start "$migration_container" >&2
                break 2
            }
        }
        next_image="docker.0k.io/postgres:${image_version}-myc"
        if ! docker_has_image "$next_image"; then
            out=$(docker pull "$next_image" 2>&1) || {
                err "Couldn't pull image $next_image:"
                printf "%s\n" "$out" | prefix "  ${GRAY}|${NORMAL} " >&2
                unexpected_stop=1
                break 2
            }
        fi
        rm -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log"
        uuid=$(openssl rand -hex 16)
        mv "$SERVICE_DATASTORE"{,".migration-${current_image_version}-${uuid}"} 2>&1 || {
            err "Couldn't move datastore to backup location for $SERVICE_NAME"
            unexpected_stop=1
            break 2
        }
        migration_backup_dirs+=("$SERVICE_DATASTORE.migration-${current_image_version}-${uuid}")
        (
            {
                cmd=(
                    compose --debug
                    --add-compose-content="$SERVICE_NAME:
  docker-compose:
    image: docker.0k.io/postgres:${image_version}-myc
    hostname: $SERVICE_NAME
"
                    run --rm --label "migration-uuid=$uuid" "$SERVICE_NAME"
                )
                echo COMMAND: "${cmd[@]}"
                "${cmd[@]}" 2>&1
                echo "ERRORLEVEL: $?"
            } | tee "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" >/dev/null
        ) &
        pid="$!"
        debug "Migration launched with PID ${YELLOW}$pid${NORMAL}"
        expected_stop=""
        unexpected_stop=""
        errlvl=""
        while read -r line; do
            case "$line" in
                *" database system is ready to accept connections"*)

                    migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')

                    ## dump all
                    debug "Start restoring in postgres database ${WHITE}$image_version${NORMAL}"
                    if ! out=$(
                        {
                            gunzip -c "$MIGRATION_TMPDIR/dump.sql.gz" |
                                docker exec -i -u 70 "${migration_container}" psql
                        } 2>&1 ); then
                        err "Failed to restore postgres database."
                        printf "%s\n" "$out" | prefix "  ${GRAY}|${NORMAL} "
                        unexpected_stop="1"
                        break
                    fi

                    current_image="$next_image"
                    current_image_version="$image_version"
                    docker:container:stop "$migration_container" || {
                        err "Failed to stop the migration container $migration_container for $image_version:"
                        break 3
                    }
                    expected_stop="1"
                    debug "Interrupting server process after migration (${YELLOW}$pid${NORMAL})" >&2
                    kill "$pid"
                    errlvl="0"
                    break
                    ;;
                *"getaddrinfo(\"$SERVICE_NAME\") failed: Name or service not known"*)
                    err "Postgres issue with resolving '$SERVICE_NAME'"
                    failed=1
                    break 3
                    ;;
                *"Replication has not yet been configured"*)
                    warn "Postgres was not setup right away, could lead to issues"
                    ;;
                "ERRORLEVEL: "*)
                    unexpected_stop="1"
                    errlvl="${line#ERRORLEVEL: }"
                    err "Unexpected stop of postgres migration container with errorlevel $errlvl"
                    failed=1
                    break
                    ;;
            esac
        done < <(tail -f "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log")

        [ "$unexpected_stop" == "1" ] && {
            migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
            if [ -n "$migration_container" ]; then
                err "Stopping dumping docker container '$migration_container' after failure"
                docker stop "$migration_container" >/dev/null 2>&1 || true
            fi
            failed=1
            break
        }

        [ "$expected_stop" == "1" ] && {
            continue 2
        }

        ##
        ## Damage control, there are some things we can solve
        ##

        if grep "UPGRADE PROBLEM: Found an invalid featureCompatibilityVersion document" "$MIGRATION_TMPDIR/migration.log" >/dev/null 2>&1; then
            if [ -z "$feature_comp" ]; then
                feature_comp=1
                info "Detected featureCompatibilityVersion issue, forcing upgrade from base version"
                echo "  This will have for effect to reload the current version and set the featureCompatibilityVersion" >&2
                upgrade_path=("$current_image_version" "$image_version" "${upgrade_path[@]}")
                break
            else
                err "Already tried to mitigate featureCompatibilityVersion without success..."
                break 2
            fi
        fi
        err "Upgrade to ${WHITE}$image_version${NORMAL} ${DARKRED}failed${NORMAL}"
        failed=1
        break 2
    done

done

if [ -z "$unexpected_stop" ] && [ -z "$expected_stop" ]; then
    migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
    if [ -n "$migration_container" ]; then
        err "Stopping migration docker container '$migration_container' after failure"
        docker stop "$migration_container"
    fi
fi

if [ -n "$failed" ]; then
    if [ -e "$MIGRATION_TMPDIR/dump-${current_image_version}.log" ]; then
        echo "${WHITE}Failing dump logs:${NORMAL}" >&2
        cat "$MIGRATION_TMPDIR/dump-${current_image_version}.log" | prefix " ${GRAY}|${NORMAL} " >&2
    fi
    if [ -e "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" ]; then
        echo "${WHITE}Failing migration logs:${NORMAL}" >&2
        cat "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" | prefix " ${GRAY}|${NORMAL} " >&2
    fi
fi

if [ -n "$unexpected_stop" ]; then
    if [ -e "$SERVICE_DATASTORE.migration-${current_image_version}-${uuid}" ]; then
        if [ -e "$SERVICE_DATASTORE" ]; then
            mv -v "$SERVICE_DATASTORE"{,".migration-${image_version}-${uuid}-failed"} || {
                err "Couldn't move back datastore to backup location for $SERVICE_NAME"
            }
        fi
        mv -v "$SERVICE_DATASTORE"{".migration-${current_image_version}-${uuid}",} || {
            err "Couldn't move back datastore to backup location for $SERVICE_NAME"
        }
    fi
    err "Migration failed unexpectedly."
    exit 1
fi

if [ "$starting_image_version" == "$current_image_version" ]; then
    warn "Database not migrated. Current version is still: ${WHITE}$current_image_version${NORMAL}" >&2
    exit 1
fi


(
    . "$CHARM_PATH/lib/common"
    cp "$MIGRATION_TMPDIR/pgpass" "$LOCAL_DB_PASSFILE"
) || {
    err "Couldn't restore postgres password file."
    exit 1
}

for migration_backup_dir in "${migration_backup_dirs[@]}"; do
    out=$(rm -rf "$migration_backup_dir") || {
        warn "Couldn't remove backup directory $migration_backup_dir"
        printf "%s\n" "$out" | prefix "  ${GRAY}|${NORMAL} "
    }
done
## This next line has to be on stdout, it is used by `vps` to get information
info "${GREEN}Successfully${NORMAL} upgraded from ${WHITE}$starting_image_version${NORMAL} to ${WHITE}$current_image_version${NORMAL}" 2>&1



if [ -z "$no_hint" ]; then
    cat <<EOF >&2
Don't forget to force the version in your \`\`compose.yml\`\`. For instance:

  ${DARKYELLOW}$SERVICE_NAME${NORMAL}:

    ${DARKGRAY}# ...${NORMAL}

    ${WHITE}docker-compose${NORMAL}:
      ${WHITE}image${NORMAL}: docker.0k.io/postgres:${current_image_version}-myc

    ${DARKGRAY}# ...${NORMAL}

You could do this automatically with:

    yq eval -i ".$SERVICE_NAME.docker-compose.image = \"docker.0k.io/postgres:${current_image_version}-myc\" // \"\"" compose.yml

And don't forget to run \`compose up\` afterwards.

EOF
fi

exit 0