0k
/
0k-charms
6
3
Fork 2
Code Issues 11 Pull Requests 6 Releases Wiki Activity
Browse Source

new: [postgres] add ``upgrade`` action

matomo
Valentin Lab 5 months ago
parent
6d8f295f64
commit
4ab63b8a74
1 changed files with 540 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 540
      postgres/actions/upgrade

540
postgres/actions/upgrade
View File

@ -0,0 +1,540 @@
#!/bin/bash
## compose: no-hooks
if [ -z "$SERVICE_DATASTORE" ]; then
echo "This script is meant to be run through 'compose' to work properly." >&2
exit 1
fi
version=0.1
usage="$exname [-h|--help] [--force|-f] [TARGET_VERSION]"
help="
USAGE:
$usage
DESCRIPTION:
Migrate the current $SERVICE_NAME service to given target version. Don't
forget to change your \`\`compose.yml\`\` accordingly afterwards.
EXAMPLES:
$exname 21.0.0
"
no_hint=
force=
target=
while [ "$1" ]; do
case "$1" in
"--help"|"-h")
print_help >&2
exit 0
;;
"--force"|"-f")
force=yes
;;
"--color"|"-c")
ansi_color yes
;;
"--no-hint")
no_hint=yes
;;
--*|-*)
err "Unexpected optional argument '$1'"
print_usage >&2
exit 1
;;
*)
[ -z "$target" ] && { target=$1 ; shift ; continue ; }
err "Unexpected positional argument '$1'"
print_usage >&2
exit 1
;;
esac
shift
done
postgres:image:version() {
local image="$1" image_version
## XXXvlab: why not always use the pragmatic second method ?
if [[ "${image}" =~ ^.*:[0-9]+.[0-9]+.[0-9]+$ ]]; then
debug "Infering postgres's version from image's name $image"
image_version="${image#*:}"
image_version="${image_version%-myc}"
else
debug "Looking for postgres's version in image $image"
if ! out=$(docker run --rm -i --entrypoint postgres "$image" --version); then
err "Couldn't infer image '$image' postgres's version."
exit 1
fi
## Expected `$out` content of the form of: 'postgres (PostgreSQL) 10.14'
out=${out%%$'\n'*}
out=${out%%$'\r'*}
image_version=${out#"postgres (PostgreSQL) "}.0
## check if this is a version
if ! [[ "$image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then
err "Couldn't infer image '$image' postgres's version: invalid version."
exit 1
fi
fi
echo "$image_version"
}
postgres:container:version() {
local container="$1"
if ! out=$(docker exec -i "$container" postgres --version); then
err "Couldn't infer container's '$container' postgres's version."
exit 1
fi
out=${out%%$'\n'*}
out=${out%%$'\r'*}
image_version=${out#"postgres (PostgreSQL) "}.0
## check if this is a version
if ! [[ "$image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then
err "Couldn't infer image '$image' postgres's version: invalid version '$image_version'."
exit 1
fi
echo "$image_version"
}
postgres:container:psql () {
local container="$1"
docker exec -i "$container" psql 2>&1
}
current_image_version=$(postgres:image:version "${DOCKER_BASE_IMAGE}") || exit 1
if ! [[ "$current_image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then
err "Current postgres version ${WHITE}$current_image_version${NORMAL} is unsupported yet."
exit 1
fi
starting_image_version=$current_image_version
## This next line has to be on stdout, it is used by `vps` to get information
info "Current postgres version: ${WHITE}$current_image_version${NORMAL}" 2>&1
last_available_versions=(
$(DEBUG= docker:tags:fetch docker.0k.io/postgres 30 '[0-9]+\.[0-9]+\.[0-9]+-myc$' |
sed -r 's/-myc$//g' |
sort -rV)
)
debug "Last available versions: ${WHITE}${last_available_versions[*]}${NORMAL}"
if [ -z "$target" ]; then
info "Latest version available: ${WHITE}${last_available_versions[0]}${NORMAL}"
fi
## XXXvlab: put this in kal-shlib-common
version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; }
last_upgradable_versions=()
for v in "${last_available_versions[@]}"; do
if version_gt "$v" "$current_image_version"; then
last_upgradable_versions+=("$v")
fi
done
if [ "${#last_upgradable_versions[@]}" == 0 ]; then
if [ -n "$target" ] && [ "$target" != "$current_image_version" ]; then
warn "Provided target version ${WHITE}$target${NORMAL} is likely incorrect."
echo " It is either non-existent and/or inferior to current version." >&2
fi
## This next line has to be on stdout, it is used by `vps` to get information
info "${DARKYELLOW}postgres${NORMAL} is already ${GREEN}up-to-date${NORMAL}." 2>&1
exit 0
fi
if [ -z "$target" ]; then
info "Target latest version: ${WHITE}${last_upgradable_versions[0]}${NORMAL}"
target=${last_upgradable_versions[0]}
else
if [[ " ${last_available_versions[*]} " != *" $target "* ]]; then
err "Invalid version $target selected, please specify one of:"
for v in "${last_upgradable_versions[@]}"; do
echo " - $v"
done >&2
exit 1
fi
last_upgradable_versions_filtered=()
info "Target version ${WHITE}$target${NORMAL}"
for v in "${last_upgradable_versions[@]}"; do
if [ "$target" == "$v" ] || version_gt "$target" "$v"; then
last_upgradable_versions_filtered+=("$v")
fi
done
last_upgradable_versions=("${last_upgradable_versions_filtered[@]}")
fi
debug "Last upgradable versions: ${WHITE}${last_upgradable_versions[*]}${NORMAL}"
upgrade_path=($(printf "%s\n" "${last_upgradable_versions[@]}" | sort -V | tail -n 1))
containers="$(get_running_containers_for_service "$SERVICE_NAME")"
debug "Running containers for service $SERVICE_NAME: $containers"
debug "Upgrade path: ${WHITE}${upgrade_path[@]}${NORMAL}"
container_stopped=()
if [ -n "$containers" ]; then
#err "Running container(s) for $DARKYELLOW$SERVICE_NAME$NORMAL are still running:"
for container in $containers; do
debug "Stopping container $container"
docker stop "$container" >/dev/null || {
err "Failed to stop container '$container'."
exit 1
}
## We need to delete it as otherwise, on final ``compose --debug up``
## it won't be recreated, and it will not be the correct version.
docker rm "$container" > /dev/null || {
err "Couldn't delete container '$container'."
}
container_stopped+=("$container")
done
fi
## XXXvlab: taking first container is probably not a good idea
container="$(echo "$containers" | head -n 1)"
settmpdir MIGRATION_TMPDIR
debug "Migration temporary directory: $MIGRATION_TMPDIR"
set -o pipefail
(
. "$CHARM_PATH/lib/common"
cp "$LOCAL_DB_PASSFILE" "$MIGRATION_TMPDIR/pgpass"
)
current_image="docker.0k.io/postgres:${current_image_version}-myc"
if ! docker_has_image "$current_image"; then
out=$(docker pull "$current_image" 2>&1) || {
err "Couldn't pull image $current_image:"
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " >&2
exit 1
}
fi
migration_backup_dirs=()
## In postgres, it seems it is not useful to have a path. So path is
## only one step here.
failed=
while [[ "${#upgrade_path[@]}" != 0 ]]; do
image_version="${upgrade_path[0]}"
if version_gt "$image_version" 12; then
docker_version=$(docker info --format '{{.ServerVersion}}')
if ! version_gt "$docker_version" 20.10.0; then
err "Sorry, ${DARKYELLOW}$SERVICE_NAME${NORMAL} ${WHITE}$image_version${NORMAL}" \
"require docker version >= 20.10 (current: $docker_version)"
break
fi
fi
upgrade_path=("${upgrade_path[@]:1}")
## Prevent build of image instead of pulling it
while true; do
info "Upgrading step ${WHITE}$current_image_version${NORMAL} => ${WHITE}$image_version${NORMAL}"
rm -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log" &&
touch "$MIGRATION_TMPDIR/dump-${current_image_version}.log"
rm -f "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" &&
touch "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log"
uuid=$(openssl rand -hex 16)
## Generate dump with pg_dumpall of current version:
(
{
cmd=(
compose --no-hooks
--add-compose-content="$SERVICE_NAME:
docker-compose:
image: docker.0k.io/postgres:${current_image_version}-myc
hostname: $SERVICE_NAME
"
run --rm --label "migration-uuid=$uuid" "$SERVICE_NAME"
)
echo COMMAND: "${cmd[@]}"
"${cmd[@]}" 2>&1
echo "ERRORLEVEL: $?"
} | tee "$MIGRATION_TMPDIR/dump-${current_image_version}.log" >/dev/null
) &
pid="$!"
debug "Dumping container of $SERVICE_NAME in ${WHITE}$current_image_version${NORMAL}" \
"launched with PID ${YELLOW}$pid${NORMAL}"
expected_stop=""
unexpected_stop=""
errlvl=""
while read -r line; do
case "$line" in
*" database system is ready to accept connections"*)
migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
if [ -z "$migration_container" ]; then
err "Couldn't find migration container for $current_image_version"
echo " despite catched line stating postgres is listening." >&2
break 3
fi
## dump all
debug "Start dumping postgres database of ${WHITE}$current_image_version${NORMAL}"
if ! out=$(
{
docker exec -u 70 "${migration_container}" pg_dumpall |
gzip > "$MIGRATION_TMPDIR/dump.sql.gz"
} 2>&1 ); then
err "Failed to dump postgres database."
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
unexpected_stop="1"
break
fi
out=$(docker stop "$migration_container" 2>&1) || {
err "Failed to stop the dump container $migration_container for $current_image_version:"
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
break 3
}
expected_stop="1"
debug "Interrupting server process after dump (${YELLOW}$pid${NORMAL})" >&2
kill "$pid"
errlvl="0"
break
;;
*"getaddrinfo(\"$SERVICE_NAME\") failed: Name or service not known"*)
err "Postgres issue with resolving '$SERVICE_NAME'"
failed=1
break 3
;;
"ERRORLEVEL: "*)
unexpected_stop="1"
errlvl="${line#ERRORLEVEL: }"
err "Unexpected stop of postgres dump container with errorlevel $errlvl"
break
;;
esac
done < <(tail -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log")
[ "$unexpected_stop" == "1" ] && {
migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
if [ -n "$migration_container" ]; then
err "Stopping dumping docker container '$migration_container' after failure"
docker stop "$migration_container" >/dev/null 2>&1 || true
fi
failed=1
break
}
[ "$expected_stop" == "1" ] && {
out=$(docker rmi "$current_image" 2>&1) || {
err "Failed to remove image $current_image"
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
break 2
}
}
next_image="docker.0k.io/postgres:${image_version}-myc"
if ! docker_has_image "$next_image"; then
out=$(docker pull "$next_image" 2>&1) || {
err "Couldn't pull image $next_image:"
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " >&2
unexpected_stop=1
break 2
}
fi
rm -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log"
uuid=$(openssl rand -hex 16)
mv "$SERVICE_DATASTORE"{,".migration-${current_image_version}-${uuid}"} 2>&1 || {
err "Couldn't move datastore to backup location for $SERVICE_NAME"
unexpected_stop=1
break 2
}
migration_backup_dirs+=("$SERVICE_DATASTORE.migration-${current_image_version}-${uuid}")
(
{
cmd=(
compose --debug
--add-compose-content="$SERVICE_NAME:
docker-compose:
image: docker.0k.io/postgres:${image_version}-myc
hostname: $SERVICE_NAME
"
run --rm --label "migration-uuid=$uuid" "$SERVICE_NAME"
)
echo COMMAND: "${cmd[@]}"
"${cmd[@]}" 2>&1
echo "ERRORLEVEL: $?"
} | tee "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" >/dev/null
) &
pid="$!"
debug "Migration launched with PID ${YELLOW}$pid${NORMAL}"
expected_stop=""
unexpected_stop=""
errlvl=""
while read -r line; do
case "$line" in
*" database system is ready to accept connections"*)
migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
## dump all
debug "Start restoring in postgres database ${WHITE}$image_version${NORMAL}"
if ! out=$(
{
gunzip -c "$MIGRATION_TMPDIR/dump.sql.gz" |
docker exec -i -u 70 "${migration_container}" psql
} 2>&1 ); then
err "Failed to restore postgres database."
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
unexpected_stop="1"
break
fi
current_image="$next_image"
current_image_version="$image_version"
out=$(docker stop "$migration_container" 2>&1) || {
err "Failed to stop the migration container $migration_container for $image_version:"
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
break 3
}
expected_stop="1"
debug "Interrupting server process after migration (${YELLOW}$pid${NORMAL})" >&2
kill "$pid"
errlvl="0"
break
;;
*"getaddrinfo(\"$SERVICE_NAME\") failed: Name or service not known"*)
err "Postgres issue with resolving '$SERVICE_NAME'"
failed=1
break 3
;;
*"Replication has not yet been configured"*)
warn "Postgres was not setup right away, could lead to issues"
;;
"ERRORLEVEL: "*)
unexpected_stop="1"
errlvl="${line#ERRORLEVEL: }"
err "Unexpected stop of postgres migration container with errorlevel $errlvl"
failed=1
break
;;
esac
done < <(tail -f "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log")
[ "$unexpected_stop" == "1" ] && {
migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
if [ -n "$migration_container" ]; then
err "Stopping dumping docker container '$migration_container' after failure"
docker stop "$migration_container" >/dev/null 2>&1 || true
fi
failed=1
break
}
[ "$expected_stop" == "1" ] && {
continue 2
}
##
## Damage control, there are some things we can solve
##
if grep "UPGRADE PROBLEM: Found an invalid featureCompatibilityVersion document" "$MIGRATION_TMPDIR/migration.log" >/dev/null 2>&1; then
if [ -z "$feature_comp" ]; then
feature_comp=1
info "Detected featureCompatibilityVersion issue, forcing upgrade from base version"
echo " This will have for effect to reload the current version and set the featureCompatibilityVersion" >&2
upgrade_path=("$current_image_version" "$image_version" "${upgrade_path[@]}")
break
else
err "Already tried to mitigate featureCompatibilityVersion without success..."
break 2
fi
fi
err "Upgrade to ${WHITE}$image_version${NORMAL} ${DARKRED}failed${NORMAL}"
failed=1
break 2
done
done
if [ -z "$unexpected_stop" ] && [ -z "$expected_stop" ]; then
migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
if [ -n "$migration_container" ]; then
err "Stopping migration docker container '$migration_container' after failure"
docker stop "$migration_container"
fi
fi
if [ -n "$failed" ]; then
if [ -e "$MIGRATION_TMPDIR/dump-${current_image_version}.log" ]; then
echo "${WHITE}Failing dump logs:${NORMAL}" >&2
cat "$MIGRATION_TMPDIR/dump-${current_image_version}.log" | prefix " ${GRAY}|${NORMAL} " >&2
fi
if [ -e "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" ]; then
echo "${WHITE}Failing migration logs:${NORMAL}" >&2
cat "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" | prefix " ${GRAY}|${NORMAL} " >&2
fi
fi
if [ -n "$unexpected_stop" ]; then
if [ -e "$SERVICE_DATASTORE.migration-${current_image_version}-${uuid}" ]; then
if [ -e "$SERVICE_DATASTORE" ]; then
mv -v "$SERVICE_DATASTORE"{,".migration-${image_version}-${uuid}-failed"} || {
err "Couldn't move back datastore to backup location for $SERVICE_NAME"
}
fi
mv -v "$SERVICE_DATASTORE"{".migration-${current_image_version}-${uuid}",} || {
err "Couldn't move back datastore to backup location for $SERVICE_NAME"
}
fi
err "Migration failed unexpectedly."
exit 1
fi
if [ "$starting_image_version" == "$current_image_version" ]; then
warn "Database not migrated. Current version is still: ${WHITE}$current_image_version${NORMAL}" >&2
exit 1
fi
(
. "$CHARM_PATH/lib/common"
cp "$MIGRATION_TMPDIR/pgpass" "$LOCAL_DB_PASSFILE"
) || {
err "Couldn't restore postgres password file."
exit 1
}
for migration_backup_dir in "${migration_backup_dirs[@]}"; do
out=$(rm -rf "$migration_backup_dir") || {
warn "Couldn't remove backup directory $migration_backup_dir"
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
}
done
## This next line has to be on stdout, it is used by `vps` to get information
info "${GREEN}Successfully${NORMAL} upgraded from ${WHITE}$starting_image_version${NORMAL} to ${WHITE}$current_image_version${NORMAL}" 2>&1
if [ -z "$no_hint" ]; then
cat <<EOF >&2
Don't forget to force the version in your \`\`compose.yml\`\`. For instance:
${DARKYELLOW}$SERVICE_NAME${NORMAL}:
${DARKGRAY}# ...${NORMAL}
${WHITE}docker-compose${NORMAL}:
${WHITE}image${NORMAL}: docker.0k.io/postgres:${current_image_version}-myc
${DARKGRAY}# ...${NORMAL}
You could do this automatically with:
yq eval -i ".$SERVICE_NAME.docker-compose.image = \"docker.0k.io/postgres:${current_image_version}-myc\" // \"\"" compose.yml
And don't forget to run \`compose up\` afterwards.
EOF
fi
exit 0
Write Preview
Loading…
Cancel
Save