forked from 0k/0k-charms
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
540 lines
20 KiB
540 lines
20 KiB
#!/bin/bash
|
|
## compose: no-hooks
|
|
|
|
|
|
if [ -z "$SERVICE_DATASTORE" ]; then
|
|
echo "This script is meant to be run through 'compose' to work properly." >&2
|
|
exit 1
|
|
fi
|
|
|
|
version=0.1
|
|
usage="$exname [-h|--help] [--force|-f] [TARGET_VERSION]"
|
|
help="
|
|
USAGE:
|
|
|
|
$usage
|
|
|
|
DESCRIPTION:
|
|
|
|
Migrate the current $SERVICE_NAME service to given target version. Don't
|
|
forget to change your \`\`compose.yml\`\` accordingly afterwards.
|
|
|
|
EXAMPLES:
|
|
|
|
$exname 21.0.0
|
|
|
|
"
|
|
|
|
no_hint=
|
|
force=
|
|
target=
|
|
while [ "$1" ]; do
|
|
case "$1" in
|
|
"--help"|"-h")
|
|
print_help >&2
|
|
exit 0
|
|
;;
|
|
"--force"|"-f")
|
|
force=yes
|
|
;;
|
|
"--color"|"-c")
|
|
ansi_color yes
|
|
;;
|
|
"--no-hint")
|
|
no_hint=yes
|
|
;;
|
|
--*|-*)
|
|
err "Unexpected optional argument '$1'"
|
|
print_usage >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
[ -z "$target" ] && { target=$1 ; shift ; continue ; }
|
|
err "Unexpected positional argument '$1'"
|
|
print_usage >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
postgres:image:version() {
|
|
local image="$1" image_version
|
|
## XXXvlab: why not always use the pragmatic second method ?
|
|
if [[ "${image}" =~ ^.*:[0-9]+.[0-9]+.[0-9]+$ ]]; then
|
|
debug "Infering postgres's version from image's name $image"
|
|
image_version="${image#*:}"
|
|
image_version="${image_version%-myc}"
|
|
else
|
|
debug "Looking for postgres's version in image $image"
|
|
if ! out=$(docker run --rm -i --entrypoint postgres "$image" --version); then
|
|
err "Couldn't infer image '$image' postgres's version."
|
|
exit 1
|
|
fi
|
|
## Expected `$out` content of the form of: 'postgres (PostgreSQL) 10.14'
|
|
out=${out%%$'\n'*}
|
|
out=${out%%$'\r'*}
|
|
image_version=${out#"postgres (PostgreSQL) "}.0
|
|
## check if this is a version
|
|
if ! [[ "$image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then
|
|
err "Couldn't infer image '$image' postgres's version: invalid version."
|
|
exit 1
|
|
fi
|
|
fi
|
|
echo "$image_version"
|
|
}
|
|
|
|
postgres:container:version() {
|
|
local container="$1"
|
|
if ! out=$(docker exec -i "$container" postgres --version); then
|
|
err "Couldn't infer container's '$container' postgres's version."
|
|
exit 1
|
|
fi
|
|
out=${out%%$'\n'*}
|
|
out=${out%%$'\r'*}
|
|
image_version=${out#"postgres (PostgreSQL) "}.0
|
|
## check if this is a version
|
|
if ! [[ "$image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then
|
|
err "Couldn't infer image '$image' postgres's version: invalid version '$image_version'."
|
|
exit 1
|
|
fi
|
|
echo "$image_version"
|
|
}
|
|
|
|
postgres:container:psql () {
|
|
local container="$1"
|
|
docker exec -i "$container" psql 2>&1
|
|
}
|
|
|
|
current_image_version=$(postgres:image:version "${DOCKER_BASE_IMAGE}") || exit 1
|
|
if ! [[ "$current_image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then
|
|
err "Current postgres version ${WHITE}$current_image_version${NORMAL} is unsupported yet."
|
|
exit 1
|
|
fi
|
|
starting_image_version=$current_image_version
|
|
## This next line has to be on stdout, it is used by `vps` to get information
|
|
info "Current postgres version: ${WHITE}$current_image_version${NORMAL}" 2>&1
|
|
|
|
last_available_versions=(
|
|
$(DEBUG= docker:tags:fetch docker.0k.io/postgres 30 '[0-9]+\.[0-9]+\.[0-9]+-myc$' |
|
|
sed -r 's/-myc$//g' |
|
|
sort -rV)
|
|
)
|
|
|
|
debug "Last available versions: ${WHITE}${last_available_versions[*]}${NORMAL}"
|
|
if [ -z "$target" ]; then
|
|
info "Latest version available: ${WHITE}${last_available_versions[0]}${NORMAL}"
|
|
fi
|
|
## XXXvlab: put this in kal-shlib-common
|
|
version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; }
|
|
|
|
last_upgradable_versions=()
|
|
for v in "${last_available_versions[@]}"; do
|
|
if version_gt "$v" "$current_image_version"; then
|
|
last_upgradable_versions+=("$v")
|
|
fi
|
|
done
|
|
|
|
if [ "${#last_upgradable_versions[@]}" == 0 ]; then
|
|
if [ -n "$target" ] && [ "$target" != "$current_image_version" ]; then
|
|
warn "Provided target version ${WHITE}$target${NORMAL} is likely incorrect."
|
|
echo " It is either non-existent and/or inferior to current version." >&2
|
|
fi
|
|
## This next line has to be on stdout, it is used by `vps` to get information
|
|
info "${DARKYELLOW}postgres${NORMAL} is already ${GREEN}up-to-date${NORMAL}." 2>&1
|
|
exit 0
|
|
fi
|
|
|
|
if [ -z "$target" ]; then
|
|
info "Target latest version: ${WHITE}${last_upgradable_versions[0]}${NORMAL}"
|
|
target=${last_upgradable_versions[0]}
|
|
else
|
|
if [[ " ${last_available_versions[*]} " != *" $target "* ]]; then
|
|
err "Invalid version $target selected, please specify one of:"
|
|
for v in "${last_upgradable_versions[@]}"; do
|
|
echo " - $v"
|
|
done >&2
|
|
exit 1
|
|
fi
|
|
last_upgradable_versions_filtered=()
|
|
info "Target version ${WHITE}$target${NORMAL}"
|
|
for v in "${last_upgradable_versions[@]}"; do
|
|
if [ "$target" == "$v" ] || version_gt "$target" "$v"; then
|
|
last_upgradable_versions_filtered+=("$v")
|
|
fi
|
|
done
|
|
last_upgradable_versions=("${last_upgradable_versions_filtered[@]}")
|
|
fi
|
|
|
|
debug "Last upgradable versions: ${WHITE}${last_upgradable_versions[*]}${NORMAL}"
|
|
|
|
upgrade_path=($(printf "%s\n" "${last_upgradable_versions[@]}" | sort -V | tail -n 1))
|
|
|
|
containers="$(get_running_containers_for_service "$SERVICE_NAME")"
|
|
debug "Running containers for service $SERVICE_NAME: $containers"
|
|
debug "Upgrade path: ${WHITE}${upgrade_path[@]}${NORMAL}"
|
|
container_stopped=()
|
|
if [ -n "$containers" ]; then
|
|
#err "Running container(s) for $DARKYELLOW$SERVICE_NAME$NORMAL are still running:"
|
|
for container in $containers; do
|
|
debug "Stopping container $container"
|
|
docker stop "$container" >/dev/null || {
|
|
err "Failed to stop container '$container'."
|
|
exit 1
|
|
}
|
|
## We need to delete it as otherwise, on final ``compose --debug up``
|
|
## it won't be recreated, and it will not be the correct version.
|
|
docker rm "$container" > /dev/null || {
|
|
err "Couldn't delete container '$container'."
|
|
}
|
|
container_stopped+=("$container")
|
|
done
|
|
fi
|
|
|
|
## XXXvlab: taking first container is probably not a good idea
|
|
container="$(echo "$containers" | head -n 1)"
|
|
|
|
settmpdir MIGRATION_TMPDIR
|
|
debug "Migration temporary directory: $MIGRATION_TMPDIR"
|
|
set -o pipefail
|
|
|
|
(
|
|
. "$CHARM_PATH/lib/common"
|
|
cp "$LOCAL_DB_PASSFILE" "$MIGRATION_TMPDIR/pgpass"
|
|
)
|
|
|
|
|
|
current_image="docker.0k.io/postgres:${current_image_version}-myc"
|
|
if ! docker_has_image "$current_image"; then
|
|
out=$(docker pull "$current_image" 2>&1) || {
|
|
err "Couldn't pull image $current_image:"
|
|
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " >&2
|
|
exit 1
|
|
}
|
|
fi
|
|
|
|
migration_backup_dirs=()
|
|
|
|
## In postgres, it seems it is not useful to have a path. So path is
|
|
## only one step here.
|
|
|
|
failed=
|
|
while [[ "${#upgrade_path[@]}" != 0 ]]; do
|
|
image_version="${upgrade_path[0]}"
|
|
|
|
if version_gt "$image_version" 12; then
|
|
docker_version=$(docker info --format '{{.ServerVersion}}')
|
|
if ! version_gt "$docker_version" 20.10.0; then
|
|
err "Sorry, ${DARKYELLOW}$SERVICE_NAME${NORMAL} ${WHITE}$image_version${NORMAL}" \
|
|
"require docker version >= 20.10 (current: $docker_version)"
|
|
break
|
|
fi
|
|
fi
|
|
upgrade_path=("${upgrade_path[@]:1}")
|
|
|
|
## Prevent build of image instead of pulling it
|
|
|
|
while true; do
|
|
|
|
info "Upgrading step ${WHITE}$current_image_version${NORMAL} => ${WHITE}$image_version${NORMAL}"
|
|
rm -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log" &&
|
|
touch "$MIGRATION_TMPDIR/dump-${current_image_version}.log"
|
|
rm -f "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" &&
|
|
touch "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log"
|
|
uuid=$(openssl rand -hex 16)
|
|
|
|
## Generate dump with pg_dumpall of current version:
|
|
|
|
(
|
|
{
|
|
cmd=(
|
|
compose --no-hooks
|
|
--add-compose-content="$SERVICE_NAME:
|
|
docker-compose:
|
|
image: docker.0k.io/postgres:${current_image_version}-myc
|
|
hostname: $SERVICE_NAME
|
|
"
|
|
run --rm --label "migration-uuid=$uuid" "$SERVICE_NAME"
|
|
)
|
|
echo COMMAND: "${cmd[@]}"
|
|
"${cmd[@]}" 2>&1
|
|
echo "ERRORLEVEL: $?"
|
|
} | tee "$MIGRATION_TMPDIR/dump-${current_image_version}.log" >/dev/null
|
|
) &
|
|
pid="$!"
|
|
debug "Dumping container of $SERVICE_NAME in ${WHITE}$current_image_version${NORMAL}" \
|
|
"launched with PID ${YELLOW}$pid${NORMAL}"
|
|
expected_stop=""
|
|
unexpected_stop=""
|
|
errlvl=""
|
|
while read -r line; do
|
|
case "$line" in
|
|
*" database system is ready to accept connections"*)
|
|
|
|
migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
|
|
if [ -z "$migration_container" ]; then
|
|
err "Couldn't find migration container for $current_image_version"
|
|
echo " despite catched line stating postgres is listening." >&2
|
|
break 3
|
|
fi
|
|
## dump all
|
|
debug "Start dumping postgres database of ${WHITE}$current_image_version${NORMAL}"
|
|
if ! out=$(
|
|
{
|
|
docker exec -u 70 "${migration_container}" pg_dumpall |
|
|
gzip > "$MIGRATION_TMPDIR/dump.sql.gz"
|
|
} 2>&1 ); then
|
|
err "Failed to dump postgres database."
|
|
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
|
|
unexpected_stop="1"
|
|
break
|
|
fi
|
|
|
|
out=$(docker stop "$migration_container" 2>&1) || {
|
|
err "Failed to stop the dump container $migration_container for $current_image_version:"
|
|
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
|
|
break 3
|
|
}
|
|
expected_stop="1"
|
|
debug "Interrupting server process after dump (${YELLOW}$pid${NORMAL})" >&2
|
|
kill "$pid"
|
|
errlvl="0"
|
|
break
|
|
;;
|
|
*"getaddrinfo(\"$SERVICE_NAME\") failed: Name or service not known"*)
|
|
err "Postgres issue with resolving '$SERVICE_NAME'"
|
|
failed=1
|
|
break 3
|
|
;;
|
|
"ERRORLEVEL: "*)
|
|
unexpected_stop="1"
|
|
errlvl="${line#ERRORLEVEL: }"
|
|
err "Unexpected stop of postgres dump container with errorlevel $errlvl"
|
|
break
|
|
;;
|
|
esac
|
|
done < <(tail -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log")
|
|
[ "$unexpected_stop" == "1" ] && {
|
|
migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
|
|
if [ -n "$migration_container" ]; then
|
|
err "Stopping dumping docker container '$migration_container' after failure"
|
|
docker stop "$migration_container" >/dev/null 2>&1 || true
|
|
fi
|
|
failed=1
|
|
break
|
|
}
|
|
|
|
[ "$expected_stop" == "1" ] && {
|
|
out=$(docker rmi "$current_image" 2>&1) || {
|
|
err "Failed to remove image $current_image"
|
|
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
|
|
break 2
|
|
}
|
|
}
|
|
next_image="docker.0k.io/postgres:${image_version}-myc"
|
|
if ! docker_has_image "$next_image"; then
|
|
out=$(docker pull "$next_image" 2>&1) || {
|
|
err "Couldn't pull image $next_image:"
|
|
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " >&2
|
|
unexpected_stop=1
|
|
break 2
|
|
}
|
|
fi
|
|
rm -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log"
|
|
uuid=$(openssl rand -hex 16)
|
|
mv "$SERVICE_DATASTORE"{,".migration-${current_image_version}-${uuid}"} 2>&1 || {
|
|
err "Couldn't move datastore to backup location for $SERVICE_NAME"
|
|
unexpected_stop=1
|
|
break 2
|
|
}
|
|
migration_backup_dirs+=("$SERVICE_DATASTORE.migration-${current_image_version}-${uuid}")
|
|
(
|
|
{
|
|
cmd=(
|
|
compose --debug
|
|
--add-compose-content="$SERVICE_NAME:
|
|
docker-compose:
|
|
image: docker.0k.io/postgres:${image_version}-myc
|
|
hostname: $SERVICE_NAME
|
|
"
|
|
run --rm --label "migration-uuid=$uuid" "$SERVICE_NAME"
|
|
)
|
|
echo COMMAND: "${cmd[@]}"
|
|
"${cmd[@]}" 2>&1
|
|
echo "ERRORLEVEL: $?"
|
|
} | tee "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" >/dev/null
|
|
) &
|
|
pid="$!"
|
|
debug "Migration launched with PID ${YELLOW}$pid${NORMAL}"
|
|
expected_stop=""
|
|
unexpected_stop=""
|
|
errlvl=""
|
|
while read -r line; do
|
|
case "$line" in
|
|
*" database system is ready to accept connections"*)
|
|
|
|
migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
|
|
|
|
## dump all
|
|
debug "Start restoring in postgres database ${WHITE}$image_version${NORMAL}"
|
|
if ! out=$(
|
|
{
|
|
gunzip -c "$MIGRATION_TMPDIR/dump.sql.gz" |
|
|
docker exec -i -u 70 "${migration_container}" psql
|
|
} 2>&1 ); then
|
|
err "Failed to restore postgres database."
|
|
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
|
|
unexpected_stop="1"
|
|
break
|
|
fi
|
|
|
|
current_image="$next_image"
|
|
current_image_version="$image_version"
|
|
out=$(docker stop "$migration_container" 2>&1) || {
|
|
err "Failed to stop the migration container $migration_container for $image_version:"
|
|
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
|
|
break 3
|
|
}
|
|
expected_stop="1"
|
|
debug "Interrupting server process after migration (${YELLOW}$pid${NORMAL})" >&2
|
|
kill "$pid"
|
|
errlvl="0"
|
|
break
|
|
;;
|
|
*"getaddrinfo(\"$SERVICE_NAME\") failed: Name or service not known"*)
|
|
err "Postgres issue with resolving '$SERVICE_NAME'"
|
|
failed=1
|
|
break 3
|
|
;;
|
|
*"Replication has not yet been configured"*)
|
|
warn "Postgres was not setup right away, could lead to issues"
|
|
;;
|
|
"ERRORLEVEL: "*)
|
|
unexpected_stop="1"
|
|
errlvl="${line#ERRORLEVEL: }"
|
|
err "Unexpected stop of postgres migration container with errorlevel $errlvl"
|
|
failed=1
|
|
break
|
|
;;
|
|
esac
|
|
done < <(tail -f "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log")
|
|
|
|
[ "$unexpected_stop" == "1" ] && {
|
|
migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
|
|
if [ -n "$migration_container" ]; then
|
|
err "Stopping dumping docker container '$migration_container' after failure"
|
|
docker stop "$migration_container" >/dev/null 2>&1 || true
|
|
fi
|
|
failed=1
|
|
break
|
|
}
|
|
|
|
[ "$expected_stop" == "1" ] && {
|
|
continue 2
|
|
}
|
|
|
|
##
|
|
## Damage control, there are some things we can solve
|
|
##
|
|
|
|
if grep "UPGRADE PROBLEM: Found an invalid featureCompatibilityVersion document" "$MIGRATION_TMPDIR/migration.log" >/dev/null 2>&1; then
|
|
if [ -z "$feature_comp" ]; then
|
|
feature_comp=1
|
|
info "Detected featureCompatibilityVersion issue, forcing upgrade from base version"
|
|
echo " This will have for effect to reload the current version and set the featureCompatibilityVersion" >&2
|
|
upgrade_path=("$current_image_version" "$image_version" "${upgrade_path[@]}")
|
|
break
|
|
else
|
|
err "Already tried to mitigate featureCompatibilityVersion without success..."
|
|
break 2
|
|
fi
|
|
fi
|
|
err "Upgrade to ${WHITE}$image_version${NORMAL} ${DARKRED}failed${NORMAL}"
|
|
failed=1
|
|
break 2
|
|
done
|
|
|
|
done
|
|
|
|
if [ -z "$unexpected_stop" ] && [ -z "$expected_stop" ]; then
|
|
migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}')
|
|
if [ -n "$migration_container" ]; then
|
|
err "Stopping migration docker container '$migration_container' after failure"
|
|
docker stop "$migration_container"
|
|
fi
|
|
fi
|
|
|
|
if [ -n "$failed" ]; then
|
|
if [ -e "$MIGRATION_TMPDIR/dump-${current_image_version}.log" ]; then
|
|
echo "${WHITE}Failing dump logs:${NORMAL}" >&2
|
|
cat "$MIGRATION_TMPDIR/dump-${current_image_version}.log" | prefix " ${GRAY}|${NORMAL} " >&2
|
|
fi
|
|
if [ -e "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" ]; then
|
|
echo "${WHITE}Failing migration logs:${NORMAL}" >&2
|
|
cat "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" | prefix " ${GRAY}|${NORMAL} " >&2
|
|
fi
|
|
fi
|
|
|
|
if [ -n "$unexpected_stop" ]; then
|
|
if [ -e "$SERVICE_DATASTORE.migration-${current_image_version}-${uuid}" ]; then
|
|
if [ -e "$SERVICE_DATASTORE" ]; then
|
|
mv -v "$SERVICE_DATASTORE"{,".migration-${image_version}-${uuid}-failed"} || {
|
|
err "Couldn't move back datastore to backup location for $SERVICE_NAME"
|
|
}
|
|
fi
|
|
mv -v "$SERVICE_DATASTORE"{".migration-${current_image_version}-${uuid}",} || {
|
|
err "Couldn't move back datastore to backup location for $SERVICE_NAME"
|
|
}
|
|
fi
|
|
err "Migration failed unexpectedly."
|
|
exit 1
|
|
fi
|
|
|
|
if [ "$starting_image_version" == "$current_image_version" ]; then
|
|
warn "Database not migrated. Current version is still: ${WHITE}$current_image_version${NORMAL}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
|
|
(
|
|
. "$CHARM_PATH/lib/common"
|
|
cp "$MIGRATION_TMPDIR/pgpass" "$LOCAL_DB_PASSFILE"
|
|
) || {
|
|
err "Couldn't restore postgres password file."
|
|
exit 1
|
|
}
|
|
|
|
for migration_backup_dir in "${migration_backup_dirs[@]}"; do
|
|
out=$(rm -rf "$migration_backup_dir") || {
|
|
warn "Couldn't remove backup directory $migration_backup_dir"
|
|
printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} "
|
|
}
|
|
done
|
|
## This next line has to be on stdout, it is used by `vps` to get information
|
|
info "${GREEN}Successfully${NORMAL} upgraded from ${WHITE}$starting_image_version${NORMAL} to ${WHITE}$current_image_version${NORMAL}" 2>&1
|
|
|
|
|
|
|
|
if [ -z "$no_hint" ]; then
|
|
cat <<EOF >&2
|
|
Don't forget to force the version in your \`\`compose.yml\`\`. For instance:
|
|
|
|
${DARKYELLOW}$SERVICE_NAME${NORMAL}:
|
|
|
|
${DARKGRAY}# ...${NORMAL}
|
|
|
|
${WHITE}docker-compose${NORMAL}:
|
|
${WHITE}image${NORMAL}: docker.0k.io/postgres:${current_image_version}-myc
|
|
|
|
${DARKGRAY}# ...${NORMAL}
|
|
|
|
You could do this automatically with:
|
|
|
|
yq eval -i ".$SERVICE_NAME.docker-compose.image = \"docker.0k.io/postgres:${current_image_version}-myc\" // \"\"" compose.yml
|
|
|
|
And don't forget to run \`compose up\` afterwards.
|
|
|
|
EOF
|
|
fi
|
|
|
|
exit 0
|