From 35f3643b48e64ae492ca2f29c4ccc34604f50ce7 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Tue, 3 Oct 2023 22:44:47 +0200 Subject: [PATCH] new: [mongo] provide ``upgrade`` action --- mongo/actions/upgrade | 377 ++++++++++++++++++++++++++++++++++++++++++ mongo/hooks/init | 38 ----- 2 files changed, 377 insertions(+), 38 deletions(-) create mode 100755 mongo/actions/upgrade diff --git a/mongo/actions/upgrade b/mongo/actions/upgrade new file mode 100755 index 0000000..be1fc73 --- /dev/null +++ b/mongo/actions/upgrade @@ -0,0 +1,377 @@ +#!/bin/bash +## compose: no-hooks + + +if [ -z "$SERVICE_DATASTORE" ]; then + echo "This script is meant to be run through 'compose' to work properly." >&2 + exit 1 +fi + +version=0.1 +usage="$exname [-h|--help] [--force|-f] [TARGET_VERSION]" +help=" +USAGE: + +$usage + +DESCRIPTION: + +Migrate the current $SERVICE_NAME service to given target version. Don't +forget to change your =compose.yml= accordingly afterwards. + +EXAMPLES: + + $exname 21.0.0 + +" + +no_hint= +force= +target= +while [ "$1" ]; do + case "$1" in + "--help"|"-h") + print_help >&2 + exit 0 + ;; + "--force"|"-f") + force=yes + ;; + "--color"|"-c") + ansi_color yes + ;; + "--no-hint") + no_hint=yes + ;; + --*|-*) + err "Unexpected optional argument '$1'" + print_usage >&2 + exit 1 + ;; + *) + [ -z "$target" ] && { target=$1 ; shift ; continue ; } + err "Unexpected positional argument '$1'" + print_usage >&2 + exit 1 + ;; + esac + shift +done + +mongo:image:version() { + local image="$1" image_version + ## XXXvlab: why not always use the pragmatic second method ? + if [[ "${image}" =~ ^.*:[0-9]+.[0-9]+.[0-9]+$ ]]; then + debug "Infering mongod's version from image's name $image" + image_version="${image#*:}" + image_version="${image_version%-myc}" + else + debug "Looking for mongod's version in image $image" + if ! out=$(docker run --rm -i --entrypoint mongod "$image" --version); then + err "Couldn't infer image '$image' mongod's version." + exit 1 + fi + out=${out%%$'\n'*} + out=${out%%$'\r'*} + image_version=${out#db version v} + fi + echo "$image_version" +} + +mongo:container:version() { + local container="$1" + if ! out=$(docker exec -i "$container" mongo --version); then + err "Couldn't infer container's '$container' mongod's version." + exit 1 + fi + out=${out%%$'\n'*} + out=${out%%$'\r'*} + image_version=${out#db version v} + echo "$image_version" +} + +mongo:container:mongo () { + local container="$1" + docker exec -i "$container" mongo 2>&1 +} + +mongo:container:fix_compat() { + local container="$1" image_version="$2" + out=$(echo "db.adminCommand( { setFeatureCompatibilityVersion: \"${image_version%.*}\" } )" | + mongo:container:mongo "$container" 2>&1) + if [[ "$out" == *"\"ok\" : 1"* ]]; then + debug "Feature Compatibility set to ${WHITE}${image_version%.*}${NORMAL}" + else + err "Failed to set feature compatibility version to ${WHITE}${image_version%.*}${NORMAL}:" + echo "$out" | prefix " | " >&2 + return 1 + fi +} + +mongo:container:enable_read() { + local container="$1" + + ## Enable read if db version >= 4.2 + if ! version=$(mongo:container:version "$container"); then + err "Couldn't get container's mongod version" + exit 1 + fi + + if version_gt "$version" 4.1; then + cmd="db.getMongo().setSecondaryOk()" + out=$(mongo:container:mongo "$container" <<<"$cmd") || { + err "Failed database command. Output:" + echo "$out" | prefix " | " >&2 + exit 1 + } + fi +} + +current_image_version=$(mongo:image:version "${DOCKER_BASE_IMAGE}") || exit 1 +if ! [[ "$current_image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then + err "Current mongo version ${WHITE}$current_image_version${NORMAL} is unsupported yet." + exit 1 +fi +starting_image_version=$current_image_version +## This next line has to be on stdout, it is used by `vps` to get information +info "Current mongo version: ${WHITE}$current_image_version${NORMAL}" 2>&1 + +last_available_versions=( + $(DEBUG= docker:tags:fetch docker.0k.io/mongo 30 '[0-9]+\.[0-9+]\.[0-9]+-myc$' | + sed -r 's/-myc$//g' | + sort -rV) +) + +debug "Last available versions: ${WHITE}${last_available_versions[*]}${NORMAL}" +if [ -z "$target" ]; then + info "Latest version available: ${WHITE}${last_available_versions[0]}${NORMAL}" +fi +## XXXvlab: put this in kal-shlib-common +version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; } + +last_upgradable_versions=() +for v in "${last_available_versions[@]}"; do + minor="${v#*.}" + minor="${minor%.**}" + ## In mongo versioning system, only even minor number are stable + (( "$minor" % 2 )) && continue + if version_gt "$v" "$current_image_version"; then + last_upgradable_versions+=("$v") + fi +done +debug "Last upgradable versions: ${WHITE}${last_upgradable_versions[*]}${NORMAL}" + +if [ "${#last_upgradable_versions[@]}" == 0 ]; then + if [ -n "$target" ] && [ "$target" != "$current_image_version" ]; then + warn "Provided target version ${WHITE}$target${NORMAL} is likely incorrect." + echo " It is either non-existent and/or inferior to current version." >&2 + fi + ## This next line has to be on stdout, it is used by `vps` to get information + info "${DARKYELLOW}mongo${NORMAL} is already ${GREEN}up-to-date${NORMAL}." 2>&1 + exit 0 +fi + +if [ -z "$target" ]; then + info "Target latest version: ${WHITE}${last_upgradable_versions[0]}${NORMAL}" + target=${last_upgradable_versions[0]} +else + if [[ " ${last_available_versions[*]} " != *" $target "* ]]; then + err "Invalid version $target selected, please specify one of:" + for v in "${last_upgradable_versions[@]}"; do + echo " - $v" + done >&2 + exit 1 + fi + last_upgradable_versions_filtered=() + info "Target version ${WHITE}$target${NORMAL}" + for v in "${last_upgradable_versions[@]}"; do + if [ "$target" == "$v" ] || version_gt "$target" "$v"; then + last_upgradable_versions_filtered+=("$v") + fi + done + last_upgradable_versions=("${last_upgradable_versions_filtered[@]}") +fi + +upgrade_path=($(printf "%s\n" "${last_upgradable_versions[@]}" | sort -rV | awk -F '.' '!seen[$1 "." $2]++ {print}' | sort -V)) + +containers="$(get_running_containers_for_service "$SERVICE_NAME")" +debug "Running containers for service $SERVICE_NAME: $containers" +debug "Upgrade path: ${WHITE}${upgrade_path[@]}${NORMAL}" +container_stopped=() +if [ -n "$containers" ]; then + #err "Running container(s) for $DARKYELLOW$SERVICE_NAME$NORMAL are still running:" + for container in $containers; do + debug "Stopping container $container" + docker stop "$container" >/dev/null || { + err "Failed to stop container '$container'." + exit 1 + } + docker rm "$container" > /dev/null || { + err "Couldn't delete container '$container'." + } + container_stopped+=("$container") + done +fi + +## XXXvlab: taking first container is probably not a good idea +container="$(echo "$containers" | head -n 1)" + +settmpdir MIGRATION_TMPDIR +set -o pipefail + +failed= +while [[ "${#upgrade_path[@]}" != 0 ]]; do + image_version="${upgrade_path[0]}" + upgrade_path=("${upgrade_path[@]:1}") + + ## Prevent build of image instead of pulling it + + next_image="docker.0k.io/mongo:${image_version}-myc" + if ! docker_has_image "$next_image"; then + out=$(docker pull "$next_image" 2>&1) || { + err "Couldn't pull image $next_image:" + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " >&2 + unexpected_stop=1 + break 1 + } + fi + while true; do + + info "Upgrading step ${WHITE}$current_image_version${NORMAL} => ${WHITE}$image_version${NORMAL}" + rm -f "$MIGRATION_TMPDIR/migration.log" && touch "$MIGRATION_TMPDIR/migration.log" + uuid=$(openssl rand -hex 16) + + ( + { + cmd=( + compose --no-hooks + --add-compose-content="$SERVICE_NAME: + docker-compose: + image: docker.0k.io/mongo:${image_version}-myc + hostname: $SERVICE_NAME +" + run --rm --label "migration-uuid=$uuid" "$SERVICE_NAME" + ) + echo COMMAND: "${cmd[@]}" + "${cmd[@]}" 2>&1 + echo "ERRORLEVEL: $?" + } | tee -a "$MIGRATION_TMPDIR/migration.log" >/dev/null + ) & + pid="$!" + debug "Migration launched with PID ${YELLOW}$pid${NORMAL}" + expected_stop="" + unexpected_stop="" + errlvl="" + while read -r line; do + case "$line" in + *" NETWORK "*"[initandlisten] waiting for connections on port"*|\ + *" NETWORK "*"[listener] waiting for connections on port"*|\ + *"\"NETWORK\""*"\"Waiting for connections\""*) + + migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}') + if [ "${image_version%.*}" != "${current_image_version%.*}" ]; then + mongo:container:enable_read "$migration_container" && + mongo:container:fix_compat "$migration_container" "$image_version" || { + failed=1 + break 3 + } + fi + current_image="$next_image" + current_image_version="$image_version" + out=$(docker stop "$migration_container" 2>&1) || { + err "Failed to stop the migration container $migration_container for $image_version:" + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " + break 3 + } + expected_stop="1" + debug "Interrupting server process after migration (${YELLOW}$pid${NORMAL})" >&2 + kill "$pid" + errlvl="0" + break + ;; + *"getaddrinfo(\"mongo\") failed: Name or service not known"*) + err "Mongo issue with resolving 'mongo'" + failed=1 + break 3 + ;; + *"Replication has not yet been configured"*) + warn "Mongo was not setup right away, could lead to issues" + ;; + "ERRORLEVEL: "*) + unexpected_stop="1" + errlvl="${line#ERRORLEVEL: }" + err "Unexpected stop of mongod migration container with errorlevel $errlvl" + break + ;; + esac + done < <(tail -f "$MIGRATION_TMPDIR/migration.log") + [ "$expected_stop" == "1" ] && { + docker rmi "$current_image" >/dev/null 2>&1 || true + continue 2 + } + + ## + ## Damage control, there are some things we can solve + ## + + if grep "UPGRADE PROBLEM: Found an invalid featureCompatibilityVersion document" "$MIGRATION_TMPDIR/migration.log" >/dev/null 2>&1; then + if [ -z "$feature_comp" ]; then + feature_comp=1 + info "Detected featureCompatibilityVersion issue, forcing upgrade from base version" + echo " This will have for effect to reload the current version and set the featureCompatibilityVersion" >&2 + upgrade_path=("$current_image_version" "$image_version" "${upgrade_path[@]}") + break + else + err "Already tried to mitigate featureCompatibilityVersion without success..." + break 2 + fi + fi + err "Upgrade to ${WHITE}$image_version${NORMAL} ${DARKRED}failed${NORMAL}" + failed=1 + break 2 + done + +done + +if [ -z "$unexpected_stop" -a -z "$expected_stop" ]; then + migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}') + err "Stopping migration docker container '$migration_container' after failure" + docker stop "$migration_container" +fi + +if [ -n "$failed" ]; then + echo "${WHITE}Failing migration logs:${NORMAL}" >&2 + cat "$MIGRATION_TMPDIR/migration.log" | prefix " ${GRAY}|${NORMAL} " >&2 +fi + +if [ "$starting_image_version" == "$current_image_version" ]; then + warn "Database not migrated. Current version is still: ${WHITE}$current_image_version${NORMAL}" >&2 + exit 1 +fi + +## This next line has to be on stdout, it is used by `vps` to get information +info "${GREEN}Successfully${NORMAL} upgraded from ${WHITE}$starting_image_version${NORMAL} to ${WHITE}$current_image_version${NORMAL}" 2>&1 + +if [ -z "$no_hint" ]; then + cat <&2 +Don't forget to force the version in your \`\`compose.yml\`\`. For instance: + + ${DARKYELLOW}$SERVICE_NAME${NORMAL}: + + ${DARKGRAY}# ...${NORMAL} + + ${WHITE}docker-compose${NORMAL}: + ${WHITE}image${NORMAL}: docker.0k.io/mongo:${current_image_version}-myc + + ${DARKGRAY}# ...${NORMAL} + +You could do this automatically with: + + yq eval -i ".$SERVICE_NAME.docker-compose.image = \"docker.0k.io/mongo:${current_image_version}-myc\" // \"\"" compose.yml + +And don't forget to run \`compose up\` afterwards. + +EOF +fi + +exit 0 \ No newline at end of file diff --git a/mongo/hooks/init b/mongo/hooks/init index 916b9da..6824288 100755 --- a/mongo/hooks/init +++ b/mongo/hooks/init @@ -72,41 +72,3 @@ else fi -## Enable read if db version >= 4.2 - -if ! version=$(mongo:db:version); then - err "Couldn't get database version" - exit 1 -fi - -echo "Current mongo database version: '$version'." >&2 - -if version_gt "$version" 4.1; then - cmd="db.getMongo().setSecondaryOk()" - debug "${WHITE}running:$NORMAL $cmd" - - out=$(ddb <<<"$cmd") || { - err "Failed database command. Output:" - echo "$out" | prefix " | " >&2 - exit 1 - } - -fi - - -## This is some sort of migrating code and should be moved to upgrade -## directory. - -major_version=${version%.*} - -cmd="db.adminCommand( { setFeatureCompatibilityVersion: \"${major_version}\" } )" -debug "${WHITE}running:$NORMAL $cmd" - -out=$(ddb <<<"$cmd") -if [[ "$out" == *"\"ok\" : 1"* ]]; then - info "Feature Compatibility set to ${major_version}. " -else - err "Failed to set feature compatibility version failed:" - echo "$out" | prefix " | " >&2 - exit 13 -fi