From 74c434c64fa42f7f0a7ae1a4aa091532837586c6 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Mon, 20 Nov 2023 11:02:30 +0100 Subject: [PATCH 1/2] new: [postgres] add ``upgrade`` action --- postgres/actions/upgrade | 540 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 540 insertions(+) create mode 100755 postgres/actions/upgrade diff --git a/postgres/actions/upgrade b/postgres/actions/upgrade new file mode 100755 index 0000000..38ef77b --- /dev/null +++ b/postgres/actions/upgrade @@ -0,0 +1,540 @@ +#!/bin/bash +## compose: no-hooks + + +if [ -z "$SERVICE_DATASTORE" ]; then + echo "This script is meant to be run through 'compose' to work properly." >&2 + exit 1 +fi + +version=0.1 +usage="$exname [-h|--help] [--force|-f] [TARGET_VERSION]" +help=" +USAGE: + +$usage + +DESCRIPTION: + +Migrate the current $SERVICE_NAME service to given target version. Don't +forget to change your \`\`compose.yml\`\` accordingly afterwards. + +EXAMPLES: + + $exname 21.0.0 + +" + +no_hint= +force= +target= +while [ "$1" ]; do + case "$1" in + "--help"|"-h") + print_help >&2 + exit 0 + ;; + "--force"|"-f") + force=yes + ;; + "--color"|"-c") + ansi_color yes + ;; + "--no-hint") + no_hint=yes + ;; + --*|-*) + err "Unexpected optional argument '$1'" + print_usage >&2 + exit 1 + ;; + *) + [ -z "$target" ] && { target=$1 ; shift ; continue ; } + err "Unexpected positional argument '$1'" + print_usage >&2 + exit 1 + ;; + esac + shift +done + +postgres:image:version() { + local image="$1" image_version + ## XXXvlab: why not always use the pragmatic second method ? + if [[ "${image}" =~ ^.*:[0-9]+.[0-9]+.[0-9]+$ ]]; then + debug "Infering postgres's version from image's name $image" + image_version="${image#*:}" + image_version="${image_version%-myc}" + else + debug "Looking for postgres's version in image $image" + if ! out=$(docker run --rm -i --entrypoint postgres "$image" --version); then + err "Couldn't infer image '$image' postgres's version." + exit 1 + fi + ## Expected `$out` content of the form of: 'postgres (PostgreSQL) 10.14' + out=${out%%$'\n'*} + out=${out%%$'\r'*} + image_version=${out#"postgres (PostgreSQL) "}.0 + ## check if this is a version + if ! [[ "$image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then + err "Couldn't infer image '$image' postgres's version: invalid version." + exit 1 + fi + fi + echo "$image_version" +} + +postgres:container:version() { + local container="$1" + if ! out=$(docker exec -i "$container" postgres --version); then + err "Couldn't infer container's '$container' postgres's version." + exit 1 + fi + out=${out%%$'\n'*} + out=${out%%$'\r'*} + image_version=${out#"postgres (PostgreSQL) "}.0 + ## check if this is a version + if ! [[ "$image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then + err "Couldn't infer image '$image' postgres's version: invalid version '$image_version'." + exit 1 + fi + echo "$image_version" +} + +postgres:container:psql () { + local container="$1" + docker exec -i "$container" psql 2>&1 +} + +current_image_version=$(postgres:image:version "${DOCKER_BASE_IMAGE}") || exit 1 +if ! [[ "$current_image_version" =~ ^[0-9]+.[0-9]+.[0-9]+$ ]]; then + err "Current postgres version ${WHITE}$current_image_version${NORMAL} is unsupported yet." + exit 1 +fi +starting_image_version=$current_image_version +## This next line has to be on stdout, it is used by `vps` to get information +info "Current postgres version: ${WHITE}$current_image_version${NORMAL}" 2>&1 + +last_available_versions=( + $(DEBUG= docker:tags:fetch docker.0k.io/postgres 30 '[0-9]+\.[0-9]+\.[0-9]+-myc$' | + sed -r 's/-myc$//g' | + sort -rV) +) + +debug "Last available versions: ${WHITE}${last_available_versions[*]}${NORMAL}" +if [ -z "$target" ]; then + info "Latest version available: ${WHITE}${last_available_versions[0]}${NORMAL}" +fi +## XXXvlab: put this in kal-shlib-common +version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; } + +last_upgradable_versions=() +for v in "${last_available_versions[@]}"; do + if version_gt "$v" "$current_image_version"; then + last_upgradable_versions+=("$v") + fi +done + +if [ "${#last_upgradable_versions[@]}" == 0 ]; then + if [ -n "$target" ] && [ "$target" != "$current_image_version" ]; then + warn "Provided target version ${WHITE}$target${NORMAL} is likely incorrect." + echo " It is either non-existent and/or inferior to current version." >&2 + fi + ## This next line has to be on stdout, it is used by `vps` to get information + info "${DARKYELLOW}postgres${NORMAL} is already ${GREEN}up-to-date${NORMAL}." 2>&1 + exit 0 +fi + +if [ -z "$target" ]; then + info "Target latest version: ${WHITE}${last_upgradable_versions[0]}${NORMAL}" + target=${last_upgradable_versions[0]} +else + if [[ " ${last_available_versions[*]} " != *" $target "* ]]; then + err "Invalid version $target selected, please specify one of:" + for v in "${last_upgradable_versions[@]}"; do + echo " - $v" + done >&2 + exit 1 + fi + last_upgradable_versions_filtered=() + info "Target version ${WHITE}$target${NORMAL}" + for v in "${last_upgradable_versions[@]}"; do + if [ "$target" == "$v" ] || version_gt "$target" "$v"; then + last_upgradable_versions_filtered+=("$v") + fi + done + last_upgradable_versions=("${last_upgradable_versions_filtered[@]}") +fi + +debug "Last upgradable versions: ${WHITE}${last_upgradable_versions[*]}${NORMAL}" + +upgrade_path=($(printf "%s\n" "${last_upgradable_versions[@]}" | sort -V | tail -n 1)) + +containers="$(get_running_containers_for_service "$SERVICE_NAME")" +debug "Running containers for service $SERVICE_NAME: $containers" +debug "Upgrade path: ${WHITE}${upgrade_path[@]}${NORMAL}" +container_stopped=() +if [ -n "$containers" ]; then + #err "Running container(s) for $DARKYELLOW$SERVICE_NAME$NORMAL are still running:" + for container in $containers; do + debug "Stopping container $container" + docker stop "$container" >/dev/null || { + err "Failed to stop container '$container'." + exit 1 + } + ## We need to delete it as otherwise, on final ``compose --debug up`` + ## it won't be recreated, and it will not be the correct version. + docker rm "$container" > /dev/null || { + err "Couldn't delete container '$container'." + } + container_stopped+=("$container") + done +fi + +## XXXvlab: taking first container is probably not a good idea +container="$(echo "$containers" | head -n 1)" + +settmpdir MIGRATION_TMPDIR +debug "Migration temporary directory: $MIGRATION_TMPDIR" +set -o pipefail + +( + . "$CHARM_PATH/lib/common" + cp "$LOCAL_DB_PASSFILE" "$MIGRATION_TMPDIR/pgpass" +) + + +current_image="docker.0k.io/postgres:${current_image_version}-myc" +if ! docker_has_image "$current_image"; then + out=$(docker pull "$current_image" 2>&1) || { + err "Couldn't pull image $current_image:" + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " >&2 + exit 1 + } +fi + +migration_backup_dirs=() + +## In postgres, it seems it is not useful to have a path. So path is +## only one step here. + +failed= +while [[ "${#upgrade_path[@]}" != 0 ]]; do + image_version="${upgrade_path[0]}" + + if version_gt "$image_version" 12; then + docker_version=$(docker info --format '{{.ServerVersion}}') + if ! version_gt "$docker_version" 20.10.0; then + err "Sorry, ${DARKYELLOW}$SERVICE_NAME${NORMAL} ${WHITE}$image_version${NORMAL}" \ + "require docker version >= 20.10 (current: $docker_version)" + break + fi + fi + upgrade_path=("${upgrade_path[@]:1}") + + ## Prevent build of image instead of pulling it + + while true; do + + info "Upgrading step ${WHITE}$current_image_version${NORMAL} => ${WHITE}$image_version${NORMAL}" + rm -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log" && + touch "$MIGRATION_TMPDIR/dump-${current_image_version}.log" + rm -f "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" && + touch "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" + uuid=$(openssl rand -hex 16) + + ## Generate dump with pg_dumpall of current version: + + ( + { + cmd=( + compose --no-hooks + --add-compose-content="$SERVICE_NAME: + docker-compose: + image: docker.0k.io/postgres:${current_image_version}-myc + hostname: $SERVICE_NAME +" + run --rm --label "migration-uuid=$uuid" "$SERVICE_NAME" + ) + echo COMMAND: "${cmd[@]}" + "${cmd[@]}" 2>&1 + echo "ERRORLEVEL: $?" + } | tee "$MIGRATION_TMPDIR/dump-${current_image_version}.log" >/dev/null + ) & + pid="$!" + debug "Dumping container of $SERVICE_NAME in ${WHITE}$current_image_version${NORMAL}" \ + "launched with PID ${YELLOW}$pid${NORMAL}" + expected_stop="" + unexpected_stop="" + errlvl="" + while read -r line; do + case "$line" in + *" database system is ready to accept connections"*) + + migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}') + if [ -z "$migration_container" ]; then + err "Couldn't find migration container for $current_image_version" + echo " despite catched line stating postgres is listening." >&2 + break 3 + fi + ## dump all + debug "Start dumping postgres database of ${WHITE}$current_image_version${NORMAL}" + if ! out=$( + { + docker exec -u 70 "${migration_container}" pg_dumpall | + gzip > "$MIGRATION_TMPDIR/dump.sql.gz" + } 2>&1 ); then + err "Failed to dump postgres database." + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " + unexpected_stop="1" + break + fi + + out=$(docker stop "$migration_container" 2>&1) || { + err "Failed to stop the dump container $migration_container for $current_image_version:" + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " + break 3 + } + expected_stop="1" + debug "Interrupting server process after dump (${YELLOW}$pid${NORMAL})" >&2 + kill "$pid" + errlvl="0" + break + ;; + *"getaddrinfo(\"$SERVICE_NAME\") failed: Name or service not known"*) + err "Postgres issue with resolving '$SERVICE_NAME'" + failed=1 + break 3 + ;; + "ERRORLEVEL: "*) + unexpected_stop="1" + errlvl="${line#ERRORLEVEL: }" + err "Unexpected stop of postgres dump container with errorlevel $errlvl" + break + ;; + esac + done < <(tail -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log") + [ "$unexpected_stop" == "1" ] && { + migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}') + if [ -n "$migration_container" ]; then + err "Stopping dumping docker container '$migration_container' after failure" + docker stop "$migration_container" >/dev/null 2>&1 || true + fi + failed=1 + break + } + + [ "$expected_stop" == "1" ] && { + out=$(docker rmi "$current_image" 2>&1) || { + err "Failed to remove image $current_image" + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " + break 2 + } + } + next_image="docker.0k.io/postgres:${image_version}-myc" + if ! docker_has_image "$next_image"; then + out=$(docker pull "$next_image" 2>&1) || { + err "Couldn't pull image $next_image:" + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " >&2 + unexpected_stop=1 + break 2 + } + fi + rm -f "$MIGRATION_TMPDIR/dump-${current_image_version}.log" + uuid=$(openssl rand -hex 16) + mv "$SERVICE_DATASTORE"{,".migration-${current_image_version}-${uuid}"} 2>&1 || { + err "Couldn't move datastore to backup location for $SERVICE_NAME" + unexpected_stop=1 + break 2 + } + migration_backup_dirs+=("$SERVICE_DATASTORE.migration-${current_image_version}-${uuid}") + ( + { + cmd=( + compose --debug + --add-compose-content="$SERVICE_NAME: + docker-compose: + image: docker.0k.io/postgres:${image_version}-myc + hostname: $SERVICE_NAME +" + run --rm --label "migration-uuid=$uuid" "$SERVICE_NAME" + ) + echo COMMAND: "${cmd[@]}" + "${cmd[@]}" 2>&1 + echo "ERRORLEVEL: $?" + } | tee "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" >/dev/null + ) & + pid="$!" + debug "Migration launched with PID ${YELLOW}$pid${NORMAL}" + expected_stop="" + unexpected_stop="" + errlvl="" + while read -r line; do + case "$line" in + *" database system is ready to accept connections"*) + + migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}') + + ## dump all + debug "Start restoring in postgres database ${WHITE}$current_image_version${NORMAL}" + if ! out=$( + { + gunzip -c "$MIGRATION_TMPDIR/dump.sql.gz" | + docker exec -i -u 70 "${migration_container}" psql + } 2>&1 ); then + err "Failed to restore postgres database." + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " + unexpected_stop="1" + break + fi + + current_image="$next_image" + current_image_version="$image_version" + out=$(docker stop "$migration_container" 2>&1) || { + err "Failed to stop the migration container $migration_container for $image_version:" + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " + break 3 + } + expected_stop="1" + debug "Interrupting server process after migration (${YELLOW}$pid${NORMAL})" >&2 + kill "$pid" + errlvl="0" + break + ;; + *"getaddrinfo(\"$SERVICE_NAME\") failed: Name or service not known"*) + err "Postgres issue with resolving '$SERVICE_NAME'" + failed=1 + break 3 + ;; + *"Replication has not yet been configured"*) + warn "Postgres was not setup right away, could lead to issues" + ;; + "ERRORLEVEL: "*) + unexpected_stop="1" + errlvl="${line#ERRORLEVEL: }" + err "Unexpected stop of postgres migration container with errorlevel $errlvl" + failed=1 + break + ;; + esac + done < <(tail -f "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log") + + [ "$unexpected_stop" == "1" ] && { + migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}') + if [ -n "$migration_container" ]; then + err "Stopping dumping docker container '$migration_container' after failure" + docker stop "$migration_container" >/dev/null 2>&1 || true + fi + failed=1 + break + } + + [ "$expected_stop" == "1" ] && { + continue 2 + } + + ## + ## Damage control, there are some things we can solve + ## + + if grep "UPGRADE PROBLEM: Found an invalid featureCompatibilityVersion document" "$MIGRATION_TMPDIR/migration.log" >/dev/null 2>&1; then + if [ -z "$feature_comp" ]; then + feature_comp=1 + info "Detected featureCompatibilityVersion issue, forcing upgrade from base version" + echo " This will have for effect to reload the current version and set the featureCompatibilityVersion" >&2 + upgrade_path=("$current_image_version" "$image_version" "${upgrade_path[@]}") + break + else + err "Already tried to mitigate featureCompatibilityVersion without success..." + break 2 + fi + fi + err "Upgrade to ${WHITE}$image_version${NORMAL} ${DARKRED}failed${NORMAL}" + failed=1 + break 2 + done + +done + +if [ -z "$unexpected_stop" ] && [ -z "$expected_stop" ]; then + migration_container=$(docker ps --filter "label=migration-uuid=$uuid" --format '{{.ID}}') + if [ -n "$migration_container" ]; then + err "Stopping migration docker container '$migration_container' after failure" + docker stop "$migration_container" + fi +fi + +if [ -n "$failed" ]; then + if [ -e "$MIGRATION_TMPDIR/dump-${current_image_version}.log" ]; then + echo "${WHITE}Failing dump logs:${NORMAL}" >&2 + cat "$MIGRATION_TMPDIR/dump-${current_image_version}.log" | prefix " ${GRAY}|${NORMAL} " >&2 + fi + if [ -e "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" ]; then + echo "${WHITE}Failing migration logs:${NORMAL}" >&2 + cat "$MIGRATION_TMPDIR/migration-${current_image_version}_${image_version}.log" | prefix " ${GRAY}|${NORMAL} " >&2 + fi +fi + +if [ -n "$unexpected_stop" ]; then + if [ -e "$SERVICE_DATASTORE.migration-${current_image_version}-${uuid}" ]; then + if [ -e "$SERVICE_DATASTORE" ]; then + mv -v "$SERVICE_DATASTORE"{,".migration-${image_version}-${uuid}-failed"} || { + err "Couldn't move back datastore to backup location for $SERVICE_NAME" + } + fi + mv -v "$SERVICE_DATASTORE"{".migration-${current_image_version}-${uuid}",} || { + err "Couldn't move back datastore to backup location for $SERVICE_NAME" + } + fi + err "Migration failed unexpectedly." + exit 1 +fi + +if [ "$starting_image_version" == "$current_image_version" ]; then + warn "Database not migrated. Current version is still: ${WHITE}$current_image_version${NORMAL}" >&2 + exit 1 +fi + + +( + . "$CHARM_PATH/lib/common" + cp "$MIGRATION_TMPDIR/pgpass" "$LOCAL_DB_PASSFILE" +) || { + err "Couldn't restore postgres password file." + exit 1 +} + +for migration_backup_dir in "${migration_backup_dirs[@]}"; do + out=$(rm -rf "$migration_backup_dir") || { + warn "Couldn't remove backup directory $migration_backup_dir" + printf "%s\n" "$out" | prefix " ${GRAY}|${NORMAL} " + } +done +## This next line has to be on stdout, it is used by `vps` to get information +info "${GREEN}Successfully${NORMAL} upgraded from ${WHITE}$starting_image_version${NORMAL} to ${WHITE}$current_image_version${NORMAL}" 2>&1 + + + +if [ -z "$no_hint" ]; then + cat <&2 +Don't forget to force the version in your \`\`compose.yml\`\`. For instance: + + ${DARKYELLOW}$SERVICE_NAME${NORMAL}: + + ${DARKGRAY}# ...${NORMAL} + + ${WHITE}docker-compose${NORMAL}: + ${WHITE}image${NORMAL}: docker.0k.io/postgres:${current_image_version}-myc + + ${DARKGRAY}# ...${NORMAL} + +You could do this automatically with: + + yq eval -i ".$SERVICE_NAME.docker-compose.image = \"docker.0k.io/postgres:${current_image_version}-myc\" // \"\"" compose.yml + +And don't forget to run \`compose up\` afterwards. + +EOF +fi + +exit 0 \ No newline at end of file -- 2.30.2 From 3e0f6df89bff419b19ae2592aa5af0965200e3cd Mon Sep 17 00:00:00 2001 From: Boris Gallet Date: Tue, 21 Nov 2023 14:39:58 +0100 Subject: [PATCH 2/2] new: [matomo] new charm --- matomo/README.org | 9 +++++++ matomo/hooks/mysql_database-relation-joined | 21 +++++++++++++++ matomo/hooks/web_proxy-relation-joined | 13 +++++++++ matomo/metadata.yml | 30 +++++++++++++++++++++ 4 files changed, 73 insertions(+) create mode 100644 matomo/README.org create mode 100755 matomo/hooks/mysql_database-relation-joined create mode 100755 matomo/hooks/web_proxy-relation-joined create mode 100644 matomo/metadata.yml diff --git a/matomo/README.org b/matomo/README.org new file mode 100644 index 0000000..3095ce8 --- /dev/null +++ b/matomo/README.org @@ -0,0 +1,9 @@ +# -*- ispell-local-dictionary: "english" -*- + +* Info + +From: https://github.com/matomo-org/docker + +* Usage + +Config info : https://hub.docker.com/_/matomo \ No newline at end of file diff --git a/matomo/hooks/mysql_database-relation-joined b/matomo/hooks/mysql_database-relation-joined new file mode 100755 index 0000000..885413c --- /dev/null +++ b/matomo/hooks/mysql_database-relation-joined @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +PASSWORD="$(relation-get password)" +USER="$(relation-get user)" +DBNAME="$(relation-get dbname)" + + +config-add "\ +services: + $MASTER_BASE_SERVICE_NAME: + environment: + MYSQL_USER: \"$USER\" + MYSQL_PASSWORD: \"$PASSWORD\" + MYSQL_DATABASE: \"$DBNAME\" + MATOMO_DATABASE_HOST: \"$TARGET_SERVICE_NAME\" +" + + +info "Configured $SERVICE_NAME code for $TARGET_SERVICE_NAME access." diff --git a/matomo/hooks/web_proxy-relation-joined b/matomo/hooks/web_proxy-relation-joined new file mode 100755 index 0000000..5a8e89a --- /dev/null +++ b/matomo/hooks/web_proxy-relation-joined @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +DOMAIN=$(relation-get domain) || exit 1 + +# config-add "\ +# services: +# $MASTER_BASE_SERVICE_NAME: +# environment: + + +# " \ No newline at end of file diff --git a/matomo/metadata.yml b/matomo/metadata.yml new file mode 100644 index 0000000..1ff43e1 --- /dev/null +++ b/matomo/metadata.yml @@ -0,0 +1,30 @@ +docker-image: docker.0k.io/matomo:4.15.1-apache +data-resources: + - /var/www/html/ + +uses: + mysql-database: + #constraint: required | recommended | optional + #auto: pair | summon | none ## default: pair + constraint: required + auto: summon + solves: + database: "main storage" + web-proxy: + constraint: required + auto: summon + solves: + proxy: "Public access" + default-options: + target: !var-expand ${MASTER_BASE_SERVICE_NAME}:80 + log-rotate: + constraint: recommended + auto: pair + solves: + disk-leak: "/matomo/logs" + backup: + constraint: recommended + auto: pair + solves: + backup: "Automatic regular backup" + default-options: -- 2.30.2