From 80fbf42b078ae6ba31e7bc0856ec6b8a3552d296 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Wed, 21 Feb 2024 15:37:17 +0100 Subject: [PATCH] new: [compose-core] cache images on ``$COMPOSE_DOCKER_REGISTRY`` --- bin/compose-core | 351 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 303 insertions(+), 48 deletions(-) diff --git a/bin/compose-core b/bin/compose-core index 1a59610..54d2844 100755 --- a/bin/compose-core +++ b/bin/compose-core @@ -466,7 +466,7 @@ export -f cached_cmd_on_image cmd_on_base_image() { local service="$1" base_image shift - base_image=$(service_base_docker_image "$service") || return 1 + base_image=$(service_ensure_image_ready "$service") || return 1 docker run -i --rm --entrypoint /bin/bash "$base_image" -c "$*" } export -f cmd_on_base_image @@ -480,10 +480,7 @@ cached_cmd_on_base_image() { quick_cat_stdin < "$cache_file" return 0 fi - base_image=$(service_base_docker_image "$service") || return 1 - if ! docker_has_image "$base_image"; then - docker pull "$base_image" 1>&2 - fi + base_image=$(service_ensure_image_ready "$service") || return 1 result=$(cached_cmd_on_image "$base_image" "$@") || return 1 echo "$result" | tee "$cache_file" } @@ -500,7 +497,7 @@ docker_update() { shift shift ## this will build it if necessary - base_image=$(service_base_docker_image "$service") || return 1 + base_image=$(service_ensure_image_ready "$service") || return 1 ## XXXvlab: there are probably ways to avoid rebuilding that each time image_id="$(docker_image_id "$base_image")" || return 1 @@ -559,10 +556,7 @@ export -f docker_image_export_dir service_base_image_export_dir() { local service="$1" src="$2" dst="$3" base_image shift - base_image=$(service_base_docker_image "$service") || return 1 - if ! docker_has_image "$base_image"; then - docker pull "$base_image" - fi + base_image=$(service_ensure_image_ready "$service") || return 1 docker_image_export_dir "$base_image" "$src" "$dst" } export -f service_base_image_export_dir @@ -571,10 +565,7 @@ export -f service_base_image_export_dir service_base_image_id() { local service="$1" src="$2" dst="$3" base_image shift - base_image=$(service_base_docker_image "$service") || return 1 - if ! docker_has_image "$base_image"; then - docker pull "$base_image" - fi + base_image=$(service_ensure_image_ready "$service") || return 1 docker inspect "$base_image" --format="{{ .Id }}" } export -f service_base_image_id @@ -830,7 +821,8 @@ ensure_db_docker_running () { else verb "Database is not locked." if ! docker_has_image "$DOCKER_BASE_IMAGE"; then - docker pull "$DOCKER_BASE_IMAGE" + err "Unexpected missing docker image $DOCKER_BASE_IMAGE." + return 1 fi _set_server_db_params || return 1 @@ -1414,51 +1406,249 @@ get_service_def () { } export -f get_service_def +get_build_hash() { + local dir="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$1")" hash + if [ -e "$cache_file" ]; then + # debug "$FUNCNAME: cache hit ($*)" + cat "$cache_file" + return 0 + fi + ## Check that there's a Dockerfile in this directory + if [ ! -e "$dir/Dockerfile" ]; then + err "No 'Dockerfile' found in '$dir'." + return 1 + fi + ## use find to md5sum all files in the directory and make a final hash + hash=$(set -o pipefail; cd "$dir"; env -i find "." -type f -exec md5sum {} \; | + sort | md5sum | awk '{print $1}') || { + err "Failed to get hash for '$dir'." + return 1 + } + printf "%s" "$hash" | tee "$cache_file" + return $? +} +export -f get_build_hash + +### Query/Get cached image from registry +## +## Returns on stdout the name of the image if found, or an empty string if not +cache:image:registry:get() { + if [ -n "$COMPOSE_DOCKER_REGISTRY" ]; then + local charm="$1" hash="$2" service="$3" + local charm_image_name="cache/charm/$charm" + local charm_image="$charm_image_name:$hash" + + Elt "pulling ${DARKPINK}$charm${NORMAL} image from $COMPOSE_DOCKER_REGISTRY" >&2 + if out=$(docker pull "$COMPOSE_DOCKER_REGISTRY/$charm_image" 2>&1); then + docker tag "$COMPOSE_DOCKER_REGISTRY/$charm_image" "$charm_image" || { + err "Failed set image '$COMPOSE_DOCKER_REGISTRY/$charm_image' as '$charm_image'" \ + "for ${DARKYELLOW}$service${NORMAL}." + return 1 + } + print_info "found" >&2 + print_status success >&2 + Feed >&2 + printf "%s" "$charm_image" | tee "$cache_file" + return $? + fi + if [[ "$out" != *"manifest unknown"* ]]; then + print_status failure >&2 + Feed >&2 + err "Failed to pull image '$COMPOSE_DOCKER_REGISTRY/$charm_image'" \ + "for ${DARKYELLOW}$service${NORMAL}:" + e "$out" | prefix " ${GRAY}|${NORMAL} " >&2 + return 1 + fi + print_info "not found" >&2 + if test "$type_method" = "long"; then + __status="[${NOOP}ABSENT${NORMAL}]" + else + echo -n "${NOOP}" + shift; shift; + echo -n "$*${NORMAL}" + fi >&2 + Feed >&2 + fi +} +export -f cache:image:registry:get + +### Store cached image on registry +## +## Returns nothing +cache:image:registry:put() { + if [ -n "$COMPOSE_DOCKER_REGISTRY" ] && [ -n "$COMPOSE_PUSH_TO_REGISTRY" ]; then + local charm="$1" hash="$2" service="$3" + local charm_image_name="cache/charm/$charm" + local charm_image="$charm_image_name:$hash" + + Wrap -d "pushing ${DARKPINK}$charm${NORMAL} image to $COMPOSE_DOCKER_REGISTRY" <&2 +} +export -f cache:image:registry:put -## Return the base docker image name of a service -service_base_docker_image() { + +### Produce docker cached charm image 'cache/charm/$charm:$hash' +## +## Either by fetching it from a registry or by building it from a +## Dockerfile. +cache:image:produce() { + local type="$1" src="$2" charm="$3" hash="$4" service="$5" + local charm_image_name="cache/charm/$charm" + local charm_image="$charm_image_name:$hash" + + case "$type" in + fetch) + local specified_image="$src" + ## will not pull upstream image if already present locally + if ! docker_has_image "${specified_image}"; then + if ! out=$(docker pull "${specified_image}" 2>&1); then + err "Failed to pull image '$specified_image' for ${DARKYELLOW}$service${NORMAL}:" + echo "$out" | prefix " | " >&2 + return 1 + fi + fi + + # specified_image_id=$(docker_image_id "$specified_image") || return 1 + # charm_image_id= + # if docker_has_image "${image_dst}"; then + # charm_image_id=$(docker_image_id "${image_dst}") || return 1 + # fi + # if [ "$specified_image_id" != "$charm_image_id" ]; then + docker tag "$specified_image" "${charm_image}" || return 1 + # fi + ;; + build) + local service_build="$src" + Wrap -v -d "Building ${DARKPINK}$charm${NORMAL}:$hash image" -- \ + docker build "$service_build" -t "${charm_image}" >&2 || { + err "Failed to build image '${charm_image}' for ${DARKYELLOW}$service${NORMAL}." + return 1 + } + ;; + *) + err "Unknown type '$type'." + return 1 + ;; + esac +} +export -f cache:image:produce + +service_ensure_image_ready() { local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \ - master_service service_def service_image service_build service_dockerfile + master_service service_def service_image service_build service_dockerfile image \ + specified_image specified_image_id charm_image_name hash \ + service_quoted + if [ -e "$cache_file" ]; then - # debug "$FUNCNAME: cache hit ($*)" + #debug "$FUNCNAME: cache hit ($*)" cat "$cache_file" return 0 fi + if [ -z "$_CURRENT_DOCKER_COMPOSE" ]; then + err "${FUNCNAME[0]} is meant to be called after"\ + "\$_CURRENT_DOCKER_COMPOSE has been calculated." + echo " Called by:" >&2 + printf " - %s\n" "${FUNCNAME[@]:1}" >&2 + return 1 + fi + master_service="$(get_top_master_service_for_service "$service")" || { err "Could not compute master service for service $DARKYELLOW$service$NORMAL." return 1 } - service_def="$(get_service_def "$master_service")" || { - err "Could not get docker-compose service definition for $DARKYELLOW$master_service$NORMAL." - return 1 - } - if service_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null); then - printf "%s" "${service_image}" | tee "$cache_file" + if [ "$master_service" != "$service" ]; then + image=$(service_ensure_image_ready "$master_service") || return 1 + printf "%s" "$image" | tee "$cache_file" return $? fi - ## According to https://stackoverflow.com/questions/32230577 , if there's a build, - ## then the built image will get name ${project}_${service} - project=$(get_default_project_name) || return 1 - image_name="${project}_${master_service}" - if docker_has_image "$image_name"; then - printf "%s" "${image_name}" | tee "$cache_file" - return $? - fi - if ! service_build=$(echo "$service_def" | shyaml get-value build 2>/dev/null); then - err "Service $DARKYELLOW$service$NORMAL has no ${WHITE}image${NORMAL} nor ${WHITE}build${NORMAL} parameter." - echo "$service_def" >&2 + ## check if \$_CURRENT_DOCKER_COMPOSE's service def is already correctly setup + local charm="$(get_service_charm "$service")" || return 1 + local charm_image_name="cache/charm/$charm" || return 1 + local service_def="$(get_service_def "$service")" || { + err "Could not get docker-compose service definition for $DARKYELLOW$service$NORMAL." return 1 + } + local service_quoted=${service//./\\.} + + if specified_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null); then + if [ "$specified_image" == "$charm_image_name"* ]; then + ## Assume we already did the change + printf "%s" "$specified_image" | tee "$cache_file" + return 0 + fi + if [[ "$specified_image" == "${COMPOSE_DOCKER_REGISTRY}/"* ]]; then + if ! docker_has_image "${specified_image}"; then + Wrap "${wrap_opts[@]}" \ + -v -d "pulling ${DARKPINK}$charm${NORMAL}'s specified image from $COMPOSE_DOCKER_REGISTRY" -- \ + docker pull "${specified_image}" >&2 || return 1 + else + if [ -n "$DEBUG" ]; then + Elt "using local ${DARKPINK}$charm${NORMAL}'s specified image from $COMPOSE_DOCKER_REGISTRY" >&2 + print_status noop >&2 + Feed >&2 + fi + fi + ## Already on the cache server + printf "%s" "$specified_image" | tee "$cache_file" + return 0 + fi + src="$specified_image" + hash=$(echo "$specified_image" | md5sum | cut -f 1 -d " ") || return 1 + type=fetch + + ## replace image by charm image + yq -i ".services.[\"${service_quoted}\"].image = \"${charm_image_name}:${hash}\"" \ + "$_CURRENT_DOCKER_COMPOSE" || return 1 + else + + if ! src=$(echo "$service_def" | shyaml get-value build 2>/dev/null); then + err "Service $DARKYELLOW$service$NORMAL has no ${WHITE}image${NORMAL} nor ${WHITE}build${NORMAL} parameter." + echo "$service_def" >&2 + return 1 + fi + + ## According to https://stackoverflow.com/questions/32230577 , if there's a build, + ## then the built image will get name ${project}_${service} + hash=$(get_build_hash "$src") || return 1 + type=build + ## delete build key from service_def and add image to charm_image_name + yq -i "del(.services.[\"${service_quoted}\"].build) | + .services.[\"${service_quoted}\"].image = \"${charm_image_name}:${hash}\"" \ + "$_CURRENT_DOCKER_COMPOSE" || return 1 + + if docker_has_image "${charm_image_name}:${hash}"; then + if [ -n "$DEBUG" ]; then + Elt "using ${DARKPINK}$charm${NORMAL}'s image from local cache" >&2 + print_status noop >&2 + Feed >&2 + fi + cache:image:registry:put "$charm" "$hash" "$service" || return 1 + printf "%s" "${charm_image_name}:${hash}" | tee "$cache_file" + return $? + fi fi - docker build "$service_build" -t "${image_name}" >&2 || { - err "Failed to build image '${image_name}' for ${DARKYELLOW}$service${NORMAL}." + ## Can we pull it ? Let's check on $COMPOSE_DOCKER_REGISTRY + + img=$(cache:image:registry:get "$charm" "$hash" "$service") || { + err "Failed to get image '$charm_image_name:$hash' from registry for ${DARKYELLOW}$service${NORMAL}." return 1 } - printf "%s" "${image_name}" | tee "$cache_file" + [ -n "$img" ] && { + printf "%s" "$img" | tee "$cache_file" + return $? + } + + cache:image:produce "$type" "$src" "$charm" "$hash" "$service" || return 1 + cache:image:registry:put "$charm" "$hash" "$service" || return 1 + printf "%s" "${charm_image_name}:$hash" | tee "$cache_file" return $? } -export -f service_base_docker_image +export -f service_ensure_image_ready get_charm_relation_def () { @@ -1613,6 +1803,31 @@ get_ordered_service_dependencies() { export -f get_ordered_service_dependencies +run_service_acquire_images () { + local service subservice subservices loaded + declare -A loaded + for service in "$@"; do + subservices=$(get_ordered_service_dependencies "$service") || return 1 + for subservice in $subservices; do + if [ "${loaded[$subservice]}" ]; then + ## Prevent double inclusion of same service if this + ## service is deps of two or more of your + ## requirements. + continue + fi + + type=$(get_service_type "$subservice") || return 1 + MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1 + if [ "$type" != "stub" ]; then + DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$MASTER_BASE_SERVICE_NAME") || return 1 + fi + + loaded[$subservice]=1 + done + done + return 0 +} + run_service_hook () { local action="$1" service subservice subservices loaded shift @@ -1635,7 +1850,7 @@ run_service_hook () { MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1 MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1 if [ "$type" != "stub" ]; then - DOCKER_BASE_IMAGE=$(service_base_docker_image "$MASTER_BASE_SERVICE_NAME") || return 1 + DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$MASTER_BASE_SERVICE_NAME") || return 1 fi Wrap "${wrap_opts[@]}" -d "running $YELLOW$action$NORMAL hook of $DARKYELLOW$subservice$NORMAL in charm $DARKPINK$charm$NORMAL" < $DARKYELLOW$target_service$NORMAL" < $DARKYELLOW$target_service$NORMAL" <