0k
/
0k-compose
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

new: [compose-core] cache images on ``$COMPOSE_DOCKER_REGISTRY``

cache-relation
Valentin Lab 9 months ago
parent
8ea4310945
commit
80fbf42b07
1 changed files with 303 additions and 48 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 341
      bin/compose-core

341
bin/compose-core
View File

@ -466,7 +466,7 @@ export -f cached_cmd_on_image
cmd_on_base_image() {
local service="$1" base_image
shift
base_image=$(service_base_docker_image "$service") || return 1
base_image=$(service_ensure_image_ready "$service") || return 1
docker run -i --rm --entrypoint /bin/bash "$base_image" -c "$*"
}
export -f cmd_on_base_image
@ -480,10 +480,7 @@ cached_cmd_on_base_image() {
quick_cat_stdin < "$cache_file"
return 0
fi
base_image=$(service_base_docker_image "$service") || return 1
if ! docker_has_image "$base_image"; then
docker pull "$base_image" 1>&2
fi
base_image=$(service_ensure_image_ready "$service") || return 1
result=$(cached_cmd_on_image "$base_image" "$@") || return 1
echo "$result" | tee "$cache_file"
}
@ -500,7 +497,7 @@ docker_update() {
shift
shift
## this will build it if necessary
base_image=$(service_base_docker_image "$service") || return 1
base_image=$(service_ensure_image_ready "$service") || return 1
## XXXvlab: there are probably ways to avoid rebuilding that each time
image_id="$(docker_image_id "$base_image")" || return 1
@ -559,10 +556,7 @@ export -f docker_image_export_dir
service_base_image_export_dir() {
local service="$1" src="$2" dst="$3" base_image
shift
base_image=$(service_base_docker_image "$service") || return 1
if ! docker_has_image "$base_image"; then
docker pull "$base_image"
fi
base_image=$(service_ensure_image_ready "$service") || return 1
docker_image_export_dir "$base_image" "$src" "$dst"
}
export -f service_base_image_export_dir
@ -571,10 +565,7 @@ export -f service_base_image_export_dir
service_base_image_id() {
local service="$1" src="$2" dst="$3" base_image
shift
base_image=$(service_base_docker_image "$service") || return 1
if ! docker_has_image "$base_image"; then
docker pull "$base_image"
fi
base_image=$(service_ensure_image_ready "$service") || return 1
docker inspect "$base_image" --format="{{ .Id }}"
}
export -f service_base_image_id
@ -830,7 +821,8 @@ ensure_db_docker_running () {
else
verb "Database is not locked."
if ! docker_has_image "$DOCKER_BASE_IMAGE"; then
docker pull "$DOCKER_BASE_IMAGE"
err "Unexpected missing docker image $DOCKER_BASE_IMAGE."
return 1
fi
_set_server_db_params || return 1
@ -1414,51 +1406,249 @@ get_service_def () {
}
export -f get_service_def
get_build_hash() {
local dir="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$(H "$1")" hash
if [ -e "$cache_file" ]; then
# debug "$FUNCNAME: cache hit ($*)"
cat "$cache_file"
return 0
fi
## Check that there's a Dockerfile in this directory
if [ ! -e "$dir/Dockerfile" ]; then
err "No 'Dockerfile' found in '$dir'."
return 1
fi
## use find to md5sum all files in the directory and make a final hash
hash=$(set -o pipefail; cd "$dir"; env -i find "." -type f -exec md5sum {} \; |
sort | md5sum | awk '{print $1}') || {
err "Failed to get hash for '$dir'."
return 1
}
printf "%s" "$hash" | tee "$cache_file"
return $?
}
export -f get_build_hash
### Query/Get cached image from registry
##
## Returns on stdout the name of the image if found, or an empty string if not
cache:image:registry:get() {
if [ -n "$COMPOSE_DOCKER_REGISTRY" ]; then
local charm="$1" hash="$2" service="$3"
local charm_image_name="cache/charm/$charm"
local charm_image="$charm_image_name:$hash"
Elt "pulling ${DARKPINK}$charm${NORMAL} image from $COMPOSE_DOCKER_REGISTRY" >&2
if out=$(docker pull "$COMPOSE_DOCKER_REGISTRY/$charm_image" 2>&1); then
docker tag "$COMPOSE_DOCKER_REGISTRY/$charm_image" "$charm_image" || {
err "Failed set image '$COMPOSE_DOCKER_REGISTRY/$charm_image' as '$charm_image'" \
"for ${DARKYELLOW}$service${NORMAL}."
return 1
}
print_info "found" >&2
print_status success >&2
Feed >&2
printf "%s" "$charm_image" | tee "$cache_file"
return $?
fi
if [[ "$out" != *"manifest unknown"* ]]; then
print_status failure >&2
Feed >&2
err "Failed to pull image '$COMPOSE_DOCKER_REGISTRY/$charm_image'" \
"for ${DARKYELLOW}$service${NORMAL}:"
e "$out" | prefix " ${GRAY}|${NORMAL} " >&2
return 1
fi
print_info "not found" >&2
if test "$type_method" = "long"; then
__status="[${NOOP}ABSENT${NORMAL}]"
else
echo -n "${NOOP}"
shift; shift;
echo -n "$*${NORMAL}"
fi >&2
Feed >&2
fi
}
export -f cache:image:registry:get
### Store cached image on registry
##
## Returns nothing
cache:image:registry:put() {
if [ -n "$COMPOSE_DOCKER_REGISTRY" ] && [ -n "$COMPOSE_PUSH_TO_REGISTRY" ]; then
local charm="$1" hash="$2" service="$3"
local charm_image_name="cache/charm/$charm"
local charm_image="$charm_image_name:$hash"
Wrap -d "pushing ${DARKPINK}$charm${NORMAL} image to $COMPOSE_DOCKER_REGISTRY" <<EOF || return 1
docker tag "$charm_image" "$COMPOSE_DOCKER_REGISTRY/$charm_image" &&
docker push "$COMPOSE_DOCKER_REGISTRY/$charm_image"
EOF
fi >&2
}
export -f cache:image:registry:put
### Produce docker cached charm image 'cache/charm/$charm:$hash'
##
## Either by fetching it from a registry or by building it from a
## Dockerfile.
cache:image:produce() {
local type="$1" src="$2" charm="$3" hash="$4" service="$5"
local charm_image_name="cache/charm/$charm"
local charm_image="$charm_image_name:$hash"
case "$type" in
fetch)
local specified_image="$src"
## will not pull upstream image if already present locally
if ! docker_has_image "${specified_image}"; then
if ! out=$(docker pull "${specified_image}" 2>&1); then
err "Failed to pull image '$specified_image' for ${DARKYELLOW}$service${NORMAL}:"
echo "$out" | prefix " | " >&2
return 1
fi
fi
# specified_image_id=$(docker_image_id "$specified_image") || return 1
# charm_image_id=
# if docker_has_image "${image_dst}"; then
# charm_image_id=$(docker_image_id "${image_dst}") || return 1
# fi
# if [ "$specified_image_id" != "$charm_image_id" ]; then
docker tag "$specified_image" "${charm_image}" || return 1
# fi
;;
build)
local service_build="$src"
Wrap -v -d "Building ${DARKPINK}$charm${NORMAL}:$hash image" -- \
docker build "$service_build" -t "${charm_image}" >&2 || {
err "Failed to build image '${charm_image}' for ${DARKYELLOW}$service${NORMAL}."
return 1
}
;;
*)
err "Unknown type '$type'."
return 1
;;
esac
}
export -f cache:image:produce
## Return the base docker image name of a service
service_base_docker_image() {
service_ensure_image_ready() {
local service="$1" cache_file="$state_tmpdir/$FUNCNAME.cache.$1" \
master_service service_def service_image service_build service_dockerfile
master_service service_def service_image service_build service_dockerfile image \
specified_image specified_image_id charm_image_name hash \
service_quoted
if [ -e "$cache_file" ]; then
#debug "$FUNCNAME: cache hit ($*)"
cat "$cache_file"
return 0
fi
if [ -z "$_CURRENT_DOCKER_COMPOSE" ]; then
err "${FUNCNAME[0]} is meant to be called after"\
"\$_CURRENT_DOCKER_COMPOSE has been calculated."
echo " Called by:" >&2
printf " - %s\n" "${FUNCNAME[@]:1}" >&2
return 1
fi
master_service="$(get_top_master_service_for_service "$service")" || {
err "Could not compute master service for service $DARKYELLOW$service$NORMAL."
return 1
}
service_def="$(get_service_def "$master_service")" || {
err "Could not get docker-compose service definition for $DARKYELLOW$master_service$NORMAL."
if [ "$master_service" != "$service" ]; then
image=$(service_ensure_image_ready "$master_service") || return 1
printf "%s" "$image" | tee "$cache_file"
return $?
fi
## check if \$_CURRENT_DOCKER_COMPOSE's service def is already correctly setup
local charm="$(get_service_charm "$service")" || return 1
local charm_image_name="cache/charm/$charm" || return 1
local service_def="$(get_service_def "$service")" || {
err "Could not get docker-compose service definition for $DARKYELLOW$service$NORMAL."
return 1
}
if service_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null); then
printf "%s" "${service_image}" | tee "$cache_file"
return $?
local service_quoted=${service//./\\.}
if specified_image=$(echo "$service_def" | shyaml get-value image 2>/dev/null); then
if [ "$specified_image" == "$charm_image_name"* ]; then
## Assume we already did the change
printf "%s" "$specified_image" | tee "$cache_file"
return 0
fi
if [[ "$specified_image" == "${COMPOSE_DOCKER_REGISTRY}/"* ]]; then
if ! docker_has_image "${specified_image}"; then
Wrap "${wrap_opts[@]}" \
-v -d "pulling ${DARKPINK}$charm${NORMAL}'s specified image from $COMPOSE_DOCKER_REGISTRY" -- \
docker pull "${specified_image}" >&2 || return 1
else
if [ -n "$DEBUG" ]; then
Elt "using local ${DARKPINK}$charm${NORMAL}'s specified image from $COMPOSE_DOCKER_REGISTRY" >&2
print_status noop >&2
Feed >&2
fi
fi
## Already on the cache server
printf "%s" "$specified_image" | tee "$cache_file"
return 0
fi
src="$specified_image"
hash=$(echo "$specified_image" | md5sum | cut -f 1 -d " ") || return 1
type=fetch
## replace image by charm image
yq -i ".services.[\"${service_quoted}\"].image = \"${charm_image_name}:${hash}\"" \
"$_CURRENT_DOCKER_COMPOSE" || return 1
else
if ! src=$(echo "$service_def" | shyaml get-value build 2>/dev/null); then
err "Service $DARKYELLOW$service$NORMAL has no ${WHITE}image${NORMAL} nor ${WHITE}build${NORMAL} parameter."
echo "$service_def" >&2
return 1
fi
## According to https://stackoverflow.com/questions/32230577 , if there's a build,
## then the built image will get name ${project}_${service}
project=$(get_default_project_name) || return 1
image_name="${project}_${master_service}"
if docker_has_image "$image_name"; then
printf "%s" "${image_name}" | tee "$cache_file"
hash=$(get_build_hash "$src") || return 1
type=build
## delete build key from service_def and add image to charm_image_name
yq -i "del(.services.[\"${service_quoted}\"].build) |
.services.[\"${service_quoted}\"].image = \"${charm_image_name}:${hash}\"" \
"$_CURRENT_DOCKER_COMPOSE" || return 1
if docker_has_image "${charm_image_name}:${hash}"; then
if [ -n "$DEBUG" ]; then
Elt "using ${DARKPINK}$charm${NORMAL}'s image from local cache" >&2
print_status noop >&2
Feed >&2
fi
cache:image:registry:put "$charm" "$hash" "$service" || return 1
printf "%s" "${charm_image_name}:${hash}" | tee "$cache_file"
return $?
fi
if ! service_build=$(echo "$service_def" | shyaml get-value build 2>/dev/null); then
err "Service $DARKYELLOW$service$NORMAL has no ${WHITE}image${NORMAL} nor ${WHITE}build${NORMAL} parameter."
echo "$service_def" >&2
return 1
fi
docker build "$service_build" -t "${image_name}" >&2 || {
err "Failed to build image '${image_name}' for ${DARKYELLOW}$service${NORMAL}."
## Can we pull it ? Let's check on $COMPOSE_DOCKER_REGISTRY
img=$(cache:image:registry:get "$charm" "$hash" "$service") || {
err "Failed to get image '$charm_image_name:$hash' from registry for ${DARKYELLOW}$service${NORMAL}."
return 1
}
printf "%s" "${image_name}" | tee "$cache_file"
[ -n "$img" ] && {
printf "%s" "$img" | tee "$cache_file"
return $?
}
cache:image:produce "$type" "$src" "$charm" "$hash" "$service" || return 1
cache:image:registry:put "$charm" "$hash" "$service" || return 1
printf "%s" "${charm_image_name}:$hash" | tee "$cache_file"
return $?
}
export -f service_base_docker_image
export -f service_ensure_image_ready
get_charm_relation_def () {
@ -1613,6 +1803,31 @@ get_ordered_service_dependencies() {
export -f get_ordered_service_dependencies
run_service_acquire_images () {
local service subservice subservices loaded
declare -A loaded
for service in "$@"; do
subservices=$(get_ordered_service_dependencies "$service") || return 1
for subservice in $subservices; do
if [ "${loaded[$subservice]}" ]; then
## Prevent double inclusion of same service if this
## service is deps of two or more of your
## requirements.
continue
fi
type=$(get_service_type "$subservice") || return 1
MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
if [ "$type" != "stub" ]; then
DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$MASTER_BASE_SERVICE_NAME") || return 1
fi
loaded[$subservice]=1
done
done
return 0
}
run_service_hook () {
local action="$1" service subservice subservices loaded
shift
@ -1635,7 +1850,7 @@ run_service_hook () {
MASTER_BASE_SERVICE_NAME=$(get_top_master_service_for_service "$subservice") || return 1
MASTER_BASE_CHARM_NAME=$(get_service_charm "$MASTER_BASE_SERVICE_NAME") || return 1
if [ "$type" != "stub" ]; then
DOCKER_BASE_IMAGE=$(service_base_docker_image "$MASTER_BASE_SERVICE_NAME") || return 1
DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$MASTER_BASE_SERVICE_NAME") || return 1
fi
Wrap "${wrap_opts[@]}" -d "running $YELLOW$action$NORMAL hook of $DARKYELLOW$subservice$NORMAL in charm $DARKPINK$charm$NORMAL" <<EOF || return 1
@ -1990,7 +2205,7 @@ _run_service_relation () {
RELATION_CONFIG="$relation_dir/config_provider"
type=$(get_service_type "$target_service") || return 1
if [ "$type" != "stub" ]; then
DOCKER_BASE_IMAGE=$(service_base_docker_image "$target_service") || return 1
DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$target_service") || return 1
fi
export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA
{
@ -2023,7 +2238,7 @@ _run_service_relation () {
"for $DARKYELLOW$service$NORMAL (charm $DARKPINK$charm$NORMAL)"
RELATION_CONFIG="$relation_dir/config_providee"
RELATION_DATA="$(cat "$RELATION_DATA_FILE")"
DOCKER_BASE_IMAGE=$(service_base_docker_image "$service") || return 1
DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$service") || return 1
export DOCKER_BASE_IMAGE RELATION_CONFIG RELATION_DATA
{
(
@ -2948,7 +3163,7 @@ run_service_relations () {
RELATION_TARGET_COMPOSE_DEF=$(get_compose_service_def "$target_service") || return 1
export RELATION_TARGET_COMPOSE_DEF MASTER_TARGET_{CHARM,SERVICE}_NAME
Wrap "${wrap_opts[@]}" -d "Building $DARKYELLOW$subservice$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW$target_service$NORMAL" <<EOF || return 1
Wrap "${wrap_opts[@]}" -d "building $DARKYELLOW$subservice$NORMAL --$DARKBLUE$relation_name$NORMAL--> $DARKYELLOW$target_service$NORMAL" <<EOF || return 1
_run_service_relation "$relation_name" "$subservice" "$target_service" "\$relation_config"
EOF
done < <(get_service_relations "$subservice") || return 1
@ -2982,7 +3197,7 @@ _run_service_action_direct() {
export ACTION_NAME=$action
export ACTION_SCRIPT_PATH="$action_script_path"
export CONTAINER_NAME=$(get_top_master_service_for_service "$service")
export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME")
export DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$CONTAINER_NAME")
export SERVICE_DATASTORE="$DATASTORE/$service"
export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
exname="$exname $ACTION_NAME $SERVICE_NAME" \
@ -3016,7 +3231,7 @@ _run_service_action_relation() {
export ACTION_NAME=$action
export ACTION_SCRIPT_PATH="$action_script_path"
export CONTAINER_NAME=$(get_top_master_service_for_service "$service")
export DOCKER_BASE_IMAGE=$(service_base_docker_image "$CONTAINER_NAME")
export DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$CONTAINER_NAME")
export SERVICE_DATASTORE="$DATASTORE/$service"
export SERVICE_CONFIGSTORE="$CONFIGSTORE/$service"
exname="$exname $ACTION_NAME $SERVICE_NAME" \
@ -3508,7 +3723,7 @@ switch_to_relation_service() {
export SERVICE_NAME="$ts"
export SERVICE_DATASTORE="$DATASTORE/$SERVICE_NAME"
DOCKER_BASE_IMAGE=$(service_base_docker_image "$SERVICE_NAME")
DOCKER_BASE_IMAGE=$(service_ensure_image_ready "$SERVICE_NAME")
export DOCKER_BASE_IMAGE
target_charm=$(get_service_charm "$ts") || return 1
@ -4060,6 +4275,7 @@ display_help() {
print_help
echo "${WHITE}Usage${NORMAL}:"
echo " $usage"
echo " $usage cache {clean|clear}"
echo "${WHITE}Options${NORMAL}:"
echo " -h, --help Print this message and quit"
echo " (ignoring any other options)"
@ -4088,6 +4304,7 @@ display_help() {
echo " --add-compose-content, -Y YAML"
echo " Will merge some direct YAML with the current compose"
echo " -c, --color Force color mode (default is to detect if in tty mode)"
echo " --push-builds Will push cached docker images to docker cache registry"
get_docker_compose_opts_help | remove_options_in_option_help_msg --version --help --verbose |
filter_docker_compose_help_message
@ -4280,7 +4497,7 @@ export -f cached_wget
trap_add "EXIT" clean_cache
export COMPOSE_DOCKER_REGISTRY="${COMPOSE_DOCKER_REGISTRY:-docker.0k.io}"
if [ -z "$DISABLE_SYSTEM_CONFIG_FILE" ]; then
if [ -r /etc/default/charm ]; then
@ -4408,6 +4625,9 @@ while read-0 arg; do
rebuild_relations_to_service+=("$value")
shift
;;
--push-builds)
export COMPOSE_PUSH_TO_REGISTRY=1
;;
--debug|-d)
export DEBUG=true
export VERBOSE=true
@ -4518,6 +4738,38 @@ while read-0 arg; do
shift
done < <(cla.normalize "$@")
case "$action" in
cache)
case "${remainder_args[0]}" in
clean)
clean_cache
exit 0
;;
clear)
[ -n "$CACHEDIR" ] || die "No cache directory defined."
[ -d "$CACHEDIR" ] || die "Cache directory '$CACHEDIR' doesn't exists."
Wrap "${wrap_opts[@]}" -v -d "clear cache directory" -- rm -rf "$CACHEDIR/"*
## clear all docker caches
## image name are like '[$COMPOSE_DOCKER_REGISTRY]cache/charm/CHARM_NAME:HASH'
Wrap "${wrap_opts[@]}" -v -d "clear docker cache" <<EOF
docker images --format "{{.Repository}}:{{.Tag}}" |
egrep "^($COMPOSE_DOCKER_REGISTRY/)?cache/charm/[a-zA-Z0-9._-]+:[0-9a-f]{32,32}$" |
while read -r image; do
docker rmi "\$image" || true
done
EOF
exit 0
;;
*)
err "Unknown cache command: ${DARKCYAN}${remainder_args[0]}${NORMAL}"
exit 1
;;
esac
;;
esac
export compose_contents
[ "${services_args[*]}" ] && debug " ${DARKWHITE}Services:$NORMAL ${DARKYELLOW}${services_args[*]}$NORMAL"
@ -4678,6 +4930,9 @@ if [ -n "$full_init" ]; then
if [[ -z "$no_init" && -z "$no_hooks" ]]; then
Section setup host resources
setup_host_resources "${services_args[@]}" || exit 1
Section "acquire charm's images"
run_service_acquire_images "${services_args[@]}" || exit 1
Feed
Section initialisation
run_service_hook init "${services_args[@]}" || exit 1
fi

Write Preview
Loading…
Cancel
Save