fix: [letsencrypt] use action ``crt {renew,create}`` to manage properly renewal.

6 years ago · 6ac5bf3656
15 changed files with 1444 additions and 100 deletions
--- a/apache/lib/common
+++ b/apache/lib/common
@ -359,8 +359,8 @@ ssl_plugin_cert-provider_prepare() {
    else
        server_aliases=()
    fi
    compose --debug --add-compose-content "$service_config" run --rm --service-ports "$service" \
            crt create "$domain" "${server_aliases[@]}" || {
    compose --debug --add-compose-content "$service_config" crt "$service" \
            create "$domain" "${server_aliases[@]}" || {
        err "Failed to launch letsencrypt for certificate creation."
        return 1
    }
--- a/apache/test/vhost_cert_provider
+++ b/apache/test/vhost_cert_provider
@ -203,7 +203,7 @@ relation-set apache-custom-rules:
  |   RewriteCond %{HTTPS} off
  |   RewriteRule ^ https://%{HTTP_HOST}%{REQUEST_URI} [R=302,L,QSA]
  |"
 is out reg 'Calling: compose .*foo: options: <merge_yaml_str\(.a., .12., )>.*run --rm --service-ports foo.*'
 is out reg 'Calling: compose .*foo: options: <merge_yaml_str\(.a., .12., )>.*crt foo create www.example.com'
 is out part 'config-add
  | $SERVICE_NAME:
  |   volumes:
@ -254,8 +254,8 @@ relation-set apache-custom-rules:
  |   RewriteCond %{HTTPS} off
  |   RewriteRule ^ https://%{HTTP_HOST}%{REQUEST_URI} [R=302,L,QSA]
  |"
 is out reg 'Calling: compose .*foo: options: <merge_yaml_str\(.a., .12., )>.*run --rm --service-ports foo crt create www.example.com\s*
 ' RTRIM
 is out reg 'Calling: compose .*foo: options: <merge_yaml_str\(.a., .12., )>.*crt foo create www.example.com\s+
 '
 is out part 'config-add
  | $SERVICE_NAME:
  |   volumes:
@ -315,7 +315,7 @@ relation-set apache-custom-rules:
  |   RewriteCond %{HTTPS} off
  |   RewriteRule ^ https://%{HTTP_HOST}%{REQUEST_URI} [R=302,L,QSA]
  |"
 is out reg 'Calling: compose .*foo: options: <merge_yaml_str\(.a., .12., )>.*run --rm --service-ports foo crt create www.example.com example.fr example.de\s+
 is out reg 'Calling: compose .*foo: options: <merge_yaml_str\(.a., .12., )>.*crt foo create www.example.com example.fr example.de\s+
 '
 is out part 'config-add
  | $SERVICE_NAME:
--- a/letsencrypt/actions/crt
+++ b/letsencrypt/actions/crt
@ -0,0 +1,53 @@
 #!/bin/bash
 if [ -z "$SERVICE_DATASTORE" ]; then
    echo "This script is meant to be run through 'compose' to work properly." >&2
    exit 1
 fi
 . /etc/shlib
 include parse
 include pretty
 . $CHARM_PATH/lib/common
 usage="
  $exname [-h|--help]
  $exname create MAIN_DOMAIN [DOMAINS..]
  $exname renew
 "
 if [ "$#" == 0 ]; then
    err "Please specify an action"
    print_usage
    exit 1
 fi
 while [ "$1" ]; do
    case "$1" in
        "--help"|"-h")
            print_usage
            exit 0
            ;;
        renew)
            exname="$exname $1"
            shift
            crt_renew "$@"
            exit $?
            ;;
        create)
            exname="$exname $1"
            shift
            crt_create "$@"
            exit $?
            ;;
        *)
            err "Wrong argument"
            print_usage
            exit 1
            ;;
    esac
    shift
 done
--- a/letsencrypt/hooks/dc-pre-run
+++ b/letsencrypt/hooks/dc-pre-run
@ -1,77 +0,0 @@
 #!/bin/bash
 ## Init is run on host
 ## For now it is run every time the script is launched, but
 ## it should be launched only once after build.
 ## Accessible variables are:
 ## - SERVICE_NAME        Name of current service
 ## - DOCKER_BASE_IMAGE   Base image from which this service might be built if any
 ## - SERVICE_DATASTORE           Location on host of the DATASTORE of this service
 ## - SERVICE_CONFIGSTORE         Location on host of the CONFIGSTORE of this service
 aimport remainder_args
 case "${remainder_args[@]:0:2}" in
    "crt info"|"crt list")
        exit 0
        ;;
 esac
 . lib/common || exit 1
 set -e
 service_def=$(get_compose_service_def "$SERVICE_NAME")
 config="
 $SERVICE_NAME:
  environment:
 "
 if USER_EMAIL=$(echo "$service_def" | shyaml get-value options.email 2>/dev/null); then
    config+="    LETSENCRYPT_USER_MAIL: $USER_EMAIL"
 fi
 if environment_def="$(printf "%s" "$service_def" | shyaml -y get-value options.env 2>/dev/null)"; then
    while read-0 key value; do
        config+="$(printf "\n    %s: %s" "$key" "$value")"
    done < <(printf "%s" "$environment_def" | yaml_opt_bash_env_ignore_first_level LEXICON)
    if ! provider=$(printf "%s" "$environment_def" | shyaml -y get-value provider 2>/dev/null); then
        provider=
        ## If no provider is given, we fallback on the first found
        while read-0 key value; do
            [[ "$(echo "$value" | shyaml get-type)" == "struct" ]] && {
                provider="$key"
                break
            }
        done < <(echo "$environment_def" | shyaml key-values-0)
        warn "No ${WHITE}provider${NORMAL} key given, had to infer it, chose '$key'."
    fi
    config+=$(echo -en "\n    LEXICON_PROVIDER: $provider")
 fi
 if ! challenge_type=$(printf "%s" "$service_def" | shyaml get-value "options.challenge-type" 2>/dev/null); then
    warn "No ${WHITE}challenge-type${NORMAL} provided, defaulting to 'http'."
    challenge_type=http
 fi
 config+=$(echo -en "\n    CHALLENGE_TYPE: $challenge_type")
 if will_need_http_access; then
    while read container_id; do
        info "Attempting to clear port 80 by stopping $container_id"
        docker stop -t 5 "$container_id"
    done < <(docker ps \
                    --filter label="compose.project=$PROJECT_NAME" \
                    --filter publish=80 \
                    --format "{{.ID}}"
            )
    config+=$(echo -en "\n  ports:
    - \"0.0.0.0:80:80\"")
 fi
 init-config-add "$config"
 mkdir -p "$SERVICE_DATASTORE/etc/letsencrypt"
--- a/letsencrypt/hooks/schedule_command-relation-joined
+++ b/letsencrypt/hooks/schedule_command-relation-joined
@ -26,7 +26,7 @@ fi
 ##   'cron' container.
 file_put "$DST" <<EOF
 $schedule     root  lock $label -D -p 10 -c "\
    dc run --rm $SERVICE_NAME crt renew" 2>&1 | ts '\%F \%T \%Z' >> $LOCAL_LOG
    compose crt $SERVICE_NAME renew" 2>&1 | ts '\%F \%T \%Z' >> $LOCAL_LOG
 EOF
 chmod +x "$DST"
--- a/letsencrypt/lib/common
+++ b/letsencrypt/lib/common
@ -24,25 +24,340 @@ yaml_opt_bash_env_ignore_first_level() {
 }
 get_dc_env() {
    local cfg="$1" action="$2" domain="$3"
    config="\
 $SERVICE_NAME:
  docker-compose:
    environment:"
    if USER_EMAIL=$(echo "$cfg" | shyaml get-value email 2>/dev/null); then
        config+=$'\n'"      LETSENCRYPT_USER_MAIL: $USER_EMAIL"
    fi
    if environment_def="$(printf "%s" "$cfg" | shyaml -y get-value env 2>/dev/null)"; then
        while read-0 key value; do
            config+="$(printf "\n      %s: %s" "$key" "$value")"
        done < <(e "$environment_def" | yaml_opt_bash_env_ignore_first_level LEXICON)
        if ! provider=$(e "$environment_def" | shyaml get-value provider 2>/dev/null); then
            provider=
            ## If no provider is given, we fallback on the first found
            while read-0 key value; do
                [[ "$(echo "$value" | shyaml get-type)" == "struct" ]] && {
                    provider="$key"
                    break
                }
            done < <(e "$environment_def" | shyaml key-values-0)
            warn "No ${WHITE}provider${NORMAL} key given, had to infer it, chose '$key'."
        fi
        if [ "$provider" ]; then
            config+=$(echo -en "\n      LEXICON_PROVIDER: $provider")
        fi
    fi
    challenge_type=$(get_challenge_type "$cfg" "$action" "$domain")
    config+=$(echo -en "\n      CHALLENGE_TYPE: $challenge_type")
    info "Challenge type is $challenge_type"
    echo "$config"
 }
 compose_get_challenge_type() {
    local cfg="$1"
    e "$cfg" | shyaml get-value "challenge-type" 2>/dev/null
 }
 letsencrypt_get_challenge_type() {
    local domain="$1" renewal_file
    renewal_file="$SERVICE_DATASTORE"/etc/letsencrypt/renewal/"$domain".conf
    [ -e "$renewal_file" ] || return 1
    grep '^pref_challs' "$renewal_file" | cut -f 2 -d "=" | xargs echo
 }
 letsencrypt_set_renew_before_expiry() {
    local domain="$1" days="$2" renewal_file
    renewal_file="$SERVICE_DATASTORE"/etc/letsencrypt/renewal/"$domain".conf
    [ -e "$renewal_file" ] || return 1
    sed -ri "s/^(#\s+)?(renew_before_expiry\s*=)\s*[0-9]+(\s+days)$/\2 $days\3/g" "$renewal_file"
 }
 letsencrypt_get_renew_before_expiry() {
    local domain="$1" renewal_file
    renewal_file="$SERVICE_DATASTORE"/etc/letsencrypt/renewal/"$domain".conf
    [ -e "$renewal_file" ] || return 1
    if out=$(egrep "^renew_before_expiry\s*=\s*[0-9]+\s+days$" "$renewal_file" 2>/dev/null); then
        e "$out" | sed -r "s/^renew_before_expiry\s*=\s*([0-9]+)\s+days$/\1/g"
    else
        err "Couldn't find 'renew_before_expiry' in letsencrypt renewal" \
            "configuration for domain '$domain'."
        return 1
    fi
 }
 get_challenge_type() {
    local cfg="$1" action="$2" domain="$3" challenge_type renewal_file challenge
    case "$action" in
        create)
            if ! challenge_type=$(compose_get_challenge_type "$cfg"); then
                warn "No ${WHITE}challenge-type${NORMAL} provided, defaulting to 'http'."
                challenge_type=http
            fi
            echo "$challenge_type"
            ;;
        renew)
            challenge=$(letsencrypt_get_challenge_type "$domain")
            if [[ "$challenge" =~ ^http ]]; then
                echo "http"
            else
                echo "$challenge"
            fi
            ;;
        *)
            err "Invalid action '$action'."
            ;;
    esac
 }
 will_need_http_access() {
    local domains args_domains remaining
    local cfg="$1" action="$2" domain="$3" domains args_domains remaining
    challenge_type=$(get_challenge_type "$cfg" "$action" "$domain")
    [ "$challenge_type" == "http" ] || return 1
    [ "${remainder_args[0]}" == "crt" ] || return 1
    [ "${remainder_args[1]}" == "create" ] || return 1
    [ -d "$SERVICE_DATASTORE/etc/letsencrypt/live/${remainder_args[2]}" ] || return 0
    info "Querying ${remainder_args[2]} for previous info..."
    out=$(compose run --rm letsencrypt crt info "${remainder_args[2]}" 2>&1 >/dev/null) || return 0
    domains=$(printf "%s" "$out" | shyaml get-value domains) || return 0
 }
 has_existing_cert() {
    local domain="$1"
    [ -d "$SERVICE_DATASTORE/etc/letsencrypt/live/$domain" ] || return 1
 }
 letsencrypt_cert_info() {
    local domain="$1"
    compose -q --no-init --no-relations run --rm "$SERVICE_NAME" \
            crt info "$domain"
 }
 letsencrypt_cert_delete() {
    local domain="$1"
    compose --debug --no-init --no-relations run --rm "$SERVICE_NAME" \
            certbot delete --cert-name "$domain"
 }
 valid_existing_cert() {
    local renew_before_expiry="$1" domain="$2" args_domains domains remaining
    shift
    args_domains=("$@")
    has_existing_cert "$domain" || return 1
    info "Querying $domain for previous info..."
    out=$(letsencrypt_cert_info "$domain") || return 1
    domains=$(e "$out" | shyaml get-value domains) || return 1
    domains=$(printf "%s " $domains | tr " " "\n" | sort)
    args_domains=$(printf "%s " ${remainder_args[*]:2} | tr " " "\n" | sort)
    info domains: "$domains"
    info args_domain: "$args_domains"
    remaining=$(printf "%s" "$out" | shyaml get-value remaining) || return 0
    ## XXXvlab: not using the variables to decide number of max days remaining
    ## for asking new certificate
    [ "$domains" != "$args_domains" ] || [ "$remaining" -lt 30 ]
    args_domains=$(printf "%s " "${args_domains[@]}" | tr " " "\n" | sort)
    # info domains: "$domains"
    # info args_domain: "$args_domains"
    remaining=$(e "$out" | shyaml get-value remaining) || return 1
    if [ "$domains" != "$args_domains" ]; then
        info "Domains mismatch:"
        info "  old: $domains"
        info "  new: $args_domains"
        return 2
    fi
    if [ "$remaining" == EXPIRED ]; then
        info "Existing certificate expired."
        return 1
    fi
    if [ "$remaining" -lt "$renew_before_expiry" ]; then
        info "Existing certificate in renew period" \
             "($remaining remaining days of validity)."
        return 1
    fi
 }
 get_domain_list() {
    compose -q --no-init --no-relations run --rm "$SERVICE_NAME" crt list
 }
 crt() {
    local cfg="$1" action="$2" domain="$3" config \
          stopped_containers container_ids
    shift
    shift
    ## expiry was checked, launch the action on the real charm, but take care of
    ## correctly running it.
    ## - provide env
    ## - declare proper ports
    ## - stop containers and restart them if necessary
    config=$(get_dc_env "$cfg" "$action" "$domain") || return 1
    stopped_containers=()
    if will_need_http_access "$cfg" "$action" "$domain"; then
        container_ids=($(docker ps \
                                --filter label="compose.project=$PROJECT_NAME" \
                                --filter publish=80 \
                                --format "{{.ID}}"
                       )) || exit 1
        for container_id in "${container_ids[@]}"; do
            info "Attempting to clear port 80 by stopping $container_id"
            docker stop -t 5 "$container_id"
            stopped_containers+=("$container_id")
        done
        config+=$(echo -en "\n    ports:
    - \"0.0.0.0:80:80\"")
    fi
    compose_opts=()
    if [ "$DEBUG" ]; then
        compose_opts+=("--debug")
    else
        compose_opts+=("--quiet")
    fi
    compose "${compose_opts[@]}" --no-init --no-relations --add-compose-content "$config" \
         run --service-ports --rm "$SERVICE_NAME" crt "$action" "$@"
    errlvl="$?"
    for container_id in "${stopped_containers[@]}"; do
        info "Attempting restart $container_id"
        docker start "$container_id"
    done
    return "$errlvl"
 }
 crt_create() {
    local force service_def cfg renew_before_expiry msg domains
    usage="
  $exname [-h|--help]
  $exname MAIN_DOMAIN [ALT_DOMAINS...]"
    force=
    domains=()
    while [ "$1" ]; do
        case "$1" in
            "--help"|"-h")
                print_usage
                return 0
                ;;
            "--force"|"-f") force=1;;
            *) domains+=("$1");;
        esac
        shift
    done
    if [ "${#domains[@]}" == 0 ]; then
        err "At least one domain should be provided as argument."
        print_usage >&2
        return 1
    fi
    service_def=$(get_compose_service_def "$SERVICE_NAME") || return 1
    cfg=$(e "$service_def" | shyaml get-value "options" 2>/dev/null)
    renew_before_expiry=$(e "$cfg" | shyaml get-value "renew-before-expiry" 30 2>/dev/null)
    renew_before_expiry=${renew_before_expiry:-30}
    valid_existing_cert "$renew_before_expiry" "${domains[@]}"
    valid_existing_cert="$?"
    if [ -z "$force" ] && [ "$valid_existing_cert" == 0 ]; then
        if [ "${#domains[@]}" -gt 1 ]; then
            msg=" (with ${domains[*]:1})"
        fi
        info "A valid cert already exists for domain ${domains[0]}$msg."
        return 0
    fi
    if [ "$valid_existing_cert" == 2 ]; then
        err "Domain mismatch detected, lets delete previous cert."
        letsencrypt_cert_delete "${domains[0]}" || return 1
        err "Previous cert for ${domains[0]} deleted."
    fi
    crt "$cfg" create "${domains[@]}" || {
        err "Certificate creation/renew failed for domain '${domains[0]}'."
        return 1
    }
    letsencrypt_set_renew_before_expiry "${domains[0]}" "$renew_before_expiry" || {
        err "Setting renew-before-expiry on '${domains[0]}' failed."
        return 1
    }
 }
 crt_renew() {
    local service_def cfg renew_before_expiry msg start domains_yml \
          domain domain_cfg
    usage="$
  $exname [-h|--help]
 "
    while [ "$1" ]; do
        case "$1" in
            "--help"|"-h")
                print_usage
                return 0
                ;;
            *)
                err "No argument required"
                print_usage >&2
                return 1
                ;;
        esac
        shift
    done
    service_def=$(get_compose_service_def "$SERVICE_NAME") || return 1
    cfg=$(e "$service_def" | shyaml get-value "options" 2>/dev/null)
    default_renew_before_expiry=$(e "$cfg" | shyaml get-value "renew-before-expiry" 2>/dev/null)
    default_renew_before_expiry=${renew_before_expiry:-30}
    if ! renew_before_expiry=$(letsencrypt_get_renew_before_expiry "$domain") || \
            [ -z "$renew_before_expiry" ]; then
        renew_before_expiry=$default_renew_before_expiry
    fi
    start="$SECONDS"
    info "Get domain list.."
    domains_yml=$(get_domain_list) || return 1
    info "  .. Done ${GRAY}($((SECONDS - start))s)${NORMAL}"
    [ "$domains_yml" ] || {
        info "No domain founds"
        return 0
    }
    failed=()
    while read-0 domain domain_cfg; do
        remaining=$(e "$domain_cfg" | shyaml get-value "remaining") || return 1
        if [ "$remaining" == EXPIRED ] || [ "$remaining" -lt "$renew_before_expiry" ]; then
            if [ "$remaining" == EXPIRED ]; then
                info "Renewing domain $domain (expired)."
            else
                info "Renewing domain $domain ($remaining days left)."
            fi
            crt "$cfg" renew "$domain"
            if [ "$?" != "0" ]; then
                failed+=("$domain")
                err "Certificate renew of '$domain' failed."
            fi
        else
            info "Domain $domain does not need renewing ($remaining days left)."
        fi
    done < <(e "$domains_yml" | shyaml key-values-0)
    if [ "${#failed[@]}" -gt 0 ]; then
        err "At least one domain failed to be renewed: ${failed[@]}"
        return 1
    fi
 }
--- a/letsencrypt/metadata.yml
+++ b/letsencrypt/metadata.yml
@ -7,6 +7,9 @@ data-resources:
  - /etc/letsencrypt        ## yes certificates are stored here, this is data
  - /var/log/letsencrypt    ## logs
  - /var/lib/tldextract     ## latest data about TLDs, this is used by lexicon...
 default-options:
  renew-before-expiry: 30
 provides:
  cert-provider:
 uses:
--- a/letsencrypt/test/crt
+++ b/letsencrypt/test/crt
@ -0,0 +1,218 @@
 #!/bin/bash
 exname=$(basename $0)
 prefix_cmd="
 . /etc/shlib
 include common
 include parse
 . ../lib/common
 get_dc_env() {
    local i
    echo \"Calling get_dc_env\" >&2
    ((i=0))
    for arg in \"\$@\"; do
        echo \"  arg\$((i++)):\"
        echo \"\$arg\" | prefix \"  | \"
    done >&2
    echo \"\$GET_DC_ENV\"
 }
 export -f get_dc_env
 will_need_http_access() {
    local i
    echo \"Calling will_need_http_access\" >&2
    ((i=0))
    for arg in \"\$@\"; do
        echo \"  arg\$((i++)):\"
        echo \"\$arg\" | prefix \"  | \"
    done >&2
    [ \"\$WILL_NEED_HTTP_ACCESS\" == 'yes' ]
 }
 export -f will_need_http_access
 "
 ##
 ## Mocks
 ##
 cfg-get-value() {
    local key="$1"
    shyaml get-value "$key" 2>/dev/null
 }
 export -f cfg-get-value
 file_put() {
    echo "file_put $1"
    cat - | prefix "  | "
 }
 export -f file_put
 docker() {
    local i
    echo "Calling: docker" >&2
    ((i=0))
    for arg in "$@"; do
        echo "  arg$((i++)):"
        echo "$arg" | prefix "  | "
    done >&2
    if [ "$1" == "ps" ]; then
        echo "$DOCKER_PS"
    fi
 }
 export -f docker
 yaml_key_val_str() {
    printf "%s:\n%s" "$1" "$(echo "$2" | prefix "  ")"
 }
 export -f yaml_key_val_str
 compose() {
    local i
    echo "Calling: compose" >&2
    ((i=0))
    for arg in "$@"; do
        echo "  arg$((i++)):"
        echo "$arg" | prefix "  | "
    done >&2
 }
 export -f compose
 try "
 SERVICE_NAME='\$SERVICE_NAME'
 WILL_NEED_HTTP_ACCESS=
 crt '' create www.example.com
 "
 is err reg 'Calling get_dc_env
  arg0:
  |
  arg1:
  | create
  arg2:
  | www.example.com
 Calling will_need_http_access
  arg0:
  |
  arg1:
  | create
  arg2:
  | www.example.com
 Calling: compose
 .*
  | run
 .*
  | letsencrypt
 .*
  | crt
 .*
  | create
 .*
  | www.example.com' RTRIM
 is errlvl 0
 is out ''
 try "
 SERVICE_NAME='\$SERVICE_NAME'
 GET_DC_ENV='
 \$SERVICE_NAME:
  environment:
    LETSENCRYPT_USER_MAIL: foo@example.com
    LEXICON_OVH_FOO: 1
    LEXICON_PROVIDER: wiz
 '
 WILL_NEED_HTTP_ACCESS=
 crt '' create www.example.com
 "
 is err reg 'Calling: compose
 .*
  | --add-compose-content
 .*
  | docker-compose:
  |   $SERVICE_NAME:
  |     environment:
  |       LETSENCRYPT_USER_MAIL: foo@example.com
  |       LEXICON_OVH_FOO: 1
  |       LEXICON_PROVIDER: wiz
 .*
  | run
 .*
  | letsencrypt
 .*
  | crt
 .*
  | create
 .*
  | www.example.com' RTRIM
 is errlvl 0
 try "
 SERVICE_NAME='\$SERVICE_NAME'
 GET_DC_ENV='
 \$SERVICE_NAME:
  environment:
    LETSENCRYPT_USER_MAIL: foo@example.com
    LEXICON_OVH_FOO: 1
    LEXICON_PROVIDER: wiz
 '
 WILL_NEED_HTTP_ACCESS=yes
 DOCKER_PS=
 crt '' create www.example.com
 " "need http acces, no docker on port 80"
 is err part 'Calling: docker
  arg0:
  | ps' RTRIM
 is errlvl 0
 is out ''
 try "
 GET_DC_ENV='
 \$SERVICE_NAME:
  environment:
    LETSENCRYPT_USER_MAIL: foo@example.com
    LEXICON_OVH_FOO: 1
    LEXICON_PROVIDER: wiz
 '
 WILL_NEED_HTTP_ACCESS=yes
 DOCKER_PS='
 docker_1
 docker_2
 '
 crt '' create www.example.com
 " "need http acces, 2 dockers on port 80"
 is err reg 'II Attempting to clear port 80 by stopping docker_1
 Calling: docker
  arg0:
  . stop
 .*
  . docker_1
 II Attempting to clear port 80 by stopping docker_2
 Calling: docker
  arg0:
  . stop
 .*
  . docker_2
 Calling: compose
 .*
 II Attempting restart docker_1
 Calling: docker
  arg0:
  . start
 .*
  . docker_1
 II Attempting restart docker_2
 Calling: docker
  arg0:
  . start
 .*
  . docker_2' RTRIM
 is errlvl 0
 is out ''
--- a/letsencrypt/test/crt_create
+++ b/letsencrypt/test/crt_create
@ -0,0 +1,296 @@
 #!/bin/bash
 exname=$(basename $0)
 prefix_cmd="
 . /etc/shlib
 include common
 include parse
 . ../lib/common
 valid_existing_cert() {
    local i
    echo \"Calling valid_existing_cert\" >&2
    ((i=0))
    for arg in \"\$@\"; do
        echo \"  arg\$((i++)):\"
        echo \"\$arg\" | prefix \"  | \"
    done >&2
    return \"\$VALID_EXISTING_CERT\"
 }
 export -f valid_existing_cert
 crt() {
    local i
    echo \"Calling crt\" >&2
    ((i=0))
    for arg in \"\$@\"; do
        echo \"  arg\$((i++)):\"
        echo \"\$arg\" | prefix \"  | \"
    done >&2
    return \$CRT
 }
 export -f crt
 letsencrypt_set_renew_before_expiry() {
    local i
    echo \"Calling letsencrypt_set_renew_before_expiry\" >&2
    ((i=0))
    for arg in \"\$@\"; do
        echo \"  arg\$((i++)):\"
        echo \"\$arg\" | prefix \"  | \"
    done >&2
    [ \"\$LETSENCRYPT_SET_RENEW_BEFORE_EXPIRY\" == \"yes\" ]
 }
 export -f letsencrypt_set_renew_before_expiry
 letsencrypt_cert_delete() {
    local i
    echo \"Calling letsencrypt_cert_delete\" >&2
    ((i=0))
    for arg in \"\$@\"; do
        echo \"  arg\$((i++)):\"
        echo \"\$arg\" | prefix \"  | \"
    done >&2
    [ \"\$LETSENCRYPT_CERT_DELETE\" == \"yes\" ]
 }
 export -f letsencrypt_cert_delete
 "
 ##
 ## Mocks
 ##
 get_compose_service_def() {
    local i
    echo "Calling: get_compose_service_def" >&2
    ((i=0))
    for arg in "$@"; do
        echo "  arg$((i++)):"
        echo "$arg" | prefix "  | "
    done >&2
    echo "$GET_COMPOSE_SERVICE_DEF"
 }
 export -f get_compose_service_def
 try "
 exname=\"crt create\"
 SERVICE_NAME='\$SERVICE_NAME'
 GET_COMPOSE_SERVICE_DEF=
 VALID_EXISTING_CERT=1
 crt_create
 "
 is err 'Error: At least one domain should be provided as argument.
 usage:
  crt create [-h|--help]
  crt create MAIN_DOMAIN [ALT_DOMAINS...]' RTRIM
 is errlvl 1
 try "
 exname=\"crt create\"
 SERVICE_NAME='\$SERVICE_NAME'
 GET_COMPOSE_SERVICE_DEF=
 VALID_EXISTING_CERT=1
 crt_create --help
 "
 is err ''
 is out 'usage:
  crt create [-h|--help]
  crt create MAIN_DOMAIN [ALT_DOMAINS...]' RTRIM
 is errlvl 0
 try "
 CRT=0
 exname=\"crt create\"
 SERVICE_NAME='\$SERVICE_NAME'
 GET_COMPOSE_SERVICE_DEF=
 VALID_EXISTING_CERT=1
 LETSENCRYPT_SET_RENEW_BEFORE_EXPIRY=yes
 crt_create www.example.com
 " "invalid cert"
 is err 'Calling: get_compose_service_def
  arg0:
  | $SERVICE_NAME
 Calling valid_existing_cert
  arg0:
  | 30
  arg1:
  | www.example.com
 Calling crt
  arg0:
  |
  arg1:
  | create
  arg2:
  | www.example.com
 Calling letsencrypt_set_renew_before_expiry
  arg0:
  | www.example.com
  arg1:
  | 30' RTRIM
 is out '' RTRIM
 is errlvl 0
 try "
 exname=\"crt create\"
 SERVICE_NAME='\$SERVICE_NAME'
 GET_COMPOSE_SERVICE_DEF=
 VALID_EXISTING_CERT=0
 LETSENCRYPT_SET_RENEW_BEFORE_EXPIRY=yes
 crt_create www.example.com
 " "valid cert"
 is err 'Calling: get_compose_service_def
  arg0:
  | $SERVICE_NAME
 Calling valid_existing_cert
  arg0:
  | 30
  arg1:
  | www.example.com
 II A valid cert already exists for domain www.example.com.' RTRIM
 is out '' RTRIM
 is errlvl 0
 try "
 exname=\"crt create\"
 SERVICE_NAME='\$SERVICE_NAME'
 GET_COMPOSE_SERVICE_DEF=
 VALID_EXISTING_CERT=0
 LETSENCRYPT_SET_RENEW_BEFORE_EXPIRY=yes
 crt_create www.example.com -f
 " "valid cert but force"
 is err 'Calling: get_compose_service_def
  arg0:
  | $SERVICE_NAME
 Calling valid_existing_cert
  arg0:
  | 30
  arg1:
  | www.example.com
 Calling crt
  arg0:
  |
  arg1:
  | create
  arg2:
  | www.example.com
 Calling letsencrypt_set_renew_before_expiry
  arg0:
  | www.example.com
  arg1:
  | 30' RTRIM
 is out '' RTRIM
 is errlvl 0
 try "
 exname=\"crt create\"
 SERVICE_NAME='\$SERVICE_NAME'
 LETSENCRYPT_SET_RENEW_BEFORE_EXPIRY=yes
 GET_COMPOSE_SERVICE_DEF='
 a: 1
 options:
  foo: bar'
 VALID_EXISTING_CERT=1
 crt_create www.example.com
 " "not valid, cfg is passed correctly"
 is err reg 'Calling crt
  arg0:
  . foo: bar
  arg1:
  . create
  arg2:
  . www.example.com' RTRIM
 is out '' RTRIM
 is errlvl 0
 try "
 exname=\"crt create\"
 SERVICE_NAME='\$SERVICE_NAME'
 GET_COMPOSE_SERVICE_DEF='
 a: 1
 options:
  foo: bar'
 VALID_EXISTING_CERT=2
 LETSENCRYPT_SET_RENEW_BEFORE_EXPIRY=yes
 LETSENCRYPT_CERT_DELETE=yes
 crt_create www.example.com
 " "not valid, already existing diff domain"
 is err 'Calling: get_compose_service_def
  arg0:
  | $SERVICE_NAME
 Calling valid_existing_cert
  arg0:
  | 30
  arg1:
  | www.example.com
 Error: Domain mismatch detected, lets delete previous cert.
 Calling letsencrypt_cert_delete
  arg0:
  | www.example.com
 Error: Previous cert for www.example.com deleted.
 Calling crt
  arg0:
  | foo: bar
  arg1:
  | create
  arg2:
  | www.example.com
 Calling letsencrypt_set_renew_before_expiry
  arg0:
  | www.example.com
  arg1:
  | 30' RTRIM
 is out '' RTRIM
 is errlvl 0
 try "
 exname=\"crt create\"
 SERVICE_NAME='\$SERVICE_NAME'
 LETSENCRYPT_SET_RENEW_BEFORE_EXPIRY=yes
 GET_COMPOSE_SERVICE_DEF='
 a: 1
 options:
  foo: bar
  renew-before-expiry: 15
 '
 VALID_EXISTING_CERT=1
 crt_create www.example.com
 " "not valid, renew-before-expiry is used"
 is err reg 'Calling valid_existing_cert
  arg0:
  . 15
  arg1:
  . www.example.com
 ' RTRIM
 is out '' RTRIM
 is errlvl 0
 try "
 crt() { return 1; }
 exname=\"crt create\"
 SERVICE_NAME='\$SERVICE_NAME'
 GET_COMPOSE_SERVICE_DEF=''
 VALID_EXISTING_CERT=1
 crt_create www.example.com
 " "valid cert but force"
 is err part "Error: Certificate creation/renew failed for domain 'www.example.com'." RTRIM
 is out '' RTRIM
 is errlvl 1
--- a/letsencrypt/test/crt_renew
+++ b/letsencrypt/test/crt_renew
@ -0,0 +1,178 @@
 #!/bin/bash
 exname=$(basename $0)
 prefix_cmd="
 . /etc/shlib
 include common
 include parse
 . ../lib/common
 valid_existing_cert() {
    local i
    echo \"Calling valid_existing_cert\" >&2
    ((i=0))
    for arg in \"\$@\"; do
        echo \"  arg\$((i++)):\"
        echo \"\$arg\" | prefix \"  | \"
    done >&2
    [ \"\$VALID_EXISTING_CERT\" == \"yes\" ]
 }
 export -f valid_existing_cert
 crt() {
    local i
    echo \"Calling crt\" >&2
    ((i=0))
    for arg in \"\$@\"; do
        echo \"  arg\$((i++)):\"
        echo \"\$arg\" | prefix \"  | \"
    done >&2
 }
 export -f crt
 get_domain_list() {
    local i
    echo \"Calling get_domain_list\" >&2
    ((i=0))
    for arg in \"\$@\"; do
        echo \"  arg\$((i++)):\"
        echo \"\$arg\" | prefix \"  | \"
    done >&2
    echo \"\$GET_DOMAIN_LIST\"
 }
 export -f get_domain_list
 "
 ##
 ## Mocks
 ##
 get_compose_service_def() {
    local i
    echo "Calling: get_compose_service_def" >&2
    ((i=0))
    for arg in "$@"; do
        echo "  arg$((i++)):"
        echo "$arg" | prefix "  | "
    done >&2
    echo "$GET_COMPOSE_SERVICE_DEF"
 }
 export -f get_compose_service_def
 try "
 exname=\"crt renew\"
 SERVICE_NAME='\$SERVICE_NAME'
 GET_COMPOSE_SERVICE_DEF=
 crt_renew xxx
 "
 is err 'Error: No argument required
 usage: $
  crt renew [-h|--help]' RTRIM
 is errlvl 1
 is out ''
 try "
 exname=\"crt renew\"
 SERVICE_NAME='\$SERVICE_NAME'
 GET_COMPOSE_SERVICE_DEF=
 GET_DOMAIN_LIST=
 crt_renew
 "
 is err part 'II No domain founds' RTRIM
 is errlvl 0
 is out ''
 try "
 exname=\"crt renew\"
 SERVICE_NAME='\$SERVICE_NAME'
 GET_COMPOSE_SERVICE_DEF='
 options:
  wiz: foo
 '
 GET_DOMAIN_LIST='
 www.example.com:
    remaining: 20
 foo.bar:
    remaining: 32
 '
 crt_renew
 " "2 certs, one need renew, one is ok"
 is err part 'II Renewing domain www.example.com (20 days left)' RTRIM
 is err part 'Calling crt
  arg0:
  | wiz: foo
  arg1:
  | renew
  arg2:
  | www.example.com
 '
 is err part 'II Domain foo.bar does not need renewing (32 days left).' RTRIM
 is errlvl 0
 is out ''
 try "
 exname=\"crt renew\"
 SERVICE_NAME='\$SERVICE_NAME'
 GET_COMPOSE_SERVICE_DEF='
 options:
  wiz: foo
  renew-before-expiry: 15
 '
 GET_DOMAIN_LIST='
 www.example.com:
    remaining: 45
 '
 crt_renew
 " "setting renew-before-expiry"
 is err part 'II Domain www.example.com does not need renewing (45 days left).' RTRIM
 is errlvl 0
 is out ''
 try "
 exname=\"crt renew\"
 SERVICE_NAME='\$SERVICE_NAME'
 GET_COMPOSE_SERVICE_DEF=
 GET_DOMAIN_LIST='
 www.example.com:
    remaining: EXPIRED
 '
 crt_renew
 " "expired cert"
 is err part 'II Renewing domain www.example.com (expired).' RTRIM
 is errlvl 0
 is out ''
 try "
 crt() { ! [[ \"\$3\" =~ ^wiz|foo$ ]]; }
 exname=\"crt renew\"
 SERVICE_NAME='\$SERVICE_NAME'
 GET_COMPOSE_SERVICE_DEF=
 GET_DOMAIN_LIST='
 www.example.com:
    remaining: EXPIRED
 foo:
    remaining: EXPIRED
 bar:
    remaining: 98
 wiz:
    remaining: 10
 '
 crt_renew
 " "some failed renewal"
 is err part 'II Renewing domain www.example.com (expired).' RTRIM
 is err part 'Error: At least one domain failed to be renewed: foo wiz' RTRIM
 is errlvl 1
 is out ''
--- a/letsencrypt/test/get_challenge_type
+++ b/letsencrypt/test/get_challenge_type
@ -0,0 +1,61 @@
 #!/bin/bash
 exname=$(basename $0)
 prefix_cmd="
 . /etc/shlib
 include common
 include parse
 . ../lib/common
 letsencrypt_get_challenge_type() {
    echo 'Calling letsencrypt_get_challenge_type' >&2
    echo \"\$LETSENCRYPT_GET_CHALLENGE_TYPE\"
 }
 export -f letsencrypt_get_challenge_type
 "
 try "
 LETSENCRYPT_GET_CHALLENGE_TYPE=foo
 get_challenge_type '' create "bar"
 "
 is errlvl 0
 is err "Warning: No challenge-type provided, defaulting to 'http'." RTRIM
 is out 'http' RTRIM
 try "
 LETSENCRYPT_GET_CHALLENGE_TYPE=foo
 get_challenge_type '
 challenge-type: wiz
 ' create "bar"
 "
 noerror
 is out 'wiz' RTRIM
 try "
 LETSENCRYPT_GET_CHALLENGE_TYPE=foo
 get_challenge_type '
 challenge-type: wiz
 ' renew "bar"
 "
 is errlvl 0
 is err 'Calling letsencrypt_get_challenge_type' RTRIM
 is out 'foo' RTRIM
 try "
 LETSENCRYPT_GET_CHALLENGE_TYPE=http-01
 get_challenge_type '
 challenge-type: wiz
 ' renew "bar"
 "
 is errlvl 0
 is err 'Calling letsencrypt_get_challenge_type' RTRIM
 is out 'http' RTRIM
--- a/letsencrypt/test/get_dc_env
+++ b/letsencrypt/test/get_dc_env
@ -0,0 +1,142 @@
 #!/bin/bash
 exname=$(basename $0)
 prefix_cmd="
 . /etc/shlib
 include common
 include parse
 . ../lib/common
 get_challenge_type() {
    local i
    echo \"Calling get_challenge_type\" >&2
    ((i=0))
    for arg in \"\$@\"; do
        echo \"  arg\$((i++)):\"
        echo \"\$arg\" | prefix \"  | \"
    done >&2
    echo \"\$GET_CHALLENGE_TYPE\"
 }
 export -f get_challenge_type
 "
 try "
 SERVICE_NAME='\$SERVICE_NAME'
 GET_CHALLENGE_TYPE=foo
 get_dc_env '' create bar
 "
 is errlvl 0
 is err part "\
 Calling get_challenge_type
  arg0:
  |
  arg1:
  | create
  arg2:
  | bar
 " RTRIM
 is out '$SERVICE_NAME:
  docker-compose:
    environment:
      CHALLENGE_TYPE: foo' RTRIM
 try "
 SERVICE_NAME='\$SERVICE_NAME'
 GET_CHALLENGE_TYPE=foo
 get_dc_env '
 email: foo@example.com
 ' create bar
 "
 is errlvl 0
 is err part "\
 Calling get_challenge_type
  arg0:
  |
  | email: foo@example.com
  |
  arg1:
  | create
  arg2:
  | bar
 " RTRIM
 is out '$SERVICE_NAME:
  docker-compose:
    environment:
      LETSENCRYPT_USER_MAIL: foo@example.com
      CHALLENGE_TYPE: foo' RTRIM
 try "
 SERVICE_NAME='\$SERVICE_NAME'
 GET_CHALLENGE_TYPE=foo
 get_dc_env '
 email: foo@example.com
 env:
 ' create bar
 " "environment def is empty"
 is errlvl 0
 is out '$SERVICE_NAME:
  docker-compose:
    environment:
      LETSENCRYPT_USER_MAIL: foo@example.com
      CHALLENGE_TYPE: foo' RTRIM
 try "
 SERVICE_NAME='\$SERVICE_NAME'
 GET_CHALLENGE_TYPE=foo
 get_dc_env '
 email: foo@example.com
 env:
  ignore: x
  ovh:
    foo: 1
    bar: 2
  wiz:
    foo: 1
 ' create bar
 " "environment def without provider"
 is errlvl 0
 is out '$SERVICE_NAME:
  docker-compose:
    environment:
      LETSENCRYPT_USER_MAIL: foo@example.com
      LEXICON_OVH_FOO: 1
      LEXICON_OVH_BAR: 2
      LEXICON_WIZ_FOO: 1
      LEXICON_PROVIDER: ovh
      CHALLENGE_TYPE: foo' RTRIM
 try "
 SERVICE_NAME='\$SERVICE_NAME'
 GET_CHALLENGE_TYPE=foo
 get_dc_env '
 email: foo@example.com
 env:
  ignore: y
  ovh:
    foo: 1
    bar: 2
  wiz:
    foo: 1
  provider: wiz
 ' create bar
 " "environment def with provider"
 is errlvl 0
 is out '$SERVICE_NAME:
  docker-compose:
    environment:
      LETSENCRYPT_USER_MAIL: foo@example.com
      LEXICON_OVH_FOO: 1
      LEXICON_OVH_BAR: 2
      LEXICON_WIZ_FOO: 1
      LEXICON_PROVIDER: wiz
      CHALLENGE_TYPE: foo' RTRIM
--- a/letsencrypt/test/valid_existing_cert
+++ b/letsencrypt/test/valid_existing_cert
@ -0,0 +1,96 @@
 #!/bin/bash
 exname=$(basename $0)
 prefix_cmd="
 . /etc/shlib
 include common
 include parse
 . ../lib/common
 has_existing_cert() {
    echo \"Calling has_existing_cert $*\" >&2
    [ \"\$HAS_EXISTING_CERT\" == 'yes' ]
 }
 export -f has_existing_cert
 letsencrypt_cert_info() {
    echo \"Calling letsencrypt_cert_info $*\" >&2
    echo \"\$LETSENCRYPT_CERT_INFO\"
 }
 export -f letsencrypt_cert_info
 "
 try "
 HAS_EXISTING_CERT=  ## False
 valid_existing_cert 30 'www.example.com'
 "
 is errlvl 1
 is err 'Calling has_existing_cert' RTRIM
 is out '' RTRIM
 try "
 HAS_EXISTING_CERT=yes  ## False
 LETSENCRYPT_CERT_INFO='
 domains: www.example.com
 remaining: 74
 '
 valid_existing_cert 30 'www.example.com'
 " "existing and valid cert"
 is errlvl 0
 is err part 'Calling has_existing_cert' RTRIM
 is err part 'Querying www.example.com for previous info...' RTRIM
 is err part 'Calling letsencrypt_cert_info' RTRIM
 is out '' RTRIM
 try "
 HAS_EXISTING_CERT=yes  ## False
 LETSENCRYPT_CERT_INFO='
 domains: www.example.com
 remaining: 74
 '
 valid_existing_cert 90 'www.example.com'
 " "days validity beneath threshold"
 is errlvl 1
 is out '' RTRIM
 try "
 HAS_EXISTING_CERT=yes  ## False
 LETSENCRYPT_CERT_INFO='
 domains: www.example.com example.com
 remaining: 74
 '
 valid_existing_cert 30 'www.example.com'
 " "domains mismatch 1"
 is errlvl 2
 is out '' RTRIM
 try "
 HAS_EXISTING_CERT=yes  ## False
 LETSENCRYPT_CERT_INFO='
 domains: www.example.com
 remaining: 74
 '
 valid_existing_cert 30 'www.example.com' example.com
 " "domains mismatch 2"
 is errlvl 2
 is out '' RTRIM
 try "
 HAS_EXISTING_CERT=yes  ## False
 LETSENCRYPT_CERT_INFO='
 domains: www.example.com
 remaining: EXPIRED
 '
 valid_existing_cert 30 www.example.com
 " "expired"
 is errlvl 1
 is out '' RTRIM
--- a/letsencrypt/test/yaml_opt_bash_env
+++ b/letsencrypt/test/yaml_opt_bash_env
@ -0,0 +1,33 @@
 #!/bin/bash
 exname=$(basename $0)
 prefix_cmd="
 . /etc/shlib
 include common
 include parse
 . ../lib/common
 "
 try "echo '
 a: b
 ' | yaml_opt_bash_env PREFIX | tr '\0' ':'"
 noerror
 is out 'PREFIX_A:b:'
 try "echo '
 x: 1
 y:
  a: 4
  b: 3
 ' | yaml_opt_bash_env PREFIX | tr '\0' ':'"
 noerror
 is out 'PREFIX_X:1:PREFIX_Y_A:4:PREFIX_Y_B:3:'
--- a/letsencrypt/test/yaml_opt_bash_env_ignore_first_level
+++ b/letsencrypt/test/yaml_opt_bash_env_ignore_first_level
@ -0,0 +1,26 @@
 #!/bin/bash
 exname=$(basename $0)
 prefix_cmd="
 . /etc/shlib
 include common
 include parse
 . ../lib/common
 "
 try "echo '
 x: 1
 y:
  a: 4
  b: 3
 ' | yaml_opt_bash_env PREFIX | tr '\0' ':'"
 noerror
 is out 'PREFIX_X:1:PREFIX_Y_A:4:PREFIX_Y_B:3:'