new: [synapse] rewrite charm for full configurability.

synapse/hooks/init

@@ -17,27 +17,52 @@
 service_def=$(get_compose_service_def "$SERVICE_NAME") || return 1
 options="$(e "$service_def" | shyaml -y get-value options)" || true
 
-
 SYNAPSE_OPTIONS=(
 
-    server-name:string       ## The server name
-    report-stats:bool        ## Enable anon stat reporting back to the Matrix project
-    enable-registration:bool ## Enable registration on the Synapse instance.
-    allow-guest:bool         ## allow guest joining this server.
-    event-cache-size:size    ## event cache size [default 10K].
-    max-upload-size:size     ## max upload size [default 10M].
+    server_name:string       ## The server name
+
+    report_stats:bool        ## Enable anon stat reporting back to the Matrix project
+    enable_registration:bool ## Enable registration on the Synapse instance.
+    allow_guest_access:bool  ## allow guest joining this server.
+    event_cache_size:size    ## event cache size [default 10K].
+    max_upload_size:size     ## max upload size [default 10M].
 
     ## shared secrets
-    registration-shared-secret:string  ## registrering users if registration is disable.
-    macaroon-secret-key:string         ## secret for signing access tokens to the server.
+    registration_shared_secret:string  ## registrering users if registration is disable.
+    macaroon_secret_key:string         ## secret for signing access tokens to the server.
 
     ## recaptcha
-    recaptcha-public-key:string  ## required in order to enable recaptcha upon registration
-    recaptcha-private-key:string ## required in order to enable recaptcha upon registration
+    recaptcha_public_key:string       ## required to have recaptcha upon registration
+    recaptcha_private_key:string      ## required to have recaptcha upon registration
+    enable_registration_captcha:bool  ## required to have recaptcha upon registration
+    recaptcha_siteverify_api:string
+
+    ## others
+    soft_file_limit:numeric
+    rc_messages_per_second:float
+    rc_message_burst_count:float
+    federation_rc_window_size:numeric
+    federation_rc_sleep_limit:numeric
+    federation_rc_sleep_delay:numeric
+    federation_rc_reject_limit:numeric
+    federation_rc_concurrent:numeric
+    max_image_pixels:size
+    dynamic_thumbnails:bool
+    url_preview_enabled:bool
+    max_spider_size:size
+    bcrypt_rounds:numeric
+    enable_group_creation:bool
+    trusted_third_party_id_servers:sequence
+    enable_metrics:bool
+    room_invite_state_types:sequence
+    expire_access_token:bool
+    key_refresh_interval:string
+    perspectives:struct
+    password_config:struct
+
+    ## NOT SUPPORTED YET
+    #thumbnail_sizes
 
-    ## turn
-    turn-uris:string             ## coma-separated list of TURN uris to enable TURN for this homeserver.
-    turn-secret:string           ## TURN shared secret if required.
 )
 
 OPTIONS_CONCAT=" ${SYNAPSE_OPTIONS[*]} "
@@ -64,6 +89,23 @@ while read-0 key val; do
                 die "Invalid value for ${WHITE}$key$NORMAL, please use numeric value."
             fi
             ;;
+        *" ${key_option}:float "*)
+            if ! is_float "$val"; then
+                die "Invalid value for ${WHITE}$key$NORMAL, please use float value."
+            fi
+            ;;
+        *" ${key_option}:struct "*)
+            val_type=$(e "$val" | shyaml get-type) || return 1
+            if [ "$val_type" != "struct" ]; then
+                die "Invalid value for ${WHITE}$key$NORMAL, please use struct value."
+            fi
+            ;;
+        *" ${key_option}:sequence "*)
+            val_type=$(e "$val" | shyaml get-type) || return 1
+            if [ "$val_type" != "sequence" ]; then
+                die "Invalid value for ${WHITE}$key$NORMAL, please use sequence value."
+            fi
+            ;;
         *" ${key_option}:string "*)
             :
             ;;
@@ -80,17 +122,86 @@ while read-0 key val; do
             ;;
     esac
     yaml_opts+=("$key" "$val")
-done < <(e "$options" | yaml_opt_flatten)
+done < <(e "$options" | shyaml key-values-0)
+
+
+setup_dirs || exit 1
+cfg-base || exit 1
+cfg-merge "$options" || exit 1
+
+
+HOST_KEY_DIR=$SERVICE_DATASTORE$DATA_DIR/keys
+for name_secret in registration_shared_secret macaroon_secret_key; do
+    secret=$(e "$options" | shyaml -q get-value "$name_secret") || true
+    if [ "$secret" == "None" ]; then
+        secret=""
+    fi
+
+    coming_from_file=
+    key_file="$HOST_KEY_DIR/${name_secret}.key"
+    if [ -z "$secret" ]; then
+        if [ -e "$key_file" ]; then
+            secret="$(cat "$key_file")"
+            coming_from_file=true
+        else
+            secret="$(gen_password 64)"
+        fi
+        cfg-merge "${name_secret}: \"$secret\"" || exit 1
+    fi
+
+    if [ -z "$coming_from_file" ]; then
+        e "$secret" > "$key_file"
+        chown -v "$uid:$gid" "$key_file" &&
+        chmod -v 600 "$key_file" || exit 1
+    fi
+done
+
+
+## XXXvlab: what to do with appservices ?
+# environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml")
+# {% if SYNAPSE_APPSERVICES %}
+# app_service_config_files:
+# {% for appservice in SYNAPSE_APPSERVICES %}    - "{{ appservice }}"
+# {% endfor %}
+# {% else %}
+# app_service_config_files: []
+# {% endif %}
+
+# ## Turn ##
+
+# {% if SYNAPSE_TURN_URIS %}
+# turn_uris:
+# {% for uri in SYNAPSE_TURN_URIS.split(',') %}    - "{{ uri }}"
+# {% endfor %}
+# turn_shared_secret: "{{ SYNAPSE_TURN_SECRET }}"
+# turn_user_lifetime: "1h"
+# turn_allow_guests: True
+# {% else %}
+# turn_uris: []
+# turn_shared_secret: "YOUR_SHARED_SECRET"
+# turn_user_lifetime: "1h"
+# turn_allow_guests: True
+# {% endif %}
+
+## XXXvlab: for SMTP relation
+# {% if SYNAPSE_SMTP_HOST %}
+# email:
+#    enable_notifs: false
+#    smtp_host: "{{ SYNAPSE_SMTP_HOST }}"
+#    smtp_port: {{ SYNAPSE_SMTP_PORT or "25" }}
+#    smtp_user: "{{ SYNAPSE_SMTP_USER }}"
+#    smtp_pass: "{{ SYNAPSE_SMTP_PASSWORD }}"
+#    require_transport_security: False
+#    notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}"
+#    app_name: Matrix
+#    # if template_dir is unset, uses the example templates that are part of
+#    # the Synapse distribution.
+#    #template_dir: res/templates
+#    notif_template_html: notif_mail.html
+#    notif_template_text: notif_mail.txt
+#    notif_for_new_users: True
+#    riot_base_url: "https://{{ SYNAPSE_SERVER_NAME }}"
+# {% endif %}
 
-config="\
-$SERVICE_NAME:
-  environment:
-    SYNAPSE_NO_TLS: \"yes\"
-"
 
-while read-0 key value; do
-    key=${key//-/_}
-    config+="$(printf "\n    SYNAPSE_%s: %s" "${key^^}" "$value")"
-done < <(array_values_to_stdin yaml_opts)
 
-init-config-add "$config"

synapse/hooks/postgres_database-relation-joined

@@ -2,25 +2,27 @@
 
 set -e
 
+. lib/common
+
 PASSWORD="$(relation-get password)"
 USER="$(relation-get user)"
 DBNAME="$(relation-get dbname)"
 
 control=$(echo -en "$USER\0$DBNAME\0$PASSWORD" | md5_compat)
 
-
-init-config-add "
-$SERVICE_NAME:
-  environment:
-    POSTGRES_HOST: $MASTER_TARGET_SERVICE_NAME 
-    POSTGRES_DB: $DBNAME
-    POSTGRES_USER: $USER
-    POSTGRES_PASSWORD: $PASSWORD
+cfg-merge "
+database:
+  name: psycopg2
+  args:
+    user: $USER
+    password: $PASSWORD
+    database: $DBNAME
+    host: $MASTER_TARGET_SERVICE_NAME
+    port: 5432
+    cp_min: 5
+    cp_max: 10
 "
 
-[ "$control" == "$(relation-get control 2>/dev/null)" ] && exit 0
-
-
 relation-set control "$control"
 
 info "Configured $SERVICE_NAME code for $TARGET_SERVICE_NAME access."

synapse/hooks/pre_deploy

@@ -0,0 +1,8 @@
+#!/bin/bash
+## Should be executable N time in a row with same result.
+
+set -e
+
+. lib/common
+
+config_hash || exit 1

synapse/hooks/web_proxy-relation-joined

@@ -5,22 +5,12 @@ set -e
 . lib/common
 
 
-env=()
-url=$(relation-get url)
-if [ "${url%://*}" == "https" ]; then
-    env+=(SYNAPSE_NO_TLS "'yes'")
-fi
-
-server_name=$(options-get "server-name") || true
+server_name=$(options-get "server_name") || true
 if [ -z "$server_name" ]; then
     DOMAIN=$(relation-get domain) || exit 1
-    env+=(SYNAPSE_SERVER_NAME "$DOMAIN")
+    cfg-merge "server_name: $DOMAIN"
 fi
+URL=$(relation-get url) || exit 1
+cfg-merge "public_baseurl: $URL"
 
-if [ "${#env[@]}" -gt 0 ]; then
-    init-config-add "
-$SERVICE_NAME:
-  environment:
-$(printf "    %s: %s\n" "${env[@]}")
-" || exit 1
-fi
+sed -ri 's/^(\s+x_forwarded:\s+)[fF]alse/\1true/g' "$HOST_CONFIG_FILE"

synapse/lib/common

@@ -18,3 +18,234 @@ yaml_opt_flatten() {
 }
 
 
+CFG_DIR=/etc/synapse
+DATA_DIR=/var/lib/synapse
+CONFIG_FILE="$CFG_DIR/config.yml"
+HOST_CONFIG_FILE="${SERVICE_CONFIGSTORE}$CONFIG_FILE"
+
+
+setup_dirs() {
+    local dirs dir
+
+    dirs=("$SERVICE_DATASTORE/var/lib/synapse")
+    uid_gid=($(docker_get_uid_gid "$SERVICE_NAME" "synapse" "synapse")) || {
+        err "Could not fetch uid/gid on image of service ${DARKYELLOW}$SERVICE_NAME${NORMAL}."
+        return 1
+    }
+    uid="${uid_gid[0]}"
+    gid="${uid_gid[1]}"
+    for dir in "${dirs[@]}"; do
+        mkdir -p "$dir"
+        find "$dir" \! -uid "$uid" -exec chown -v "$uid" {} \;
+        find "$dir" \! -gid "$gid" -exec chgrp -v "$gid" {} \;
+    done
+
+    dirs=(
+        "${SERVICE_CONFIGSTORE}/$CFG_DIR"
+        "${SERVICE_DATASTORE}/var/lib/synapse/keys"
+    )
+    for dir in "${dirs[@]}"; do
+        mkdir -p "$dir"
+        chown "$uid:$gid" "$dir"
+    done
+}
+
+
+cfg-merge() {
+    local yaml="$1"
+    merge_yaml_str "$(cat "$HOST_CONFIG_FILE" 2>/dev/null)" \
+                    "$yaml" > "$HOST_CONFIG_FILE.tmp" || return 1
+    mv "$HOST_CONFIG_FILE.tmp" "$HOST_CONFIG_FILE"
+}
+
+
+cfg-base() {
+    cat <<EOF > "$HOST_CONFIG_FILE"
+
+## Server
+
+## Not running as a daemon
+# pid_file: /var/run/synapse/synapse.pid
+web_client: False
+soft_file_limit: 0
+log_config: "$CFG_DIR/logging.yml"
+
+## Ports
+
+listeners:
+  - port: 8008
+    tls: false
+    bind_addresses: ['::']
+    type: http
+    x_forwarded: false
+
+    resources:
+      - names: [client]
+        compress: true
+      - names: [federation]
+        compress: false
+
+## Database ##
+
+database:
+  name: "sqlite3"
+  args:
+    database: "$DATA_DIR/homeserver.db"
+
+## Performance ##
+
+event_cache_size: 10K
+
+## Ratelimiting ##
+
+rc_messages_per_second: 0.2
+rc_message_burst_count: 10.0
+federation_rc_window_size: 1000
+federation_rc_sleep_limit: 10
+federation_rc_sleep_delay: 500
+federation_rc_reject_limit: 50
+federation_rc_concurrent: 3
+
+## Files ##
+
+media_store_path: "$DATA_DIR/media"
+uploads_path: "$DATA_DIR/uploads"
+max_upload_size: "10M"
+max_image_pixels: "32M"
+dynamic_thumbnails: false
+
+# List of thumbnail to precalculate when an image is uploaded.
+thumbnail_sizes:
+- width: 32
+  height: 32
+  method: crop
+- width: 96
+  height: 96
+  method: crop
+- width: 320
+  height: 240
+  method: scale
+- width: 640
+  height: 480
+  method: scale
+- width: 800
+  height: 600
+  method: scale
+
+url_preview_enabled: false
+max_spider_size: "10M"
+
+## Registration ##
+
+enable_registration: false
+enable_registration_captcha: false
+
+bcrypt_rounds: 12
+allow_guest_access: true
+enable_group_creation: true
+
+## TURN
+
+turn_allow_guests: true
+turn_shared_secret: YOUR_SHARED_SECRET
+turn_uris: []
+turn_user_lifetime: 1h
+
+
+# The list of identity servers trusted to verify third party
+# identifiers by this server.
+#
+# Also defines the ID server which will be called when an account is
+# deactivated (one will be picked arbitrarily).
+trusted_third_party_id_servers:
+    - matrix.org
+    - vector.im
+
+## Metrics
+
+enable_metrics: false
+report_stats: false
+
+## API Configuration
+
+room_invite_state_types:
+    - "m.room.join_rules"
+    - "m.room.canonical_alias"
+    - "m.room.avatar"
+    - "m.room.name"
+
+expire_access_token: False
+
+## Signing Keys ##
+
+signing_key_path: "$DATA_DIR/keys/synapse.signing.key"
+old_signing_keys: {}
+key_refresh_interval: "1d" # 1 Day.
+
+
+# The trusted servers to download signing keys from.
+perspectives:
+  servers:
+    "matrix.org":
+      verify_keys:
+        "ed25519:auto":
+          key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"
+
+
+password_config:
+   enabled: true
+
+
+recaptcha_siteverify_api: https://www.google.com/recaptcha/api/siteverify
+
+app_service_config_files: []
+
+EOF
+
+    cat <<EOF > "$SERVICE_CONFIGSTORE$CFG_DIR"/logging.yml
+version: 1
+
+formatters:
+  precise:
+   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s'
+
+filters:
+  context:
+    (): synapse.util.logcontext.LoggingContextFilter
+    request: ""
+
+handlers:
+  console:
+    class: logging.StreamHandler
+    formatter: precise
+    filters: [context]
+
+loggers:
+    synapse:
+        level: WARNING
+
+    synapse.storage.SQL:
+        # beware: increasing this to DEBUG will make synapse log sensitive
+        # information such as access tokens.
+        level: WARNING
+
+root:
+    level: WARNING
+    handlers: [console]
+
+EOF
+}
+
+
+config_hash() {
+    debug "Adding config hash to enable recreating upon config change."
+    config_hash=$({
+                     cat "$HOST_CONFIG_FILE"
+                  } | md5_compat) || exit 1
+    init-config-add "
+$SERVICE_NAME:
+  labels:
+  - compose.config_hash=$config_hash
+"
+}
+

synapse/metadata.yml

@@ -1,25 +1,28 @@
 description: Synapse
 maintainer: "Valentin Lab <valentin.lab@kalysto.org>"
-docker-image: docker.0k.io/synapse:py3.6
+docker-image: docker.0k.io/synapse:1.0
+config-resources:
+  - /etc/synapse
 data-resources:
-  - /data
+  - /var/lib/synapse
 host-resources:
   - /etc/localtime:ro
 
 default-options:
-  report-stats: no
-  enable-registration: no
-  allow-guest: no
-  event-cache-size: 10K
-  max-upload-size: 10M
+  ## ensure that this virtualhost is joinable on https, and with a
+  ## valid cert.
+  # server_name: XXX
 
-  # recaptcha:
-  #   ## https://www.google.com/recaptcha/admin/create
-  #   public-key: XXX
-  #   private-key: XXX
-  # turn:
-  #   uris:
-  #   secret:
+  # report_stats: no
+
+  # enable_registration: no
+  # registration_shared_secret: My secret
+
+  # allow_guest_access: no
+
+  # enable_registration_captcha: yes
+  # recaptcha_public_key:   XXX
+  # recaptcha_private_key:  YYY
 
 # docker-compose:
 #   ports:
@@ -44,3 +47,16 @@ uses:
       ## ``nocanon`` is mandatory
       ## see: https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.rst
       apache-proxy-pass-options: retry=0 nocanon
+      apache-core-rules: !var-expand |
+        SSLProxyEngine on
+        <Location "/.well-known/matrix" >
+            ## ../server implementation added with a patch in docker image
+            ProxyPass "http://${MASTER_BASE_SERVICE_NAME}:8008/.well-known/matrix" retry=0 nocanon
+            Order deny,allow
+            Allow from all
+        </Location>
+        #<Location "/_matrix" >
+        #    ProxyPass "http://${MASTER_BASE_SERVICE_NAME}:8008/_matrix" retry=0 nocanon
+        #    #Order deny,allow
+        #    #Allow from all
+        #</Location>