From 33bf52d49fc921e154718f0fd7cf776ac43b49f8 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Thu, 28 Feb 2019 13:47:34 +0100 Subject: [PATCH] new: [synapse] rewrite charm for full configurability. --- synapse/hooks/init | 161 ++++++++++-- .../hooks/postgres_database-relation-joined | 24 +- synapse/hooks/pre_deploy | 8 + synapse/hooks/web_proxy-relation-joined | 20 +- synapse/lib/common | 231 ++++++++++++++++++ synapse/metadata.yml | 46 ++-- 6 files changed, 424 insertions(+), 66 deletions(-) create mode 100755 synapse/hooks/pre_deploy diff --git a/synapse/hooks/init b/synapse/hooks/init index dfdcbf8..53e5c9f 100755 --- a/synapse/hooks/init +++ b/synapse/hooks/init @@ -17,27 +17,52 @@ service_def=$(get_compose_service_def "$SERVICE_NAME") || return 1 options="$(e "$service_def" | shyaml -y get-value options)" || true - SYNAPSE_OPTIONS=( - server-name:string ## The server name - report-stats:bool ## Enable anon stat reporting back to the Matrix project - enable-registration:bool ## Enable registration on the Synapse instance. - allow-guest:bool ## allow guest joining this server. - event-cache-size:size ## event cache size [default 10K]. - max-upload-size:size ## max upload size [default 10M]. + server_name:string ## The server name + + report_stats:bool ## Enable anon stat reporting back to the Matrix project + enable_registration:bool ## Enable registration on the Synapse instance. + allow_guest_access:bool ## allow guest joining this server. + event_cache_size:size ## event cache size [default 10K]. + max_upload_size:size ## max upload size [default 10M]. ## shared secrets - registration-shared-secret:string ## registrering users if registration is disable. - macaroon-secret-key:string ## secret for signing access tokens to the server. + registration_shared_secret:string ## registrering users if registration is disable. + macaroon_secret_key:string ## secret for signing access tokens to the server. ## recaptcha - recaptcha-public-key:string ## required in order to enable recaptcha upon registration - recaptcha-private-key:string ## required in order to enable recaptcha upon registration + recaptcha_public_key:string ## required to have recaptcha upon registration + recaptcha_private_key:string ## required to have recaptcha upon registration + enable_registration_captcha:bool ## required to have recaptcha upon registration + recaptcha_siteverify_api:string + + ## others + soft_file_limit:numeric + rc_messages_per_second:float + rc_message_burst_count:float + federation_rc_window_size:numeric + federation_rc_sleep_limit:numeric + federation_rc_sleep_delay:numeric + federation_rc_reject_limit:numeric + federation_rc_concurrent:numeric + max_image_pixels:size + dynamic_thumbnails:bool + url_preview_enabled:bool + max_spider_size:size + bcrypt_rounds:numeric + enable_group_creation:bool + trusted_third_party_id_servers:sequence + enable_metrics:bool + room_invite_state_types:sequence + expire_access_token:bool + key_refresh_interval:string + perspectives:struct + password_config:struct + + ## NOT SUPPORTED YET + #thumbnail_sizes - ## turn - turn-uris:string ## coma-separated list of TURN uris to enable TURN for this homeserver. - turn-secret:string ## TURN shared secret if required. ) OPTIONS_CONCAT=" ${SYNAPSE_OPTIONS[*]} " @@ -64,6 +89,23 @@ while read-0 key val; do die "Invalid value for ${WHITE}$key$NORMAL, please use numeric value." fi ;; + *" ${key_option}:float "*) + if ! is_float "$val"; then + die "Invalid value for ${WHITE}$key$NORMAL, please use float value." + fi + ;; + *" ${key_option}:struct "*) + val_type=$(e "$val" | shyaml get-type) || return 1 + if [ "$val_type" != "struct" ]; then + die "Invalid value for ${WHITE}$key$NORMAL, please use struct value." + fi + ;; + *" ${key_option}:sequence "*) + val_type=$(e "$val" | shyaml get-type) || return 1 + if [ "$val_type" != "sequence" ]; then + die "Invalid value for ${WHITE}$key$NORMAL, please use sequence value." + fi + ;; *" ${key_option}:string "*) : ;; @@ -80,17 +122,86 @@ while read-0 key val; do ;; esac yaml_opts+=("$key" "$val") -done < <(e "$options" | yaml_opt_flatten) +done < <(e "$options" | shyaml key-values-0) + + +setup_dirs || exit 1 +cfg-base || exit 1 +cfg-merge "$options" || exit 1 + + +HOST_KEY_DIR=$SERVICE_DATASTORE$DATA_DIR/keys +for name_secret in registration_shared_secret macaroon_secret_key; do + secret=$(e "$options" | shyaml -q get-value "$name_secret") || true + if [ "$secret" == "None" ]; then + secret="" + fi + + coming_from_file= + key_file="$HOST_KEY_DIR/${name_secret}.key" + if [ -z "$secret" ]; then + if [ -e "$key_file" ]; then + secret="$(cat "$key_file")" + coming_from_file=true + else + secret="$(gen_password 64)" + fi + cfg-merge "${name_secret}: \"$secret\"" || exit 1 + fi + + if [ -z "$coming_from_file" ]; then + e "$secret" > "$key_file" + chown -v "$uid:$gid" "$key_file" && + chmod -v 600 "$key_file" || exit 1 + fi +done + + +## XXXvlab: what to do with appservices ? +# environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") +# {% if SYNAPSE_APPSERVICES %} +# app_service_config_files: +# {% for appservice in SYNAPSE_APPSERVICES %} - "{{ appservice }}" +# {% endfor %} +# {% else %} +# app_service_config_files: [] +# {% endif %} + +# ## Turn ## + +# {% if SYNAPSE_TURN_URIS %} +# turn_uris: +# {% for uri in SYNAPSE_TURN_URIS.split(',') %} - "{{ uri }}" +# {% endfor %} +# turn_shared_secret: "{{ SYNAPSE_TURN_SECRET }}" +# turn_user_lifetime: "1h" +# turn_allow_guests: True +# {% else %} +# turn_uris: [] +# turn_shared_secret: "YOUR_SHARED_SECRET" +# turn_user_lifetime: "1h" +# turn_allow_guests: True +# {% endif %} + +## XXXvlab: for SMTP relation +# {% if SYNAPSE_SMTP_HOST %} +# email: +# enable_notifs: false +# smtp_host: "{{ SYNAPSE_SMTP_HOST }}" +# smtp_port: {{ SYNAPSE_SMTP_PORT or "25" }} +# smtp_user: "{{ SYNAPSE_SMTP_USER }}" +# smtp_pass: "{{ SYNAPSE_SMTP_PASSWORD }}" +# require_transport_security: False +# notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}" +# app_name: Matrix +# # if template_dir is unset, uses the example templates that are part of +# # the Synapse distribution. +# #template_dir: res/templates +# notif_template_html: notif_mail.html +# notif_template_text: notif_mail.txt +# notif_for_new_users: True +# riot_base_url: "https://{{ SYNAPSE_SERVER_NAME }}" +# {% endif %} -config="\ -$SERVICE_NAME: - environment: - SYNAPSE_NO_TLS: \"yes\" -" -while read-0 key value; do - key=${key//-/_} - config+="$(printf "\n SYNAPSE_%s: %s" "${key^^}" "$value")" -done < <(array_values_to_stdin yaml_opts) -init-config-add "$config" diff --git a/synapse/hooks/postgres_database-relation-joined b/synapse/hooks/postgres_database-relation-joined index 2d2d530..cba3782 100755 --- a/synapse/hooks/postgres_database-relation-joined +++ b/synapse/hooks/postgres_database-relation-joined @@ -2,25 +2,27 @@ set -e +. lib/common + PASSWORD="$(relation-get password)" USER="$(relation-get user)" DBNAME="$(relation-get dbname)" control=$(echo -en "$USER\0$DBNAME\0$PASSWORD" | md5_compat) - -init-config-add " -$SERVICE_NAME: - environment: - POSTGRES_HOST: $MASTER_TARGET_SERVICE_NAME - POSTGRES_DB: $DBNAME - POSTGRES_USER: $USER - POSTGRES_PASSWORD: $PASSWORD +cfg-merge " +database: + name: psycopg2 + args: + user: $USER + password: $PASSWORD + database: $DBNAME + host: $MASTER_TARGET_SERVICE_NAME + port: 5432 + cp_min: 5 + cp_max: 10 " -[ "$control" == "$(relation-get control 2>/dev/null)" ] && exit 0 - - relation-set control "$control" info "Configured $SERVICE_NAME code for $TARGET_SERVICE_NAME access." diff --git a/synapse/hooks/pre_deploy b/synapse/hooks/pre_deploy new file mode 100755 index 0000000..5a777a5 --- /dev/null +++ b/synapse/hooks/pre_deploy @@ -0,0 +1,8 @@ +#!/bin/bash +## Should be executable N time in a row with same result. + +set -e + +. lib/common + +config_hash || exit 1 diff --git a/synapse/hooks/web_proxy-relation-joined b/synapse/hooks/web_proxy-relation-joined index 742f0e0..7201512 100755 --- a/synapse/hooks/web_proxy-relation-joined +++ b/synapse/hooks/web_proxy-relation-joined @@ -5,22 +5,12 @@ set -e . lib/common -env=() -url=$(relation-get url) -if [ "${url%://*}" == "https" ]; then - env+=(SYNAPSE_NO_TLS "'yes'") -fi - -server_name=$(options-get "server-name") || true +server_name=$(options-get "server_name") || true if [ -z "$server_name" ]; then DOMAIN=$(relation-get domain) || exit 1 - env+=(SYNAPSE_SERVER_NAME "$DOMAIN") + cfg-merge "server_name: $DOMAIN" fi +URL=$(relation-get url) || exit 1 +cfg-merge "public_baseurl: $URL" -if [ "${#env[@]}" -gt 0 ]; then - init-config-add " -$SERVICE_NAME: - environment: -$(printf " %s: %s\n" "${env[@]}") -" || exit 1 -fi \ No newline at end of file +sed -ri 's/^(\s+x_forwarded:\s+)[fF]alse/\1true/g' "$HOST_CONFIG_FILE" diff --git a/synapse/lib/common b/synapse/lib/common index 3d1cb8d..2ee6d02 100644 --- a/synapse/lib/common +++ b/synapse/lib/common @@ -18,3 +18,234 @@ yaml_opt_flatten() { } +CFG_DIR=/etc/synapse +DATA_DIR=/var/lib/synapse +CONFIG_FILE="$CFG_DIR/config.yml" +HOST_CONFIG_FILE="${SERVICE_CONFIGSTORE}$CONFIG_FILE" + + +setup_dirs() { + local dirs dir + + dirs=("$SERVICE_DATASTORE/var/lib/synapse") + uid_gid=($(docker_get_uid_gid "$SERVICE_NAME" "synapse" "synapse")) || { + err "Could not fetch uid/gid on image of service ${DARKYELLOW}$SERVICE_NAME${NORMAL}." + return 1 + } + uid="${uid_gid[0]}" + gid="${uid_gid[1]}" + for dir in "${dirs[@]}"; do + mkdir -p "$dir" + find "$dir" \! -uid "$uid" -exec chown -v "$uid" {} \; + find "$dir" \! -gid "$gid" -exec chgrp -v "$gid" {} \; + done + + dirs=( + "${SERVICE_CONFIGSTORE}/$CFG_DIR" + "${SERVICE_DATASTORE}/var/lib/synapse/keys" + ) + for dir in "${dirs[@]}"; do + mkdir -p "$dir" + chown "$uid:$gid" "$dir" + done +} + + +cfg-merge() { + local yaml="$1" + merge_yaml_str "$(cat "$HOST_CONFIG_FILE" 2>/dev/null)" \ + "$yaml" > "$HOST_CONFIG_FILE.tmp" || return 1 + mv "$HOST_CONFIG_FILE.tmp" "$HOST_CONFIG_FILE" +} + + +cfg-base() { + cat < "$HOST_CONFIG_FILE" + +## Server + +## Not running as a daemon +# pid_file: /var/run/synapse/synapse.pid +web_client: False +soft_file_limit: 0 +log_config: "$CFG_DIR/logging.yml" + +## Ports + +listeners: + - port: 8008 + tls: false + bind_addresses: ['::'] + type: http + x_forwarded: false + + resources: + - names: [client] + compress: true + - names: [federation] + compress: false + +## Database ## + +database: + name: "sqlite3" + args: + database: "$DATA_DIR/homeserver.db" + +## Performance ## + +event_cache_size: 10K + +## Ratelimiting ## + +rc_messages_per_second: 0.2 +rc_message_burst_count: 10.0 +federation_rc_window_size: 1000 +federation_rc_sleep_limit: 10 +federation_rc_sleep_delay: 500 +federation_rc_reject_limit: 50 +federation_rc_concurrent: 3 + +## Files ## + +media_store_path: "$DATA_DIR/media" +uploads_path: "$DATA_DIR/uploads" +max_upload_size: "10M" +max_image_pixels: "32M" +dynamic_thumbnails: false + +# List of thumbnail to precalculate when an image is uploaded. +thumbnail_sizes: +- width: 32 + height: 32 + method: crop +- width: 96 + height: 96 + method: crop +- width: 320 + height: 240 + method: scale +- width: 640 + height: 480 + method: scale +- width: 800 + height: 600 + method: scale + +url_preview_enabled: false +max_spider_size: "10M" + +## Registration ## + +enable_registration: false +enable_registration_captcha: false + +bcrypt_rounds: 12 +allow_guest_access: true +enable_group_creation: true + +## TURN + +turn_allow_guests: true +turn_shared_secret: YOUR_SHARED_SECRET +turn_uris: [] +turn_user_lifetime: 1h + + +# The list of identity servers trusted to verify third party +# identifiers by this server. +# +# Also defines the ID server which will be called when an account is +# deactivated (one will be picked arbitrarily). +trusted_third_party_id_servers: + - matrix.org + - vector.im + +## Metrics + +enable_metrics: false +report_stats: false + +## API Configuration + +room_invite_state_types: + - "m.room.join_rules" + - "m.room.canonical_alias" + - "m.room.avatar" + - "m.room.name" + +expire_access_token: False + +## Signing Keys ## + +signing_key_path: "$DATA_DIR/keys/synapse.signing.key" +old_signing_keys: {} +key_refresh_interval: "1d" # 1 Day. + + +# The trusted servers to download signing keys from. +perspectives: + servers: + "matrix.org": + verify_keys: + "ed25519:auto": + key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw" + + +password_config: + enabled: true + + +recaptcha_siteverify_api: https://www.google.com/recaptcha/api/siteverify + +app_service_config_files: [] + +EOF + + cat < "$SERVICE_CONFIGSTORE$CFG_DIR"/logging.yml +version: 1 + +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s' + +filters: + context: + (): synapse.util.logcontext.LoggingContextFilter + request: "" + +handlers: + console: + class: logging.StreamHandler + formatter: precise + filters: [context] + +loggers: + synapse: + level: WARNING + + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: WARNING + +root: + level: WARNING + handlers: [console] + +EOF +} + + +config_hash() { + debug "Adding config hash to enable recreating upon config change." + config_hash=$({ + cat "$HOST_CONFIG_FILE" + } | md5_compat) || exit 1 + init-config-add " +$SERVICE_NAME: + labels: + - compose.config_hash=$config_hash +" +} + diff --git a/synapse/metadata.yml b/synapse/metadata.yml index 6b22bd4..c51366c 100644 --- a/synapse/metadata.yml +++ b/synapse/metadata.yml @@ -1,25 +1,28 @@ description: Synapse maintainer: "Valentin Lab " -docker-image: docker.0k.io/synapse:py3.6 +docker-image: docker.0k.io/synapse:1.0 +config-resources: + - /etc/synapse data-resources: - - /data + - /var/lib/synapse host-resources: - /etc/localtime:ro default-options: - report-stats: no - enable-registration: no - allow-guest: no - event-cache-size: 10K - max-upload-size: 10M + ## ensure that this virtualhost is joinable on https, and with a + ## valid cert. + # server_name: XXX - # recaptcha: - # ## https://www.google.com/recaptcha/admin/create - # public-key: XXX - # private-key: XXX - # turn: - # uris: - # secret: + # report_stats: no + + # enable_registration: no + # registration_shared_secret: My secret + + # allow_guest_access: no + + # enable_registration_captcha: yes + # recaptcha_public_key: XXX + # recaptcha_private_key: YYY # docker-compose: # ports: @@ -43,4 +46,17 @@ uses: default-options: ## ``nocanon`` is mandatory ## see: https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.rst - apache-proxy-pass-options: retry=0 nocanon \ No newline at end of file + apache-proxy-pass-options: retry=0 nocanon + apache-core-rules: !var-expand | + SSLProxyEngine on + + ## ../server implementation added with a patch in docker image + ProxyPass "http://${MASTER_BASE_SERVICE_NAME}:8008/.well-known/matrix" retry=0 nocanon + Order deny,allow + Allow from all + + # + # ProxyPass "http://${MASTER_BASE_SERVICE_NAME}:8008/_matrix" retry=0 nocanon + # #Order deny,allow + # #Allow from all + #