From 41684e1dfbe05109d1e678047afff31798a0826c Mon Sep 17 00:00:00 2001
From: Hans van Zijst <hans@procolix.com>
Date: Mon, 30 Dec 2024 12:00:51 +0100
Subject: [PATCH] Proxy-optimizations added, several worker config files added
 plus loggin template.

---
 matrix/nginx/workers/README.md                | 57 +++++++++++++++--
 matrix/synapse/temp                           | 64 -------------------
 .../synapse/{workers.md => workers/README.md} |  0
 .../synapse/workers/federation_receiver1.yaml | 15 +++++
 .../synapse/workers/federation_sender1.yaml   | 10 +++
 matrix/synapse/workers/initial_sync1.yaml     | 19 ++++++
 matrix/synapse/workers/login-log.yaml         | 41 ++++++++++++
 matrix/synapse/workers/login.yaml             | 19 ++++++
 matrix/synapse/workers/media-log.yaml         | 41 ++++++++++++
 matrix/synapse/workers/media.yaml             | 15 +++++
 10 files changed, 212 insertions(+), 69 deletions(-)
 delete mode 100644 matrix/synapse/temp
 rename matrix/synapse/{workers.md => workers/README.md} (100%)
 create mode 100644 matrix/synapse/workers/federation_receiver1.yaml
 create mode 100644 matrix/synapse/workers/federation_sender1.yaml
 create mode 100644 matrix/synapse/workers/initial_sync1.yaml
 create mode 100644 matrix/synapse/workers/login-log.yaml
 create mode 100644 matrix/synapse/workers/login.yaml
 create mode 100644 matrix/synapse/workers/media-log.yaml
 create mode 100644 matrix/synapse/workers/media.yaml

diff --git a/matrix/nginx/workers/README.md b/matrix/nginx/workers/README.md
index c1f487e..a26dbcd 100644
--- a/matrix/nginx/workers/README.md
+++ b/matrix/nginx/workers/README.md
@@ -8,8 +8,8 @@ include_toc: true
 Changing nginx's configuration from a reverse proxy for a normal, monolithic
 Synapse to one for a Synapse that uses workers, quite a lot has to be changed.
 
-As mentioned in [Synapse with workers](../../synapse/workers.md#synapse), we're
-changing the "backend" from network sockets to UNIX sockets.
+As mentioned in [Synapse with workers](../../synapse/workers/README.md#synapse),
+we're changing the "backend" from network sockets to UNIX sockets.
 
 Because we're going to have to forward a lot of specific requests to all kinds
 of workers, we'll split the configuration into a few bits:
@@ -26,6 +26,42 @@ configuration of nginx itself, others go into `/etc/nginx/snippets` because we
 need to include them several times in different places.
 
 
+# Optimizations
+
+In the quest for speed, we are going to tweak several settings in nginx. To
+keep things manageable, most of those tweaks go into separate configuration
+files that are either automatically included (those under `/etc/nginx/conf.d`)
+or explicitly where we need them (those under `/etc/nginx/snippets`).
+
+For every `proxy_forward` we want to configure several settings, and because
+we don't want to include the same list of settings every time, we put all of
+them in one snippet of code, that we can include every time we need it.
+
+Create `/etc/nginx/snippets/proxy.conf` and put this in it:
+
+```
+proxy_connect_timeout 2s;
+proxy_buffering off;
+proxy_http_version 1.1;
+proxy_read_timeout 3600s;
+proxy_redirect off;
+proxy_send_timeout 120s;
+proxy_socket_keepalive on;
+proxy_ssl_verify off;
+
+proxy_set_header Accept-Encoding "";
+proxy_set_header Host $host;
+proxy_set_header X-Forwarded-For $remote_addr;
+proxy_set_header X-Forwarded-Proto $scheme;
+proxy_set_header Connection $connection_upgrade;
+proxy_set_header Upgrade $http_upgrade;
+
+client_max_body_size 50M;
+```
+
+Every time we use a `proxy_forward`, we include this snippet.
+
+
 # Maps
 
 A map sets a variable based on, usually, another variable. One case we use this
@@ -76,6 +112,14 @@ Just like what `haproxy` does, it can forward requests to "servers" behind it.
 Such a server is the inbound UNIX socket of a worker, and there can be several
 of them in one group.
 
+Let's start with a simple one, the `login` worker, that handles the login
+process for clients.
+
+```
+login worker komt hier...
+```
+
+
 Two of these upstreams are the sync workers: `normal_sync` and `initial_sync`,
 both consisting of several "servers":
 
@@ -119,11 +163,14 @@ requests:
 ^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$
 ```
 
-Now, if we had only one worker type for synchronisations, named `sync`, not
+Now, if we had only one worker type for synchronisations, named `syncworkers`, not
 splitting those requests up in normal and initial, we would direct all
-sync-requests to that worker with this `location`:
+sync-requests to that worker pool with this `location`:
 
 ```
 location ~ ^(/_matrix/client/(r0|v3)/sync|/_matrix/client/(api/v1|r0|v3)/events|/_matrix/client/(api/v1|r0|v3)/initialSync|/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync)$ {
-    proxy_pass http://sync;
+    proxy_pass http://syncworkers;
 }
+
+That's the concept. 
+
diff --git a/matrix/synapse/temp b/matrix/synapse/temp
deleted file mode 100644
index 8cc6d76..0000000
--- a/matrix/synapse/temp
+++ /dev/null
@@ -1,64 +0,0 @@
-## Temporary block
-
-We're going to configure a few different workers:
-
-* client-sync
-* roomworker
-* federation-sender
-* mediaworker
-
-
-### Client-sync
-
-This type needs both an inbound socket to receive stuff from nginx, and a
-replication socket to communicate with the rest. We probably want a few of
-these workers. The configuration should look like this:
-
-```
-worker_app: "synapse.app.generic_worker" # Always this unless
-"synapse.app.media_repository"
-worker_name: "clientsync1" # Name of worker specified in instance map
-worker_log_config: "/data/log.config/client_sync.log.config" # Log config file
-
-worker_listeners:
-  # Include for any worker in the instance map above:
-  - path: "/run/matrix-synapse/replication_clientsync1.sock"
-    type: http
-    resources:
-      - names: [replication]
-        compress: false
-  # Include for any worker that receives requests in Nginx:
-  - path: "/run/matrix-synapse/synapse_inbound_client_sync1.sock"
-    type: http
-    x_forwarded: true # Trust the X-Forwarded-For header from Nginx
-    resources:
-      - names:
-        - client
-        - consent
-```
-
-### Roomworker
-
-These don't need a replication socket as they're not in the instance map, but
-they do need an inboud socket for nginx to pass stuff to them. We want a few
-of these workers, we may even configure a worker for one specific busy room...
-
-Configuration should look like this:
-
-```
-worker_app: "synapse.app.generic_worker"
-worker_name: "roomworker1"
-worker_log_config: "/data/log.config/rooms.log.config"
-
-worker_listeners:
-  - path: "/run/matrix-synapse/inbound_roomworker1.sock"
-    type: http
-    x_forwarded: true
-    resources:
-      - names:
-        - client
-        - consent
-        - federation
-        compress: false
-```
-
diff --git a/matrix/synapse/workers.md b/matrix/synapse/workers/README.md
similarity index 100%
rename from matrix/synapse/workers.md
rename to matrix/synapse/workers/README.md
diff --git a/matrix/synapse/workers/federation_receiver1.yaml b/matrix/synapse/workers/federation_receiver1.yaml
new file mode 100644
index 0000000..64f394f
--- /dev/null
+++ b/matrix/synapse/workers/federation_receiver1.yaml
@@ -0,0 +1,15 @@
+worker_app: "synapse.app.generic_worker"
+worker_name: "federation_reader1"
+worker_log_config: "/etc/matrix-synapse/logconf.d/federation_reader-log.yaml"
+
+worker_listeners:
+  - path: "/run/matrix-synapse/replication_federation_reader1.sock"
+    type: http
+    resources:
+      - names: [replication]
+
+  - path: "/run/matrix-synapse/inbound_federation_reader1.sock"
+    type: http
+    resources:
+      - names: [federation]
+
diff --git a/matrix/synapse/workers/federation_sender1.yaml b/matrix/synapse/workers/federation_sender1.yaml
new file mode 100644
index 0000000..d2b0399
--- /dev/null
+++ b/matrix/synapse/workers/federation_sender1.yaml
@@ -0,0 +1,10 @@
+worker_app: "synapse.app.generic_worker"
+worker_name: "federation_sender1"
+worker_log_config: "/etc/matrix-synapse/logconf.d/federation_sender-log.yaml"
+
+worker_listeners:
+  - path: "/run/matrix-synapse/replication_federation_sender1.sock"
+    type: http
+    resources:
+      - names: [replication]
+
diff --git a/matrix/synapse/workers/initial_sync1.yaml b/matrix/synapse/workers/initial_sync1.yaml
new file mode 100644
index 0000000..45d9b85
--- /dev/null
+++ b/matrix/synapse/workers/initial_sync1.yaml
@@ -0,0 +1,19 @@
+worker_app: "synapse.app.generic_worker"
+worker_name: "initial_sync1"
+worker_log_config: "/etc/matrix-synapse/logconf.d/initial_sync-log.yaml"
+
+worker_listeners:
+
+  - path: "/run/matrix-synapse/inbound_initial_sync1.sock"
+    type: http
+    resources:
+      - names:
+        - client
+        - consent
+        - federation
+
+  - path: "/run/matrix-synapse/replication_initial_sync1.sock"
+    type: http
+    resources:
+      - names: [replication]
+
diff --git a/matrix/synapse/workers/login-log.yaml b/matrix/synapse/workers/login-log.yaml
new file mode 100644
index 0000000..7cb5975
--- /dev/null
+++ b/matrix/synapse/workers/login-log.yaml
@@ -0,0 +1,41 @@
+version: 1
+formatters:
+  precise:
+    format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+handlers:
+  file:
+    class: logging.handlers.TimedRotatingFileHandler
+    formatter: precise
+    filename: /var/log/matrix-synapse/login.log
+    when: midnight
+    backupCount: 3
+    encoding: utf8
+
+  buffer:
+    class: synapse.logging.handlers.PeriodicallyFlushingMemoryHandler
+    target: file
+    capacity: 10
+    flushLevel: 30
+    period: 5
+
+loggers:
+  synapse.metrics:
+    level: WARN
+    handlers: [buffer]
+  synapse.replication.tcp:
+    level: WARN
+    handlers: [buffer]
+  synapse.util.caches.lrucache:
+    level: WARN
+    handlers: [buffer]
+  twisted:
+    level: WARN
+    handlers: [buffer]
+  synapse:
+    level: INFO
+    handlers: [buffer]
+
+root:
+  level: INFO
+  handlers: [buffer]
+
diff --git a/matrix/synapse/workers/login.yaml b/matrix/synapse/workers/login.yaml
new file mode 100644
index 0000000..c21bd54
--- /dev/null
+++ b/matrix/synapse/workers/login.yaml
@@ -0,0 +1,19 @@
+worker_app: "synapse.app.generic_worker"
+worker_name: "login"
+worker_log_config: "/etc/matrix-synapse/logconf.d/login-log.yaml"
+
+worker_listeners:
+
+  - path: "/run/matrix-synapse/inbound_login.sock"
+    type: http
+    resources:
+      - names:
+        - client
+        - consent
+        - federation
+
+  - path: "/run/matrix-synapse/replication_login.sock"
+    type: http
+    resources:
+      - names: [replication]
+
diff --git a/matrix/synapse/workers/media-log.yaml b/matrix/synapse/workers/media-log.yaml
new file mode 100644
index 0000000..bbddbc1
--- /dev/null
+++ b/matrix/synapse/workers/media-log.yaml
@@ -0,0 +1,41 @@
+version: 1
+formatters:
+  precise:
+    format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+handlers:
+  file:
+    class: logging.handlers.TimedRotatingFileHandler
+    formatter: precise
+    filename: /var/log/matrix-synapse/media.log
+    when: midnight
+    backupCount: 3
+    encoding: utf8
+
+  buffer:
+    class: synapse.logging.handlers.PeriodicallyFlushingMemoryHandler
+    target: file
+    capacity: 10
+    flushLevel: 30
+    period: 5
+
+loggers:
+  synapse.metrics:
+    level: WARN
+    handlers: [buffer]
+  synapse.replication.tcp:
+    level: WARN
+    handlers: [buffer]
+  synapse.util.caches.lrucache:
+    level: WARN
+    handlers: [buffer]
+  twisted:
+    level: WARN
+    handlers: [buffer]
+  synapse:
+    level: INFO
+    handlers: [buffer]
+
+root:
+  level: INFO
+  handlers: [buffer]
+
diff --git a/matrix/synapse/workers/media.yaml b/matrix/synapse/workers/media.yaml
new file mode 100644
index 0000000..65b3bf1
--- /dev/null
+++ b/matrix/synapse/workers/media.yaml
@@ -0,0 +1,15 @@
+worker_app: "synapse.app.media_repository"
+worker_name: "mediaworker"
+worker_log_config: "/etc/matrix-synapse/logconf.d/media-log.yaml"
+
+worker_listeners:
+  - path: "/run/matrix-synapse/inbound_mediaworker.sock"
+    type: http
+    resources:
+      - names: [media]
+
+  - path: "/run/matrix-synapse/replication_mediaworker.sock"
+    type: http
+    resources:
+      - names: [replication]
+