protect small servers by rejecting new addresses under resource constraints / capping smtp/imap connection limits

README.md

@@ -1,7 +1,7 @@
 
 # Chatmail relays for end-to-end encrypted email
 
-Chatmail relay servers are interoperable Mail Transport Agents (MTAs) designed for: 
+Chatmail relay servers are interoperable Mail Transport Agents (MTAs) designed for:
 
 -  **Zero State:** no private data or metadata collected, messages are auto-deleted, low disk usage
 
@@ -18,7 +18,7 @@ Chatmail relay servers are interoperable Mail Transport Agents (MTAs) designed f
 -  **Reliable Federation and Decentralization:** No spam or IP reputation checks, federating
    depends on established IETF standards and protocols.
 
-This repository contains everything needed to setup a ready-to-use chatmail relay on an ssh-reachable host. 
+This repository contains everything needed to setup a ready-to-use chatmail relay on an ssh-reachable host.
 For getting started and more information please refer to the web version of this repositories' documentation at
 
 [https://chatmail.at/doc/relay](https://chatmail.at/doc/relay)

chatmaild/pyproject.toml

@@ -8,6 +8,7 @@ version = "0.3"
 dependencies = [
   "iniconfig",
   "filelock",
+  "psutil",
   "requests",
   "crypt-r >= 3.13.1 ; python_version >= '3.13'",
 ]

chatmaild/src/chatmaild/config.py

@@ -75,6 +75,16 @@ def __init__(self, inipath, params):
         self.privacy_pdo = params.pop("privacy_pdo", None)
         self.privacy_supervisor = params.pop("privacy_supervisor", None)
 
+        self.max_load_1m = float(params.pop("max_load_1m", 5))
+        self.min_available_memory_mb = parse_size_mb(
+            params.pop("min_available_memory", "200M")
+        )
+        self.min_free_disk_space_mb = parse_size_mb(
+            params.pop("min_free_disk_space", "1G")
+        )
+        self.max_imap_connections = int(params.pop("max_imap_connections", 10000))
+        self.max_smtp_connections = int(params.pop("max_smtp_connections", 1000))
+
         # TLS certificate management.
         # If tls_external_cert_and_key is set, use externally managed certs.
         # Otherwise derived from the domain name:

chatmaild/src/chatmaild/doveauth.py

@@ -14,6 +14,7 @@
 from .config import Config, read_config
 from .dictproxy import DictProxy
 from .migrate_db import migrate_from_db_to_maildir
+from .syslimits import has_sufficient_resources
 
 NOCREATE_FILE = "/etc/chatmail-nocreate"
 VALID_LOCALPART_RE = re.compile(r"^[a-z0-9._-]+$")
@@ -147,6 +148,8 @@ def lookup_passdb(self, addr, cleartext_password):
             return userdata
         if not is_allowed_to_create(self.config, addr, cleartext_password):
             return
+        if not has_sufficient_resources(self.config):
+            return
 
         lock = filelock.FileLock(str(user.password_path) + ".lock", timeout=5)
         with lock:

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -42,6 +42,33 @@
 # minimum length a password must have
 #password_min_length = 9
 
+#
+# System resource limits
+#
+
+# The following three limits refuse creation of new addresses
+# while existing addresses keep working.
+# Rejections are logged by the doveauth service.
+
+# Maximum 1-minute load average, as reported by "uptime";
+# it counts processes waiting for disk I/O as well as for CPU.
+#max_load_1m = 5
+
+# Minimum memory available without swapping.
+#min_available_memory = 200M
+
+# Minimum free disk space on the file system holding the mailboxes.
+#min_free_disk_space = 1G
+
+# Maximum number of concurrent IMAP connections
+# (the Dovecot imap process limit).
+#max_imap_connections = 10000
+
+# Maximum number of concurrent SMTP connections
+# on each of the submission and smtps ports (the Postfix process limit).
+# A single client IP may use up to a fifth of this.
+#max_smtp_connections = 1000
+
 # Use externally managed TLS certificates instead of built-in acmetool.
 # Paths refer to files on the deployment server (not the build machine).
 # Both files must already exist before running cmdeploy.

chatmaild/src/chatmaild/syslimits.py

@@ -0,0 +1,32 @@
+"""Detect whether the system is at its limits."""
+
+import logging
+
+import psutil
+
+MB = 1024 * 1024
+
+
+def read_value(getter):
+    try:
+        return getter()
+    except Exception as e:
+        logging.warning("ignoring unreadable system limit: %s", e)
+        return None
+
+
+def has_sufficient_resources(config):
+    """Return False if load, memory or disk exceeds a configured limit."""
+    load = read_value(lambda: psutil.getloadavg()[0])
+    mem = read_value(lambda: psutil.virtual_memory().available // MB)
+    disk = read_value(lambda: psutil.disk_usage(str(config.mailboxes_dir)).free // MB)
+    if load is not None and load > config.max_load_1m:
+        msg = f"load avg {load:.2f} > {config.max_load_1m:.2f}"
+    elif mem is not None and mem < config.min_available_memory_mb:
+        msg = f"available memory {mem}MB < {config.min_available_memory_mb}MB"
+    elif disk is not None and disk < config.min_free_disk_space_mb:
+        msg = f"free disk {disk}MB < {config.min_free_disk_space_mb}MB"
+    else:
+        return True
+    logging.warning("registration rejected: %s", msg)
+    return False

chatmaild/src/chatmaild/tests/plugin.py

@@ -20,6 +20,10 @@ def make_conf(mail_domain, settings=None):
         basedir.mkdir(parents=True, exist_ok=True)
         overrides = settings.copy() if settings else {}
         overrides["mailboxes_dir"] = str(basedir)
+        # permissive resource limits so tests never depend on host load/memory/disk
+        overrides.setdefault("max_load_1m", "99999")
+        overrides.setdefault("min_available_memory", "0")
+        overrides.setdefault("min_free_disk_space", "0")
         write_initial_config(inipath, mail_domain, overrides=overrides)
         return read_config(inipath)
 

chatmaild/src/chatmaild/tests/test_config.py

@@ -45,6 +45,8 @@ def test_read_config_basic_using_defaults(tmp_path, maildomain):
     assert example_config.username_min_length == 9
     assert example_config.username_max_length == 9
     assert example_config.password_min_length == 9
+    assert example_config.max_imap_connections == 10000
+    assert example_config.max_smtp_connections == 1000
     assert example_config._unused_keys == []
 
 

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -202,3 +202,17 @@ def lookup():
         res = results.get()
         if res is not None:
             pytest.fail(f"concurrent lookup failed\n{res}")
+
+
+def test_insufficient_resources_block_creation_not_existing_logins(
+    dictproxy, gencreds, monkeypatch
+):
+    addr, password = gencreds()
+    assert dictproxy.lookup_passdb(addr, password)
+
+    monkeypatch.setattr(
+        chatmaild.doveauth, "has_sufficient_resources", lambda config: False
+    )
+    newaddr, newpassword = gencreds()
+    assert not dictproxy.lookup_passdb(newaddr, newpassword)
+    assert dictproxy.lookup_passdb(addr, password)

chatmaild/src/chatmaild/tests/test_syslimits.py

@@ -0,0 +1,45 @@
+import shutil
+
+import psutil
+
+from chatmaild.syslimits import has_sufficient_resources
+
+PERMISSIVE = {
+    "max_load_1m": "99999",
+    "min_available_memory": "0",
+    "min_free_disk_space": "0",
+}
+
+
+def test_rejects_constrained_system(make_config, caplog):
+    assert has_sufficient_resources(make_config("chat.example.org", PERMISSIVE))
+    for settings in (
+        {"max_load_1m": "-1.0"},
+        {"min_available_memory": "99999999G"},
+        {"min_free_disk_space": "99999999G"},
+    ):
+        config = make_config("chat.example.org", PERMISSIVE | settings)
+        caplog.clear()
+        assert not has_sufficient_resources(config), settings
+        assert "registration rejected" in caplog.text
+
+
+def test_unreadable_disk_does_not_reject(make_config, caplog):
+    config = make_config(
+        "chat.example.org", PERMISSIVE | {"min_free_disk_space": "99999999G"}
+    )
+    shutil.rmtree(config.mailboxes_dir)
+    assert has_sufficient_resources(config)
+    assert "ignoring" in caplog.text
+
+
+def test_one_unreadable_value_keeps_other_checks(make_config, monkeypatch, caplog):
+    def raise_error(*args):
+        raise psutil.Error("dud")
+
+    monkeypatch.setattr(psutil, "getloadavg", raise_error)
+    config = make_config(
+        "chat.example.org", PERMISSIVE | {"min_free_disk_space": "99999999G"}
+    )
+    assert not has_sufficient_resources(config)
+    assert "ignoring" in caplog.text

cmdeploy/src/cmdeploy/deployers.py

@@ -356,6 +356,8 @@ class ChatmailVenvDeployer(Deployer):
     def __init__(self, config):
         self.config = config
         self.units = (
+            # doveauth must restart when chatmaild/ini file changes
+            "doveauth",
             "chatmail-metadata",
             "lastlogin",
             "chatmail-expire",

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -34,7 +34,7 @@ class DovecotDeployer(Deployer):
     def __init__(self, config, disable_mail):
         self.config = config
         self.disable_mail = disable_mail
-        self.units = ["doveauth"]
+        self.units = []
 
     def install(self):
         arch = host.get_fact(Arch)

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -37,7 +37,7 @@ default_client_limit = 20000
 # the following warning will be logged:
 #   Warning: service(imap): process_limit (1024) reached, client connections are being dropped
 service imap {
-  process_limit = 50000
+  process_limit = {{ config.max_imap_connections }}
 }
 
 {% if config.privacy_mail %}

cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail

@@ -28,6 +28,13 @@ counter created_nonci_accounts
   }
 }
 
+# doveauth refusing new addresses because a chatmail.ini
+# system resource limit is exceeded.
+counter rejected_registrations
+/registration rejected: / {
+  rejected_registrations++
+}
+
 counter postfix_timeouts
 /timeout after DATA/ {
   postfix_timeouts++

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -18,7 +18,7 @@ smtp      inet  n       -       y       -       -       smtpd
   -o smtpd_tls_mandatory_protocols=>=TLSv1.2
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port_incoming }}
   -o smtpd_relay_restrictions=reject_unauth_destination
-submission inet n       -       y       -       5000    smtpd
+submission inet n       -       y       -       {{ config.max_smtp_connections }}    smtpd
   -o syslog_name=postfix/submission
   -o smtpd_tls_security_level=encrypt
   -o smtpd_tls_mandatory_protocols=>=TLSv1.3
@@ -32,9 +32,9 @@ submission inet n       -       y       -       5000    smtpd
   -o smtpd_sender_restrictions=$mua_sender_restrictions
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
-  -o smtpd_client_connection_count_limit=1000
+  -o smtpd_client_connection_count_limit={{ config.max_smtp_connections // 5 }}
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
-smtps     inet  n       -       y       -       5000    smtpd
+smtps     inet  n       -       y       -       {{ config.max_smtp_connections }}    smtpd
   -o syslog_name=postfix/smtps
   -o smtpd_tls_wrappermode=yes
   -o smtpd_tls_security_level=encrypt
@@ -48,7 +48,7 @@ smtps     inet  n       -       y       -       5000    smtpd
   -o smtpd_sender_restrictions=$mua_sender_restrictions
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
-  -o smtpd_client_connection_count_limit=1000
+  -o smtpd_client_connection_count_limit={{ config.max_smtp_connections // 5 }}
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup

doc/source/getting_started.rst

@@ -199,6 +199,93 @@ creating addresses, login with ssh to the deployment machine and run:
 Chatmail address creation will be denied while this file is present.
 
 
+.. _system-limits:
+
+Configurable System Limits
+--------------------------
+
+Limits for auto-rejecting address creation
+..........................................
+
+A relay refuses creation of new addresses
+when the machine runs low on resources,
+but existing addresses keep working.
+
+Three ``chatmail.ini`` settings control this,
+shown here with their defaults::
+
+    max_load_1m = 5
+    min_available_memory = 200M
+    min_free_disk_space = 1G
+
+- ``max_load_1m`` is the maximum 1-minute load average,
+  as reported by ``uptime``;
+  it counts processes waiting for disk I/O as well as for CPU.
+  It is deliberately not scaled by the number of CPUs
+  because I/O rather than CPU is what typically limits a relay.
+
+- ``min_available_memory`` is the minimum memory available without swapping.
+
+- ``min_free_disk_space`` is the minimum free disk space
+  on the file system holding the mailboxes.
+
+The defaults suit the small machine described in
+`Minimal requirements and prerequisites`_.
+
+.. note::
+
+   If you run a bigger machine,
+   raise ``max_load_1m`` after watching ``uptime`` under typical load.
+
+Rejections are logged by the ``doveauth`` service,
+so you can check whether a limit is set too tightly::
+
+    journalctl -u doveauth --grep 'registration rejected'
+
+If ``mtail_address`` is set, rejections are also counted
+in the ``rejected_registrations`` metric.
+
+
+Overall IMAP and SMTP connection limits
+.......................................
+
+Two further settings bound how many connections
+the relay accepts at all, again shown with their defaults::
+
+    max_imap_connections = 10000
+    max_smtp_connections = 1000
+
+``max_imap_connections`` becomes the Dovecot imap process limit,
+and ``max_smtp_connections`` the Postfix process limit
+on each of the submission and smtps ports.
+A single client IP may use up to a fifth of ``max_smtp_connections``.
+Each connection costs memory,
+so these limits defend the relay against running out of RAM.
+
+Unless ``imap_compress`` is enabled,
+an IMAP connection that is idle for ``imap_hibernate_timeout``
+is handed over to the ``imap-hibernate`` process
+and does not count towards ``max_imap_connections``,
+which is why a relay can serve far more IMAP clients
+than this setting suggests.
+
+If you run a large relay with 10k or 100k's of addresses,
+check current connection counts before upgrading
+and set the limits accordingly.
+
+To see how close a running relay is to these two limits,
+copy ``scripts/check-connections.sh`` from the relay repository
+onto the relay and run it there::
+
+    imap             5  ports 143,993   (max_imap_connections)
+                     5  dovecot sessions, 0 of them in an active imap process
+    submission       0  ports 465,587   (max_smtp_connections per port)
+    incoming         0  port 25         (from other relays, no chatmail.ini limit)
+
+It counts established sockets with ``ss``
+and cross-checks the IMAP number against ``doveadm who``.
+
+
 Running a relay with self-signed certificates
 ----------------------------------------------