From a18e10bb28a3d6073790eda38044858dbd4f0f7c Mon Sep 17 00:00:00 2001 From: Omar Sy Date: Sat, 13 Jun 2026 19:11:18 +0200 Subject: [PATCH] fix(nginx): reload via SIGHUP instead of starting a second master nginx.sh "reloaded" by running bare `nginx`, which starts a second master that cannot bind 0.0.0.0:443 ("Address in use") and exits. So after the first start, no cert or config change ever took effect: cert rotations were silently dropped until a full container restart, and the manager's periodic CRL copy into /root triggered a failed-reload storm every sync interval (incident 2026-06-13). Use `nginx -s reload` (keeps the listening sockets, re-execs workers) when a master is already running, and `nginx` only to start one. Start once up front so a container restart with certs already on disk brings nginx up without waiting for an event, and exclude *.crl from the watch since this nginx never references the CRL. Watch in --monitor mode so no event is lost: the manager installs the cert as three separate writes, and a one-shot inotifywait could exit on the .crt event and miss the .key write while `nginx -t` runs, leaving nginx un-(re)loaded until an unrelated later change. An outer loop respawns the watch if it dies. Surface `nginx -t` failures when the cert is present instead of silently doing nothing. Co-Authored-By: Claude Opus 4.8 --- nginx/nginx.sh | 63 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/nginx/nginx.sh b/nginx/nginx.sh index d3d4422..9b93d6d 100644 --- a/nginx/nginx.sh +++ b/nginx/nginx.sh @@ -1,12 +1,55 @@ #!/bin/sh +# Watch /root for cert/config changes and (re)load nginx accordingly. +# +# The previous version "reloaded" by running bare `nginx`, which starts a +# SECOND master that cannot bind 0.0.0.0:443 ("Address in use") and exits, so +# after the first start NO change ever took effect: cert rotations were silently +# dropped until a full container restart. SIGHUP (`nginx -s reload`) keeps the +# listening sockets and re-execs workers with the new config. +# +# We also exclude *.crl: the manager's CRL is copied into /root for historical +# reasons but this nginx never references it (no ssl_crl here), so reacting to +# it just churned a reload every sync interval (incident 2026-06-13). +# +# We watch in --monitor mode so NO event is ever lost. The manager installs the +# cert as separate Docker writes (pem.crt, then cert.key, then ca.pem); a +# one-shot inotifywait exits after the first event and stops watching while +# `nginx -t` runs, so a key write landing in that window is missed and nginx +# never (re)loads until some unrelated later change. In monitor mode the kernel +# queues events while we process, so the iteration where both cert and key are +# present always converges. The outer loop respawns the watch if it ever dies. + +start_or_reload() { + # `nginx -t` fails when the cert material is not present yet (a fresh + # container has an empty /root until the manager installs the cert), which + # is expected. Distinguish that from a genuinely broken config so a real + # error is not swallowed silently. + if ! nginx -t >/dev/null 2>&1; then + if [ -s /root/pem.crt ]; then + echo "nginx -t failed with cert present (bad config?), not (re)loading:" + nginx -t + fi + return 0 + fi + if pidof nginx >/dev/null 2>&1; then + echo "Reloading Nginx Configuration" + nginx -s reload + else + echo "Starting Nginx" + nginx + fi +} + +# Bring nginx up if usable config + certs are already in place (e.g. a container +# restart with the certs still on disk); otherwise the first event starts it. +start_or_reload + while true; do - inotifywait --exclude .crt -e create -e modify -e delete -e move /root/ - # Check NGINX Configuration Test - # Only Reload NGINX If NGINX Configuration Test Pass - nginx -t - if [ $? -eq 0 ] - then - echo "Reloading Nginx Configuration" - nginx - fi -done \ No newline at end of file + inotifywait -m -q --exclude '\.crl$' -e create -e modify -e delete -e move /root/ | + while read -r _; do + start_or_reload + done + # inotifywait exited (watch lost / error); back off so we can't busy-spin, + # then respawn the monitor. + sleep 5 +done