feat(ci): dashboard TLS + kill /grind + multi-org dashboard sections

ci.sh

@@ -124,11 +124,11 @@ export RUN_ID=${RUN_ID:-$(date +%s%3N)}
 
 function multi_job_run {
   if [[ -z "${CI_DASHBOARD:-}" ]]; then
-    if [[ "${REF_NAME:-}" == "main" ]]; then
-      export CI_DASHBOARD="main"
-    else
-      export CI_DASHBOARD="prs"
-    fi
+    # Section = a mainline branch's own name (the default branch, v5, ...), "tags" for
+    # any tag, else "prs" (see ci_dashboard_section in source_refname). log_ci_run
+    # prefixes this with the repo. The trigger sets CI_DASHBOARD from .ci3.yml's
+    # push_branches; this is the fallback for direct/local ci.sh runs.
+    export CI_DASHBOARD="$(ci_dashboard_section)"
   fi
   export AWS_SHUTDOWN_TIME=${AWS_SHUTDOWN_TIME:-75}
   export AWS_SHUTDOWN_TIME_ARM=${AWS_SHUTDOWN_TIME_ARM:-90}
@@ -230,7 +230,9 @@ case "$cmd" in
     # Uses same hash as run_test_cmd's test_hash for consistency
     test_cmd="${full_cmd#* }"
     test_hash=$(hash_str_orig "$test_cmd")
-    export CI_DASHBOARD="deflake"
+    # Grind is a dev tool; its runs go to the repo's "local" section (the deflake
+    # section was retired along with the /grind web endpoint).
+    export CI_DASHBOARD="local"
     export JOB_ID="grind-test-$test_hash"
     export INSTANCE_POSTFIX=$JOB_ID
     export CPUS=${CPUS:-192}
@@ -383,7 +385,7 @@ case "$cmd" in
   release)
     # Spin up ec2 instances (amd64 + arm64) and run the full release flow: backwards-compat e2e
     # checks, build, and publish. Set DRY_RUN=1 to exercise the whole flow without publishing.
-    export CI_DASHBOARD="releases"
+    export CI_DASHBOARD="tags"
     # Roomier instance lifetime than a standard run: the amd64 job builds, runs the backwards-compat
     # e2e suite, and then publishes, which together exceed the default 75 min shutdown.
     export AWS_SHUTDOWN_TIME=${AWS_SHUTDOWN_TIME:-180}
@@ -483,7 +485,9 @@ case "$cmd" in
         echo "No redis available and CI_PASSWORD not set for http fallback."
         exit 1
       fi
-      curl -sf "http://aztec:$CI_PASSWORD@ci.aztec-labs.com/$key.txt" | $pager
+      # https, not http: the dashboard now redirects 80->443, and curl won't resend
+      # inline basic-auth across an http->https redirect (would need --location-trusted).
+      curl -sf "https://aztec:$CI_PASSWORD@ci.aztec-labs.com/$key.txt" | $pager
       if [ ${PIPESTATUS[0]} -ne 0 ]; then
         echo "Failed to fetch log via http."
         exit 1

ci3/dashboard/Caddyfile

@@ -0,0 +1,11 @@
+ci.aztec-labs.com {
+	encode zstd gzip
+	# Long timeouts: the dashboard streams/long-polls live logs. Port 80 stays open
+	# (ACME HTTP-01 + redirect); TLS terminates here, app is plaintext on loopback.
+	reverse_proxy 127.0.0.1:8080 {
+		transport http {
+			read_timeout 3600s
+			write_timeout 3600s
+		}
+	}
+}

ci3/dashboard/ci-metrics/metrics.py

@@ -431,7 +431,6 @@ def get_phases(date_from: str, date_to: str, dashboard: str = '',
 # ---- Sync failed_tests_{section} lists from Redis into SQLite ----
 
 _ANSI_STRIP = re.compile(r'\x1b\[[^m]*m|\x1b\]8;;[^\x07]*\x07')
-_GRIND_CMD_RE = re.compile(r'/grind\?cmd=([^&\x07"]+)')
 _LOG_KEY_RE = re.compile(r'ci\.aztec-labs\.com/([a-f0-9]{16})')
 _INLINE_CMD_RE = re.compile(r'(?:grind\)|[0-9a-f]{16}\)):?\s+(.+?)\s+\(\d+s\)')
 _DURATION_RE = re.compile(r'\((\d+)s\)')
@@ -444,7 +443,6 @@ def get_phases(date_from: str, date_to: str, dashboard: str = '',
 
 def _parse_failed_test_entry(raw: str, section: str) -> dict | None:
     """Parse an ANSI-formatted failed_tests_{section} entry into structured data."""
-    from urllib.parse import unquote
     clean = _ANSI_STRIP.sub('', raw)
 
     # Status
@@ -478,23 +476,12 @@ def _parse_failed_test_entry(raw: str, section: str) -> dict | None:
     if m:
         log_key = m.group(1)
 
-    # Test command: try grind link first, then inline text
+    # Test command: extract from the inline text after the log key.
+    # (Historically a /grind?cmd= link was parsed first; that link was removed.)
     test_cmd = ''
-    m = _GRIND_CMD_RE.search(raw)
+    m = _INLINE_CMD_RE.search(clean)
     if m:
-        cmd_raw = unquote(m.group(1))
-        # Format: "hash:KEY=VAL:KEY=VAL actual_command"
-        # Strip the hash:KEY=VAL prefix to get the actual test command
-        parts = cmd_raw.split(' ', 1)
-        if len(parts) == 2 and ':' in parts[0]:
-            test_cmd = parts[1].strip()
-        else:
-            test_cmd = cmd_raw
-    else:
-        # Fallback: extract from inline text after log key
-        m = _INLINE_CMD_RE.search(clean)
-        if m:
-            test_cmd = m.group(1).strip()
+        test_cmd = m.group(1).strip()
 
     # Duration
     duration = None

ci3/dashboard/deploy.sh

@@ -1,13 +1,52 @@
 #!/bin/bash
+# Deploy the CI dashboard (rkapp + Caddy TLS) to the ci host.
+#
+# Safe to run repeatedly. The first run also performs the one-time cutover from the
+# legacy systemd `rkapp` unit (which bound :80 directly) to the compose stack (rkapp on
+# loopback, Caddy terminating TLS on 443 and redirecting 80). Ordering is chosen so a
+# failure leaves the current service up: the new image is built BEFORE the old unit is
+# retired.
+#
+# Prerequisite: /etc/rkapp.env (mode 600) on the host — the app secrets. deploy.sh
+# refuses to proceed without it rather than bring the app up unconfigured.
 set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+HOST=${1:-ubuntu@ci.aztec-labs.com}
+KEY=~/.ssh/build_instance_key
 
-# Sync dashboard (rkapp) files, including ci-metrics subdirectory
-rsync -avz --exclude='deploy.sh' -e "ssh -i ~/.ssh/build_instance_key" "$SCRIPT_DIR"/* ubuntu@ci.aztec-labs.com:rk
+rsync -avz --exclude='deploy.sh' -e "ssh -i $KEY" "$SCRIPT_DIR"/ "$HOST":rk
 
-ssh -i ~/.ssh/build_instance_key ubuntu@ci.aztec-labs.com "
+ssh -i "$KEY" "$HOST" '
+  set -euo pipefail
+  if [ ! -f /etc/rkapp.env ]; then
+    echo "ERROR: /etc/rkapp.env missing. Create it (mode 600) before deploying." >&2
+    exit 1
+  fi
+  mkdir -p /home/ubuntu/rk/caddy/data /home/ubuntu/rk/caddy/config
   cd rk
-  docker build -t rkapp .
-  sudo systemctl restart rkapp
-"
+
+  # Build the new image first — nothing running is disturbed if this fails.
+  docker compose build
+
+  # Retire the legacy systemd rkapp so Caddy can bind 80/443. Idempotent: a no-op
+  # once it is already gone, so steady-state redeploys skip it.
+  if systemctl list-unit-files rkapp.service >/dev/null 2>&1; then
+    echo "Retiring legacy systemd rkapp unit..."
+    sudo systemctl disable --now rkapp 2>/dev/null || true
+  fi
+
+  docker compose up -d
+
+  # Liveness: the app answers on loopback (401 = up-and-auth-gated, which is fine;
+  # 000 = not listening). Caddy issues its cert on first boot, so https may lag a few
+  # seconds — check it in a browser.
+  sleep 3
+  code=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8080/ || echo 000)
+  if [ "$code" = "000" ]; then
+    echo "ERROR: app not responding on 127.0.0.1:8080" >&2
+    docker compose logs --tail=30 rkapp >&2
+    exit 1
+  fi
+  echo "Dashboard app up (http $code on loopback). Caddy fronting 443; verify https in a browser."
+'

ci3/dashboard/docker-compose.yml

@@ -0,0 +1,41 @@
+# Runs the CI dashboard behind Caddy (TLS) in one stack, replacing the hand-written
+# `docker run` systemd unit that lived only on the ci host. Secrets are NOT inline:
+# they come from /etc/rkapp.env (mode 600, off-repo) — the same pattern as ci3-trigger.
+#
+# Both services use host networking, deliberately unchanged from the old unit: the app
+# reaches ElastiCache and IMDS through the host's network identity, and IMDSv2's default
+# hop limit rejects bridged containers. Caddy therefore reaches the app on loopback.
+#
+# deploy.sh does the whole rollout (build, retire the old rkapp unit, compose up).
+# It requires /etc/rkapp.env (mode 600) to already exist on the host — the app secrets
+# (REDIS_HOST, CI_REDIS, DASHBOARD_PASSWORD, the GCP creds path, AWS region). GH_TOKEN
+# and REPO_PATH from the old unit are deliberately dropped: GH_TOKEN is dead and fed a
+# now-misdirected GitHub poller; REPO_PATH only served the removed /grind route.
+# 443 must be open on bastion_sg (iac change, already applied).
+services:
+  rkapp:
+    build: .
+    image: rkapp
+    network_mode: host
+    restart: always
+    env_file: /etc/rkapp.env
+    volumes:
+      - /home/ubuntu/rk/ci-metrics-gcp-credentials.json:/app/ci-metrics-gcp-credentials.json:ro
+      - /logs-disk:/logs-disk
+      - /home/ubuntu/rk/data:/data
+      - /home/ubuntu/.aws:/root/.aws:ro
+    # Bind loopback only: the app is no longer publicly reachable; Caddy fronts it.
+    command: gunicorn -w 50 -b 127.0.0.1:8080 rk:app
+    logging:
+      driver: json-file
+      options: {max-size: "10m", max-file: "3"}
+
+  caddy:
+    image: caddy:2-alpine
+    network_mode: host
+    restart: always
+    depends_on: [rkapp]
+    volumes:
+      - ./Caddyfile:/etc/caddy/Caddyfile:ro
+      - /home/ubuntu/rk/caddy/data:/data
+      - /home/ubuntu/rk/caddy/config:/config