test(opal-server): git leak/resilience test environment (PR1)

.gitignore

@@ -137,3 +137,6 @@ dmypy.json
 *.iml
 
 .DS_Store
+
+# Private Claude Code working artifacts (plans/specs) — never commit
+.claude/

app-tests/git-leak/README.md

@@ -0,0 +1,48 @@
+# OPAL git-leak / resilience test bed
+
+Reproduces (as failing tests on `master`) the four issues fixed by PR2–PR5:
+memory leak, offline-repo hang, slow serial boot, broadcaster no-reconnect.
+
+## Stack
+- `opal_server` (2 workers, scopes on, Postgres broadcaster, built from `docker/Dockerfile`)
+- `redis`, `postgres`, `gitea` (+ one-shot `gitea-admin` and `seed` sidecars)
+
+Only `opal_server` (`:7002`) and `gitea` (`:13000` on the host) are published;
+Postgres is internal to the compose network.
+
+## Helpers (`helpers.py`)
+- `OpalServerClient` — drive opal over HTTP (`stats`, `put_scope`, `delete_scope`, `refresh_all`).
+- `GiteaAdmin` — host-side Gitea admin client (`list_repos`, `repo_exists`,
+  `create_repo`, `delete_repo`); also exposed as the `gitea_admin` pytest fixture.
+- `make_repo_unreachable(name)` — git URL on a routable-but-dead host (TEST-NET-1) for the offline-repo test.
+- `bounce_postgres(down_seconds)` — stop/start Postgres to simulate a broadcaster outage.
+
+## Run
+```bash
+cd app-tests/git-leak
+python -m pytest -v --boot-scopes=50              # full set
+python -m pytest test_leak.py -v --boot-scopes=20 # just the leak gates
+```
+Useful flags: `--boot-scopes=N` (any N), `--keep-stack` (skip teardown),
+env `BOOT_TARGET_SECONDS=120` (tighten the boot gate).
+
+## Expected on master
+The leak tests (#1, #2) and the offline-repo test (#4) FAIL on master — they
+target unfixed bugs and become the regression gates for PR2/PR3. The boot test
+(#3) passes but only fails when `BOOT_TARGET_SECONDS` is set low (PR4's gate).
+The Postgres-bounce test (#5) PASSES on master: it is a recovery guard — when
+the broadcaster drops, the worker is respawned by gunicorn while the sibling
+worker keeps serving, so the HTTP surface recovers. It guards that property
+against regression rather than reproducing a current failure.
+
+## Requires
+Docker + docker compose v2, plus host Python with `pytest pytest-timeout requests GitPython`.
+
+## Notes
+- Auth is disabled in the stack: `OPAL_AUTH_PUBLIC_KEY` is left unset so the JWT
+  verifier is disabled and the harness can call scope routes without minting JWTs.
+  Local test bed only; never a production setting.
+- The server runs 2 uvicorn workers with a Postgres broadcaster, mirroring a
+  realistic multi-worker deployment. The `GitPolicyFetcher` caches read by the
+  `/internal/git-fetcher-cache-stats` endpoint are per-process, so the harness
+  polls with generous timeouts to let the leader worker converge.

app-tests/git-leak/conftest.py

@@ -0,0 +1,49 @@
+import os
+
+import pytest
+from helpers import GiteaAdmin, OpalServerClient, compose
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--boot-scopes",
+        action="store",
+        default="50",
+        help="number of repos to seed/boot (default 50)",
+    )
+    parser.addoption(
+        "--keep-stack",
+        action="store_true",
+        default=False,
+        help="do not tear the compose stack down after the run",
+    )
+
+
+@pytest.fixture(scope="session")
+def repo_count(request) -> int:
+    return int(request.config.getoption("--boot-scopes"))
+
+
+@pytest.fixture(scope="session")
+def stack(request, repo_count):
+    os.environ["REPO_COUNT"] = str(repo_count)
+    # build + start infra; seed runs to completion then exits
+    compose("up", "-d", "--build")
+    # block until seeding sidecar has finished creating repos
+    compose("wait", "seed")
+    client = OpalServerClient()
+    client.wait_healthy()
+    yield client
+    if not request.config.getoption("--keep-stack"):
+        compose("down", "-v")
+
+
+@pytest.fixture()
+def opal(stack) -> OpalServerClient:
+    return stack
+
+
+@pytest.fixture(scope="session")
+def gitea_admin(stack) -> GiteaAdmin:
+    """Host-side Gitea admin client (depends on `stack` so Gitea is up)."""
+    return GiteaAdmin()

app-tests/git-leak/docker-compose.yml

@@ -0,0 +1,106 @@
+name: opal-git-leak-test
+
+services:
+  redis:
+    image: redis:7-alpine
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 2s
+      timeout: 3s
+      retries: 30
+
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER: opal
+      POSTGRES_PASSWORD: opal
+      POSTGRES_DB: opal
+    # not published to the host: only opal_server reaches it over the compose
+    # network, and bounce_postgres() uses `docker compose stop/start`. Publishing
+    # 5432 would collide with any Postgres already running on the host.
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U opal"]
+      interval: 2s
+      timeout: 3s
+      retries: 30
+
+  gitea:
+    image: gitea/gitea:1.21
+    environment:
+      GITEA__security__INSTALL_LOCK: "true"
+      GITEA__server__ROOT_URL: "http://gitea:3000/"
+      GITEA__database__DB_TYPE: "sqlite3"
+    # published on 13000 (not 3000) for the host-side GiteaAdmin helper; the
+    # uncommon port avoids the usual :3000 clash. opal_server and the seed
+    # sidecar still reach it over the compose network via http://gitea:3000.
+    ports:
+      - "13000:3000"
+    volumes:
+      - gitea-data:/data
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://localhost:3000/api/v1/version || exit 1"]
+      interval: 3s
+      timeout: 5s
+      retries: 40
+
+  gitea-admin:
+    # creates the admin user once gitea is healthy
+    image: gitea/gitea:1.21
+    depends_on:
+      gitea:
+        condition: service_healthy
+    user: git
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - >
+        gitea admin user create --username opaladmin --password opaladmin
+        --email admin@example.com --admin --must-change-password=false
+        --config /data/gitea/conf/app.ini || true
+    volumes:
+      - gitea-data:/data
+    restart: "no"
+
+  seed:
+    build: ./seed
+    depends_on:
+      gitea:
+        condition: service_healthy
+      gitea-admin:
+        condition: service_completed_successfully
+    environment:
+      GITEA_URL: "http://gitea:3000"
+      GITEA_ADMIN_USER: "opaladmin"
+      GITEA_ADMIN_PASSWORD: "opaladmin"
+      REPO_COUNT: "${REPO_COUNT:-50}"
+    volumes:
+      - seed-output:/seed-output
+    restart: "no"
+
+  opal_server:
+    build:
+      context: ../..
+      dockerfile: docker/Dockerfile
+      target: server
+    environment:
+      UVICORN_NUM_WORKERS: "2"
+      OPAL_SCOPES: "1"
+      OPAL_REDIS_URL: "redis://redis:6379"
+      OPAL_BROADCAST_URI: "postgres://opal:opal@postgres:5432/opal"
+      OPAL_BASE_DIR: "/opal"
+      OPAL_POLICY_REFRESH_INTERVAL: "0"
+      OPAL_DEBUG_INTERNAL_STATS: "1"
+      # OPAL_AUTH_PUBLIC_KEY is intentionally left unset: with no public key the
+      # JWT verifier is disabled, so the harness can call scope routes without
+      # minting JWTs. Local test bed only; never a production setting.
+      OPAL_LOG_FORMAT_INCLUDE_PID: "true"
+    ports:
+      - "7002:7002"
+    depends_on:
+      redis:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+
+volumes:
+  gitea-data:
+  seed-output:

app-tests/git-leak/helpers.py

@@ -0,0 +1,185 @@
+"""HTTP + infra helpers for the git-leak test bed."""
+import subprocess
+import time
+from pathlib import Path
+from typing import Dict, List
+
+import requests
+
+OPAL_URL = "http://localhost:7002"
+# reachable from inside the opal_server container (compose network)
+GITEA_INTERNAL_URL = "http://gitea:3000"
+# reachable from the host-side test harness (published port, see docker-compose.yml)
+GITEA_HOST_URL = "http://localhost:13000"
+GITEA_USER = "opaladmin"
+GITEA_PASSWORD = "opaladmin"
+
+# TEST-NET-1 (RFC 5737): routable but runs no git server, so a clone hangs
+# instead of failing fast — used to simulate an offline/unreachable repo.
+UNREACHABLE_HOST = "192.0.2.1"
+
+# the compose project lives next to this file; compose() runs from here
+_COMPOSE_DIR = str(Path(__file__).resolve().parent)
+
+
+class OpalServerClient:
+    def __init__(self, base_url: str = OPAL_URL):
+        self.base_url = base_url.rstrip("/")
+
+    def wait_healthy(self, timeout: int = 180) -> None:
+        deadline = time.time() + timeout
+        last = None
+        while time.time() < deadline:
+            try:
+                if (
+                    requests.get(f"{self.base_url}/healthcheck", timeout=5).status_code
+                    == 200
+                ):
+                    return
+            except requests.RequestException as exc:
+                last = exc
+            time.sleep(2)
+        raise RuntimeError(f"opal-server not healthy in {timeout}s (last: {last})")
+
+    def stats(self) -> Dict[str, int]:
+        resp = requests.get(
+            f"{self.base_url}/internal/git-fetcher-cache-stats", timeout=10
+        )
+        resp.raise_for_status()
+        return resp.json()
+
+    def put_scope(self, scope_id: str, repo_url: str, branch: str = "main") -> None:
+        body = {
+            "scope_id": scope_id,
+            "policy": {
+                "source_type": "git",
+                "url": repo_url,
+                "auth": {"auth_type": "none"},
+                "branch": branch,
+                "directories": ["."],
+                "extensions": [".rego", ".json"],
+                "manifest": ".manifest",
+                "poll_updates": False,
+            },
+            "data": {"entries": []},
+        }
+        # the scope router mounts at prefix="/scopes" with @router.put("")
+        resp = requests.put(f"{self.base_url}/scopes", json=body, timeout=30)
+        resp.raise_for_status()
+
+    def delete_scope(self, scope_id: str) -> None:
+        resp = requests.delete(f"{self.base_url}/scopes/{scope_id}", timeout=30)
+        if resp.status_code not in (200, 204, 404):
+            resp.raise_for_status()
+
+    def refresh_all(self) -> None:
+        # publishes a refresh on the webhook topic; leader pulls all scopes
+        resp = requests.post(f"{self.base_url}/scopes/refresh", timeout=30)
+        if resp.status_code == 404:
+            return  # endpoint name differs across versions; caller falls back to poll
+        resp.raise_for_status()
+
+
+class GiteaAdmin:
+    """Host-side admin client for the test bed's Gitea.
+
+    The ``seed`` sidecar does the bulk repo creation from inside the compose
+    network; this class lets a test inspect or mutate Gitea repos directly
+    from the host (e.g. assert seeding happened, or add/remove a single repo
+    for a specific scenario). It authenticates with the admin user that the
+    ``gitea-admin`` sidecar created, over the published host port.
+    """
+
+    def __init__(
+        self,
+        base_url: str = GITEA_HOST_URL,
+        user: str = GITEA_USER,
+        password: str = GITEA_PASSWORD,
+    ):
+        self.base_url = base_url.rstrip("/")
+        self._user = user
+        self._auth = (user, password)
+
+    def repo_exists(self, name: str) -> bool:
+        resp = requests.get(
+            f"{self.base_url}/api/v1/repos/{self._user}/{name}",
+            auth=self._auth,
+            timeout=10,
+        )
+        return resp.status_code == 200
+
+    def list_repos(self) -> List[str]:
+        names: List[str] = []
+        page = 1
+        while True:
+            resp = requests.get(
+                f"{self.base_url}/api/v1/users/{self._user}/repos",
+                params={"page": page, "limit": 50},
+                auth=self._auth,
+                timeout=10,
+            )
+            resp.raise_for_status()
+            batch = resp.json()
+            if not batch:
+                break
+            names.extend(r["name"] for r in batch)
+            page += 1
+        return names
+
+    def create_repo(self, name: str) -> None:
+        if self.repo_exists(name):
+            return
+        resp = requests.post(
+            f"{self.base_url}/api/v1/user/repos",
+            json={"name": name, "private": False, "auto_init": True},
+            auth=self._auth,
+            timeout=10,
+        )
+        resp.raise_for_status()
+
+    def delete_repo(self, name: str) -> None:
+        resp = requests.delete(
+            f"{self.base_url}/api/v1/repos/{self._user}/{name}",
+            auth=self._auth,
+            timeout=10,
+        )
+        if resp.status_code not in (204, 404):
+            resp.raise_for_status()
+
+
+def gitea_repo_url(name: str) -> str:
+    # url reachable from inside the opal_server container
+    return f"{GITEA_INTERNAL_URL}/{GITEA_USER}/{name}.git"
+
+
+def make_repo_unreachable(name: str) -> str:
+    """Return a git URL for ``name`` pointing at a routable-but-dead host.
+
+    Simulates an offline/unreachable policy repo: the address is in
+    TEST-NET-1 (RFC 5737), which is routable but runs no git server, so a
+    clone hangs rather than failing fast — exercising the missing fetch
+    timeout on the scopes path (the bug PR3 fixes). The URL keeps the same
+    ``/{user}/{name}.git`` shape as a real Gitea repo so the scope looks
+    ordinary apart from the unreachable host.
+    """
+    return f"http://{UNREACHABLE_HOST}/{GITEA_USER}/{name}.git"
+
+
+def compose(*args: str) -> subprocess.CompletedProcess:
+    return subprocess.run(
+        ["docker", "compose", *args],
+        cwd=_COMPOSE_DIR,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+
+
+def bounce_postgres(down_seconds: int = 5) -> None:
+    compose("stop", "postgres")
+    time.sleep(down_seconds)
+    compose("start", "postgres")
+
+
+def list_seeded_repos(count: int) -> List[str]:
+    return [f"policy-repo-{i:04d}" for i in range(count)]