dev: local-stack — full tier-routing dev environment (real CPU engine, real token counts)

dev/local-stack/.gitignore

@@ -0,0 +1,6 @@
+cache/
+run/
+*.log
+*.pid
+.api-key
+SATURATE-*

dev/local-stack/Makefile

@@ -0,0 +1,42 @@
+SHELL := /bin/bash
+STACK_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+REPO_ROOT := $(abspath $(STACK_DIR)/../..)
+MODEL_PROXY_DIR ?= $(REPO_ROOT)/../model-proxy
+
+.PHONY: up shims api seed demo proxy-demo down logs
+
+## Bring up the containers (Postgres + llama.cpp engine; first start downloads
+## the ~400MB GGUF into ./cache) and the two tier shims.
+up:
+	cd $(STACK_DIR) && docker compose up -d --wait
+	cd $(STACK_DIR) && (python3 tier_shim.py --tier base --port 18100 --ctx-window 1000 --model-id z-ai/glm-5.2-local > shim-base.log 2>&1 & echo $$! > .shim-base.pid)
+	cd $(STACK_DIR) && (python3 tier_shim.py --tier long --port 18101 --ctx-window 8000 --model-id z-ai/glm-5.2-local > shim-long.log 2>&1 & echo $$! > .shim-long.pid)
+	@sleep 1 && curl -sf http://127.0.0.1:18100/health >/dev/null && curl -sf http://127.0.0.1:18101/health >/dev/null \
+		&& echo "engine + shims up (base :18100 ctx=1000, long :18101 ctx=8000)"
+
+## Run cloud-api in the foreground against the stack (debug build; DEV=1
+## ephemeral signing keys). Ctrl-C to stop.
+api:
+	cd $(REPO_ROOT) && set -a && source $(STACK_DIR)/env.local-stack && set +a && cargo run --bin api
+
+## Seed model + org + API key (needs `make api` running in another terminal).
+seed:
+	$(STACK_DIR)/seed.sh
+
+## Tier-routing demo (small/oversize/boundary/streaming/saturated).
+demo:
+	$(STACK_DIR)/demo.sh
+
+## Model-proxy control-plane demo (dual-probe registration, shared-backend
+## ownership guard, health-gated stub + circuit breaker). Needs a model-proxy
+## checkout at $(MODEL_PROXY_DIR) (override: make proxy-demo MODEL_PROXY_DIR=…).
+proxy-demo:
+	MODEL_PROXY_DIR=$(MODEL_PROXY_DIR) $(STACK_DIR)/model-proxy-demo/run.sh
+
+down:
+	-cd $(STACK_DIR) && for f in .shim-base.pid .shim-long.pid; do [ -f $$f ] && kill $$(cat $$f) 2>/dev/null; rm -f $$f; done
+	-cd $(STACK_DIR) && rm -f SATURATE-base SATURATE-long
+	cd $(STACK_DIR) && docker compose down
+
+logs:
+	@cd $(STACK_DIR) && tail -n 40 shim-base.log shim-long.log 2>/dev/null || true

dev/local-stack/README.md

@@ -0,0 +1,82 @@
+# Local dev stack — full tier-routing environment on one machine
+
+Runs the real cloud-api against a **real inference engine on CPU** with the
+GLM-5.2-style two-tier context topology (base fleet + long-context tier), so
+routing changes can be exercised end to end without touching staging or any
+GPU host:
+
+```
+curl ──► cloud-api :13000 (cargo run, mock auth, ephemeral signing keys)
+             │                        ▲ Postgres :15432 (docker)
+             ├─ base tier ─► tier_shim :18100 (ctx 1000) ──┐
+             └─ long tier ─► tier_shim :18101 (ctx 8000) ──┴─► llama.cpp :18090
+                                                (docker, CPU, Qwen2.5-0.5B GGUF)
+```
+
+The tier shims play the role of the per-tier SGLang fleets: they advertise the
+canonical model id, serve **real completions** and **real token counts**
+(llama.cpp's tokenizer) through the vLLM/SGLang-shaped `/v1/tokenize`, enforce
+their tier's context window with the exact SGLang error phrasing (so
+cloud-api's context-400 fall-through matcher fires as in prod), tag responses
+(`"model": "<id>+base"` / `+long`) so you can SEE which tier served, and offer
+a saturation drill (`touch SATURATE-long` → 503 "queue full").
+
+## Quick start
+
+```bash
+cd dev/local-stack
+make up          # postgres + llama.cpp (first run downloads ~400MB GGUF) + shims
+make api         # cloud-api in the foreground (separate terminal)
+make seed        # mock admin user, two-tier model, org + credits, API key
+make demo        # the tier-routing demo table (8 checks)
+make down        # stop everything
+```
+
+`make demo` exercises: small→base (long tier untouched), oversize→long,
+boundary-sized→exact-tokenize decides (real counts beat the byte heuristic),
+streaming→long over SSE, and saturated-long→retryable 429/5xx (never a
+misleading context-length 400).
+
+Manual poking:
+
+```bash
+KEY=$(cat .api-key)
+curl -s localhost:13000/v1/chat/completions -H "Authorization: Bearer $KEY" \
+  -d '{"model":"z-ai/glm-5.2-local","messages":[{"role":"user","content":"hi"}],"max_tokens":16}' \
+  | jq .model     # → "z-ai/glm-5.2-local+base"
+make logs         # shim access logs: tier decisions, tokenize hits, 400s/503s
+```
+
+## model-proxy control plane
+
+`make proxy-demo` (needs a model-proxy checkout, default `../model-proxy`,
+override with `MODEL_PROXY_DIR=…`) runs the REAL model-proxy binary with fast
+discovery/health intervals and drives the GLM-5.2 long-context registration
+flow against it: dual probes on one host (canonical id on `:18000`, synthetic
+`-long` id on `:18001`) sharing one routed backend, the probe-cleanup
+ownership guard (unregistering one probe must not drop the shared backend),
+and the health-gated discovery stub + circuit breaker (non-2xx = unhealthy).
+
+## What is intentionally NOT local
+
+- **The TLS/SNI data path through model-proxy.** cloud-api's providers
+  fail-closed on HTTPS to backends whose attestation didn't verify
+  (fingerprint pinning) — that's the TEE trust model working. Locally the
+  data path is plain HTTP direct to the tier shims (the provider's
+  non-rotation fallback-client path, same code that serves IP-literal URLs);
+  model-proxy is exercised on its control plane beside it.
+- **The Chutes wire client** (ML-KEM E2EE + TDX quote verification against
+  vetted measurements — not mockable at the HTTP level by design). The
+  fallback CHAIN including a pinned attested provider is covered at the pool
+  boundary in `crates/api/tests/e2e_all/glm52_tier_routing.rs`; the wire
+  client runs in prod today for GLM-5.1/5.2.
+
+## Engine notes
+
+llama.cpp was chosen because it starts in seconds on CPU with a tiny GGUF and
+exposes an OpenAI-compatible API plus a native `/tokenize`. The shims speak
+OpenAI upstream, so any OpenAI-compatible engine (e.g. a vLLM CPU build) can
+be swapped into `docker-compose.yml` without touching anything else.
+
+Ports used: 13000 (api), 15432 (postgres), 18090 (engine), 18100/18101
+(shims) — chosen not to clash with the e2e `test-postgres` on 5432.

dev/local-stack/demo.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# Tier-routing demo against the local stack: shows which tier served each
+# request (the shims tag the response `.model` with +base/+long) and prints a
+# pass/fail table. Requires seed.sh to have run.
+set -uo pipefail
+cd "$(dirname "$0")"
+
+API=${API:-http://127.0.0.1:13000}
+MODEL_ID=${MODEL_ID:-z-ai/glm-5.2-local}
+API_KEY=${API_KEY:-$(cat .api-key 2>/dev/null || true)}
+[ -n "$API_KEY" ] || { echo "no API key — run seed.sh first"; exit 1; }
+
+PASS=0; FAIL=0; ROWS=""
+
+chat() { # prompt max_tokens stream
+  curl -s -w '\n%{http_code}' "$API/v1/chat/completions" \
+    -H "Authorization: Bearer $API_KEY" -H "Content-Type: application/json" \
+    -d "{\"model\":\"$MODEL_ID\",\"messages\":[{\"role\":\"user\",\"content\":\"$1\"}],\"max_tokens\":$2,\"stream\":$3}"
+}
+
+check() { # name expect_pattern got
+  local name=$1 expect=$2 got=$3
+  if echo "$got" | grep -Eq "$expect"; then
+    PASS=$((PASS+1)); ROWS+="  PASS  $name\n"
+  else
+    FAIL=$((FAIL+1)); ROWS+="  FAIL  $name  (wanted /$expect/, got: $(echo "$got" | head -c 200))\n"
+  fi
+}
+
+# Word-repeat prompts give predictable real token counts (~1 token per word).
+words() { python3 -c "print('hello '*$1)"; }
+
+echo "== 1. small request -> base tier"
+OUT=$(chat "Say hi in one word." 16 false)
+check "small -> +base tag"        '"model": ?"[^"]*\+base"'   "$OUT"
+check "small -> HTTP 200"         '^200$'                  "$(echo "$OUT" | tail -1)"
+
+echo "== 2. oversize request (~3000 real tokens > base 1000) -> long tier"
+OUT=$(chat "$(words 3000)" 16 false)
+check "oversize -> +long tag"     '"model": ?"[^"]*\+long"'   "$OUT"
+check "oversize -> HTTP 200"      '^200$'                  "$(echo "$OUT" | tail -1)"
+
+echo "== 3. boundary request (~800 real tokens, heuristic ambiguous) -> exact /v1/tokenize decides -> base"
+OUT=$(chat "$(words 800)" 16 false)
+check "boundary -> +base tag"     '"model": ?"[^"]*\+base"'   "$OUT"
+
+echo "== 4. streaming oversize -> long tier serves (SSE)"
+OUT=$(chat "$(words 3000)" 16 true)
+check "stream -> HTTP 200"        '^200$'                  "$(echo "$OUT" | tail -1)"
+check "stream -> content"         'data:'                  "$OUT"
+
+echo "== 5. saturate the long tier -> oversize gets a RETRYABLE error (not a context-400)"
+touch SATURATE-long
+OUT=$(chat "$(words 3000)" 16 false)
+CODE=$(echo "$OUT" | tail -1)
+rm -f SATURATE-long
+if [ "$CODE" = "429" ] || [ "${CODE:0:1}" = "5" ]; then
+  PASS=$((PASS+1)); ROWS+="  PASS  saturated long -> retryable $CODE\n"
+else
+  FAIL=$((FAIL+1)); ROWS+="  FAIL  saturated long -> got $CODE (must be 429/5xx, never 400)\n"
+fi
+
+echo
+echo "================ local-stack tier-routing demo ================"
+printf "%b" "$ROWS"
+echo "==============================================================="
+echo "PASS=$PASS FAIL=$FAIL"
+[ "$FAIL" = 0 ]

dev/local-stack/docker-compose.yml

@@ -0,0 +1,42 @@
+# Local dev stack: real engine + isolated Postgres. Everything binds to
+# 127.0.0.1 on ports chosen not to clash with the e2e test-postgres (5432).
+services:
+  postgres:
+    image: postgres:16
+    container_name: local-stack-postgres
+    environment:
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: platform_api
+    ports:
+      - "127.0.0.1:15432:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 2s
+      timeout: 3s
+      retries: 30
+
+  # Real inference engine on CPU: llama.cpp's OpenAI-compatible server with a
+  # tiny instruct model, auto-downloaded from HF on first start and cached in
+  # ./cache. Engine-agnostic seam: the tier shims speak OpenAI upstream, so a
+  # vLLM CPU image can be swapped in here later.
+  llama:
+    image: ghcr.io/ggml-org/llama.cpp:server
+    container_name: local-stack-llama
+    command: >
+      -hf Qwen/Qwen2.5-0.5B-Instruct-GGUF:Q4_K_M
+      --alias local
+      --host 0.0.0.0 --port 8090
+      --ctx-size 8192
+      --parallel 2
+    environment:
+      LLAMA_CACHE: /cache
+    volumes:
+      - ./cache:/cache
+    ports:
+      - "127.0.0.1:18090:8090"
+    healthcheck:
+      test: ["CMD", "curl", "-sf", "http://127.0.0.1:8090/health"]
+      interval: 5s
+      timeout: 5s
+      retries: 120
+      start_period: 30s

dev/local-stack/env.local-stack

@@ -0,0 +1,50 @@
+# Environment for running cloud-api against the local dev stack.
+# Usage:  set -a; source dev/local-stack/env.local-stack; set +a; cargo run --bin api
+# (the Makefile's `make api` does exactly this)
+
+SERVER_HOST=127.0.0.1
+SERVER_PORT=13000
+
+# Local-stack Postgres (docker-compose.yml, NOT the e2e test-postgres on 5432)
+POSTGRES_PRIMARY_APP_ID=postgres-test
+GATEWAY_SUBDOMAIN=localhost
+DATABASE_HOST=127.0.0.1
+DATABASE_PORT=15432
+DATABASE_NAME=platform_api
+DATABASE_USERNAME=postgres
+DATABASE_PASSWORD=postgres
+DATABASE_MAX_CONNECTIONS=5
+
+# Mock auth: sessions are `rt_<user-uuid>` + User-Agent "Mock User Agent".
+# admin@test.com (seeded by seed.sh) gets admin via AUTH_ADMIN_DOMAINS.
+AUTH_MOCK=true
+AUTH_ENCODING_KEY=local-stack-not-a-secret
+AUTH_ADMIN_DOMAINS=test.com
+GITHUB_CLIENT_ID=unused
+GITHUB_CLIENT_SECRET=unused
+GITHUB_REDIRECT_URL=http://localhost:13000/v1/auth/callback
+GOOGLE_CLIENT_ID=unused
+GOOGLE_CLIENT_SECRET=unused
+GOOGLE_REDIRECT_URL=http://localhost:13000/v1/auth/callback
+
+# DEV (debug builds only): fall back to ephemeral signing keys when dstack is
+# absent — this stack runs outside a TEE by definition.
+DEV=1
+DSTACK_CLIENT_URL=http://127.0.0.1:1
+
+# Model discovery: nothing to discover locally; models are seeded via the
+# admin API with explicit inferenceUrls (the tier shims).
+MODEL_DISCOVERY_SERVER_URL=http://127.0.0.1:1/models
+MODEL_DISCOVERY_API_KEY=local-stack-inference-key
+MODEL_DISCOVERY_REFRESH_INTERVAL=300
+MODEL_DISCOVERY_TIMEOUT=2
+
+# Placeholders for integrations that must not be exercised locally.
+BRAVE_SEARCH_PRO_API_KEY=local-stack-not-a-secret
+AWS_S3_BUCKET=local-stack-unused
+AWS_S3_REGION=us-east-1
+S3_ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000
+INVITATION_EMAIL_ENABLED=false
+
+LOG_LEVEL=info
+LOG_FORMAT=compact

dev/local-stack/model-proxy-demo/config.yaml

@@ -0,0 +1,32 @@
+# Minimal localhost model-proxy config for the control-plane demo.
+# Fast intervals so the demo asserts within seconds; no TLS listener
+# (cert/key/base_domain unset), no peer sync (urls empty), no admin token.
+proxy:
+  bind_address: "127.0.0.1:18443"
+  client_hello_timeout_ms: 2000
+  backend_connect_timeout_ms: 5000
+  max_connections_per_ip: 100
+  max_backend_retries: 2
+
+discovery:
+  refresh_interval_s: 2
+  probe_timeout_ms: 5000
+  routing_port: 18444
+  forget_after_s: 600
+
+health_check:
+  interval_s: 2
+  timeout_ms: 3000
+  health_check_path: "/v1/models"
+  slow_threshold_ms: 0
+
+circuit_breaker:
+  failure_threshold: 2
+  recovery_threshold: 2
+  cooldown_s: 5
+
+admin:
+  bind_address: "127.0.0.1:19090"
+  metrics_enabled: true
+
+peers: {}

dev/local-stack/model-proxy-demo/probe_stub.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""Probe endpoints for the model-proxy control-plane demo.
+
+Emulates the GLM-5.2 TP8 host's two discovery endpoints:
+  :18000 -> /v1/models advertising the canonical id  (the serving probe)
+  :18001 -> /v1/models advertising the synthetic -long id, HEALTH-GATED the
+            same way the PR-#129 nginx stub is (auth_request to the engine):
+            when ./ENGINE_DOWN exists, both answer 5xx — engine dead.
+
+Stdlib only; both servers run in one process.
+"""
+
+import json
+import os
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+GATE_FILE = os.path.join(HERE, "ENGINE_DOWN")
+
+
+def make_handler(model_id: str):
+    class Handler(BaseHTTPRequestHandler):
+        protocol_version = "HTTP/1.1"
+
+        def log_message(self, fmt, *args):
+            print(f"[{model_id}] {fmt % args}", flush=True)
+
+        def do_GET(self):
+            if os.path.exists(GATE_FILE):
+                body = json.dumps({"error": {"message": "engine down"}}).encode()
+                self.send_response(502)
+            elif self.path == "/v1/models":
+                body = json.dumps(
+                    {"object": "list",
+                     "data": [{"id": model_id, "object": "model", "owned_by": "nearai"}]}
+                ).encode()
+                self.send_response(200)
+            else:
+                body = b"{}"
+                self.send_response(404)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+
+    return Handler
+
+
+def serve(port: int, model_id: str):
+    ThreadingHTTPServer(("127.0.0.1", port), make_handler(model_id)).serve_forever()
+
+
+if __name__ == "__main__":
+    threading.Thread(target=serve, args=(18000, "z-ai/glm-5.2-e2e"), daemon=True).start()
+    print("probe stubs on :18000 (canonical) and :18001 (synthetic -long)", flush=True)
+    serve(18001, "z-ai/glm-5.2-e2e-long")