diff --git a/dev/local-stack/.gitignore b/dev/local-stack/.gitignore new file mode 100644 index 000000000..13f35a21c --- /dev/null +++ b/dev/local-stack/.gitignore @@ -0,0 +1,6 @@ +cache/ +run/ +*.log +*.pid +.api-key +SATURATE-* diff --git a/dev/local-stack/Makefile b/dev/local-stack/Makefile new file mode 100644 index 000000000..c7f45f681 --- /dev/null +++ b/dev/local-stack/Makefile @@ -0,0 +1,42 @@ +SHELL := /bin/bash +STACK_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +REPO_ROOT := $(abspath $(STACK_DIR)/../..) +MODEL_PROXY_DIR ?= $(REPO_ROOT)/../model-proxy + +.PHONY: up shims api seed demo proxy-demo down logs + +## Bring up the containers (Postgres + llama.cpp engine; first start downloads +## the ~400MB GGUF into ./cache) and the two tier shims. +up: + cd $(STACK_DIR) && docker compose up -d --wait + cd $(STACK_DIR) && (python3 tier_shim.py --tier base --port 18100 --ctx-window 1000 --model-id z-ai/glm-5.2-local > shim-base.log 2>&1 & echo $$! > .shim-base.pid) + cd $(STACK_DIR) && (python3 tier_shim.py --tier long --port 18101 --ctx-window 8000 --model-id z-ai/glm-5.2-local > shim-long.log 2>&1 & echo $$! > .shim-long.pid) + @sleep 1 && curl -sf http://127.0.0.1:18100/health >/dev/null && curl -sf http://127.0.0.1:18101/health >/dev/null \ + && echo "engine + shims up (base :18100 ctx=1000, long :18101 ctx=8000)" + +## Run cloud-api in the foreground against the stack (debug build; DEV=1 +## ephemeral signing keys). Ctrl-C to stop. +api: + cd $(REPO_ROOT) && set -a && source $(STACK_DIR)/env.local-stack && set +a && cargo run --bin api + +## Seed model + org + API key (needs `make api` running in another terminal). +seed: + $(STACK_DIR)/seed.sh + +## Tier-routing demo (small/oversize/boundary/streaming/saturated). +demo: + $(STACK_DIR)/demo.sh + +## Model-proxy control-plane demo (dual-probe registration, shared-backend +## ownership guard, health-gated stub + circuit breaker). Needs a model-proxy +## checkout at $(MODEL_PROXY_DIR) (override: make proxy-demo MODEL_PROXY_DIR=…). +proxy-demo: + MODEL_PROXY_DIR=$(MODEL_PROXY_DIR) $(STACK_DIR)/model-proxy-demo/run.sh + +down: + -cd $(STACK_DIR) && for f in .shim-base.pid .shim-long.pid; do [ -f $$f ] && kill $$(cat $$f) 2>/dev/null; rm -f $$f; done + -cd $(STACK_DIR) && rm -f SATURATE-base SATURATE-long + cd $(STACK_DIR) && docker compose down + +logs: + @cd $(STACK_DIR) && tail -n 40 shim-base.log shim-long.log 2>/dev/null || true diff --git a/dev/local-stack/README.md b/dev/local-stack/README.md new file mode 100644 index 000000000..3eeaaa40f --- /dev/null +++ b/dev/local-stack/README.md @@ -0,0 +1,82 @@ +# Local dev stack — full tier-routing environment on one machine + +Runs the real cloud-api against a **real inference engine on CPU** with the +GLM-5.2-style two-tier context topology (base fleet + long-context tier), so +routing changes can be exercised end to end without touching staging or any +GPU host: + +``` +curl ──► cloud-api :13000 (cargo run, mock auth, ephemeral signing keys) + │ ▲ Postgres :15432 (docker) + ├─ base tier ─► tier_shim :18100 (ctx 1000) ──┐ + └─ long tier ─► tier_shim :18101 (ctx 8000) ──┴─► llama.cpp :18090 + (docker, CPU, Qwen2.5-0.5B GGUF) +``` + +The tier shims play the role of the per-tier SGLang fleets: they advertise the +canonical model id, serve **real completions** and **real token counts** +(llama.cpp's tokenizer) through the vLLM/SGLang-shaped `/v1/tokenize`, enforce +their tier's context window with the exact SGLang error phrasing (so +cloud-api's context-400 fall-through matcher fires as in prod), tag responses +(`"model": "+base"` / `+long`) so you can SEE which tier served, and offer +a saturation drill (`touch SATURATE-long` → 503 "queue full"). + +## Quick start + +```bash +cd dev/local-stack +make up # postgres + llama.cpp (first run downloads ~400MB GGUF) + shims +make api # cloud-api in the foreground (separate terminal) +make seed # mock admin user, two-tier model, org + credits, API key +make demo # the tier-routing demo table (8 checks) +make down # stop everything +``` + +`make demo` exercises: small→base (long tier untouched), oversize→long, +boundary-sized→exact-tokenize decides (real counts beat the byte heuristic), +streaming→long over SSE, and saturated-long→retryable 429/5xx (never a +misleading context-length 400). + +Manual poking: + +```bash +KEY=$(cat .api-key) +curl -s localhost:13000/v1/chat/completions -H "Authorization: Bearer $KEY" \ + -d '{"model":"z-ai/glm-5.2-local","messages":[{"role":"user","content":"hi"}],"max_tokens":16}' \ + | jq .model # → "z-ai/glm-5.2-local+base" +make logs # shim access logs: tier decisions, tokenize hits, 400s/503s +``` + +## model-proxy control plane + +`make proxy-demo` (needs a model-proxy checkout, default `../model-proxy`, +override with `MODEL_PROXY_DIR=…`) runs the REAL model-proxy binary with fast +discovery/health intervals and drives the GLM-5.2 long-context registration +flow against it: dual probes on one host (canonical id on `:18000`, synthetic +`-long` id on `:18001`) sharing one routed backend, the probe-cleanup +ownership guard (unregistering one probe must not drop the shared backend), +and the health-gated discovery stub + circuit breaker (non-2xx = unhealthy). + +## What is intentionally NOT local + +- **The TLS/SNI data path through model-proxy.** cloud-api's providers + fail-closed on HTTPS to backends whose attestation didn't verify + (fingerprint pinning) — that's the TEE trust model working. Locally the + data path is plain HTTP direct to the tier shims (the provider's + non-rotation fallback-client path, same code that serves IP-literal URLs); + model-proxy is exercised on its control plane beside it. +- **The Chutes wire client** (ML-KEM E2EE + TDX quote verification against + vetted measurements — not mockable at the HTTP level by design). The + fallback CHAIN including a pinned attested provider is covered at the pool + boundary in `crates/api/tests/e2e_all/glm52_tier_routing.rs`; the wire + client runs in prod today for GLM-5.1/5.2. + +## Engine notes + +llama.cpp was chosen because it starts in seconds on CPU with a tiny GGUF and +exposes an OpenAI-compatible API plus a native `/tokenize`. The shims speak +OpenAI upstream, so any OpenAI-compatible engine (e.g. a vLLM CPU build) can +be swapped into `docker-compose.yml` without touching anything else. + +Ports used: 13000 (api), 15432 (postgres), 18090 (engine), 18100/18101 +(shims) — chosen not to clash with the e2e `test-postgres` on 5432. diff --git a/dev/local-stack/demo.sh b/dev/local-stack/demo.sh new file mode 100755 index 000000000..5a9d77cf1 --- /dev/null +++ b/dev/local-stack/demo.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Tier-routing demo against the local stack: shows which tier served each +# request (the shims tag the response `.model` with +base/+long) and prints a +# pass/fail table. Requires seed.sh to have run. +set -uo pipefail +cd "$(dirname "$0")" + +API=${API:-http://127.0.0.1:13000} +MODEL_ID=${MODEL_ID:-z-ai/glm-5.2-local} +API_KEY=${API_KEY:-$(cat .api-key 2>/dev/null || true)} +[ -n "$API_KEY" ] || { echo "no API key — run seed.sh first"; exit 1; } + +PASS=0; FAIL=0; ROWS="" + +chat() { # prompt max_tokens stream + curl -s -w '\n%{http_code}' "$API/v1/chat/completions" \ + -H "Authorization: Bearer $API_KEY" -H "Content-Type: application/json" \ + -d "{\"model\":\"$MODEL_ID\",\"messages\":[{\"role\":\"user\",\"content\":\"$1\"}],\"max_tokens\":$2,\"stream\":$3}" +} + +check() { # name expect_pattern got + local name=$1 expect=$2 got=$3 + if echo "$got" | grep -Eq "$expect"; then + PASS=$((PASS+1)); ROWS+=" PASS $name\n" + else + FAIL=$((FAIL+1)); ROWS+=" FAIL $name (wanted /$expect/, got: $(echo "$got" | head -c 200))\n" + fi +} + +# Word-repeat prompts give predictable real token counts (~1 token per word). +words() { python3 -c "print('hello '*$1)"; } + +echo "== 1. small request -> base tier" +OUT=$(chat "Say hi in one word." 16 false) +check "small -> +base tag" '"model": ?"[^"]*\+base"' "$OUT" +check "small -> HTTP 200" '^200$' "$(echo "$OUT" | tail -1)" + +echo "== 2. oversize request (~3000 real tokens > base 1000) -> long tier" +OUT=$(chat "$(words 3000)" 16 false) +check "oversize -> +long tag" '"model": ?"[^"]*\+long"' "$OUT" +check "oversize -> HTTP 200" '^200$' "$(echo "$OUT" | tail -1)" + +echo "== 3. boundary request (~800 real tokens, heuristic ambiguous) -> exact /v1/tokenize decides -> base" +OUT=$(chat "$(words 800)" 16 false) +check "boundary -> +base tag" '"model": ?"[^"]*\+base"' "$OUT" + +echo "== 4. streaming oversize -> long tier serves (SSE)" +OUT=$(chat "$(words 3000)" 16 true) +check "stream -> HTTP 200" '^200$' "$(echo "$OUT" | tail -1)" +check "stream -> content" 'data:' "$OUT" + +echo "== 5. saturate the long tier -> oversize gets a RETRYABLE error (not a context-400)" +touch SATURATE-long +OUT=$(chat "$(words 3000)" 16 false) +CODE=$(echo "$OUT" | tail -1) +rm -f SATURATE-long +if [ "$CODE" = "429" ] || [ "${CODE:0:1}" = "5" ]; then + PASS=$((PASS+1)); ROWS+=" PASS saturated long -> retryable $CODE\n" +else + FAIL=$((FAIL+1)); ROWS+=" FAIL saturated long -> got $CODE (must be 429/5xx, never 400)\n" +fi + +echo +echo "================ local-stack tier-routing demo ================" +printf "%b" "$ROWS" +echo "===============================================================" +echo "PASS=$PASS FAIL=$FAIL" +[ "$FAIL" = 0 ] diff --git a/dev/local-stack/docker-compose.yml b/dev/local-stack/docker-compose.yml new file mode 100644 index 000000000..d0d330443 --- /dev/null +++ b/dev/local-stack/docker-compose.yml @@ -0,0 +1,42 @@ +# Local dev stack: real engine + isolated Postgres. Everything binds to +# 127.0.0.1 on ports chosen not to clash with the e2e test-postgres (5432). +services: + postgres: + image: postgres:16 + container_name: local-stack-postgres + environment: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: platform_api + ports: + - "127.0.0.1:15432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 2s + timeout: 3s + retries: 30 + + # Real inference engine on CPU: llama.cpp's OpenAI-compatible server with a + # tiny instruct model, auto-downloaded from HF on first start and cached in + # ./cache. Engine-agnostic seam: the tier shims speak OpenAI upstream, so a + # vLLM CPU image can be swapped in here later. + llama: + image: ghcr.io/ggml-org/llama.cpp:server + container_name: local-stack-llama + command: > + -hf Qwen/Qwen2.5-0.5B-Instruct-GGUF:Q4_K_M + --alias local + --host 0.0.0.0 --port 8090 + --ctx-size 8192 + --parallel 2 + environment: + LLAMA_CACHE: /cache + volumes: + - ./cache:/cache + ports: + - "127.0.0.1:18090:8090" + healthcheck: + test: ["CMD", "curl", "-sf", "http://127.0.0.1:8090/health"] + interval: 5s + timeout: 5s + retries: 120 + start_period: 30s diff --git a/dev/local-stack/env.local-stack b/dev/local-stack/env.local-stack new file mode 100644 index 000000000..4325c4a4a --- /dev/null +++ b/dev/local-stack/env.local-stack @@ -0,0 +1,50 @@ +# Environment for running cloud-api against the local dev stack. +# Usage: set -a; source dev/local-stack/env.local-stack; set +a; cargo run --bin api +# (the Makefile's `make api` does exactly this) + +SERVER_HOST=127.0.0.1 +SERVER_PORT=13000 + +# Local-stack Postgres (docker-compose.yml, NOT the e2e test-postgres on 5432) +POSTGRES_PRIMARY_APP_ID=postgres-test +GATEWAY_SUBDOMAIN=localhost +DATABASE_HOST=127.0.0.1 +DATABASE_PORT=15432 +DATABASE_NAME=platform_api +DATABASE_USERNAME=postgres +DATABASE_PASSWORD=postgres +DATABASE_MAX_CONNECTIONS=5 + +# Mock auth: sessions are `rt_` + User-Agent "Mock User Agent". +# admin@test.com (seeded by seed.sh) gets admin via AUTH_ADMIN_DOMAINS. +AUTH_MOCK=true +AUTH_ENCODING_KEY=local-stack-not-a-secret +AUTH_ADMIN_DOMAINS=test.com +GITHUB_CLIENT_ID=unused +GITHUB_CLIENT_SECRET=unused +GITHUB_REDIRECT_URL=http://localhost:13000/v1/auth/callback +GOOGLE_CLIENT_ID=unused +GOOGLE_CLIENT_SECRET=unused +GOOGLE_REDIRECT_URL=http://localhost:13000/v1/auth/callback + +# DEV (debug builds only): fall back to ephemeral signing keys when dstack is +# absent — this stack runs outside a TEE by definition. +DEV=1 +DSTACK_CLIENT_URL=http://127.0.0.1:1 + +# Model discovery: nothing to discover locally; models are seeded via the +# admin API with explicit inferenceUrls (the tier shims). +MODEL_DISCOVERY_SERVER_URL=http://127.0.0.1:1/models +MODEL_DISCOVERY_API_KEY=local-stack-inference-key +MODEL_DISCOVERY_REFRESH_INTERVAL=300 +MODEL_DISCOVERY_TIMEOUT=2 + +# Placeholders for integrations that must not be exercised locally. +BRAVE_SEARCH_PRO_API_KEY=local-stack-not-a-secret +AWS_S3_BUCKET=local-stack-unused +AWS_S3_REGION=us-east-1 +S3_ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 +INVITATION_EMAIL_ENABLED=false + +LOG_LEVEL=info +LOG_FORMAT=compact diff --git a/dev/local-stack/model-proxy-demo/config.yaml b/dev/local-stack/model-proxy-demo/config.yaml new file mode 100644 index 000000000..57e93cfff --- /dev/null +++ b/dev/local-stack/model-proxy-demo/config.yaml @@ -0,0 +1,32 @@ +# Minimal localhost model-proxy config for the control-plane demo. +# Fast intervals so the demo asserts within seconds; no TLS listener +# (cert/key/base_domain unset), no peer sync (urls empty), no admin token. +proxy: + bind_address: "127.0.0.1:18443" + client_hello_timeout_ms: 2000 + backend_connect_timeout_ms: 5000 + max_connections_per_ip: 100 + max_backend_retries: 2 + +discovery: + refresh_interval_s: 2 + probe_timeout_ms: 5000 + routing_port: 18444 + forget_after_s: 600 + +health_check: + interval_s: 2 + timeout_ms: 3000 + health_check_path: "/v1/models" + slow_threshold_ms: 0 + +circuit_breaker: + failure_threshold: 2 + recovery_threshold: 2 + cooldown_s: 5 + +admin: + bind_address: "127.0.0.1:19090" + metrics_enabled: true + +peers: {} diff --git a/dev/local-stack/model-proxy-demo/probe_stub.py b/dev/local-stack/model-proxy-demo/probe_stub.py new file mode 100755 index 000000000..12b90cfb0 --- /dev/null +++ b/dev/local-stack/model-proxy-demo/probe_stub.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Probe endpoints for the model-proxy control-plane demo. + +Emulates the GLM-5.2 TP8 host's two discovery endpoints: + :18000 -> /v1/models advertising the canonical id (the serving probe) + :18001 -> /v1/models advertising the synthetic -long id, HEALTH-GATED the + same way the PR-#129 nginx stub is (auth_request to the engine): + when ./ENGINE_DOWN exists, both answer 5xx — engine dead. + +Stdlib only; both servers run in one process. +""" + +import json +import os +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +HERE = os.path.dirname(os.path.abspath(__file__)) +GATE_FILE = os.path.join(HERE, "ENGINE_DOWN") + + +def make_handler(model_id: str): + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, fmt, *args): + print(f"[{model_id}] {fmt % args}", flush=True) + + def do_GET(self): + if os.path.exists(GATE_FILE): + body = json.dumps({"error": {"message": "engine down"}}).encode() + self.send_response(502) + elif self.path == "/v1/models": + body = json.dumps( + {"object": "list", + "data": [{"id": model_id, "object": "model", "owned_by": "nearai"}]} + ).encode() + self.send_response(200) + else: + body = b"{}" + self.send_response(404) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + return Handler + + +def serve(port: int, model_id: str): + ThreadingHTTPServer(("127.0.0.1", port), make_handler(model_id)).serve_forever() + + +if __name__ == "__main__": + threading.Thread(target=serve, args=(18000, "z-ai/glm-5.2-e2e"), daemon=True).start() + print("probe stubs on :18000 (canonical) and :18001 (synthetic -long)", flush=True) + serve(18001, "z-ai/glm-5.2-e2e-long") diff --git a/dev/local-stack/model-proxy-demo/run.sh b/dev/local-stack/model-proxy-demo/run.sh new file mode 100755 index 000000000..1a2e8c6a0 --- /dev/null +++ b/dev/local-stack/model-proxy-demo/run.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# model-proxy control-plane demo: drives the REAL model-proxy binary through +# the GLM-5.2 long-context registration flow. +# +# 1. dual probes on one "host": canonical id on :18000, synthetic -long id +# on :18001, sharing one routed backend (:18444) +# 2. shared-backend dedupe: both domains -> the SAME backend, total_backends=1 +# 3. probe-cleanup OWNERSHIP GUARD (model-proxy PR #42): unregistering the +# :18001 probe must NOT drop the shared backend the :18000 probe still owns +# 4. health-gated stub + circuit breaker: when the "engine" dies, the stub +# 5xxes -> the -long domain drains within a discovery cycle and the +# breaker opens (non-2xx = unhealthy, also PR #42) +# +# Requires a model-proxy checkout (branch probe-cleanup-ownership for the +# guard steps): MODEL_PROXY_DIR=… ./run.sh +set -euo pipefail +cd "$(dirname "$0")" + +MODEL_PROXY_DIR=${MODEL_PROXY_DIR:-$(cd ../../../.. && pwd)/model-proxy} +ADMIN=http://127.0.0.1:19090 +PASS=0; FAIL=0 + +say() { echo; echo "== $*"; } +check() { # name cond-cmd + if eval "$2" >/dev/null 2>&1; then PASS=$((PASS+1)); echo " PASS $1"; + else FAIL=$((FAIL+1)); echo " FAIL $1"; fi +} +reg() { curl -sf -X POST "$ADMIN/$1" -H 'Content-Type: application/json' -d "$2" >/dev/null; } +registry() { curl -sf "$ADMIN/registry"; } +cleanup_all() { + [ -n "${MP_PID:-}" ] && kill "$MP_PID" 2>/dev/null || true + [ -n "${STUB_PID:-}" ] && kill "$STUB_PID" 2>/dev/null || true + [ -n "${ENGINE_PID:-}" ] && kill "$ENGINE_PID" 2>/dev/null || true +} +trap cleanup_all EXIT + +say "building model-proxy from $MODEL_PROXY_DIR" +(cd "$MODEL_PROXY_DIR" && cargo build -q 2>&1 | tail -1 || true) +MP_BIN="$MODEL_PROXY_DIR/target/debug/model-proxy" +[ -x "$MP_BIN" ] || { echo "model-proxy binary not found at $MP_BIN"; exit 1; } + +say "starting probe stubs (:18000 canonical, :18001 health-gated synthetic) + fake engine (:18010)" +python3 probe_stub.py > probe_stub.log 2>&1 & +STUB_PID=$! +sleep 0.5 + +say "starting model-proxy (discovery 2s, health 2s, breaker threshold 2)" +rm -rf run && mkdir -p run && cp config.yaml run/ +(cd run && exec "$MP_BIN" > model-proxy.log 2>&1) & +MP_PID=$! +for _ in $(seq 1 20); do curl -sf "$ADMIN/health" >/dev/null 2>&1 && break; sleep 0.5; done +curl -sf "$ADMIN/health" >/dev/null || { echo "model-proxy admin API not up"; tail -20 run/model-proxy.log; exit 1; } + +say "registering dual probes + model->domain mappings" +reg register/endpoint '{"endpoint":"127.0.0.1:18000","routing_port":18444}' +reg register/endpoint '{"endpoint":"127.0.0.1:18001","routing_port":18444}' +reg register/model '{"model":"z-ai/glm-5.2-e2e","domain":"glm-5-2-e2e.local"}' +reg register/model '{"model":"z-ai/glm-5.2-e2e-long","domain":"glm-5-2-e2e-long.local"}' +sleep 5 # ≥2 discovery cycles + +say "1+2: both domains share ONE routed backend" +REG=$(registry) +check "base domain has the backend" "echo '$REG' | python3 -c 'import sys,json; d=json.load(sys.stdin); assert d[\"domains\"][\"glm-5-2-e2e.local\"][0][\"address\"]==\"127.0.0.1:18444\"'" +check "long domain has the backend" "echo '$REG' | python3 -c 'import sys,json; d=json.load(sys.stdin); assert d[\"domains\"][\"glm-5-2-e2e-long.local\"][0][\"address\"]==\"127.0.0.1:18444\"'" +check "backend deduped (total=1)" "echo '$REG' | python3 -c 'import sys,json; assert json.load(sys.stdin)[\"total_backends\"]==1'" + +say "3: ownership guard — unregister the :18001 probe, shared backend must survive" +reg unregister/endpoint '{"endpoint":"127.0.0.1:18001"}' +REG=$(registry) +check "shared backend survives (total=1)" "echo '$REG' | python3 -c 'import sys,json; assert json.load(sys.stdin)[\"total_backends\"]==1'" +check "base domain still routable" "curl -sf '$ADMIN/backends/count?domain=glm-5-2-e2e.local' | python3 -c 'import sys,json; assert json.load(sys.stdin)[\"total\"]==1'" +sleep 3 +check "long domain drained by next rebuild" "! curl -sf '$ADMIN/backends/count?domain=glm-5-2-e2e-long.local' >/dev/null 2>&1 || curl -s '$ADMIN/backends/count?domain=glm-5-2-e2e-long.local' | python3 -c 'import sys,json; assert json.load(sys.stdin)[\"total\"]==0'" + +say "4: health-gated stub — kill the engine, breaker must OPEN and long domain drain" +reg register/endpoint '{"endpoint":"127.0.0.1:18001","routing_port":18444}' +sleep 5 +touch ENGINE_DOWN # probe_stub gates :18001 (and :18000) on this file, like the nginx auth_request gate +sleep 8 # a few health cycles at threshold 2 +REG=$(registry) +check "circuit breaker opened on engine death" "echo '$REG' | python3 -c 'import sys,json; d=json.load(sys.stdin); assert d[\"healthy_backends\"]==0, d'" +check "long domain drained" "echo '$REG' | python3 -c 'import sys,json; d=json.load(sys.stdin); assert \"glm-5-2-e2e-long.local\" not in d[\"domains\"] or not d[\"domains\"][\"glm-5-2-e2e-long.local\"], d'" +rm -f ENGINE_DOWN +sleep 10 # cooldown 5s + half-open recovery +REG=$(registry) +check "breaker recovers when engine returns" "echo '$REG' | python3 -c 'import sys,json; d=json.load(sys.stdin); assert d[\"healthy_backends\"]==1, d'" + +echo +echo "================ model-proxy control-plane demo ================" +echo "PASS=$PASS FAIL=$FAIL" +[ "$FAIL" = 0 ] diff --git a/dev/local-stack/seed.sh b/dev/local-stack/seed.sh new file mode 100755 index 000000000..a37ae8703 --- /dev/null +++ b/dev/local-stack/seed.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Seed the local stack: mock admin user, two-tier model (base shim 1k ctx, +# long shim 8k ctx), org with credits, API key. Prints the API key. +set -euo pipefail +cd "$(dirname "$0")" + +API=${API:-http://127.0.0.1:13000} +MODEL_ID=${MODEL_ID:-z-ai/glm-5.2-local} +BASE_SHIM=${BASE_SHIM:-http://127.0.0.1:18100} +LONG_SHIM=${LONG_SHIM:-http://127.0.0.1:18101} +BASE_CTX=${BASE_CTX:-1000} +LONG_CTX=${LONG_CTX:-8000} + +MOCK_USER_ID="11111111-1111-1111-1111-111111111111" +SESSION="rt_${MOCK_USER_ID}" +UA="Mock User Agent" + +req() { # method path [json] + local method=$1 path=$2 body=${3:-} + curl -sf -X "$method" "$API$path" \ + -H "Authorization: Bearer $SESSION" \ + -H "User-Agent: $UA" \ + -H "Content-Type: application/json" \ + ${body:+-d "$body"} +} + +echo "== waiting for cloud-api at $API" +for _ in $(seq 1 60); do curl -sf "$API/v1/health" >/dev/null 2>&1 && break; sleep 1; done +curl -sf "$API/v1/health" >/dev/null || { echo "cloud-api not reachable"; exit 1; } + +echo "== seeding mock admin user (admin@test.com)" +docker exec local-stack-postgres psql -U postgres -d platform_api -q -c " + INSERT INTO users (id, email, username, display_name, avatar_url, auth_provider, provider_user_id, created_at, updated_at) + VALUES ('${MOCK_USER_ID}', 'admin@test.com', 'localdev', 'Local Dev', NULL, 'mock', 'mock_123', NOW(), NOW()) + ON CONFLICT (id) DO UPDATE SET email = EXCLUDED.email;" + +echo "== upserting two-tier model ${MODEL_ID} (base ${BASE_CTX} @ ${BASE_SHIM}, long ${LONG_CTX} @ ${LONG_SHIM})" +req PATCH /v1/admin/models "$(cat </dev/null + +echo "== creating org + credits + API key" +ORG_ID=$(req POST /v1/organizations '{"name":"local-stack-'"$RANDOM"'","description":"local dev"}' | python3 -c 'import sys,json;print(json.load(sys.stdin)["id"])') +req PATCH "/v1/admin/organizations/${ORG_ID}/limits" \ + '{"type":"payment","spendLimit":{"amount":10000000000,"currency":"USD"},"changedBy":"admin@test.com","changeReason":"local dev credits"}' >/dev/null +WS_ID=$(req GET "/v1/organizations/${ORG_ID}/workspaces" | python3 -c 'import sys,json;print(json.load(sys.stdin)["workspaces"][0]["id"])') +API_KEY=$(req POST "/v1/workspaces/${WS_ID}/api-keys" '{"name":"local-stack"}' | python3 -c 'import sys,json;print(json.load(sys.stdin)["key"])') + +echo +echo "model: ${MODEL_ID}" +echo "api key: ${API_KEY}" +echo "${API_KEY}" > .api-key +echo "(saved to dev/local-stack/.api-key)" diff --git a/dev/local-stack/tier_shim.py b/dev/local-stack/tier_shim.py new file mode 100644 index 000000000..e128ef10a --- /dev/null +++ b/dev/local-stack/tier_shim.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Per-tier backend shim for the local dev stack. + +Sits between cloud-api and a single shared llama.cpp server, playing the role +one SGLang fleet plays in prod: + + * GET /v1/models -> advertises the canonical model id + * POST /v1/tokenize -> REAL token counts (proxied to llama-server's + native /tokenize), vLLM/SGLang response shape + {"count": N} — this is what drives cloud-api's + exact-count boundary refinement + * POST /v1/chat/completions -> enforces THIS TIER's context window with the + SGLang-phrased 400 (so cloud-api's + context-400 fall-through matcher fires exactly + as in prod), otherwise proxies to the engine. + Non-streaming responses get the tier tag + appended to `.model` ("+base"/"+long") + so curl output shows which tier served. + +Saturation drill: `touch /SATURATE-` flips this shim to 503 +("queue full"), emulating SGLang's bounded queue — cloud-api then walks its +fallback chain. Remove the file to recover. + +Stdlib only — no venv needed. +""" + +import argparse +import json +import os +import sys +import threading +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +ARGS = None + + +def log(msg: str) -> None: + print(f"[{ARGS.tier}] {msg}", flush=True) + + +def engine(path: str, body: dict | None = None, raw: bytes | None = None, + headers: dict | None = None, stream: bool = False): + data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None) + req = urllib.request.Request( + f"{ARGS.engine}{path}", + data=data, + headers={"Content-Type": "application/json", **(headers or {})}, + method="POST" if data is not None else "GET", + ) + return urllib.request.urlopen(req, timeout=ARGS.engine_timeout) + + +def count_tokens(text: str) -> int: + with engine("/tokenize", body={"content": text}) as r: + return len(json.load(r).get("tokens", [])) + + +def request_text(payload: dict) -> str: + parts = [] + for m in payload.get("messages", []): + c = m.get("content") + if isinstance(c, str): + parts.append(c) + elif isinstance(c, list): + parts.extend(p.get("text", "") for p in c if isinstance(p, dict)) + if payload.get("tools"): + parts.append(json.dumps(payload["tools"])) + return "\n".join(parts) + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_): # quiet the default access log; we log ourselves + pass + + def _json(self, code: int, obj: dict): + body = json.dumps(obj).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _read_body(self) -> bytes: + return self.rfile.read(int(self.headers.get("Content-Length", 0))) + + def do_GET(self): + if self.path == "/v1/models": + self._json(200, {"object": "list", + "data": [{"id": ARGS.model_id, "object": "model", + "owned_by": "nearai"}]}) + elif self.path in ("/", "/health", "/healthz"): + self._json(200, {"status": "ok", "tier": ARGS.tier}) + else: + self._json(404, {"error": {"message": f"no route {self.path}"}}) + + def do_POST(self): + try: + if self.path == "/v1/tokenize": + payload = json.loads(self._read_body()) + n = count_tokens(payload.get("prompt") or payload.get("content") or "") + log(f"POST /v1/tokenize -> count={n}") + self._json(200, {"count": n}) + elif self.path == "/v1/chat/completions": + self.handle_completion() + else: + self._json(404, {"error": {"message": f"no route {self.path}"}}) + except Exception as e: # keep the shim alive; surface as a 500 + log(f"ERROR {self.path}: {e}") + try: + self._json(500, {"error": {"message": str(e)}}) + except Exception: + pass + + def handle_completion(self): + payload = json.loads(self._read_body()) + + if os.path.exists(ARGS.saturate_file): + log("POST /v1/chat/completions -> 503 (SATURATED)") + self._json(503, {"error": {"message": "queue full", "type": "server_error"}}) + return + + # Enforce this tier's window like SGLang does, with its phrasing so + # cloud-api's fall-through matcher behaves exactly as in prod. + n_input = count_tokens(request_text(payload)) + max_new = int(payload.get("max_tokens") or payload.get("max_completion_tokens") or 0) + if n_input + max_new > ARGS.ctx_window: + log(f"POST /v1/chat/completions -> 400 context ({n_input}+{max_new} > {ARGS.ctx_window})") + self._json(400, {"error": { + "message": (f"This model's maximum context length is {ARGS.ctx_window} tokens. " + f"However, you requested {n_input + max_new} tokens " + f"({n_input} in the messages, {max_new} in the completion). " + f"Please reduce the length of the messages or completion."), + "type": "invalid_request_error"}}) + return + + stream = bool(payload.get("stream")) + upstream = dict(payload) + upstream["model"] = ARGS.engine_model + with engine("/v1/chat/completions", body=upstream) as r: + if stream: + # SSE passthrough (tier tag only on non-streaming .model). + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Connection", "close") + self.end_headers() + sent = 0 + while chunk := r.read(8192): + self.wfile.write(chunk) + sent += len(chunk) + log(f"POST /v1/chat/completions [stream] -> 200 ({n_input} tok in, {sent}B out)") + else: + resp = json.load(r) + resp["model"] = f"{ARGS.model_id}+{ARGS.tier}" + log(f"POST /v1/chat/completions -> 200 ({n_input} tok in)") + self._json(200, resp) + + +def main(): + global ARGS + p = argparse.ArgumentParser() + p.add_argument("--tier", required=True, choices=["base", "long"]) + p.add_argument("--port", type=int, required=True) + p.add_argument("--ctx-window", type=int, required=True) + p.add_argument("--model-id", required=True) + p.add_argument("--engine", default="http://127.0.0.1:18090") + p.add_argument("--engine-model", default="local") + p.add_argument("--engine-timeout", type=int, default=120) + p.add_argument("--state-dir", default=os.path.dirname(os.path.abspath(__file__))) + ARGS = p.parse_args() + ARGS.saturate_file = os.path.join(ARGS.state_dir, f"SATURATE-{ARGS.tier}") + + srv = ThreadingHTTPServer(("127.0.0.1", ARGS.port), Handler) + log(f"tier shim on :{ARGS.port} ctx={ARGS.ctx_window} engine={ARGS.engine} " + f"(saturate file: {ARGS.saturate_file})") + try: + srv.serve_forever() + except KeyboardInterrupt: + pass + + +if __name__ == "__main__": + main()