feat: enable PostHog observability for the demo

anomstack/observability.py

@@ -0,0 +1,100 @@
+"""Opt-in PostHog error tracking and OpenTelemetry application logs."""
+
+import atexit
+import logging
+import os
+import time
+
+_client = None
+
+
+def configure_observability():
+    """Initialize once per process; leave non-demo installations unchanged."""
+    global _client
+    if _client is not None:
+        return _client
+    key = os.getenv("POSTHOG_API_KEY")
+    if not key or os.getenv("POSTHOG_OBSERVABILITY_ENABLED", "false").lower() != "true":
+        return None
+
+    from posthog import Posthog
+
+    host = os.getenv("POSTHOG_HOST", "https://us.i.posthog.com").rstrip("/")
+    _client = Posthog(
+        key,
+        host=host,
+        enable_exception_autocapture=True,
+        super_properties={"app": "anomstack", "environment": "demo"},
+    )
+    atexit.register(_client.shutdown)
+
+    from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
+    from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+    from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+    from opentelemetry.sdk.resources import Resource
+
+    provider = LoggerProvider(
+        resource=Resource.create(
+            {
+                "service.name": "anomstack-dashboard",
+                "deployment.environment.name": "demo",
+            }
+        )
+    )
+    provider.add_log_record_processor(
+        BatchLogRecordProcessor(
+            OTLPLogExporter(
+                endpoint=f"{host}/i/v1/logs",
+                headers={"Authorization": f"Bearer {key}"},
+                timeout=5,
+            )
+        )
+    )
+    handler = LoggingHandler(level=logging.INFO, logger_provider=provider)
+    # Export application logs only, avoiding exporter recursion and SDK payload logs.
+    for name in ("anomstack", "anomstack_dashboard"):
+        logger = logging.getLogger(name)
+        logger.addHandler(handler)
+        logger.setLevel(logging.INFO)
+    atexit.register(provider.shutdown)
+    logging.getLogger("anomstack").info("PostHog observability initialized")
+    return _client
+
+
+class ObservabilityMiddleware:
+    """Capture request failures without changing responses or recording request bodies."""
+
+    def __init__(self, app, client):
+        self.app = app
+        self.client = client
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http":
+            return await self.app(scope, receive, send)
+        started = time.monotonic()
+        status = 500
+
+        async def track_send(message):
+            nonlocal status
+            if message["type"] == "http.response.start":
+                status = message["status"]
+            await send(message)
+
+        try:
+            await self.app(scope, receive, track_send)
+        except Exception as exc:
+            self.client.capture_exception(exc, distinct_id="anomstack-demo-server")
+            raise
+        finally:
+            if scope.get("path") != "/health":
+                route = scope.get("route")
+                logging.getLogger("anomstack.http").log(
+                    logging.ERROR if status >= 500 else logging.INFO,
+                    "HTTP request completed",
+                    extra={
+                        "http.request.method": scope["method"],
+                        "http.route": getattr(route, "path", "unmatched"),
+                        "http.response.status_code": status,
+                        "duration_ms": round((time.monotonic() - started) * 1000, 2),
+                    },
+                )

constraints.txt

@@ -289,6 +289,7 @@ googleapis-common-protos==1.75.3
     # via
     #   google-api-core
     #   grpcio-status
+    #   opentelemetry-exporter-otlp-proto-http
 gql==4.0.0
     # via dagster-graphql
 graphene==3.4.3
@@ -604,7 +605,28 @@ oauthlib==3.3.1
 openai==3.8.0
     # via langchain-openai
 opentelemetry-api==1.44.0
-    # via google-api-core
+    # via
+    #   google-api-core
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
+opentelemetry-exporter-otlp-proto-common==1.44.0
+    # via opentelemetry-exporter-otlp-proto-http
+opentelemetry-exporter-otlp-proto-http==1.44.0
+    # via
+    #   -r requirements.txt
+    #   anomstack
+opentelemetry-proto==1.44.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-common
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-sdk==1.44.0
+    # via
+    #   -r requirements.txt
+    #   anomstack
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-semantic-conventions==0.65b0
+    # via opentelemetry-sdk
 orjson==3.12.0
     # via
     #   langgraph-sdk
@@ -714,6 +736,7 @@ protobuf==6.33.6
     #   googleapis-common-protos
     #   grpcio-health-checking
     #   grpcio-status
+    #   opentelemetry-proto
     #   proto-plus
     #   streamlit
 psutil==7.2.2
@@ -892,6 +915,7 @@ requests==2.34.2
     #   gql
     #   jupyterlab-server
     #   langsmith
+    #   opentelemetry-exporter-otlp-proto-http
     #   posthog
     #   requests-oauthlib
     #   requests-toolbelt
@@ -1068,6 +1092,9 @@ typing-extensions==4.16.0
     #   langsmith
     #   openai
     #   opentelemetry-api
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
     #   posthog
     #   pydantic
     #   pydantic-core

dashboard/README.md

@@ -28,3 +28,32 @@ To stop:
 ```bash
 make kill-dashboardd
 ```
+
+### Demo observability
+
+The Fly demo sends browser analytics and server telemetry to Deepwell's PostHog
+`dev` project (148051). `profiles/demo.env` holds the deployment configuration.
+
+- `POSTHOG_FRONTEND_API_KEY`: browser analytics, exceptions, and session replay.
+- `POSTHOG_API_KEY` / `POSTHOG_HOST`: existing LLM telemetry and server errors/logs.
+- `POSTHOG_OBSERVABILITY_ENABLED=true`: opt in to dashboard request logs and server
+  exception capture. Other installations remain opted out by default.
+
+Application logs use OpenTelemetry HTTP export to `/i/v1/logs`, with service name
+`anomstack-dashboard`. HTTP logs include method, route template, status, and duration;
+request bodies, query strings, and headers are not included. This covers dashboard
+application logs, not nginx, Dagster daemon, or machine-wide stdout.
+
+Replay uses the project's recording settings and masks input values. The existing
+Replay Vision scanners process eligible recordings and emit Signals. Those scanners
+are shared with other apps using the dev project. Filter analytics/replays to
+`anomstack-live-demo.fly.dev` when inspecting this demo.
+
+Useful project links:
+- https://us.posthog.com/project/148051/error_tracking
+- https://us.posthog.com/project/148051/replay
+- https://us.posthog.com/project/148051/replay-vision
+- https://us.posthog.com/project/148051/logs
+
+Setup references: https://posthog.com/docs/logs/installation/python and
+https://posthog.com/docs/error-tracking/installation/python.

dashboard/app.py

@@ -88,6 +88,12 @@ def get_posthog_script():
     log=log,
 )
 
+from anomstack.observability import ObservabilityMiddleware, configure_observability
+
+telemetry = configure_observability()
+if telemetry is not None:
+    app.add_middleware(ObservabilityMiddleware, client=telemetry)
+
 # Set the app state
 app.state = AppState()
 

dashboard/constants.py

@@ -14,6 +14,12 @@
     !function(t,e){var o,n,p,r;e.__SV||(window.posthog=e,e._i=[],e.init=function(i,s,a){function g(t,e){var o=e.split(".");2==o.length&&(t=t[o[0]],e=o[1]),t[e]=function(){t.push([e].concat(Array.prototype.slice.call(arguments,0)))}}(p=t.createElement("script")).type="text/javascript",p.crossOrigin="anonymous",p.async=!0,p.src=s.api_host.replace(".i.posthog.com","-assets.i.posthog.com")+"/static/array.js",(r=t.getElementsByTagName("script")[0]).parentNode.insertBefore(p,r);var u=e;for(void 0!==a?u=e[a]=[]:a="posthog",u.people=u.people||[],u.toString=function(t){var e="posthog";return"posthog"!==a&&(e+="."+a),t||(e+=" (stub)"),e},u.people.toString=function(){return u.toString(1)+".people (stub)"},o="init capture register register_once register_for_session unregister unregister_for_session getFeatureFlag getFeatureFlagPayload isFeatureEnabled reloadFeatureFlags updateEarlyAccessFeatureEnrollment getEarlyAccessFeatures on onFeatureFlags onSurveysLoaded onSessionId getSurveys getActiveMatchingSurveys renderSurvey canRenderSurvey canRenderSurveyAsync identify setPersonProperties group resetGroups setPersonPropertiesForFlags resetPersonPropertiesForFlags setGroupPropertiesForFlags resetGroupPropertiesForFlags reset get_distinct_id getGroups get_session_id get_session_replay_url alias set_config startSessionRecording stopSessionRecording sessionRecordingStarted captureException loadToolbar get_property getSessionProperty createPersonProfile opt_in_capturing opt_out_capturing has_opted_in_capturing has_opted_out_capturing clear_opt_in_out_capturing debug getPageViewId captureTraceFeedback captureTraceMetric".split(" "),n=0;n<o.length;n++)g(u,o[n]);e._i.push([i,s,a])},e.__SV=1)}(document,window.posthog||[]);
     posthog.init(window.POSTHOG_API_KEY || '', {
         api_host: 'https://us.i.posthog.com',
+        defaults: '2026-05-30',
+        capture_exceptions: true,
+        capture_pageview: 'history_change',
+        capture_pageleave: true,
+        session_recording: { maskAllInputs: true },
+        loaded: function(ph) { ph.register({app: 'anomstack', deployment: 'demo'}); },
         person_profiles: 'identified_only', // or 'always' to create profiles for anonymous users as well
     })
 """

profiles/demo.env

@@ -101,12 +101,12 @@ ANOMSTACK__EIRGRID__DELETE_DEFAULT_SCHEDULE_STATUS=RUNNING
 ANOMSTACK__EIRGRID__LLMALERT_DEFAULT_SCHEDULE_STATUS=RUNNING
 
 # PostHog frontend analytics tracking
-POSTHOG_FRONTEND_API_KEY=phc_TudOaC1O5X0Ycl3GtIR0A2cjREW6B6Zp3xDd6c8qxUM
+POSTHOG_FRONTEND_API_KEY=phc_ABOAagCSNfMOUWin6A6Tda0WuhzWLFSXjSgSiq9KKBs
 
 # PostHog LLM analytics tracking for anomaly-agent
 # Tracks LLM calls (tokens, latency, costs) via posthog.ai.langchain integration
 POSTHOG_ENABLED=true
-POSTHOG_API_KEY=phc_TudOaC1O5X0Ycl3GtIR0A2cjREW6B6Zp3xDd6c8qxUM
+POSTHOG_API_KEY=phc_ABOAagCSNfMOUWin6A6Tda0WuhzWLFSXjSgSiq9KKBs
 POSTHOG_HOST=https://us.i.posthog.com
 POSTHOG_DISTINCT_ID=anomstack-demo
 
@@ -177,3 +177,6 @@ ANOMSTACK__PYTHON_INGEST_SIMPLE__ALERT_DEFAULT_SCHEDULE_STATUS=RUNNING
 ANOMSTACK__PYTHON_INGEST_SIMPLE__CHANGE_DEFAULT_SCHEDULE_STATUS=RUNNING
 ANOMSTACK__PYTHON_INGEST_SIMPLE__DELETE_DEFAULT_SCHEDULE_STATUS=RUNNING
 ANOMSTACK__PYTHON_INGEST_SIMPLE__LLMALERT_DEFAULT_SCHEDULE_STATUS=RUNNING
+
+# Application error tracking and OTLP logs
+POSTHOG_OBSERVABILITY_ENABLED=true

requirements.txt

@@ -31,3 +31,7 @@ slack_sdk
 snowflake-connector-python[pandas]
 sqlglot
 streamlit
+
+# PostHog application log export
+opentelemetry-sdk
+opentelemetry-exporter-otlp-proto-http

tests/test_observability.py

@@ -0,0 +1,50 @@
+"""Request telemetry must preserve application behavior."""
+
+import asyncio
+from unittest.mock import Mock
+
+import pytest
+
+from anomstack.observability import ObservabilityMiddleware, configure_observability
+
+
+def test_disabled_by_default(monkeypatch):
+    monkeypatch.delenv("POSTHOG_OBSERVABILITY_ENABLED", raising=False)
+    assert configure_observability() is None
+
+
+def test_response_is_preserved():
+    client = Mock()
+    messages = []
+
+    async def send(message):
+        messages.append(message)
+
+    async def application(scope, receive, send):
+        await send({"type": "http.response.start", "status": 204})
+        await send({"type": "http.response.body", "body": b""})
+
+    asyncio.run(
+        ObservabilityMiddleware(application, client)(
+            {"type": "http", "method": "GET", "path": "/"}, None, send
+        )
+    )
+    assert messages[0]["status"] == 204
+    assert messages[1]["body"] == b""
+    client.capture_exception.assert_not_called()
+
+
+def test_exception_is_captured_and_reraised():
+    client = Mock()
+    error = ValueError("controlled test")
+
+    async def application(scope, receive, send):
+        raise error
+
+    with pytest.raises(ValueError, match="controlled test"):
+        asyncio.run(
+            ObservabilityMiddleware(application, client)(
+                {"type": "http", "method": "GET", "path": "/"}, None, None
+            )
+        )
+    client.capture_exception.assert_called_once_with(error, distinct_id="anomstack-demo-server")