Guard against web-code/DB schema drift (Aug 24/25 outage)

.github/workflows/ci.yml

@@ -595,6 +595,13 @@ jobs:
       - name: Install dependencies
         run: bun install --frozen-lockfile
 
+      # Web code auto-deploys on merge while DB migrations are dispatched
+      # separately, so destructive SQL needs an explicit deploy/migrate
+      # ordering plan (acknowledged in the .sql file). See the Aug 24/25 2026
+      # schema-drift outage.
+      - name: Lint migrations for additive SQL
+        run: bun scripts/lint-migrations-additive.ts
+
       - name: Apply migrations
         env:
           DATABASE_URL: postgres://cmux:cmux@localhost:5432/cmux_test

.github/workflows/cloud-vm-migrate-auto.yml

@@ -0,0 +1,94 @@
+name: Cloud VM DB migration auto-dispatch
+
+# Web code auto-deploys to Vercel when a PR merges to main, but Cloud VM
+# database migrations only run when someone dispatches cloud-vm-migrate.yml.
+# On Aug 24/25 2026 that gap broke production: three additive migrations had
+# merged, the deployed code queried the new columns, and account deletion
+# failed with VmDatabaseError until the migration was finally dispatched.
+#
+# This workflow closes the gap: any push to main that touches migrations
+# dispatches a staging migration, waits for it, then dispatches a production
+# migration. The production run still pauses on the cloud-vm-production
+# environment approval; the point is that the run now exists and waits for a
+# reviewer instead of relying on somebody remembering to dispatch it.
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - web/db/migrations/**
+
+run-name: "Auto-dispatch Cloud VM migrations for ${{ github.sha }}"
+
+permissions:
+  contents: read
+  actions: write
+
+concurrency:
+  group: cloud-vm-migrate-auto
+  cancel-in-progress: false
+
+jobs:
+  dispatch:
+    runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
+    timeout-minutes: 45
+    steps:
+      - name: Dispatch staging migration
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          # Buffer for clock skew between this runner and GitHub's run
+          # timestamps; the newest-first pick below tolerates the overlap.
+          dispatched_after="$(date -u -d '60 seconds ago' '+%Y-%m-%dT%H:%M:%S+00:00')"
+          echo "DISPATCHED_AFTER=$dispatched_after" >> "$GITHUB_ENV"
+          gh workflow run cloud-vm-migrate.yml \
+            --repo "$GITHUB_REPOSITORY" \
+            --ref main \
+            -f target=staging
+
+      - name: Wait for the staging migration to succeed
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          # gh workflow run does not return a run id (see reload-build.yml);
+          # cloud-vm-migrate.yml takes no nonce input, so match the newest
+          # dispatch-event run created after our dispatch instant.
+          run_id=""
+          for _ in $(seq 1 24); do
+            sleep 5
+            run_id="$(gh run list \
+              --repo "$GITHUB_REPOSITORY" \
+              --workflow cloud-vm-migrate.yml \
+              --event workflow_dispatch \
+              --branch main \
+              --created ">=$DISPATCHED_AFTER" \
+              --limit 1 \
+              --json databaseId \
+              --jq '.[0].databaseId // empty')"
+            if [ -n "$run_id" ]; then
+              break
+            fi
+          done
+          if [ -z "$run_id" ]; then
+            echo "::error::Dispatched a staging cloud-vm-migrate run but could not find it; dispatch target=staging and target=production manually."
+            exit 1
+          fi
+          echo "Watching staging migration run $run_id"
+          gh run watch "$run_id" --repo "$GITHUB_REPOSITORY" --interval 15 --exit-status
+
+      # The production run re-applies staging first (that is how
+      # cloud-vm-migrate.yml sequences target=production), then waits on the
+      # cloud-vm-production environment approval before touching production.
+      - name: Dispatch production migration (pauses for approval)
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          gh workflow run cloud-vm-migrate.yml \
+            --repo "$GITHUB_REPOSITORY" \
+            --ref main \
+            -f target=production
+          echo "Production migration dispatched; it waits on the cloud-vm-production environment approval." >> "$GITHUB_STEP_SUMMARY"

.github/workflows/cloud-vm-schema-parity.yml

@@ -0,0 +1,46 @@
+name: Cloud VM schema parity
+
+# Daily drift alert for the gap that broke production on Aug 24/25 2026: web
+# code auto-deploys on merge while database migrations are applied manually,
+# so deployed code can query columns whose migrations were never applied.
+# /api/health/schema-parity compares the migrations bundled in the deployed
+# code with the drizzle.__drizzle_migrations table and reports "behind" when
+# the database lags. This run fails loudly in that case.
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "37 8 * * *"
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
+    timeout-minutes: 5
+    steps:
+      - name: Check production schema parity
+        run: |
+          set -euo pipefail
+          url="https://cmux.com/api/health/schema-parity"
+          if ! body="$(curl -sS --max-time 30 --retry 3 "$url")"; then
+            echo "::error::$url is unreachable; cannot verify schema parity."
+            exit 1
+          fi
+          echo "$body"
+          status="$(printf '%s' "$body" | jq -r '.status // "invalid"')"
+          case "$status" in
+            ok|ahead)
+              echo "Schema parity: $status (codeHead=$(printf '%s' "$body" | jq -r '.codeHead'), dbHead=$(printf '%s' "$body" | jq -r '.dbHead'))"
+              ;;
+            behind)
+              pending="$(printf '%s' "$body" | jq -r '.pending | join(", ")')"
+              echo "::error::Production database schema is BEHIND the deployed code. Pending migrations: $pending. Approve the waiting cloud-vm-migrate production run, or dispatch cloud-vm-migrate.yml with target=production."
+              exit 1
+              ;;
+            *)
+              echo "::error::Unexpected schema-parity response (status=$status); treat as drift until proven otherwise."
+              exit 1
+              ;;
+          esac

web/app/api/health/schema-parity/route.ts

@@ -0,0 +1,23 @@
+import { connection } from "next/server";
+import { schemaParityResponse } from "../../../../services/health/schemaParity";
+
+/**
+ * Public, unauthenticated drift check between the migrations bundled in the
+ * deployed code and the migrations applied to the Cloud VM database. Web code
+ * auto-deploys on merge while database migrations are applied through the
+ * cloud-vm-migrate workflow, so the two can drift (Aug 24/25 2026: deployed
+ * code queried columns whose migrations were merged but not applied, and
+ * account deletion failed with VmDatabaseError until the migration ran).
+ *
+ * 200 {"status":"ok"|"ahead",...} when the database is not lagging the code;
+ * 503 {"status":"behind","pending":[...]} when applied migrations lag;
+ * 503 {"status":"unavailable"} when the database cannot be queried. The body
+ * only ever contains migration names, never connection or error details.
+ */
+export async function GET(): Promise<Response> {
+  // Cache Components is enabled, so a GET handler can be prerendered at build
+  // time (`export const dynamic` no longer exists). Parity must be measured
+  // per request, never baked into the build.
+  await connection();
+  return schemaParityResponse();
+}

web/next.config.ts

@@ -216,6 +216,9 @@ const nextConfig: NextConfig = {
       "./public/logo.png",
     ],
     "**/browser-opengraph-image": ["./public/logo.png"],
+    // The schema-parity health route reads bundled migration names (and, for
+    // legacy rows, sql hashes) from the migrations folder at request time.
+    "**/api/health/schema-parity": ["./db/migrations/*/migration.sql"],
     // Changelog versions outside generateStaticParams render at request time
     // and read the copy that tools/sync-changelog.ts writes before the build.
     "**/docs/changelog": ["./CHANGELOG.md"],