From c85623a77ea9b79cc7f4d3d2100f9831e7ce1465 Mon Sep 17 00:00:00 2001 From: sebi Date: Tue, 10 Nov 2026 00:59:13 -0500 Subject: [PATCH 1/2] =?UTF-8?q?feat(v31):=20plan=20101=20wave=20F=20?= =?UTF-8?q?=E2=80=94=20SAML=20SSO,=20partitions,=20ledger,=20locks,=20sear?= =?UTF-8?q?ch=20index,=20upload=20scan,=20pdf(),=20fast=20boot,=20verify?= =?UTF-8?q?=20--changed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scale and enterprise features, each with a port or factory over an existing primitive, plus the replacement review (bug-hunter x2, concurrency, security) and every confirmed finding fixed. Breaking #50-#53 in CHANGELOG and wiki/Upgrading.md (OAuth routes move to /auth/sso/). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 15 +- CHANGELOG.md | 19 + bun.lock | 3 + docs/history/cli-file-map.md | 87 + docs/idea/17-scale-ladder.md | 31 +- docs/idea/18-build-vs-wrap.md | 14 + .../22-scale.md | 2 +- .../apps/web/e2e/built-site.e2e.test.ts | 5 +- dummy/docs-site/package.json | 1 + .../apps/web/e2e/smoke.e2e.test.ts | 5 +- dummy/social-media-clone/package.json | 1 + .../03_sequences/x_job_lease_fence.sql | 3 + .../framework/04_tables/x_job_leases.sql | 1 + examples/dummy/apps/web/api/index.ts | 6 + .../dummy/apps/web/app/auth/login.test.ts | 10 +- examples/dummy/apps/web/app/auth/login.ts | 4 +- .../web/app/billing/jobs/post-charge.test.ts | 33 + .../apps/web/app/billing/jobs/post-charge.ts | 34 + .../web/app/billing/jobs/settle-upgrade.ts | 5 + .../app/billing/settle-upgrade.job.test.ts | 19 +- .../web/app/posts/actions/print-post.test.ts | 42 + .../apps/web/app/posts/actions/print-post.ts | 29 + .../posts/backfills/reindex-posts.job.test.ts | 56 + .../web/app/posts/backfills/reindex-posts.ts | 26 + .../dummy/apps/web/clients/postly/index.ts | 7 +- .../dummy/apps/web/clients/postly/types.ts | 4 +- .../dummy/apps/web/e2e/fixtures/postly.ts | 5 +- examples/dummy/apps/web/exercise.ts | 18 + examples/dummy/openapi.json | 112 + examples/dummy/packages/db/migrations.test.ts | 8 +- ...19_ledger_lines_and_post_title_search.hash | 1 + ..._lines_and_post_title_search.snapshot.json | 1871 +++++++++++++++++ ...219_ledger_lines_and_post_title_search.sql | 27 + .../db/schema/04_tables/ledger_lines.sql | 16 + .../packages/db/schema/04_tables/posts.sql | 1 + .../packages/db/schema/05_indexes/posts.sql | 2 + .../schema/06_foreign_keys/ledger_lines.sql | 3 + .../03_sequences/x_job_lease_fence.sql | 3 + .../framework/04_tables/x_job_leases.sql | 1 + examples/dummy/packages/db/src/client.ts | 2 + examples/dummy/packages/db/src/index.ts | 2 + .../packages/db/src/schema/ledger-lines.ts | 25 + .../dummy/packages/db/src/schema/posts.ts | 3 +- examples/dummy/x.manifest.json | 206 +- framework.manifest.json | 145 +- packages/ai/README.md | 25 +- packages/ai/src/adapter.ts | 13 +- packages/ai/src/llm-cache.test.ts | 71 +- packages/ai/src/runtime.test.ts | 74 +- packages/ai/src/runtime.ts | 103 +- packages/auth/CLAUDE.md | 31 +- packages/auth/README.md | 88 +- packages/auth/src/auth.ts | 2 +- packages/auth/src/client-scope.ts | 2 +- packages/auth/src/errors.ts | 8 + packages/auth/src/identity-provider-cookie.ts | 89 + .../auth/src/identity-provider-oidc.test.ts | 68 + packages/auth/src/identity-provider-oidc.ts | 142 ++ .../auth/src/identity-provider-route-core.ts | 253 +++ .../auth/src/identity-provider-routes.test.ts | 221 ++ packages/auth/src/identity-provider-routes.ts | 78 + packages/auth/src/identity-provider-status.ts | 45 + packages/auth/src/identity-provider.ts | 112 + packages/auth/src/index.ts | 32 +- packages/auth/src/oauth-discovery.ts | 2 +- packages/auth/src/oauth-errors.test.ts | 4 +- packages/auth/src/oauth-errors.ts | 28 +- packages/auth/src/oauth-paths.ts | 7 +- packages/auth/src/oauth-route-keys.test.ts | 4 +- packages/auth/src/oauth-route-public.test.ts | 18 +- .../auth/src/oauth-route-refusals.test.ts | 8 +- packages/auth/src/oauth-route.test.ts | 58 +- packages/auth/src/oauth-route.ts | 357 +--- packages/auth/src/saml-assertion.ts | 156 ++ packages/auth/src/saml-c14n.test.ts | 104 + packages/auth/src/saml-c14n.ts | 118 ++ packages/auth/src/saml-cert-fixture.ts | 66 + packages/auth/src/saml-certificate.ts | 86 + packages/auth/src/saml-dom.ts | 66 + packages/auth/src/saml-dsig.ts | 165 ++ packages/auth/src/saml-email-trust.ts | 49 + packages/auth/src/saml-errors.ts | 96 + packages/auth/src/saml-hostile.test.ts | 158 ++ packages/auth/src/saml-idp-fixture.ts | 173 ++ packages/auth/src/saml-metadata.test.ts | 67 + packages/auth/src/saml-metadata.ts | 80 + packages/auth/src/saml-provider.test.ts | 169 ++ packages/auth/src/saml-provider.ts | 179 ++ packages/auth/src/saml-request.ts | 51 + packages/auth/src/saml-response.ts | 131 ++ .../auth/src/saml-test-support-fixture.ts | 98 + packages/auth/src/saml-vectors-fixture.ts | 20 + packages/auth/src/saml-xml.test.ts | 91 + packages/auth/src/saml-xml.ts | 328 +++ packages/cache/CLAUDE.md | 11 +- packages/cache/README.md | 22 + packages/cache/src/index.ts | 9 + packages/cache/src/redis-pubsub.test.ts | 260 +++ packages/cache/src/redis-pubsub.ts | 215 ++ packages/cli/CLAUDE.md | 38 +- packages/cli/e2e/cdp-shot.e2e.test.ts | 3 +- packages/cli/e2e/client-navigation-fixture.ts | 2 +- packages/cli/e2e/client-vitals.e2e.test.ts | 2 +- packages/cli/e2e/data-table-live.e2e.test.ts | 2 +- packages/cli/e2e/service-worker.e2e.test.ts | 2 +- packages/cli/e2e/six-deploys.e2e.test.ts | 5 +- packages/cli/e2e/static-cards.e2e.test.ts | 3 +- packages/cli/e2e/stream-hole.e2e.test.ts | 2 +- packages/cli/e2e/ui-as-prop.e2e.test.ts | 2 +- packages/cli/e2e/ui-chart-live.e2e.test.ts | 3 +- packages/cli/e2e/ui-confirm.e2e.test.ts | 3 +- packages/cli/e2e/ui-date-pickers.e2e.test.ts | 3 +- packages/cli/e2e/ui-drag.e2e.test.ts | 2 +- packages/cli/e2e/ui-mobile-frame.e2e.test.ts | 3 +- packages/cli/e2e/ui-rtl.e2e.test.ts | 3 +- .../cli/e2e/ui-searchable-select.e2e.test.ts | 3 +- packages/cli/e2e/ui-viewers.e2e.test.ts | 2 +- packages/cli/e2e/ui-virtual-list.e2e.test.ts | 3 +- packages/cli/e2e/zone-detect.e2e.test.ts | 2 +- packages/cli/src/admin-registration.ts | 2 +- packages/cli/src/affected.test.ts | 13 + packages/cli/src/affected.ts | 6 +- .../cli/src/app-boundaries-workspace.test.ts | 89 + packages/cli/src/app-boundaries.ts | 52 +- packages/cli/src/app-load-async.test.ts | 39 + packages/cli/src/app-load-async.ts | 133 ++ packages/cli/src/app-load-batch.ts | 202 ++ packages/cli/src/app-load-order.test.ts | 269 +++ packages/cli/src/app-load-prebuilt.ts | 106 + packages/cli/src/app-load-register.ts | 173 ++ packages/cli/src/app-load.ts | 318 ++- .../cli/src/browser-launcher-fake-fixture.ts | 2 +- packages/cli/src/browser-launcher.test.ts | 2 +- packages/cli/src/browser-launcher.ts | 2 +- packages/cli/src/budgets-preload.test.ts | 74 + packages/cli/src/budgets-scan.ts | 8 + packages/cli/src/budgets.test.ts | 8 +- packages/cli/src/budgets.ts | 11 +- packages/cli/src/build-registry.test.ts | 237 +++ packages/cli/src/build-registry.ts | 124 ++ packages/cli/src/cdp-shot-driver.test.ts | 2 +- packages/cli/src/cdp-shot-driver.ts | 5 +- packages/cli/src/cdp-shot-page.ts | 4 +- packages/cli/src/cdp-shot-popup.e2e.test.ts | 3 +- packages/cli/src/cdp-shot-watch.ts | 2 +- packages/cli/src/cdp-shot-wire-fixture.ts | 4 +- packages/cli/src/cmd-generate-refresh.test.ts | 86 + packages/cli/src/cmd-generate-spec.ts | 6 + packages/cli/src/cmd-generate.ts | 26 +- packages/cli/src/cmd-verify-changed.test.ts | 214 ++ packages/cli/src/cmd-verify-spec.ts | 22 +- packages/cli/src/cmd-verify.test.ts | 8 +- packages/cli/src/cmd-verify.ts | 20 +- packages/cli/src/document-options.ts | 12 +- packages/cli/src/error-codes.ts | 10 + packages/cli/src/exercise-subject.test.ts | 36 + packages/cli/src/exercise-subject.ts | 51 +- packages/cli/src/generate-files.ts | 34 +- packages/cli/src/generate-kinds.ts | 5 + packages/cli/src/generate-output.test.ts | 5 +- packages/cli/src/generate-refresh.test.ts | 38 + packages/cli/src/generate-refresh.ts | 53 + packages/cli/src/generate-workspace.test.ts | 149 ++ packages/cli/src/generate-workspace.ts | 144 ++ packages/cli/src/image-prepare.test.ts | 73 + packages/cli/src/image-prepare.ts | 8 +- packages/cli/src/island-bundle.test.ts | 14 + packages/cli/src/island-bundle.ts | 10 +- packages/cli/src/job-registration.test.ts | 37 + packages/cli/src/job-registration.ts | 49 +- packages/cli/src/mcp-errors.ts | 7 + packages/cli/src/otlp-export.ts | 15 +- .../cli/src/route-islands-preload.test.ts | 104 + packages/cli/src/route-islands-preload.ts | 46 + packages/cli/src/route-islands.ts | 8 +- packages/cli/src/runtime-bus.test.ts | 18 + packages/cli/src/runtime-bus.ts | 8 +- packages/cli/src/runtime-cache-bus.test.ts | 108 + packages/cli/src/runtime-cache-bus.ts | 53 + .../src/runtime-cache-redis-peer-fixture.ts | 21 + .../cli/src/runtime-cache-redis.live.test.ts | 119 ++ packages/cli/src/runtime-cache.ts | 42 +- packages/cli/src/runtime-overrides.ts | 32 +- packages/cli/src/runtime-port-bindings.ts | 20 +- packages/cli/src/runtime-ports.ts | 5 + packages/cli/src/runtime-render-document.ts | 5 +- packages/cli/src/runtime-services.ts | 1 + packages/cli/src/scaffold-fixture.ts | 5 + packages/cli/src/serve-boot.ts | 11 +- .../cli/src/serve-graph-ceilings-fixture.ts | 34 +- .../src/serve-graph-web-ceiling-fixture.ts | 7 +- packages/cli/src/serve-prebuilt-paths.ts | 3 + packages/cli/src/shot-browser.test.ts | 2 +- packages/cli/src/templates/entity.test.ts | 14 + packages/cli/src/templates/entity.ts | 3 + packages/cli/src/templates/route.ts | 8 +- .../templates/scaffold-coverage-floor.test.ts | 2 +- packages/cli/src/templates/scaffold-repo.ts | 104 +- .../src/templates/scaffold-verify-floor.ts | 97 + .../cli/src/templates/sign-in-oidc.test.ts | 34 + packages/cli/src/templates/sign-in-oidc.ts | 8 +- packages/cli/src/templates/workspace.test.ts | 64 + packages/cli/src/templates/workspace.ts | 75 + packages/cli/src/verify-changed-facts.test.ts | 65 + packages/cli/src/verify-changed-facts.ts | 53 + packages/cli/src/verify-changed-scope.ts | 161 ++ packages/cli/src/verify-changed.test.ts | 154 ++ packages/cli/src/verify-changed.ts | 174 ++ packages/cli/src/verify-checks.ts | 6 +- .../cli/src/verify-drift-owner-step.test.ts | 34 + packages/cli/src/verify-drift-owner.test.ts | 35 + packages/cli/src/verify-drift-owner.ts | 30 + packages/cli/src/verify-e2e.ts | 3 +- packages/cli/src/verify-step.ts | 13 + packages/cli/src/verify-tests-changed.test.ts | 116 + packages/cli/src/verify-tests.ts | 35 +- packages/core/README.md | 5 +- packages/core/src/instrument-events.ts | 4 + packages/core/src/metrics-export.test.ts | 236 +++ packages/core/src/metrics.test.ts | 184 -- packages/core/src/metrics.ts | 31 +- packages/core/src/registrar.test.ts | 9 + packages/core/src/registrar.ts | 40 + packages/db/CLAUDE.md | 1 + packages/db/README.md | 17 + packages/db/src/branch-errors.ts | 24 + packages/db/src/branch.ts | 3 +- packages/db/src/entity-shape.ts | 20 + packages/db/src/errors.ts | 22 +- .../src/generate-partition-embedded.test.ts | 33 + packages/db/src/generate-partition-fixture.ts | 230 ++ .../src/generate-partition-lock.live.test.ts | 71 + packages/db/src/generate-partition-run.ts | 97 + .../db/src/generate-partition-sidecar.test.ts | 66 + .../db/src/generate-partition.live.test.ts | 30 + packages/db/src/generate-partition.test.ts | 94 + packages/db/src/generate-partition.ts | 190 ++ packages/db/src/generate-snapshot.ts | 9 + packages/db/src/generate.ts | 7 +- packages/db/src/index.ts | 14 +- packages/db/src/introspect.ts | 7 + packages/db/src/pglite-branch.ts | 3 +- packages/db/src/snapshot-parse.ts | 12 + packages/entity/CLAUDE.md | 7 +- packages/entity/README.md | 99 + packages/entity/src/adapter.ts | 11 + packages/entity/src/column.ts | 12 + packages/entity/src/describe.ts | 5 + packages/entity/src/entity-error.ts | 3 + packages/entity/src/entity.ts | 17 + packages/entity/src/index.ts | 29 + .../entity/src/partition-describe.test.ts | 44 + packages/entity/src/partition-entity.test.ts | 52 + packages/entity/src/partition.test.ts | 99 + packages/entity/src/partition.ts | 140 ++ packages/entity/src/personal-data-kind.ts | 15 + packages/entity/src/personal-data.test.ts | 92 + packages/entity/src/personal-data.ts | 66 + packages/entity/src/query.ts | 17 +- packages/entity/src/refuse.test.ts | 2 + packages/entity/src/registry.ts | 3 + packages/entity/src/search-index-installed.ts | 31 + packages/entity/src/search-index-memory.ts | 103 + packages/entity/src/search-index-postgres.ts | 85 + packages/entity/src/search-index-sync.test.ts | 137 ++ packages/entity/src/search-index-sync.ts | 96 + packages/entity/src/search-index-version.ts | 42 + packages/entity/src/search-index.test.ts | 193 ++ packages/entity/src/search-index.ts | 107 + packages/entity/src/search-repo.test.ts | 87 + packages/entity/src/search-repo.ts | 91 + packages/entity/src/types.ts | 12 + packages/http/src/error-map-tier-1.ts | 4 + packages/http/src/error-map-tier-2.ts | 6 + packages/http/src/error-map-tier-3.ts | 12 + packages/http/src/error-map-tier-4.ts | 15 + packages/jobs/CLAUDE.md | 6 + packages/jobs/README.md | 155 ++ packages/jobs/src/driver-pg-ddl.ts | 5 + packages/jobs/src/driver-pg-sql.ts | 29 +- packages/jobs/src/driver-pg-stores.test.ts | 6 +- packages/jobs/src/driver-pg.ts | 15 +- packages/jobs/src/error-codes.ts | 27 + packages/jobs/src/gdpr-audit.ts | 75 + packages/jobs/src/gdpr-coverage.ts | 94 + packages/jobs/src/gdpr-errors.ts | 35 + packages/jobs/src/gdpr.job.test.ts | 239 +++ packages/jobs/src/index.ts | 33 + packages/jobs/src/leases.test.ts | 19 +- packages/jobs/src/leases.ts | 12 +- packages/jobs/src/ledger-book.ts | 168 ++ packages/jobs/src/ledger-errors.ts | 48 + packages/jobs/src/ledger-tenant.job.test.ts | 122 ++ packages/jobs/src/ledger.live.test.ts | 154 ++ packages/jobs/src/ledger.test.ts | 174 ++ packages/jobs/src/ledger.ts | 145 ++ packages/jobs/src/lock-errors.ts | 54 + packages/jobs/src/lock.live.test.ts | 144 ++ packages/jobs/src/lock.test.ts | 357 ++++ packages/jobs/src/lock.ts | 176 ++ packages/jobs/src/partitions.live.test.ts | 166 ++ packages/jobs/src/partitions.test.ts | 60 + packages/jobs/src/partitions.ts | 119 ++ packages/jobs/src/reindex.test.ts | 143 ++ packages/jobs/src/reindex.ts | 70 + .../jobs/src/scheduler-failover.live.test.ts | 243 +++ packages/jobs/src/subject-jobs.ts | 12 +- packages/jobs/src/worker-backlog.test.ts | 133 ++ packages/jobs/src/worker-loop.ts | 33 +- packages/jobs/src/worker.ts | 14 +- packages/notify/CLAUDE.md | 1 + packages/notify/README.md | 33 + packages/notify/src/adapter.ts | 7 +- packages/notify/src/errors.test.ts | 2 + packages/notify/src/errors.ts | 8 + packages/notify/src/index.ts | 7 + packages/notify/src/sms-channel.test.ts | 109 + packages/notify/src/sms-channel.ts | 72 + packages/notify/src/sms-errors.ts | 32 + packages/notify/src/sms-transport.test.ts | 78 + packages/notify/src/sms-transport.ts | 109 + packages/realtime/CLAUDE.md | 3 +- .../realtime/src/replicator-reconnect.test.ts | 177 ++ packages/realtime/src/replicator-reconnect.ts | 85 + .../src/replicator-supervision.test.ts | 11 +- packages/realtime/src/replicator.ts | 32 +- packages/render/src/module-loader.test.ts | 5 +- packages/render/src/surfaces.test.ts | 27 + packages/render/src/surfaces.ts | 20 +- packages/scraping/CLAUDE.md | 16 +- packages/scraping/README.md | 57 + packages/scraping/package.json | 4 +- .../src/cdp-connection.test.ts | 0 .../src/cdp-connection.ts | 21 +- packages/scraping/src/cdp-error-codes.ts | 28 + .../{testing => scraping}/src/cdp-errors.ts | 14 +- .../src/cdp-launch-attempt.ts | 10 + .../src/cdp-launch-candidates.ts | 0 .../src/cdp-launch-probe.test.ts | 0 .../src/cdp-launch-probe.ts | 0 .../src/cdp-launch-profile.test.ts | 0 .../src/cdp-launch-profile.ts | 0 .../src/cdp-launch-reap.test.ts | 0 .../src/cdp-launch-reap.ts | 0 .../src/cdp-launch-relaunch.test.ts | 0 .../src/cdp-launch-wire.test.ts | 0 .../src/cdp-launch-wire.ts | 0 .../src/cdp-launch.test.ts | 38 + .../{testing => scraping}/src/cdp-launch.ts | 40 +- .../src/cdp-pipe.test.ts | 0 .../{testing => scraping}/src/cdp-pipe.ts | 0 .../src/cdp-wire-watch.test.ts | 2 +- .../src/cdp-wire-watch.ts | 0 packages/scraping/src/cdp.ts | 46 + .../scraping/src/clock-discipline.test.ts | 18 +- packages/scraping/src/error-throws-pdf.ts | 45 + packages/scraping/src/errors.ts | 16 + packages/scraping/src/index.ts | 9 + packages/scraping/src/pdf-action.test.ts | 121 ++ packages/scraping/src/pdf-action.ts | 116 + packages/scraping/src/pdf-cdp-params.ts | 65 + packages/scraping/src/pdf-fence.ts | 86 + .../scraping/src/pdf-render-fence.test.ts | 81 + packages/scraping/src/pdf-render-fixture.ts | 103 + .../scraping/src/pdf-render-slots.test.ts | 104 + packages/scraping/src/pdf-render.test.ts | 72 + packages/scraping/src/pdf-render.ts | 241 +++ packages/scraping/src/pdf-slots.ts | 68 + packages/scraping/tsconfig.json | 3 + packages/storage/CLAUDE.md | 7 +- packages/storage/README.md | 54 +- packages/storage/src/adapter.test.ts | 9 +- packages/storage/src/adapter.ts | 10 +- .../storage/src/attachment-scan-race.test.ts | 100 + .../storage/src/attachment-scan-read.test.ts | 93 + packages/storage/src/attachment-scan.test.ts | 150 ++ packages/storage/src/attachment.ts | 50 +- packages/storage/src/driver-local-list.ts | 2 +- packages/storage/src/errors-scan.ts | 35 + packages/storage/src/errors-variant.ts | 40 + packages/storage/src/errors.ts | 4 + packages/storage/src/image-variant.test.ts | 170 ++ packages/storage/src/image-variant.ts | 106 + packages/storage/src/index.ts | 12 + packages/storage/src/upload-scan.ts | 78 + packages/storage/src/upload-scanner-memory.ts | 62 + packages/storage/src/upload-scanner.test.ts | 68 + packages/storage/src/upload-scanner.ts | 52 + packages/storage/src/upload.ts | 24 +- packages/testing/CLAUDE.md | 5 +- packages/testing/README.md | 10 +- packages/testing/e2e/cdp-browser.e2e.test.ts | 2 +- .../e2e/cdp-launch-targets.e2e.test.ts | 2 +- packages/testing/e2e/cdp-session.e2e.test.ts | 2 +- .../e2e/service-worker-handover.e2e.test.ts | 2 +- .../testing/src/cdp-browser-lease.test.ts | 2 +- packages/testing/src/cdp-browser.test.ts | 21 +- packages/testing/src/cdp-browser.ts | 21 +- packages/testing/src/cdp-e2e-page.test.ts | 4 +- packages/testing/src/cdp-e2e-page.ts | 4 +- .../testing/src/cdp-e2e-session-edges.test.ts | 2 +- .../src/cdp-e2e-session-fake-fixture.ts | 4 +- packages/testing/src/cdp-e2e-session.test.ts | 2 +- packages/testing/src/cdp-e2e-session.ts | 4 +- packages/testing/src/cdp-network-probe.ts | 2 +- .../testing/src/cdp-offline-script.test.ts | 2 +- packages/testing/src/cdp-offline-script.ts | 4 +- packages/testing/src/conformance.ts | 57 +- packages/testing/src/e2e-a11y-run.test.ts | 2 +- packages/testing/src/e2e-a11y.e2e.test.ts | 6 +- packages/testing/src/e2e-browser-handle.ts | 3 +- .../testing/src/e2e-browser-hooks.test.ts | 4 +- packages/testing/src/e2e-error-codes.test.ts | 2 +- packages/testing/src/e2e-error-codes.ts | 33 +- packages/testing/src/e2e-run.test.ts | 9 +- packages/testing/src/e2e-run.ts | 7 +- packages/testing/src/e2e-teardown.test.ts | 2 +- .../testing/src/embedder-conformance.test.ts | 51 + packages/testing/src/embedder-conformance.ts | 101 + .../src/identity-provider-conformance.test.ts | 141 ++ .../src/identity-provider-conformance.ts | 144 ++ packages/testing/src/index.ts | 36 +- .../src/lease-store-conformance.test.ts | 19 +- .../testing/src/lease-store-conformance.ts | 30 + ...ch-index-conformance-postgres.live.test.ts | 88 + .../src/search-index-conformance-support.ts | 64 + .../src/search-index-conformance-versions.ts | 49 + .../src/search-index-conformance.test.ts | 112 + .../testing/src/search-index-conformance.ts | 124 ++ ...semantic-cache-factory-conformance.test.ts | 35 + .../src/semantic-cache-factory-conformance.ts | 86 + .../src/sms-transport-conformance.test.ts | 53 + .../testing/src/sms-transport-conformance.ts | 102 + packages/testing/src/swap-adapter.ts | 16 +- .../src/upload-scanner-conformance.test.ts | 86 + .../testing/src/upload-scanner-conformance.ts | 108 + scripts/bench-claims.test.ts | 8 +- scripts/bench-claims.ts | 2 +- scripts/bench/boot-vs-files-child.ts | 38 + scripts/bench/boot-vs-files-shape.test.ts | 28 + scripts/bench/boot-vs-files-shape.ts | 43 + scripts/bench/boot-vs-files.bench.ts | 129 ++ scripts/bench/fanout-3-nodes.bench.ts | 176 ++ scripts/bench/fanout-node-child.ts | 78 + scripts/bench/fanout-shape.test.ts | 33 + scripts/bench/fanout-shape.ts | 60 + scripts/bench/http-prod-image.bench.ts | 168 ++ scripts/bench/results/README.md | 42 + scripts/bench/results/boot-vs-files.json | 34 + scripts/bench/results/fanout-3-nodes.json | 44 + scripts/bench/results/http-prod-image.json | 87 + scripts/bench/results/worker-throughput.json | 70 + scripts/bench/scale-claims.ts | 143 ++ scripts/bench/worker-throughput-child.ts | 67 + scripts/bench/worker-throughput-shape.test.ts | 46 + scripts/bench/worker-throughput-shape.ts | 78 + scripts/bench/worker-throughput.bench.ts | 196 ++ scripts/ci-workflow-shape.test.ts | 15 +- scripts/ci-workflow-windows.test.ts | 2 +- scripts/error-map-backlog.ts | 6 + scripts/identity-route-status.test.ts | 26 + scripts/identity-routes-csrf.test.ts | 146 ++ scripts/lib/adapter-port-pins.ts | 6 +- scripts/lib/adapter-port-references.ts | 25 + scripts/lib/bench-claims-table.ts | 3 + scripts/lib/fix-prose-pins.ts | 8 +- scripts/lib/proto-index-pins.ts | 2 +- scripts/lib/source-header-pins.ts | 1 - scripts/lib/spawn-timeout-pins.ts | 7 +- scripts/lib/test-real-sleep-pins.ts | 6 +- scripts/primitive-factories-used.test.ts | 5 +- scripts/primitive-factories.test.ts | 4 + scripts/set-cookie-literals.ts | 4 +- scripts/wiki-fences-backlog.ts | 1 - tsconfig.base.json | 1 + wiki/Adapters.md | 60 + wiki/Auth.md | 78 +- wiki/CLI-Reference.md | 30 +- wiki/Caching-And-Invalidation.md | 10 + wiki/Configuration.md | 2 +- wiki/Deployment.md | 27 + wiki/Entities-And-Migrations.md | 17 + wiki/Error-Codes.md | 43 +- wiki/Jobs-And-Workflows.md | 100 + wiki/Known-Gaps.md | 1 + wiki/Migrations-And-Backfills.md | 33 +- wiki/Notify.md | 21 + wiki/Observability.md | 17 +- wiki/Scraping.md | 41 + wiki/Storage-And-Uploads.md | 50 + wiki/Upgrading.md | 16 +- 491 files changed, 25317 insertions(+), 1452 deletions(-) create mode 100644 docs/history/cli-file-map.md create mode 100644 dummy/social-media-clone/packages/db/schema/framework/03_sequences/x_job_lease_fence.sql create mode 100644 examples/dummy/apps/web/app/billing/jobs/post-charge.test.ts create mode 100644 examples/dummy/apps/web/app/billing/jobs/post-charge.ts create mode 100644 examples/dummy/apps/web/app/posts/actions/print-post.test.ts create mode 100644 examples/dummy/apps/web/app/posts/actions/print-post.ts create mode 100644 examples/dummy/apps/web/app/posts/backfills/reindex-posts.job.test.ts create mode 100644 examples/dummy/apps/web/app/posts/backfills/reindex-posts.ts create mode 100644 examples/dummy/packages/db/migrations/20261110045219_ledger_lines_and_post_title_search.hash create mode 100644 examples/dummy/packages/db/migrations/20261110045219_ledger_lines_and_post_title_search.snapshot.json create mode 100644 examples/dummy/packages/db/migrations/20261110045219_ledger_lines_and_post_title_search.sql create mode 100644 examples/dummy/packages/db/schema/04_tables/ledger_lines.sql create mode 100644 examples/dummy/packages/db/schema/06_foreign_keys/ledger_lines.sql create mode 100644 examples/dummy/packages/db/schema/framework/03_sequences/x_job_lease_fence.sql create mode 100644 examples/dummy/packages/db/src/schema/ledger-lines.ts create mode 100644 packages/auth/src/identity-provider-cookie.ts create mode 100644 packages/auth/src/identity-provider-oidc.test.ts create mode 100644 packages/auth/src/identity-provider-oidc.ts create mode 100644 packages/auth/src/identity-provider-route-core.ts create mode 100644 packages/auth/src/identity-provider-routes.test.ts create mode 100644 packages/auth/src/identity-provider-routes.ts create mode 100644 packages/auth/src/identity-provider-status.ts create mode 100644 packages/auth/src/identity-provider.ts create mode 100644 packages/auth/src/saml-assertion.ts create mode 100644 packages/auth/src/saml-c14n.test.ts create mode 100644 packages/auth/src/saml-c14n.ts create mode 100644 packages/auth/src/saml-cert-fixture.ts create mode 100644 packages/auth/src/saml-certificate.ts create mode 100644 packages/auth/src/saml-dom.ts create mode 100644 packages/auth/src/saml-dsig.ts create mode 100644 packages/auth/src/saml-email-trust.ts create mode 100644 packages/auth/src/saml-errors.ts create mode 100644 packages/auth/src/saml-hostile.test.ts create mode 100644 packages/auth/src/saml-idp-fixture.ts create mode 100644 packages/auth/src/saml-metadata.test.ts create mode 100644 packages/auth/src/saml-metadata.ts create mode 100644 packages/auth/src/saml-provider.test.ts create mode 100644 packages/auth/src/saml-provider.ts create mode 100644 packages/auth/src/saml-request.ts create mode 100644 packages/auth/src/saml-response.ts create mode 100644 packages/auth/src/saml-test-support-fixture.ts create mode 100644 packages/auth/src/saml-vectors-fixture.ts create mode 100644 packages/auth/src/saml-xml.test.ts create mode 100644 packages/auth/src/saml-xml.ts create mode 100644 packages/cache/src/redis-pubsub.test.ts create mode 100644 packages/cache/src/redis-pubsub.ts create mode 100644 packages/cli/src/app-boundaries-workspace.test.ts create mode 100644 packages/cli/src/app-load-async.test.ts create mode 100644 packages/cli/src/app-load-async.ts create mode 100644 packages/cli/src/app-load-batch.ts create mode 100644 packages/cli/src/app-load-order.test.ts create mode 100644 packages/cli/src/app-load-prebuilt.ts create mode 100644 packages/cli/src/app-load-register.ts create mode 100644 packages/cli/src/budgets-preload.test.ts create mode 100644 packages/cli/src/build-registry.test.ts create mode 100644 packages/cli/src/build-registry.ts create mode 100644 packages/cli/src/cmd-generate-refresh.test.ts create mode 100644 packages/cli/src/cmd-verify-changed.test.ts create mode 100644 packages/cli/src/generate-refresh.test.ts create mode 100644 packages/cli/src/generate-refresh.ts create mode 100644 packages/cli/src/generate-workspace.test.ts create mode 100644 packages/cli/src/generate-workspace.ts create mode 100644 packages/cli/src/image-prepare.test.ts create mode 100644 packages/cli/src/route-islands-preload.test.ts create mode 100644 packages/cli/src/route-islands-preload.ts create mode 100644 packages/cli/src/runtime-cache-bus.test.ts create mode 100644 packages/cli/src/runtime-cache-bus.ts create mode 100644 packages/cli/src/runtime-cache-redis-peer-fixture.ts create mode 100644 packages/cli/src/runtime-cache-redis.live.test.ts create mode 100644 packages/cli/src/templates/scaffold-verify-floor.ts create mode 100644 packages/cli/src/templates/sign-in-oidc.test.ts create mode 100644 packages/cli/src/templates/workspace.test.ts create mode 100644 packages/cli/src/templates/workspace.ts create mode 100644 packages/cli/src/verify-changed-facts.test.ts create mode 100644 packages/cli/src/verify-changed-facts.ts create mode 100644 packages/cli/src/verify-changed-scope.ts create mode 100644 packages/cli/src/verify-changed.test.ts create mode 100644 packages/cli/src/verify-changed.ts create mode 100644 packages/cli/src/verify-drift-owner-step.test.ts create mode 100644 packages/cli/src/verify-drift-owner.test.ts create mode 100644 packages/cli/src/verify-drift-owner.ts create mode 100644 packages/cli/src/verify-tests-changed.test.ts create mode 100644 packages/core/src/metrics-export.test.ts create mode 100644 packages/db/src/branch-errors.ts create mode 100644 packages/db/src/generate-partition-embedded.test.ts create mode 100644 packages/db/src/generate-partition-fixture.ts create mode 100644 packages/db/src/generate-partition-lock.live.test.ts create mode 100644 packages/db/src/generate-partition-run.ts create mode 100644 packages/db/src/generate-partition-sidecar.test.ts create mode 100644 packages/db/src/generate-partition.live.test.ts create mode 100644 packages/db/src/generate-partition.test.ts create mode 100644 packages/db/src/generate-partition.ts create mode 100644 packages/entity/src/partition-describe.test.ts create mode 100644 packages/entity/src/partition-entity.test.ts create mode 100644 packages/entity/src/partition.test.ts create mode 100644 packages/entity/src/partition.ts create mode 100644 packages/entity/src/personal-data-kind.ts create mode 100644 packages/entity/src/personal-data.test.ts create mode 100644 packages/entity/src/personal-data.ts create mode 100644 packages/entity/src/search-index-installed.ts create mode 100644 packages/entity/src/search-index-memory.ts create mode 100644 packages/entity/src/search-index-postgres.ts create mode 100644 packages/entity/src/search-index-sync.test.ts create mode 100644 packages/entity/src/search-index-sync.ts create mode 100644 packages/entity/src/search-index-version.ts create mode 100644 packages/entity/src/search-index.test.ts create mode 100644 packages/entity/src/search-index.ts create mode 100644 packages/entity/src/search-repo.test.ts create mode 100644 packages/entity/src/search-repo.ts create mode 100644 packages/jobs/src/gdpr-audit.ts create mode 100644 packages/jobs/src/gdpr-coverage.ts create mode 100644 packages/jobs/src/gdpr-errors.ts create mode 100644 packages/jobs/src/gdpr.job.test.ts create mode 100644 packages/jobs/src/ledger-book.ts create mode 100644 packages/jobs/src/ledger-errors.ts create mode 100644 packages/jobs/src/ledger-tenant.job.test.ts create mode 100644 packages/jobs/src/ledger.live.test.ts create mode 100644 packages/jobs/src/ledger.test.ts create mode 100644 packages/jobs/src/ledger.ts create mode 100644 packages/jobs/src/lock-errors.ts create mode 100644 packages/jobs/src/lock.live.test.ts create mode 100644 packages/jobs/src/lock.test.ts create mode 100644 packages/jobs/src/lock.ts create mode 100644 packages/jobs/src/partitions.live.test.ts create mode 100644 packages/jobs/src/partitions.test.ts create mode 100644 packages/jobs/src/partitions.ts create mode 100644 packages/jobs/src/reindex.test.ts create mode 100644 packages/jobs/src/reindex.ts create mode 100644 packages/jobs/src/scheduler-failover.live.test.ts create mode 100644 packages/jobs/src/worker-backlog.test.ts create mode 100644 packages/notify/src/sms-channel.test.ts create mode 100644 packages/notify/src/sms-channel.ts create mode 100644 packages/notify/src/sms-errors.ts create mode 100644 packages/notify/src/sms-transport.test.ts create mode 100644 packages/notify/src/sms-transport.ts create mode 100644 packages/realtime/src/replicator-reconnect.test.ts create mode 100644 packages/realtime/src/replicator-reconnect.ts rename packages/{testing => scraping}/src/cdp-connection.test.ts (100%) rename packages/{testing => scraping}/src/cdp-connection.ts (93%) create mode 100644 packages/scraping/src/cdp-error-codes.ts rename packages/{testing => scraping}/src/cdp-errors.ts (90%) rename packages/{testing => scraping}/src/cdp-launch-attempt.ts (94%) rename packages/{testing => scraping}/src/cdp-launch-candidates.ts (100%) rename packages/{testing => scraping}/src/cdp-launch-probe.test.ts (100%) rename packages/{testing => scraping}/src/cdp-launch-probe.ts (100%) rename packages/{testing => scraping}/src/cdp-launch-profile.test.ts (100%) rename packages/{testing => scraping}/src/cdp-launch-profile.ts (100%) rename packages/{testing => scraping}/src/cdp-launch-reap.test.ts (100%) rename packages/{testing => scraping}/src/cdp-launch-reap.ts (100%) rename packages/{testing => scraping}/src/cdp-launch-relaunch.test.ts (100%) rename packages/{testing => scraping}/src/cdp-launch-wire.test.ts (100%) rename packages/{testing => scraping}/src/cdp-launch-wire.ts (100%) rename packages/{testing => scraping}/src/cdp-launch.test.ts (92%) rename packages/{testing => scraping}/src/cdp-launch.ts (84%) rename packages/{testing => scraping}/src/cdp-pipe.test.ts (100%) rename packages/{testing => scraping}/src/cdp-pipe.ts (100%) rename packages/{testing => scraping}/src/cdp-wire-watch.test.ts (98%) rename packages/{testing => scraping}/src/cdp-wire-watch.ts (100%) create mode 100644 packages/scraping/src/cdp.ts create mode 100644 packages/scraping/src/pdf-action.test.ts create mode 100644 packages/scraping/src/pdf-action.ts create mode 100644 packages/scraping/src/pdf-cdp-params.ts create mode 100644 packages/scraping/src/pdf-fence.ts create mode 100644 packages/scraping/src/pdf-render-fence.test.ts create mode 100644 packages/scraping/src/pdf-render-fixture.ts create mode 100644 packages/scraping/src/pdf-render-slots.test.ts create mode 100644 packages/scraping/src/pdf-render.test.ts create mode 100644 packages/scraping/src/pdf-render.ts create mode 100644 packages/scraping/src/pdf-slots.ts create mode 100644 packages/storage/src/attachment-scan-race.test.ts create mode 100644 packages/storage/src/attachment-scan-read.test.ts create mode 100644 packages/storage/src/attachment-scan.test.ts create mode 100644 packages/storage/src/errors-scan.ts create mode 100644 packages/storage/src/errors-variant.ts create mode 100644 packages/storage/src/image-variant.test.ts create mode 100644 packages/storage/src/image-variant.ts create mode 100644 packages/storage/src/upload-scan.ts create mode 100644 packages/storage/src/upload-scanner-memory.ts create mode 100644 packages/storage/src/upload-scanner.test.ts create mode 100644 packages/storage/src/upload-scanner.ts create mode 100644 packages/testing/src/embedder-conformance.test.ts create mode 100644 packages/testing/src/embedder-conformance.ts create mode 100644 packages/testing/src/identity-provider-conformance.test.ts create mode 100644 packages/testing/src/identity-provider-conformance.ts create mode 100644 packages/testing/src/search-index-conformance-postgres.live.test.ts create mode 100644 packages/testing/src/search-index-conformance-support.ts create mode 100644 packages/testing/src/search-index-conformance-versions.ts create mode 100644 packages/testing/src/search-index-conformance.test.ts create mode 100644 packages/testing/src/search-index-conformance.ts create mode 100644 packages/testing/src/semantic-cache-factory-conformance.test.ts create mode 100644 packages/testing/src/semantic-cache-factory-conformance.ts create mode 100644 packages/testing/src/sms-transport-conformance.test.ts create mode 100644 packages/testing/src/sms-transport-conformance.ts create mode 100644 packages/testing/src/upload-scanner-conformance.test.ts create mode 100644 packages/testing/src/upload-scanner-conformance.ts create mode 100644 scripts/bench/boot-vs-files-child.ts create mode 100644 scripts/bench/boot-vs-files-shape.test.ts create mode 100644 scripts/bench/boot-vs-files-shape.ts create mode 100644 scripts/bench/boot-vs-files.bench.ts create mode 100644 scripts/bench/fanout-3-nodes.bench.ts create mode 100644 scripts/bench/fanout-node-child.ts create mode 100644 scripts/bench/fanout-shape.test.ts create mode 100644 scripts/bench/fanout-shape.ts create mode 100644 scripts/bench/http-prod-image.bench.ts create mode 100644 scripts/bench/results/boot-vs-files.json create mode 100644 scripts/bench/results/fanout-3-nodes.json create mode 100644 scripts/bench/results/http-prod-image.json create mode 100644 scripts/bench/results/worker-throughput.json create mode 100644 scripts/bench/scale-claims.ts create mode 100644 scripts/bench/worker-throughput-child.ts create mode 100644 scripts/bench/worker-throughput-shape.test.ts create mode 100644 scripts/bench/worker-throughput-shape.ts create mode 100644 scripts/bench/worker-throughput.bench.ts create mode 100644 scripts/identity-route-status.test.ts create mode 100644 scripts/identity-routes-csrf.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5bacee45..145decd5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -581,6 +581,10 @@ jobs: flags: --shape blog - app: dashboardapp flags: --shape dashboard + # The monorepo shape (plan 101 slice 22, M1): web + admin + a third app, domain + ui + a + # third package — each one command, gated by the same two passes below. + - app: multiapp + flags: '' env: APP: ${{ matrix.app }} steps: @@ -630,6 +634,15 @@ jobs: # to every exit code in this job. The defect that bought it broke 4 generators of the 13, # found by a human scaffolding an app — so the list is projected from the CLI's own # `GENERATORS` and a fourteenth generator is covered the day it lands. + # A workspace is added before the sweep, so every generator after it writes into a repo that + # already holds three apps and three packages, and both gates run over all of them. + - name: x g app reports && x g package billing + if: matrix.app == 'multiapp' + working-directory: ${{ runner.temp }}/${{ matrix.app }} + run: | + bunx x g app reports --json + bunx x g package billing --json + bun install - name: every generator run: bun run scripts/scaffold-first-run.ts "$RUNNER_TEMP/$APP" # The same two scripts again, and the second run is not a repeat: `bun run setup` applies the @@ -1286,7 +1299,7 @@ jobs: # W3: a CRLF migration must hash to the checksum its LF twin wrote (`./packages/db/...`). - name: unit · the path-heavy subset, every core if: ${{ !cancelled() && steps.setup.outcome == 'success' }} - run: bun test --isolate --parallel --timeout=30000 ./packages/core ./packages/db/src/migrate.test.ts ./packages/db/src/migration-ledger.test.ts ./packages/policy ./packages/render ./scripts/lib ./packages/cli/src/app-load.test.ts ./packages/cli/src/drift.test.ts ./packages/cli/src/app-boundaries.test.ts ./packages/cli/src/path-segments.test.ts ./packages/cli/src/templates/scaffold-gitattributes.test.ts ./packages/testing/src/cdp-launch.test.ts ./packages/testing/src/cdp-launch-wire.test.ts ./packages/testing/src/cdp-launch-reap.test.ts ./packages/testing/src/cdp-launch-probe.test.ts + run: bun test --isolate --parallel --timeout=30000 ./packages/core ./packages/db/src/migrate.test.ts ./packages/db/src/migration-ledger.test.ts ./packages/policy ./packages/render ./scripts/lib ./packages/cli/src/app-load.test.ts ./packages/cli/src/drift.test.ts ./packages/cli/src/app-boundaries.test.ts ./packages/cli/src/path-segments.test.ts ./packages/cli/src/templates/scaffold-gitattributes.test.ts ./packages/scraping/src/cdp-launch.test.ts ./packages/scraping/src/cdp-launch-wire.test.ts ./packages/scraping/src/cdp-launch-reap.test.ts ./packages/scraping/src/cdp-launch-probe.test.ts - name: x new winapp id: new if: ${{ !cancelled() && steps.setup.outcome == 'success' }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 5552d39b7..3c425fcda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,19 @@ Semver applies from 1.0.0. A breaking change to a documented API needs a major ### Changed +- `@ultimat3/jobs`: a worker drains a backlog at the speed of its jobs, not one batch per poll interval. A pass that found work used to wait the whole floor (250 ms) and a freed slot could not cut it short, so the defaults capped one worker at 8 jobs × 4 passes = 31 claims/s; a pass whose every claimed row started now marks a backlog and the next freed slot claims at once — 799 / 1,618 / 2,277 claims/s at 1 / 4 / 16 workers on the same bench (`scripts/bench/results/worker-throughput.json`). The poll interval is the idle backoff only. +- Scale ladder, measured (plan 101 slice 22): `scripts/bench/{worker-throughput,fanout-3-nodes,boot-vs-files,http-prod-image}.bench.ts` with committed results, held to their prose by `bun run scripts/bench-claims.ts` (`X_BENCH_CLAIM_STALE`); `docs/idea/17-scale-ladder.md` cites one measurement per rung and drops two stale rows (read replicas are wired per request; `redisJobDriver` ships). `@ultimat3/jobs`: `scheduler-failover.live.test.ts` kills the scheduler leader mid-fire — no double fire, no skip. +- `@ultimat3/render`, `@ultimat3/cli`: an app workspace may be named `api`, `site`, `app` or `shared` — the surface is the directory directly under `apps//` (`locateSurface`), where it took the first such segment and `apps/api/site/x/page.tsx` was `X_ROUTE_FILE_INVALID`. The `boundaries` step resolves `@//…` onto `apps//…`, so a `site/` page importing a sibling app's `app/` is `X_BOUNDARY_SITE_TO_APP`. An `app/` module importing a SIBLING app's `api/` registry by value (`apps/admin` projecting `apps/web`'s actions) is allowed: `app/ → api/` stays types-only inside one app, where it is a registration cycle. New `manifest` finding `X_PARTITIONS_UNMAINTAINED`: an entity declaring `partitionBy` with no `partitions()` job registered. +- `@ultimat3/auth`: SAML 2.0 SSO, no dependency, beside OIDC behind one `IdentityProvider` port (`begin` / `complete` → one `OAuthProfile`; `signInWithIdentity` is the one way to a session). `samlIdentityProvider({ id, idp: parseSamlMetadata(xml), spEntityId, trustEmail })`: SP-initiated (AuthnRequest over HTTP-Redirect, RelayState), ACS over HTTP-POST, XML-DSig verified in-package (strict XML reader refusing DOCTYPE/entities/PIs, exclusive c14n proven against xmlsec-signed vectors, RSA-SHA256 ≥ 2048, keys from metadata only), the assertion itself signed and the only one in the document, Recipient/Destination/InResponseTo/Audience/NotBefore/NotOnOrAfter checked, each assertion claimed once in `auth.totpReplay`. `oidcIdentityProvider({ provider })` puts the existing OAuth/OIDC handshake behind the same port. `identityProviderRoutes(auth, { providers })` serves `GET /auth/sso/:provider`, `GET|POST /auth/sso/:provider/callback`. New codes `X_SAML_RESPONSE_INVALID` (400, `meta.check`), `X_SAML_METADATA_INVALID`, `X_IDENTITY_PROVIDER_UNKNOWN` (404). `identityProviderConformance` in `@ultimat3/testing` holds an adapter of your own to the contract. +- `@ultimat3/cli`: `x g app ` / `x g package ` add a workspace (`apps//`, `packages//`) named `@/` off the root manifest, and print `bun install` then `bunx x verify --changed --dirty`; a name the other kind holds is `X_WORKSPACE_NAME_TAKEN`. `--app ` (default `web`) lands `x g route|island|query` in another app workspace; a generator that registers in `apps/web` refuses it (`X_CLI_BAD_FLAG`) +- `@ultimat3/cli`: the app scan imports its modules in batches of 256 — one generated entry module per batch, statically importing it in scan order, so every module evaluates in the order the serial loop evaluated it (`app-load-order.test.ts` snapshots both), then registers in that order; a batch with a module that fails falls back to one import at a time for its own finding. `x build --target prebuilt` can write a module registry (`node_modules/.cache/ultimate/registry/`) that `web`, `sync` and `replicator` import at boot instead of scanning; a registry whose module list is not the disk's is logged as `X_PREBUILT_REGISTRY_STALE` and the boot scans. `x g` loads the app to refresh `x.manifest.json` only when a written file is an app module, declares an `X_*` code or sits at the app root — `x g island` no longer loads the app. 5,000-module fixture (2026-11-09, loaded host): untracked scan 2.2 s → 1.6 s, prebuilt boot 1.1 s, tool load (`loadApp`) 4.8 s → 2.2–3.1 s; `x g island` on the reference app 1.16 s → 0.59 s +- `@ultimat3/scraping`: `pdf({ input, policy, route, key, storage, layout?, headers?, origin? })` — a factory over `action` (a `PRIMITIVE_FACTORIES` row) that renders one of the app's own routes to PDF through Chrome over raw CDP (`Page.printToPDF`), writes it to a disk and answers `{ key, bytes, contentType }`; schema, policy, rate limit, meter and audit are the action's. A route leaving the origin is `X_SCRAPE_PDF_ROUTE_INVALID`, no `APP_URL`/`origin` `X_SCRAPE_PDF_ORIGIN_MISSING`, a 4xx/5xx page `X_SCRAPE_PDF_ROUTE_FAILED`. The raw-CDP launcher and connection moved from `@ultimat3/testing` to a new entry point, `@ultimat3/scraping/cdp`, which now registers the four `X_CDP_*` codes; `@ultimat3/testing` re-exports every name unchanged. +- `@ultimat3/notify`: a new adapter port, `smsTransport` (`SmsTransport` on `@ultimat3/notify/adapter`, `smsTransport:` in `runtime.ts`, `smsTransportConformance` in `@ultimat3/testing/conformance`), and the reference `smsChannel({ message })`: one SMS per recipient, the body a catalog key rendered in the recipient's locale, one idempotency key per delivery. `memorySmsTransport()` is the default in development/test; elsewhere a send with none installed is `X_NOTIFY_SMS_TRANSPORT_MISSING`, and a non-E.164 number `X_NOTIFY_SMS_NUMBER_INVALID` (both terminal). Providers (Twilio, Vonage, SNS) are the app's adapters. +- `@ultimat3/cli`: `x verify --changed [--base ] [--dirty]` runs only the steps whose inputs the diff touches — each changed file gets one rule (`doc`, `test`, `source`, `drift-input`, `generated`, `root-config`, …), each rule selects steps, `data.changed` names the files behind every step; NOT A GATE RUN, refused beside `--only`, fail-open on any path no rule speaks for. `drift` runs only when an entity, a module an entity imports, `packages/db/**`, a generated client or `openapi.json` moved. On a 3-app + 2-package scaffold (full gate 52 s): a docs-only diff 8.9 s, a test-only diff 9.3 s, a `packages/ui` edit 39.8 s. `x new`'s root tsconfig reaches every workspace through one alias (`@/*` → `packages/*/src`, then `apps/*`), so a new `apps/` typechecks with no tsconfig edit. New code `X_APP_ROOT_NESTED`: an `app.config.ts` inside a workspace is a second app root — one repository is one app, one image, one migration owner. +- `@ultimat3/jobs`: four mechanisms over existing primitives. `ledger({ name, tenant, book })` — a job posting double-entry Money, every currency summing to zero on its own, refused before any write (`X_LEDGER_UNBALANCED`, `X_LEDGER_ENTRY_INVALID`, terminal), written in one step and idempotent by `(tenant, postingKey, line)` through `entityLedgerBook` over the app's own entity; a key re-used with other entries is `X_LEDGER_POSTING_CONFLICT`, nothing written. `withLock(key, fn, { ttl, wait? })` — one holder fleet-wide over the driver's `LeaseStore`, a fencing `token` (the lease store's number, `x_job_leases.fence` from one sequence — never a client clock) and an abort `signal` handed to `fn`, a hung renewal a loss once the ttl lapses (`X_LOCK_NOT_ACQUIRED`, `X_LOCK_LOST`, `X_LOCK_STORE_MISSING`). `reindex({ name, entity, source, tenant })` — a `backfill()` rebuilding the `searchIndex` port. `personalDataProblems()` — proof the `defineSubject()` declarations erase every `personalData()` column (`X_PERSONAL_DATA_UNERASED`, `X_PERSONAL_DATA_UNREACHED`); `subjectExport`/`subjectErase` stay the one export/erase path. A subject run refuses an incomplete declaration before touching a row, `x exercise` lists both codes, and `subjectErase` writes its erase to the installed `auditSink` (`allowed` once per completed erase, `failed` per attempt that threw; `input: { subject, run }`). `partitions()` — a job plus its `.schedule` task (03:17 UTC daily) running `@ultimat3/db`'s `partitionMaintenance` for every `entity({ partitionBy })` at the job's clock: partitions created `ahead`, expired ones dropped. +- `@ultimat3/cache`, `@ultimat3/cli`: cache and ISR invalidations reach every replica on rung 1 (Redis, no NATS). With the in-process bus and `REDIS_URL` (else `VALKEY_URL`) set, the hop rides Redis pub/sub (`redisPubSub`, `redisPubSubAt`); NATS or a supplied `transport` stays the path when configured. One rule, one boot line: `cache.broadcast bus= source=`. A dropped subscriber connection is resubscribed and the process's own tiers flushed, because pub/sub keeps nothing for the window it was away. Before, a write on one web replica left the others serving stale pages and `cache:` reads until TTL. +- `@ultimat3/entity`: `personalData(kind, column)` marks a column as personal data (`identifier` / `contact` / `free-text`) without moving its DDL; `personalDataOf(entity)` and `personalDataEntities()` list every marked column per registered entity, for a GDPR export or erase job. A new adapter port, `searchIndex` (`SearchIndex`: `index` / `remove` / `query`, tenant-scoped): `postgresSearchIndex()` — `.searchable()`'s generated `tsvector`, the default — and `memorySearchIndex()`, the reference; Meilisearch or Typesense is the app's, proved by `searchIndexConformance` (`@ultimat3/testing/conformance`) and returned as `searchIndex` from `runtime.ts`. +- `@ultimat3/storage`: a new adapter port, `uploadScanner` (`UploadScanner`: `scan({ key, size, contentType, body })` → clean / infected), returned from `runtime.ts` and asked by `promoteAttachment` about a per-attempt copy (`pending/scan/`), which is what gets attached when clean — a refused confirm never touches the row's key. Infected: the upload and its copy deleted, `X_STORAGE_UPLOAD_INFECTED` (422). No verdict: fails closed in every environment, `X_STORAGE_SCAN_UNAVAILABLE` (503), the upload left pending for a retry — `uploadPolicy({ onScanUnavailable: 'accept' })` promotes unscanned instead. `memoryUploadScanner()` flags the EICAR test string (`EICAR_TEST_STRING`); `uploadScannerConformance` (`@ultimat3/testing/conformance`) proves an app's ClamAV adapter. No scanner installed, nothing changes. And `variant(key, { disk, images, width, format? })`: one resized image made by the `images` driver passed in (any `ImageTransformDriver`) and kept under `variantKey(…, { revision })` — a second call is a `stat`; a non-raster source is `X_IMAGE_UNSUPPORTED`. - `@ultimat3/testing`: conformance suites for the realtime, AI, scraping, secret and clock ports — `transportConformance` (in-process, NATS on the in-memory server and a real nats-server), `gatewayConformance` (budget scope and ledger included), `providerConformance` plus `providerTransportConformance` for an HTTP provider (abort reaches `fetch`, an echoed key never reaches an error, an endless error body ends, a context overflow is `X_AI_CONTEXT_OVERFLOW`), `vectorStoreConformance` (memory, pgvector), `scrapeDriverConformance` (fake, fixture), `secretSourceConformance` (env, file, default, an app's KMS source) and `sleepClockConformance` (system, frozen, no-wait — the abort reason verbatim, no listener left behind). `@ultimat3/realtime`'s `transport-parity.test.ts` is deleted in their favour, and the socket cases leave `@ultimat3/ai`'s `provider-parity*.test.ts`. - `@ultimat3/testing`: a new entry point, `@ultimat3/testing/conformance`, holds every port's suite (`behavesLike(Conformance, () => yours)`; the README's table lists them). `entityDriverConformance` holds an entity `Driver` to what `db.` promises — writes, keys, values, reads, upserts, soft delete, append-only, seeds, aggregates, tenancy, preload, locks, every column kind — over memory, Postgres on PGlite and a real server, and a `defineDriver` store; a capability the subject does not declare is asserted refused. Ten of `@ultimat3/entity`'s `*-parity.test.ts` are deleted in its favour; seven keep only what no driver promises. - `@ultimat3/testing`: five conformance suites for credential and limit stores — `rateLimitStoreConformance` (memory, Postgres, Redis), `authLimiterFactoryConformance` and `totpReplayGuardConformance` (memory, Postgres), `authServerStoreConformance` (memory, Postgres) and `authAdapterConformance` (every `AuthAdapter` sub-store; memory, Postgres on PGlite and a real server). `behavesLike(suite, () => yours)` proves a hand-written adapter; `@ultimat3/auth`'s `adapter-parity*.test.ts` and `authorization-server-store.contract.test.ts` are deleted in their favour. @@ -90,8 +103,14 @@ Semver applies from 1.0.0. A breaking change to a documented API needs a major - **BREAKING — (#47) `@ultimat3/entity`: memory answers what Postgres answers, found by `entityDriverConformance`.** A malformed uuid in a repository write is `X_INVARIANT_VIOLATED` before the statement in both drivers (Postgres answered `X_DB_STATEMENT_FAILED`; memory stored it). Also fixed, not breaking: an unknown operator and a whole-money comparison are refused at plan time on an empty table too; an `integer()` operand written as text finds its row in memory; `defineDriver`'s `update(id)` answers the row when the patch moves the key or soft-deletes it. [Upgrading](https://github.com/developerz-ai/ultimate/wiki/Upgrading) #47. - **BREAKING — (#48) `@ultimat3/pwa`: `PushSubscriptionRecord.actorId` is `string`, never `null`.** `x_push_subscriptions.actor_id` was always `not null` and `pushSubscribe()` always stores `ctx.actor.id`; the memory store accepted what Postgres refused. Both now refuse a missing or empty actor with `X_PWA_PUSH_SUBSCRIPTION_INVALID` before writing, held by `pushSubscriptionStoreConformance`. Port suites added on `@ultimat3/testing/conformance`: `mailDriverConformance` (memory, log, smtp, ses, resend), `notifyStoresConformance` and one per store, `pushDeviceStoreConformance`, `pushSubscriptionStoreConformance`; the mail and notify parity tests they replace are deleted. [Upgrading](https://github.com/developerz-ai/ultimate/wiki/Upgrading) #48. - **BREAKING — (#49) `@ultimat3/notify`: no memory delivery ledger outside development.** With no `notifyStores.ledger` installed, staging and production refuse the first claim with `X_NOTIFY_STORE_MISSING` (fix: `postgresDeliveryLedger({ executor: dbExecutor() })` in `runtime.ts`); development and test keep the memory default. A per-process ledger behind replicas or across a restart let a replayed attempt send twice. [Upgrading](https://github.com/developerz-ai/ultimate/wiki/Upgrading) #49. +- **BREAKING — (#50) `@ultimat3/ai`: the embedder and the semantic-cache factory are adapter ports, `embedder` and `semanticCache`.** Set them in `runtime.ts`; the boot installs them with `installAiEmbedder` / `installSemanticCache` (`@ultimat3/ai/adapter`), each install answering its release and clearing the per-scope caches. `configureAi` keeps `redact` only. New suites: `embedderConformance`, `semanticCacheFactoryConformance` (`@ultimat3/testing/conformance`). Upgrading entry 50. +- **BREAKING — (#51) `@ultimat3/core`: a metric exporter is a `metric.export` subscriber on the instrumentation bus.** `exportMetrics()` publishes the collection (and collects nothing while nobody subscribes); `configureMetrics` keeps `clock` and `enabled`. The OTLP boot subscribes `otlpMetricExporter()` and its release is the unsubscribe, as for spans and logs. Upgrading entry 51. +- **BREAKING — (#52) `@ultimat3/auth`: `oauthLogin` is a front door over `identityProviderRoutes`, at `/auth/sso/:provider`.** One sign-in route implementation for OAuth and SAML: the paths move from `/auth/oauth/…` (`oauthStartPath`, `oauthCallbackPath`, `OAUTH_*_ROUTE_PATH` and `restartAt` answer the new ones), the state cookie is `__Host-x_sso_` and each callback's state is claimed once in `auth.totpReplay`. `oauthLogin(auth, options)` keeps its signature and refusals. Re-register the callback URL at each IdP before deploying. Upgrading entry 52. +- **BREAKING — (#53) `@ultimat3/testing`: the raw-CDP launcher is imported from `@ultimat3/scraping/cdp`, never `@ultimat3/testing`.** `findChrome`, `launchChrome`, `launchFoundChrome`, `cdpConnect`, `cdpConnectOver`, `CHROME_CANDIDATES`, `CHROME_PATH_ENV`, `CONTAINER_CHROME_ARGS`, `LAUNCH_ATTEMPTS`, `LAUNCH_BUDGET_MS`, `LAUNCH_TIMEOUT_MS`, the four `Cdp*Error` classes and the `CdpConnection`/`LaunchedBrowser`/`LaunchOptions` types left testing's barrel (one value, one import path: `X_HELPER_COPY`), and `E2E_BROWSER_CLOSE_MS` is `LAUNCHED_CLOSE_MS` from the same entry. An app's e2e suite adds `@ultimat3/scraping` to its devDependencies and moves those names into `import { … } from '@ultimat3/scraping/cdp'`. +- `@ultimat3/realtime`: the replicator reconnects within milliseconds of a database restart, not ~73 s. A lost connection — SQLSTATE 57P01/57P02/57P03, `ECONNRESET`/`ECONNREFUSED`/`EPIPE`, a dial that found no host, a lost advisory-lock session — redials at once, then on a curve capped at `REPLICATOR_RECONNECT_MAX_MS` (2 s, `equal` jitter); a refused change, a standby and a stream that ended politely keep the 30 s curve. The slot-release wait before `START_REPLICATION` is unchanged. - Adapter ports, plan 101 slice 21 part B: `scripts/adapter-ports.ts` on the `boundaries` step raises `X_ADAPTER_PORT_INCOMPLETE` for an `ADAPTER_PORTS` row missing its `RuntimeOverrides` field, its boot binding, its interface (and installer) on the package's `./adapter`, its `@ultimat3/testing/conformance` suite or its `wiki/Adapters.md` section; for a `set*`/`configure*`/`install*` on a `.` barrel; and for a backend-shaped type on a `./adapter` no row lists. Parts still being built are a ratchet in `scripts/lib/adapter-port-pins.ts`, each with its reason; a pin that excuses nothing is the same finding. `bun run scripts/adapter-ports.ts --json` lists every port's five parts. New `wiki/Adapters.md`, generated from the table (`--write`): per port the interface, the reference implementation to copy, the conformance one-liner, the default and the `runtime.ts` field; `wiki/Upgrading.md` gains a removed-setter → `runtime.ts` table for 30 → 31. - Instrumentation, plan 101 slice 20 (edge): `render.page` (keyed on the route pattern, `http.route` on its span), `render.hole` and `render.isr.regenerate` (once per generation) from `@ultimat3/render`; `mail.deliver` from `@ultimat3/mail` — inline and queued sends through one `deliverMail`, a recipient count and no address; `outbound.request` and an `outbound.` client span per `outbound()` call from `@ultimat3/core/outbound`, never the path or query. `X_INSTRUMENT_SLOT_FORBIDDEN` runs on the `boundaries` step. `packages/cli/src/otlp-export.live.test.ts` proves a request → action → Postgres reaches an OTLP collector as a server span over a `db.select` client span; `scripts/bench/span-overhead.bench.ts` measures an unsampled span (3.3 µs, target 1 µs — known gap). +- `@ultimat3/entity` + `@ultimat3/db`: `entity({ partitionBy: { column, interval, ahead?, retain? } })` — range partitions by a `timestamp()`/`date()` in the primary key, one per `day`/`week`/`month`/`year`, UTC bounds. Postgres' rules are refused at declaration (`X_ENTITY_PARTITION_INVALID`); `x db gen` writes `partition by range` and the function `ultimate_partitions`, which creates the current partition plus `ahead` at apply time; `partitionMaintenance(description, now)` creates ahead and detaches-then-drops whole partitions `retain.after` expired. Gaining it on a plain table or losing it on a partitioned one is `X_MIGRATION_PARTITION_CHANGE`. Not breaking: an entity without it describes and generates byte-identically. ### Fixed diff --git a/bun.lock b/bun.lock index 73627857d..3ddf9a286 100644 --- a/bun.lock +++ b/bun.lock @@ -44,6 +44,7 @@ "@biomejs/biome": "2.6.3", "@electric-sql/pglite": "^0.5.11", "@types/bun": "^1.4.0", + "@ultimat3/scraping": "30.9.0", "@ultimat3/testing": "30.9.0", "typescript": "^7.0.2", }, @@ -124,6 +125,7 @@ "@biomejs/biome": "2.6.3", "@electric-sql/pglite": "^0.5.11", "@types/bun": "1.4.4", + "@ultimat3/scraping": "30.9.0", "@ultimat3/testing": "30.9.0", "typescript": "^7.0.2", }, @@ -604,6 +606,7 @@ "name": "@ultimat3/scraping", "version": "30.9.0", "dependencies": { + "@ultimat3/action": "30.9.0", "@ultimat3/core": "30.9.0", "@ultimat3/jobs": "30.9.0", "@ultimat3/schema": "30.9.0", diff --git a/docs/history/cli-file-map.md b/docs/history/cli-file-map.md new file mode 100644 index 000000000..df637b8c9 --- /dev/null +++ b/docs/history/cli-file-map.md @@ -0,0 +1,87 @@ +# cli: file-map detail moved out of `packages/cli/CLAUDE.md` + +`As of 2026-11-09`. The rows below are the full text of `packages/cli/CLAUDE.md` file-map cells cut +to keep that file under its 24,576 B ceiling (`bun run scripts/claude-md-size.ts`). Where the two +disagree, `packages/cli/CLAUDE.md` wins. + +## `cmd-mcp.ts` / `mcp-host.ts` / `mcp-errors.ts` / `mcp-db-target.ts` + +- `x mcp serve`: 19 tools, two transports. +- A taken port is `X_PORT_IN_USE`. +- Over HTTP a non-loopback `Host` is 421 `X_MCP_HOST_REFUSED` first; `GET`/`DELETE` are http's 405. +- `mcp-test-run.ts` folds `bun test`'s `N errors` line and exit code into `tests.run`. + +## `cmd-deploy-verify.ts` / `deploy-verify-*.ts` + +- `x deploy verify`: the wait (samples, window, queues), the pure verdict, flags refused before a + sample, the `--json` shape, the real sources — lazy, so the build verdict loads neither. +- `DATABASE_URL` from the env only, never `resolveServices` (full rule: `docs/history/cli.md`). + +## `runtime-cache-bus.ts` (v31, plan 101 slice 22) + +Which bus carries cache and ISR invalidations between replicas — one rule, one boot line +(`cache.broadcast {bus, source}`, never the URL): + +| Deployment | Bus | +|---|---| +| `NATS_URL` set, or `runtime.ts` supplies a `transport` | that transport | +| in-process bus, `REDIS_URL` (else `VALKEY_URL`) set — rung 1 | Redis pub/sub (`@ultimat3/cache`'s `redisPubSubAt`) | +| neither | in-process: this replica only | + +- Why: rung 1 (roles split, Redis, no NATS) with 2+ web replicas published busts on the in-process + bus, so a write on one replica left the others serving stale pages and `cache:` reads until TTL. +- A transport that crosses processes always wins: a second hop beside NATS would deliver each bust + twice through two systems that fail independently. +- Loss: a dropped subscriber is resubscribed and `onReconnect` flushes the process's own tiers — + pub/sub keeps nothing for the window it was away. `runtime-cache-redis.live.test.ts` proves both + halves with two processes on one Redis. +- `runtime-bus.ts`'s production in-process warning drops "cache invalidations" from what stays + local when Redis carries them, and points at the `cache.broadcast` line. + +## `coverage-floor.ts` / `coverage-lcov.ts` / `coverage-source.ts` / `verify-coverage-run.ts` + +- an APP's `unit` step holds `coverage` in `x.verify.json` over its whole source tree, in fixed slices, never `--parallel --coverage`; `coverage-lcov.ts` is the ONE lcov reader. Full rule: `docs/history/cli.md` + +## `verify-deadline.ts` / `verify-stalled.ts` / `verify-progress.ts` + +- a step past its deadline is `X_VERIFY_STEP_TIMEOUT`: its children carry `ULTIMATE_VERIFY_STEP` and are killed by that tag; the finding names the test file `bun test` had not finished. Full rule: `docs/history/cli.md` + +## `runtime-render.ts` / `runtime-render-document.ts` / `runtime-isr-outcome.ts` / `route-islands.ts` / `runtime-assets.ts` / `runtime-storage.ts` / `runtime-hooks.ts` / `api-routes.ts` + +- the HTTP surface: pages, `/icons` + `/media`, `/_storage`, authz, the app's API, bearer mounts and pages' bound `POST`s, mounted by BOTH boots. `isr` and `static` render as nobody. `route-islands-preload.ts`: modulepreload for each rendered eager island's imports. Full rule: `docs/history/cli.md` + +## `retained-assets.ts` / `-layout.ts` / `-sweep.ts` / `-read.ts` / `-boot.ts` + +- `assets.retain`: the web boot publishes this build's hashed files before `/readyz`; `withRetainedAssets` answers an assets 404 a retained manifest names. `x dev` never wraps. Full rule: `docs/history/cli.md` + +## `island-bundle.ts` / `island-store.ts` / `island-sources.ts` / `island-realtime.ts` / `island-realtime-state.ts` / `island-runtime.ts` / `solid-loader.ts` + +- islands: one `Bun.build` each, source-addressed; the image build writes a verified store, and an edit since the build is a rebuild, never the old chunk + +## `db-schema-dump.ts` / `db-schema-refresh.ts` / `db-replay-engine.ts` / `migration-extensions.ts` / `db-object-drift.ts` + +- the schema dump: replay on a scratch database (full rule: history), written by `x db gen` and `x db migrate`. `migration-extensions.ts` is also what `runtime-queue.ts` links `x dev`'s database with + +## `verify-role-load.ts` / `verify-role-load-probe.ts` + +- `manifest`: a child imports the app as a worker does, then the rest module by module; a registration only a document-reaching module makes is `X_ROLE_LOAD_INCOMPLETE` with the module and the import the API index lacks + +## `db-rolling.ts` / `doctor-rolling.ts` + +- `drift`'s fifth rail: what the previous release trips over (`X_MIGRATION_ROLLING_UNSAFE`, `@ultimat3/db`'s classifier), for migrations after `rollingSince` in `x.verify.json` (`x new` writes `"0"`). Full rule: `docs/history/cli.md` + +## `framework-schema-apply.ts` / `framework-schema-stamp.ts` + +- the framework tables: `ROLE=migrate`, `x dev` and every CLI command APPLY them in one transaction behind `MIGRATION_LOCK_KEY` (`pg_advisory_xact_lock`) with a bounded `lock_timeout`. Full rule: `docs/history/cli.md` + +## `app-config-load.ts` + +- the ONE reader of an app's `app.config.ts`: `loadAppConfig(root)` — the `config` export through core's `defineConfig`, so validated and default-merged; `undefined` with no file; no `config` object is `X_CONFIG_INVALID`. Full rule: `docs/history/cli.md` + +## `serve-prebuilt.ts` / `serve-prebuilt-paths.ts` + +- the image's prebuilt store (`node_modules/.cache/ultimate`, never `.x/` — state, tmpfs): every role adopts its compiled stylesheets before importing the app; a boot that built an island or ran Sass logs `X_IMAGE_NOT_PREBUILT` once + +## `db-diagnostics.ts` + +- `x db diagnostics`: `startDb` (never `startQueue`, which applies the framework tables), `@ultimat3/db/diagnostics`'s `dbDiagnostics`, one table per section; exit 0 whatever it says; an embedded database `x dev` holds is `X_DEV_ALREADY_RUNNING` diff --git a/docs/idea/17-scale-ladder.md b/docs/idea/17-scale-ladder.md index 65a485747..7d22dd6b0 100644 --- a/docs/idea/17-scale-ladder.md +++ b/docs/idea/17-scale-ladder.md @@ -28,6 +28,22 @@ Each rung below states what you run, what forces the next one, and what the app Costs are order-of-magnitude, not quotes. No price in this table is verified against a vendor bill. +## Each rung, measured + +`As of 2026-11-09`. Every figure is one committed result, held to the prose by `bun run scripts/bench-claims.ts` (`X_BENCH_CLAIM_STALE`); the box, the method and the caveats are on [the claims page](../../scripts/bench/results/README.md). One busy 12-core machine unless the row says otherwise — an order of magnitude, never a ceiling. + +| Rung | What was measured | Figure | Result | +|---|---|---|---| +| 0 | `x dev` on a fresh scaffold, `/` | 267 req/s median, p99 181.85 ms | [`http-hello.json`](../../scripts/bench/results/http-hello.json) | +| 0 → 2 | `x build --target docker` on a fresh scaffold, the production image, one `web` pod: 1,439 req/s | p99 on the claims page; three pods on one host did not scale, and no scaling figure is claimed | [`http-prod-image.json`](../../scripts/bench/results/http-prod-image.json) | +| 1 | the Postgres job queue, 1 / 4 / 16 worker processes | defaults: 799 / 1,618 / 2,277 claims/s; tuned, the queue itself plateaus near 3,724 claims/s | [`worker-throughput.json`](../../scripts/bench/results/worker-throughput.json) | +| 1 | scheduler failover | a live test, not a bench: no double fire, no skip | [`scheduler-failover.live.test.ts`](../../packages/jobs/src/scheduler-failover.live.test.ts) | +| 1–3 | boot: module load against file count | 5,000 modules batched vs serial on the claims page | [`boot-vs-files.json`](../../scripts/bench/results/boot-vs-files.json) | +| 3 | 10,000 subscribers across 3 `sync` nodes over NATS: p99 23.26 ms | every one of 200,000 deliveries arrived | [`fanout-3-nodes.json`](../../scripts/bench/results/fanout-3-nodes.json) | +| 3 | one `sync` node, 50,000 sockets SIGKILLed | p50 54.0s to the first patch after reconnect | [`50k-restart.json`](../../scripts/bench/results/50k-restart.json) | +| 3 | a `helm upgrade` under load on kind | 0 of 8,686 requests failed | `deploy-proof`, [`14-roadmap.md`](./14-roadmap.md) milestone 11 | +| 4 | — | **nothing measured**: no distributed SQL, no multi-region run, no clustered cache node | — | + --- ## Rung 0 — a free PaaS plan, not a container you operate @@ -72,6 +88,8 @@ Same platform. The change is `ROLE` and two env vars. This is also the rung a fr | `scheduler` | cron dispatch, fixed at one — leader election is an expiring lease row, so a second instance is an idle standby. **Must be always-on**: a plan that sleeps it drops the cron | | `migrate` | run-once, before anything serves the new schema | +**Invalidations reach every replica on this rung**, `As of 2026-11-09`. With no `NATS_URL` and `REDIS_URL` (else `VALKEY_URL`) set, cache and ISR invalidations ride Redis pub/sub; NATS, once configured, stays the path, never both. The boot says which in one `cache.broadcast` log line, naming the bus and the variable that chose it, and a replica whose subscription dropped flushes its own tiers on reconnect rather than serve what it may have missed ([`runtime-cache-bus.ts`](../../packages/cli/src/runtime-cache-bus.ts), [`redis-pubsub.ts`](../../packages/cache/src/redis-pubsub.ts)). Until then this rung's second web replica served a stale page or `cache:` read until its TTL: the bus was the in-process one. + Add the shared cache tier when there is more than one web replica **and** a measured cross-replica miss. `cache.tiers: ['request-memo', 'lru', 'redis']` plus `REDIS_URL`. Redis, Valkey or Dragonfly `As of 2026-08` — the key layout is slot-clean, so no engine is excluded and no server flag is needed ([below](#dragonfly-honestly)). **App code change: none.** The cache tiers are read-through and ordered by core's `CACHE_TIERS`; adding one changes where a value is found, never how it is asked for ([`packages/cache/src/tiers.ts`](../../packages/cache/src/tiers.ts)). @@ -136,13 +154,13 @@ Every scale component, and exactly what to swap. | Embedded dev DB | `@ultimat3/db` · `PgliteClient` | `pgliteClient()` | — | unset `DATABASE_URL` | shipped, `x dev` only | | Repository | `@ultimat3/entity` · `Repo`, `Driver` | `postgresRepo()`; `memoryRepo()` for tests | — | — | shipped | | Pool sizing | `@ultimat3/db` · `POOL_PROFILES` | per-`ROLE` max / statement timeout / idle timeout | — | `ROLE`, `DATABASE_POOL_MAX` | shipped | -| Read replicas | `@ultimat3/db` · `replicatedClient()`, `isPlainRead()`, `withReplicaReads()` | primary + standby behind one `DbClient`; a plain read inside an open scope goes to the standby, everything else and every transaction to the primary. Read-your-writes by scope, 3-failure/10s breaker back to the primary | — | `DATABASE_REPLICA_URL` | shipped `As of 2026-08-24` — the pool is env, the SCOPE is the app's, and nothing opens one yet | +| Read replicas | `@ultimat3/db` · `replicatedClient()`, `isPlainRead()`, `withReplicaReads()`; `@ultimat3/cli` · `attachReplica()`, `positionStage()` | primary + standby behind one `DbClient`; a plain read inside an open scope goes to the standby, everything else and every transaction to the primary, 3-failure/10s breaker back to the primary. **The boot opens the scope per request** ([`runtime-replica.ts`](../../packages/cli/src/runtime-replica.ts)): a write early in a request pins its later reads to the primary, and the write's WAL position rides a `__Host-x-pos` cookie / `x-ultimate-position` header, so the NEXT request reads a standby only once it has replayed past it ([`replica-position.ts`](../../packages/db/src/replica-position.ts), [`runtime-replica-position.ts`](../../packages/cli/src/runtime-replica-position.ts)) | — | `DATABASE_REPLICA_URL` | shipped and wired `As of 2026-11-09` — the env is the only switch; `replica-position.live.test.ts` and `runtime-replica-roundtrip.live.test.ts` run against the standby in `docker-compose.test.yml` | | Cache, per-request | `@ultimat3/cache` · `CacheTier` | request memo | `cache.tiers: ['request-memo']` | — | shipped | | Cache, per-process | `@ultimat3/cache` · `CacheTier` | LRU | `cache.tiers: ['lru']` | — | shipped | | Cache, cross-node | `@ultimat3/cache` · `CacheTier`, `RedisLike` | `redisTier()` over `Bun.redis` | `cache.tiers: ['redis']` | `REDIS_URL` | shipped | | Cache, edge | `@ultimat3/cache` · `CacheTier` | CDN headers + purge (Cloudflare, Fastly, HTTP) | `cache.tiers: ['cdn']` | purge-provider env | shipped | | Job queue | `@ultimat3/jobs` · `JobDriver` (`enqueue`/`claim`/`ack`/`nack`/`heartbeat`/`stats`) | `postgresJobDriver()`, which the boot installs | **none** — `jobs.driver` was deleted in 5.0.0; `jobs:` in `apps//runtime.ts` is the only switch | `DATABASE_URL` | shipped | -| Job queue, Redis | same interface | a future `redisJobDriver()` — Streams + consumer groups + `XAUTOCLAIM` | none; `jobs: redisJobDriver()` in `apps//runtime.ts` | `REDIS_URL` | **not shipped** — the all-throw stub was deleted in 25.0.0; a real driver arrives as a minor | +| Job queue, Redis | same interface | `redisJobDriver()` on `@ultimat3/jobs/redis` — one Lua script per operation, every key under one hash tag, over core's `redisClient()`; held to the pg driver's contract by `@ultimat3/testing`'s `jobDriverConformance` | none; `jobs: redisJobDriver()` in `apps//runtime.ts` | `REDIS_URL` | shipped `As of 2026-11-09` ([`driver-redis.ts`](../../packages/jobs/src/driver-redis.ts)); unmeasured — the worker bench below is the Postgres queue's | | Scheduler leader | `@ultimat3/jobs` · `postgresLeaseLeader` | `SQL_LEADER_ACQUIRE` / `SQL_LEADER_RELEASE` — an expiring row in `x_scheduler_leader`, TTL 30s, `acquire()` doubling as the renewal. **Never `pg_try_advisory_lock`**: it is session-scoped, and the executor is a pool, so the grant dies when the connection returns and every node reads itself as leader | — | — | shipped | | Realtime fanout | `@ultimat3/realtime` · `Transport`, via `selectTransport(env)` | `InProcessTransport` \| `NatsTransport` | `NATS_URL` set or not — **decides**, as of 31.0.0 (`transport` in `runtime.ts` for a bus of the app's own) | `NATS_URL` | shipped | | Presence | `@ultimat3/realtime` · JetStream KV | bucket, default `x_presence`, TTL 30s | — | `NATS_KV_BUCKET` | shipped | @@ -323,7 +341,7 @@ The most valuable advice on this page. | your own box (rung 2) | the PaaS bill exceeds a server plus the hours to run it, or a platform limit actually blocks you | | NATS (rung 3) | you run more than one `sync` replica. `InProcessTransport` is correct and free at one node — and the measured 50k-socket result was taken on one node, in process | | the `replicator` role (rung 3) | you use live queries. The chart ships it disabled | -| a read replica (rung 3) | the primary's CPU is read-bound and you have measured which reads. It is two opt-ins — `DATABASE_REPLICA_URL` and a `withReplicaReads` scope — and it doubles the connection arithmetic, because the standby inherits the role's pool profile and `DATABASE_POOL_MAX` | +| a read replica (rung 3) | the primary's CPU is read-bound and you have measured which reads. It is one opt-in — `DATABASE_REPLICA_URL`; the boot opens the per-request scope and carries read-your-writes across requests — and it doubles the connection arithmetic, because the standby inherits the role's pool profile and `DATABASE_POOL_MAX` | | Kubernetes (rung 3) | you have roughly three nodes of real workload. One 6-vCPU / 12-GiB host runs Postgres, a cache, NATS and all six roles under Compose | | splitting a database off (rung 4) | a database sustains >50% CPU or >75% IO utilisation, or an app needs an extension that is not safe cluster-wide, or compliance forces separation. Until then: share | | distributed SQL (rung 4) | you need multi-region write locality, or must survive a zone loss without a manual failover. Throughput alone is not a reason | @@ -340,10 +358,15 @@ Two rules underneath all of them: |---|---| | one image, `ROLE` selects the process | **true** — `packages/core/src/roles.ts`, `docker/Dockerfile` | | Postgres job queue with `SKIP LOCKED`, partial-unique idempotency, lease-row leader | **true** — `packages/jobs/src/driver-pg-sql.ts` | -| Redis job driver | **stub** — interface-complete, every method throws `X_NOT_IMPLEMENTED`. There is no NATS job driver (deleted in 25.0.0) | +| Redis job driver | **true, unmeasured** — `redisJobDriver()` on `@ultimat3/jobs/redis`, the conformance suite's contract. There is no NATS job driver (deleted in 25.0.0) | | shared cache tier over the Redis protocol, tag sets, declared-key invalidation | **true** — `packages/cache/src/redis.ts`; one call per tag over `{entity}`-tagged buckets, then the value keys one `DEL` each, plus single-flight, TTL jitter and a bucket lease | | NATS fanout and JetStream KV presence | **true** — `packages/realtime/src/nats-transport.ts`, selected by `NATS_URL` | | Postgres logical replication change feed, `pgoutput`, own wire client | **true** — `packages/realtime/src/pg-replication.ts` | +| Postgres job queue throughput | **measured, one box** — defaults bounded by the queue, not the poll floor, since a full pass marks a backlog (As of 2026-11-09), plateau of the queue itself between 4 and 16 tuned workers, `scripts/bench/results/worker-throughput.json` | +| scheduler failover: the leader dies mid-fire, a standby takes over | **proved live** — `packages/jobs/src/scheduler-failover.live.test.ts`: the standby fires only after the lease expires, fires the occurrence the dead leader held, the dead leader's late fire is refused by the watermark, and every occurrence is queued exactly once | +| cross-node fan-out over NATS | **measured at 10k subscribers, 3 nodes, one box** — `scripts/bench/results/fanout-3-nodes.json`; in-process sockets, a real nats-server | +| read replicas with read-your-writes | **true and wired** — the boot opens the scope per request and carries the write's WAL position to the next one (`runtime-replica.ts`) | +| cache/ISR invalidation across replicas without NATS | **true** — Redis pub/sub on `REDIS_URL` (`runtime-cache-bus.ts`) | | 50k sockets, forced restart, p50 54.0s to the first patch on the reconnected socket | **measured** — one node, in-process transport, `scripts/bench/results/50k-restart.json`. Reachability; it cannot see a lost patch | | channel patches delivered across a forced restart, no client observing a gap | **measured at 10k, not at 50k** — 1,666,882 received, 0 observed sequence gaps, a lower bound on loss, `scripts/bench/results/10k-restart-seq.json` | | Compose prod topology | **true, and emitted by `x new`** — `web` and `sync` at `replicas: 1` because each publishes a host port; `worker` scales freely | diff --git a/docs/idea/18-build-vs-wrap.md b/docs/idea/18-build-vs-wrap.md index 881ae5d5d..2921d5ef5 100644 --- a/docs/idea/18-build-vs-wrap.md +++ b/docs/idea/18-build-vs-wrap.md @@ -154,6 +154,20 @@ every axis, which is what makes the criterion a criterion rather than a preferen The one-line form: NATS was protocol with no integration; the agent layer is integration with almost no protocol. +### SAML response verification — BUILD, decided 2026-11-09 + +`samlIdentityProvider()` (`packages/auth/src/saml-*.ts`) verifies XML-DSig itself; no dependency. + +| Axis | Reading | +|---|---| +| a Bun native | none — no SAML or XML-DSig in Bun | +| the candidates | `xml-crypto` + `@xmldom/xmldom`, `node-saml`, `samlify`: Node-centric, with a long CVE history on exactly this surface — signature wrapping (XSW), comment injection into a signed value, `DigestValue` confusion, multi-root documents | +| integration benefit of owning it | high: sessions, the shared replay store (`auth.totpReplay`), coded refusals with `meta.check` — all ours whichever library parsed the bytes | +| the XML actually needed | a small subset: SP-initiated, one signed assertion, exc-c14n + enveloped, RSA-SHA256 / SHA-256. Everything outside it is refused, not parsed (DOCTYPE, entities, processing instructions, encrypted assertions, SHA-1) | +| what earns it | known-answer tests against a REAL signer: our c14n reproduces every `DigestValue` of three xmlsec-signed responses (python3-saml's vectors, MIT) and their `SignedInfo` verifies under the published certificate, so a bug symmetric between our signer and our verifier cannot pass; the test IdP writes canonical form by hand | + +Revisit if a Bun-native, maintained XML-DSig verifier appears. + ## Dependency ledger Every dependency admitted at a driver/transport seam under this criterion, in one place. diff --git a/docs/plans/2026/11/09/101-final-hardening-before-v31/22-scale.md b/docs/plans/2026/11/09/101-final-hardening-before-v31/22-scale.md index 8c36dcbf7..1e90206e0 100644 --- a/docs/plans/2026/11/09/101-final-hardening-before-v31/22-scale.md +++ b/docs/plans/2026/11/09/101-final-hardening-before-v31/22-scale.md @@ -23,7 +23,7 @@ Nothing grows super-linearly. Fixed costs dominate. | B1 | **High** | `cli/src/app-load.ts:207-231` (`scanAppModules`): serial `await import()`; dev also reads each file's text. Production boots through the same scan (`:260`), so pod start and rollout grow with the app (projected ~60 s at 12k files) | import concurrently (`Promise.all`), register in sorted order (deterministic). **v31:** `x build` writes a prebuilt registry; production boots from it, never scans | | B2 | Med | no affected-only verify (`cmd-verify-spec.ts`; `x affected` is per workspace) | `x verify --changed`: map changed files to the steps whose inputs they touch (registries → contract/policy/i18n; entities → drift) | | B3 | Low | `drift` step: fixed ~8 s (PGlite boot + migration replay) | skip when no entity/migration changed (with B2), or reuse a cached template database | -| B4 | Med | static `islands/`: each island bundles the framework client prelude (63 KB per generated form island, no shared chunk); N islands on a page download it N times | one hashed shared chunk for framework client code; re-measure route budgets (`bun run budget-raises` if any rise, axiom 9) | +| B4 | Med | static `islands/`, As of 2026-11-09: one shared framework chunk is **falsified** — one global chunk carries the union of what any island uses (examples/dummy 183,281 B, docs-site 46,924 B; `/pricing` 25,557 → ~185 KB). Two shared files already exist: opt-in `islands.sharedChunks` (`cli/src/island-bundle.ts`) and the hashed `page-runtime..js` (63,928 B, the plan's "63 KB"). Only `/posts/[id]` renders 3 framework-heavy islands; split mode there 96,250 → 70,462 B, but +910…+9,154 B on six single-island routes. No preload: an entry's shared chunks were fetched only after it parsed | **per-route island grouping and modulepreload.** Shipped: `` for each rendered `idle`/held island's imports, deduped, the runtime chunk skipped on a booted document, none on a no-island page (`cli/src/route-islands-preload.ts`); JS bytes unchanged (budgets weigh scripts and entries, never ``). Not built, design: the static export knows which islands each route rendered → build islands that co-render on a route in ONE split build, every other island solo. Cost: two-pass export (render, group, rebuild, re-render: island URLs are in the documents), dynamic routes' sets vary by params, the container has no render pass to learn sets from (needs the export's grouping in the image), the layout's `update-banner` co-renders with every app/ island and must be kept out of grouping. Benefit, measured: `/posts/[id]` −25,788 B, every other route unchanged | | B5 | Low | `x g` loads the whole app per call (1.6 s → 4.0 s) | skip the app load when the generator's checks do not need it | ## Monorepo layout (owner, 2026-11-09: a powerful layout to build anything web) — audit first, then fix diff --git a/dummy/docs-site/apps/web/e2e/built-site.e2e.test.ts b/dummy/docs-site/apps/web/e2e/built-site.e2e.test.ts index 5e86a17bd..8e0dd1e37 100644 --- a/dummy/docs-site/apps/web/e2e/built-site.e2e.test.ts +++ b/dummy/docs-site/apps/web/e2e/built-site.e2e.test.ts @@ -12,15 +12,14 @@ import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { findChrome, LAUNCHED_CLOSE_MS } from '@ultimat3/scraping/cdp'; import type { E2eTab } from '@ultimat3/testing'; import { allowHost, DEFAULT_CDP_TIMEOUT_MS, - E2E_BROWSER_CLOSE_MS, E2E_BROWSER_OPEN_MS, E2E_GOTO_MS, E2E_TAB_OPEN_MS, - findChrome, openE2eBrowser, } from '@ultimat3/testing'; @@ -80,7 +79,7 @@ describe.skipIf(noBrowser)('the published site, as files', () => { await browser?.close(); server?.stop(true); if (out !== undefined) await rm(out, { recursive: true, force: true }); - }, E2E_BROWSER_CLOSE_MS + DEFAULT_CDP_TIMEOUT_MS); + }, LAUNCHED_CLOSE_MS + DEFAULT_CDP_TIMEOUT_MS); test('/errors/X_DB_DRIFT is 200 with its row, and its .md sibling is served', async () => { const page = await fetch(`${base}/errors/X_DB_DRIFT`, { signal: AbortSignal.timeout(5_000) }); diff --git a/dummy/docs-site/package.json b/dummy/docs-site/package.json index 601fc5e8e..6ac51fb0c 100644 --- a/dummy/docs-site/package.json +++ b/dummy/docs-site/package.json @@ -21,6 +21,7 @@ "@biomejs/biome": "2.6.3", "@electric-sql/pglite": "^0.5.11", "@types/bun": "^1.4.0", + "@ultimat3/scraping": "30.9.0", "@ultimat3/testing": "30.9.0", "typescript": "^7.0.2" }, diff --git a/dummy/social-media-clone/apps/web/e2e/smoke.e2e.test.ts b/dummy/social-media-clone/apps/web/e2e/smoke.e2e.test.ts index 5cc4275b2..d620d82c8 100644 --- a/dummy/social-media-clone/apps/web/e2e/smoke.e2e.test.ts +++ b/dummy/social-media-clone/apps/web/e2e/smoke.e2e.test.ts @@ -9,17 +9,16 @@ */ import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { findChrome, LAUNCHED_CLOSE_MS } from '@ultimat3/scraping/cdp'; import type { E2eApp, E2eTab } from '@ultimat3/testing'; import { allowHost, DEFAULT_CDP_TIMEOUT_MS, E2E_APP_START_MS, E2E_APP_STOP_MS, - E2E_BROWSER_CLOSE_MS, E2E_BROWSER_OPEN_MS, E2E_GOTO_MS, E2E_TAB_OPEN_MS, - findChrome, openE2eBrowser, startE2eApp, } from '@ultimat3/testing'; @@ -54,7 +53,7 @@ describe.skipIf(noBrowser)('smoke: sign in and land where you were going', () => await browser?.close(); await app?.stop(); }, - E2E_BROWSER_CLOSE_MS + E2E_APP_STOP_MS + DEFAULT_CDP_TIMEOUT_MS, + LAUNCHED_CLOSE_MS + E2E_APP_STOP_MS + DEFAULT_CDP_TIMEOUT_MS, ); test('a signed-out visit to /dashboard signs in and comes back to /dashboard', async () => { diff --git a/dummy/social-media-clone/package.json b/dummy/social-media-clone/package.json index e03cd9df0..400fe8f24 100644 --- a/dummy/social-media-clone/package.json +++ b/dummy/social-media-clone/package.json @@ -21,6 +21,7 @@ "@biomejs/biome": "2.6.3", "@electric-sql/pglite": "^0.5.11", "@types/bun": "1.4.4", + "@ultimat3/scraping": "30.9.0", "@ultimat3/testing": "30.9.0", "typescript": "^7.0.2" }, diff --git a/dummy/social-media-clone/packages/db/schema/framework/03_sequences/x_job_lease_fence.sql b/dummy/social-media-clone/packages/db/schema/framework/03_sequences/x_job_lease_fence.sql new file mode 100644 index 000000000..6e5f00054 --- /dev/null +++ b/dummy/social-media-clone/packages/db/schema/framework/03_sequences/x_job_lease_fence.sql @@ -0,0 +1,3 @@ +-- generated by `x db gen` from the migrations; never edited by hand (X_SCHEMA_DUMP_DRIFT) + +create sequence "x_job_lease_fence" as bigint start with 1 increment by 1 minvalue 1 maxvalue 9223372036854775807 cache 1 no cycle; diff --git a/dummy/social-media-clone/packages/db/schema/framework/04_tables/x_job_leases.sql b/dummy/social-media-clone/packages/db/schema/framework/04_tables/x_job_leases.sql index aefb1b8c8..f547c6e5c 100644 --- a/dummy/social-media-clone/packages/db/schema/framework/04_tables/x_job_leases.sql +++ b/dummy/social-media-clone/packages/db/schema/framework/04_tables/x_job_leases.sql @@ -5,5 +5,6 @@ create table "x_job_leases" ( "slot" integer not null, "holder" text not null, "expires_at" timestamp with time zone not null, + "fence" bigint, constraint "x_job_leases_pkey" PRIMARY KEY (lease_key, slot) ); diff --git a/examples/dummy/apps/web/api/index.ts b/examples/dummy/apps/web/api/index.ts index 55b39466d..5c08c5cbb 100644 --- a/examples/dummy/apps/web/api/index.ts +++ b/examples/dummy/apps/web/api/index.ts @@ -37,6 +37,7 @@ import * as passkeySignInOptions from '../app/auth/actions/passkey-sign-in-optio import * as registerPasskey from '../app/auth/actions/register-passkey'; import * as planCatalog from '../app/billing/backfills/plan-catalog'; import * as planPrices from '../app/billing/jobs/plan-prices'; +import * as postCharge from '../app/billing/jobs/post-charge'; import * as settleUpgrade from '../app/billing/jobs/settle-upgrade'; import * as contactSales from '../app/contact/actions/contact-sales'; import * as sendSalesReceipt from '../app/contact/jobs/send-sales-receipt'; @@ -63,6 +64,7 @@ import * as keepDraftReview from '../app/posts/actions/keep-draft-review'; import * as likePost from '../app/posts/actions/like-post'; import * as listPostsImportParts from '../app/posts/actions/list-posts-import-parts'; import * as movePostStatus from '../app/posts/actions/move-post-status'; +import * as printPost from '../app/posts/actions/print-post'; import * as publishPost from '../app/posts/actions/publish-post'; import * as requestDraftReview from '../app/posts/actions/request-draft-review'; import * as requestPostsExport from '../app/posts/actions/request-posts-export'; @@ -72,6 +74,7 @@ import * as summarize from '../app/posts/actions/summarize'; import * as summarizePosts from '../app/posts/actions/summarize-posts'; import * as withdrawPost from '../app/posts/actions/withdraw-post'; import * as postExcerpts from '../app/posts/backfills/post-excerpts'; +import * as reindexPosts from '../app/posts/backfills/reindex-posts'; import * as commentPosted from '../app/posts/jobs/comment-posted'; import * as dailyLikesRollup from '../app/posts/jobs/daily-likes'; import * as exportPosts from '../app/posts/jobs/export-posts'; @@ -146,6 +149,7 @@ export const api = defineApi({ memberAvatar, createPost, publishPost, + printPost, withdrawPost, createComment, requestPostsExport, @@ -206,6 +210,7 @@ export const api = defineApi({ commentPosted, reviewDraftLater, postExcerpts, + reindexPosts, dailyLikesRollup, likeTotalsRefresh, onboardOrg, @@ -224,6 +229,7 @@ export const api = defineApi({ planPrices, planCatalog, settleUpgrade, + postCharge, openErrorIssue, exportMember, eraseMember, diff --git a/examples/dummy/apps/web/app/auth/login.test.ts b/examples/dummy/apps/web/app/auth/login.test.ts index 1e1e0789f..ab76dece4 100644 --- a/examples/dummy/apps/web/app/auth/login.test.ts +++ b/examples/dummy/apps/web/app/auth/login.test.ts @@ -74,7 +74,7 @@ afterEach(() => { setLogSink(previousSink); }); -/** `/auth/oauth/:provider` → the URL a browser is actually sent to. The mount, filled in. */ +/** `/auth/sso/:provider` → the URL a browser is actually sent to. The mount, filled in. */ const mounted = (pattern: string, provider: string): string => pattern.replace(':provider', provider); @@ -171,15 +171,17 @@ describe('log in with GitHub', () => { // at the start descriptor's own path, which it already holds — never at text parsed out of // a failure body. expect(body['fix']).toBe('x errors explain X_OAUTH_STATE_INVALID --json'); - expect(JSON.stringify(body)).not.toContain('/auth/oauth/'); + expect(JSON.stringify(body)).not.toContain('/auth/sso/'); // Axiom 4, checked as a round trip, where the authored fix now lives: the `auth.oauth.refused` // log line. The path it tells the developer to restart at has to be the one this app mounts. const refused = logged.find((line) => line.includes('auth.oauth.refused')) ?? ''; expect(/GET (\/[^\s"\\]+)/.exec(refused)?.[1] ?? '').toBe(mounted(login.start.path, 'github')); - // The code the handshake authorised is spent whether or not the callback succeeded. - expect(done.headers.getSetCookie().some((c) => c.includes('=;'))).toBe(true); + // A forged state is not this browser's answer, so its sign-in state cookie stays: a request + // carrying the cookie cross-site cannot end the real sign-in in flight. Only an answer bound + // to the state (`answers()`) clears it, on any outcome. + expect(done.headers.getSetCookie().some((c) => c.includes('=;'))).toBe(false); }); test('a completed login lands on Postly and mints a session Postly can authenticate', async () => { diff --git a/examples/dummy/apps/web/app/auth/login.ts b/examples/dummy/apps/web/app/auth/login.ts index 9f548bdd9..d8438559a 100644 --- a/examples/dummy/apps/web/app/auth/login.ts +++ b/examples/dummy/apps/web/app/auth/login.ts @@ -63,8 +63,8 @@ export function postlyAuth(seams: PostlyAuthSeams = {}): Auth { export type PostlyLoginOptions = Omit; /** - * The two legs of the login as route descriptors — `start.path` is `/auth/oauth/:provider` and - * `callback.path` is `/auth/oauth/:provider/callback`, the one declaration every `X_OAUTH_*` fix + * The two legs of the login as route descriptors — `start.path` is `/auth/sso/:provider` and + * `callback.path` is `/auth/sso/:provider/callback`, the one declaration every `X_OAUTH_*` fix * line quotes. `options` carries the seams the framework already injects (credentials, the token * endpoint's `fetch`, the handshake secret), so a test drives THIS login and not a copy of it. * diff --git a/examples/dummy/apps/web/app/billing/jobs/post-charge.test.ts b/examples/dummy/apps/web/app/billing/jobs/post-charge.test.ts new file mode 100644 index 000000000..046933821 --- /dev/null +++ b/examples/dummy/apps/web/app/billing/jobs/post-charge.test.ts @@ -0,0 +1,33 @@ +/** + * `chargePosting` is the whole of what Postly decides about its books: which accounts a charge + * moves, and by how much. The framework refuses an unbalanced posting, so the proof here is that + * every plan, in every billing currency, posts one that nets to zero. + */ + +import type { BillingCurrency, PlanCode } from '@postly/domain'; +import { PLAN_CODES, priceOf } from '@postly/domain'; +import { assertBalancedPosting } from '@ultimat3/jobs'; +import { describe, expect, test } from '@ultimat3/testing'; +import { chargePosting, postCharge } from './post-charge'; + +const ORG = '00000000-0000-4000-8000-0000000000aa'; +const CURRENCIES: readonly BillingCurrency[] = ['USD', 'EUR']; + +describe('chargePosting', () => { + test('every plan in every currency posts cash in and revenue out, netting to zero', () => { + for (const plan of PLAN_CODES as readonly PlanCode[]) { + for (const currency of CURRENCIES) { + const posting = chargePosting(ORG, 'ch_1', plan, currency); + expect(() => assertBalancedPosting(posting)).not.toThrow(); + expect(posting.entries.map((entry) => entry.account)).toEqual(['cash', 'revenue']); + expect(posting.entries[0]?.amount).toEqual(priceOf(plan, currency)); + } + } + }); + + test('the charge id is the key and the org the tenant: one posting per charge, per org', () => { + const posting = chargePosting(ORG, 'ch_42', 'team', 'EUR'); + expect([posting.key, posting.orgId]).toEqual(['ch_42', ORG]); + expect(postCharge.idempotencyKeyFor(posting)).toBe('billing.post-charge:ch_42'); + }); +}); diff --git a/examples/dummy/apps/web/app/billing/jobs/post-charge.ts b/examples/dummy/apps/web/app/billing/jobs/post-charge.ts new file mode 100644 index 000000000..8b5a180d0 --- /dev/null +++ b/examples/dummy/apps/web/app/billing/jobs/post-charge.ts @@ -0,0 +1,34 @@ +// `postCharge`, the billing slice's books: a settled upgrade's charge as a double-entry posting — +// cash in, revenue out, the same amount — written by `ledger()` into `ledger_lines`. The account +// names are Postly's; the framework refuses a posting that does not net to zero per currency. + +import { driver, ledgerLines } from '@postly/db'; +import type { BillingCurrency, PlanCode } from '@postly/domain'; +import { priceOf } from '@postly/domain'; +import type { LedgerPosting } from '@ultimat3/jobs'; +import { entityLedgerBook, ledger } from '@ultimat3/jobs'; + +export const postCharge = ledger({ + name: 'billing.post-charge', + // The org the posting names: its lines live under it. + tenant: (posting) => posting.orgId ?? 'none', + book: () => entityLedgerBook({ driver, entity: ledgerLines }), +}); + +/** The charge as a posting, keyed by the vendor's charge id: one posting per charge, per org. */ +export const chargePosting = ( + orgId: string, + chargeId: string, + plan: PlanCode, + currency: BillingCurrency, +): LedgerPosting => { + const price = priceOf(plan, currency); + return { + key: chargeId, + orgId, + entries: [ + { account: 'cash', amount: price }, + { account: 'revenue', amount: { ...price, minor: -price.minor } }, + ], + }; +}; diff --git a/examples/dummy/apps/web/app/billing/jobs/settle-upgrade.ts b/examples/dummy/apps/web/app/billing/jobs/settle-upgrade.ts index b40e20f94..5b73c8b75 100644 --- a/examples/dummy/apps/web/app/billing/jobs/settle-upgrade.ts +++ b/examples/dummy/apps/web/app/billing/jobs/settle-upgrade.ts @@ -13,6 +13,7 @@ import { releaseSeats, reserveSeats, } from '../vendor-payments'; +import { chargePosting, postCharge } from './post-charge'; export const settleUpgrade = job({ input: t.object({ orgId: t.uuid, plan: t.enumerated(...PLAN_CODES) }), @@ -36,5 +37,9 @@ export const settleUpgrade = job({ compensate: (reservationId) => releaseSeats(reservationId), }); await step.run('invoice', () => issueInvoice(org.id, charge)); + // The books, once the money is final: a posting keyed by the charge, so a replay is one posting. + await step.run('post-ledger', () => + postCharge.enqueue(chargePosting(org.id, charge, input.plan, org.billingCurrency)), + ); }, }); diff --git a/examples/dummy/apps/web/app/billing/settle-upgrade.job.test.ts b/examples/dummy/apps/web/app/billing/settle-upgrade.job.test.ts index 033d9c608..1f9569846 100644 --- a/examples/dummy/apps/web/app/billing/settle-upgrade.job.test.ts +++ b/examples/dummy/apps/web/app/billing/settle-upgrade.job.test.ts @@ -4,6 +4,8 @@ * nothing. Read off the vendor's own call log, the only witness a refund has. */ +import { db } from '@postly/db'; +import { priceOf } from '@postly/domain'; import { expect, jobTest } from '@ultimat3/testing'; import { settleUpgrade } from './jobs/settle-upgrade'; import { closeVendorAccount, vendorCallsFor } from './vendor-payments'; @@ -33,17 +35,30 @@ jobTest( ); jobTest( - 'a settled upgrade is charged, reserved and invoiced, and undoes nothing', + 'a settled upgrade is charged, reserved, invoiced and booked, and undoes nothing', async ({ seed, runJobs }) => { const { org } = await seed('dev').pick({ org: 'org:tinta' }); const trace = await runJobs(settleUpgrade, { orgId: org.id, plan: 'team' }); - expect(outcomes(trace)).toEqual([['completed', null]]); + // The upgrade, then the posting it enqueued (`postCharge`). + expect(outcomes(trace)).toEqual([ + ['completed', null], + ['completed', null], + ]); + const [charge] = vendorCallsFor(org.id); expect(vendorCallsFor(org.id).map((call) => call.split(':')[0])).toEqual([ 'charge', 'reserve', 'invoice', ]); + // One posting per charge: cash in, revenue out, the plan's price in the org's currency. + const price = priceOf('team', org.billingCurrency); + const chargeId = idOf(charge) ?? expect.unreachable('the charge call left no id'); + const booked = await db.ledgerLines.where({ orgId: org.id }).all(); + expect(booked.map((line) => [line.postingKey, line.account, line.amount])).toEqual([ + [chargeId, 'cash', price], + [chargeId, 'revenue', { ...price, minor: -price.minor }], + ]); }, ); diff --git a/examples/dummy/apps/web/app/posts/actions/print-post.test.ts b/examples/dummy/apps/web/app/posts/actions/print-post.test.ts new file mode 100644 index 000000000..169a3b61b --- /dev/null +++ b/examples/dummy/apps/web/app/posts/actions/print-post.test.ts @@ -0,0 +1,42 @@ +/** + * unit — `printPost`, the app's half of `pdf()`: who may print, into which org, and where the file + * lands. Every refusal here happens before a browser is asked — a reader or another org's owner + * that reached Chrome would fail on a missing browser instead of `X_FORBIDDEN`. The print itself + * is the framework's (`packages/scraping/src/pdf-action.test.ts`). + */ + +import { expect, test } from '@ultimat3/testing'; +import { printedPostKey, printPost } from './print-post'; + +test('printing is an export: an author without post:export is refused', async ({ + seed, + actorFor, +}) => { + const { post, bruno } = await seed('dev').pick({ post: 'post:tenancy', bruno: 'member:bruno' }); + + await expect( + printPost.as(actorFor(bruno), { orgId: post.orgId, slug: 'tenancy' }), + ).rejects.toBeUltimateError('X_FORBIDDEN'); +}); + +test('another org’s owner cannot print into this org', async ({ seed, actorFor }) => { + const { post, mara } = await seed('dev').pick({ post: 'post:tenancy', mara: 'member:mara' }); + + await expect( + printPost.as(actorFor(mara), { orgId: post.orgId, slug: 'tenancy' }), + ).rejects.toBeUltimateError('X_FORBIDDEN'); +}); + +test('a slug that is not one is refused by the schema', async ({ seed, actorFor }) => { + const { post, ada } = await seed('dev').pick({ post: 'post:tenancy', ada: 'member:ada' }); + + await expect( + printPost.as(actorFor(ada), { orgId: post.orgId, slug: '../../etc/passwd' }), + ).rejects.toBeUltimateError('X_INPUT_INVALID'); +}); + +test('the file lands under the org’s own prefix, one per post', () => { + const orgId = '3f1b7f4a-5c2d-4a5b-9f6e-0a1b2c3d4e5f'; + expect(printedPostKey(orgId, 'tenancy')).toStartWith(`org/${orgId}/`); + expect(printedPostKey(orgId, 'tenancy')).toEndWith('/posts/printed/tenancy.pdf'); +}); diff --git a/examples/dummy/apps/web/app/posts/actions/print-post.ts b/examples/dummy/apps/web/app/posts/actions/print-post.ts new file mode 100644 index 000000000..1cbd0dfa8 --- /dev/null +++ b/examples/dummy/apps/web/app/posts/actions/print-post.ts @@ -0,0 +1,29 @@ +// `printPost`, the posts slice's action — one primitive per file, the layout `x g` writes. +// +// A published post, printed to PDF on the org's disk. `pdf()` is a factory that RETURNS an +// `action`, so this is an action like its neighbours: schema, policy and rate limit are the +// action's, and HTTP, OpenAPI, the typed client and `.job()` come with it. It prints the post's +// PUBLIC page (`/blog/`): the printing browser carries no session, so a private route would +// need a short-lived token through `headers` (wiki Known gaps, "`pdf()` prints with no session"). + +import { t } from '@ultimat3/action'; +import { pdf } from '@ultimat3/scraping'; +import { disk, scopedKey } from '@ultimat3/storage'; +import { blogHref } from '../../../shared/entities'; +import { postExport } from '../policy'; + +/** The key a post's print lands under — the org's own prefix, one file per post. */ +export const printedPostKey = (orgId: string, slug: string): string => + scopedKey(orgId, 'posts', 'printed', `${slug}.pdf`); + +export const printPost = pdf({ + input: t.object({ orgId: t.uuid, slug: t.slug }), + // No `mcp:` — `pdf()` takes none, so this is not agent-callable: each call launches a browser, + // and an agent wanting a copy has `requestPostsExport`. + // A copy of the org's content leaving the app as a file is an export: `post:export`'s holder. + policy: postExport, + route: ({ input }) => blogHref({ slug: input.slug }), + key: ({ input }) => printedPostKey(input.orgId, input.slug), + storage: () => disk(), + layout: { format: 'a4', marginMm: { top: 15, right: 15, bottom: 15, left: 15 } }, +}); diff --git a/examples/dummy/apps/web/app/posts/backfills/reindex-posts.job.test.ts b/examples/dummy/apps/web/app/posts/backfills/reindex-posts.job.test.ts new file mode 100644 index 000000000..fb579290d --- /dev/null +++ b/examples/dummy/apps/web/app/posts/backfills/reindex-posts.job.test.ts @@ -0,0 +1,56 @@ +/** + * `reindexPosts` is the post search index rebuilt from the table. What is worth failing on: its + * durable identity (one live pass per name), and the document each post becomes — the title and + * nothing else of the row, under the post's own org, versioned by `updatedAt` so an index keeps + * the newer of two writes. + */ + +import { type Post, posts } from '@postly/db'; +import { postId as toPostId } from '@postly/domain'; +import { searchDocumentOf } from '@ultimat3/entity'; +import { memoryJobDriver } from '@ultimat3/jobs'; +import { installJobDriver } from '@ultimat3/jobs/adapter'; +import { afterAll, beforeAll, expect, jobTest } from '@ultimat3/testing'; +import { reindexPosts } from './reindex-posts'; + +let release = (): void => {}; +beforeAll(() => { + release = installJobDriver(memoryJobDriver()); +}); +afterAll(() => release()); + +const row = (over: Partial = {}): Post => + ({ + id: toPostId('00000000-0000-4000-8000-000000000001'), + orgId: '00000000-0000-4000-8000-000000000002', + authorId: '00000000-0000-4000-8000-000000000003', + slug: 'a-post', + title: 'Shipping on Fridays', + excerpt: 'never', + body: 'A body the index never sees.', + coverUrl: null, + status: 'published', + likeCount: 0, + publishedAt: new Date(0), + createdAt: new Date(0), + updatedAt: new Date(5_000), + ...over, + }) as Post; + +jobTest('reindexPosts is a backfill job under one durable key, forced or not', () => { + expect(reindexPosts.kind).toBe('job'); + expect(reindexPosts.idempotencyKeyFor({})).toBe('reindex-posts'); + expect(reindexPosts.idempotencyKeyFor({ force: true })).toBe('reindex-posts'); +}); + +jobTest('a post is indexed as its title alone, under its org, versioned by updatedAt', () => { + const post = row(); + const document = searchDocumentOf(posts, post); + expect(document.fields).toEqual({ title: 'Shipping on Fridays' }); + expect(document.tenant).toBe(post.orgId); + expect(document.version).toBe(5_000); + // A later edit carries a greater version: the index keeps it over the pass's older read. + expect(searchDocumentOf(posts, row({ updatedAt: new Date(9_000) })).version).toBeGreaterThan( + document.version, + ); +}); diff --git a/examples/dummy/apps/web/app/posts/backfills/reindex-posts.ts b/examples/dummy/apps/web/app/posts/backfills/reindex-posts.ts new file mode 100644 index 000000000..a30f9884f --- /dev/null +++ b/examples/dummy/apps/web/app/posts/backfills/reindex-posts.ts @@ -0,0 +1,26 @@ +/** + * `reindexPosts`: rebuilds the post search index from the table — what an operator runs after + * pointing `runtime.ts`'s `searchIndex` at a new engine, or after one lost data. `reindex()` is a + * `backfill()`, so it inherits the checkpoints, the throttle and `x db backfill`. Each document is + * versioned by `updatedAt`, so a pass racing a live edit can never put an older title back. + */ + +import { db, posts } from '@postly/db'; +import { assert, hasScope } from '@ultimat3/core'; +import { CROSS_TENANT_SCOPE } from '@ultimat3/entity'; +import { reindex } from '@ultimat3/jobs'; + +export const reindexPosts = reindex({ + name: 'reindex-posts', + entity: posts, + // Every org's posts in one pass; each document keeps its own org (`tenant` below is the run's). + tenant: 'none', + source: ({ ctx }) => { + assert( + hasScope(ctx.actor, CROSS_TENANT_SCOPE), + 'reindex-posts: this pass spans every tenant and its actor holds no tenancy:cross', + 'x db backfill reindex-posts --write --json', + ); + return db.posts; + }, +}); diff --git a/examples/dummy/apps/web/clients/postly/index.ts b/examples/dummy/apps/web/clients/postly/index.ts index 36c480fcf..5755bddfd 100644 --- a/examples/dummy/apps/web/clients/postly/index.ts +++ b/examples/dummy/apps/web/clients/postly/index.ts @@ -1,6 +1,6 @@ // biome-ignore-all format: generated by x client; its bytes are hashed in the stamp below // GENERATED from openapi.json by x client: never edit it, wrap it in a file of your own. -// @x-client {"command":"x client customer --out apps/web/clients/postly --only posts,query","source":"openapi.json","sourceHash":"32c95dbb3eca4c79dd983bd6e230aaa900bd74e2cafdfe696bf46a28f55a5872","bodyHash":"ef0478216dc24100ddbeb7258e90c797f181d95cf373047f8e9a928d5dd94efd"} +// @x-client {"command":"x client customer --out apps/web/clients/postly --only posts,query","source":"openapi.json","sourceHash":"5ff2b997a87912585bc5b56c5ac669f2411150fa618c7a0976d71c451fb111e8","bodyHash":"8cbd256c95c6f93a9a859cf09aa38de602b8d90b3d65d191fd33757a3fbf0288"} // The client for @postly/root 0.0.1: one method per published operation. import type * as T from './types'; @@ -152,6 +152,7 @@ export interface Refusals { postLikesByDay: never; postRecord: never; postReview: never; + printPost: never; publicPost: never; publicPostSlugs: never; publicPosts: never; @@ -174,6 +175,7 @@ const DECLARED: { readonly [K in keyof Refusals]: readonly string[] } = { postLikesByDay: [], postRecord: [], postReview: [], + printPost: [], publicPost: [], publicPostSlugs: [], publicPosts: [], @@ -228,6 +230,9 @@ export function createClient(options: ClientOptions) { /** Read the latest review of a post */ postReview: (input: T.PostReviewRequest, call: CallOptions = {}): Promise => send({ operation: "postReview", method: "GET", path: "/_x/query/post-review", pathParams: [], query: ["orgId","postId","_first","_after"], headers: [], body: 'none', empty: false }, input, call) as Promise, + /** printPost */ + printPost: (input: T.PrintPostInput, call: CallOptions = {}): Promise => + send({ operation: "printPost", method: "POST", path: "/api/posts/print", pathParams: [], query: [], headers: [], body: ["orgId","slug"], empty: false }, input, call) as Promise, /** Read one published post from the public blog */ publicPost: (input: T.PublicPostRequest, call: CallOptions = {}): Promise => send({ operation: "publicPost", method: "GET", path: "/_x/query/public-post", pathParams: [], query: ["slug","_first","_after"], headers: [], body: 'none', empty: false }, input, call) as Promise, diff --git a/examples/dummy/apps/web/clients/postly/types.ts b/examples/dummy/apps/web/clients/postly/types.ts index 8b09a8c92..97180780a 100644 --- a/examples/dummy/apps/web/clients/postly/types.ts +++ b/examples/dummy/apps/web/clients/postly/types.ts @@ -1,12 +1,14 @@ // biome-ignore-all format: generated by x client; its bytes are hashed in the stamp below // GENERATED from openapi.json by x client: never edit it, wrap it in a file of your own. -// @x-client {"command":"x client customer --out apps/web/clients/postly --only posts,query","source":"openapi.json","sourceHash":"32c95dbb3eca4c79dd983bd6e230aaa900bd74e2cafdfe696bf46a28f55a5872","bodyHash":"a39926998352f3c5046bd54fb96051dfdad943bef737dbe3098db5b62fb07e83"} +// @x-client {"command":"x client customer --out apps/web/clients/postly --only posts,query","source":"openapi.json","sourceHash":"5ff2b997a87912585bc5b56c5ac669f2411150fa618c7a0976d71c451fb111e8","bodyHash":"3f976fbb1a7280b4d52040e53d9ca54e6338e1a2d0b16e6d2ceacf6ef053671b"} // The types of @postly/root 0.0.1, one per schema the operations reach. export type CreatePostInput = { body: string; orgId: string; slug?: string | undefined; title: string }; export type CreatePostOutput = { authorId: string; authorName: string; body: string; coverUrl: string | null; excerpt: string; id: string; likeCount: number; orgId: string; publishedAt: string | null; slug: string; status: "draft" | "scheduled" | "published"; title: string }; export type LikePostInput = { orgId: string; postId: string }; export type LikePostOutput = { authorId: string; authorName: string; body: string; coverUrl: string | null; excerpt: string; id: string; likeCount: number; orgId: string; publishedAt: string | null; slug: string; status: "draft" | "scheduled" | "published"; title: string }; +export type PrintPostInput = { orgId: string; slug: string }; +export type PrintPostOutput = { bytes: number; contentType: string; key: string }; export type PublishPostInput = { notify?: boolean | undefined; orgId: string }; export type PublishPostOutput = { authorId: string; authorName: string; body: string; coverUrl: string | null; excerpt: string; id: string; likeCount: number; orgId: string; publishedAt: string | null; slug: string; status: "draft" | "scheduled" | "published"; title: string }; export type SummarizePostsInput = { orgId: string; postIds: string[] }; diff --git a/examples/dummy/apps/web/e2e/fixtures/postly.ts b/examples/dummy/apps/web/e2e/fixtures/postly.ts index 4258953f1..e7845eca9 100644 --- a/examples/dummy/apps/web/e2e/fixtures/postly.ts +++ b/examples/dummy/apps/web/e2e/fixtures/postly.ts @@ -7,15 +7,14 @@ import { expect } from 'bun:test'; import { seedId } from '@ultimat3/entity'; +import { findChrome, LAUNCHED_CLOSE_MS } from '@ultimat3/scraping/cdp'; import type { E2eApp, E2eSession } from '@ultimat3/testing'; import { allowHost, DEFAULT_CDP_TIMEOUT_MS, E2E_APP_START_MS, E2E_APP_STOP_MS, - E2E_BROWSER_CLOSE_MS, E2E_BROWSER_OPEN_MS, - findChrome, openE2eBrowser, startE2eApp, } from '@ultimat3/testing'; @@ -50,7 +49,7 @@ export const SIGN_IN_MS = DEFAULT_CDP_TIMEOUT_MS; * bounded stop. Left at Bun's 5 s default, a close cut short by the run's end leaves the Chrome * profile half-removed in the temp root. A suite that closes a tab first adds one call. */ -export const ACCEPTANCE_CLOSE_MS = E2E_BROWSER_CLOSE_MS + E2E_APP_STOP_MS; +export const ACCEPTANCE_CLOSE_MS = LAUNCHED_CLOSE_MS + E2E_APP_STOP_MS; /** The demo sign-in (`app/auth/demo-actor.ts`): a cookie naming one of four seeded members. */ export const DEMO_MEMBER_COOKIE = 'postly_demo_member'; diff --git a/examples/dummy/apps/web/exercise.ts b/examples/dummy/apps/web/exercise.ts index daeabef6c..a4a8b19a5 100644 --- a/examples/dummy/apps/web/exercise.ts +++ b/examples/dummy/apps/web/exercise.ts @@ -22,6 +22,9 @@ const author = userActor({ id: 'author', roles: ['author'] }); /** The one service a server's boot installs that `x exercise` cannot: a real model behind `llm()`. */ const NO_GATEWAY = 'calls a real LLM: x exercise installs no AI gateway'; +/** `pdf()` prints through Chrome over CDP, which the pass neither launches nor fakes. */ +const NO_BROWSER = 'prints through Chrome: x exercise launches no browser'; + /** * The passkey ceremonies sign what the SERVER issued to one actor, which no draw can forge: each * input is computed from the pass's own state, inside the request the call runs in — so the actor @@ -164,6 +167,20 @@ export const exercise = defineExercise( }), }, + // A posting nets to zero per currency, which no schema can say and a draw never does: one + // charge, cash in and revenue out, keyed per org so each tenant's pass posts its own. + 'billing.post-charge': { + actor: null, + input: (drawn) => ({ + ...drawn, + key: `ch_exercise_${String(drawn['orgId'])}`, + entries: [ + { account: 'cash', amount: { minor: 1900, currency: 'USD' } }, + { account: 'revenue', amount: { minor: -1900, currency: 'USD' } }, + ], + }), + }, + answerPrompt: { off: 'answers a prompt a live run is waiting on; only runs.sync writes one' }, cancelRun: { off: 'cancels the queued job behind a run; the seed writes the run row alone' }, confirmAgentPublish: { @@ -280,6 +297,7 @@ export const exercise = defineExercise( reviewDraft: { off: NO_GATEWAY }, summarize: { off: NO_GATEWAY }, 'posts.review': { off: NO_GATEWAY }, + printPost: { off: NO_BROWSER }, }, { actor: members }, ); diff --git a/examples/dummy/openapi.json b/examples/dummy/openapi.json index 88a1a872b..d723afc24 100644 --- a/examples/dummy/openapi.json +++ b/examples/dummy/openapi.json @@ -1407,6 +1407,49 @@ ], "type": "object" }, + "PrintPostInput": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "orgId": { + "format": "uuid", + "type": "string" + }, + "slug": { + "format": "slug", + "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$", + "type": "string" + } + }, + "required": [ + "orgId", + "slug" + ], + "type": "object" + }, + "PrintPostOutput": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "bytes": { + "type": "number" + }, + "contentType": { + "minLength": 1, + "type": "string" + }, + "key": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "key", + "bytes", + "contentType" + ], + "type": "object" + }, "Problem": { "properties": { "cause": { @@ -7702,6 +7745,75 @@ } } }, + "/api/posts/print": { + "post": { + "operationId": "printPost", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PrintPostInput" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PrintPostOutput" + } + } + }, + "description": "ok" + }, + "400": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "X_INPUT_INVALID" + }, + "403": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "policy denied" + }, + "422": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "X_BODY_INVALID" + } + }, + "summary": "printPost", + "tags": [ + "posts" + ], + "x-ultimate": { + "capability": "post:export", + "idempotent": false, + "invalidates": [], + "mcpTool": null, + "rateLimit": null + } + } + }, "/api/posts/summarize": { "post": { "operationId": "summarizePosts", diff --git a/examples/dummy/packages/db/migrations.test.ts b/examples/dummy/packages/db/migrations.test.ts index 593c9b124..9dc646a64 100644 --- a/examples/dummy/packages/db/migrations.test.ts +++ b/examples/dummy/packages/db/migrations.test.ts @@ -202,7 +202,8 @@ function walk(statements: readonly string[]): ChainState { continue; } const index = statement.match( - /^create (unique )?index (?:concurrently )?(?:if not exists )?"?(\w+)"? on "?(\w+)"? ?\(([^)]*)\)/i, + // `using gin` (a `.searchable()` vector's index) names the method between table and columns. + /^create (unique )?index (?:concurrently )?(?:if not exists )?"?(\w+)"? on "?(\w+)"? (?:using \w+ )?\(([^)]*)\)/i, ); if (index !== null) { indexes.set(index[2] ?? '', { @@ -235,11 +236,13 @@ function walk(statements: readonly string[]): ChainState { // Everything before an inline column CHECK is the type and its modifiers; the predicate // itself says `is null`, which a nullability test run over the whole tail would misread. const rest = (column[3] ?? '').split(/\s+check\s*\(/i)[0] ?? ''; + // A generated column's expression is not its type: `tsvector generated always as (…) stored`. + const typed = rest.split(/\s+generated\s+always\b/i)[0] ?? ''; const fallback = rest.match(/\bdefault\s+(\S+)/i); added.push({ table: column[1] ?? '', name: column[2] ?? '', - dataType: (rest.split(/\s+(?:not null|null|default)\b/i)[0] ?? '').trim(), + dataType: (typed.split(/\s+(?:not null|null|default)\b/i)[0] ?? '').trim(), nullable: !/\bnot null\b/i.test(rest), fallback: fallback === null ? null : (fallback[1] ?? ''), }); @@ -340,6 +343,7 @@ describe('the newest migration records the schema this chain creates', () => { 'comments', 'connections', 'daily_likes', + 'ledger_lines', 'likes', 'members', 'orgs', diff --git a/examples/dummy/packages/db/migrations/20261110045219_ledger_lines_and_post_title_search.hash b/examples/dummy/packages/db/migrations/20261110045219_ledger_lines_and_post_title_search.hash new file mode 100644 index 000000000..441d81893 --- /dev/null +++ b/examples/dummy/packages/db/migrations/20261110045219_ledger_lines_and_post_title_search.hash @@ -0,0 +1 @@ +bb2de7a369292a58 diff --git a/examples/dummy/packages/db/migrations/20261110045219_ledger_lines_and_post_title_search.snapshot.json b/examples/dummy/packages/db/migrations/20261110045219_ledger_lines_and_post_title_search.snapshot.json new file mode 100644 index 000000000..c961ce978 --- /dev/null +++ b/examples/dummy/packages/db/migrations/20261110045219_ledger_lines_and_post_title_search.snapshot.json @@ -0,0 +1,1871 @@ +{ + "tables": [ + { + "schema": "public", + "name": "comments", + "columns": [ + { + "name": "author_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 1 + }, + { + "name": "body", + "dataType": "text", + "nullable": false, + "default": null, + "position": 2 + }, + { + "name": "created_at", + "dataType": "timestamptz", + "nullable": false, + "default": "now()", + "position": 3 + }, + { + "name": "id", + "dataType": "uuid", + "nullable": false, + "default": "gen_random_uuid()", + "position": 4 + }, + { + "name": "org_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 5 + }, + { + "name": "post_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 6 + } + ], + "primaryKey": ["id"], + "indexes": [ + { + "name": "comments_post_id_idx", + "columns": ["post_id"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "comments_author_id_idx", + "columns": ["author_id"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "comments_post_id_created_at_idx", + "columns": ["post_id", "created_at"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "comments_org_id_id_idx", + "columns": ["org_id", "id"], + "unique": false, + "primary": false, + "where": null, + "order": null + } + ], + "foreignKeys": [ + { + "name": "comments_author_id_tenant_fkey", + "columns": ["org_id", "author_id"], + "referencedTable": "members", + "referencedColumns": ["org_id", "id"], + "onDelete": "restrict" + }, + { + "name": "comments_org_id_fkey", + "columns": ["org_id"], + "referencedTable": "orgs", + "referencedColumns": ["id"], + "onDelete": "cascade" + }, + { + "name": "comments_post_id_tenant_fkey", + "columns": ["org_id", "post_id"], + "referencedTable": "posts", + "referencedColumns": ["org_id", "id"], + "onDelete": "cascade" + } + ], + "checks": [ + { + "name": "comments_body_check", + "expression": "char_length(body) <= 2000" + }, + { + "name": "comments_comment_body_present_check", + "expression": "char_length(btrim(body)) >= 1" + } + ] + }, + { + "schema": "public", + "name": "connections", + "columns": [ + { + "name": "created_at", + "dataType": "timestamptz", + "nullable": false, + "default": "now()", + "position": 1 + }, + { + "name": "credential", + "dataType": "text", + "nullable": false, + "default": null, + "position": 2 + }, + { + "name": "exit", + "dataType": "text", + "nullable": true, + "default": null, + "position": 3 + }, + { + "name": "id", + "dataType": "uuid", + "nullable": false, + "default": "gen_random_uuid()", + "position": 4 + }, + { + "name": "label", + "dataType": "text", + "nullable": false, + "default": null, + "position": 5 + }, + { + "name": "org_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 6 + } + ], + "primaryKey": ["id"], + "indexes": [ + { + "name": "connections_org_id_created_at_idx", + "columns": ["org_id", "created_at"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "connections_org_id_id_tenant_key", + "columns": ["org_id", "id"], + "unique": true, + "primary": false, + "where": null, + "order": null + } + ], + "foreignKeys": [ + { + "name": "connections_org_id_fkey", + "columns": ["org_id"], + "referencedTable": "orgs", + "referencedColumns": ["id"], + "onDelete": "cascade" + } + ], + "checks": [ + { + "name": "connections_label_check", + "expression": "char_length(label) <= 80" + }, + { + "name": "connections_connection_label_present_check", + "expression": "char_length(btrim(label)) >= 1" + } + ] + }, + { + "schema": "public", + "name": "daily_likes", + "columns": [ + { + "name": "day", + "dataType": "timestamptz", + "nullable": false, + "default": null, + "position": 1 + }, + { + "name": "likes", + "dataType": "integer", + "nullable": false, + "default": null, + "position": 2 + } + ], + "primaryKey": ["day"], + "indexes": [], + "foreignKeys": [] + }, + { + "schema": "public", + "name": "ledger_lines", + "columns": [ + { + "name": "account", + "dataType": "text", + "nullable": false, + "default": null, + "position": 1 + }, + { + "name": "amount_currency", + "dataType": "char(3)", + "nullable": false, + "default": null, + "position": 2 + }, + { + "name": "amount_minor", + "dataType": "bigint", + "nullable": false, + "default": null, + "position": 3 + }, + { + "name": "amount_scale", + "dataType": "integer", + "nullable": true, + "default": null, + "position": 4 + }, + { + "name": "line", + "dataType": "integer", + "nullable": false, + "default": null, + "position": 5 + }, + { + "name": "org_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 6 + }, + { + "name": "posting_key", + "dataType": "text", + "nullable": false, + "default": null, + "position": 7 + } + ], + "primaryKey": ["org_id", "posting_key", "line"], + "indexes": [], + "foreignKeys": [ + { + "name": "ledger_lines_org_id_fkey", + "columns": ["org_id"], + "referencedTable": "orgs", + "referencedColumns": ["id"], + "onDelete": "cascade" + } + ], + "checks": [ + { + "name": "ledger_lines_posting_key_check", + "expression": "char_length(posting_key) <= 120" + }, + { + "name": "ledger_lines_account_check", + "expression": "char_length(account) <= 60" + }, + { + "name": "ledger_lines_amount_currency_check", + "expression": "amount_currency ~ '^[A-Z]{3}$'" + }, + { + "name": "ledger_lines_amount_scale_check", + "expression": "amount_scale is null or (amount_scale >= 0 and amount_scale <= 15)" + } + ] + }, + { + "schema": "public", + "name": "likes", + "columns": [ + { + "name": "created_at", + "dataType": "timestamptz", + "nullable": false, + "default": "now()", + "position": 1 + }, + { + "name": "member_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 2 + }, + { + "name": "org_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 3 + }, + { + "name": "post_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 4 + } + ], + "primaryKey": ["post_id", "member_id"], + "indexes": [ + { + "name": "likes_post_id_idx", + "columns": ["post_id"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "likes_member_id_idx", + "columns": ["member_id"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "likes_org_id_post_id_member_id_idx", + "columns": ["org_id", "post_id", "member_id"], + "unique": false, + "primary": false, + "where": null, + "order": null + } + ], + "foreignKeys": [ + { + "name": "likes_member_id_tenant_fkey", + "columns": ["org_id", "member_id"], + "referencedTable": "members", + "referencedColumns": ["org_id", "id"], + "onDelete": "cascade" + }, + { + "name": "likes_org_id_fkey", + "columns": ["org_id"], + "referencedTable": "orgs", + "referencedColumns": ["id"], + "onDelete": "cascade" + }, + { + "name": "likes_post_id_tenant_fkey", + "columns": ["org_id", "post_id"], + "referencedTable": "posts", + "referencedColumns": ["org_id", "id"], + "onDelete": "cascade" + } + ] + }, + { + "schema": "public", + "name": "members", + "columns": [ + { + "name": "created_at", + "dataType": "timestamptz", + "nullable": false, + "default": "now()", + "position": 1 + }, + { + "name": "digest_opt_in", + "dataType": "boolean", + "nullable": false, + "default": "true", + "position": 2 + }, + { + "name": "email", + "dataType": "text", + "nullable": false, + "default": null, + "position": 3 + }, + { + "name": "id", + "dataType": "uuid", + "nullable": false, + "default": "gen_random_uuid()", + "position": 4 + }, + { + "name": "locale", + "dataType": "text", + "nullable": false, + "default": "'en'", + "position": 5 + }, + { + "name": "name", + "dataType": "text", + "nullable": false, + "default": null, + "position": 6 + }, + { + "name": "org_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 7 + }, + { + "name": "role", + "dataType": "text", + "nullable": false, + "default": "'author'", + "position": 8 + }, + { + "name": "theme", + "dataType": "text", + "nullable": false, + "default": "'system'", + "position": 9 + }, + { + "name": "tz", + "dataType": "text", + "nullable": false, + "default": "'UTC'", + "position": 10 + }, + { + "name": "user_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 11 + } + ], + "primaryKey": ["id"], + "indexes": [ + { + "name": "members_org_id_role_idx", + "columns": ["org_id", "role"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "members_tz_8dcfa966_idx", + "columns": ["tz"], + "unique": false, + "primary": false, + "where": "digest_opt_in", + "order": null + }, + { + "name": "members_digest_opt_in_org_id_id_idx", + "columns": ["digest_opt_in", "org_id", "id"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "members_org_id_id_tenant_key", + "columns": ["org_id", "id"], + "unique": true, + "primary": false, + "where": null, + "order": null + }, + { + "name": "members_member_unique_per_org_key", + "columns": ["org_id", "user_id"], + "unique": true, + "primary": false, + "where": null, + "order": null + } + ], + "foreignKeys": [ + { + "name": "members_org_id_fkey", + "columns": ["org_id"], + "referencedTable": "orgs", + "referencedColumns": ["id"], + "onDelete": "cascade" + } + ], + "checks": [ + { + "name": "members_email_check", + "expression": "char_length(email) <= 254" + }, + { + "name": "members_name_check", + "expression": "char_length(name) <= 80" + }, + { + "name": "members_role_check", + "expression": "role in ('owner', 'admin', 'author', 'reader')" + }, + { + "name": "members_tz_check", + "expression": "tz in ('UTC', 'America/New_York', 'America/Sao_Paulo', 'Europe/Madrid', 'Europe/Berlin', 'Asia/Tokyo', 'Australia/Sydney', 'Pacific/Auckland')" + }, + { + "name": "members_locale_check", + "expression": "locale in ('en', 'es')" + }, + { + "name": "members_theme_check", + "expression": "theme in ('system', 'light', 'dark')" + }, + { + "name": "members_member_email_shape_check", + "expression": "email ~ '^[^@]+@[^@]+$'" + } + ] + }, + { + "schema": "public", + "name": "orgs", + "columns": [ + { + "name": "billing_currency", + "dataType": "text", + "nullable": false, + "default": "'USD'", + "position": 1 + }, + { + "name": "created_at", + "dataType": "timestamptz", + "nullable": false, + "default": "now()", + "position": 2 + }, + { + "name": "id", + "dataType": "uuid", + "nullable": false, + "default": "gen_random_uuid()", + "position": 3 + }, + { + "name": "name", + "dataType": "text", + "nullable": false, + "default": null, + "position": 4 + }, + { + "name": "plan_code", + "dataType": "text", + "nullable": false, + "default": "'free'", + "position": 5 + }, + { + "name": "slug", + "dataType": "text", + "nullable": false, + "default": null, + "position": 6 + }, + { + "name": "updated_at", + "dataType": "timestamptz", + "nullable": false, + "default": "now()", + "position": 7 + } + ], + "primaryKey": ["id"], + "indexes": [ + { + "name": "orgs_slug_key", + "columns": ["slug"], + "unique": true, + "primary": false, + "where": null, + "order": null + }, + { + "name": "orgs_plan_code_idx", + "columns": ["plan_code"], + "unique": false, + "primary": false, + "where": null, + "order": null + } + ], + "foreignKeys": [], + "checks": [ + { + "name": "orgs_slug_check", + "expression": "char_length(slug) <= 80" + }, + { + "name": "orgs_name_check", + "expression": "char_length(name) <= 80" + }, + { + "name": "orgs_plan_code_check", + "expression": "plan_code in ('free', 'team', 'business')" + }, + { + "name": "orgs_billing_currency_check", + "expression": "billing_currency in ('USD', 'EUR')" + }, + { + "name": "orgs_org_slug_shape_check", + "expression": "slug ~ '^[a-z0-9]+(?:-[a-z0-9]+)*$'" + }, + { + "name": "orgs_org_name_present_check", + "expression": "char_length(btrim(name)) >= 1" + } + ] + }, + { + "schema": "public", + "name": "plans", + "columns": [ + { + "name": "code", + "dataType": "text", + "nullable": false, + "default": null, + "position": 1 + }, + { + "name": "currency", + "dataType": "text", + "nullable": false, + "default": null, + "position": 2 + }, + { + "name": "monthly_currency", + "dataType": "char(3)", + "nullable": false, + "default": null, + "position": 3 + }, + { + "name": "monthly_minor", + "dataType": "bigint", + "nullable": false, + "default": null, + "position": 4 + }, + { + "name": "monthly_scale", + "dataType": "integer", + "nullable": true, + "default": null, + "position": 5 + }, + { + "name": "seats", + "dataType": "integer", + "nullable": false, + "default": null, + "position": 6 + }, + { + "name": "tagline", + "dataType": "text", + "nullable": true, + "default": null, + "position": 7, + "retired": true + } + ], + "primaryKey": ["code", "currency"], + "indexes": [], + "foreignKeys": [], + "checks": [ + { + "name": "plans_code_check", + "expression": "code in ('free', 'team', 'business')" + }, + { + "name": "plans_currency_check", + "expression": "currency in ('USD', 'EUR')" + }, + { + "name": "plans_monthly_currency_check", + "expression": "monthly_currency ~ '^[A-Z]{3}$'" + }, + { + "name": "plans_monthly_scale_check", + "expression": "monthly_scale is null or (monthly_scale >= 0 and monthly_scale <= 15)" + }, + { + "name": "plans_plan_price_non_negative_check", + "expression": "monthly_minor >= 0" + }, + { + "name": "plans_plan_currency_matches_price_check", + "expression": "monthly_currency = currency" + }, + { + "name": "plans_plan_seats_positive_check", + "expression": "seats >= 1" + } + ] + }, + { + "schema": "public", + "name": "post_review_runs", + "columns": [ + { + "name": "checkpoint", + "dataType": "jsonb", + "nullable": true, + "default": null, + "position": 1 + }, + { + "name": "draft", + "dataType": "text", + "nullable": false, + "default": null, + "position": 2 + }, + { + "name": "opened_at", + "dataType": "timestamptz", + "nullable": false, + "default": "now()", + "position": 3 + }, + { + "name": "org_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 4 + }, + { + "name": "post_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 5 + } + ], + "primaryKey": ["org_id", "post_id"], + "indexes": [ + { + "name": "post_review_runs_post_id_idx", + "columns": ["post_id"], + "unique": false, + "primary": false, + "where": null, + "order": null + } + ], + "foreignKeys": [ + { + "name": "post_review_runs_org_id_fkey", + "columns": ["org_id"], + "referencedTable": "orgs", + "referencedColumns": ["id"], + "onDelete": "cascade" + }, + { + "name": "post_review_runs_post_id_tenant_fkey", + "columns": ["org_id", "post_id"], + "referencedTable": "posts", + "referencedColumns": ["org_id", "id"], + "onDelete": "cascade" + } + ], + "checks": [ + { + "name": "post_review_runs_draft_check", + "expression": "char_length(draft) <= 64" + } + ] + }, + { + "schema": "public", + "name": "post_reviews", + "columns": [ + { + "name": "notes", + "dataType": "text", + "nullable": false, + "default": null, + "position": 1 + }, + { + "name": "org_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 2 + }, + { + "name": "post_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 3 + }, + { + "name": "reviewed_at", + "dataType": "timestamptz", + "nullable": false, + "default": "now()", + "position": 4 + }, + { + "name": "reviewed_by", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 5 + }, + { + "name": "verdict", + "dataType": "text", + "nullable": false, + "default": null, + "position": 6 + } + ], + "primaryKey": ["org_id", "post_id"], + "indexes": [ + { + "name": "post_reviews_post_id_idx", + "columns": ["post_id"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "post_reviews_reviewed_by_idx", + "columns": ["reviewed_by"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "post_reviews_org_id_reviewed_at_idx", + "columns": ["org_id", "reviewed_at"], + "unique": false, + "primary": false, + "where": null, + "order": null + } + ], + "foreignKeys": [ + { + "name": "post_reviews_org_id_fkey", + "columns": ["org_id"], + "referencedTable": "orgs", + "referencedColumns": ["id"], + "onDelete": "cascade" + }, + { + "name": "post_reviews_post_id_tenant_fkey", + "columns": ["org_id", "post_id"], + "referencedTable": "posts", + "referencedColumns": ["org_id", "id"], + "onDelete": "cascade" + }, + { + "name": "post_reviews_reviewed_by_tenant_fkey", + "columns": ["org_id", "reviewed_by"], + "referencedTable": "members", + "referencedColumns": ["org_id", "id"], + "onDelete": "cascade" + } + ], + "checks": [ + { + "name": "post_reviews_verdict_check", + "expression": "verdict in ('ready', 'revise')" + }, + { + "name": "post_reviews_notes_check", + "expression": "char_length(notes) <= 600" + } + ] + }, + { + "schema": "public", + "name": "posts", + "columns": [ + { + "name": "author_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 1 + }, + { + "name": "body", + "dataType": "text", + "nullable": false, + "default": null, + "position": 2 + }, + { + "name": "cover_url", + "dataType": "text", + "nullable": true, + "default": null, + "position": 3 + }, + { + "name": "created_at", + "dataType": "timestamptz", + "nullable": false, + "default": "now()", + "position": 4 + }, + { + "name": "excerpt", + "dataType": "text", + "nullable": false, + "default": null, + "position": 5 + }, + { + "name": "id", + "dataType": "uuid", + "nullable": false, + "default": "gen_random_uuid()", + "position": 6 + }, + { + "name": "like_count", + "dataType": "integer", + "nullable": false, + "default": "0", + "position": 7 + }, + { + "name": "org_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 8 + }, + { + "name": "published_at", + "dataType": "timestamptz", + "nullable": true, + "default": null, + "position": 9 + }, + { + "name": "search_tsv", + "dataType": "tsvector", + "nullable": false, + "default": null, + "position": 10, + "generated": "setweight(to_tsvector('english', coalesce(\"title\", '')), 'A')" + }, + { + "name": "slug", + "dataType": "text", + "nullable": false, + "default": null, + "position": 11 + }, + { + "name": "status", + "dataType": "text", + "nullable": false, + "default": "'draft'", + "position": 12 + }, + { + "name": "title", + "dataType": "text", + "nullable": false, + "default": null, + "position": 13 + }, + { + "name": "updated_at", + "dataType": "timestamptz", + "nullable": false, + "default": "now()", + "position": 14 + } + ], + "primaryKey": ["id"], + "indexes": [ + { + "name": "posts_author_id_idx", + "columns": ["author_id"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "posts_org_id_published_at_647c63aa_idx", + "columns": ["org_id", "published_at"], + "unique": false, + "primary": false, + "where": "status = 'published'", + "order": "desc" + }, + { + "name": "posts_org_id_created_at_id_idx", + "columns": ["org_id", "created_at", "id"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "posts_status_published_at_id_idx", + "columns": ["status", "published_at", "id"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "posts_search_tsv_879c084e_idx", + "columns": ["search_tsv"], + "unique": false, + "primary": false, + "where": null, + "order": null, + "using": "gin" + }, + { + "name": "posts_org_id_id_tenant_key", + "columns": ["org_id", "id"], + "unique": true, + "primary": false, + "where": null, + "order": null + }, + { + "name": "posts_post_slug_unique_key", + "columns": ["slug"], + "unique": true, + "primary": false, + "where": null, + "order": null + } + ], + "foreignKeys": [ + { + "name": "posts_author_id_tenant_fkey", + "columns": ["org_id", "author_id"], + "referencedTable": "members", + "referencedColumns": ["org_id", "id"], + "onDelete": "restrict" + }, + { + "name": "posts_org_id_fkey", + "columns": ["org_id"], + "referencedTable": "orgs", + "referencedColumns": ["id"], + "onDelete": "cascade" + } + ], + "checks": [ + { + "name": "posts_slug_check", + "expression": "char_length(slug) <= 80" + }, + { + "name": "posts_title_check", + "expression": "char_length(title) <= 120" + }, + { + "name": "posts_excerpt_check", + "expression": "char_length(excerpt) <= 200" + }, + { + "name": "posts_cover_url_check", + "expression": "cover_url ~ '^https?://'" + }, + { + "name": "posts_status_check", + "expression": "status in ('draft', 'scheduled', 'published')" + }, + { + "name": "posts_post_slug_shape_check", + "expression": "slug ~ '^[a-z0-9]+(?:-[a-z0-9]+)*$'" + }, + { + "name": "posts_post_like_count_non_negative_check", + "expression": "like_count >= 0" + }, + { + "name": "posts_post_publish_coherent_check", + "expression": "(status = 'published') = (published_at is not null)" + } + ], + "replicaIdentityFull": true, + "triggers": [ + { + "name": "ultimate_history", + "install": [ + "create or replace function \"ultimate_record_history\"() returns trigger language plpgsql as $history$ declare source jsonb := case when tg_op = 'DELETE' then to_jsonb(old) else to_jsonb(new) end; prior jsonb := case when tg_op = 'INSERT' then null else to_jsonb(old) end; ignored text[] := tg_argv[3]::text[]; digits text[] := coalesce(tg_argv[4], '{}')::text[]; instants text[] := coalesce(tg_argv[5], '{}')::text[]; k text; begin if tg_op = 'UPDATE' and (prior - ignored) = (source - ignored) then return null; end if; foreach k in array digits loop if jsonb_typeof(source -> k) = 'number' then source := jsonb_set(source, array[k], coalesce(to_jsonb(source ->> k), source -> k)); end if; if jsonb_typeof(prior -> k) = 'number' then prior := jsonb_set(prior, array[k], coalesce(to_jsonb(prior ->> k), prior -> k)); end if; end loop; foreach k in array instants loop if jsonb_typeof(source -> k) = 'string' then source := jsonb_set(source, array[k], coalesce(to_jsonb(to_char((source ->> k)::timestamptz at time zone 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')), source -> k)); end if; if jsonb_typeof(prior -> k) = 'string' then prior := jsonb_set(prior, array[k], coalesce(to_jsonb(to_char((prior ->> k)::timestamptz at time zone 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')), prior -> k)); end if; end loop; execute format('insert into %I select * from jsonb_populate_record(null::%I, $1)', tg_argv[0], tg_argv[0]) using jsonb_build_object('id', gen_random_uuid(), 'row_id', source ->> tg_argv[1], 'op', lower(tg_op), 'at', now(), 'actor_id', nullif(current_setting('x.actor', true), ''), 'request_id', nullif(current_setting('x.request', true), ''), 'before', prior, 'after', case when tg_op = 'DELETE' then null else source end) || case when tg_argv[2] = '' then '{}'::jsonb else jsonb_build_object(tg_argv[2], source -> tg_argv[2]) end; return null; end; $history$;", + "drop trigger if exists \"ultimate_history\" on \"posts\";", + "create trigger \"ultimate_history\" after insert or update or delete on \"posts\" for each row execute function \"ultimate_record_history\"('posts_history', 'id', 'org_id', '{}', '{}', '{\"published_at\",\"created_at\",\"updated_at\"}');" + ] + } + ] + }, + { + "schema": "public", + "name": "posts_history", + "columns": [ + { + "name": "actor_id", + "dataType": "text", + "nullable": true, + "default": null, + "position": 1 + }, + { + "name": "after", + "dataType": "jsonb", + "nullable": true, + "default": null, + "position": 2 + }, + { + "name": "at", + "dataType": "timestamptz", + "nullable": false, + "default": "now()", + "position": 3 + }, + { + "name": "before", + "dataType": "jsonb", + "nullable": true, + "default": null, + "position": 4 + }, + { + "name": "id", + "dataType": "uuid", + "nullable": false, + "default": "gen_random_uuid()", + "position": 5 + }, + { + "name": "op", + "dataType": "text", + "nullable": false, + "default": null, + "position": 6 + }, + { + "name": "org_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 7 + }, + { + "name": "request_id", + "dataType": "text", + "nullable": true, + "default": null, + "position": 8 + }, + { + "name": "row_id", + "dataType": "text", + "nullable": false, + "default": null, + "position": 9 + } + ], + "primaryKey": ["id"], + "indexes": [ + { + "name": "posts_history_org_id_row_id_at_idx", + "columns": ["org_id", "row_id", "at"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "posts_history_at_idx", + "columns": ["at"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "posts_history_org_id_id_idx", + "columns": ["org_id", "id"], + "unique": false, + "primary": false, + "where": null, + "order": null + } + ], + "foreignKeys": [], + "triggers": [ + { + "name": "ultimate_history_guard", + "install": [ + "create or replace function \"ultimate_guard_history\"() returns trigger language plpgsql as $history_guard$ begin if tg_op = 'DELETE' and current_setting('x.purge', true) = tg_table_name then return old; end if; raise exception 'X_ENTITY_APPEND_ONLY: % on %.% is refused, the table is history', tg_op, tg_table_schema, tg_table_name using errcode = '23001', hint = 'history is written by its trigger and removed only by purge({ entities: ''declared'' })'; end; $history_guard$;", + "drop trigger if exists \"ultimate_history_guard\" on \"posts_history\";", + "create trigger \"ultimate_history_guard\" before update or delete on \"posts_history\" for each row execute function \"ultimate_guard_history\"();" + ] + } + ], + "retain": true + }, + { + "schema": "public", + "name": "run_events", + "columns": [ + { + "name": "at", + "dataType": "timestamptz", + "nullable": false, + "default": "now()", + "position": 1 + }, + { + "name": "id", + "dataType": "uuid", + "nullable": false, + "default": "gen_random_uuid()", + "position": 2 + }, + { + "name": "kind", + "dataType": "text", + "nullable": false, + "default": null, + "position": 3 + }, + { + "name": "message", + "dataType": "text", + "nullable": false, + "default": null, + "position": 4 + }, + { + "name": "org_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 5 + }, + { + "name": "prompt", + "dataType": "integer", + "nullable": true, + "default": null, + "position": 6 + }, + { + "name": "run_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 7 + }, + { + "name": "seq", + "dataType": "integer", + "nullable": false, + "default": null, + "position": 8 + }, + { + "name": "usage", + "dataType": "jsonb", + "nullable": true, + "default": null, + "position": 9 + } + ], + "primaryKey": ["id"], + "indexes": [ + { + "name": "run_events_run_id_idx", + "columns": ["run_id"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "run_events_org_id_run_id_seq_idx", + "columns": ["org_id", "run_id", "seq"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "run_events_at_idx", + "columns": ["at"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "run_events_org_id_id_idx", + "columns": ["org_id", "id"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "run_events_run_event_seq_unique_key", + "columns": ["run_id", "seq"], + "unique": true, + "primary": false, + "where": null, + "order": null + } + ], + "foreignKeys": [ + { + "name": "run_events_org_id_fkey", + "columns": ["org_id"], + "referencedTable": "orgs", + "referencedColumns": ["id"], + "onDelete": "cascade" + }, + { + "name": "run_events_run_id_tenant_fkey", + "columns": ["org_id", "run_id"], + "referencedTable": "runs", + "referencedColumns": ["org_id", "id"], + "onDelete": "cascade" + } + ], + "checks": [ + { + "name": "run_events_kind_check", + "expression": "kind in ('prompt', 'answered', 'navigated', 'extracted', 'done', 'failed', 'usage')" + }, + { + "name": "run_events_message_check", + "expression": "char_length(message) <= 200" + }, + { + "name": "run_events_run_event_seq_from_one_check", + "expression": "seq >= 1" + } + ], + "replicaIdentityFull": true, + "triggers": [ + { + "name": "ultimate_count_9f9b3faf", + "install": [ + "create or replace function \"ultimate_count_9f9b3faf\"() returns trigger language plpgsql as $counter$ begin if tg_op <> 'INSERT' and old.\"run_id\" is not null then update \"runs\" set \"event_count\" = \"event_count\" - 1 where \"id\" = old.\"run_id\"; end if; if tg_op <> 'DELETE' and new.\"run_id\" is not null then update \"runs\" set \"event_count\" = \"event_count\" + 1 where \"id\" = new.\"run_id\"; end if; return null; end; $counter$;", + "drop trigger if exists \"ultimate_count_9f9b3faf\" on \"run_events\";", + "create trigger \"ultimate_count_9f9b3faf\" after insert or delete or update of \"run_id\" on \"run_events\" for each row execute function \"ultimate_count_9f9b3faf\"();" + ] + } + ], + "retain": true + }, + { + "schema": "public", + "name": "runs", + "columns": [ + { + "name": "code", + "dataType": "text", + "nullable": true, + "default": null, + "position": 1 + }, + { + "name": "connection_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 2 + }, + { + "name": "event_count", + "dataType": "integer", + "nullable": false, + "default": "0", + "position": 3 + }, + { + "name": "id", + "dataType": "uuid", + "nullable": false, + "default": "gen_random_uuid()", + "position": 4 + }, + { + "name": "job_id", + "dataType": "text", + "nullable": false, + "default": null, + "position": 5 + }, + { + "name": "org_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 6 + }, + { + "name": "started_at", + "dataType": "timestamptz", + "nullable": false, + "default": "now()", + "position": 7 + }, + { + "name": "status", + "dataType": "text", + "nullable": false, + "default": null, + "position": 8 + } + ], + "primaryKey": ["id"], + "indexes": [ + { + "name": "runs_connection_id_idx", + "columns": ["connection_id"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "runs_org_id_started_at_idx", + "columns": ["org_id", "started_at"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "runs_org_id_status_idx", + "columns": ["org_id", "status"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "runs_org_id_id_tenant_key", + "columns": ["org_id", "id"], + "unique": true, + "primary": false, + "where": null, + "order": null + } + ], + "foreignKeys": [ + { + "name": "runs_connection_id_tenant_fkey", + "columns": ["org_id", "connection_id"], + "referencedTable": "connections", + "referencedColumns": ["org_id", "id"], + "onDelete": "cascade" + }, + { + "name": "runs_org_id_fkey", + "columns": ["org_id"], + "referencedTable": "orgs", + "referencedColumns": ["id"], + "onDelete": "cascade" + } + ], + "checks": [ + { + "name": "runs_job_id_check", + "expression": "char_length(job_id) <= 64" + }, + { + "name": "runs_status_check", + "expression": "status in ('queued', 'running', 'done', 'failed')" + }, + { + "name": "runs_code_check", + "expression": "char_length(code) <= 64" + } + ] + }, + { + "schema": "public", + "name": "stream_cursors", + "columns": [ + { + "name": "position", + "dataType": "integer", + "nullable": false, + "default": null, + "position": 1 + }, + { + "name": "stream", + "dataType": "text", + "nullable": false, + "default": null, + "position": 2 + } + ], + "primaryKey": ["stream"], + "indexes": [], + "foreignKeys": [], + "checks": [ + { + "name": "stream_cursors_stream_check", + "expression": "char_length(stream) <= 160" + } + ] + }, + { + "schema": "public", + "name": "webhook_deliveries", + "columns": [ + { + "name": "at", + "dataType": "timestamptz", + "nullable": false, + "default": null, + "position": 1 + }, + { + "name": "attempt", + "dataType": "integer", + "nullable": false, + "default": null, + "position": 2 + }, + { + "name": "consecutive_failures", + "dataType": "integer", + "nullable": false, + "default": null, + "position": 3 + }, + { + "name": "duration_ms", + "dataType": "integer", + "nullable": false, + "default": null, + "position": 4 + }, + { + "name": "endpoint_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 5 + }, + { + "name": "error", + "dataType": "text", + "nullable": true, + "default": null, + "position": 6 + }, + { + "name": "event_id", + "dataType": "text", + "nullable": false, + "default": null, + "position": 7 + }, + { + "name": "id", + "dataType": "uuid", + "nullable": false, + "default": "gen_random_uuid()", + "position": 8 + }, + { + "name": "ok", + "dataType": "boolean", + "nullable": false, + "default": null, + "position": 9 + }, + { + "name": "org_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 10 + }, + { + "name": "seq", + "dataType": "integer", + "nullable": false, + "default": null, + "position": 11 + }, + { + "name": "status", + "dataType": "integer", + "nullable": true, + "default": null, + "position": 12 + }, + { + "name": "topic", + "dataType": "text", + "nullable": false, + "default": null, + "position": 13 + }, + { + "name": "webhook", + "dataType": "text", + "nullable": false, + "default": null, + "position": 14 + } + ], + "primaryKey": ["id"], + "indexes": [ + { + "name": "webhook_deliveries_endpoint_id_idx", + "columns": ["endpoint_id"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "webhook_deliveries_org_id_endpoint_id_seq_idx", + "columns": ["org_id", "endpoint_id", "seq"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "webhook_deliveries_org_id_id_idx", + "columns": ["org_id", "id"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "webhook_deliveries_webhook_delivery_seq_unique_key", + "columns": ["endpoint_id", "seq"], + "unique": true, + "primary": false, + "where": null, + "order": null + } + ], + "foreignKeys": [ + { + "name": "webhook_deliveries_endpoint_id_tenant_fkey", + "columns": ["org_id", "endpoint_id"], + "referencedTable": "webhook_endpoints", + "referencedColumns": ["org_id", "id"], + "onDelete": "cascade" + }, + { + "name": "webhook_deliveries_org_id_fkey", + "columns": ["org_id"], + "referencedTable": "orgs", + "referencedColumns": ["id"], + "onDelete": "cascade" + } + ], + "checks": [ + { + "name": "webhook_deliveries_webhook_check", + "expression": "char_length(webhook) <= 80" + }, + { + "name": "webhook_deliveries_event_id_check", + "expression": "char_length(event_id) <= 128" + }, + { + "name": "webhook_deliveries_topic_check", + "expression": "char_length(topic) <= 80" + }, + { + "name": "webhook_deliveries_error_check", + "expression": "char_length(error) <= 500" + }, + { + "name": "webhook_deliveries_webhook_delivery_seq_from_one_check", + "expression": "seq >= 1" + } + ] + }, + { + "schema": "public", + "name": "webhook_endpoints", + "columns": [ + { + "name": "created_at", + "dataType": "timestamptz", + "nullable": false, + "default": "now()", + "position": 1 + }, + { + "name": "disabled_reason", + "dataType": "text", + "nullable": true, + "default": null, + "position": 2 + }, + { + "name": "id", + "dataType": "uuid", + "nullable": false, + "default": "gen_random_uuid()", + "position": 3 + }, + { + "name": "org_id", + "dataType": "uuid", + "nullable": false, + "default": null, + "position": 4 + }, + { + "name": "secret", + "dataType": "text", + "nullable": false, + "default": null, + "position": 5 + }, + { + "name": "slot", + "dataType": "text", + "nullable": false, + "default": null, + "position": 6 + }, + { + "name": "url", + "dataType": "text", + "nullable": false, + "default": null, + "position": 7 + } + ], + "primaryKey": ["id"], + "indexes": [ + { + "name": "webhook_endpoints_org_id_created_at_idx", + "columns": ["org_id", "created_at"], + "unique": false, + "primary": false, + "where": null, + "order": null + }, + { + "name": "webhook_endpoints_org_id_id_tenant_key", + "columns": ["org_id", "id"], + "unique": true, + "primary": false, + "where": null, + "order": null + }, + { + "name": "webhook_endpoints_webhook_endpoint_slot_unique_key", + "columns": ["org_id", "slot"], + "unique": true, + "primary": false, + "where": null, + "order": null + } + ], + "foreignKeys": [ + { + "name": "webhook_endpoints_org_id_fkey", + "columns": ["org_id"], + "referencedTable": "orgs", + "referencedColumns": ["id"], + "onDelete": "cascade" + } + ], + "checks": [ + { + "name": "webhook_endpoints_slot_check", + "expression": "slot in ('1', '2', '3', '4', '5', '6', '7', '8', '9', '10')" + }, + { + "name": "webhook_endpoints_url_check", + "expression": "url ~ '^https?://'" + }, + { + "name": "webhook_endpoints_disabled_reason_check", + "expression": "char_length(disabled_reason) <= 200" + } + ] + } + ], + "views": [ + { + "name": "post_like_totals", + "materialized": true, + "create": [ + "create materialized view \"post_like_totals\" as select org_id, post_id, count(*)::integer as likes from likes group by org_id, post_id;", + "create unique index \"post_like_totals_post_id_key\" on \"post_like_totals\" (\"post_id\");", + "create index \"post_like_totals_org_id_post_id_idx\" on \"post_like_totals\" (\"org_id\", \"post_id\");" + ] + } + ] +} diff --git a/examples/dummy/packages/db/migrations/20261110045219_ledger_lines_and_post_title_search.sql b/examples/dummy/packages/db/migrations/20261110045219_ledger_lines_and_post_title_search.sql new file mode 100644 index 000000000..95daf70e5 --- /dev/null +++ b/examples/dummy/packages/db/migrations/20261110045219_ledger_lines_and_post_title_search.sql @@ -0,0 +1,27 @@ +-- 20261110045219_ledger_lines_and_post_title_search +-- GENERATED by `x db gen` from the app's entities — do not edit. +-- Editing an applied migration changes its checksum: X_MIGRATION_CONFLICT on the next apply. + +create table "ledger_lines" ( + "org_id" uuid not null, + "posting_key" text not null, + "line" integer not null, + "account" text not null, + "amount_minor" bigint not null, + "amount_currency" char(3) not null, + "amount_scale" integer, + primary key ("org_id", "posting_key", "line"), + constraint "ledger_lines_posting_key_check" check (char_length(posting_key) <= 120), + constraint "ledger_lines_account_check" check (char_length(account) <= 60), + constraint "ledger_lines_amount_currency_check" check (amount_currency ~ '^[A-Z]{3}$'), + constraint "ledger_lines_amount_scale_check" check (amount_scale is null or (amount_scale >= 0 and amount_scale <= 15)) +); +alter table "posts" add column "search_tsv" tsvector generated always as (setweight(to_tsvector('english', coalesce("title", '')), 'A')) stored not null; +create index "posts_search_tsv_879c084e_idx" on "posts" using gin ("search_tsv"); +alter table "ledger_lines" add constraint "ledger_lines_org_id_fkey" foreign key ("org_id") references "orgs" ("id") on delete cascade; + +-- down +alter table "ledger_lines" drop constraint "ledger_lines_org_id_fkey"; +drop index "posts_search_tsv_879c084e_idx"; +alter table "posts" drop column "search_tsv"; +drop table "ledger_lines"; diff --git a/examples/dummy/packages/db/schema/04_tables/ledger_lines.sql b/examples/dummy/packages/db/schema/04_tables/ledger_lines.sql new file mode 100644 index 000000000..715eb9f9e --- /dev/null +++ b/examples/dummy/packages/db/schema/04_tables/ledger_lines.sql @@ -0,0 +1,16 @@ +-- generated by `x db gen` from the migrations; never edited by hand (X_SCHEMA_DUMP_DRIFT) + +create table "ledger_lines" ( + "org_id" uuid not null, + "posting_key" text not null, + "line" integer not null, + "account" text not null, + "amount_minor" bigint not null, + "amount_currency" character(3) not null, + "amount_scale" integer, + constraint "ledger_lines_pkey" PRIMARY KEY (org_id, posting_key, line), + constraint "ledger_lines_account_check" CHECK ((char_length(account) <= 60)), + constraint "ledger_lines_amount_currency_check" CHECK ((amount_currency ~ '^[A-Z]{3}$'::text)), + constraint "ledger_lines_amount_scale_check" CHECK (((amount_scale IS NULL) OR ((amount_scale >= 0) AND (amount_scale <= 15)))), + constraint "ledger_lines_posting_key_check" CHECK ((char_length(posting_key) <= 120)) +); diff --git a/examples/dummy/packages/db/schema/04_tables/posts.sql b/examples/dummy/packages/db/schema/04_tables/posts.sql index 1e4ffba1f..7d0cfbbdf 100644 --- a/examples/dummy/packages/db/schema/04_tables/posts.sql +++ b/examples/dummy/packages/db/schema/04_tables/posts.sql @@ -14,6 +14,7 @@ create table "posts" ( "published_at" timestamp with time zone, "created_at" timestamp with time zone default now() not null, "updated_at" timestamp with time zone default now() not null, + "search_tsv" tsvector generated always as (setweight(to_tsvector('english'::regconfig, COALESCE(title, ''::text)), 'A'::"char")) stored not null, constraint "posts_pkey" PRIMARY KEY (id), constraint "posts_cover_url_check" CHECK ((cover_url ~ '^https?://'::text)), constraint "posts_excerpt_check" CHECK ((char_length(excerpt) <= 200)), diff --git a/examples/dummy/packages/db/schema/05_indexes/posts.sql b/examples/dummy/packages/db/schema/05_indexes/posts.sql index f9e06f701..5e0a1d498 100644 --- a/examples/dummy/packages/db/schema/05_indexes/posts.sql +++ b/examples/dummy/packages/db/schema/05_indexes/posts.sql @@ -10,4 +10,6 @@ CREATE INDEX posts_org_id_published_at_647c63aa_idx ON public.posts USING btree CREATE UNIQUE INDEX posts_post_slug_unique_key ON public.posts USING btree (slug); +CREATE INDEX posts_search_tsv_879c084e_idx ON public.posts USING gin (search_tsv); + CREATE INDEX posts_status_published_at_id_idx ON public.posts USING btree (status, published_at, id); diff --git a/examples/dummy/packages/db/schema/06_foreign_keys/ledger_lines.sql b/examples/dummy/packages/db/schema/06_foreign_keys/ledger_lines.sql new file mode 100644 index 000000000..e53221582 --- /dev/null +++ b/examples/dummy/packages/db/schema/06_foreign_keys/ledger_lines.sql @@ -0,0 +1,3 @@ +-- generated by `x db gen` from the migrations; never edited by hand (X_SCHEMA_DUMP_DRIFT) + +alter table "ledger_lines" add constraint "ledger_lines_org_id_fkey" FOREIGN KEY (org_id) REFERENCES orgs(id) ON DELETE CASCADE; diff --git a/examples/dummy/packages/db/schema/framework/03_sequences/x_job_lease_fence.sql b/examples/dummy/packages/db/schema/framework/03_sequences/x_job_lease_fence.sql new file mode 100644 index 000000000..6e5f00054 --- /dev/null +++ b/examples/dummy/packages/db/schema/framework/03_sequences/x_job_lease_fence.sql @@ -0,0 +1,3 @@ +-- generated by `x db gen` from the migrations; never edited by hand (X_SCHEMA_DUMP_DRIFT) + +create sequence "x_job_lease_fence" as bigint start with 1 increment by 1 minvalue 1 maxvalue 9223372036854775807 cache 1 no cycle; diff --git a/examples/dummy/packages/db/schema/framework/04_tables/x_job_leases.sql b/examples/dummy/packages/db/schema/framework/04_tables/x_job_leases.sql index aefb1b8c8..f547c6e5c 100644 --- a/examples/dummy/packages/db/schema/framework/04_tables/x_job_leases.sql +++ b/examples/dummy/packages/db/schema/framework/04_tables/x_job_leases.sql @@ -5,5 +5,6 @@ create table "x_job_leases" ( "slot" integer not null, "holder" text not null, "expires_at" timestamp with time zone not null, + "fence" bigint, constraint "x_job_leases_pkey" PRIMARY KEY (lease_key, slot) ); diff --git a/examples/dummy/packages/db/src/client.ts b/examples/dummy/packages/db/src/client.ts index 2e826eb83..48345c26a 100644 --- a/examples/dummy/packages/db/src/client.ts +++ b/examples/dummy/packages/db/src/client.ts @@ -8,6 +8,7 @@ import { type Driver, database, postgresDriver, testDriver } from '@ultimat3/ent import { comments } from './schema/comments'; import { connections } from './schema/connections'; import { dailyLikes } from './schema/daily-likes'; +import { ledgerLines } from './schema/ledger-lines'; import { likes } from './schema/likes'; import { members } from './schema/members'; import { orgs } from './schema/orgs'; @@ -59,6 +60,7 @@ const entities = { comments, connections, dailyLikes, + ledgerLines, likes, members, orgs, diff --git a/examples/dummy/packages/db/src/index.ts b/examples/dummy/packages/db/src/index.ts index eaa02ab6e..5c7b635a8 100644 --- a/examples/dummy/packages/db/src/index.ts +++ b/examples/dummy/packages/db/src/index.ts @@ -14,6 +14,8 @@ export { } from './schema/connections'; export type { DailyLikes } from './schema/daily-likes'; export { dailyLikes } from './schema/daily-likes'; +export type { LedgerLine } from './schema/ledger-lines'; +export { ledgerLines } from './schema/ledger-lines'; export type { Like } from './schema/likes'; export { likes } from './schema/likes'; export type { Member } from './schema/members'; diff --git a/examples/dummy/packages/db/src/schema/ledger-lines.ts b/examples/dummy/packages/db/src/schema/ledger-lines.ts new file mode 100644 index 000000000..3c79bc82d --- /dev/null +++ b/examples/dummy/packages/db/src/schema/ledger-lines.ts @@ -0,0 +1,25 @@ +/** + * The org's money book: one row per leg of a double-entry posting, which `postCharge` + * (`apps/web/app/billing/jobs/post-charge.ts`) writes through `@ultimat3/jobs`' `ledger()`. The + * accounts are Postly's own words; the framework only checks that each posting nets to zero. + */ + +import { entity, integer, money, text, uuid } from '@ultimat3/entity'; +import { orgs } from './orgs'; + +export const ledgerLines = entity('ledger_lines', { + columns: { + orgId: uuid() + .references(() => orgs.id, { onDelete: 'cascade' }) + .tenant(), + /** The posting's identity — the vendor's charge id — and so the replay key, per org. */ + postingKey: text({ max: 120 }), + line: integer(), + account: text({ max: 60 }), + amount: money(), + }, + // One posting per (org, key): a replayed charge lands on the lines it already wrote. + primaryKey: ['orgId', 'postingKey', 'line'], +}); + +export type LedgerLine = typeof ledgerLines.$row; diff --git a/examples/dummy/packages/db/src/schema/posts.ts b/examples/dummy/packages/db/src/schema/posts.ts index f21d6b930..328448b60 100644 --- a/examples/dummy/packages/db/src/schema/posts.ts +++ b/examples/dummy/packages/db/src/schema/posts.ts @@ -28,7 +28,8 @@ export const posts = entity('posts', { .tenant(), authorId: uuid().references(() => members.id, { onDelete: 'restrict' }), slug: text({ max: SLUG_MAX }), - title: text({ max: TITLE_MAX }), + /** What a search box matches, ranked first (`reindex-posts` rebuilds the index from it). */ + title: text({ max: TITLE_MAX }).searchable('A'), excerpt: text({ max: EXCERPT_MAX }), body: text(), /** Nullable: a post without a cover renders the generated OG image instead. */ diff --git a/examples/dummy/x.manifest.json b/examples/dummy/x.manifest.json index a2b44f714..8482bb11e 100644 --- a/examples/dummy/x.manifest.json +++ b/examples/dummy/x.manifest.json @@ -1,6 +1,6 @@ { "manifestVersion": 1, - "buildId": "08366c8bb04227af", + "buildId": "8905100dc383093c", "app": { "name": "@postly/root", "version": "0.0.1" @@ -724,6 +724,56 @@ ], "invariants": [] }, + { + "name": "ledger_lines", + "table": "ledger_lines", + "columns": [ + { + "name": "account", + "type": "text", + "nullable": false, + "primaryKey": false + }, + { + "name": "amount_currency", + "type": "char", + "nullable": false, + "primaryKey": false + }, + { + "name": "amount_minor", + "type": "bigint", + "nullable": false, + "primaryKey": false + }, + { + "name": "amount_scale", + "type": "integer", + "nullable": true, + "primaryKey": false + }, + { + "name": "line", + "type": "integer", + "nullable": false, + "primaryKey": true + }, + { + "name": "org_id", + "type": "uuid", + "nullable": false, + "primaryKey": true, + "references": "orgs.id" + }, + { + "name": "posting_key", + "type": "text", + "nullable": false, + "primaryKey": true + } + ], + "invariants": [] + }, { "name": "likes", "table": "likes", @@ -1125,6 +1175,12 @@ "nullable": true, "primaryKey": false }, + { + "name": "search_tsv", + "type": "tsvector", + "nullable": false, + "primaryKey": false + }, { "name": "slug", "type": "text", @@ -3241,6 +3297,60 @@ "expose": false } }, + { + "name": "printPost", + "input": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "orgId": { + "type": "string", + "format": "uuid" + }, + "slug": { + "type": "string", + "format": "slug", + "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" + } + }, + "required": [ + "orgId", + "slug" + ], + "additionalProperties": false + }, + "output": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "key": { + "type": "string", + "minLength": 1 + }, + "bytes": { + "type": "number" + }, + "contentType": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "key", + "bytes", + "contentType" + ], + "additionalProperties": false + }, + "policy": "post:export", + "permissions": [ + "post:export" + ], + "cacheInvalidates": [], + "mcp": { + "expose": false + } + }, { "name": "publishPost", "input": { @@ -7180,6 +7290,79 @@ }, "steps": [] }, + { + "name": "billing.post-charge", + "input": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "key": { + "type": "string", + "minLength": 1 + }, + "entries": { + "type": "array", + "items": { + "type": "object", + "properties": { + "account": { + "type": "string", + "minLength": 1 + }, + "amount": { + "type": "object", + "properties": { + "minor": { + "type": "integer", + "description": "amount in minor units, never a float", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "currency": { + "type": "string", + "pattern": "^[A-Z]{3}$" + }, + "scale": { + "type": "integer", + "description": "decimal places `minor` counts; absent means the currency’s own", + "minimum": 0, + "maximum": 15 + } + }, + "required": [ + "minor", + "currency" + ], + "additionalProperties": false, + "description": "integer minor units plus an ISO 4217 currency code" + } + }, + "required": [ + "account", + "amount" + ], + "additionalProperties": false + }, + "minItems": 2 + }, + "orgId": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "key", + "entries" + ], + "additionalProperties": false + }, + "queue": "default", + "retry": { + "attempts": 3, + "backoff": "exponential" + }, + "steps": [] + }, { "name": "deliverDigest", "input": { @@ -7933,6 +8116,26 @@ }, "steps": [] }, + { + "name": "reindex-posts", + "input": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "force": { + "type": "boolean" + } + }, + "required": [], + "additionalProperties": false + }, + "queue": "default", + "retry": { + "attempts": 3, + "backoff": "exponential" + }, + "steps": [] + }, { "name": "runs.statement", "input": { @@ -8459,6 +8662,7 @@ "action:completePostsImport", "action:grantPostsImport", "action:listPostsImportParts", + "action:printPost", "action:requestPostsExport", "action:signPostsImportPart" ] diff --git a/framework.manifest.json b/framework.manifest.json index c5c929f5b..8539a3d73 100644 --- a/framework.manifest.json +++ b/framework.manifest.json @@ -1,6 +1,6 @@ { "version": 1, - "buildId": "9ef63267ec866704a68588df5ed80ced17d4d07dc292419fe6a82092879ca211", + "buildId": "2ddd151207d184bf092e7e28b4df66fbf7ceb012a458a926f63ec0f1b5f43a90", "tiers": { "0": [ "core", @@ -559,6 +559,11 @@ "owner": "cli", "at": "packages/cli/src/error-codes.ts" }, + { + "code": "X_APP_ROOT_NESTED", + "owner": "cli", + "at": "packages/cli/src/error-codes.ts" + }, { "code": "X_APP_URL_PORT_MISMATCH", "owner": "cli", @@ -697,7 +702,7 @@ { "code": "X_BENCH_RUN_FAILED", "owner": "scripts", - "at": "scripts/bench/first-hour.ts" + "at": "scripts/bench/boot-vs-files-child.ts" }, { "code": "X_BENCH_RUN_TOO_SHORT", @@ -966,23 +971,23 @@ }, { "code": "X_CDP_BROWSER_MISSING", - "owner": "testing", - "at": "packages/testing/src/e2e-error-codes.ts" + "owner": "scraping", + "at": "packages/scraping/src/cdp-error-codes.ts" }, { "code": "X_CDP_CALL_FAILED", - "owner": "testing", - "at": "packages/testing/src/e2e-error-codes.ts" + "owner": "scraping", + "at": "packages/scraping/src/cdp-error-codes.ts" }, { "code": "X_CDP_LAUNCH_FAILED", - "owner": "testing", - "at": "packages/testing/src/e2e-error-codes.ts" + "owner": "scraping", + "at": "packages/scraping/src/cdp-error-codes.ts" }, { "code": "X_CDP_TIMEOUT", - "owner": "testing", - "at": "packages/testing/src/e2e-error-codes.ts" + "owner": "scraping", + "at": "packages/scraping/src/cdp-error-codes.ts" }, { "code": "X_CHANGELOG_HIGHLIGHTS_MISSING", @@ -1884,6 +1889,11 @@ "owner": "entity", "at": "packages/entity/src/entity-error.ts" }, + { + "code": "X_ENTITY_PARTITION_INVALID", + "owner": "entity", + "at": "packages/entity/src/entity-error.ts" + }, { "code": "X_ENTITY_READ_ONLY", "owner": "entity", @@ -2464,6 +2474,11 @@ "owner": "action", "at": "packages/action/src/error-titles.ts" }, + { + "code": "X_IDENTITY_PROVIDER_UNKNOWN", + "owner": "auth", + "at": "packages/auth/src/errors.ts" + }, { "code": "X_IMAGE_CONTENT_UNIGNORED", "owner": "scripts", @@ -2809,6 +2824,21 @@ "owner": "seo", "at": "packages/seo/src/errors.ts" }, + { + "code": "X_LEDGER_ENTRY_INVALID", + "owner": "jobs", + "at": "packages/jobs/src/error-codes.ts" + }, + { + "code": "X_LEDGER_POSTING_CONFLICT", + "owner": "jobs", + "at": "packages/jobs/src/error-codes.ts" + }, + { + "code": "X_LEDGER_UNBALANCED", + "owner": "jobs", + "at": "packages/jobs/src/error-codes.ts" + }, { "code": "X_LIFECYCLE_DRAINED", "owner": "core", @@ -2899,6 +2929,21 @@ "owner": "i18n", "at": "packages/i18n/src/errors.ts" }, + { + "code": "X_LOCK_LOST", + "owner": "jobs", + "at": "packages/jobs/src/error-codes.ts" + }, + { + "code": "X_LOCK_NOT_ACQUIRED", + "owner": "jobs", + "at": "packages/jobs/src/error-codes.ts" + }, + { + "code": "X_LOCK_STORE_MISSING", + "owner": "jobs", + "at": "packages/jobs/src/error-codes.ts" + }, { "code": "X_LOCK_UNSUPPORTED", "owner": "entity", @@ -3389,6 +3434,11 @@ "owner": "db", "at": "packages/db/src/errors.ts" }, + { + "code": "X_MIGRATION_PARTITION_CHANGE", + "owner": "db", + "at": "packages/db/src/errors.ts" + }, { "code": "X_MIGRATION_ROLLBACK_DESTRUCTIVE", "owner": "db", @@ -3589,6 +3639,16 @@ "owner": "notify", "at": "packages/notify/src/errors.ts" }, + { + "code": "X_NOTIFY_SMS_NUMBER_INVALID", + "owner": "notify", + "at": "packages/notify/src/errors.ts" + }, + { + "code": "X_NOTIFY_SMS_TRANSPORT_MISSING", + "owner": "notify", + "at": "packages/notify/src/errors.ts" + }, { "code": "X_NOTIFY_STORE_MISSING", "owner": "notify", @@ -3749,6 +3809,11 @@ "owner": "cli", "at": "packages/cli/src/error-codes.ts" }, + { + "code": "X_PARTITIONS_UNMAINTAINED", + "owner": "cli", + "at": "packages/cli/src/error-codes.ts" + }, { "code": "X_PASSKEY_CHALLENGE_EXPIRED", "owner": "auth", @@ -3794,6 +3859,16 @@ "owner": "policy", "at": "packages/policy/src/errors.ts" }, + { + "code": "X_PERSONAL_DATA_UNERASED", + "owner": "jobs", + "at": "packages/jobs/src/error-codes.ts" + }, + { + "code": "X_PERSONAL_DATA_UNREACHED", + "owner": "jobs", + "at": "packages/jobs/src/error-codes.ts" + }, { "code": "X_PIN_BASE_MISSING", "owner": "scripts", @@ -3854,6 +3929,11 @@ "owner": "cli", "at": "packages/cli/src/error-codes.ts" }, + { + "code": "X_PREBUILT_REGISTRY_STALE", + "owner": "cli", + "at": "packages/cli/src/error-codes.ts" + }, { "code": "X_PRELOAD_UNKNOWN_RELATION", "owner": "entity", @@ -4529,6 +4609,16 @@ "owner": "cli", "at": "packages/cli/src/error-codes.ts" }, + { + "code": "X_SAML_METADATA_INVALID", + "owner": "auth", + "at": "packages/auth/src/errors.ts" + }, + { + "code": "X_SAML_RESPONSE_INVALID", + "owner": "auth", + "at": "packages/auth/src/errors.ts" + }, { "code": "X_SCAFFOLD_FIRST_RUN_FAILED", "owner": "scripts", @@ -4699,11 +4789,31 @@ "owner": "scraping", "at": "packages/scraping/src/errors.ts" }, + { + "code": "X_SCRAPE_PDF_BUSY", + "owner": "scraping", + "at": "packages/scraping/src/errors.ts" + }, { "code": "X_SCRAPE_PDF_OPTIONS_INVALID", "owner": "scraping", "at": "packages/scraping/src/errors.ts" }, + { + "code": "X_SCRAPE_PDF_ORIGIN_MISSING", + "owner": "scraping", + "at": "packages/scraping/src/errors.ts" + }, + { + "code": "X_SCRAPE_PDF_ROUTE_FAILED", + "owner": "scraping", + "at": "packages/scraping/src/errors.ts" + }, + { + "code": "X_SCRAPE_PDF_ROUTE_INVALID", + "owner": "scraping", + "at": "packages/scraping/src/errors.ts" + }, { "code": "X_SCRAPE_PROFILE_LOCKED", "owner": "scraping", @@ -5334,6 +5444,11 @@ "owner": "storage", "at": "packages/storage/src/errors.ts" }, + { + "code": "X_STORAGE_SCAN_UNAVAILABLE", + "owner": "storage", + "at": "packages/storage/src/errors.ts" + }, { "code": "X_STORAGE_SECRET_DEV", "owner": "cli", @@ -5359,6 +5474,11 @@ "owner": "storage", "at": "packages/storage/src/errors.ts" }, + { + "code": "X_STORAGE_UPLOAD_INFECTED", + "owner": "storage", + "at": "packages/storage/src/errors.ts" + }, { "code": "X_STORAGE_UPLOAD_SIZE_UNDECLARED", "owner": "storage", @@ -6249,6 +6369,11 @@ "owner": "scripts", "at": "scripts/lib/workspaces.ts" }, + { + "code": "X_WORKSPACE_NAME_TAKEN", + "owner": "cli", + "at": "packages/cli/src/error-codes.ts" + }, { "code": "X_WRITE_UNFILTERED", "owner": "entity", diff --git a/packages/ai/README.md b/packages/ai/README.md index 4fe1a5b39..e5f34d55b 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -870,8 +870,29 @@ The boot installs it through `installAiGateway(gateway): () => void`, on `@ultim only — with the `Gateway` interface, its request/answer types and the errors a gateway driver throws (`AiProviderUnavailableError`, `AiKeyMissingError`, `AiModelUnknownError`, `AiRequestInvalidError`). A test swaps the gateway the same way and calls the release to put the -previous one back. `configureAi` configures the rest (`embedder`, `semanticCache`, `redact`) and -takes no `gateway`. +previous one back. + +The embedder and the semantic cache are ports the same way — `embedder` (an `Embedder`; default +`hashEmbedder()`, offline and lexical) and `semanticCache` (a `SemanticCacheFactory`, +`(scope) => SemanticCache`; default a `memorySemanticCache()` per scope), installed with +`installAiEmbedder` / `installSemanticCache` from `@ultimat3/ai/adapter`. Each install clears the +per-scope caches: vectors from two embedders are not comparable. Prove one with +`embedderConformance` / `semanticCacheFactoryConformance` (`@ultimat3/testing/conformance`). + +```ts +// apps/web/runtime.ts +import { remoteEmbedder } from '@ultimat3/ai'; +import { memorySemanticCache } from '@ultimat3/cache'; + +export const runtime = { + embedder: remoteEmbedder({ name: 'voyage-3', dimension: 1_024 }), // EMBEDDINGS_API_KEY + // One cache per scope; a pgvector-backed factory of your own takes the same shape. + semanticCache: () => memorySemanticCache({ threshold: 0.95 }), +}; +``` + +`configureAi` configures the rest — `redact` only — and takes no `gateway`, `embedder` or +`semanticCache` (31.0.0). Missing at call time is `X_AI_GATEWAY_MISSING`, never a silent default provider. diff --git a/packages/ai/src/adapter.ts b/packages/ai/src/adapter.ts index c1d3b4f7d..6a597fe00 100644 --- a/packages/ai/src/adapter.ts +++ b/packages/ai/src/adapter.ts @@ -1,9 +1,13 @@ -// `@ultimat3/ai/adapter` — the `aiGateway` port: the interface a gateway implements, the request -// and answer shapes it speaks, the errors a gateway driver throws, and the boot's installer. The -// installer is here and never on the `.` barrel; the boot fills it from `apps//runtime.ts`. +// `@ultimat3/ai/adapter` — the `aiGateway`, `embedder` and `semanticCache` ports: the interfaces +// an adapter implements, the shapes a gateway speaks, the errors a driver throws, and the boot's +// installers. Installers live here and never on the `.` barrel; the boot fills them from +// `apps//runtime.ts`. +export type { SemanticCache } from '@ultimat3/cache'; export type { BudgetKeys, BudgetLedger } from './budget'; +export type { Embedder } from './embeddings'; export { + AiEmbedderInvalidError, AiKeyMissingError, AiModelUnknownError, AiProviderUnavailableError, @@ -12,5 +16,6 @@ export { export type { Gateway } from './gateway'; export type { ModelId } from './models'; export type { GenerateRequest, GenerateResult, StreamChunk } from './provider'; -export { installAiGateway } from './runtime'; +export type { SemanticCacheFactory } from './runtime'; +export { installAiEmbedder, installAiGateway, installSemanticCache } from './runtime'; export type { SamplingWire } from './sampling'; diff --git a/packages/ai/src/llm-cache.test.ts b/packages/ai/src/llm-cache.test.ts index 0609f25d4..b11cf0828 100644 --- a/packages/ai/src/llm-cache.test.ts +++ b/packages/ai/src/llm-cache.test.ts @@ -23,7 +23,12 @@ import { } from './llm-fixture'; import { FIXTURE_MODEL, useFixtureModels } from './model-fixture'; import { definePrompt } from './prompt'; -import { configureAi, installAiGateway, resetAiRuntime } from './runtime'; +import { + installAiEmbedder, + installAiGateway, + installSemanticCache, + resetAiRuntime, +} from './runtime'; // The framework registers no model: this suite registers the rows it names (`model-fixture.ts`). useFixtureModels(); @@ -238,28 +243,26 @@ describe('the entry key', () => { const inner = memorySemanticCache(); const { provider } = stub({ answer: 'ok' }, { answer: 'ok' }); installAiGateway(providerGateway({ defaultModel: FIXTURE_MODEL, providers: [provider] })); - configureAi({ - // Orthogonal per first letter, so the two prompts never answer each other's lookup and the - // entry KEY is the only thing under test. - embedder: { - name: 'first-letter', - dimension: 26, - embed: async (texts) => - texts.map((text) => { - const vector = new Float32Array(26); - vector[(text.charCodeAt(0) - 97 + 26) % 26] = 1; - return vector; - }), - }, - semanticCache: () => ({ - ...inner, - lookup: (embedding, threshold) => inner.lookup(embedding, threshold), - remember(key, embedding, value, options) { - keys.push(key); - return inner.remember(key, embedding, value, options); - }, - }), + // Orthogonal per first letter, so the two prompts never answer each other's lookup and the + // entry KEY is the only thing under test. + installAiEmbedder({ + name: 'first-letter', + dimension: 26, + embed: async (texts) => + texts.map((text) => { + const vector = new Float32Array(26); + vector[(text.charCodeAt(0) - 97 + 26) % 26] = 1; + return vector; + }), }); + installSemanticCache(() => ({ + ...inner, + lookup: (embedding, threshold) => inner.lookup(embedding, threshold), + remember(key, embedding, value, options) { + keys.push(key); + return inner.remember(key, embedding, value, options); + }, + })); const ask = llm({ input: t.object({ q: t.string }), output: t.object({ answer: t.string }), @@ -291,19 +294,17 @@ describe('a broken embedding never serves another prompt answer', () => { test('two unrelated prompts get two answers, even with a directionless embedding', async () => { const { provider, seen } = stub(ANSWER, { summary: 'the other post', tags: ['other'] }); installAiGateway(providerGateway({ defaultModel: FIXTURE_MODEL, providers: [provider] })); - configureAi({ - embedder: { - name: 'nan-component', - dimension: 3, - embed: async (texts) => - texts.map((text) => { - const vector = new Float32Array(3); - vector[0] = 1; - vector[1] = Number.NaN; - vector[2] = text.includes(POST_ID) ? 1 : 2; - return vector; - }), - }, + installAiEmbedder({ + name: 'nan-component', + dimension: 3, + embed: async (texts) => + texts.map((text) => { + const vector = new Float32Array(3); + vector[0] = 1; + vector[1] = Number.NaN; + vector[2] = text.includes(POST_ID) ? 1 : 2; + return vector; + }), }); const summarize = declare(promptFor(), { cache: { semantic: { threshold: 0.99 } } }); diff --git a/packages/ai/src/runtime.test.ts b/packages/ai/src/runtime.test.ts index d5d05172c..12545329e 100644 --- a/packages/ai/src/runtime.test.ts +++ b/packages/ai/src/runtime.test.ts @@ -7,8 +7,17 @@ import { beforeEach, describe, expect, test } from 'bun:test'; import { memorySemanticCache } from '@ultimat3/cache'; import { MAX_CACHED_FORMATTERS } from '@ultimat3/core'; import { echoProvider } from './echo-provider'; +import { hashEmbedder } from './embeddings'; import { providerGateway } from './gateway'; -import { configureAi, installAiGateway, resetAiRuntime, semanticCacheFor } from './runtime'; +import { + aiEmbedder, + configureAi, + installAiEmbedder, + installAiGateway, + installSemanticCache, + resetAiRuntime, + semanticCacheFor, +} from './runtime'; const built: string[] = []; @@ -16,11 +25,9 @@ beforeEach(() => { resetAiRuntime(); built.length = 0; installAiGateway(providerGateway({ providers: [echoProvider()] })); - configureAi({ - semanticCache: (scope) => { - built.push(scope); - return memorySemanticCache(); - }, + installSemanticCache((scope) => { + built.push(scope); + return memorySemanticCache(); }); }); @@ -55,16 +62,55 @@ describe('semanticCacheFor', () => { // its first caller rather than its contract. }); - test('a new runtime drops every instance — vectors from two embedders are not comparable', () => { + test('a new embedder drops every instance — vectors from two embedders are not comparable', () => { semanticCacheFor('org-a'); - installAiGateway(providerGateway({ providers: [echoProvider()] })); - configureAi({ - semanticCache: (scope) => { - built.push(scope); - return memorySemanticCache(); - }, - }); + const release = installAiEmbedder(hashEmbedder({ dimension: 8 })); + semanticCacheFor('org-a'); + // And back: the release restores the old embedder, whose vectors the new caches never held. + release(); + semanticCacheFor('org-a'); + expect(built).toEqual(['org-a', 'org-a', 'org-a']); + }); + + test('configureAi() starts from empty caches and keeps the installed factory', () => { + semanticCacheFor('org-a'); + configureAi(); semanticCacheFor('org-a'); expect(built).toEqual(['org-a', 'org-a']); }); }); + +describe('the embedder and semanticCache ports', () => { + test('nothing installed: the hash embedder and a memory cache per scope', () => { + resetAiRuntime(); + expect(aiEmbedder().name).toBe('hash'); + expect(semanticCacheFor('org-a').name).toBe('memory'); + }); + + test('installs stack and each release removes its own frame, in any order', () => { + resetAiRuntime(); + const a = hashEmbedder({ dimension: 4 }); + const b = hashEmbedder({ dimension: 8 }); + const releaseA = installAiEmbedder(a); + const releaseB = installAiEmbedder(b); + expect(aiEmbedder()).toBe(b); + releaseA(); + expect(aiEmbedder()).toBe(b); + releaseB(); + expect(aiEmbedder().name).toBe('hash'); + expect(aiEmbedder().dimension).toBe(256); + }); + + test('a released cache factory builds nothing more', () => { + resetAiRuntime(); + const scopes: string[] = []; + const release = installSemanticCache((scope) => { + scopes.push(scope); + return memorySemanticCache(); + }); + semanticCacheFor('org-a'); + release(); + semanticCacheFor('org-a'); + expect(scopes).toEqual(['org-a']); + }); +}); diff --git a/packages/ai/src/runtime.ts b/packages/ai/src/runtime.ts index 0a110e9ee..96b38e7f2 100644 --- a/packages/ai/src/runtime.ts +++ b/packages/ai/src/runtime.ts @@ -1,5 +1,6 @@ // The ambient AI runtime an `llm()` action calls into: the gateway that serves every model -// call, the embedder the semantic cache keys on, and one semantic cache per declared scope. +// call, the embedder the semantic cache keys on, and one semantic cache per declared scope. All +// three are adapter ports, filled from `apps//runtime.ts` through `@ultimat3/ai/adapter`. // // Ambient rather than injected, for the same reason the budget ledger is: an `llm()` // declaration is evaluated at module scope, long before a provider, an API key or a request @@ -10,26 +11,19 @@ import type { SemanticCache } from '@ultimat3/cache'; import { memorySemanticCache } from '@ultimat3/cache'; import { asyncContext, cachedFormatter } from '@ultimat3/core'; -import { adapterSlot } from '@ultimat3/core/adapter'; +import { type AdapterSlot, adapterSlot } from '@ultimat3/core/adapter'; import type { Embedder } from './embeddings'; import { hashEmbedder } from './embeddings'; import { AiGatewayMissingError } from './errors'; import type { Gateway } from './gateway'; /** - * Everything about the AI runtime EXCEPT the gateway, which is a port with one way in: - * `aiGateway` in `apps//runtime.ts`, installed by the boot through `installAiGateway` - * (`@ultimat3/ai/adapter`). `gateway` left this input in 31.0.0 — two ways to set one slot. + * The AI runtime's configuration — the redactor, and nothing that selects a backend. The gateway, + * the embedder and the semantic-cache factory are ports with one way in each: `aiGateway`, + * `embedder` and `semanticCache` in `apps//runtime.ts`, installed by the boot through + * `@ultimat3/ai/adapter`. `gateway`, `embedder` and `semanticCache` left this input in 31.0.0. */ export interface AiRuntimeInput { - /** - * Keys the semantic cache. Defaults to `HashEmbedder`: deterministic, offline, and lexical - * rather than semantic — at the thresholds an LLM cache runs at that is a near-exact-match - * cache, which is the safe way to be wrong. Install a real embedder to get paraphrase hits. - */ - readonly embedder?: Embedder; - /** One cache per scope key. Defaults to the in-memory driver; pgvector in production. */ - readonly semanticCache?: (scope: string) => SemanticCache; /** * The last thing that runs over a prompt before it leaves the process. Declared here, in the * one place the framework already owns, because `vars()` is the one declared place a model call @@ -54,25 +48,39 @@ export type Redactor = (text: string) => string; /** No app redactor installed. Named rather than inline so `ultimate.llm.redacted` has one thing to mean. */ const noRedaction: Redactor = (text) => text; -interface AiRuntime { - readonly embedder: Embedder; - readonly semanticCache: (scope: string) => SemanticCache; - readonly redact: Redactor; -} +/** + * The `semanticCache` port: one cache per scope key. A FACTORY, because scopes are separate cache + * instances (`semanticCacheFor`). Default: a `memorySemanticCache()` per scope, per process. + */ +export type SemanticCacheFactory = (scope: string) => SemanticCache; -let runtime: AiRuntime | undefined; +let redactor: Redactor = noRedaction; /** The `aiGateway` port's ONE slot, written only by `installAiGateway`. */ const gatewaySlot = /* @__PURE__ */ adapterSlot(() => undefined); +/** + * The `embedder` port. Default `hashEmbedder()`: deterministic, offline and lexical rather than + * semantic — at the thresholds an LLM cache runs at that is a near-exact-match cache, which is the + * safe way to be wrong. Install a real embedder (`remoteEmbedder`) to get paraphrase hits. + */ +const embedderSlot = /* @__PURE__ */ adapterSlot(() => fallbackEmbedder()); +/** The `semanticCache` port. Default the in-memory driver; pgvector in production. */ +const semanticCacheSlot = /* @__PURE__ */ adapterSlot(() => memoryPerScope); +const memoryPerScope: SemanticCacheFactory = () => memorySemanticCache(); +let hashed: Embedder | undefined; +/** One default embedder per process, built on first use rather than at import. */ +const fallbackEmbedder = (): Embedder => { + hashed ??= hashEmbedder(); + return hashed; +}; const caches = new Map(); /** - * The boot's install of the `aiGateway` port, from `@ultimat3/ai/adapter` only. Answers the release - * that removes THIS install and no other (core's `adapterSlot`), in any order, once. + * Install into a slot whose value every cached `SemanticCache` was built against, and clear the + * caches on the install and on the release alike: vectors from two embedders are not comparable, + * and a test's double must never be served what the real backend answered (and back). */ -export function installAiGateway(gateway: Gateway): () => void { - const release = gatewaySlot.install(gateway); - // A cached answer is one gateway's: kept across a swap, a test's double would be served what - // the real gateway answered (and back). Cleared on install and on release alike. +function installClearing(slot: AdapterSlot, value: T): () => void { + const release = slot.install(value); caches.clear(); return () => { release(); @@ -80,14 +88,30 @@ export function installAiGateway(gateway: Gateway): () => void { }; } +/** + * The boot's install of the `aiGateway` port, from `@ultimat3/ai/adapter` only. Answers the release + * that removes THIS install and no other (core's `adapterSlot`), in any order, once. + */ +export function installAiGateway(gateway: Gateway): () => void { + // A cached answer is one gateway's: kept across a swap, a test's double would be served what + // the real gateway answered (and back). + return installClearing(gatewaySlot, gateway); +} + +/** The boot's install of the `embedder` port, from `@ultimat3/ai/adapter` only. Answers its release. */ +export function installAiEmbedder(embedder: Embedder): () => void { + return installClearing(embedderSlot, embedder); +} + +/** The boot's install of the `semanticCache` port, from `@ultimat3/ai/adapter` only. */ +export function installSemanticCache(factory: SemanticCacheFactory): () => void { + return installClearing(semanticCacheSlot, factory); +} + +/** Replace the runtime's configuration; an omitted field takes its default. Clears the caches. */ export function configureAi(input: AiRuntimeInput = {}): void { - runtime = { - embedder: input.embedder ?? hashEmbedder(), - semanticCache: input.semanticCache ?? (() => memorySemanticCache()), - redact: input.redact ?? noRedaction, - }; - // A new runtime means a new embedder; vectors from the old one are not - // comparable to vectors from the new one, and a stale hit would answer the wrong question. + redactor = input.redact ?? noRedaction; + // What a test calls between cases to start from empty caches; harmless anywhere else. caches.clear(); } @@ -138,13 +162,14 @@ export function installedGateway(): Gateway | undefined { return evalScopes.get()?.gateway ?? gatewaySlot.current(); } +/** The installed embedder (`embedder` port), or the process's hash embedder. */ export function aiEmbedder(): Embedder { - return runtime?.embedder ?? hashEmbedder(); + return embedderSlot.current(); } /** The installed redactor, or the identity. Never absent, so the call site has no branch. */ export function aiRedactor(): Redactor { - return runtime?.redact ?? noRedaction; + return redactor; } /** @@ -160,16 +185,14 @@ export function aiRedactor(): Redactor { * no entries at all and the memory one loses a cache. */ export function semanticCacheFor(scope: string): SemanticCache { - return cachedFormatter( - caches, - scope, - () => runtime?.semanticCache(scope) ?? memorySemanticCache(), - ); + return cachedFormatter(caches, scope, () => semanticCacheSlot.current()(scope)); } /** Test-only reset. Module-level state otherwise leaks between test files. */ export function resetAiRuntime(): void { - runtime = undefined; + redactor = noRedaction; gatewaySlot.reset(); + embedderSlot.reset(); + semanticCacheSlot.reset(); caches.clear(); } diff --git a/packages/auth/CLAUDE.md b/packages/auth/CLAUDE.md index 11e533f68..6a7c70719 100644 --- a/packages/auth/CLAUDE.md +++ b/packages/auth/CLAUDE.md @@ -96,9 +96,8 @@ Tier 2. Produces the `Actor`; produces nothing else. Authorization is `@ultimat3 `deleteSessionsForOrg` joins through `x_users`; `x_sessions` does **not** gain an `org_id`. - The code flow carries `nonce` inside the id token. `assertOAuthCallback` checks an echoed one when present and never requires it; `verifyIdToken` is the real gate. -- The handshake is sealed (`sealHandshake`); `openHandshake` takes the provider as an argument. - Expiry is the server's clock, not `Max-Age`. One handshake cookie **per provider** - (`handshakeCookieName`), and `clearHandshakeCookie(provider)`. +- `sealHandshake`/`openHandshake` (a hand-built flow's cookie, not `oauthLogin`'s): the provider is + an argument, expiry the server's clock, one cookie per provider (`handshakeCookieName`). - `takeVerification(purpose, identifier, tokenHash)` consumes **only on a hash match**, in one conditional statement (`consumed_at is null` on the UPDATE and its subselect, `order by created_at desc limit 1`); `consumeVerification` still compares in constant time. @@ -113,14 +112,13 @@ Tier 2. Produces the `Actor`; produces nothing else. Authorization is `@ultimat3 - A token endpoint's HTTP 200 is not success — read `error`. - Link by address only when the provider **and** the local account both verified it (`link: 'verified-email'`, default; `'never'` is the only other value). -- **The OAuth route paths are not configurable.** `oauth-paths.ts` imports nothing and is read by - both `oauth-errors.ts` and `oauth-route.ts`; every "start over" fix is `restartAt(provider)`. -- The routes are **descriptors** (`AuthRouteDescriptor`), never mounted handlers. -- The callback answers failure as **coded JSON** — a fixed body per code (`code`, `title`, `docs`, - one `cause`, `fix: x errors explain --json`); the authored cause and fix are the - `auth.oauth.refused` log line — and success redirects to a fixed `successPath` — - never `?next=` (`nextAfterSignIn` in `@ultimat3/http` is the one open-redirect check). -- The handshake cookie is cleared on **every** callback outcome. +- **One sign-in route implementation** (`identity-provider-route-core.ts`): `identityProviderRoutes` + and `oauthLogin` are two policies over it, at `/auth/sso/:provider[/callback]` (`oauth-paths.ts`, + imports nothing, not configurable; every "start over" fix is `restartAt(provider)`). Descriptors, + never mounted handlers. Failure is **coded JSON** (`code`, `title`, `docs`, one fixed `cause`, + `fix: x errors explain --json`); the authored cause/fix is the `.refused` log line. + Success is 303 to a fixed `successPath`, never `?next=`. `__Host-x_sso_` clears only for an + answer bound to it (`answers()`). The ACS `meta` is `csrf: 'cookieless'`; mount `meta: leg.meta`. - Refresh is **not implemented**, so the framework **stores no provider token**: `accountFor` (`oauth-login.ts`) writes `accessToken: null, refreshToken: null`. The columns and `AuthAccount` fields stay; an app that stores tokens implements `linkAccount` itself. @@ -200,14 +198,14 @@ Tier 2. Produces the `Actor`; produces nothing else. Authorization is `@ultimat3 compare-and-set, challenges single-use in `x_passkey_challenges` (never a process map). A step-up challenge is DERIVED (`stepUpChallenge`), never supplied. Browser half: `./client`, two modules (`client-bundle.test.ts`); never import the barrel there. `docs/history/auth.md`. -- SAML is out of scope permanently (XML-DSig canonicalisation has no Bun native). Put an - OIDC-speaking bridge in front and register that. +- **SAML = `samlIdentityProvider`**, no dependency: assertion signature required, one assertion, + metadata keys only, replay via `auth.totpReplay`, `trustEmail: true` needs `emailDomains`. ## Files | File | Job | |---|---| -| `auth.ts` | `defineAuth`, entity schemas, `login`/`register`/`authenticate`/`logout`. Three suites, split at the line ceiling along the three subjects: `auth.test.ts` (the credential flow), `auth-config.test.ts` (`defineAuth`'s defaults and refusals) and `auth-lockout.test.ts` (the tenant and address buckets) | +| `auth.ts` | `defineAuth`, entity schemas, `login`/`register`/`authenticate`/`logout`. Suites: `auth.test.ts` (credential flow), `auth-config.test.ts` (defaults, refusals), `auth-lockout.test.ts` (tenant and address buckets) | | `policy-bridge.ts` | the one funnel: identity → `Actor`, all four `ActorKind`s | | `session.ts` | two expiries, rotation, revocation, device list, the cookie | | `adapter.ts` | the seam; `builtin-adapter.ts` (Postgres) + `memory-adapter.ts` | @@ -238,10 +236,9 @@ Tier 2. Produces the `Actor`; produces nothing else. Authorization is `@ultimat3 | `oauth-profile.ts` | claims or userinfo → one `OAuthProfile`; `userinfoBody`, the one userinfo reader | | `oauth-grant-context.ts` | `resolveGrants`' second argument: tokens, every id-token claim, a lazy subject-checked `userinfo()` | | `oauth-login.ts` | profile → account link → session. `completeOAuthLogin` is the entry point | -| `oauth-paths.ts` | the one declaration of where the two routes live. Imports nothing | | `errors.ts` | the codes this package owns and borrows, their titles, the one `registerErrorCodes()` call, `AuthError`, and every non-OAuth factory | | `oauth-errors.ts` | the OAuth refusals more than one OAuth module raises, and `restartAt`. A refusal with one thrower lives beside it; declares no code and registers nothing | -| `oauth-route.ts` | `oauthLogin(auth)` — the redirect out and the callback back | +| `oauth-route.ts` | `oauthLogin(auth)`, the OAuth policy over the sign-in route core | | `kdf-gate.ts` | the one bound on concurrent argon2 work: core's `flightGate`, auth's `X_OVERLOADED` | | `email.ts` | `normaliseEmail` — the one normalisation an address gets before it is an identity key | | `json.ts` | reading untrusted JSON: `isRecord`, and a base64url JWT segment as an object or `null` | @@ -249,6 +246,8 @@ Tier 2. Produces the `Actor`; produces nothing else. Authorization is `@ultimat3 | `subject-target.ts` | `authSubjectTarget({ executor, user })`: the identity tables as a `defineSubject({ targets })` entry, structural (`@ultimat3/jobs` imports nothing from here). Export = `AUTH_SUBJECT_COLUMNS` (never `password_hash`, `mfa_secret`, a recovery digest); erase = api keys, pending verifications, then the user (sessions, accounts cascade). Three idempotent statements, no transaction of its own | | `passkey-login.ts` / `passkey-step-up.ts` / `passkey-evidence.ts` | the factories; the offline re-check. `passkey-verify.ts` + `passkey-assertion.ts` are every check; `*-postgres`/`*-memory` the store; `soft-authenticator.ts` (`./soft-authenticator`) the WebCrypto authenticator tests sign with | | `client-scope.ts` | `clientScopeOf(actor)`, the OPAQUE id for ``: keyed SHA-256 (`SESSION_SECRET`, else a per-process key and one warning) over kind, id, impersonator; `''` anonymous | +| `identity-provider*.ts` | SSO port, `signInWithIdentity`, OIDC adapter, route core, state cookie, status table | +| `saml-*.ts` | SAML adapter: strict XML reader, exc-c14n, XML-DSig, response checks, metadata, AuthnRequest | ```bash bun test ./packages/auth/ diff --git a/packages/auth/README.md b/packages/auth/README.md index 7bca0b4c3..2a25a4423 100644 --- a/packages/auth/README.md +++ b/packages/auth/README.md @@ -24,7 +24,7 @@ const { actor, token, cookie } = await login(auth, { email, password, ip }); // a normalised handle to the auth user's id, and `register(auth, { id, ... })` links the two rows. const byHandle = await login(auth, { handle: 'ada', password, ip }); -// "Log in with GitHub" is a link to /auth/oauth/github. These two routes are what serves it. +// "Log in with GitHub" is a link to /auth/sso/github. These two routes are what serves it. const { start, callback } = oauthLogin(auth); ``` @@ -317,11 +317,9 @@ registerOAuthProvider( PKCE stays mandatory for all three: for a public client it is the only thing binding the code to the browser that started the flow. -**SAML is out of scope and will stay out of scope.** XML-DSig canonicalisation has no Bun native -and implementing it would mean a real dependency in the primitive vocabulary, which -[`docs/idea/18-build-vs-wrap.md`](../../docs/idea/18-build-vs-wrap.md) does not permit. The honest -answer is to put an OIDC-speaking SAML bridge (Okta, Entra, Keycloak, Dex, a SAML-to-OIDC proxy) in -front and register **that** as a provider. +**SAML 2.0 is supported natively since 31.0.0** — see [SSO: one IdentityProvider port, OIDC and +SAML](#sso-one-identityprovider-port-oidc-and-saml). No dependency: the XML reader, exclusive c14n +and XML-DSig verification are this package's own, held to xmlsec-signed vectors. ## Id token signatures @@ -845,7 +843,9 @@ const mounted = (leg: AuthRouteDescriptor): Route => ({ method: leg.method, path: leg.path, handler: (request) => leg.handle(request.raw), - meta: { name: leg.name, auth: leg.auth }, + // Verbatim: the SAML ACS (POST) declares `csrf: 'cookieless'` + `authenticate: () => null`, + // because the IdP answers with a cross-site form. Hand-copy `name`/`auth` and it is a 403. + meta: leg.meta, }); export const runtime: NonNullable = { @@ -855,8 +855,8 @@ export const runtime: NonNullable = { | | `start` | `callback` | |---|---|---| -| path | `/auth/oauth/:provider` | `/auth/oauth/:provider/callback` | -| success | `302` to the provider, `Set-Cookie: __Host-x_oauth_` | `303` to `successPath`, `Set-Cookie: __Host-x_session` **and** the handshake cleared | +| path | `/auth/sso/:provider` | `/auth/sso/:provider/callback` | +| success | `302` to the provider, `Set-Cookie: __Host-x_sso_` | `303` to `successPath`, `Set-Cookie: __Host-x_session` **and** the handshake cleared | | failure | coded JSON, status per code: `code` + `title` + `docs`, one fixed `cause`, `fix: x errors explain --json` | the same, handshake cleared either way | A **descriptor**, never a mounted handler — the same category as `mcpHttpRoute()`. `@ultimat3/http` @@ -866,9 +866,11 @@ router that can match a `:param` — and `runtime.routes` is the one an app has. framework's own routes and before the pages; every field it shares the file with is in [`wiki/Configuration.md`](../../wiki/Configuration.md#runtime-overrides--appsappruntimets). -**The paths are not configurable.** `X_OAUTH_STATE_INVALID` has always told the caller to restart -at `GET /auth/oauth/`; it now quotes `oauthStartPath()`, the same declaration the mount -reads. A movable base path is that sentence going stale again. +**The paths are not configurable.** `X_OAUTH_STATE_INVALID` quotes `oauthStartPath()`, the same +declaration the mount reads. A movable base path is that sentence going stale again. Since 31.0.0 +the base is `/auth/sso`: `oauthLogin` is a front door over `identityProviderRoutes`' legs, so OAuth +and SAML share the paths, the sealed state cookie and the refusal body (re-register the callback +URL at each IdP — `wiki/Upgrading.md`). `oauthLogin` needs an origin to build the provider's `redirect_uri` from: `baseUrl`, or `APP_URL`. With neither, the start leg refuses with `X_ENV_MISSING` rather than trust a request header. @@ -905,6 +907,10 @@ SameSite=Lax` under a `__Host-` name, and expired against the server's clock rat client's copy of `Max-Age`. `sealHandshake` / `openHandshake` are the same codec without the cookie, for an app that would rather keep it server-side. +`oauthLogin` itself no longer uses this cookie (31.0.0): its state rides the sign-in routes' own +`__Host-x_sso_`, sealed the same way and claimed once in `auth.totpReplay`. The +`handshakeCookie` family stays for a hand-built flow over `beginOAuth` / `completeOAuthLogin`. + **One cookie per provider:** `handshakeCookieName(provider)` → `__Host-x_oauth_github`. A browser is one cookie jar and a user is allowed two tabs, so a single shared name means the `google` redirect overwrites a `github` handshake still in flight — and the github callback then opens @@ -949,6 +955,64 @@ both callback routes are GET. An app that needs it registers its own with `regis `loginFailed()` the taken-address case answers. Minting one is a pre-hijack: the owner's `register()` then collides and their verified login is refused, with the attacker still linked. +## SSO: one IdentityProvider port, OIDC and SAML + +Every federated sign-in is an `IdentityProvider`: `begin` sends the browser out, `complete` takes +the answer back and proves one `OAuthProfile`. Two adapters, one set of routes, one session path. + +```ts +import { + defineAuth, + identityProviderRoutes, + memoryAuthAdapter, + oidcIdentityProvider, + parseSamlMetadata, + samlIdentityProvider, +} from '@ultimat3/auth'; + +// The app's group → role mapping: business convention, so it is yours (axiom 8). +declare function groupsToRoles(claims: Readonly>): string[]; + +const auth = defineAuth({ adapter: memoryAuthAdapter(), providers: ['corp', 'github'] }); +const routes = identityProviderRoutes(auth, { + providers: [ + samlIdentityProvider({ + id: 'corp', + idp: parseSamlMetadata(await Bun.file('config/corp-idp-metadata.xml').text()), + spEntityId: 'https://app.example.com/saml/metadata', + trustEmail: true, // required: does this IdP vouch for the addresses it asserts? + emailDomains: ['corp.example'], // required with true: only these domains are believed + }), + oidcIdentityProvider({ provider: 'github' }), + ], + resolveGrants: (identity) => ({ roles: groupsToRoles(identity.claims) }), // yours, never shipped +}); +// routes.start GET /auth/sso/:provider → 302 to the IdP +// routes.acs POST /auth/sso/:provider/callback → SAML ACS (register this URL at the IdP) +// routes.callback GET /auth/sso/:provider/callback → OIDC redirect_uri +``` + +| Rule | Why | +|---|---| +| The assertion **itself** must be signed; a signed response around an unsigned assertion is refused | the node that is read is the node that was verified — no wrapping | +| Exactly one `saml:Assertion` in the whole document, a direct child of the Response; no ID twice; no `ds:Object` | the signature-wrapping (XSW) attacks all need a second candidate | +| The `Reference` must name the enveloping element's own `ID`; transforms exactly enveloped-signature + exc-c14n | no XPath/XSLT, no detached reference | +| RSA-SHA256 with SHA-256 digests, RSA ≥ 2048 | SHA-1 is refused | +| Keys come from metadata only; `KeyInfo` is never read | a self-signed cert in the response proves nothing | +| NameID text is every text node concatenated | `a@corp.com.evil.com` reads as what was signed | +| Recipient, Destination, InResponseTo, Audience, NotBefore/NotOnOrAfter (±`skewMs`, 60 s) | SP-initiated only: an unsolicited response has no InResponseTo to match | +| One answer completes once: claimed in `auth.totpReplay` (the shared replay store) until it expires | replay across replicas | +| DOCTYPE, entities, processing instructions refused by the reader | XXE, billion laughs | +| An address is verified only inside `emailDomains` (exact; list subdomains); outside, it can neither link nor create an account. `trustEmail: true` with no domains does not compile, and refuses at construction (`X_CONFIG_INVALID`) | an IdP's admin can assert ANY address — unbound, one tenant's IdP signs in as another tenant's verified user | +| Transient NameIDs refused | linked by a per-login id, every sign-in is a stranger | +| Encrypted assertions refused (`meta.check: 'encrypted'`) | not supported | + +Every SAML refusal is `X_SAML_RESPONSE_INVALID` with the failed check in `meta.check` and a fix per +check, logged as `auth.sso.refused`; the anonymous caller gets the code and title only. Bad metadata +is `X_SAML_METADATA_INVALID` at boot. An id the routes do not serve is `X_IDENTITY_PROVIDER_UNKNOWN`. +An adapter of your own is held to the same contract by +`identityProviderConformance` (`@ultimat3/testing/conformance`). + ## API keys — how an agent authenticates `ult___`. The plaintext is shown once; the row holds `sha256(secret)` and is diff --git a/packages/auth/src/auth.ts b/packages/auth/src/auth.ts index ca4d9b228..166feedcc 100644 --- a/packages/auth/src/auth.ts +++ b/packages/auth/src/auth.ts @@ -162,7 +162,7 @@ export interface AuthConfigInput { readonly totpReplay?: TotpReplayGuard | undefined; /** * The OAuth providers this app serves login routes for. Defaults to `[]` — an empty list is - * "no OAuth", and every `/auth/oauth/` answers `X_OAUTH_PROVIDER_UNKNOWN`. Never the live + * "no OAuth", and every `/auth/sso/` answers `X_OAUTH_PROVIDER_UNKNOWN`. Never the live * registry, which a dependency can write into. */ readonly providers?: readonly OAuthProviderId[] | undefined; diff --git a/packages/auth/src/client-scope.ts b/packages/auth/src/client-scope.ts index 0e472b60e..b00cfc880 100644 --- a/packages/auth/src/client-scope.ts +++ b/packages/auth/src/client-scope.ts @@ -47,7 +47,7 @@ function scopeKey(options: ClientScopeOptions): string { throw new AuthError({ code: 'X_CONFIG_INVALID', cause: `clientScopeOf({ secret }) was passed a key shorter than the ${String(MIN_SECRET_LENGTH)} characters required, so the scope id would be keyed by a guessable secret`, - fix: 'omit secret so clientScopeOf reads SESSION_SECRET, or pass one generated with: openssl rand -hex 32', + fix: 'openssl rand -hex 32 # pass the output as clientScopeOf({ secret }), or omit secret to read SESSION_SECRET', meta: { option: 'secret', minLength: MIN_SECRET_LENGTH }, }); } diff --git a/packages/auth/src/errors.ts b/packages/auth/src/errors.ts index 837107bca..f8048b410 100644 --- a/packages/auth/src/errors.ts +++ b/packages/auth/src/errors.ts @@ -39,6 +39,9 @@ export const AUTH_OWNED_ERROR_CODES = [ 'X_PASSKEY_INVALID', 'X_PASSKEY_CHALLENGE_EXPIRED', 'X_PASSKEY_STEP_UP_REQUIRED', + 'X_SAML_RESPONSE_INVALID', + 'X_SAML_METADATA_INVALID', + 'X_IDENTITY_PROVIDER_UNKNOWN', ] as const; /** @@ -99,6 +102,11 @@ export const AUTH_ERROR_TITLES: Readonly> = { 'a passkey ceremony named a challenge that is unknown, already used or past its ttl', X_PASSKEY_STEP_UP_REQUIRED: 'a sensitive action ran without a fresh user-verified passkey proof for its digest', + X_SAML_RESPONSE_INVALID: + 'a saml response failed its signature, wrapping, audience, recipient, time, InResponseTo or replay check', + X_SAML_METADATA_INVALID: + 'saml IdP metadata has no entity id, HTTP-Redirect sign-on endpoint or signing certificate', + X_IDENTITY_PROVIDER_UNKNOWN: 'the URL named an identity provider the sso routes were not given', }; // Registered unconditionally, in one call: a second package claiming a code auth owns has to fail diff --git a/packages/auth/src/identity-provider-cookie.ts b/packages/auth/src/identity-provider-cookie.ts new file mode 100644 index 000000000..ef3d3666b --- /dev/null +++ b/packages/auth/src/identity-provider-cookie.ts @@ -0,0 +1,89 @@ +// Single responsibility: where an `IdentityRedirect.state` lives between the two legs — HMAC- +// signed with `SESSION_SECRET`, bound to its provider, aged on the server's clock, one cookie per +// provider. SAML answers with a cross-site POST no `SameSite=Lax` cookie rides, so a POST-callback +// provider's cookie is `SameSite=None`; `Secure` and `__Host-` keep it off every other site. + +import { type Clock, hmac, readCookie, serializeSetCookie, timingSafeEqual } from '@ultimat3/core'; +import type { IdentityProvider } from './identity-provider'; +import { DEFAULT_HANDSHAKE_TTL_MS, handshakeSecret } from './oauth-cookie'; +import { oauthStateInvalid } from './oauth-errors'; +import { base64Url, base64UrlBytes } from './tokens'; + +export const IDENTITY_STATE_COOKIE_PREFIX = '__Host-x_sso'; + +export const identityStateCookieName = (provider: string): string => + `${IDENTITY_STATE_COOKIE_PREFIX}_${provider}`; + +export interface IdentityStateOptions { + readonly clock: Clock; + /** Defaults to `SESSION_SECRET`. */ + readonly secret?: string | undefined; +} + +const sign = (body: string, secret: string): string => hmac('sha256', secret, body, 'hex'); + +export function identityStateCookie( + provider: IdentityProvider, + state: string, + options: IdentityStateOptions, +): string { + const body = base64Url( + new TextEncoder().encode(JSON.stringify([provider.id, state, options.clock.now().getTime()])), + ); + const sealed = `${body}.${sign(body, options.secret ?? handshakeSecret())}`; + return serializeSetCookie(identityStateCookieName(provider.id), sealed, { + maxAge: Math.floor(DEFAULT_HANDSHAKE_TTL_MS / 1000), + sameSite: provider.callbackMethod === 'POST' ? 'None' : 'Lax', + }); +} + +/** + * Sent with every callback answer: the state it carried is spent whatever the outcome. Takes the id + * and method rather than a provider, so a refusal before the provider resolved can still clear it. + */ +export const clearIdentityStateCookie = ( + id: string, + callbackMethod: IdentityProvider['callbackMethod'], +): string => + serializeSetCookie(identityStateCookieName(id), '', { + maxAge: 0, + sameSite: callbackMethod === 'POST' ? 'None' : 'Lax', + }); + +/** The state `begin` returned, or `X_OAUTH_STATE_INVALID` — one refusal for every way it fails. */ +export function readIdentityState( + request: Request, + provider: IdentityProvider, + options: IdentityStateOptions, +): string { + const sealed = readCookie(request.headers.get('cookie'), identityStateCookieName(provider.id)); + if (sealed === null) throw oauthStateInvalid(provider.id, 'no sign-in state cookie arrived'); + const dot = sealed.lastIndexOf('.'); + const body = sealed.slice(0, Math.max(dot, 0)); + if ( + dot <= 0 || + !timingSafeEqual(sign(body, options.secret ?? handshakeSecret()), sealed.slice(dot + 1)) + ) { + throw oauthStateInvalid( + provider.id, + 'the sign-in state was tampered with, or the secret rotated', + ); + } + let parsed: unknown; + try { + parsed = JSON.parse(new TextDecoder().decode(base64UrlBytes(body) ?? new Uint8Array())); + } catch { + parsed = null; + } + if (!Array.isArray(parsed) || parsed.length !== 3) { + throw oauthStateInvalid(provider.id, 'the sign-in state is not readable'); + } + const [id, state, issuedAt] = parsed as readonly unknown[]; + if (id !== provider.id || typeof state !== 'string' || typeof issuedAt !== 'number') { + throw oauthStateInvalid(provider.id, 'the sign-in state belongs to another provider'); + } + if (options.clock.now().getTime() - issuedAt > DEFAULT_HANDSHAKE_TTL_MS) { + throw oauthStateInvalid(provider.id, 'the sign-in state expired before the IdP answered'); + } + return state; +} diff --git a/packages/auth/src/identity-provider-oidc.test.ts b/packages/auth/src/identity-provider-oidc.test.ts new file mode 100644 index 000000000..bd062ba9f --- /dev/null +++ b/packages/auth/src/identity-provider-oidc.test.ts @@ -0,0 +1,68 @@ +// The OIDC adapter's single-use claim, across time: a callback replayed later inside the state's +// ten-minute life must still meet its first claim. The claim's step is fixed by when the handshake +// was ISSUED, never by when a completion arrives — or each 30 s step is a fresh row. + +import { describe, expect, test } from 'bun:test'; +import { frozenClock, isUltimateError } from '@ultimat3/core'; +import { oidcIdentityProvider } from './identity-provider-oidc'; +import { memoryTotpReplayGuard } from './mfa'; +import { DEFAULT_HANDSHAKE_TTL_MS } from './oauth-cookie'; +import type { OAuthFetch } from './oauth-exchange'; +import { credentials, json } from './oauth-login-fixture'; + +const CALLBACK = 'https://app.example.test/auth/sso/github/callback'; + +const githubFetch: OAuthFetch = async (url) => { + if (url.endsWith('/access_token')) return json({ access_token: 'gho_x', token_type: 'bearer' }); + if (url.endsWith('/user')) return json({ id: 583231, login: 'octocat' }); + return json([{ email: 'octo@example.com', primary: true, verified: true }]); +}; + +const outcome = (call: Promise): Promise => + call.then( + () => 'accepted', + (error: unknown) => (isUltimateError(error) ? error.code : 'uncoded'), + ); + +describe('oidcIdentityProvider: a completed handshake stays completed', () => { + for (const laterMs of [61_000, 5 * 60_000, DEFAULT_HANDSHAKE_TTL_MS - 1_000]) { + test(`a callback replayed ${laterMs / 1000} s later is refused`, async () => { + const clock = frozenClock('2026-11-09T12:00:00.000Z'); + const replay = memoryTotpReplayGuard(); + const provider = oidcIdentityProvider({ + provider: 'github', + credentials, + fetch: githubFetch, + }); + const redirect = await provider.begin({ callbackUrl: CALLBACK, clock }); + const state = new URL(redirect.location).searchParams.get('state') ?? ''; + const complete = () => + provider.complete({ + state: redirect.state, + params: { state, code: 'c0de' }, + callbackUrl: CALLBACK, + clock, + replay, + }); + expect(await outcome(complete())).toBe('accepted'); + clock.advance(laterMs); + expect(await outcome(complete())).toBe('X_OAUTH_STATE_INVALID'); + }); + } + + test('a handshake past its ten minutes is refused before anything is claimed', async () => { + const clock = frozenClock('2026-11-09T12:00:00.000Z'); + const provider = oidcIdentityProvider({ provider: 'github', credentials, fetch: githubFetch }); + const redirect = await provider.begin({ callbackUrl: CALLBACK, clock }); + const state = new URL(redirect.location).searchParams.get('state') ?? ''; + clock.advance(DEFAULT_HANDSHAKE_TTL_MS + 1); + const call = provider.complete({ + state: redirect.state, + params: { state, code: 'c0de' }, + callbackUrl: CALLBACK, + clock, + replay: memoryTotpReplayGuard(), + }); + expect(await outcome(call)).toBe('X_OAUTH_STATE_INVALID'); + }); +}); diff --git a/packages/auth/src/identity-provider-oidc.ts b/packages/auth/src/identity-provider-oidc.ts new file mode 100644 index 000000000..0181af069 --- /dev/null +++ b/packages/auth/src/identity-provider-oidc.ts @@ -0,0 +1,142 @@ +// Single responsibility: a registered OAuth2/OIDC provider as an `IdentityProvider` — the existing +// handshake (`beginOAuth`), exchange (`exchangeOAuthCode`) and profile read (`readOAuthProfile`) +// behind the port, no protocol of its own. The port adds one thing: the handshake's `state` is +// claimed ONCE in the shared replay store, keyed on when it was issued, before the code is spent. + +import { timingSafeEqual } from '@ultimat3/core'; +import { + type IdentityProvider, + type IdentityRedirect, + replayStepFor, + type VerifiedIdentity, +} from './identity-provider'; +import { isRecord } from './json'; +import type { IdTokenKeys } from './jwks'; +import { + assertOAuthCallback, + beginOAuth, + type OAuthHandshake, + type OAuthProviderId, +} from './oauth'; +import { DEFAULT_HANDSHAKE_TTL_MS } from './oauth-cookie'; +import { oauthDenied, oauthStateInvalid } from './oauth-errors'; +import { + exchangeOAuthCode, + type OAuthClientCredentials, + type OAuthFetch, + oauthCredentials, +} from './oauth-exchange'; +import { oauthGrantContext } from './oauth-grant-context'; +import { readOAuthProfile } from './oauth-profile'; + +export interface OidcIdentityProviderConfig { + /** A registered `OAuthProvider` id (`registerOAuthProvider`, or a built-in). */ + readonly provider: OAuthProviderId; + /** Default: the provider's two env vars, read per call, never at import. */ + readonly credentials?: OAuthClientCredentials | undefined; + readonly scopes?: readonly string[] | undefined; + readonly fetch?: OAuthFetch | undefined; + readonly timeoutMs?: number | undefined; + readonly idTokenKeys?: IdTokenKeys | undefined; +} + +const FIELDS = ['state', 'nonce', 'verifier', 'redirectUri', 'authorizeUrl'] as const; + +/** The handshake, and when it was issued — the instant its single-use claim is keyed on. */ +interface StoredHandshake { + readonly handshake: OAuthHandshake; + readonly issuedAt: number; +} + +function readHandshake(provider: OAuthProviderId, state: string): StoredHandshake { + let parsed: unknown; + try { + parsed = JSON.parse(state); + } catch { + parsed = null; + } + if ( + !isRecord(parsed) || + !FIELDS.every((field) => typeof parsed[field] === 'string') || + typeof parsed['issuedAt'] !== 'number' || + !Number.isFinite(parsed['issuedAt']) + ) { + throw oauthStateInvalid(provider, 'the stored handshake is not a handshake'); + } + const field = (name: (typeof FIELDS)[number]): string => String(parsed[name]); + return { + issuedAt: parsed['issuedAt'], + handshake: { + provider, + state: field('state'), + nonce: field('nonce'), + verifier: field('verifier'), + redirectUri: field('redirectUri'), + authorizeUrl: field('authorizeUrl'), + }, + }; +} + +export function oidcIdentityProvider(config: OidcIdentityProviderConfig): IdentityProvider { + const id = config.provider; + const credentials = (): OAuthClientCredentials => config.credentials ?? oauthCredentials(id); + const provider: IdentityProvider = { + id, + protocol: 'oidc', + callbackMethod: 'GET', + async begin({ callbackUrl, clock }): Promise { + const handshake = beginOAuth({ + provider: id, + clientId: credentials().clientId, + redirectUri: callbackUrl, + scopes: config.scopes, + }); + const state = { + ...Object.fromEntries(FIELDS.map((field) => [field, handshake[field]])), + issuedAt: clock.now().getTime(), + }; + return { location: handshake.authorizeUrl, state: JSON.stringify(state) }; + }, + answers(state, params) { + try { + return timingSafeEqual(readHandshake(id, state).handshake.state, params['state'] ?? ''); + } catch { + return false; + } + }, + async complete({ state, params, callbackUrl, clock, replay }): Promise { + const { handshake, issuedAt } = readHandshake(id, state); + if (handshake.redirectUri !== callbackUrl) { + throw oauthStateInvalid(id, 'the handshake was started for another callback URL'); + } + const declined = params['error']; + if (declined !== undefined && declined !== '') { + throw oauthDenied(id, declined, params['error_description'] ?? null); + } + const callback = { state: params['state'] ?? '', code: params['code'] ?? '' }; + // The state check first, so a forged callback cannot spend the real browser's handshake. + assertOAuthCallback(handshake, callback); + const now = clock.now(); + // Keyed on when the handshake was ISSUED, the way SAML's claim is keyed on NotOnOrAfter: a + // horizon read off `now` moved with every retry, so each 30 s step was a fresh claim row. + const horizon = new Date(issuedAt + DEFAULT_HANDSHAKE_TTL_MS); + if (horizon.getTime() < now.getTime()) { + throw oauthStateInvalid(id, 'the handshake expired before the provider answered'); + } + if (!(await replay.claim(`oidc:${id}:${handshake.state}`, replayStepFor(horizon), now))) { + throw oauthStateInvalid(id, 'this handshake was already completed'); + } + const options = { fetch: config.fetch, timeoutMs: config.timeoutMs }; + const tokens = await exchangeOAuthCode(handshake, callback, { + credentials: credentials(), + clock, + ...options, + keys: config.idTokenKeys, + }); + const { profile, userinfo } = await readOAuthProfile(id, tokens, options); + const oauth = oauthGrantContext(profile, tokens, options, userinfo); + return { profile, expiresAt: tokens.expiresAt, claims: oauth.idTokenClaims ?? {}, oauth }; + }, + }; + return Object.freeze(provider); +} diff --git a/packages/auth/src/identity-provider-route-core.ts b/packages/auth/src/identity-provider-route-core.ts new file mode 100644 index 000000000..adf2a3d9b --- /dev/null +++ b/packages/auth/src/identity-provider-route-core.ts @@ -0,0 +1,253 @@ +// Single responsibility: the ONE implementation of the two sign-in legs over any `IdentityProvider` +// — start (`begin`, seal the state, 302) and callback (open it, `complete`, `signInWithIdentity`, +// 303; coded JSON on refusal). `identityProviderRoutes` and `oauthLogin` are this function with two +// policies, so the state cookie, the refusal body and the redirect rules cannot drift apart. + +import { + EnvMissingError, + isUltimateError, + logger, + renderFixShellArg, + type UltimateError, +} from '@ultimat3/core'; +import type { Auth } from './auth'; +import { + type IdentityProvider, + signInWithIdentity, + type VerifiedIdentity, +} from './identity-provider'; +import { + clearIdentityStateCookie, + identityStateCookie, + readIdentityState, +} from './identity-provider-cookie'; +import { oauthRouteStatus } from './identity-provider-status'; +import { oauthStateInvalid } from './oauth-errors'; +import type { OAuthGrants } from './oauth-login'; +import { + OAUTH_CALLBACK_ROUTE_PATH, + OAUTH_START_ROUTE_PATH, + oauthCallbackPath, +} from './oauth-paths'; +import { samlResponseInvalid } from './saml-errors'; + +export interface SignInRouteDescriptor { + readonly method: 'GET' | 'POST'; + /** The mount pattern, `:provider` included. The handler re-reads it off the request itself. */ + readonly path: string; + readonly name: string; + /** Both legs are public by definition — they are how an anonymous visitor stops being one. */ + readonly auth: 'public'; + /** What a router mounts as the route's `meta`, verbatim: `meta: leg.meta`. */ + readonly meta: SignInRouteMeta; + handle(request: Request): Promise; +} + +/** + * The route meta, structurally `@ultimat3/http`'s (`Route['meta']`): this package is its tier and + * cannot import it. The POST leg — SAML's ACS — is `csrf: 'cookieless'` with its own + * `authenticate: () => null`: the IdP answers with an auto-submitted CROSS-SITE form, which the csrf + * stage refuses on any other route. CSRF is still defended there, by the protocol: the answer must + * carry this browser's RelayState (`answers()`, timing-safe against the HMAC-sealed state cookie) + * and its assertion must name this request's ID as InResponseTo — an attacker's form has neither. + * And no session cookie decides anything on it. + */ +export interface SignInRouteMeta { + readonly name: string; + readonly auth: 'public'; + // No `| undefined`: http's `Route['meta']` takes these absent or set, never `undefined`. + readonly csrf?: 'cookieless'; + readonly authenticate?: () => null; +} + +/** What both front doors accept. */ +export interface SignInRouteOptions { + /** The origin the IdP answers to. Defaults to `APP_URL`; never the request's `Host`. */ + readonly baseUrl?: string | undefined; + /** Defaults to `SESSION_SECRET`. Seals the state across the two requests. */ + readonly secret?: string | undefined; + /** A fixed landing path, never `?next=`: `nextAfterSignIn` in `@ultimat3/http` is the one check. */ + readonly successPath?: string | undefined; + /** The client address recorded on the session; `@ultimat3/http` knows the proxy chain. */ + readonly clientIp?: ((request: Request) => string | null | undefined) | undefined; + readonly env?: Readonly> | undefined; +} + +/** What differs between the two front doors. */ +export interface SignInRoutePolicy { + /** The provider a URL segment names and this app enabled, or `unknown(segment)` for both. */ + resolve(segment: string, leg: 'start' | 'callback'): IdentityProvider; + /** The one 404: unknown, not enabled, or answered by the wrong method — never told apart. */ + unknown(segment: string): UltimateError; + /** `auth.sso` / `auth.oauth`: the log event prefix (`.refused`). */ + readonly event: string; + readonly names: { readonly start: string; readonly callback: string; readonly acs: string }; + /** A throw with no code, logged and turned into the refusal served. Its text never is. */ + uncoded(error: unknown): UltimateError; + grants(identity: VerifiedIdentity): Promise | OAuthGrants | undefined; +} + +export interface SignInRoutes { + readonly start: SignInRouteDescriptor; + /** OIDC's redirect back. */ + readonly callback: SignInRouteDescriptor; + /** SAML's Assertion Consumer Service: the same path, POSTed. */ + readonly acs: SignInRouteDescriptor; +} + +const MAX_FORM_BYTES = 512 * 1024; +const FRAMEWORK_CODE = /^X_[A-Z0-9_]+$/; + +function problem(policy: SignInRoutePolicy, error: unknown, cookies: readonly string[]): Response { + const refusal = isUltimateError(error) ? error : policy.uncoded(error); + const status = oauthRouteStatus(refusal.code); + const fields = { code: refusal.code, status, cause: refusal.cause, fix: refusal.fix }; + if (status >= 500) logger.error(`${policy.event}.refused`, fields); + else logger.info(`${policy.event}.refused`, fields); + const headers = new Headers({ 'content-type': 'application/json; charset=utf-8' }); + for (const cookie of cookies) headers.append('set-cookie', cookie); + // An anonymous caller reads the code, its title and docs, and one fixed sentence; the authored + // cause and fix name env vars and what an IdP answered, so they go to the log only. + const body = { + code: refusal.code, + title: refusal.title, + cause: `the sign-in was refused; its cause and its fix are in this server’s log under ${policy.event}.refused`, + fix: FRAMEWORK_CODE.test(refusal.code) + ? `x errors explain ${renderFixShellArg(refusal.code, '')} --json` + : 'x errors list --json', + docs: refusal.docs, + }; + return new Response(JSON.stringify(body), { status, headers }); +} + +function segmentOf(request: Request, leg: 'start' | 'callback'): string { + const parts = new URL(request.url).pathname.split('/').filter((part) => part !== ''); + return parts[leg === 'start' ? parts.length - 1 : parts.length - 2] ?? ''; +} + +function origin(options: SignInRouteOptions): string { + const declared = (options.baseUrl ?? (options.env ?? Bun.env)['APP_URL'] ?? '').trim(); + if (declared === '') { + throw new EnvMissingError({ + cause: + 'APP_URL is not set and the sign-in routes were given no baseUrl, so there is no declared origin to build the IdP callback URL from', + fix: 'export APP_URL=http://localhost:3000 # the origin this app is served from', + meta: { key: 'APP_URL' }, + }); + } + return declared.replace(/\/+$/, ''); +} + +/** + * The form body, never more than `MAX_FORM_BYTES` of it: a declared length over the cap refuses + * before one byte is read, and an undeclared or lying one is cut off at the cap as it streams. + */ +async function formParams(request: Request, provider: string): Promise> { + const tooLarge = () => samlResponseInvalid(provider, 'encoding', 'the posted form is too large'); + const declared = Number(request.headers.get('content-length') ?? '0'); + if (!Number.isFinite(declared) || declared > MAX_FORM_BYTES) throw tooLarge(); + const decoder = new TextDecoder(); + let text = ''; + let size = 0; + const reader = request.body?.getReader(); + for (;;) { + const next = await reader?.read(); + if (next === undefined || next.done) break; + size += next.value.byteLength; + if (size > MAX_FORM_BYTES) { + await reader?.cancel(); + throw tooLarge(); + } + text += decoder.decode(next.value, { stream: true }); + } + text += decoder.decode(); + return Object.fromEntries(new URLSearchParams(text)); +} + +export function signInRoutes( + auth: Auth, + options: SignInRouteOptions, + policy: SignInRoutePolicy, +): SignInRoutes { + const seal = { + clock: auth.clock, + ...(options.secret === undefined ? {} : { secret: options.secret }), + }; + + const start = async (request: Request): Promise => { + try { + // Before the provider: asked after, a missing origin answered 500 for an enabled provider + // and 404 for one that is not — an oracle for the app's configuration. + const base = origin(options); + const provider = policy.resolve(segmentOf(request, 'start'), 'start'); + const callbackUrl = `${base}${oauthCallbackPath(provider.id)}`; + const redirect = await provider.begin({ callbackUrl, clock: auth.clock }); + const headers = new Headers({ location: redirect.location }); + headers.append('set-cookie', identityStateCookie(provider, redirect.state, seal)); + return new Response(null, { status: 302, headers }); + } catch (error) { + return problem(policy, error, []); + } + }; + + const callback = async (request: Request): Promise => { + const segment = segmentOf(request, 'callback'); + // Set only once the answer is bound to this browser's state; then cleared on every outcome, + // since that answer is spent either way. An unbound answer — a junk cross-site POST riding the + // SameSite=None cookie — clears nothing, so it cannot end a sign-in in flight. + const clear: string[] = []; + try { + const base = origin(options); + const provider = policy.resolve(segment, 'callback'); + // A provider answers by one method: SAML's POST binding, or OIDC's redirect. + if (request.method !== provider.callbackMethod) throw policy.unknown(provider.id); + const params = + request.method === 'POST' + ? await formParams(request, provider.id) + : Object.fromEntries(new URL(request.url).searchParams); + const state = readIdentityState(request, provider, seal); + if (!provider.answers(state, params)) { + throw oauthStateInvalid(provider.id, 'the answer does not carry this browser\u2019s state'); + } + clear.push(clearIdentityStateCookie(provider.id, provider.callbackMethod)); + const identity = await provider.complete({ + state, + params, + callbackUrl: `${base}${oauthCallbackPath(provider.id)}`, + clock: auth.clock, + replay: auth.totpReplay, + }); + const grants = await policy.grants(identity); + const result = await signInWithIdentity(auth, identity, { + ...(grants === undefined ? {} : { grants }), + ip: options.clientIp?.(request) ?? null, + userAgent: request.headers.get('user-agent'), + }); + const headers = new Headers({ location: options.successPath ?? '/' }); + headers.append('set-cookie', result.cookie); + for (const cookie of clear) headers.append('set-cookie', cookie); + // 303: the answer may arrive as a POST, and the destination is a GET either way. + return new Response(null, { status: 303, headers }); + } catch (error) { + return problem(policy, error, clear); + } + }; + + const route = ( + method: 'GET' | 'POST', + path: string, + name: string, + handle: (request: Request) => Promise, + ): SignInRouteDescriptor => { + const meta: SignInRouteMeta = + method === 'POST' + ? { name, auth: 'public', csrf: 'cookieless', authenticate: () => null } + : { name, auth: 'public' }; + return Object.freeze({ method, path, name, auth: 'public', meta: Object.freeze(meta), handle }); + }; + return Object.freeze({ + start: route('GET', OAUTH_START_ROUTE_PATH, policy.names.start, start), + callback: route('GET', OAUTH_CALLBACK_ROUTE_PATH, policy.names.callback, callback), + acs: route('POST', OAUTH_CALLBACK_ROUTE_PATH, policy.names.acs, callback), + }); +} diff --git a/packages/auth/src/identity-provider-routes.test.ts b/packages/auth/src/identity-provider-routes.test.ts new file mode 100644 index 000000000..8e5d4df3a --- /dev/null +++ b/packages/auth/src/identity-provider-routes.test.ts @@ -0,0 +1,221 @@ +// The two legs over both adapters, as a browser drives them: start → IdP → callback → session, +// and every way the callback refuses (no state cookie, a replay, an unknown id, a wrong method). + +import { describe, expect, test } from 'bun:test'; +import { frozenClock } from '@ultimat3/core'; +import { defineAuth } from './auth'; +import type { IdentityProvider } from './identity-provider'; +import { oidcIdentityProvider } from './identity-provider-oidc'; +import { identityProviderRoutes } from './identity-provider-routes'; +import { memoryAuthAdapter } from './memory-adapter'; +import type { OAuthFetch } from './oauth-exchange'; +import { credentials, json } from './oauth-login-fixture'; +import { encodeResponse, readAuthnRequest, samlTestIdp, type TestIdp } from './saml-idp-fixture'; +import { samlIdentityProvider } from './saml-provider'; +import { ACS_URL, SAML_NOW, SP_ENTITY_ID } from './saml-test-support-fixture'; + +const BASE = 'https://app.example.test'; +const SECRET = 's'.repeat(40); + +const githubFetch: OAuthFetch = async (url) => { + if (url === 'https://github.com/login/oauth/access_token') + return json({ access_token: 'gho_x', token_type: 'bearer' }); + if (url === 'https://api.github.com/user') return json({ id: 583231, login: 'octocat' }); + if (url === 'https://api.github.com/user/emails') { + return json([{ email: 'octo@example.com', primary: true, verified: true }]); + } + return expect.unreachable(`unexpected endpoint: ${url}`); +}; + +async function setup(extra: readonly IdentityProvider[] = []) { + const idp: TestIdp = await samlTestIdp(); + const adapter = memoryAuthAdapter(); + const corp = samlIdentityProvider({ + id: 'corp', + idp: idp.idp, + spEntityId: SP_ENTITY_ID, + trustEmail: true, + emailDomains: ['corp.example'], + }); + const github = oidcIdentityProvider({ provider: 'github', credentials, fetch: githubFetch }); + const auth = defineAuth({ adapter, clock: frozenClock(SAML_NOW), providers: ['corp', 'github'] }); + const routes = identityProviderRoutes(auth, { + providers: [corp, github, ...extra], + baseUrl: BASE, + secret: SECRET, + successPath: '/home', + }); + return { idp, adapter, routes }; +} + +const cookiePair = (setCookie: string | null): string => (setCookie ?? '').split(';')[0] ?? ''; + +describe('identityProviderRoutes: SAML', () => { + async function started() { + const env = await setup(); + const out = await env.routes.start.handle(new Request(`${BASE}/auth/sso/corp`)); + const request = readAuthnRequest(out.headers.get('location') ?? ''); + const xml = await env.idp.responseXml({ + requestId: request.id, + acsUrl: ACS_URL, + audience: SP_ENTITY_ID, + now: SAML_NOW, + }); + const post = (cookie: string) => + env.routes.acs.handle( + new Request(`${BASE}/auth/sso/corp/callback`, { + method: 'POST', + headers: { cookie, 'content-type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ + SAMLResponse: encodeResponse(xml), + RelayState: request.relayState, + }), + }), + ); + return { ...env, out, post, cookie: cookiePair(out.headers.get('set-cookie')) }; + } + + test('start redirects to the IdP and seals the request in a SameSite=None cookie', async () => { + const { out } = await started(); + expect(out.status).toBe(302); + expect(out.headers.get('location')).toStartWith('https://idp.example.test/sso?SAMLRequest='); + expect(out.headers.get('set-cookie')).toContain('SameSite=None'); + expect(out.headers.get('set-cookie')).toStartWith('__Host-x_sso_corp='); + }); + + test('the ACS POST signs the user in and lands on the fixed path', async () => { + const { post, cookie, adapter } = await started(); + const answer = await post(cookie); + expect(answer.status).toBe(303); + expect(answer.headers.get('location')).toBe('/home'); + expect(answer.headers.getSetCookie().some((c) => c.startsWith('__Host-x_session='))).toBe(true); + expect((await adapter.findUserByEmail('ada@corp.example'))?.email).toBe('ada@corp.example'); + }); + + test('the same POST twice is refused the second time', async () => { + const { post, cookie } = await started(); + expect((await post(cookie)).status).toBe(303); + const replay = await post(cookie); + expect(replay.status).toBe(400); + expect(((await replay.json()) as { code: string }).code).toBe('X_SAML_RESPONSE_INVALID'); + }); + + test("a junk cross-site POST carrying the victim's cookie leaves their sign-in alive", async () => { + const { routes, post, cookie } = await started(); + const junk = await routes.acs.handle( + new Request(`${BASE}/auth/sso/corp/callback`, { + method: 'POST', + headers: { cookie, 'content-type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ SAMLResponse: 'junk', RelayState: 'not-theirs' }), + }), + ); + expect(junk.status).toBe(400); + expect(junk.headers.getSetCookie().some((c) => c.startsWith('__Host-x_sso_corp=;'))).toBe( + false, + ); + expect((await post(cookie)).status).toBe(303); + }); + + test('a body over the cap is refused from its content-length, before it is read', async () => { + const { routes, cookie } = await started(); + let pulled = 0; + const body = new ReadableStream({ + pull(controller) { + pulled += 1; + controller.enqueue(new Uint8Array(64 * 1024)); + if (pulled > 64) controller.close(); + }, + }); + const answer = await routes.acs.handle( + new Request(`${BASE}/auth/sso/corp/callback`, { + method: 'POST', + headers: { cookie, 'content-length': String(4 * 1024 * 1024) }, + body, + }), + ); + expect(answer.status).toBe(400); + expect(pulled).toBeLessThanOrEqual(1); + }); + + test('a POST with no state cookie (another browser) is refused', async () => { + const { post } = await started(); + const answer = await post(''); + expect(answer.status).toBe(400); + expect(((await answer.json()) as { code: string }).code).toBe('X_OAUTH_STATE_INVALID'); + }); + + test("a tenant's own IdP asserting another tenant's verified address cannot sign in as them", async () => { + const env = await setup(); + const victim = await env.adapter.createUser({ + id: '00000000-0000-7000-8000-000000000001', + email: 'ceo@other-tenant.example', + passwordHash: null, + orgId: 'other-tenant', + roles: ['owner'], + scopes: [], + externalId: null, + createdAt: SAML_NOW, + }); + await env.adapter.updateUser(victim.id, { emailVerifiedAt: SAML_NOW }); + const out = await env.routes.start.handle(new Request(`${BASE}/auth/sso/corp`)); + const request = readAuthnRequest(out.headers.get('location') ?? ''); + const xml = await env.idp.responseXml({ + requestId: request.id, + acsUrl: ACS_URL, + audience: SP_ENTITY_ID, + now: SAML_NOW, + email: 'ceo@other-tenant.example', + }); + const answer = await env.routes.acs.handle( + new Request(`${BASE}/auth/sso/corp/callback`, { + method: 'POST', + headers: { cookie: cookiePair(out.headers.get('set-cookie')) }, + body: new URLSearchParams({ + SAMLResponse: encodeResponse(xml), + RelayState: request.relayState, + }), + }), + ); + expect(answer.status).toBe(401); + expect(answer.headers.getSetCookie().some((c) => c.startsWith('__Host-x_session='))).toBe( + false, + ); + expect(await env.adapter.findAccount('corp', 'u-1001')).toBeNull(); + }); + + test('a GET on the SAML callback, and an unknown id, are one 404', async () => { + const { routes } = await setup(); + const get = await routes.callback.handle(new Request(`${BASE}/auth/sso/corp/callback`)); + const unknown = await routes.start.handle(new Request(`${BASE}/auth/sso/okta`)); + expect([get.status, unknown.status]).toEqual([404, 404]); + }); + + test('two providers under one id refuse at construction', async () => { + const idp = await samlTestIdp(); + const twin = samlIdentityProvider({ + id: 'corp', + idp: idp.idp, + spEntityId: 'x', + trustEmail: false, + }); + await expect(setup([twin])).rejects.toMatchObject({ code: 'X_CONFIG_INVALID' }); + }); +}); + +describe('identityProviderRoutes: OIDC behind the same port', () => { + test('start → callback signs in, and the callback cannot be replayed', async () => { + const { routes, adapter } = await setup(); + const out = await routes.start.handle(new Request(`${BASE}/auth/sso/github`)); + expect(out.headers.get('set-cookie')).toContain('SameSite=Lax'); + const state = new URL(out.headers.get('location') ?? '').searchParams.get('state') ?? ''; + const callback = () => + routes.callback.handle( + new Request(`${BASE}/auth/sso/github/callback?state=${state}&code=c0de`, { + headers: { cookie: cookiePair(out.headers.get('set-cookie')) }, + }), + ); + expect((await callback()).status).toBe(303); + expect((await adapter.findUserByEmail('octo@example.com'))?.email).toBe('octo@example.com'); + expect((await callback()).status).toBe(400); + }); +}); diff --git a/packages/auth/src/identity-provider-routes.ts b/packages/auth/src/identity-provider-routes.ts new file mode 100644 index 000000000..a8b008de7 --- /dev/null +++ b/packages/auth/src/identity-provider-routes.ts @@ -0,0 +1,78 @@ +// Single responsibility: `identityProviderRoutes(auth, { providers })` — the sign-in routes over a +// fixed list of `IdentityProvider`s (SAML, OIDC, an app's own). The legs themselves are +// `identity-provider-route-core.ts`, shared with `oauthLogin`; this file is the list, its one 404, +// and the grant seam over a `VerifiedIdentity`. + +import { + ConfigInvalidError, + logger, + renderCauseValue, + renderThrowable, + type UltimateError, +} from '@ultimat3/core'; +import type { Auth } from './auth'; +import { AuthError } from './errors'; +import type { IdentityProvider, ResolveIdentityGrants } from './identity-provider'; +import { + type SignInRouteDescriptor, + type SignInRouteOptions, + type SignInRoutes, + signInRoutes, +} from './identity-provider-route-core'; + +export type IdentityRouteDescriptor = SignInRouteDescriptor; +export type IdentityProviderRoutes = SignInRoutes; + +export interface IdentityProviderRoutesOptions extends SignInRouteOptions { + readonly providers: readonly IdentityProvider[]; + /** What a verified identity is entitled to — business convention, never shipped (axiom 8). */ + readonly resolveGrants?: ResolveIdentityGrants | undefined; +} + +export const identityProviderUnknown = (segment: string): AuthError => + new AuthError({ + code: 'X_IDENTITY_PROVIDER_UNKNOWN', + cause: `no identity provider ${renderCauseValue(segment)} is served by these routes`, + fix: 'identityProviderRoutes(auth, { providers: [samlIdentityProvider({ id, … })] }) // and add the id to defineAuth({ providers }), or correct the id in the link', + }); + +/** An uncoded throw's text goes to the log only — never into a public body. */ +function uncoded(error: unknown): UltimateError { + logger.error('auth.sso.uncoded_failure', { detail: renderThrowable(error) }); + return new ConfigInvalidError({ + cause: + 'a provider, adapter or fetch this package does not own threw a value with no error code', + fix: 'new AuthError({ code, cause, fix }) // throw it from the provider or adapter that threw; its value is logged as auth.sso.uncoded_failure', + }); +} + +export function identityProviderRoutes( + auth: Auth, + options: IdentityProviderRoutesOptions, +): IdentityProviderRoutes { + const byId = new Map(); + for (const provider of options.providers) { + if (byId.has(provider.id)) { + throw new ConfigInvalidError({ + cause: `two identity providers claim the id ${renderCauseValue(provider.id)}`, + fix: 'identityProviderRoutes(auth, { providers: [first, second] }) // each with its own id', + }); + } + byId.set(provider.id, provider); + } + return signInRoutes(auth, options, { + // Unknown and not-enabled are one 404, so the routes describe no configuration. + resolve(segment) { + const provider = byId.get(segment); + if (provider === undefined || !auth.providers.includes(segment)) { + throw identityProviderUnknown(segment); + } + return provider; + }, + unknown: identityProviderUnknown, + event: 'auth.sso', + names: { start: 'auth.sso.start', callback: 'auth.sso.callback', acs: 'auth.sso.acs' }, + uncoded, + grants: (identity) => options.resolveGrants?.(identity), + }); +} diff --git a/packages/auth/src/identity-provider-status.ts b/packages/auth/src/identity-provider-status.ts new file mode 100644 index 000000000..873ce43d8 --- /dev/null +++ b/packages/auth/src/identity-provider-status.ts @@ -0,0 +1,45 @@ +// Single responsibility: the code → HTTP status table the sign-in route descriptors answer with +// when driven outside a pipeline — one table for every IdentityProvider route, OIDC and SAML alike, +// pinned to `@ultimat3/http`'s `statusFor` by `scripts/oauth-route-status.test.ts` and +// `scripts/identity-route-status.test.ts`. + +/** + * HTTP status per code for the sign-in descriptors (`identityProviderRoutes`, and `oauthLogin` over + * it), driven OUTSIDE a pipeline — a bare `Request` in, a + * `Response` out, which is the whole point of a descriptor. `@ultimat3/http`'s `error-map.ts` OWNS + * these numbers and is where a new one is declared; this package is the same tier and can never + * import it, so the table is a copy the pin `scripts/oauth-route-status.test.ts` holds identical to + * `statusFor()`. Everything absent is the provider's fault until proven otherwise: 502. + */ +export const OAUTH_ROUTE_STATUS: Readonly> = { + X_OAUTH_PROVIDER_UNKNOWN: 404, + X_OAUTH_DENIED: 403, + X_OAUTH_STATE_INVALID: 400, + X_OAUTH_TOKEN_INVALID: 400, + X_UNAUTHENTICATED: 401, + X_MFA_REQUIRED: 401, + X_ACCOUNT_LOCKED: 429, + X_ENV_MISSING: 500, + X_CONFIG_INVALID: 500, + X_SAML_RESPONSE_INVALID: 400, + X_SAML_METADATA_INVALID: 500, + X_IDENTITY_PROVIDER_UNKNOWN: 404, +}; + +/** + * The status for a code, read as an OWN key of the table above. + * + * `OAUTH_ROUTE_STATUS[code] ?? 502` is an index read on a plain object, so `Object.prototype` + * answered for `toString`, `constructor`, `hasOwnProperty` and `__proto__` — and `toString` + * yields a FUNCTION, which was handed straight to `ResponseInit.status`. `code` is a plain + * `string` on every `UltimateError`, so no cast makes that unreachable. Fourth instance of the + * class in the framework, after i18n's catalog lookup, schema's `coerce` and mcp's `validate-args`. + * + * Exported so a test can drive it without building a `Response`; `OAUTH_ROUTE_STATUS` stays + * exported unchanged, because `scripts/oauth-route-status.test.ts` pins it against + * `@ultimat3/http`'s `statusFor`. + */ +export function oauthRouteStatus(code: string): number { + // 502 is the default: an uncoded throw on this path came out of the provider conversation. + return Object.hasOwn(OAUTH_ROUTE_STATUS, code) ? (OAUTH_ROUTE_STATUS[code] ?? 502) : 502; +} diff --git a/packages/auth/src/identity-provider.ts b/packages/auth/src/identity-provider.ts new file mode 100644 index 000000000..608e82a4f --- /dev/null +++ b/packages/auth/src/identity-provider.ts @@ -0,0 +1,112 @@ +// Single responsibility: the IdentityProvider port — the one shape every federated sign-in +// protocol takes: send the browser out (`begin`), take its answer back (`complete`), hand over one +// proven `OAuthProfile`. OIDC (`oidcIdentityProvider`) and SAML (`samlIdentityProvider`) are its two +// adapters; `signInWithIdentity` is the one way either becomes a session. + +import type { Clock } from '@ultimat3/core'; +import type { Auth, LoginResult } from './auth'; +import { type TotpReplayGuard, totpStep } from './mfa'; +import type { OAuthGrantContext } from './oauth-grant-context'; +import { type OAuthGrants, signInWithOAuth } from './oauth-login'; +import type { OAuthProfile } from './oauth-profile'; + +export interface IdentityBeginInput { + /** The absolute URL the IdP answers to: the callback route for this provider. */ + readonly callbackUrl: string; + readonly clock: Clock; +} + +export interface IdentityRedirect { + /** Where the browser is sent. */ + readonly location: string; + /** + * What the second leg must be handed back, opaque to the caller. Not secret from the browser + * that started the flow, but it must not be FORGEABLE: the routes seal it in a signed cookie. + */ + readonly state: string; +} + +export interface IdentityCompleteInput { + /** `IdentityRedirect.state`, as the caller sealed and re-opened it. */ + readonly state: string; + /** The IdP's answer: the callback's query (GET) or form body (POST). */ + readonly params: Readonly>; + /** The same URL `begin` was given. */ + readonly callbackUrl: string; + readonly clock: Clock; + /** + * The single-use claim store: an answer completes ONCE. The routes pass `auth.totpReplay`, + * the store a framework boot installs shared across replicas (`installTotpReplay`). + */ + readonly replay: TotpReplayGuard; +} + +export interface VerifiedIdentity { + readonly profile: OAuthProfile; + /** When the IdP says the federated session ends, if it says. */ + readonly expiresAt: Date | null; + /** Everything else the IdP asserted — id-token claims or SAML attributes — for the grant seam. */ + readonly claims: Readonly>; + /** OIDC only: the token set and a lazy, subject-checked `userinfo()` (`oauthLogin`'s grant seam). */ + readonly oauth?: OAuthGrantContext | undefined; +} + +export interface IdentityProvider { + /** Stable id: the route segment, `OAuthProfile.provider`, and the `x_accounts.provider` value. */ + readonly id: string; + readonly protocol: 'oidc' | 'saml'; + /** How the IdP returns: SAML's HTTP-POST binding is a cross-site POST, OIDC's a GET. */ + readonly callbackMethod: 'GET' | 'POST'; + begin(input: IdentityBeginInput): Promise; + /** + * Whether `params` answer THIS state — RelayState for SAML, `state` for OIDC — compared in + * constant time, nothing verified beyond it. The routes clear a browser's state cookie only on an + * answer bound to it: the ACS cookie rides any cross-site POST, so clearing on every outcome let + * any site end a sign-in in flight with one junk form. + */ + answers(state: string, params: Readonly>): boolean; + /** Every refusal is a coded `AuthError`; nothing here creates a session. */ + complete(input: IdentityCompleteInput): Promise; +} + +/** The app's grant seam over a verified identity — business convention, never shipped (axiom 8). */ +export type ResolveIdentityGrants = ( + identity: VerifiedIdentity, +) => Promise | OAuthGrants; + +export interface IdentitySignInOptions { + readonly grants?: OAuthGrants | undefined; + readonly ip?: string | null | undefined; + readonly userAgent?: string | null | undefined; +} + +/** + * A verified identity → a session, through the same account-link, MFA and session rules an OAuth + * login runs (`signInWithOAuth`). The provider id must be in `defineAuth({ providers })`. + */ +export async function signInWithIdentity( + auth: Auth, + identity: VerifiedIdentity, + options: IdentitySignInOptions = {}, +): Promise { + return await signInWithOAuth(auth, { + profile: identity.profile, + // The framework stores no provider credential (`accountFor`); only the expiry is carried. + tokens: { + accessToken: '', + refreshToken: null, + expiresAt: identity.expiresAt, + idToken: null, + claims: null, + }, + ip: options.ip, + userAgent: options.userAgent, + ...(options.grants === undefined ? {} : { grants: options.grants }), + }); +} + +/** + * The replay store forgets a claim once its step falls behind the clock, so a claim is filed + * under the step of the LAST instant the answer is acceptable: it outlives its own window. + */ +export const replayStepFor = (until: Date): number => totpStep(until); diff --git a/packages/auth/src/index.ts b/packages/auth/src/index.ts index 68d68c844..d91061540 100644 --- a/packages/auth/src/index.ts +++ b/packages/auth/src/index.ts @@ -149,6 +149,25 @@ export { idTokenEmailVerified, verifyIdToken, } from './id-token'; +export type { + IdentityBeginInput, + IdentityCompleteInput, + IdentityProvider, + IdentityRedirect, + IdentitySignInOptions, + ResolveIdentityGrants, + VerifiedIdentity, +} from './identity-provider'; +export { signInWithIdentity } from './identity-provider'; +export type { OidcIdentityProviderConfig } from './identity-provider-oidc'; +export { oidcIdentityProvider } from './identity-provider-oidc'; +export type { + IdentityProviderRoutes, + IdentityProviderRoutesOptions, + IdentityRouteDescriptor, +} from './identity-provider-routes'; +export { identityProviderRoutes } from './identity-provider-routes'; +export { OAUTH_ROUTE_STATUS } from './identity-provider-status'; export type { IdTokenKeys, JwksClientOptions, @@ -265,6 +284,7 @@ export { discoverOAuthProvider, discoveryUrl } from './oauth-discovery'; // from that module, below. export type { OAuthExchangeFailure } from './oauth-errors'; export { + oauthDenied, oauthExchangeFailed, oauthProviderUnknown, oauthStateInvalid, @@ -309,7 +329,7 @@ export { registerOAuthProvider, } from './oauth-registry'; export type { AuthRouteDescriptor, OAuthLoginOptions, OAuthLoginRoutes } from './oauth-route'; -export { OAUTH_ROUTE_STATUS, oauthDenied, oauthLogin } from './oauth-route'; +export { oauthLogin } from './oauth-route'; // Passkeys (WebAuthn): sign-in and digest-bound step-up. The browser half is `@ultimat3/auth/client`. export type { PasskeyPolicy, @@ -391,6 +411,16 @@ export { revokeSessionsCreatedBefore, revokeUserSessions, } from './revocation'; +export type { SamlCheck } from './saml-errors'; +export type { SamlIdpMetadata } from './saml-metadata'; +export { parseSamlMetadata } from './saml-metadata'; +export type { SamlIdentityProviderConfig } from './saml-provider'; +export { + DEFAULT_SAML_EMAIL_ATTRIBUTES, + DEFAULT_SAML_NAME_ATTRIBUTES, + DEFAULT_SAML_SKEW_MS, + samlIdentityProvider, +} from './saml-provider'; export type { CookieJar, CreateSessionInput, diff --git a/packages/auth/src/oauth-discovery.ts b/packages/auth/src/oauth-discovery.ts index b36d5e7ef..c68ef282c 100644 --- a/packages/auth/src/oauth-discovery.ts +++ b/packages/auth/src/oauth-discovery.ts @@ -30,7 +30,7 @@ export const discoveryUrl = (issuer: string): string => `${bareIssuer(issuer)}/.well-known/openid-configuration`; export interface DiscoverOAuthProviderInput { - /** The id the URL segment carries: `/auth/oauth/`. Also names the two env vars. */ + /** The id the URL segment carries: `/auth/sso/`. Also names the two env vars. */ readonly id: string; readonly issuer: string; /** Defaults to the OIDC minimum this package needs to identify somebody. */ diff --git a/packages/auth/src/oauth-errors.test.ts b/packages/auth/src/oauth-errors.test.ts index d3d1f1748..50638bd98 100644 --- a/packages/auth/src/oauth-errors.test.ts +++ b/packages/auth/src/oauth-errors.test.ts @@ -13,6 +13,7 @@ import { describe, expect, test } from 'bun:test'; import { hasErrorCode } from '@ultimat3/core'; import { + oauthDenied, oauthExchangeFailed, oauthProviderUnknown, oauthStateInvalid, @@ -21,7 +22,6 @@ import { } from './oauth-errors'; import { emailVerifiedNotStored, oauthAccountNotLinked, oauthLinkingDisabled } from './oauth-login'; import { oauthProviderDuplicate } from './oauth-registry'; -import { oauthDenied } from './oauth-route'; /** * Every OAuth factory, against the code it carries. Codes are forever; `emailVerifiedNotStored` @@ -71,7 +71,7 @@ describe('the oauth error factories, wherever each lives', () => { test('restartAt still quotes the mounted start path, never a hand-written one', () => { // The whole reason this phrase is a function: the fix line named a route the package did not // mount, and every caller who followed it got a 404. - expect(restartAt('github')).toContain('/auth/oauth/github'); + expect(restartAt('github')).toContain('/auth/sso/github'); expect(oauthStateInvalid('github', 'nonce').fix).toContain(restartAt('github')); }); diff --git a/packages/auth/src/oauth-errors.ts b/packages/auth/src/oauth-errors.ts index e17f675af..b5c00dc3c 100644 --- a/packages/auth/src/oauth-errors.ts +++ b/packages/auth/src/oauth-errors.ts @@ -1,6 +1,6 @@ // Single responsibility: the OAuth refusals more than one OAuth module raises, and the one phrase -// every "start over" fix is built from. A refusal with one thrower lives beside it (`oauthDenied` -// in `oauth-route.ts`, the linking ones in `oauth-login.ts`); the codes, titles and the single +// every "start over" fix is built from. A refusal with one thrower lives beside it (the linking ones +// in `oauth-login.ts`); the codes, titles and the single // `registerErrorCodes()` call stay in `errors.ts`, so this file adds no code and registers nothing. import { renderCauseValue, renderFixLiteral } from '@ultimat3/core'; @@ -96,3 +96,27 @@ export const oauthTokenInvalid = (provider: string, reason: string, fix: string) fix, meta: { provider }, }); + +/** + * The provider came back with `error=` and no code — almost always the user pressing Cancel. + * A separate code from `X_OAUTH_EXCHANGE_FAILED` on purpose: nothing was exchanged, nothing is + * misconfigured, and folding the single commonest non-success outcome of a login into the code + * that means "the client secret is wrong" makes both unreadable in a log and pages the wrong person. + */ +export const oauthDenied = ( + provider: string, + reason: string, + description: string | null, +): AuthError => + new AuthError({ + code: 'X_OAUTH_DENIED', + // `reason` and `description` are query parameters off the callback URL — whatever the browser + // was redirected with, newlines and quotes included. `renderCauseValue` renders them as JSON + // string literals, so a forged `error_description` cannot forge a second log line or break the + // sentence around it. Both are already `string` by type: this is escaping, not throw-safety. + cause: `${provider} declined the authorization: ${renderCauseValue(reason)}${ + description === null ? '' : ` (${renderCauseValue(description)})` + }`, + fix: `${restartAt(provider)} and approve the ${provider} consent screen`, + meta: { provider, reason }, + }); diff --git a/packages/auth/src/oauth-paths.ts b/packages/auth/src/oauth-paths.ts index 65ee76135..98d639aa4 100644 --- a/packages/auth/src/oauth-paths.ts +++ b/packages/auth/src/oauth-paths.ts @@ -4,8 +4,11 @@ // while no route was mounted anywhere. Deriving both the mount and the fix from here makes a fix // line that names a route nothing serves impossible rather than merely discouraged. -/** Not configurable, on purpose: a movable base path is a `fix:` line that can go stale again. */ -export const OAUTH_BASE_PATH = '/auth/oauth'; +/** + * Not configurable, on purpose: a movable base path is a `fix:` line that can go stale again. + * `/auth/sso` since 31.0.0: every `IdentityProvider` — OIDC and SAML — signs in through it. + */ +export const OAUTH_BASE_PATH = '/auth/sso'; /** The pattern a router mounts. `:provider` is the segment `oauthStartPath` fills in. */ export const OAUTH_START_ROUTE_PATH = `${OAUTH_BASE_PATH}/:provider` as const; diff --git a/packages/auth/src/oauth-route-keys.test.ts b/packages/auth/src/oauth-route-keys.test.ts index b99665404..151383a36 100644 --- a/packages/auth/src/oauth-route-keys.test.ts +++ b/packages/auth/src/oauth-route-keys.test.ts @@ -47,12 +47,12 @@ async function callback( baseUrl: 'https://app.test', ...(idTokenKeys === undefined ? {} : { idTokenKeys }), }); - const start = await login.start.handle(new Request('https://app.test/auth/oauth/google')); + const start = await login.start.handle(new Request('https://app.test/auth/sso/google')); const authorize = new URL(start.headers.get('location') ?? ''); nonce = authorize.searchParams.get('nonce') ?? ''; return await login.callback.handle( new Request( - `https://app.test/auth/oauth/google/callback?code=c&state=${authorize.searchParams.get('state')}`, + `https://app.test/auth/sso/google/callback?code=c&state=${authorize.searchParams.get('state')}`, { headers: { cookie: cookiePair(start.headers.getSetCookie()[0] ?? '') } }, ), ); diff --git a/packages/auth/src/oauth-route-public.test.ts b/packages/auth/src/oauth-route-public.test.ts index 3a6b1ba43..48913bcef 100644 --- a/packages/auth/src/oauth-route-public.test.ts +++ b/packages/auth/src/oauth-route-public.test.ts @@ -50,7 +50,7 @@ describe('the body a stranger reads is the code and a fixed sentence', () => { await ( await login().callback.handle( new Request( - `https://app.test/auth/oauth/github/callback?error=${reason}&error_description=${reason}-text`, + `https://app.test/auth/sso/github/callback?error=${reason}&error_description=${reason}-text`, ), ) ).text(); @@ -61,7 +61,7 @@ describe('the body a stranger reads is the code and a fixed sentence', () => { test('the cause and the fix go to the log, under one event', async () => { const response = await login().callback.handle( - new Request('https://app.test/auth/oauth/github/callback?code=the-code&state=forged'), + new Request('https://app.test/auth/sso/github/callback?code=the-code&state=forged'), ); const body = await bodyOf(response); expect(response.status).toBe(400); @@ -76,7 +76,7 @@ describe('the body a stranger reads is the code and a fixed sentence', () => { expect(String(body['title'])).not.toBe(''); // The developer's two lines are where only an operator reads them. expect(refusedLine()).toContain('X_OAUTH_STATE_INVALID'); - expect(refusedLine()).toContain('GET /auth/oauth/github'); + expect(refusedLine()).toContain('GET /auth/sso/github'); }); test('a short SESSION_SECRET is refused without its length, in the body or the cause', async () => { @@ -85,7 +85,7 @@ describe('the body a stranger reads is the code and a fixed sentence', () => { credentials, baseUrl: 'https://app.test', env: { SESSION_SECRET: short }, - }).start.handle(new Request('https://app.test/auth/oauth/github')); + }).start.handle(new Request('https://app.test/auth/sso/github')); expect(response.status).toBe(500); expect(await response.text()).not.toContain('17'); @@ -104,7 +104,7 @@ describe('the body a stranger reads is the code and a fixed sentence', () => { describe('redirect_uri is built from a declared origin, never from the Host header', () => { test('with neither baseUrl nor APP_URL the start leg refuses, whatever the Host says', async () => { const response = await oauthLogin(auth, { credentials, secret: SECRET, env: {} }).start.handle( - new Request('https://attacker.test/auth/oauth/github'), + new Request('https://attacker.test/auth/sso/github'), ); expect(response.status).toBe(500); expect(response.headers.get('location')).toBeNull(); @@ -114,8 +114,8 @@ describe('redirect_uri is built from a declared origin, never from the Host head test('an unknown provider answers that same refusal, so the gap is not an oracle', async () => { const routes = oauthLogin(auth, { credentials, secret: SECRET, env: { APP_URL: ' ' } }); - const known = await routes.start.handle(new Request('https://app.test/auth/oauth/github')); - const unknown = await routes.start.handle(new Request('https://app.test/auth/oauth/nope')); + const known = await routes.start.handle(new Request('https://app.test/auth/sso/github')); + const unknown = await routes.start.handle(new Request('https://app.test/auth/sso/nope')); expect(unknown.status).toBe(known.status); expect(await unknown.text()).toBe(await known.text()); }); @@ -129,11 +129,11 @@ describe('redirect_uri is built from a declared origin, never from the Host head credentials, secret: SECRET, ...origin, - }).start.handle(new Request('https://attacker.test/auth/oauth/github')); + }).start.handle(new Request('https://attacker.test/auth/sso/github')); expect(response.status).toBe(302); const authorize = new URL(response.headers.get('location') ?? ''); expect(authorize.searchParams.get('redirect_uri')).toBe( - 'https://app.test/auth/oauth/github/callback', + 'https://app.test/auth/sso/github/callback', ); }); }); diff --git a/packages/auth/src/oauth-route-refusals.test.ts b/packages/auth/src/oauth-route-refusals.test.ts index f12359efa..1ef85342e 100644 --- a/packages/auth/src/oauth-route-refusals.test.ts +++ b/packages/auth/src/oauth-route-refusals.test.ts @@ -86,11 +86,11 @@ describe('an internal exception is logged, never published', () => { const callbackBody = async ( login: ReturnType, ): Promise> => { - const start = await login.start.handle(new Request('https://app.test/auth/oauth/github')); + const start = await login.start.handle(new Request('https://app.test/auth/sso/github')); const state = new URL(start.headers.get('location') ?? '').searchParams.get('state') ?? ''; return bodyOf( await login.callback.handle( - new Request(`https://app.test/auth/oauth/github/callback?code=c&state=${state}`, { + new Request(`https://app.test/auth/sso/github/callback?code=c&state=${state}`, { headers: { cookie: cookiePair(start.headers.getSetCookie()[0] ?? '') }, }), ), @@ -196,11 +196,11 @@ describe('the token leg quotes only the OAuth error fields', () => { secret: SECRET, baseUrl: 'https://app.test', }); - const start = await login.start.handle(new Request('https://app.test/auth/oauth/github')); + const start = await login.start.handle(new Request('https://app.test/auth/sso/github')); const state = new URL(start.headers.get('location') ?? '').searchParams.get('state') ?? ''; return bodyOf( await login.callback.handle( - new Request(`https://app.test/auth/oauth/github/callback?code=c&state=${state}`, { + new Request(`https://app.test/auth/sso/github/callback?code=c&state=${state}`, { headers: { cookie: cookiePair(start.headers.getSetCookie()[0] ?? '') }, }), ), diff --git a/packages/auth/src/oauth-route.test.ts b/packages/auth/src/oauth-route.test.ts index 8b7ff7f8a..50834bde0 100644 --- a/packages/auth/src/oauth-route.test.ts +++ b/packages/auth/src/oauth-route.test.ts @@ -5,11 +5,12 @@ import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { frozenClock, setLogSink } from '@ultimat3/core'; import { type Auth, defineAuth } from './auth'; +import { oauthRouteStatus } from './identity-provider-status'; import { type MemoryAuthAdapter, memoryAuthAdapter } from './memory-adapter'; import type { OAuthFetch } from './oauth-exchange'; import { oauthCallbackPath, oauthStartPath } from './oauth-paths'; import { registerOAuthProvider } from './oauth-registry'; -import { oauthLogin, oauthRouteStatus } from './oauth-route'; +import { oauthLogin } from './oauth-route'; const NOW = new Date('2026-08-15T12:00:00.000Z'); @@ -79,11 +80,11 @@ const bodyOf = async (response: Response): Promise> => { describe('oauthLogin', () => { test('a forged state is refused with the code and the fix that restarts the flow', async () => { const login = routes(); - const start = await login.start.handle(new Request('https://app.test/auth/oauth/github')); + const start = await login.start.handle(new Request('https://app.test/auth/sso/github')); const sealed = cookiePair(start.headers.getSetCookie()[0] ?? ''); const done = await login.callback.handle( - new Request('https://app.test/auth/oauth/github/callback?code=the-code&state=forged', { + new Request('https://app.test/auth/sso/github/callback?code=the-code&state=forged', { headers: { cookie: sealed }, }), ); @@ -93,17 +94,18 @@ describe('oauthLogin', () => { expect(body['code']).toBe('X_OAUTH_STATE_INVALID'); // Axiom 4: the fix names a route this package actually mounts, not one it wishes existed. expect(refused()).toContain(`GET ${oauthStartPath('github')}`); - expect(login.start.path).toBe('/auth/oauth/:provider'); - // A spent handshake must not survive its own failure — the code it authorised is gone. - expect(done.headers.getSetCookie().some((c) => c.startsWith('__Host-x_oauth_github=;'))).toBe( - true, + expect(login.start.path).toBe('/auth/sso/:provider'); + // A forged state is not this browser's answer: the cookie survives, so a cross-site request + // carrying it cannot end the real sign-in in flight. A bound answer clears it on any outcome. + expect(done.headers.getSetCookie().some((c) => c.startsWith('__Host-x_sso_github=;'))).toBe( + false, ); }); test('redirect to session: a github login round-trips through the two mounted routes', async () => { const login = routes(); - const start = await login.start.handle(new Request('https://app.test/auth/oauth/github')); + const start = await login.start.handle(new Request('https://app.test/auth/sso/github')); expect(start.status).toBe(302); const authorize = new URL(start.headers.get('location') ?? ''); expect(`${authorize.origin}${authorize.pathname}`).toBe( @@ -118,11 +120,11 @@ describe('oauthLogin', () => { const setStart = start.headers.getSetCookie(); expect(setStart).toHaveLength(1); - expect(setStart[0]).toStartWith('__Host-x_oauth_github='); + expect(setStart[0]).toStartWith('__Host-x_sso_github='); expect(setStart[0]).toContain('HttpOnly'); const done = await login.callback.handle( - new Request(`https://app.test/auth/oauth/github/callback?code=the-code&state=${state}`, { + new Request(`https://app.test/auth/sso/github/callback?code=the-code&state=${state}`, { headers: { cookie: cookiePair(setStart[0] ?? ''), 'user-agent': 'Firefox/1' }, }), ); @@ -131,7 +133,7 @@ describe('oauthLogin', () => { expect(done.headers.get('location')).toBe('/'); const setDone = done.headers.getSetCookie(); expect(setDone.some((c) => c.startsWith('__Host-x_session='))).toBe(true); - expect(setDone.some((c) => c.startsWith('__Host-x_oauth_github=;'))).toBe(true); + expect(setDone.some((c) => c.startsWith('__Host-x_sso_github=;'))).toBe(true); const user = await adapter.findUserByEmail('ada@example.com'); expect(user?.emailVerifiedAt).toEqual(NOW); @@ -144,7 +146,7 @@ describe('oauthLogin', () => { test('a provider the app did not enable never reaches the provider', async () => { const login = routes(); - const response = await login.start.handle(new Request('https://app.test/auth/oauth/apple')); + const response = await login.start.handle(new Request('https://app.test/auth/sso/apple')); expect(response.status).toBe(404); expect((await bodyOf(response))['code']).toBe('X_OAUTH_PROVIDER_UNKNOWN'); @@ -153,7 +155,7 @@ describe('oauthLogin', () => { test('a provider Ultimate has never heard of is the same refusal', async () => { const login = routes(); - const response = await login.start.handle(new Request('https://app.test/auth/oauth/facebook')); + const response = await login.start.handle(new Request('https://app.test/auth/sso/facebook')); expect(response.status).toBe(404); expect((await bodyOf(response))['code']).toBe('X_OAUTH_PROVIDER_UNKNOWN'); @@ -161,10 +163,12 @@ describe('oauthLogin', () => { test('a user pressing Cancel is its own code, never a broken-app one', async () => { const login = routes(); - const start = await login.start.handle(new Request('https://app.test/auth/oauth/github')); + const start = await login.start.handle(new Request('https://app.test/auth/sso/github')); + // RFC 6749 4.1.2.1: the error redirect echoes `state`, which binds it to this browser. + const state = new URL(start.headers.get('location') ?? '').searchParams.get('state') ?? ''; const done = await login.callback.handle( - new Request('https://app.test/auth/oauth/github/callback?error=access_denied', { + new Request(`https://app.test/auth/sso/github/callback?error=access_denied&state=${state}`, { headers: { cookie: cookiePair(start.headers.getSetCookie()[0] ?? '') }, }), ); @@ -180,7 +184,7 @@ describe('oauthLogin', () => { test('a callback with no handshake cookie has nothing to check state against', async () => { const login = routes(); const done = await login.callback.handle( - new Request('https://app.test/auth/oauth/github/callback?code=c&state=s'), + new Request('https://app.test/auth/sso/github/callback?code=c&state=s'), ); expect(done.status).toBe(400); @@ -189,14 +193,14 @@ describe('oauthLogin', () => { test('a github handshake may not finish a google callback', async () => { const login = routes(); - const start = await login.start.handle(new Request('https://app.test/auth/oauth/github')); + const start = await login.start.handle(new Request('https://app.test/auth/sso/github')); const sealed = cookiePair(start.headers.getSetCookie()[0] ?? '').replace( - '__Host-x_oauth_github=', - '__Host-x_oauth_google=', + '__Host-x_sso_github=', + '__Host-x_sso_google=', ); const done = await login.callback.handle( - new Request('https://app.test/auth/oauth/google/callback?code=c&state=s', { + new Request('https://app.test/auth/sso/google/callback?code=c&state=s', { headers: { cookie: sealed }, }), ); @@ -212,12 +216,12 @@ describe('oauthLogin', () => { baseUrl: 'https://app.test', fetch: () => Promise.reject(new Error('boom')), }); - const start = await login.start.handle(new Request('https://app.test/auth/oauth/github')); + const start = await login.start.handle(new Request('https://app.test/auth/sso/github')); const authorize = new URL(start.headers.get('location') ?? ''); const done = await login.callback.handle( new Request( - `https://app.test/auth/oauth/github/callback?code=c&state=${authorize.searchParams.get('state')}`, + `https://app.test/auth/sso/github/callback?code=c&state=${authorize.searchParams.get('state')}`, { headers: { cookie: cookiePair(start.headers.getSetCookie()[0] ?? '') } }, ), ); @@ -256,7 +260,7 @@ describe('oauthLogin', () => { ); const body = await bodyOf( - await login.start.handle(new Request('https://app.test/auth/oauth/apple')), + await login.start.handle(new Request('https://app.test/auth/sso/apple')), ); // Both legs are public by definition, so `toJSON()` — which carries `meta` and `stack` for a @@ -308,7 +312,7 @@ describe('credentials read from options.env', () => { }, ); - const started = await route.start.handle(new Request('https://app.test/auth/oauth/github')); + const started = await route.start.handle(new Request('https://app.test/auth/sso/github')); expect(started.status).toBe(302); const location = new URL(started.headers.get('location') ?? ''); expect(location.searchParams.get('client_id')).toBe('from-options-env'); @@ -316,7 +320,7 @@ describe('credentials read from options.env', () => { const state = location.searchParams.get('state') ?? ''; const back = await route.callback.handle( - new Request(`https://app.test/auth/oauth/github/callback?code=abc&state=${state}`, { + new Request(`https://app.test/auth/sso/github/callback?code=abc&state=${state}`, { headers: { cookie }, }), ); @@ -336,7 +340,7 @@ describe('a provider with no credentials is 404, not a 500 naming its env vars', test('the start leg answers the same 404 an unknown provider gets', async () => { const response = await noCredentials().start.handle( - new Request('https://app.test/auth/oauth/github'), + new Request('https://app.test/auth/sso/github'), ); const body = await bodyOf(response); @@ -349,7 +353,7 @@ describe('a provider with no credentials is 404, not a 500 naming its env vars', test('the callback leg answers it too — one oracle closed on both halves', async () => { const response = await noCredentials().callback.handle( - new Request('https://app.test/auth/oauth/github/callback?code=x&state=y'), + new Request('https://app.test/auth/sso/github/callback?code=x&state=y'), ); expect(response.status).toBe(404); expect((await bodyOf(response))['code']).toBe('X_OAUTH_PROVIDER_UNKNOWN'); diff --git a/packages/auth/src/oauth-route.ts b/packages/auth/src/oauth-route.ts index 525b358fd..878d2634d 100644 --- a/packages/auth/src/oauth-route.ts +++ b/packages/auth/src/oauth-route.ts @@ -1,7 +1,7 @@ -// Single responsibility: the two HTTP routes the OAuth library functions have always been -// missing — the redirect out and the callback back — mounted at one fixed pair of paths so the -// `fix:` lines that name them cannot go stale. Everything below composes `beginOAuth`, -// `handshakeCookie` and `completeOAuthLogin`; no new protocol lives here. +// Single responsibility: `oauthLogin(auth)` — "log in with GitHub" in one call, as a front door +// over the ONE sign-in route implementation (`identity-provider-route-core.ts`): every registered +// OAuth provider this app enabled becomes an `oidcIdentityProvider` per request, so OAuth and SAML +// share the paths (`/auth/sso/:provider`), the sealed state cookie and the refusal body. // // A route DESCRIPTOR, never a mounted handler, for the reason `mcpHttpRoute()` is one: // `@ultimat3/http` is tier 2 like this package, so auth may not import it — and `defineRoute` @@ -9,56 +9,28 @@ // a test and mountable by any router that can match a `:param`. import { - type Clock, - EnvMissingError, + ConfigInvalidError, isUltimateError, logger, - renderCauseValue, - renderFixShellArg, renderThrowable, type UltimateError, } from '@ultimat3/core'; -import type { Auth, LoginResult } from './auth'; -import { AuthError } from './errors'; +import type { Auth } from './auth'; +import { oidcIdentityProvider } from './identity-provider-oidc'; +import { + type SignInRouteDescriptor, + type SignInRouteMeta, + signInRoutes, +} from './identity-provider-route-core'; import type { IdTokenKeys } from './jwks'; -import { beginOAuth, type OAuthProviderId } from './oauth'; +import type { OAuthProviderId } from './oauth'; import { BUILTIN_OAUTH_PROVIDER_IDS } from './oauth-builtins'; -import { clearHandshakeCookie, handshakeCookie, readHandshakeCookie } from './oauth-cookie'; -import { oauthExchangeFailed, oauthProviderUnknown, restartAt } from './oauth-errors'; +import { oauthExchangeFailed, oauthProviderUnknown } from './oauth-errors'; import type { OAuthClientCredentials, OAuthFetch } from './oauth-exchange'; import { oauthCredentials } from './oauth-exchange'; -import { completeOAuthLogin, type ResolveOAuthGrants } from './oauth-login'; -import { - OAUTH_CALLBACK_ROUTE_PATH, - OAUTH_START_ROUTE_PATH, - oauthCallbackPath, -} from './oauth-paths'; +import type { ResolveOAuthGrants } from './oauth-login'; import { hasOAuthProvider } from './oauth-registry'; -/** - * The provider came back with `error=` and no code — almost always the user pressing Cancel. - * A separate code from `X_OAUTH_EXCHANGE_FAILED` on purpose: nothing was exchanged, nothing is - * misconfigured, and folding the single commonest non-success outcome of a login into the code - * that means "the client secret is wrong" makes both unreadable in a log and pages the wrong person. - */ -export const oauthDenied = ( - provider: string, - reason: string, - description: string | null, -): AuthError => - new AuthError({ - code: 'X_OAUTH_DENIED', - // `reason` and `description` are query parameters off the callback URL — whatever the browser - // was redirected with, newlines and quotes included. `renderCauseValue` renders them as JSON - // string literals, so a forged `error_description` cannot forge a second log line or break the - // sentence around it. Both are already `string` by type: this is escaping, not throw-safety. - cause: `${provider} declined the authorization: ${renderCauseValue(reason)}${ - description === null ? '' : ` (${renderCauseValue(description)})` - }`, - fix: `${restartAt(provider)} and approve the ${provider} consent screen`, - meta: { provider, reason }, - }); - /** * What a router needs to mount one of these. Structural, like `RequestLike` in `session.ts`: * `@ultimat3/http` binds to this shape, this package never binds to `@ultimat3/http`. @@ -71,6 +43,8 @@ export interface AuthRouteDescriptor { readonly name: string; /** Both legs are public by definition — they are how an anonymous visitor stops being one. */ readonly auth: 'public'; + /** What a router mounts as the route's `meta`, verbatim: `meta: leg.meta`. */ + readonly meta: SignInRouteMeta; handle(request: Request): Promise; } @@ -126,97 +100,10 @@ export interface OAuthLoginOptions { readonly env?: Readonly> | undefined; } -/** - * HTTP status per code, for a descriptor driven OUTSIDE a pipeline — a bare `Request` in, a - * `Response` out, which is the whole point of a descriptor. `@ultimat3/http`'s `error-map.ts` OWNS - * these numbers and is where a new one is declared; this package is the same tier and can never - * import it, so the table is a copy the pin `scripts/oauth-route-status.test.ts` holds identical to - * `statusFor()`. Everything absent is the provider's fault until proven otherwise: 502. - */ -export const OAUTH_ROUTE_STATUS: Readonly> = { - X_OAUTH_PROVIDER_UNKNOWN: 404, - X_OAUTH_DENIED: 403, - X_OAUTH_STATE_INVALID: 400, - X_OAUTH_TOKEN_INVALID: 400, - X_UNAUTHENTICATED: 401, - X_MFA_REQUIRED: 401, - X_ACCOUNT_LOCKED: 429, - X_ENV_MISSING: 500, - X_CONFIG_INVALID: 500, -}; - -/** - * The status for a code, read as an OWN key of the table above. - * - * `OAUTH_ROUTE_STATUS[code] ?? 502` is an index read on a plain object, so `Object.prototype` - * answered for `toString`, `constructor`, `hasOwnProperty` and `__proto__` — and `toString` - * yields a FUNCTION, which was handed straight to `ResponseInit.status`. `code` is a plain - * `string` on every `UltimateError`, so no cast makes that unreachable. Fourth instance of the - * class in the framework, after i18n's catalog lookup, schema's `coerce` and mcp's `validate-args`. - * - * Exported so a test can drive it without building a `Response`; `OAUTH_ROUTE_STATUS` stays - * exported unchanged, because `scripts/oauth-route-status.test.ts` pins it against - * `@ultimat3/http`'s `statusFor`. - */ -export function oauthRouteStatus(code: string): number { - // 502 is the default: an uncoded throw on this path came out of the provider conversation. - return Object.hasOwn(OAUTH_ROUTE_STATUS, code) ? (OAUTH_ROUTE_STATUS[code] ?? 502) : 502; -} - -/** The one sentence every refusal carries outward. The event name is where the rest went. */ -const PUBLIC_CAUSE = - 'the sign-in was refused; its cause and its fix are in this server\u2019s log under auth.oauth.refused'; - -/** A code this framework would have minted — the only kind spliced into a command. */ -const FRAMEWORK_CODE = /^X_[A-Z0-9_]+$/; - -/** - * What an anonymous caller is allowed to read: the code, its registered title and docs link, and - * one fixed sentence. Both legs are public by definition, and an authored `cause`/`fix` is written - * for the developer — it names env vars, endpoints and what a provider answered — so neither is - * served, on any code: no per-code judgement about which sentence is safe today. - */ -const publicBody = (coded: UltimateError): Record => ({ - code: coded.code, - title: coded.title, - cause: PUBLIC_CAUSE, - fix: FRAMEWORK_CODE.test(coded.code) - ? `x errors explain ${renderFixShellArg(coded.code, '')} --json` - : 'x errors list --json', - docs: coded.docs, -}); - -/** - * Failure is JSON, never a redirect to a login page carrying `?error=`: the status and the code - * are the answer a client branches on. The developer's half — the cause and the fix line — is - * logged as `auth.oauth.refused`, where only an operator reads it. An app that wants a rendered - * page wraps these two descriptors. - */ -function problem(error: unknown, extraCookies: readonly string[]): Response { - const coded = isUltimateError(error) ? error : uncoded(error); - const status = oauthRouteStatus(coded.code); - const fields = { code: coded.code, status, cause: coded.cause, fix: coded.fix }; - if (status >= 500) logger.error('auth.oauth.refused', fields); - else logger.info('auth.oauth.refused', fields); - const headers = new Headers({ 'content-type': 'application/json; charset=utf-8' }); - for (const cookie of extraCookies) headers.append('set-cookie', cookie); - return new Response(JSON.stringify(publicBody(coded)), { status, headers }); -} - /** * A throw with no code: a bug in an `AuthAdapter` or an `OAuthFetch` this package does not own. - * - * Its text goes to the LOG and never into the response. `publicBody` publishes `cause`, and the - * rendered value went straight into it — so a bare - * `Error('connect ECONNREFUSED 10.4.2.17:5432 (postgres://app:hunter2@db.internal/prod)')` from a - * pool inside a callback was served to whoever typed the URL, as a 502 with the connection string - * in the body. The comment above `publicBody` says a stack and `meta` must never reach that - * reader; the field carrying the message was the one exception nobody meant to make. - * - * `renderThrowable`, never `error.message`: the value came from code this package does not own, - * and a getter on `message` — or a `Proxy` trapping `getPrototypeOf` — would make the callback's - * last answer throw instead of send. Same shape `rate-limit.ts`'s `loginFailed()` uses: one fixed - * sentence outward, the discriminating detail kept where only an operator can read it. + * Its text goes to the LOG and never into the response — a pool error carrying a connection + * string was once served to whoever typed the URL. `renderThrowable`, never `error.message`. */ function uncoded(error: unknown): UltimateError { logger.error('auth.oauth.uncoded_failure', { detail: renderThrowable(error) }); @@ -230,45 +117,12 @@ function uncoded(error: unknown): UltimateError { }); } -/** `/auth/oauth/github` → `github`; `/auth/oauth/github/callback` → `github`. */ -function providerSegment(request: Request, leg: 'start' | 'callback'): string { - const segments = new URL(request.url).pathname.split('/').filter((s) => s.length > 0); - const index = leg === 'start' ? segments.length - 1 : segments.length - 2; - return segments[index] ?? ''; -} - -/** The app's answer for this provider, or nothing — so `undefined` falls through to the default. */ -function idTokenKeysFor( - options: OAuthLoginOptions, - provider: OAuthProviderId, -): { readonly idTokenKeys?: IdTokenKeys } { - const keys = options.idTokenKeys?.(provider); - return keys === undefined ? {} : { idTokenKeys: keys }; -} - /** - * Both halves of "is this a provider", in one refusal. An unknown segment and a known provider - * the app left out of `defineAuth({ providers })` are the same 404 on purpose — telling an - * anonymous caller which of the two it hit describes the app's configuration for free. - * - * The list in the refusal is the BUILT-INS (`BUILTIN_OAUTH_PROVIDER_IDS`), never the live - * registry and never `defineAuth({ providers })`. Both of the latter are this deployment's own configuration, and - * this caller is an anonymous stranger who typed a URL: an app that registered an internal OP has - * put its own vocabulary into the registry, and echoing it back names a system the stranger had no - * way to know exists. The built-in list is a framework constant already in the public docs, so it - * discloses nothing while still making the fix executable — and `registerOAuthProvider` in the - * same sentence covers the other branch without enumerating anything. - * - * `providerFor()` keeps the full registered list for the same reason in reverse: its reader is a - * developer holding a stack trace, and there the list is exactly what makes the fix runnable. + * Unknown and not-enabled are one 404, listing the BUILT-INS only: the caller is an anonymous + * stranger, and the live registry is this deployment's own vocabulary. */ -function assertEnabled(auth: Auth, segment: string): OAuthProviderId { - const supported = BUILTIN_OAUTH_PROVIDER_IDS; - if (!hasOAuthProvider(segment)) throw oauthProviderUnknown(segment, supported); - const provider: OAuthProviderId = segment; - if (!auth.providers.includes(provider)) throw oauthProviderUnknown(segment, supported); - return provider; -} +const unknown = (segment: string): UltimateError => + oauthProviderUnknown(segment, BUILTIN_OAUTH_PROVIDER_IDS); /** * The credentials for a provider, or the SAME 404 an unknown provider gets. @@ -297,145 +151,54 @@ function credentialsFor( cause: error.cause, fix: error.fix, }); - throw oauthProviderUnknown(provider, BUILTIN_OAUTH_PROVIDER_IDS); - } -} - -/** - * The declared origin, or a refusal. The request's own origin is the `Host` header, so a - * `redirect_uri` built from it is one the caller chose. - */ -function declaredOrigin(options: OAuthLoginOptions): string { - const env = options.env ?? Bun.env; - const declared = (options.baseUrl ?? env['APP_URL'] ?? '').trim(); - if (declared === '') { - throw new EnvMissingError({ - cause: - 'APP_URL is not set and oauthLogin() was given no baseUrl, so there is no declared origin to build the provider redirect_uri from', - fix: 'export APP_URL=http://localhost:3000 # the origin this app is served from', - meta: { key: 'APP_URL' }, - }); - } - return declared.replace(/\/+$/, ''); -} - -/** The handshake's own options, assembled once so both legs seal and open it identically. */ -const sealOptions = ( - clock: Clock, - options: OAuthLoginOptions, -): { clock: Clock; secret?: string | undefined } => ({ - clock, - ...(options.secret === undefined ? {} : { secret: options.secret }), -}); - -async function startHandler( - auth: Auth, - options: OAuthLoginOptions, - request: Request, -): Promise { - try { - // Before the provider is looked at: asked after, a missing origin answered 500 for a provider - // this app enabled and 404 for one it did not. - const origin = declaredOrigin(options); - const provider = assertEnabled(auth, providerSegment(request, 'start')); - const credentials = credentialsFor(provider, options); - const handshake = beginOAuth({ - provider, - clientId: credentials.clientId, - redirectUri: `${origin}${oauthCallbackPath(provider)}`, - scopes: options.scopes, - }); - return new Response(null, { - // 302, the status every OAuth client already expects on this hop. - status: 302, - headers: { - location: handshake.authorizeUrl, - 'set-cookie': handshakeCookie(handshake, sealOptions(auth.clock, options)), - }, - }); - } catch (error) { - return problem(error, []); + throw unknown(provider); } } -/** The provider declining is `error=` on the redirect, and there is no code to exchange. */ -function assertNoProviderError(url: URL, provider: string): void { - const declined = url.searchParams.get('error'); - if (declined === null || declined === '') return; - throw oauthDenied(provider, declined, url.searchParams.get('error_description')); -} - -async function callbackHandler( - auth: Auth, - options: OAuthLoginOptions, - request: Request, -): Promise { - const segment = providerSegment(request, 'callback'); - // Cleared on every outcome, success and failure alike: the code it authorised is spent either - // way, so a handshake that outlives its own callback is a replay window and nothing else. - const clear = hasOAuthProvider(segment) ? [clearHandshakeCookie(segment)] : []; - try { - const provider = assertEnabled(auth, segment); - // A provider with no credentials answers 404 on THIS leg too: one oracle, closed on both halves - // — a 404 on start and a 500 on callback is the same enumeration one request later. And the - // answer is what the exchange uses, or `options.env` would answer the start leg alone. - const credentials = credentialsFor(provider, options); - const url = new URL(request.url); - assertNoProviderError(url, provider); - const result: LoginResult = await completeOAuthLogin(auth, { - handshake: readHandshakeCookie(request, provider, sealOptions(auth.clock, options)), - callback: { - state: url.searchParams.get('state') ?? '', - code: url.searchParams.get('code') ?? '', - }, - credentials, - ...(options.fetch === undefined ? {} : { fetch: options.fetch }), - ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }), - ...(options.resolveGrants === undefined ? {} : { resolveGrants: options.resolveGrants }), - ...idTokenKeysFor(options, provider), - ip: options.clientIp?.(request) ?? null, - userAgent: request.headers.get('user-agent'), - }); - const headers = new Headers({ - // A fixed path, never `?next=`: an attacker-supplied return target on the one endpoint whose - // job is to hand out a session is the classic open redirect, and `@ultimat3/http`'s - // `nextAfterSignIn` is the one implementation of that check. Two copies is one that drifts. - location: options.successPath ?? '/', - }); - // 303: the callback may arrive as a `form_post`, and the destination is a GET either way. - headers.append('set-cookie', result.cookie); - for (const cookie of clear) headers.append('set-cookie', cookie); - return new Response(null, { status: 303, headers }); - } catch (error) { - return problem(error, clear); - } -} +/** A descriptor of a GET leg, narrowed to the `AuthRouteDescriptor` the type has always promised. */ +const asGet = (descriptor: SignInRouteDescriptor): AuthRouteDescriptor => + Object.freeze({ ...descriptor, method: 'GET' as const }); /** * The two routes. Mount them and "log in with GitHub" is a button pointing at - * `/auth/oauth/github` — no handshake store, no PKCE bookkeeping, no state check to forget. + * `/auth/sso/github` — no handshake store, no PKCE bookkeeping, no state check to forget. * * ```ts * const { start, callback } = oauthLogin(auth); - * // start.path → '/auth/oauth/:provider' - * // callback.path → '/auth/oauth/:provider/callback' + * // start.path → '/auth/sso/:provider' + * // callback.path → '/auth/sso/:provider/callback' * ``` */ export function oauthLogin(auth: Auth, options: OAuthLoginOptions = {}): OAuthLoginRoutes { - return Object.freeze({ - start: Object.freeze({ - method: 'GET', - path: OAUTH_START_ROUTE_PATH, - name: 'auth.oauth.start', - auth: 'public', - handle: (request: Request) => startHandler(auth, options, request), - } as const), - callback: Object.freeze({ - method: 'GET', - path: OAUTH_CALLBACK_ROUTE_PATH, - name: 'auth.oauth.callback', - auth: 'public', - handle: (request: Request) => callbackHandler(auth, options, request), - } as const), + const routes = signInRoutes(auth, options, { + resolve(segment, leg) { + if (!hasOAuthProvider(segment) || !auth.providers.includes(segment)) throw unknown(segment); + // Asked on the callback only: the start leg verifies nothing. + const keys = leg === 'callback' ? options.idTokenKeys?.(segment) : undefined; + return oidcIdentityProvider({ + provider: segment, + credentials: credentialsFor(segment, options), + scopes: options.scopes, + fetch: options.fetch, + timeoutMs: options.timeoutMs, + ...(keys === undefined ? {} : { idTokenKeys: keys }), + }); + }, + unknown, + event: 'auth.oauth', + names: { start: 'auth.oauth.start', callback: 'auth.oauth.callback', acs: 'auth.oauth.acs' }, + uncoded, + grants(identity) { + const resolve = options.resolveGrants; + if (resolve === undefined) return undefined; + if (identity.oauth === undefined) { + throw new ConfigInvalidError({ + cause: `${identity.profile.provider} returned no OAuth grant context, so resolveGrants(profile, context) cannot be called`, + fix: 'identityProviderRoutes(auth, { providers, resolveGrants: (identity) => ({ roles }) }) // for a provider that is not OAuth', + }); + } + return resolve(identity.profile, identity.oauth); + }, }); + return Object.freeze({ start: asGet(routes.start), callback: asGet(routes.callback) }); } diff --git a/packages/auth/src/saml-assertion.ts b/packages/auth/src/saml-assertion.ts new file mode 100644 index 000000000..70d5c97c8 --- /dev/null +++ b/packages/auth/src/saml-assertion.ts @@ -0,0 +1,156 @@ +// Single responsibility: what a VERIFIED assertion says, checked — issuer, the bearer subject +// confirmation (recipient, InResponseTo, its window), the conditions (window, audience) — and read +// out as one `SamlAssertion`. It only ever sees the node whose signature `saml-response.ts` just +// verified: it is handed that node, never a document to search. + +import { attribute, BEARER, childElements, NAMEID_TRANSIENT, SAML_NS, textOf } from './saml-dom'; +import type { SamlCheck } from './saml-errors'; +import type { XmlElement } from './saml-xml'; + +export interface SamlAssertion { + readonly id: string; + readonly issuer: string; + readonly nameId: string; + readonly nameIdFormat: string | null; + /** Every Attribute by `Name`, values in document order. */ + readonly attributes: Readonly>; + /** The last instant this assertion is acceptable: the earlier of the two NotOnOrAfter. */ + readonly notOnOrAfter: Date; + /** `SessionNotOnOrAfter`, when the IdP bounds the session. */ + readonly sessionNotOnOrAfter: Date | null; + readonly sessionIndex: string | null; +} + +export interface AssertionExpectations { + readonly issuer: string; + readonly audience: string; + readonly recipient: string; + readonly requestId: string; + readonly now: Date; + readonly skewMs: number; +} + +export class AssertionRefusal { + constructor( + readonly check: SamlCheck, + readonly detail: string, + ) {} +} +const refuse = (check: SamlCheck, detail: string): never => { + throw new AssertionRefusal(check, detail); +}; + +/** xs:dateTime, which SAML requires in UTC; anything unparseable is absent-and-wrong. */ +const instant = (value: string | undefined): Date | null => { + if (value === undefined || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$/.test(value)) + return null; + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; +}; + +const one = (parent: XmlElement, local: string, check: SamlCheck): XmlElement => { + const found = childElements(parent, SAML_NS, local); + const first = found[0]; + return found.length === 1 && first !== undefined + ? first + : refuse(check, `expected one saml:${local}`); +}; + +const leaf = (element: XmlElement, check: SamlCheck): string => + textOf(element) ?? refuse(check, `saml:${element.local} holds an element`); + +function window(element: XmlElement, expect: AssertionExpectations, required: boolean): Date { + const now = expect.now.getTime(); + const notBefore = attribute(element, 'NotBefore'); + if (notBefore !== undefined) { + const at = instant(notBefore) ?? refuse('not-before', 'NotBefore is not a UTC instant'); + if (at.getTime() - expect.skewMs > now) + refuse('not-before', `${element.local} is not yet valid`); + } + const raw = attribute(element, 'NotOnOrAfter'); + if (raw === undefined && !required) return new Date(Number.POSITIVE_INFINITY); + const end = instant(raw) ?? refuse('expired', `${element.local} has no UTC NotOnOrAfter`); + if (end.getTime() + expect.skewMs <= now) refuse('expired', `${element.local} has expired`); + return end; +} + +/** At least one bearer confirmation must hold — SAML Profiles §4.1.4.2. */ +function confirmation(subject: XmlElement, expect: AssertionExpectations): Date { + let last: AssertionRefusal = new AssertionRefusal('subject', 'no bearer SubjectConfirmation'); + for (const confirmation of childElements(subject, SAML_NS, 'SubjectConfirmation')) { + if (attribute(confirmation, 'Method') !== BEARER) continue; + try { + const data = one(confirmation, 'SubjectConfirmationData', 'subject'); + if (attribute(data, 'Recipient') !== expect.recipient) { + refuse('recipient', 'the Recipient is not this ACS URL'); + } + if (attribute(data, 'InResponseTo') !== expect.requestId) { + refuse('in-response-to', 'the confirmation answers another request'); + } + return window(data, expect, true); + } catch (error) { + if (!(error instanceof AssertionRefusal)) throw error; + last = error; + } + } + throw last; +} + +function conditions(assertion: XmlElement, expect: AssertionExpectations): Date { + const element = one(assertion, 'Conditions', 'audience'); + const end = window(element, expect, false); + const restrictions = childElements(element, SAML_NS, 'AudienceRestriction'); + if (restrictions.length === 0) refuse('audience', 'the assertion names no audience'); + for (const restriction of restrictions) { + const audiences = childElements(restriction, SAML_NS, 'Audience').map((a) => + leaf(a, 'audience'), + ); + if (!audiences.includes(expect.audience)) refuse('audience', 'this SP is not an audience'); + } + return end; +} + +function attributes(assertion: XmlElement): Record { + const out: Record = Object.create(null); + for (const statement of childElements(assertion, SAML_NS, 'AttributeStatement')) { + for (const item of childElements(statement, SAML_NS, 'Attribute')) { + const name = attribute(item, 'Name'); + if (name === undefined) continue; + const values = childElements(item, SAML_NS, 'AttributeValue').map((v) => leaf(v, 'subject')); + out[name] = [...(out[name] ?? []), ...values]; + } + } + return out; +} + +/** Check and read. Throws `AssertionRefusal`, which the caller turns into the coded error. */ +export function readAssertion(assertion: XmlElement, expect: AssertionExpectations): SamlAssertion { + const issuer = leaf(one(assertion, 'Issuer', 'issuer'), 'issuer'); + if (issuer !== expect.issuer) + refuse('issuer', 'the assertion Issuer is not the metadata entityID'); + const subject = one(assertion, 'Subject', 'subject'); + const nameIdElement = one(subject, 'NameID', 'subject'); + const nameId = leaf(nameIdElement, 'subject').trim(); + const nameIdFormat = attribute(nameIdElement, 'Format') ?? null; + if (nameId === '') refuse('subject', 'the NameID is empty'); + // A transient NameID is a new id per login: linked by it, every sign-in is a stranger. + if (nameIdFormat === NAMEID_TRANSIENT) refuse('subject', 'the NameID is transient'); + const confirmed = confirmation(subject, expect); + const conditioned = conditions(assertion, expect); + const authn = childElements(assertion, SAML_NS, 'AuthnStatement'); + const statement = authn[0] ?? refuse('structure', 'the assertion carries no AuthnStatement'); + const session = attribute(statement, 'SessionNotOnOrAfter'); + return { + id: attribute(assertion, 'ID') ?? '', + issuer, + nameId, + nameIdFormat, + attributes: attributes(assertion), + notOnOrAfter: new Date(Math.min(confirmed.getTime(), conditioned.getTime())), + sessionNotOnOrAfter: + session === undefined + ? null + : (instant(session) ?? refuse('expired', 'bad SessionNotOnOrAfter')), + sessionIndex: attribute(statement, 'SessionIndex') ?? null, + }; +} diff --git a/packages/auth/src/saml-c14n.test.ts b/packages/auth/src/saml-c14n.test.ts new file mode 100644 index 000000000..4167c6841 --- /dev/null +++ b/packages/auth/src/saml-c14n.test.ts @@ -0,0 +1,104 @@ +// Exclusive c14n, proven against bytes another implementation signed: every DigestValue in the +// three xmlsec-signed vectors must come out of our canonical form, and every SignedInfo must +// verify under the vector's certificate. SHA-1 here is the vectors' algorithm, never production's. + +import { describe, expect, test } from 'bun:test'; +import { exclusiveC14n } from './saml-c14n'; +import { base64Bytes, certificateSpki } from './saml-certificate'; +import { childElements, DSIG_NS, descendants } from './saml-dom'; +import { + DOUBLE_SIGNED_RESPONSE, + SIGNED_ASSERTION_RESPONSE, + SIGNED_MESSAGE_RESPONSE, + VECTOR_CERTIFICATE, +} from './saml-vectors-fixture'; +import { parseXml, type XmlElement } from './saml-xml'; + +const parsed = (xml: string): XmlElement => { + const result = parseXml(xml); + if (!result.ok) return expect.unreachable(result.reason); + return result.root; +}; + +const signaturesIn = (root: XmlElement): XmlElement[] => + [root, ...descendants(root)].filter((el) => el.ns === DSIG_NS && el.local === 'Signature'); + +const child = (element: XmlElement, local: string): XmlElement => { + const found = childElements(element, DSIG_NS, local)[0]; + return found ?? expect.unreachable(`no ds:${local}`); +}; + +const text = (element: XmlElement): string => + element.children.map((node) => (node.kind === 'text' ? node.value : '')).join(''); + +async function sha1Base64(value: string): Promise { + const digest = await crypto.subtle.digest('SHA-1', new TextEncoder().encode(value)); + return btoa(String.fromCharCode(...new Uint8Array(digest))); +} + +const VECTORS = [ + ['signed assertion', SIGNED_ASSERTION_RESPONSE, 1], + ['signed message', SIGNED_MESSAGE_RESPONSE, 1], + ['double signed', DOUBLE_SIGNED_RESPONSE, 2], +] as const; + +describe('exclusive c14n against xmlsec-signed vectors', () => { + for (const [name, xml, count] of VECTORS) { + test(`${name}: every reference digest matches`, async () => { + const signatures = signaturesIn(parsed(xml)); + expect(signatures).toHaveLength(count); + for (const signature of signatures) { + const signed = signature.parent ?? expect.unreachable('a detached signature'); + const reference = child(child(signature, 'SignedInfo'), 'Reference'); + const canonical = exclusiveC14n(signed, { exclude: signature }); + expect(await sha1Base64(canonical)).toBe(text(child(reference, 'DigestValue')).trim()); + } + }); + + test(`${name}: every SignedInfo verifies under the vector certificate`, async () => { + const spki = certificateSpki(VECTOR_CERTIFICATE) ?? expect.unreachable('no spki'); + const key = await crypto.subtle.importKey( + 'spki', + spki, + { name: 'RSASSA-PKCS1-v1_5', hash: 'SHA-1' }, + false, + ['verify'], + ); + for (const signature of signaturesIn(parsed(xml))) { + const signedInfo = exclusiveC14n(child(signature, 'SignedInfo')); + const value = base64Bytes(text(child(signature, 'SignatureValue'))) ?? new Uint8Array(); + const bytes = new TextEncoder().encode(signedInfo); + expect(await crypto.subtle.verify('RSASSA-PKCS1-v1_5', key, value, bytes)).toBe(true); + } + }); + } +}); + +describe('exclusive c14n rules', () => { + const canonical = (xml: string, prefixes?: readonly string[]): string => { + const root = parsed(xml); + const apex = childElements(root)[0] ?? root; + return exclusiveC14n(apex, { inclusivePrefixes: prefixes }); + }; + + test('an ancestor namespace is rendered only where visibly used', () => { + const xml = ''; + expect(canonical(xml)).toBe(''); + }); + + test('an unused namespace is dropped unless the InclusiveNamespaces list names it', () => { + const xml = ''; + expect(canonical(xml)).toBe(''); + expect(canonical(xml, ['xs'])).toBe(''); + }); + + test('comments are dropped, text and attributes escaped, empty elements opened and closed', () => { + const xml = 'ab & > '; + expect(canonical(xml)).toBe('ab & > '); + }); + + test('a default namespace is undeclared below an ancestor that rendered one', () => { + const xml = ''; + expect(canonical(xml)).toBe(''); + }); +}); diff --git a/packages/auth/src/saml-c14n.ts b/packages/auth/src/saml-c14n.ts new file mode 100644 index 000000000..ff890f8ce --- /dev/null +++ b/packages/auth/src/saml-c14n.ts @@ -0,0 +1,118 @@ +// Single responsibility: Exclusive XML Canonicalization 1.0 without comments over one element's +// subtree — the bytes a SAML signature covers. Comments are dropped, which is why a reader taking +// only a NameID's FIRST text node is fooled by `ada@corp.com.evil.com`: the signature +// covers the concatenation, so `saml-xml.ts` readers concatenate too. + +import type { XmlAttribute, XmlElement } from './saml-xml'; + +export interface C14nOptions { + /** The enveloped `ds:Signature`: it and its subtree are not part of the node set. */ + readonly exclude?: XmlElement | undefined; + /** `InclusiveNamespaces PrefixList`; `#default` names the default namespace. */ + readonly inclusivePrefixes?: readonly string[] | undefined; +} + +/** `&name;` — one XML entity or character reference, as Exclusive C14N §2.3 writes it. */ +const ref = (name: string): string => `&${name};`; + +/** + * C14N's own tables, NOT core's `escapeHtml`: the spec fixes them byte for byte — `>` escaped in + * text but never in an attribute, `'` never, and CR/TAB/LF as ` `/` `/` ` — and one + * byte off is a digest mismatch. `escapeHtml` is right for markup and wrong here. + */ +const TEXT_REFS: ReadonlyMap = new Map([ + ['&', ref('amp')], + ['<', ref('lt')], + ['>', ref('gt')], + ['\r', ref('#xD')], +]); +const ATTRIBUTE_REFS: ReadonlyMap = new Map([ + ['&', ref('amp')], + ['<', ref('lt')], + ['"', ref('quot')], + ['\t', ref('#x9')], + ['\n', ref('#xA')], + ['\r', ref('#xD')], +]); + +/** A text node in canonical form. */ +export const c14nText = (value: string): string => + value.replace(/[&<>\r]/g, (char) => TEXT_REFS.get(char) ?? char); + +/** An attribute value in canonical form, for between double quotes. */ +export const c14nAttribute = (value: string): string => + value.replace(/[&<"\t\n\r]/g, (char) => ATTRIBUTE_REFS.get(char) ?? char); + +/** Code-unit order, which for the ASCII names SAML uses is the spec's code-point order. */ +const byKey = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0); + +/** + * The prefixes this element visibly utilises: its own, and its prefixed attributes' (`xml:` is + * never declared). An InclusiveNamespaces prefix is rendered by the inclusive rule instead — when + * it is in scope at all. + */ +function wanted(element: XmlElement, inclusive: ReadonlySet): Set { + const prefixes = new Set([element.prefix ?? '']); + for (const attribute of element.attributes) { + if (attribute.prefix !== null && attribute.prefix !== 'xml') prefixes.add(attribute.prefix); + } + for (const prefix of inclusive) if (element.scope.has(prefix)) prefixes.add(prefix); + return prefixes; +} + +function namespaces( + element: XmlElement, + rendered: Map, + inclusive: ReadonlySet, +): string { + const out: [string, string][] = []; + for (const prefix of wanted(element, inclusive)) { + const uri = element.scope.get(prefix) ?? ''; + // An empty default is only worth saying when an output ancestor said a non-empty one. + const previous = rendered.get(prefix) ?? ''; + if (prefix === '' ? uri === previous : rendered.get(prefix) === uri) continue; + rendered.set(prefix, uri); + out.push([prefix, uri]); + } + out.sort((a, b) => byKey(a[0], b[0])); + return out + .map( + ([prefix, uri]) => ` ${prefix === '' ? 'xmlns' : `xmlns:${prefix}`}="${c14nAttribute(uri)}"`, + ) + .join(''); +} + +const attributeOrder = (a: XmlAttribute, b: XmlAttribute): number => + byKey(a.ns, b.ns) || byKey(a.local, b.local); + +function render( + element: XmlElement, + rendered: ReadonlyMap, + options: C14nOptions, + inclusive: ReadonlySet, + out: string[], +): void { + const scope = new Map(rendered); + out.push(`<${element.qname}${namespaces(element, scope, inclusive)}`); + for (const attribute of [...element.attributes].sort(attributeOrder)) { + out.push(` ${attribute.qname}="${c14nAttribute(attribute.value)}"`); + } + out.push('>'); + for (const child of element.children) { + if (child.kind === 'text') out.push(c14nText(child.value)); + else if (child.kind === 'element' && child !== options.exclude) { + render(child, scope, options, inclusive, out); + } + } + out.push(``); +} + +/** The canonical form of `apex`'s subtree, as the string whose UTF-8 bytes are digested. */ +export function exclusiveC14n(apex: XmlElement, options: C14nOptions = {}): string { + const inclusive = new Set( + (options.inclusivePrefixes ?? []).map((prefix) => (prefix === '#default' ? '' : prefix)), + ); + const out: string[] = []; + render(apex, new Map(), options, inclusive, out); + return out.join(''); +} diff --git a/packages/auth/src/saml-cert-fixture.ts b/packages/auth/src/saml-cert-fixture.ts new file mode 100644 index 000000000..04d5a36c1 --- /dev/null +++ b/packages/auth/src/saml-cert-fixture.ts @@ -0,0 +1,66 @@ +// TEST-ONLY. An RSA-2048 key pair and a self-signed X.509 certificate for it, minted in-process +// with Web Crypto and hand-written DER: what a SAML IdP's metadata publishes and signs with. + +export interface TestRsaSigner { + /** Base64 DER, as `` carries it. */ + readonly certificate: string; + sign(data: string): Promise; +} + +const concat = (parts: readonly Uint8Array[]): Uint8Array => { + const out = new Uint8Array(parts.reduce((sum, part) => sum + part.length, 0)); + let at = 0; + for (const part of parts) { + out.set(part, at); + at += part.length; + } + return out; +}; + +const lengthOf = (n: number): Uint8Array => + n < 0x80 ? Uint8Array.of(n) : n < 0x100 ? Uint8Array.of(0x81, n) : Uint8Array.of(0x82, n >> 8, n); +const tlv = (tag: number, ...parts: readonly Uint8Array[]): Uint8Array => { + const body = concat(parts); + return concat([Uint8Array.of(tag), lengthOf(body.length), body]); +}; +const seq = (...parts: readonly Uint8Array[]): Uint8Array => tlv(0x30, ...parts); +const text = (value: string): Uint8Array => new TextEncoder().encode(value); + +/** sha256WithRSAEncryption, NULL parameters. */ +const SHA256_RSA = seq( + Uint8Array.of(0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x0b), + Uint8Array.of(0x05, 0x00), +); +const COMMON_NAME = Uint8Array.of(0x06, 0x03, 0x55, 0x04, 0x03); +const NAME = seq(tlv(0x31, seq(COMMON_NAME, tlv(0x0c, text('test-idp'))))); +const VALIDITY = seq(tlv(0x17, text('000101000000Z')), tlv(0x18, text('20991231235959Z'))); + +const base64 = (bytes: Uint8Array): string => btoa(String.fromCharCode(...bytes)); + +export async function testRsaSigner(): Promise { + const algorithm = { + name: 'RSASSA-PKCS1-v1_5', + modulusLength: 2048, + publicExponent: Uint8Array.of(1, 0, 1), + hash: 'SHA-256', + }; + const pair = await crypto.subtle.generateKey(algorithm, true, ['sign', 'verify']); + const spki = new Uint8Array(await crypto.subtle.exportKey('spki', pair.publicKey)); + const tbs = seq( + tlv(0xa0, tlv(0x02, Uint8Array.of(2))), + tlv(0x02, Uint8Array.of(1)), + SHA256_RSA, + NAME, + VALIDITY, + NAME, + spki, + ); + const signRaw = async (data: Uint8Array): Promise => + new Uint8Array(await crypto.subtle.sign('RSASSA-PKCS1-v1_5', pair.privateKey, data)); + const signature = await signRaw(tbs); + const certificate = seq(tbs, SHA256_RSA, tlv(0x03, Uint8Array.of(0), signature)); + return { + certificate: base64(certificate), + sign: async (data) => base64(await signRaw(new TextEncoder().encode(data))), + }; +} diff --git a/packages/auth/src/saml-certificate.ts b/packages/auth/src/saml-certificate.ts new file mode 100644 index 000000000..4ece6be54 --- /dev/null +++ b/packages/auth/src/saml-certificate.ts @@ -0,0 +1,86 @@ +// Single responsibility: an IdP's X.509 signing certificate (base64 DER, as metadata carries it) +// → the RSA public key Web Crypto verifies with. Web Crypto takes no certificate, so this walks +// just enough DER to lift `subjectPublicKeyInfo` out of `tbsCertificate` (RFC 5280 §4.1). +// Only the key is trusted: the certificate's dates and issuer are the metadata's business. + +interface Tlv { + readonly tag: number; + readonly start: number; + readonly body: number; + readonly end: number; +} + +const SEQUENCE = 0x30; +const VERSION = 0xa0; +/** `serialNumber`, `signature`, `issuer`, `validity`, `subject` — skipped to reach the key. */ +const FIELDS_BEFORE_KEY = 5; +/** NIST SP 800-131A: an RSA key under 2048 bits signs nothing an SP should believe. */ +export const MIN_SAML_RSA_BITS = 2048; + +function readTlv(bytes: Uint8Array, at: number, limit: number): Tlv | undefined { + if (at + 2 > limit) return undefined; + const tag = bytes[at] ?? 0; + const first = bytes[at + 1] ?? 0; + let body = at + 2; + let length = first; + if (first & 0x80) { + const count = first & 0x7f; + if (count === 0 || count > 4 || body + count > limit) return undefined; + length = 0; + for (let index = 0; index < count; index += 1) + length = length * 256 + (bytes[body + index] ?? 0); + body += count; + } + const end = body + length; + return end > limit ? undefined : { tag, start: at, body, end }; +} + +/** Strict base64 with the white space metadata wraps it in; `undefined` for anything else. */ +export function base64Bytes(text: string): Uint8Array | undefined { + const compact = text.replace(/[ \t\r\n]+/g, ''); + if (compact === '' || !/^[A-Za-z0-9+/]+={0,2}$/.test(compact) || compact.length % 4 !== 0) { + return undefined; + } + const binary = atob(compact); + return Uint8Array.from(binary, (char) => char.charCodeAt(0)); +} + +/** `Certificate → tbsCertificate → subjectPublicKeyInfo`, tag included, or `undefined`. */ +export function certificateSpki(base64Der: string): Uint8Array | undefined { + const der = base64Bytes(base64Der); + if (der === undefined) return undefined; + const certificate = readTlv(der, 0, der.byteLength); + if (certificate?.tag !== SEQUENCE || certificate.end !== der.byteLength) return undefined; + const tbs = readTlv(der, certificate.body, certificate.end); + if (tbs?.tag !== SEQUENCE) return undefined; + let field = readTlv(der, tbs.body, tbs.end); + if (field?.tag === VERSION) field = readTlv(der, field.end, tbs.end); + for (let skipped = 0; skipped < FIELDS_BEFORE_KEY; skipped += 1) { + if (field === undefined) return undefined; + field = readTlv(der, field.end, tbs.end); + } + if (field?.tag !== SEQUENCE) return undefined; + return der.slice(field.start, field.end); +} + +/** + * The verification key for RSA-SHA256, or `undefined` for a certificate that is unreadable, not + * RSA, or under `MIN_SAML_RSA_BITS`. The caller names the refusal; this file throws nothing. + */ +export async function samlVerifyKey(base64Der: string): Promise { + const spki = certificateSpki(base64Der); + if (spki === undefined) return undefined; + try { + const key = await crypto.subtle.importKey( + 'spki', + spki, + { name: 'RSASSA-PKCS1-v1_5', hash: 'SHA-256' }, + false, + ['verify'], + ); + const { modulusLength } = key.algorithm as RsaHashedKeyAlgorithm; + return modulusLength >= MIN_SAML_RSA_BITS ? key : undefined; + } catch { + return undefined; + } +} diff --git a/packages/auth/src/saml-dom.ts b/packages/auth/src/saml-dom.ts new file mode 100644 index 000000000..b29305c7f --- /dev/null +++ b/packages/auth/src/saml-dom.ts @@ -0,0 +1,66 @@ +// Single responsibility: reading a parsed SAML tree — the namespaces and algorithm URIs SAML +// speaks, and the four lookups every reader needs. Matched by namespace URI and local name, never +// by prefix: a prefix is the sender's choice, and matching `saml:` would let `evil:Assertion` +// bound to the same URI walk past a check, or `saml:` bound to another URI walk into one. + +import type { XmlElement } from './saml-xml'; + +export const SAMLP_NS = 'urn:oasis:names:tc:SAML:2.0:protocol'; +export const SAML_NS = 'urn:oasis:names:tc:SAML:2.0:assertion'; +export const MD_NS = 'urn:oasis:names:tc:SAML:2.0:metadata'; +export const DSIG_NS = 'http://www.w3.org/2000/09/xmldsig#'; +export const EXC_C14N = 'http://www.w3.org/2001/10/xml-exc-c14n#'; +export const ENVELOPED = 'http://www.w3.org/2000/09/xmldsig#enveloped-signature'; +export const RSA_SHA256 = 'http://www.w3.org/2001/04/xmldsig-more#rsa-sha256'; +export const SHA256 = 'http://www.w3.org/2001/04/xmlenc#sha256'; +export const BINDING_REDIRECT = 'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect'; +export const BINDING_POST = 'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST'; +export const STATUS_SUCCESS = 'urn:oasis:names:tc:SAML:2.0:status:Success'; +export const BEARER = 'urn:oasis:names:tc:SAML:2.0:cm:bearer'; +export const NAMEID_PERSISTENT = 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent'; +export const NAMEID_TRANSIENT = 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient'; +export const NAMEID_EMAIL = 'urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress'; + +export const isElement = (element: XmlElement, ns: string, local: string): boolean => + element.ns === ns && element.local === local; + +export function childElements(element: XmlElement, ns?: string, local?: string): XmlElement[] { + const out: XmlElement[] = []; + for (const child of element.children) { + if (child.kind !== 'element') continue; + if (ns !== undefined && local !== undefined && !isElement(child, ns, local)) continue; + out.push(child); + } + return out; +} + +/** Every element below `element`, in document order, `element` excluded. */ +export function descendants(element: XmlElement): XmlElement[] { + const out: XmlElement[] = []; + const walk = (node: XmlElement): void => { + for (const child of childElements(node)) { + out.push(child); + walk(child); + } + }; + walk(element); + return out; +} + +/** An unqualified attribute — SAML's own attributes are never namespaced. */ +export function attribute(element: XmlElement, local: string): string | undefined { + return element.attributes.find((one) => one.ns === '' && one.local === local)?.value; +} + +/** + * The element's text: EVERY text child, concatenated — what the signature covered, since c14n + * drops comments. `undefined` when it holds an element, which no SAML leaf value does. + */ +export function textOf(element: XmlElement): string | undefined { + let text = ''; + for (const child of element.children) { + if (child.kind === 'element') return undefined; + if (child.kind === 'text') text += child.value; + } + return text; +} diff --git a/packages/auth/src/saml-dsig.ts b/packages/auth/src/saml-dsig.ts new file mode 100644 index 000000000..fa0b77032 --- /dev/null +++ b/packages/auth/src/saml-dsig.ts @@ -0,0 +1,165 @@ +// Single responsibility: verify the ONE enveloped XML-DSig signature that is a direct child of an +// element, over that element alone. Wrapping makes the verifier and the reader look at two nodes; +// here there is no lookup to fool — the caller hands in the node it will read, the Reference must +// name THAT node's ID, and the key comes from metadata, never from the document's KeyInfo. + +import { exclusiveC14n } from './saml-c14n'; +import { base64Bytes } from './saml-certificate'; +import { + attribute, + childElements, + DSIG_NS, + ENVELOPED, + EXC_C14N, + isElement, + RSA_SHA256, + SHA256, + textOf, +} from './saml-dom'; +import type { SamlCheck } from './saml-errors'; +import type { XmlElement } from './saml-xml'; + +export type DsigVerdict = + | { readonly ok: true } + | { readonly ok: false; readonly check: SamlCheck; readonly detail: string }; + +const fail = (check: SamlCheck, detail: string): DsigVerdict => ({ ok: false, check, detail }); + +/** The ds:* element children, white space skipped; any other child is a refusal. */ +function dsChildren(element: XmlElement): XmlElement[] | undefined { + for (const child of element.children) { + if (child.kind === 'text' && child.value.trim() !== '') return undefined; + if (child.kind === 'element' && child.ns !== DSIG_NS) return undefined; + } + return childElements(element); +} + +/** The algorithm, plus the `InclusiveNamespaces PrefixList` exc-c14n allows under it. */ +function c14nPrefixes(method: XmlElement): readonly string[] | undefined { + if (attribute(method, 'Algorithm') !== EXC_C14N) return undefined; + const inner = childElements(method); + if (inner.length === 0) return []; + const only = inner[0]; + if (inner.length !== 1 || only === undefined || only.ns !== EXC_C14N) return undefined; + if (only.local !== 'InclusiveNamespaces') return undefined; + return (attribute(only, 'PrefixList') ?? '').split(/[ \t\n]+/).filter((p) => p !== ''); +} + +interface ReferenceShape { + readonly prefixes: readonly string[]; + readonly digest: Uint8Array; +} + +function readReference(reference: XmlElement, signed: XmlElement): ReferenceShape | DsigVerdict { + const id = attribute(signed, 'ID'); + if (id === undefined || id === '') return fail('reference', 'the signed element carries no ID'); + if (attribute(reference, 'URI') !== `#${id}`) { + return fail('reference', 'the Reference URI does not name the element it is enveloped in'); + } + const parts = dsChildren(reference); + const [transforms, method, value] = parts ?? []; + if ( + parts?.length !== 3 || + transforms === undefined || + method === undefined || + value === undefined + ) { + return fail('reference', 'the Reference is not Transforms, DigestMethod, DigestValue'); + } + if (!isElement(transforms, DSIG_NS, 'Transforms')) return fail('reference', 'no Transforms'); + // Exactly enveloped-signature then exc-c14n: an XPath or XSLT transform is how a signature is + // made to cover a different node set than the one the reader reads. + const steps = dsChildren(transforms) ?? []; + const [first, second] = steps; + if (steps.length !== 2 || first === undefined || second === undefined) { + return fail('algorithm', 'the transforms are not enveloped-signature then exc-c14n'); + } + const prefixes = c14nPrefixes(second); + if (attribute(first, 'Algorithm') !== ENVELOPED || childElements(first).length > 0) { + return fail('algorithm', 'the first transform is not enveloped-signature'); + } + if (prefixes === undefined) return fail('algorithm', 'the second transform is not exc-c14n'); + if (!isElement(method, DSIG_NS, 'DigestMethod') || attribute(method, 'Algorithm') !== SHA256) { + return fail('algorithm', 'the digest is not SHA-256'); + } + const digest = isElement(value, DSIG_NS, 'DigestValue') + ? base64Bytes(textOf(value) ?? '') + : undefined; + return digest === undefined + ? fail('reference', 'the DigestValue is not base64') + : { prefixes, digest }; +} + +const equalBytes = (a: Uint8Array, b: Uint8Array): boolean => + a.length === b.length && a.every((byte, index) => byte === b[index]); + +const utf8 = (text: string): Uint8Array => new TextEncoder().encode(text); + +/** + * Verify `signed`'s own enveloped signature against the metadata keys. `{ ok: false }` names the + * check; `signature-missing` when the element carries none (the caller decides if that is fatal). + */ +export async function verifyEnvelopedSignature( + signed: XmlElement, + keys: readonly CryptoKey[], +): Promise { + const signatures = childElements(signed, DSIG_NS, 'Signature'); + const signature = signatures[0]; + if (signature === undefined) return fail('signature-missing', 'the element carries no signature'); + if (signatures.length > 1) return fail('structure', 'the element carries two signatures'); + // SignedInfo, SignatureValue, then an optional KeyInfo that is never read. A ds:Object is where + // wrapping attacks hide the original assertion, so its presence alone is a refusal. + const parts = dsChildren(signature) ?? []; + const [signedInfo, signatureValue, keyInfo] = parts; + if ( + signedInfo === undefined || + signatureValue === undefined || + !isElement(signedInfo, DSIG_NS, 'SignedInfo') || + !isElement(signatureValue, DSIG_NS, 'SignatureValue') || + parts.length > 3 || + (keyInfo !== undefined && !isElement(keyInfo, DSIG_NS, 'KeyInfo')) + ) { + return fail('structure', 'the Signature is not SignedInfo, SignatureValue[, KeyInfo]'); + } + const info = dsChildren(signedInfo) ?? []; + const [c14nMethod, signatureMethod, reference] = info; + if ( + info.length !== 3 || + c14nMethod === undefined || + signatureMethod === undefined || + reference === undefined + ) { + return fail('reference', 'SignedInfo must hold exactly one Reference'); + } + const infoPrefixes = isElement(c14nMethod, DSIG_NS, 'CanonicalizationMethod') + ? c14nPrefixes(c14nMethod) + : undefined; + if (infoPrefixes === undefined) return fail('algorithm', 'SignedInfo is not exc-c14n'); + if ( + !isElement(signatureMethod, DSIG_NS, 'SignatureMethod') || + attribute(signatureMethod, 'Algorithm') !== RSA_SHA256 || + childElements(signatureMethod).length > 0 + ) { + return fail('algorithm', 'the signature is not RSA-SHA256'); + } + if (!isElement(reference, DSIG_NS, 'Reference')) return fail('reference', 'no Reference'); + const shape = readReference(reference, signed); + if ('ok' in shape) return shape; + + const value = base64Bytes(textOf(signatureValue) ?? ''); + if (value === undefined) return fail('signature', 'the SignatureValue is not base64'); + const canonicalInfo = utf8(exclusiveC14n(signedInfo, { inclusivePrefixes: infoPrefixes })); + let verified = false; + for (const key of keys) { + if (await crypto.subtle.verify('RSASSA-PKCS1-v1_5', key, value, canonicalInfo)) verified = true; + } + if (!verified) return fail('signature', 'no metadata certificate verifies the SignedInfo'); + + const canonical = exclusiveC14n(signed, { + exclude: signature, + inclusivePrefixes: shape.prefixes, + }); + const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', utf8(canonical))); + if (!equalBytes(digest, shape.digest)) return fail('digest', 'the signed content was altered'); + return { ok: true }; +} diff --git a/packages/auth/src/saml-email-trust.ts b/packages/auth/src/saml-email-trust.ts new file mode 100644 index 000000000..6f808f5a1 --- /dev/null +++ b/packages/auth/src/saml-email-trust.ts @@ -0,0 +1,49 @@ +// Single responsibility: which asserted addresses a SAML IdP may vouch for. An IdP is one tenant's +// directory, and its admin can assert ANY address — so "trust this IdP's email" without a domain +// is "let one customer's admin sign in as another customer's verified user" (`resolveUser` links a +// verified provider address to the existing account). Trust is therefore per domain, exact. + +import { renderCauseValue } from '@ultimat3/core'; +import { AuthError } from './errors'; + +/** The trust half of `SamlIdentityProviderConfig`: `true` cannot be written without its domains. */ +export type SamlEmailTrust = + | { + /** The IdP's addresses are never treated as verified: they cannot link or create accounts. */ + readonly trustEmail: false; + readonly emailDomains?: undefined; + } + | { + /** Addresses in `emailDomains` are verified; any other is not, and cannot link. */ + readonly trustEmail: true; + /** The domains this IdP owns, exactly — a subdomain is listed on its own. Never empty. */ + readonly emailDomains: readonly string[]; + }; + +/** A DNS name: labels of letters, digits and inner hyphens. No wildcard, no trailing dot. */ +const DOMAIN = /^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z][a-z0-9-]{0,62}$/; + +/** The screened, lower-cased domain set; refuses at construction, never at the first login. */ +export function samlTrustedDomains(provider: string, trust: SamlEmailTrust): ReadonlySet { + if (!trust.trustEmail) return new Set(); + const domains = (trust.emailDomains ?? []).map((domain) => domain.trim().toLowerCase()); + const bad = domains.find((domain) => !DOMAIN.test(domain)); + if (domains.length === 0 || bad !== undefined) { + throw new AuthError({ + code: 'X_CONFIG_INVALID', + cause: + bad === undefined + ? `samlIdentityProvider ${renderCauseValue(provider)} trusts asserted addresses with no emailDomains, so its admin could assert any tenant's address` + : `samlIdentityProvider ${renderCauseValue(provider)} lists ${renderCauseValue(bad)} in emailDomains, which is not a plain domain`, + fix: "samlIdentityProvider({ trustEmail: true, emailDomains: ['corp.example'] }) // the exact domains this IdP owns, or trustEmail: false", + meta: { provider }, + }); + } + return new Set(domains); +} + +/** Whether `email`'s domain is one the IdP owns. Only the part after the LAST `@` is a domain. */ +export function isTrustedAddress(email: string, domains: ReadonlySet): boolean { + const at = email.lastIndexOf('@'); + return at > 0 && domains.has(email.slice(at + 1).toLowerCase()); +} diff --git a/packages/auth/src/saml-errors.ts b/packages/auth/src/saml-errors.ts new file mode 100644 index 000000000..e4624d813 --- /dev/null +++ b/packages/auth/src/saml-errors.ts @@ -0,0 +1,96 @@ +// Single responsibility: the SAML refusals. One code for every check a response can fail — +// `X_SAML_RESPONSE_INVALID` with the check in `meta.check` — because the operator needs to know +// WHICH check (an audience mismatch is the commonest setup bug there is) and the anonymous caller +// gets none of it: the route serves the code and its title only, like the OAuth legs. + +import { renderCauseValue } from '@ultimat3/core'; +import { AuthError } from './errors'; + +/** Every check a response is put through, in the order `verifySamlResponse` runs them. */ +export type SamlCheck = + | 'encoding' + | 'xml' + | 'structure' + | 'id-duplicate' + | 'encrypted' + | 'signature-missing' + | 'algorithm' + | 'reference' + | 'signature' + | 'digest' + | 'status' + | 'issuer' + | 'destination' + | 'in-response-to' + | 'recipient' + | 'audience' + | 'not-before' + | 'expired' + | 'subject' + | 'relay-state' + | 'replay'; + +const SP = 'samlIdentityProvider({ id, idp: parseSamlMetadata(xml), spEntityId, trustEmail })'; +const IDP_APP = `${SP} // unchanged here; in the IdP application:`; +const METADATA = + "parseSamlMetadata(await Bun.file('idp-metadata.xml').text()) // the IdP current metadata"; + +/** + * The repair per check. Each opens with the code to paste (`X_FIX_PROSE`): the SP side as a call, + * an IdP-side setting as that call's trailing comment, a retry as the route to start again at. + */ +const FIXES: Readonly> = { + encoding: + 'identityProviderRoutes(auth, { providers }).acs // mount as-is: SAMLResponse arrives base64 and is decoded once', + xml: `${IDP_APP} send a plain SAML 2.0 Response, no DOCTYPE or processing instruction`, + structure: `${IDP_APP} one samlp:Response holding exactly one saml:Assertion`, + 'id-duplicate': + 'oauthStartPath(id) // start again here; a response carrying one ID twice is a wrapping attempt, never accepted', + encrypted: `${IDP_APP} turn assertion encryption off (not supported)`, + 'signature-missing': `${IDP_APP} turn on "sign assertion" (a signed response alone is refused)`, + algorithm: `${IDP_APP} RSA-SHA256 signatures, SHA-256 digests, exclusive c14n`, + reference: `${IDP_APP} sign the assertion by its ID with the enveloped-signature transform`, + signature: `${METADATA}, pasted into samlIdentityProvider({ idp }): its signing certificate has likely rotated`, + digest: + 'oauthStartPath(id) // start again; if it repeats, remove the proxy rewriting the POST body between the IdP and this app', + status: `${IDP_APP} assign this user to the application (the IdP answer is in meta.status)`, + issuer: `${METADATA}: its entityID must be the assertion Issuer`, + destination: 'oauthCallbackPath(id) // the IdP application ACS URL is APP_URL + this path', + 'in-response-to': + 'oauthStartPath(id) // start the sign-in here, not from the IdP dashboard: IdP-initiated responses are refused', + recipient: 'oauthCallbackPath(id) // the IdP application ACS URL is APP_URL + this path', + audience: + 'samlIdentityProvider({ spEntityId }) // equal to the audience (SP entity id) the IdP application names', + 'not-before': + 'samlIdentityProvider({ skewMs: 120_000 }) // or sync the server clock first: chronyc makestep', + expired: 'oauthStartPath(id) // sync the server clock (chronyc makestep), then start again here', + subject: `${IDP_APP} NameID format persistent (or emailAddress)`, + 'relay-state': 'oauthStartPath(id) // start again here, in the same browser', + replay: 'oauthStartPath(id) // start again here: an assertion is accepted once', +}; + +/** + * Read as a `Map` (`proto-index`): the `Record` above is what makes a missing check a compile + * error; the lookup never indexes a plain object. + */ +const FIX_BY_CHECK: ReadonlyMap = new Map(Object.entries(FIXES)); + +export const samlResponseInvalid = ( + provider: string, + check: SamlCheck, + detail: string, + meta: Readonly> = {}, +): AuthError => + new AuthError({ + code: 'X_SAML_RESPONSE_INVALID', + cause: `the ${renderCauseValue(provider)} SAML response failed its ${check} check: ${detail}`, + fix: FIX_BY_CHECK.get(check) ?? FIXES.xml, + meta: { ...meta, provider, check }, + }); + +export const samlMetadataInvalid = (detail: string): AuthError => + new AuthError({ + code: 'X_SAML_METADATA_INVALID', + cause: `the SAML IdP metadata is unusable: ${detail}`, + fix: `${METADATA} XML, whole, downloaded again from its federation metadata URL`, + }); diff --git a/packages/auth/src/saml-hostile.test.ts b/packages/auth/src/saml-hostile.test.ts new file mode 100644 index 000000000..235c7371c --- /dev/null +++ b/packages/auth/src/saml-hostile.test.ts @@ -0,0 +1,158 @@ +// Hostile SAML responses: each is a published attack class against XML-DSig SPs (signature +// wrapping XSW1–XSW8, comment injection in NameID, unsigned assertion in a signed response, key +// substitution through KeyInfo, XXE / entity expansion), and each must be refused — or, for the +// comment, read as exactly what was signed. + +import { describe, expect, test } from 'bun:test'; +import { testRsaSigner } from './saml-cert-fixture'; +import { NAMEID_EMAIL } from './saml-dom'; +import { samlTestIdp } from './saml-idp-fixture'; +import { refusalOf, samlHarness } from './saml-test-support-fixture'; +import { SIGNED_MESSAGE_RESPONSE } from './saml-vectors-fixture'; + +const ASSERTION = //; +const assertionOf = (xml: string): string => + ASSERTION.exec(xml)?.[0] ?? expect.unreachable('no assertion'); +const INVALID = 'X_SAML_RESPONSE_INVALID'; + +describe('signature wrapping', () => { + test('an evil assertion beside the signed one (XSW: two assertions) is refused', async () => { + const login = await (await samlHarness()).begin(); + const signed = await login.xml(); + const evil = assertionOf(await login.xml({ nameId: 'root', signAssertion: false })); + const wrapped = signed.replace(' { + const login = await (await samlHarness()).begin(); + const signed = await login.xml({ assertionId: '_same' }); + const evil = assertionOf( + await login.xml({ nameId: 'root', assertionId: '_same', signAssertion: false }), + ); + const wrapped = signed.replace('
_pYYYYMMDD` (lower bound), UTC; `week` is ISO; `to_char`, so `DateStyle`/`TimeZone` never move a bound. Hand-made children are never touched | +| Snapshot | the parent records `partitioned: true` and `partitionInterval` (snapshot only, never read off the catalog) | +| Refusals | `X_MIGRATION_PARTITION_CHANGE`: from `x db gen` when a recorded partitioned table loses `partitionBy` (`partitionDropped`), from the function at migrate time when it meets a plain table (adoption on a table the hand recipe already partitioned passes) | +| Interval change | refused by `x db gen` with `X_MIGRATION_PARTITION_CHANGE` (`partitionRewidened`): the snapshot records `partitionInterval` (parsed back by `parseSnapshot`), and partitions of another width would overlap | +| No default partition | by decision: a default partition forbids `detach … concurrently`, and creating a month it holds rows for scans and moves them under ACCESS EXCLUSIVE. A row past every partition is `23514`; `ahead` is the scheduler-outage budget | + +Proof: `generate-partition-fixture.ts`, on PGlite (`generate-partition-embedded.test.ts`) and Postgres (`generate-partition.live.test.ts`); a reader holding the parent: `generate-partition-lock.live.test.ts`; the sidecar round trip: `generate-partition-sidecar.test.ts`. + ## The schema dump **Imported from `@ultimat3/db/schema-dump`, never from the barrel** (23.0.0): `introspectCatalog`, diff --git a/packages/db/src/branch-errors.ts b/packages/db/src/branch-errors.ts new file mode 100644 index 000000000..b7c742643 --- /dev/null +++ b/packages/db/src/branch-errors.ts @@ -0,0 +1,24 @@ +// Single responsibility: the two refusals `x db branch` raises — a branch that exists, and a name +// that is not one. Split from `errors.ts` at its line ceiling; `index.ts` re-exports both. + +import { DbError } from './errors'; + +export const branchExists = (branch: string): DbError => + new DbError({ + code: 'X_BRANCH_EXISTS', + cause: `database "${branch}" already exists`, + fix: `x db branch delete ${branch} # then re-create, or pick another name`, + meta: { branch }, + }); + +/** + * An unvalidated branch name is spliced into `CREATE DATABASE ""`, so a bad name is an + * injection vector, not a typo — hence `X_SQL_UNSAFE` rather than a validation code. + */ +export const branchNameInvalid = (branch: string): DbError => + new DbError({ + code: 'X_SQL_UNSAFE', + cause: `branch name "${branch}" is not [a-z0-9_-]+`, + fix: 'x db branch create # lowercase letters, digits, underscore and dash only', + meta: { branch }, + }); diff --git a/packages/db/src/branch.ts b/packages/db/src/branch.ts index 0b056f747..edddf033d 100644 --- a/packages/db/src/branch.ts +++ b/packages/db/src/branch.ts @@ -4,8 +4,9 @@ // Branches are cheap and forgettable, so `reapBranches()` is part of the design, not an add-on. import { finiteCount, systemClock } from '@ultimat3/core'; +import { branchExists, branchNameInvalid } from './branch-errors'; import { baseClient, type DbClient } from './client'; -import { branchExists, branchNameInvalid, DbError } from './errors'; +import { DbError } from './errors'; import { identifier, literal, sql } from './sql'; const BRANCH_NAME = /^[a-z0-9_-]+$/; diff --git a/packages/db/src/entity-shape.ts b/packages/db/src/entity-shape.ts index 2ec543297..d61f5b414 100644 --- a/packages/db/src/entity-shape.ts +++ b/packages/db/src/entity-shape.ts @@ -157,6 +157,26 @@ export interface EntityDescriptionLike { readonly counts?: readonly CounterDescriptionLike[] | undefined; /** `entity({ retain })`: rows older than the window are swept by `purge()`. Absent is kept. */ readonly retain?: RetainDescriptionLike | undefined; + /** + * `entity({ partitionBy })`: a range-partitioned table (`generate-partition.ts`). Absent is a + * plain table, for the reason `invariants` is optional. + */ + readonly partition?: PartitionDescriptionLike | undefined; +} + +/** The bucket widths. `week` is ISO: it starts on a Monday, as `date_trunc('week', …)` does. */ +export const PARTITION_INTERVALS = ['day', 'week', 'month', 'year'] as const; +export type PartitionInterval = (typeof PARTITION_INTERVALS)[number]; + +/** Structurally assignment-compatible with `@ultimat3/entity`'s `PartitionDescription`. */ +export interface PartitionDescriptionLike { + /** The physical partition column, part of the primary key. */ + readonly column: string; + readonly interval: PartitionInterval; + /** Partitions kept ready past the current one. */ + readonly ahead: number; + /** Whole partitions whose upper bound is older than `afterMs` are detached, then dropped. */ + readonly retain?: { readonly after: string; readonly afterMs: number } | undefined; } /** Where a table's row images go, and what its trigger needs to write one. Physical names. */ diff --git a/packages/db/src/errors.ts b/packages/db/src/errors.ts index 94b964369..06181d188 100644 --- a/packages/db/src/errors.ts +++ b/packages/db/src/errors.ts @@ -58,6 +58,7 @@ export const DB_OWNED_ERROR_CODES = [ 'X_MIGRATION_ROLLBACK_DESTRUCTIVE', 'X_ENTITY_RETAIN_INVALID', 'X_LIFECYCLE_TRIGGER_MISSING', + 'X_MIGRATION_PARTITION_CHANGE', ] as const; /** @@ -131,6 +132,7 @@ export const DB_ERROR_TITLES: Readonly> = { 'entity({ retain }) has no window, sits beside appendOnly, or is on a partitioned table', X_LIFECYCLE_TRIGGER_MISSING: 'a history or counter trigger the migrations created is missing from the database', + X_MIGRATION_PARTITION_CHANGE: 'an existing table gained or lost entity({ partitionBy })', }; // Registered unconditionally, in one call, so a second package claiming one of db's codes fails @@ -462,26 +464,6 @@ export const indexMethodInvalid = (received: unknown): DbError => fix: "indexes: [{ on: ['tags'], using: 'gin' }] # or leave it out for the btree default", }); -export const branchExists = (branch: string): DbError => - new DbError({ - code: 'X_BRANCH_EXISTS', - cause: `database "${branch}" already exists`, - fix: `x db branch delete ${branch} # then re-create, or pick another name`, - meta: { branch }, - }); - -/** - * An unvalidated branch name is spliced into `CREATE DATABASE ""`, so a bad name is an - * injection vector, not a typo — hence `X_SQL_UNSAFE` rather than a validation code. - */ -export const branchNameInvalid = (branch: string): DbError => - new DbError({ - code: 'X_SQL_UNSAFE', - cause: `branch name "${branch}" is not [a-z0-9_-]+`, - fix: 'x db branch create # lowercase letters, digits, underscore and dash only', - meta: { branch }, - }); - /** * An isolation level that is not one of the three. `X_SQL_UNSAFE` for the reason * `branchNameInvalid` uses it: `BEGIN` takes no parameters, so the level is SPLICED into the diff --git a/packages/db/src/generate-partition-embedded.test.ts b/packages/db/src/generate-partition-embedded.test.ts new file mode 100644 index 000000000..769784696 --- /dev/null +++ b/packages/db/src/generate-partition-embedded.test.ts @@ -0,0 +1,33 @@ +// The partitioning proof on the embedded PGlite — the dev database and every drift replay — so the +// function `x db gen` writes is one both engines run, plpgsql included. + +import { afterAll, describe, test } from 'bun:test'; +import { proveBackfillFrom, provePartitioning } from './generate-partition-fixture'; +import { pgliteClient } from './pglite'; + +// A WASM compile plus an initdb, against bun's 5s default — a hang detector, not a budget. +const PGLITE_BOOT_MS = 30_000; + +describe('embedded · pglite · entity({ partitionBy })', () => { + const client = pgliteClient(); + + afterAll(async () => { + await client.close(); + }); + + test( + 'monthly partitions are created, an insert routes, retention drops a partition', + async () => { + await provePartitioning(client, 'gp_events_embedded'); + }, + PGLITE_BOOT_MS, + ); + + test( + 'a backfill target gets every partition back to its oldest row (from)', + async () => { + await proveBackfillFrom(client, 'gp_back_embedded'); + }, + PGLITE_BOOT_MS, + ); +}); diff --git a/packages/db/src/generate-partition-fixture.ts b/packages/db/src/generate-partition-fixture.ts new file mode 100644 index 000000000..121cf19f5 --- /dev/null +++ b/packages/db/src/generate-partition-fixture.ts @@ -0,0 +1,230 @@ +// TEST-ONLY. The one proof of `entity({ partitionBy })`'s SQL, run against a real server by two +// suites: Postgres (`generate-partition.live.test.ts`) and the embedded PGlite +// (`generate-partition-embedded.test.ts`). Monthly partitions are created, an insert routes to its +// month, the maintenance call creates ahead and its retention drops whole partitions. + +import { expect } from 'bun:test'; +import { renderThrowable } from '@ultimat3/core'; +import type { DbClient } from './client'; +import { diffSchema } from './drift'; +import type { + ColumnDescriptionLike, + EntityDescriptionLike, + PartitionDescriptionLike, +} from './entity-shape'; +import { generateMigration } from './generate'; +import { partitionMaintenance, RETIRING_COMMENT } from './generate-partition'; +import { maintainPartitions } from './generate-partition-run'; +import { introspectSchema } from './introspect'; +import { raw } from './sql'; +import { statementsOf } from './statement-split'; + +const column = (name: string, kind: string, primaryKey = false): ColumnDescriptionLike => ({ + property: name, + column: name, + kind, + notNull: true, + primaryKey, + unique: false, + hasDefault: false, + check: null, + references: null, +}); + +const DAY_MS = 86_400_000; + +/** A monthly `events` table keyed `(id, at)`, as `entity({ partitionBy })` describes one. */ +export const partitionedEvents = ( + table: string, + partition: PartitionDescriptionLike | null = { column: 'at', interval: 'month', ahead: 2 }, +): EntityDescriptionLike => ({ + name: table, + table, + primaryKey: ['id', 'at'], + columns: [column('id', 'uuid', true), column('at', 'timestamptz', true), column('kind', 'text')], + indexes: [ + { name: `${table}_kind_idx`, columns: ['kind'], unique: false, where: null, order: null }, + ], + ...(partition === null ? {} : { partition }), +}); + +/** `
_pYYYYMM01` for the month `months` after `at`'s, UTC — the function's own naming. */ +export const monthPartition = (table: string, at: Date, months: number): string => { + const first = new Date(Date.UTC(at.getUTCFullYear(), at.getUTCMonth() + months, 1)); + return `${table}_p${first.toISOString().slice(0, 10).replaceAll('-', '')}`; +}; + +/** The partitions of `table`, by name, as the catalog holds them. */ +export const partitionsOf = async (client: DbClient, table: string): Promise => + ( + await client.query<{ name: string }>( + raw( + 'select c.relname::text as name from pg_inherits h join pg_class c on c.oid = h.inhrelid ' + + `where h.inhparent = '"${table}"'::regclass order by 1`, + ), + ) + ).map((row) => row.name); + +const apply = async (client: DbClient, script: string): Promise => { + for (const statement of statementsOf(script)) await client.execute(raw(statement)); +}; + +/** The text a refused statement carried, server error included. */ +const refusal = async (send: Promise): Promise => { + try { + await send; + } catch (error) { + const source = (error as { sourceError?: unknown }).sourceError ?? error; + return `${renderThrowable(error)} ${renderThrowable(source)}`; + } + return expect.unreachable('the statement was not refused'); +}; + +export async function provePartitioning(client: DbClient, table: string): Promise { + await client.execute(raw(`drop table if exists "${table}" cascade`)); + const entity = partitionedEvents(table); + const migration = generateMigration({ entities: [entity], name: 'events', now: new Date() }); + expect(migration.destructive).toBe(false); + await apply(client, migration.up); + + // The migration created this month and two ahead, judged at APPLY time by the SERVER's clock — + // the test runner's `Date` may be pinned, the database's never is. + const clock = await client.one<{ now: string }>(raw(`select to_json(now())#>>'{}' as now`)); + const now = new Date(clock?.now ?? ''); + const created = [0, 1, 2].map((months) => monthPartition(table, now, months)); + const [first = '', second = ''] = created; + expect(await partitionsOf(client, table)).toEqual(created); + + // An insert routes to its month's partition; a read of the parent sees both. + const next = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 15)); + await client.execute( + raw( + `insert into "${table}" (id, at, kind) values ` + + `('00000000-0000-7000-8000-000000000001', '${now.toISOString()}', 'a'), ` + + `('00000000-0000-7000-8000-000000000002', '${next.toISOString()}', 'b')`, + ), + ); + const routed = await client.query<{ kind: string; part: string }>( + raw(`select kind, tableoid::regclass::text as part from "${table}" order by kind`), + ); + expect(routed.map((row) => [row.kind, row.part.replaceAll('"', '')])).toEqual([ + ['a', first], + ['b', second], + ]); + + // A row past every partition is refused by Postgres — no default partition hides it. + const far = new Date(now.getTime() + 400 * DAY_MS).toISOString(); + expect( + await refusal( + client.execute( + raw( + `insert into "${table}" values ('00000000-0000-7000-8000-000000000003', '${far}', 'c')`, + ), + ), + ), + ).toContain('partition'); + + // Sixteen months on, a 400-day window (about thirteen months): this month and two ahead of then + // are created, and the first two months — whose upper bounds the window passed — are detached + // and dropped with their rows. + const later = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 16, 2)); + const retained: EntityDescriptionLike = { + ...entity, + partition: { + column: 'at', + interval: 'month', + ahead: 2, + retain: { after: '400d', afterMs: 400 * DAY_MS }, + }, + }; + const run = await maintainPartitions(client, retained, later); + expect(run.created).toEqual([0, 1, 2].map((m) => monthPartition(table, later, m))); + expect(run.dropped).toEqual([first, second]); + expect(await partitionsOf(client, table)).not.toContain(first); + const left = await client.query<{ kind: string }>(raw(`select kind from "${table}"`)); + expect(left).toEqual([]); + + // A second run at the same instant is a no-op both ways. + expect(await maintainPartitions(client, retained, later)).toEqual({ created: [], dropped: [] }); + + // A run that died after its detach left a marked, detached table: the next run drops it. An + // unmarked table of the same shape of name is not this package's and is never touched. + const third = monthPartition(table, now, 2); + await client.execute(raw(`comment on table "${third}" is '${RETIRING_COMMENT}'`)); + await client.execute(raw(`alter table "${table}" detach partition "${third}"`)); + const stray = monthPartition(table, now, -30); + await client.execute(raw(`create table "${stray}" (id int)`)); + const resumed = await maintainPartitions( + client, + retained, + new Date(later.getTime() + 62 * DAY_MS), + ); + expect(resumed.dropped).toContain(third); + expect(resumed.dropped).not.toContain(stray); + expect(await client.one(raw(`select to_regclass('"${third}"') as t`))).toEqual({ t: null }); + await client.execute(raw(`drop table "${stray}"`)); + + // Drift judges the parent: still the table the snapshot records, partitions or not. + const live = await introspectSchema({ client }); + const mine = { tables: live.tables.filter((each) => each.name === table) }; + expect(diffSchema(mine, migration.snapshot).differences).toEqual([]); + + // Adopting partitionBy on a PLAIN table is refused at migrate time, with the code. + const plain = `${table}_plain`; + await client.execute(raw(`drop table if exists "${plain}" cascade`)); + const created2 = generateMigration({ + entities: [partitionedEvents(plain, null)], + name: 'plain', + now: new Date(), + }); + await apply(client, created2.up); + const adopt = generateMigration({ + entities: [partitionedEvents(plain)], + current: created2.snapshot, + name: 'adopt', + now: new Date(), + }); + expect(await refusal(apply(client, adopt.up))).toContain('X_MIGRATION_PARTITION_CHANGE'); + + await client.execute(raw(`drop table if exists "${plain}" cascade`)); + await client.execute(raw(`drop table "${table}" cascade`)); +} + +/** + * The recovery recipe, runnable: a new partitioned table receives rows older than its first + * partition — a `backfill()` from the table it replaces. `partitionMaintenance(…, { from })` + * creates every partition back to the oldest row first, so the copy routes instead of `23514`. + */ +export async function proveBackfillFrom(client: DbClient, table: string): Promise { + await client.execute(raw(`drop table if exists "${table}" cascade`)); + const entity = partitionedEvents(table); + await apply( + client, + generateMigration({ entities: [entity], name: 'events', now: new Date() }).up, + ); + const clock = await client.one<{ now: string }>(raw(`select to_json(now())#>>'{}' as now`)); + const now = new Date(clock?.now ?? ''); + const oldest = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 5, 20)); + const insertOld = () => + client.execute( + raw( + `insert into "${table}" (id, at, kind) values ` + + `('00000000-0000-7000-8000-000000000009', '${oldest.toISOString()}', 'old')`, + ), + ); + // Without the bound: the recipe's first copied row has no partition. + expect(await refusal(insertOld())).toContain('partition'); + + const back = partitionMaintenance(entity, now, { from: oldest }); + if (back === null) return expect.unreachable('a partitioned entity has maintenance'); + const [answer] = await client.query<{ result: { created: string[] } }>(back); + expect(answer?.result.created).toEqual( + [-5, -4, -3, -2, -1].map((months) => monthPartition(table, now, months)), + ); + await insertOld(); + const [row] = await client.query<{ part: string }>( + raw(`select tableoid::regclass::text as part from "${table}" where kind = 'old'`), + ); + expect(row?.part.replaceAll('"', '')).toBe(monthPartition(table, now, -5)); + await client.execute(raw(`drop table "${table}" cascade`)); +} diff --git a/packages/db/src/generate-partition-lock.live.test.ts b/packages/db/src/generate-partition-lock.live.test.ts new file mode 100644 index 000000000..b1b4edb20 --- /dev/null +++ b/packages/db/src/generate-partition-lock.live.test.ts @@ -0,0 +1,71 @@ +// Partition maintenance never takes a table down: a long reader holding the parent makes the +// maintenance step fail fast with `X_DB_LOCK_TIMEOUT` (retryable — the job's retry absorbs it) +// instead of queueing its ACCESS EXCLUSIVE lock and every insert and select behind it. + +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { isUltimateError } from '@ultimat3/core'; +import { type PostgresClient, postgresClient } from './client'; +import { generateMigration } from './generate'; +import { partitionedEvents } from './generate-partition-fixture'; +import { maintainPartitions } from './generate-partition-run'; +import { raw } from './sql'; +import { statementsOf } from './statement-split'; +import { withTransaction } from './transaction'; + +const url = Bun.env['TEST_DATABASE_URL']; +const hasPostgres = typeof url === 'string' && url.length > 0; +const TABLE = 'gp_lock_live'; + +describe.skipIf(!hasPostgres)('live · postgres · partition maintenance under a reader', () => { + let client: PostgresClient; + let reader: PostgresClient; + + beforeAll(async () => { + client = postgresClient({ url: url ?? '' }); + reader = postgresClient({ url: url ?? '' }); + await client.execute(raw(`drop table if exists "${TABLE}" cascade`)); + const { up } = generateMigration({ entities: [partitionedEvents(TABLE)], name: 'lock' }); + for (const statement of statementsOf(up)) await client.execute(raw(statement)); + }); + + afterAll(async () => { + await client.execute(raw(`drop table if exists "${TABLE}" cascade`)); + await reader.close(); + await client.close(); + }); + + test('a reader holding the parent makes creation fail fast, and the table keeps serving', async () => { + let release = (): void => undefined; + const held = new Promise((resolve) => { + release = resolve; + }); + let opened = (): void => undefined; + const reading = new Promise((resolve) => { + opened = resolve; + }); + const transaction = withTransaction( + async (tx) => { + await tx.query(raw(`select count(*) from "${TABLE}"`)); + opened(); + await held; + }, + { client: reader }, + ); + await reading; + const started = performance.now(); + let code: unknown = 'resolved'; + try { + // A year on: a partition must be created, which needs ACCESS EXCLUSIVE on the parent. + await maintainPartitions(client, partitionedEvents(TABLE), new Date(Date.now() + 4e10), { + lockTimeoutMs: 200, + }); + } catch (error) { + code = isUltimateError(error) ? error.code : String(error); + } + const waited = performance.now() - started; + release(); + await transaction; + expect(code).toBe('X_DB_LOCK_TIMEOUT'); + expect(waited).toBeLessThan(3000); + }); +}); diff --git a/packages/db/src/generate-partition-run.ts b/packages/db/src/generate-partition-run.ts new file mode 100644 index 000000000..1703bf581 --- /dev/null +++ b/packages/db/src/generate-partition-run.ts @@ -0,0 +1,97 @@ +// Single responsibility: one partitioned table's maintenance, run — create ahead under a bounded +// `lock_timeout`, then retire each expired partition without blocking the parent: mark it, `detach +// … concurrently` (outside any transaction, as Postgres requires), drop it. A `55P03` is +// `X_DB_LOCK_TIMEOUT`, retryable: the caller's retry absorbs a long report query, never the table. + +import { finiteCount } from '@ultimat3/core'; +import type { DbClient } from './client'; +import type { EntityDescriptionLike } from './entity-shape'; +import { + type PartitionMaintenanceOptions, + partitionMaintenance, + RETIRING_COMMENT, +} from './generate-partition'; +import { identifier, literal, raw } from './sql'; +import { withTransaction } from './transaction'; +import type { DbTx } from './transaction-options'; + +/** How long one maintenance statement waits for a lock on the parent before giving up. */ +export const PARTITION_LOCK_TIMEOUT_MS = 5000; + +export interface PartitionRunOptions extends PartitionMaintenanceOptions { + readonly lockTimeoutMs?: number | undefined; +} + +export interface PartitionRun { + readonly created: readonly string[]; + readonly dropped: readonly string[]; +} + +type Expired = { readonly name: string; readonly state: 'attached' | 'pending' | 'detached' }; +const RETIREMENT_STATES: readonly string[] = ['attached', 'pending', 'detached']; + +const names = (value: unknown): readonly string[] => + Array.isArray(value) ? value.filter((each): each is string => typeof each === 'string') : []; + +const expiredOf = (value: unknown): readonly Expired[] => + (Array.isArray(value) ? value : []).flatMap((each: unknown): Expired[] => { + if (typeof each !== 'object' || each === null) return []; + const { name, state: stage } = each as Record; + return typeof name === 'string' && + typeof stage === 'string' && + RETIREMENT_STATES.includes(stage) + ? [{ name, state: stage as Expired['state'] }] + : []; + }); + +/** One transaction whose every lock wait is bounded: a blocked DDL fails fast, the table keeps serving. */ +const bounded = (client: DbClient, ms: number, work: (tx: DbTx) => Promise): Promise => + withTransaction( + async (tx) => { + await tx.execute(raw(`set local lock_timeout = ${ms}`)); + return work(tx); + }, + { client }, + ); + +/** + * Creates what `partitionBy` (and `from`) need, then retires each expired partition, resuming a + * run that died anywhere in between: a `pending` detach is finalized, a `detached` one is dropped. + * Never inside a caller's transaction — `detach … concurrently` refuses one (`25001`). + */ +export async function maintainPartitions( + client: DbClient, + entity: EntityDescriptionLike, + instant: Date, + options: PartitionRunOptions = {}, +): Promise { + const statement = partitionMaintenance(entity, instant, options); + if (statement === null) return { created: [], dropped: [] }; + const ms = finiteCount( + 'maintainPartitions', + 'lockTimeoutMs', + options.lockTimeoutMs ?? PARTITION_LOCK_TIMEOUT_MS, + 1, + ); + const [row] = await bounded(client, ms, (tx) => tx.query<{ result: unknown }>(statement)); + const result = + typeof row?.result === 'object' && row.result !== null + ? (row.result as Record) + : {}; + const parent = identifier(entity.table).text; + const dropped: string[] = []; + for (const { name, state } of expiredOf(result['expired'])) { + const child = identifier(name).text; + if (state === 'attached') { + await bounded(client, ms, (tx) => + tx.execute(raw(`comment on table ${child} is ${literal(RETIRING_COMMENT).text}`)), + ); + await client.execute(raw(`alter table ${parent} detach partition ${child} concurrently`)); + } else if (state === 'pending') { + await client.execute(raw(`alter table ${parent} detach partition ${child} finalize`)); + } + await bounded(client, ms, (tx) => tx.execute(raw(`drop table if exists ${child}`))); + dropped.push(name); + } + return { created: names(result['created']), dropped }; +} diff --git a/packages/db/src/generate-partition-sidecar.test.ts b/packages/db/src/generate-partition-sidecar.test.ts new file mode 100644 index 000000000..62fbe15c4 --- /dev/null +++ b/packages/db/src/generate-partition-sidecar.test.ts @@ -0,0 +1,66 @@ +// A partitioned table's snapshot through the sidecar, as `x db gen` reads it: generate, write the +// snapshot as JSON, `parseSnapshot` it back, generate again. Every `partitionPlan` judgment has to +// survive the disk, or it judges nothing in a real app. + +import { describe, expect, test } from 'bun:test'; +import { isUltimateError } from '@ultimat3/core'; +import type { PartitionDescriptionLike } from './entity-shape'; +import { generateMigration } from './generate'; +import { partitionedEvents } from './generate-partition-fixture'; +import type { SchemaDescription } from './introspect'; +import { parseSnapshot } from './snapshot-parse'; + +const NOW = new Date('2026-11-09T00:00:00.000Z'); +const MONTHLY: PartitionDescriptionLike = { column: 'at', interval: 'month', ahead: 2 }; + +/** The previous migration's snapshot, as `x db gen` finds it: through JSON and the parser. */ +const fromDisk = (partition: PartitionDescriptionLike | null): SchemaDescription => { + const snapshot = generateMigration({ + entities: [partitionedEvents('events', partition)], + name: 'events', + now: NOW, + }).snapshot; + const parsed = parseSnapshot(JSON.parse(JSON.stringify(snapshot))); + if (parsed === undefined) return expect.unreachable('the sidecar did not parse'); + return parsed; +}; + +const next = (partition: PartitionDescriptionLike | null) => { + try { + return generateMigration({ + entities: [partitionedEvents('events', partition)], + current: fromDisk(MONTHLY), + name: 'again', + now: NOW, + }).up; + } catch (error) { + return isUltimateError(error) ? error.code : expect.unreachable(String(error)); + } +}; + +describe('a partitioned snapshot, through the sidecar', () => { + test('the parsed snapshot still records partitioned and its interval', () => { + expect(fromDisk(MONTHLY).tables[0]).toMatchObject({ + partitioned: true, + partitionInterval: 'month', + }); + }); + + test('an unchanged partitioned entity writes nothing', () => { + expect(next(MONTHLY)).toBe(''); + }); + + test('removing partitionBy is refused', () => { + expect(next(null)).toBe('X_MIGRATION_PARTITION_CHANGE'); + }); + + test('changing the interval is refused', () => { + expect(next({ ...MONTHLY, interval: 'day' })).toBe('X_MIGRATION_PARTITION_CHANGE'); + }); + + test('an interval outside the set takes the sidecar with it', () => { + const bad = JSON.parse(JSON.stringify(fromDisk(MONTHLY))); + bad.tables[0].partitionInterval = 'hour'; + expect(parseSnapshot(bad)).toBeUndefined(); + }); +}); diff --git a/packages/db/src/generate-partition.live.test.ts b/packages/db/src/generate-partition.live.test.ts new file mode 100644 index 000000000..6031e6cab --- /dev/null +++ b/packages/db/src/generate-partition.live.test.ts @@ -0,0 +1,30 @@ +// `entity({ partitionBy })` on a real Postgres: monthly partitions created at apply time, an insert +// routed to its month, a row past every partition refused, retention detaching and dropping whole +// partitions, drift clean on the parent, and adoption on a plain table refused with the code. + +import { afterAll, beforeAll, describe, test } from 'bun:test'; +import { type PostgresClient, postgresClient } from './client'; +import { proveBackfillFrom, provePartitioning } from './generate-partition-fixture'; + +const url = Bun.env['TEST_DATABASE_URL']; +const hasPostgres = typeof url === 'string' && url.length > 0; + +describe.skipIf(!hasPostgres)('live · postgres · entity({ partitionBy })', () => { + let client: PostgresClient; + + beforeAll(() => { + client = postgresClient({ url: url ?? '' }); + }); + + afterAll(async () => { + await client.close(); + }); + + test('monthly partitions are created, an insert routes, retention drops a partition', async () => { + await provePartitioning(client, 'gp_events_live'); + }); + + test('a backfill target gets every partition back to its oldest row (from)', async () => { + await proveBackfillFrom(client, 'gp_back_live'); + }); +}); diff --git a/packages/db/src/generate-partition.test.ts b/packages/db/src/generate-partition.test.ts new file mode 100644 index 000000000..d37e05176 --- /dev/null +++ b/packages/db/src/generate-partition.test.ts @@ -0,0 +1,94 @@ +// `generateMigration` over a description carrying `partition`: the parent's `partition by range`, +// the function and its first call after the table, the snapshot's `partitioned` record, and the +// removal refused. A plain table's migration is untouched by any of it. + +import { describe, expect, test } from 'bun:test'; +import { isUltimateError } from '@ultimat3/core'; +import { generateMigration } from './generate'; +import { PARTITION_FUNCTION, partitionMaintenance } from './generate-partition'; +import { partitionedEvents } from './generate-partition-fixture'; + +const NOW = new Date('2026-11-09T00:00:00.000Z'); +const gen = (partitioned: boolean, current?: ReturnType) => + generateMigration({ + entities: [partitionedEvents('events', partitioned ? undefined : null)], + name: 'events', + now: NOW, + ...(current === undefined ? {} : { current: current.snapshot }), + }); + +describe('generateMigration · entity({ partitionBy })', () => { + test('a new table is created partitioned, then its function and first partitions', () => { + const up = gen(true).up; + expect(up).toContain('\n) partition by range ("at");'); + const fn = up.indexOf(`create or replace function "${PARTITION_FUNCTION}"`); + const call = up.indexOf( + `select "${PARTITION_FUNCTION}"('events', 'timestamptz', 'month', 2, null);`, + ); + const index = up.indexOf('create index'); + expect(index).toBeGreaterThanOrEqual(0); + expect(fn).toBeGreaterThan(index); + expect(call).toBeGreaterThan(fn); + }); + + test('the migration is not destructive: the retention drop lives in a function body', () => { + expect(gen(true).destructive).toBe(false); + }); + + test('the snapshot records the parent partitioned; a plain table records nothing new', () => { + expect(gen(true).snapshot.tables[0]?.partitioned).toBe(true); + expect(gen(false).snapshot.tables[0]).not.toHaveProperty('partitioned'); + expect(gen(false).up).not.toContain(PARTITION_FUNCTION); + }); + + test('regenerating an unchanged partitioned table writes nothing', () => { + expect(gen(true, gen(true)).up).toBe(''); + }); + + test('dropping partitionBy from a partitioned table is X_MIGRATION_PARTITION_CHANGE', () => { + let code: unknown; + try { + gen(false, gen(true)); + } catch (error) { + code = isUltimateError(error) ? error.code : error; + } + expect(code).toBe('X_MIGRATION_PARTITION_CHANGE'); + }); + + test('changing the interval of a partitioned table is X_MIGRATION_PARTITION_CHANGE', () => { + const monthly = gen(true); + const daily = { column: 'at', interval: 'day', ahead: 2 } as const; + let refused: unknown; + try { + generateMigration({ + entities: [partitionedEvents('events', daily)], + current: monthly.snapshot, + name: 'daily', + now: NOW, + }); + } catch (error) { + refused = isUltimateError(error) ? [error.code, error.cause, error.fix] : error; + } + expect(refused).toEqual([ + 'X_MIGRATION_PARTITION_CHANGE', + expect.stringContaining('from month to day'), + expect.stringContaining('backfill()'), + ]); + }); + + test('maintenance binds every value and is absent on a plain table', () => { + const plain = partitionMaintenance(partitionedEvents('events', null), NOW); + expect(plain).toBeNull(); + const bound = partitionMaintenance(partitionedEvents('events'), NOW); + expect(bound?.text).not.toContain('events'); + expect(bound?.values).toEqual([ + 'events', + 'timestamptz', + 'month', + 2, + null, + NOW.toISOString(), + null, + ]); + }); +}); diff --git a/packages/db/src/generate-partition.ts b/packages/db/src/generate-partition.ts new file mode 100644 index 000000000..251c569b2 --- /dev/null +++ b/packages/db/src/generate-partition.ts @@ -0,0 +1,190 @@ +// Single responsibility: `entity({ partitionBy })` as SQL — the parent's `partition by range` +// clause and the ONE plpgsql function that creates the partitions ahead and drops expired ones. The +// migration creating the table and the maintenance job both call it, so both name and bound a +// partition identically, and both read the database's clock unless handed one. + +import { renderFixShellArg } from '@ultimat3/core'; +import type { EntityDescriptionLike, PartitionDescriptionLike } from './entity-shape'; +import { DbError } from './errors'; +import { findTable, type SchemaDescription } from './introspect'; +import { identifier, literal, raw, type SqlFragment, sql } from './sql'; + +/** The column kinds a range can be bounded by: `timestamp()` and `date()`. */ +export const PARTITION_KINDS = ['timestamptz', 'date'] as const; + +/** `_p20261101`: the suffix every partition carries — its lower bound, UTC, `YYYYMMDD`. */ +export const PARTITION_SUFFIX_BYTES = 10; + +export const PARTITION_FUNCTION = 'ultimate_partitions'; + +/** A bound, spelled independently of `DateStyle` and `TimeZone`: UTC, ISO. */ +const bound = (at: string): string => + `case when kind = 'date' then to_char(${at}, 'YYYY-MM-DD') ` + + `else to_char(${at}, 'YYYY-MM-DD HH24:MI:SS') || '+00' end`; + +/** What a retiring partition's comment says, so a detached orphan is known to be this package's. */ +export const RETIRING_COMMENT = 'ultimate:partition-retiring'; + +/** + * Creates every partition from `since`'s (the current one when `since` is null) through `ahead` + * past the current one (`if not exists`, so a double fire is a no-op). With `keep_ms`, it also + * NAMES — never touches — every `_pYYYYMMDD` whose upper bound is `keep_ms` before + * `instant`, with its state: `attached`, `pending` (a `detach … concurrently` interrupted) or + * `detached` (detached by a run that died before its drop, known by `RETIRING_COMMENT`). The caller + * retires them outside any transaction (`generate-partition-run.ts`): `detach … concurrently` + * cannot run inside a function. Refuses a parent that is not partitioned with this package's code. + * Answers `{ created, expired: [{ name, state }] }`. + */ +export const PARTITION_FUNCTION_SQL = + `create or replace function ${identifier(PARTITION_FUNCTION).text}(parent text, kind text, ` + + 'unit text, ahead integer, keep_ms bigint, instant timestamptz default now(), ' + + 'since timestamptz default null) returns jsonb ' + + 'language plpgsql as $partitions$ declare ' + + "step interval := ('1 ' || unit)::interval; " + + "head timestamp := date_trunc(unit, instant at time zone 'UTC'); " + + 'low timestamp; part text; child record; ' + + "created text[] := '{}'; expired jsonb := '[]'; " + + 'begin ' + + 'if (select c.relkind from pg_class c where c.oid = to_regclass(quote_ident(parent))) ' + + "is distinct from 'p' then raise exception 'X_MIGRATION_PARTITION_CHANGE: table % is not " + + "partitioned, and its entity declares partitionBy', parent using errcode = '42809', " + + "hint = 'Postgres cannot partition a table in place: declare a new entity with partitionBy, " + + "run partitions() with from: , then backfill() it'; end if; " + + "low := least(head, coalesce(date_trunc(unit, since at time zone 'UTC'), head)); " + + 'while low <= head + step * ahead loop ' + + "part := parent || '_p' || to_char(low, 'YYYYMMDD'); " + + 'if to_regclass(quote_ident(part)) is null then ' + + "execute format('create table if not exists %I partition of %I for values from (%L) to (%L)', " + + `part, parent, ${bound('low')}, ${bound('low + step')}); ` + + 'created := created || part; end if; low := low + step; end loop; ' + + 'if keep_ms is not null then ' + + 'for child in select c.relname::text as name, case when h.inhrelid is null then ' + + "'detached' when h.inhdetachpending then 'pending' else 'attached' end as state " + + 'from pg_class c left join pg_inherits h on h.inhrelid = c.oid ' + + 'and h.inhparent = to_regclass(quote_ident(parent)) ' + + "where c.relkind = 'r' and c.relnamespace = (select p.relnamespace from pg_class p " + + 'where p.oid = to_regclass(quote_ident(parent))) ' + + `and length(c.relname) = length(parent) + ${PARTITION_SUFFIX_BYTES} ` + + "and left(c.relname, length(parent) + 2) = parent || '_p' " + + "and right(c.relname, 8) ~ '^[0-9]{8}$' " + + 'and (h.inhrelid is not null or (not c.relispartition and ' + + `obj_description(c.oid, 'pg_class') = '${RETIRING_COMMENT}')) order by c.relname loop ` + + "if to_date(right(child.name, 8), 'YYYYMMDD')::timestamp + step <= " + + "(instant at time zone 'UTC') - make_interval(secs => keep_ms / 1000.0) then " + + "expired := expired || jsonb_build_object('name', child.name, 'state', child.state); " + + 'end if; end loop; end if; ' + + "return jsonb_build_object('created', to_jsonb(created), 'expired', expired); " + + 'end; $partitions$;'; + +/** + * The function's first signature, before `since` existed: `create or replace` with another argument + * list makes an OVERLOAD, and a call naming five arguments then matches both (`42725`). + */ +export const PARTITION_FUNCTION_RETIRED_SQL = + `drop function if exists ${identifier(PARTITION_FUNCTION).text}` + + '(text, text, text, integer, bigint, timestamptz);'; + +/** ` partition by range ("at")` — appended to the parent's `create table (…)`. */ +export const partitionByClause = (partition: PartitionDescriptionLike): string => + ` partition by range (${identifier(partition.column).text})`; + +/** The partition column's kind, read off the description's own column. */ +const kindOf = (entity: EntityDescriptionLike, partition: PartitionDescriptionLike): string => + entity.columns.find((column) => column.column === partition.column)?.kind ?? 'timestamptz'; + +/** + * The migration's call: partitions from the APPLY time onward, never a retention sweep — a + * migration replayed a year later (a fresh clone, a test database) still has this month's. + */ +export function partitionMigrationCall(entity: EntityDescriptionLike): string | null { + const partition = entity.partition; + if (partition === undefined) return null; + const args = [ + literal(entity.table).text, + literal(kindOf(entity, partition)).text, + literal(partition.interval).text, + String(Math.trunc(partition.ahead)), + 'null', + ].join(', '); + return `select ${identifier(PARTITION_FUNCTION).text}(${args});`; +} + +/** `from`: also create every partition back to this instant's — before a `backfill()` copies old rows. */ +export interface PartitionMaintenanceOptions { + readonly from?: Date | undefined; +} + +/** + * The maintenance statement: create ahead, and name what `retain` expired (retired by + * `maintainPartitions`, never here), judged at `instant` — the framework clock's now, never the server's, so a test can move it. + * With `from`, it also creates every partition back to `from`'s and sweeps NOTHING: a backfill's + * preparation must not drop the partitions it just made for rows older than the window. + * Answers one row, `{ result: { created, expired } }`. `null` on a plain table. + */ +export function partitionMaintenance( + entity: EntityDescriptionLike, + instant: Date, + options: PartitionMaintenanceOptions = {}, +): SqlFragment | null { + const partition = entity.partition; + if (partition === undefined) return null; + const from = options.from === undefined ? null : options.from.toISOString(); + const keep = + partition.retain === undefined || from !== null ? null : Math.trunc(partition.retain.afterMs); + return sql`select ${raw(identifier(PARTITION_FUNCTION).text)}(${entity.table}, ${kindOf(entity, partition)}, ${partition.interval}, ${Math.trunc(partition.ahead)}::integer, ${keep}::bigint, ${instant.toISOString()}::timestamptz, ${from}::timestamptz) as result`; +} + +/** + * A NEW partitioned table gets the function and its first partitions; an existing one that newly + * declares `partitionBy` gets them too — if the hand-edit recipe already partitioned it, that is + * adoption, and if it is a plain table the function refuses it at migrate time with + * `X_MIGRATION_PARTITION_CHANGE`. A table the snapshot records partitioned and the entity no + * longer declares is refused here: no statement turns it back. + */ +export function partitionPlan( + plan: { up: string[] }, + entities: readonly EntityDescriptionLike[], + current: SchemaDescription, +): void { + let installed = false; + for (const entity of entities) { + const recorded = findTable(current, entity.table); + const call = partitionMigrationCall(entity); + if (call === null) { + if (recorded?.partitioned === true) throw partitionDropped(entity.name, entity.table); + continue; + } + const was = recorded?.partitionInterval; + const now = entity.partition?.interval; + if (was !== undefined && now !== undefined && was !== now) { + throw partitionRewidened(entity.name, entity.table, was, now); + } + if (recorded?.partitioned === true) continue; + if (!installed) plan.up.push(PARTITION_FUNCTION_RETIRED_SQL, PARTITION_FUNCTION_SQL); + installed = true; + plan.up.push(call); + } +} + +/** An existing partitioned table losing `partitionBy`: no in-place statement does it. */ +export const partitionDropped = (entity: string, table: string): DbError => + new DbError({ + code: 'X_MIGRATION_PARTITION_CHANGE', + cause: `entity "${entity}" no longer declares partitionBy, and its table "${table}" was created partitioned: Postgres cannot turn it back into a plain table in place`, + fix: `x db gen "unpartition" # after putting partitionBy back on entity('${renderFixShellArg(entity, '')}'), or moving its rows to a new plain entity with backfill()`, + meta: { entity, table }, + }); + +/** A partitioned table's interval changing: partitions of the new width would overlap the old. */ +export const partitionRewidened = ( + entity: string, + table: string, + was: string, + now: string, +): DbError => + new DbError({ + code: 'X_MIGRATION_PARTITION_CHANGE', + cause: `entity "${entity}" changes its partition interval from ${was} to ${now}, and "${table}" already holds ${was} partitions: Postgres cannot repartition in place, and new ones would overlap`, + fix: `x db gen "repartition" # after putting interval: '${was}' back on entity('${renderFixShellArg(entity, '')}'), or declaring a new entity with interval: '${now}', running partitions() with { from: }, then backfill() into it`, + meta: { entity, table }, + }); diff --git a/packages/db/src/generate-snapshot.ts b/packages/db/src/generate-snapshot.ts index e4cfdc3db..5f63ea429 100644 --- a/packages/db/src/generate-snapshot.ts +++ b/packages/db/src/generate-snapshot.ts @@ -22,6 +22,11 @@ export function snapshotOf( ): SchemaDescription { // A view is recorded as its statements (`generate-view.ts`), never as a table it is not. const split = splitViews(entities); + const partitioned = new Map( + entities.flatMap((entity) => + entity.partition === undefined ? [] : [[entity.table, entity.partition.interval] as const], + ), + ); const tables = [...withSameTenantIndexes(split.tables)] .sort((a, b) => (a.table < b.table ? -1 : 1)) .map((entity): TableDescription => { @@ -73,6 +78,10 @@ export function snapshotOf( ...(entity.appendOnly === true ? { appendOnly: true as const } : {}), // And again: the record that makes the tenant policy a one-time statement. ...(rowSecurity.has(entity.table) ? { rowSecurity: true as const } : {}), + // `entity({ partitionBy })`: the record the next `x db gen` refuses its removal against. + ...(partitioned.has(entity.table) + ? { partitioned: true as const, partitionInterval: partitioned.get(entity.table) } + : {}), }; }); // `views` absent when none is declared, so no snapshot written before views existed moves. diff --git a/packages/db/src/generate.ts b/packages/db/src/generate.ts index a051d39b0..0ae95cf81 100644 --- a/packages/db/src/generate.ts +++ b/packages/db/src/generate.ts @@ -12,6 +12,7 @@ import { dropOrder } from './drop-order'; import type { ColumnDescriptionLike, EntityDescriptionLike } from './entity-shape'; import { type ConstraintPlans, foreignKeyPlan, type Plan } from './foreign-key-plan'; import { appendOnlyBackfillRefused, appendOnlyPlan } from './generate-append-only'; +import { partitionByClause, partitionPlan } from './generate-partition'; import { type RowSecurityOption, rowSecurityInput } from './generate-row-security'; import { snapshotOf } from './generate-snapshot'; import { assembleSteps } from './generate-steps'; @@ -76,8 +77,9 @@ function createTable(entity: EntityDescriptionLike): readonly string[] { } // After the key, so a table declaring no invariant emits the statement it always emitted. clauses.push(...checkClauses(entity)); + const by = entity.partition === undefined ? '' : partitionByClause(entity.partition); const statements = [ - `create table ${identifier(entity.table).text} (\n ${clauses.join(',\n ')}\n);`, + `create table ${identifier(entity.table).text} (\n ${clauses.join(',\n ')}\n)${by};`, ]; // Every column of a new table carries its own clause, so every `unique` one brings its index. const added = new Set(entity.columns.map((column) => column.column)); @@ -205,6 +207,7 @@ function diffTable( export { snapshotOf } from './generate-snapshot'; export interface GenerateOptions { + /** `partition` rides on the description when `entity({ partitionBy })` declares one. */ readonly entities: readonly EntityDescriptionLike[]; /** The schema migrations already declare — `expectedSchema(migrations, ledger)`. */ readonly current?: SchemaDescription | undefined; @@ -405,6 +408,8 @@ export function generateMigration(given: GenerateOptions): GeneratedMigration { appendOnlyPlan(plan, options.entities, current, created); // History, its guard and counters, the same way: after every table they name exists. triggerPlan(plan, options.entities, current); + // The partitions a new partitioned table starts with, after its indexes so each clones them. + partitionPlan(plan, options.entities, current); // Then each tenant table's policy, both directions (`row-security-plan.ts`), on tables that exist. const rowSecurity = rowSecurityInput(options.entities, current, options.rowSecurity); rowSecurityPlan(plan, rowSecurity); diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index b73de85ec..416038ae4 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -14,6 +14,7 @@ export { listBranches, reapBranches, } from './branch'; +export { branchExists, branchNameInvalid } from './branch-errors'; export { checkClauses, checkPlan, @@ -84,12 +85,13 @@ export type { HistoryDescriptionLike, IndexDescriptionLike, InvariantDescriptionLike, + PartitionDescriptionLike, + PartitionInterval, RetainDescriptionLike, } from './entity-shape'; +export { PARTITION_INTERVALS } from './entity-shape'; export type { DbErrorCode, DbErrorInit } from './errors'; export { - branchExists, - branchNameInvalid, DB_ERROR_CODES, DB_ERROR_RETRY, DB_ERROR_TITLES, @@ -134,6 +136,14 @@ export { PURGE_SETTING, REQUEST_SETTING, } from './generate-history'; +export type { PartitionMaintenanceOptions } from './generate-partition'; +export { + PARTITION_FUNCTION, + partitionDropped, + partitionMaintenance, +} from './generate-partition'; +export type { PartitionRun, PartitionRunOptions } from './generate-partition-run'; +export { maintainPartitions, PARTITION_LOCK_TIMEOUT_MS } from './generate-partition-run'; export type { RowSecurityOption, TenantColumnLike } from './generate-row-security'; export { declaredTriggers } from './generate-triggers'; export type { IndexMethod } from './index-method'; diff --git a/packages/db/src/introspect.ts b/packages/db/src/introspect.ts index 8674e85fe..0468c3d9f 100644 --- a/packages/db/src/introspect.ts +++ b/packages/db/src/introspect.ts @@ -5,6 +5,7 @@ import { nonAppRelations } from './app-relation'; import { type DbClient, db } from './client'; +import type { PartitionInterval } from './entity-shape'; import { type CheckRow, type ColumnRow, @@ -168,6 +169,12 @@ export interface TableDescription { * descriptions are unchanged. Its partitions are Postgres' and appear in no description. */ readonly partitioned?: true | undefined; + /** + * `entity({ partitionBy })`'s interval, as the migration that created the table recorded it. The + * SNAPSHOT's half only, never the catalog's: `x db gen` refuses a change to it, because partitions + * of another width would overlap the ones that exist (`generate-partition.ts`). + */ + readonly partitionInterval?: PartitionInterval | undefined; /** * That a migration gave this table the tenant policy, enabled and forced (`row-security-plan.ts`). * The SNAPSHOT's half, `true` or absent for `appendOnly`'s reason; the catalog's is `policyNames`. diff --git a/packages/db/src/pglite-branch.ts b/packages/db/src/pglite-branch.ts index 39316ff81..db4f1ba01 100644 --- a/packages/db/src/pglite-branch.ts +++ b/packages/db/src/pglite-branch.ts @@ -8,7 +8,8 @@ import { basename, dirname, join } from 'node:path'; import { NotImplementedError, systemClock } from '@ultimat3/core'; import type { BranchInfo } from './branch'; import { assertBranchName } from './branch'; -import { branchExists, dbUnavailable } from './errors'; +import { branchExists } from './branch-errors'; +import { dbUnavailable } from './errors'; import { PGLITE_MEMORY, pgliteDataDir } from './pglite'; export interface PgliteBranchOptions { diff --git a/packages/db/src/snapshot-parse.ts b/packages/db/src/snapshot-parse.ts index 20b7ad07f..1482a9886 100644 --- a/packages/db/src/snapshot-parse.ts +++ b/packages/db/src/snapshot-parse.ts @@ -3,6 +3,7 @@ // this package's answer and the reader on disk is `@ultimat3/cli`'s — two owners of that question // is a sidecar one of them accepts and the other cannot use. +import { PARTITION_INTERVALS, type PartitionInterval } from './entity-shape'; import type { CheckDescription, ColumnDescription, @@ -25,6 +26,9 @@ const nullableStr = (value: unknown): value is string | null => value === null | const strings = (value: unknown): value is readonly string[] => Array.isArray(value) && value.every(str); +const INTERVALS: readonly unknown[] = PARTITION_INTERVALS; +const interval = (value: unknown): value is PartitionInterval => INTERVALS.includes(value); + /** `null` is Postgres' own default; anything else must be one of the two directions. */ const order = (value: unknown): value is 'asc' | 'desc' | null => value === null || value === 'asc' || value === 'desc'; @@ -138,6 +142,12 @@ function tableOf(value: unknown): TableDescription | undefined { if (!(rowSecurity === undefined || bool(rowSecurity))) return undefined; const retain = value['retain']; if (!(retain === undefined || bool(retain))) return undefined; + // `partitioned` by the same rule; its interval absent, or one of the set — the record + // `partitionPlan` refuses a removal and a change of width against. + const partitioned = value['partitioned']; + if (!(partitioned === undefined || bool(partitioned))) return undefined; + const width = value['partitionInterval']; + if (!(width === undefined || interval(width))) return undefined; // Absent is none recorded; anything present must be a list of `{ name, install }`. const rawTriggers = value['triggers']; const triggers = rawTriggers === undefined ? undefined : all(rawTriggers, trigger); @@ -148,6 +158,8 @@ function tableOf(value: unknown): TableDescription | undefined { ...(identity === true ? { replicaIdentityFull: true as const } : {}), ...(appendOnly === true ? { appendOnly: true as const } : {}), ...(rowSecurity === true ? { rowSecurity: true as const } : {}), + ...(partitioned === true ? { partitioned: true as const } : {}), + ...(width === undefined ? {} : { partitionInterval: width }), }; const raw = value['checks']; if (raw === undefined) { diff --git a/packages/entity/CLAUDE.md b/packages/entity/CLAUDE.md index c19a14b82..d918c41a4 100644 --- a/packages/entity/CLAUDE.md +++ b/packages/entity/CLAUDE.md @@ -228,12 +228,13 @@ Columns + invariants; the row type is derived from the columns. Tier 2. | `history: true` | `history.ts`, `history-actor.ts` | `
_history` is a REAL registered entity (`$historyOf`), tenant column copied without its key. Written by `@ultimat3/db`'s trigger; `stampHistoryActor` sets `x.actor`/`x.request` once per transaction, never on a pinned repository — no transaction records `null`. `bigint`/`decimal`/`timestamp` columns ride `text`/`instants`: the image is the memory driver's strings (`history-values.test.ts`) | | `countedIn` on a key | `counter.ts` | the target is a NOT NULL `integer()` of the entity the key points at (Postgres' `NULL + 1` stays NULL), by its single primary key (`X_ENTITY_COUNTS_INVALID`); resolved at describe time, memoised on `registryGeneration()` | | all three, described | `lifecycle.ts` | each `$describe()` key ABSENT unless declared, so no other entity's hash moves | -| the memory driver | `memory-lifecycle.ts` | mirrors both triggers through the table's raw store, so a unit test answers what Postgres answers | +| the memory driver | `memory-lifecycle.ts` | mirrors both triggers through the table's raw store | +| `partitionBy` | `partition.ts` | Postgres' rules, at declaration (`X_ENTITY_PARTITION_INVALID`); SQL in db | - **Known gap: the row observer does not see trigger writes.** A history row and a counter move are the trigger's (memory: `memoryLifecycle`'s raw store), never a repository write, so `entity.row` reports neither; a live counter on PGlite or memory moves on the parent's next - write. The WAL replicator does report them. As of 2026-11. + write. The WAL replicator reports them. As of 2026-11. - **`read-capture.ts` is failure capture's read half**: `observedRepo`'s `findById`/`findMany` hand rows to core's `recordFailureFact` — sealed properties dropped first; one check when off. @@ -274,7 +275,7 @@ Columns + invariants; the row type is derived from the columns. Tier 2. ## Files -Each file's header states its one job; `ls packages/entity/src` is the map, kept by no hand. +Each file's header states its job; `ls packages/entity/src` is the map. ## Commands diff --git a/packages/entity/README.md b/packages/entity/README.md index 4fd7ccfd0..7a42344d2 100644 --- a/packages/entity/README.md +++ b/packages/entity/README.md @@ -585,6 +585,51 @@ await db.posts.where({ orgId }).search(term).orderBy('createdAt', 'desc').limit( | `memoryDriver()` **refuses** it — `X_SEARCH_IN_MEMORY` | stemming, stop words and a phrase parser are not a JS token comparison, and an answer memory could give is one Postgres would contradict. Assert a search in a `.live.test.ts` | | relevance is **not** an order the chain serves | `ts_rank` is a computed value and a cursor carries columns; the order is the one you declared, and it pages | +## Search index (the `searchIndex` port) + +`.searchable()` columns are searched through one port, `SearchIndex` (`index` / `remove` / `query`, +every call tenant-scoped by `{ entity, tenant }`). + +| Index | What it is | +|---|---| +| `postgresSearchIndex()` | the default: the generated `tsvector`; `index`/`remove` send nothing (the server keeps the vector), `query` is the repository's own `matches` read — tenancy, actor check and soft delete included. Hits in primary-key order (no rank) | +| `memorySearchIndex()` | the reference to copy and the unit-test index: words, case-folded, every word required; no stemming | +| Meilisearch / Typesense | the app's: implement `SearchIndex`, prove it with `behavesLike(searchIndexConformance, …)` from `@ultimat3/testing/conformance`, return it as `searchIndex` from `runtime.ts` | + +```ts +import { database, entity, searchIndex, text, timestamp, uuid } from '@ultimat3/entity'; + +const posts = entity('posts', { + columns: { + id: uuid().primaryKey(), + orgId: uuid().tenant(), + title: text().searchable('A'), + updatedAt: timestamp().defaultNow().onUpdateNow(), // the document's version + }, +}); +const db = database({ posts }); + +export const find = async (orgId: string, q: string) => { + const hits = await searchIndex().query({ entity: 'posts', tenant: orgId, term: q, limit: 20 }); + const page = await db.posts.where({ orgId }).search(q).page(); // the same port, one path + return { hits, page }; +}; +``` + +| Rule | Detail | +|---|---| +| `.search(term)` asks the installed index | built-in: the `matches` predicate in the statement (today's SQL). Any other: ids first (at most `MAX_PAGE_SIZE`), then the chain reads those rows with its own tenancy, soft delete, order and cursor | +| a document has a **version** | the row's `onUpdateNow()` instant, epoch ms. A searchable entity indexed by anything declares one — `updatedAt: timestamp().defaultNow().onUpdateNow()` — or `searchDocumentOf` and `reindex()` refuse (`X_INVARIANT_VIOLATED`) | +| keep only newer | an index applies a document only when its version is newer than what it holds for that id: a document, or the tombstone `remove(scope, ids, deletedAt)` leaves. Equal = replay, a no-op. Millisecond resolution: two writes in one ms are one version | +| the write path | while an external index is installed (`runtime.ts`'s `searchIndex`, `installSearchIndex`), every COMMITTED `insert`/`insertAll`/`upsertAll`/`update`/transition of a searchable entity indexes the row, and a `delete` (soft or hard) removes it at the delete's instant. Fed by `entity.row`, emitted after commit: a rollback indexes nothing | +| a failed mirror never fails the write | logged as `entity.search-index.write-failed` (`entity`, `op`, `code`, `cause`, `fix`); `reindex()` (`@ultimat3/jobs`) is the repair | +| not mirrored, logged the same way | `updateWhere`/`deleteWhere` (a count, no rows), and a delete on an entity whose key is not `id` (no row read). `reindex()` re-indexes updated rows; it never removes a deleted one | +| the built-in index costs nothing | `postgresSearchIndex()` subscribes to nothing — not even `entity.row`'s `before` read | + +`searchDocumentOf(entity, row)` is the one document builder (only `.searchable()` columns leave). +`searchVersionColumn` and `searchVersionOf` are on the `.` barrel; `installSearchIndex` is on +`@ultimat3/entity/adapter`. + ## A state machine over a column `.transitions()` on an `enumerated()` column. The states are yours; the machine is the framework's. @@ -769,6 +814,60 @@ Three declarations, each a trigger or a purge target `x db gen` and `purge()` co Memory parity: the memory driver records history and moves counters in its repository (`memory-lifecycle.ts`); `history.test.ts` and `counter.test.ts` are the parity tables. +## Partitioned tables — `partitionBy` + +A range by one `timestamp()`/`date()` column, one partition per `interval`, bounds in UTC. `As of 2026-11-09`. + +```ts +import { entity, text, timestamp, uuid } from '@ultimat3/entity'; + +export const events = entity('events', { + columns: { id: uuid(), at: timestamp().defaultNow(), orgId: uuid().tenant(), kind: text() }, + primaryKey: ['id', 'at'], // must hold the partition column + partitionBy: { column: 'at', interval: 'month', ahead: 3, retain: { after: '400d' } }, +}); +``` + +| Question | Answer | +|---|---| +| Refused at declaration (`X_ENTITY_PARTITION_INVALID`) | key or a unique index/invariant without the column (Postgres' rule), a column that is no `timestamp()`/`date()`, an interval outside `day`·`week`·`month`·`year`, `ahead` not 0–1000, a bad `retain.after`, row `retain` beside it, a `view`, a table name over 53 bytes | +| `x db gen` | `create table … partition by range ("at")`, then `select ultimate_partitions(…)` — the current partition and `ahead` more, judged at APPLY time, so a migration replayed a year later still has this month's | +| Partition names | `
_pYYYYMMDD`, the lower bound. `week` is ISO (Monday) | +| Maintenance | `partitions()` from `@ultimat3/jobs`: a job plus a daily UTC `.schedule` task. Per table `maintainPartitions` (`@ultimat3/db`) creates `ahead` under a 5 s `lock_timeout` (a blocked step is `X_DB_LOCK_TIMEOUT`, retried — never a queue of inserts behind it), then retires each partition `retain.after` passed: marked, `detach … concurrently`, dropped; a run that died midway is resumed. Only `_pYYYYMMDD` children are touched | +| A row past every partition | refused by Postgres (`23514`). No default partition, by decision: it forbids `detach … concurrently`, and creating a month it caught rows for scans and moves them under ACCESS EXCLUSIVE. Size `ahead` so `ahead × interval` outlasts the longest scheduler outage you would tolerate (default 3); a failing `partitions()` run dead-letters loudly long before | +| Repository, tenancy, cursors | unchanged — partitions are storage. The memory driver has none to simulate | +| Turning it on for an existing plain table | refused at migrate time by the function (`X_MIGRATION_PARTITION_CHANGE`): Postgres cannot partition in place. Taking it off a partitioned table is refused by `x db gen` (same code) | +| Changing `interval` later | refused by `x db gen` (`X_MIGRATION_PARTITION_CHANGE`): the existing partitions would overlap the new width | +| Moving rows in (a new entity replacing a plain or re-widened one) | enqueue `partitions()` once with `{ from: '' }` — it creates every partition back to that row's and sweeps nothing — then `backfill()` the copy. Rows older than `retain.after` are dropped by the next scheduled run | +| A reference INTO it | a composite key by hand; a same-tenant key needs `crossTenant: ''` — `wiki/Migrations-And-Backfills.md#partitioned-tables` | + +## Personal data (GDPR) + +`.personalData(kind)` marks a column as holding personal data. `kind` is one of `identifier` (ties +rows to a person: a handle, a national id), `contact` (reaches one: email, phone, address) or +`free-text` (may hold either: a bio, a message). + +```ts +import { entity, text, uuid } from '@ultimat3/entity'; + +export const members = entity('members', { + columns: { + id: uuid().primaryKey(), + email: text().unique().personalData('contact'), + bio: text({ max: 400 }).nullable().personalData('free-text'), + }, +}); +``` + +- Changes no DDL and no `$describe()` hash: marking an existing column is not a migration. +- Every chain link after the marker keeps it. +- `personalDataOf(entity)` lists the marked columns (`property`, physical `column`, `kind`, + `columnKind`, `nullable`); `personalDataEntities()` walks the registry, each entity with its + `tenantColumn`. `@ultimat3/jobs`' `accountExport`/`accountErase` read these. +- Not wired into log redaction: `redactKeys()` (core) is by key name, and a `free-text` `name` column + would redact every `name` log field. An app wanting it calls + `redactKeys(personalDataEntities().flatMap((e) => e.columns.map((c) => c.property)))`. + ## Writing many rows Each call is all or nothing, `As of 2026-11-02`. diff --git a/packages/entity/src/adapter.ts b/packages/entity/src/adapter.ts index 710b71afc..8103947d0 100644 --- a/packages/entity/src/adapter.ts +++ b/packages/entity/src/adapter.ts @@ -40,6 +40,17 @@ export { updatePlan, } from './plan'; export { rowKey } from './row-key'; +// The searchIndex port: what an app's Meilisearch/Typesense index implements, and its installer. +export type { + SearchDocument, + SearchHit, + SearchIndex, + SearchQuery, + SearchScope, +} from './search-index'; +export { searchLimitOf } from './search-index'; +export { installSearchIndex } from './search-index-installed'; +export { searchWords } from './search-index-memory'; export type { Predicate, QueryPlan, SortKey } from './tenancy'; export { assertRowTenant, scopedPlan } from './tenancy'; export type { UniqueConstraint } from './unique-constraints'; diff --git a/packages/entity/src/column.ts b/packages/entity/src/column.ts index f84df978c..725fd71ab 100644 --- a/packages/entity/src/column.ts +++ b/packages/entity/src/column.ts @@ -7,6 +7,7 @@ // physical name even though two schema modules import each other in a cycle. import { invariantViolated } from './errors'; +import { isPersonalDataKind, PERSONAL_DATA_KINDS } from './personal-data-kind'; import { refuseColumn } from './refuse'; import { DEFAULT_SEARCH_WEIGHT, isSearchWeight } from './search'; import type { @@ -216,6 +217,17 @@ export const makeColumn = ( return makeColumn({ ...meta, searchable: weight }, parse, optional); }, + personalData: (kind) => { + if (!isPersonalDataKind(kind)) { + refuseColumn( + 'personal-data', + `"${String(kind)}" is not a personal data kind`, + `text().personalData('contact') — one of: ${PERSONAL_DATA_KINDS.join(', ')}`, + ); + } + return makeColumn({ ...meta, personalData: kind }, parse, optional); + }, + tenant: () => makeColumn({ ...meta, tenant: true, index: true }, parse, optional), references: (target, options = {}) => { diff --git a/packages/entity/src/describe.ts b/packages/entity/src/describe.ts index 8bc415244..268039498 100644 --- a/packages/entity/src/describe.ts +++ b/packages/entity/src/describe.ts @@ -10,6 +10,7 @@ import { currencyCheck, scaleCheck } from './columns'; import { EntityError, invariantViolated } from './errors'; import type { Invariant } from './invariants'; import type { LifecycleDescription } from './lifecycle'; +import type { PartitionDescription } from './partition'; import type { ColumnDescription, EntityDescription, @@ -48,6 +49,8 @@ export interface DescribeInput { readonly view?: ViewQuery | null | undefined; /** `retain`, `history`, `countedIn` counters — each projected only when declared (`lifecycle.ts`). */ readonly lifecycle?: LifecycleDescription | undefined; + /** `entity({ partitionBy })`, checked (`partition.ts`) — projected only when declared. */ + readonly partition?: PartitionDescription | null | undefined; } /** @@ -385,5 +388,7 @@ export const describeEntity = (input: DescribeInput): EntityDescriptio }, }), ...input.lifecycle, + // Absent unless declared: no entity that never said `partitionBy` moves the hash. + ...(input.partition == null ? {} : { partition: input.partition }), }; }; diff --git a/packages/entity/src/entity-error.ts b/packages/entity/src/entity-error.ts index 8f6dd6f44..9fa082af2 100644 --- a/packages/entity/src/entity-error.ts +++ b/packages/entity/src/entity-error.ts @@ -52,6 +52,7 @@ export const ENTITY_ERROR_CODES = [ 'X_DRIVER_UNSUPPORTED', 'X_DRIVER_CAPABILITY_MISSING', 'X_DRIVER_ROW_OUT_OF_SCOPE', + 'X_ENTITY_PARTITION_INVALID', ] as const; export type EntityErrorCode = (typeof ENTITY_ERROR_CODES)[number]; @@ -100,6 +101,8 @@ export const ENTITY_ERROR_TITLES: Readonly> = { X_DRIVER_CAPABILITY_MISSING: 'a custom driver declares a capability its raw repo does not implement', X_DRIVER_ROW_OUT_OF_SCOPE: 'a custom driver returned a row outside the plan it was given', + X_ENTITY_PARTITION_INVALID: + 'entity({ partitionBy }) declares a partitioning Postgres cannot build', }; // Registered at module load, unconditionally, in one call. Without this the registry humanises the diff --git a/packages/entity/src/entity.ts b/packages/entity/src/entity.ts index 22b5a4cec..648d4140d 100644 --- a/packages/entity/src/entity.ts +++ b/packages/entity/src/entity.ts @@ -27,6 +27,7 @@ import { indexName } from './index-name'; import type { Invariant, InvariantDef } from './invariants'; import { assertInvariants, bindInvariant } from './invariants'; import { describeLifecycle, lifecycleHistory } from './lifecycle'; +import { type PartitionInit, partitionOf } from './partition'; import { assertReadOnly } from './read-only'; import { recordProjection } from './record-projection'; import type { EntityDescription, ReferenceDescription } from './registry'; @@ -127,6 +128,12 @@ export interface EntityInit { readonly retain?: RetainInit; /** Every row change recorded into `
_history` by trigger (`history.ts`, `$history`). */ readonly history?: HistoryInit; + /** + * Range-partitioned by a `timestamp()`/`date()` in the primary key, one partition per `interval` + * (`partition.ts`). `x db gen` writes `partition by range` and the first partitions; a scheduled + * maintenance run creates `ahead` more and drops what `retain` expired. Never beside `retain`. + */ + readonly partitionBy?: PartitionInit; /** Set by `history.ts` on the history entity it declares: the parent's name. Never by hand. */ readonly historyOf?: string; } @@ -354,6 +361,15 @@ export const entity = ( ); const retain = retainOf(name, init.retain, entries, indexes, appendOnly); + const partition = partitionOf(name, init.partitionBy, { + table, + columns: entries, + primaryKey, + indexes, + uniques: invariants.filter((inv) => inv.kind === 'unique').map((inv) => inv.columns), + retain: init.retain !== undefined, + view: init.view != null, + }); const sealed = sealedFields({ $name: name, $table: table, $columns: init.columns }); assertSealedDeclaration({ name, @@ -393,6 +409,7 @@ export const entity = ( readOnly: init.readOnly, view: viewQuery, lifecycle: describeLifecycle(core as EntityCore, retain, init.historyOf), + partition, }); assertNotRetired(name, entries, init.retired); const references = (): readonly ReferenceDescription[] => describeReferences(name, entries); diff --git a/packages/entity/src/index.ts b/packages/entity/src/index.ts index 98cc9d635..63a1058a4 100644 --- a/packages/entity/src/index.ts +++ b/packages/entity/src/index.ts @@ -101,7 +101,22 @@ export { memoryRepo, memoryTransactor } from './memory-repo'; export type { StatementLoop } from './n-plus-one'; export { N_PLUS_ONE_THRESHOLD, nPlusOne, preloadsFor } from './n-plus-one'; export { isOperator, OPERATORS } from './operator'; +// `entity({ partitionBy })`, checked at declaration (`partition.ts`); the SQL is `@ultimat3/db`'s. +export type { + PartitionContext, + PartitionDescription, + PartitionInit, +} from './partition'; +export { DEFAULT_PARTITIONS_AHEAD, partitionInvalid, partitionOf } from './partition'; export { persistedRecordTypes } from './persisted-types'; +// GDPR: the personal-data column marker and its readers (`personal-data.ts`). +export type { PersonalDataColumn, PersonalDataEntity, PersonalDataKind } from './personal-data'; +export { + PERSONAL_DATA_KINDS, + personalDataEntities, + personalDataKindOf, + personalDataOf, +} from './personal-data'; // For a change feed holding a table name and raw columns: `entityForTable` + `decodeRow` is the row // the app declared, money and all. Exported for `@ultimat3/realtime` (plan 101, 06 i). export type { PgArray } from './pg-array-literal'; @@ -182,6 +197,20 @@ export { SEARCH_LANGUAGES, SEARCH_WEIGHTS, } from './search'; +// The searchIndex port: types, the document builder, the installed index, the two shipped ones. +export type { + SearchDocument, + SearchHit, + SearchIndex, + SearchQuery, + SearchScope, +} from './search-index'; +export { DEFAULT_SEARCH_LIMIT, searchDocumentOf } from './search-index'; +export { searchIndex } from './search-index-installed'; +export { memorySearchIndex } from './search-index-memory'; +export type { PostgresSearchIndexOptions } from './search-index-postgres'; +export { postgresSearchIndex } from './search-index-postgres'; +export { searchVersionColumn, searchVersionOf } from './search-index-version'; export type { Seed, SeedContext, diff --git a/packages/entity/src/partition-describe.test.ts b/packages/entity/src/partition-describe.test.ts new file mode 100644 index 000000000..dc36c38df --- /dev/null +++ b/packages/entity/src/partition-describe.test.ts @@ -0,0 +1,44 @@ +// `$describe()`'s `partition` key: present exactly when `partitionBy` was declared, so no entity +// that never said it moves `x verify`'s source-drift hash, and `@ultimat3/db` reads it as it is. + +import { describe, expect, test } from 'bun:test'; +import { timestamp, uuid } from './columns'; +import { describeEntity } from './describe'; +import { partitionOf } from './partition'; + +const COLUMNS = Object.entries({ id: uuid().primaryKey(), at: timestamp() }); +const base = { + name: 'pd_events', + table: 'pd_events', + columns: COLUMNS, + primaryKey: ['id', 'at'], + invariants: [], + indexes: [], + tags: ['entity:pd_events'], + cacheTag: 'entity:pd_events', + softDelete: false, + tenantColumn: null, +}; + +describe('describeEntity · partition', () => { + test('absent when not declared', () => { + expect(describeEntity(base)).not.toHaveProperty('partition'); + expect(describeEntity({ ...base, partition: null })).not.toHaveProperty('partition'); + }); + + test('the checked declaration, physical, when declared', () => { + const partition = partitionOf( + 'pd_events', + { column: 'at', interval: 'month', retain: { after: '400d' } }, + { ...base, retain: false, view: false, uniques: [] }, + ); + expect(describeEntity({ ...base, partition })).toMatchObject({ + partition: { + column: 'at', + interval: 'month', + ahead: 3, + retain: { after: '400d', afterMs: 400 * 86_400_000 }, + }, + }); + }); +}); diff --git a/packages/entity/src/partition-entity.test.ts b/packages/entity/src/partition-entity.test.ts new file mode 100644 index 000000000..d53398b88 --- /dev/null +++ b/packages/entity/src/partition-entity.test.ts @@ -0,0 +1,52 @@ +// `entity({ partitionBy })` end to end through the declaration: refused at `entity()` when the key +// lacks the partition column, and otherwise described so `x db gen` writes a partitioned table. +// Applied with the `entity.ts` / `registry.ts` wiring (FENT-A-OUTSIDE.patch). + +import { afterAll, describe, expect, test } from 'bun:test'; +import { isUltimateError } from '@ultimat3/core'; +import { generateMigration } from '@ultimat3/db'; +import { text, timestamp, uuid } from './columns'; +import { entity } from './entity'; +import { clearRegistry } from './registry'; + +afterAll(() => { + clearRegistry(); +}); + +describe('entity({ partitionBy })', () => { + test('an id-only key is refused at declaration, the fix naming the composite key', () => { + let refused: unknown; + try { + entity('pe_bad', { + columns: { id: uuid().primaryKey(), at: timestamp(), kind: text() }, + partitionBy: { column: 'at', interval: 'month' }, + }); + } catch (error) { + refused = isUltimateError(error) ? [error.code, error.fix] : String(error); + } + expect(refused).toEqual([ + 'X_ENTITY_PARTITION_INVALID', + expect.stringContaining("primaryKey: ['id', 'at']"), + ]); + }); + + test('a composite key is described and generates a partitioned table', () => { + const events = entity('pe_events', { + columns: { id: uuid(), at: timestamp(), kind: text() }, + primaryKey: ['id', 'at'], + partitionBy: { column: 'at', interval: 'month', ahead: 2, retain: { after: '400d' } }, + }); + const described = events.$describe(); + expect(described.partition).toEqual({ + column: 'at', + interval: 'month', + ahead: 2, + retain: { after: '400d', afterMs: 400 * 86_400_000 }, + }); + const up = generateMigration({ entities: [described], name: 'events' }).up; + expect(up).toContain(') partition by range ("at");'); + expect(up).toContain( + `select "ultimate_partitions"('pe_events', 'timestamptz', 'month', 2, null);`, + ); + }); +}); diff --git a/packages/entity/src/partition.test.ts b/packages/entity/src/partition.test.ts new file mode 100644 index 000000000..db30a670b --- /dev/null +++ b/packages/entity/src/partition.test.ts @@ -0,0 +1,99 @@ +// `entity({ partitionBy })` checked at declaration: Postgres' rules — the key and every unique +// index hold the partition column, the column is a timestamp or a date — refused with +// `X_ENTITY_PARTITION_INVALID` and the edit, before any migration exists. + +import { describe, expect, test } from 'bun:test'; +import { isUltimateError } from '@ultimat3/core'; +import { text, timestamp, uuid } from './columns'; +import { date } from './columns-data'; +import { type PartitionContext, type PartitionInit, partitionOf } from './partition'; + +const COLUMNS = Object.entries({ + id: uuid().primaryKey(), + at: timestamp(), + day: date(), + kind: text(), +}); + +const context = (over: Partial = {}): PartitionContext => ({ + table: 'events', + columns: COLUMNS, + primaryKey: ['id', 'at'], + indexes: [], + uniques: [], + retain: false, + view: false, + ...over, +}); + +const MONTHLY: PartitionInit = { column: 'at', interval: 'month' }; + +const refusal = (init: PartitionInit, over: Partial = {}) => { + try { + partitionOf('Event', init, context(over)); + } catch (error) { + if (isUltimateError(error)) return { code: error.code, cause: error.cause, fix: error.fix }; + return expect.unreachable(`refused without a code: ${String(error)}`); + } + return expect.unreachable('the declaration was accepted'); +}; + +describe('partitionOf', () => { + test('absent is no partitioning', () => { + expect(partitionOf('Event', undefined, context())).toBeNull(); + }); + + test('a monthly range by a timestamp in the key: physical column, three ahead by default', () => { + expect(partitionOf('Event', MONTHLY, context())).toEqual({ + column: 'at', + interval: 'month', + ahead: 3, + }); + }); + + test('a date column, a retention window and an explicit ahead', () => { + const init: PartitionInit = { + column: 'day', + interval: 'day', + ahead: 7, + retain: { after: '90d' }, + }; + expect(partitionOf('Event', init, context({ primaryKey: ['id', 'day'] }))).toEqual({ + column: 'day', + interval: 'day', + ahead: 7, + retain: { after: '90d', afterMs: 90 * 86_400_000 }, + }); + }); + + test('a key without the partition column is refused, the fix naming the composite key', () => { + const refused = refusal(MONTHLY, { primaryKey: ['id'] }); + expect(refused.code).toBe('X_ENTITY_PARTITION_INVALID'); + expect(refused.cause).toContain('does not hold "at"'); + expect(refused.fix).toContain("primaryKey: ['id', 'at']"); + }); + + test('a unique index or unique invariant without the partition column is refused', () => { + const index = { name: 'events_kind_key', columns: ['kind'], unique: true }; + expect(refusal(MONTHLY, { indexes: [index] }).cause).toContain('unique index on (kind)'); + expect(refusal(MONTHLY, { uniques: [['kind']] }).code).toBe('X_ENTITY_PARTITION_INVALID'); + expect(partitionOf('Event', MONTHLY, context({ uniques: [['kind', 'at']] }))).not.toBeNull(); + }); + + test.each([ + ['a text column', { column: 'kind', interval: 'month' }, {}, 'no timestamp() or date()'], + ['an unknown column', { column: 'nope', interval: 'month' }, {}, 'no timestamp() or date()'], + ['an unknown interval', { column: 'at', interval: 'hour' }, {}, 'interval "hour"'], + ['a negative ahead', { column: 'at', interval: 'month', ahead: -1 }, {}, 'ahead'], + ['a fractional ahead', { column: 'at', interval: 'month', ahead: 1.5 }, {}, 'ahead'], + ['a zero window', { ...MONTHLY, retain: { after: '0d' } }, {}, 'retain.after'], + ['a bad window', { ...MONTHLY, retain: { after: 'soon' } }, {}, 'retain.after'], + ['row retain beside it', MONTHLY, { retain: true }, 'never rows'], + ['a view', MONTHLY, { view: true }, 'is a view'], + ['a long table name', MONTHLY, { table: 'e'.repeat(54) }, 'over 53 bytes'], + ] as const)('%s is refused', (_, init, over, cause) => { + const refused = refusal(init as PartitionInit, over); + expect(refused.code).toBe('X_ENTITY_PARTITION_INVALID'); + expect(refused.cause).toContain(cause); + }); +}); diff --git a/packages/entity/src/partition.ts b/packages/entity/src/partition.ts new file mode 100644 index 000000000..f8a88a63d --- /dev/null +++ b/packages/entity/src/partition.ts @@ -0,0 +1,140 @@ +// Single responsibility: `entity({ partitionBy })` checked where it is written. Postgres' rules +// become declaration errors here — the key and every unique index hold the partition column, the +// column is a timestamp or a date — so `x db gen` never writes a migration the server refuses. +// The SQL (the `partition by` clause, creating ahead, retention) is `@ultimat3/db`'s. + +import { renderFixShellArg } from '@ultimat3/core'; +import { PARTITION_INTERVALS, type PartitionInterval } from '@ultimat3/db'; +import { parseDuration } from '@ultimat3/time'; +import { columnName } from './column'; +import { EntityError } from './errors'; +import type { AnyColumn, IndexDef } from './types'; + +const INTERVALS: readonly string[] = PARTITION_INTERVALS; +const KINDS: readonly string[] = ['timestamptz', 'date']; + +/** Partitions kept ready past the current one, when `ahead` is not said. */ +export const DEFAULT_PARTITIONS_AHEAD = 3; +/** Postgres' NAMEDATALEN - 1, less the `_pYYYYMMDD` every partition name adds. */ +const MAX_PARENT_BYTES = 63 - 10; + +/** What `partitionBy` declares. A range by `column`, one partition per `interval`, UTC bounds. */ +export interface PartitionInit { + readonly column: K; + readonly interval: PartitionInterval; + /** Partitions created past the current one, by the migration and by every maintenance run. */ + readonly ahead?: number; + /** Whole partitions whose upper bound is older than `after` are detached, then dropped. */ + readonly retain?: { readonly after: string }; +} + +/** The checked declaration, physical column. Structurally `@ultimat3/db`'s `PartitionDescriptionLike`. */ +export interface PartitionDescription { + readonly column: string; + readonly interval: PartitionInterval; + readonly ahead: number; + readonly retain?: { readonly after: string; readonly afterMs: number } | undefined; +} + +/** The rest of the declaration a partitioning has to agree with. */ +export interface PartitionContext { + readonly table: string; + readonly columns: readonly (readonly [string, AnyColumn])[]; + /** Properties, as `primaryKey` names them. */ + readonly primaryKey: readonly string[]; + readonly indexes: readonly IndexDef[]; + /** Unique invariants' physical columns. */ + readonly uniques: readonly (readonly string[])[]; + readonly retain: boolean; + readonly view: boolean; +} + +export const partitionInvalid = (entity: string, reason: string, edit: string): EntityError => + new EntityError({ + code: 'X_ENTITY_PARTITION_INVALID', + cause: `entity "${entity}" declares partitionBy, and ${reason}`, + fix: `x db gen "partition" # after ${edit} in entity('${renderFixShellArg(entity, '')}')`, + }); + +const retainOf = (entity: string, after: unknown): PartitionDescription['retain'] => { + let afterMs = Number.NaN; + try { + afterMs = typeof after === 'string' ? parseDuration(after) : Number.NaN; + } catch { + // Reported below with the edit, the one place a bad window is described. + } + if (!(afterMs > 0)) { + throw partitionInvalid( + entity, + 'partitionBy.retain.after is no window longer than zero', + "partitionBy.retain: { after: '400d' }", + ); + } + return { after: String(after), afterMs }; +}; + +/** `null` when `partitionBy` is absent. Every refusal raised here is at declaration. */ +export function partitionOf( + entity: string, + init: PartitionInit | undefined, + context: PartitionContext, +): PartitionDescription | null { + if (init === undefined) return null; + const refuse = (reason: string, edit: string): never => { + throw partitionInvalid(entity, reason, edit); + }; + if (context.view) refuse('is a view: a view has no partitions', 'removing partitionBy'); + if (context.retain) { + refuse( + 'retain sits beside it: a partitioned table drops whole partitions, never rows', + 'moving the window to partitionBy.retain: { after }', + ); + } + if (!INTERVALS.includes(init.interval)) { + refuse( + `interval "${String(init.interval)}" is none of ${INTERVALS.join(', ')}`, + "interval: 'month'", + ); + } + const ahead = init.ahead ?? DEFAULT_PARTITIONS_AHEAD; + if (!Number.isInteger(ahead) || ahead < 0 || ahead > 1000) { + refuse('partitionBy.ahead is no whole number from 0 to 1000', 'ahead: 3'); + } + if (new TextEncoder().encode(context.table).length > MAX_PARENT_BYTES) { + refuse( + `its table name is over ${MAX_PARENT_BYTES} bytes, and each partition adds 10 (_pYYYYMMDD)`, + "table: ''", + ); + } + const column = context.columns.find(([property]) => property === init.column)?.[1]; + if (column === undefined || !KINDS.includes(column.$meta.kind)) { + throw partitionInvalid( + entity, + `partitionBy.column "${String(init.column)}" is no timestamp() or date() column of it`, + "column: 'at'", + ); + } + if (!context.primaryKey.includes(init.column)) { + refuse( + `its primary key (${context.primaryKey.join(', ')}) does not hold "${init.column}": Postgres requires it`, + `primaryKey: [${[...context.primaryKey, init.column].map((key) => `'${key}'`).join(', ')}]`, + ); + } + const physical = columnName(init.column, column.$meta); + const unique = [ + ...context.indexes.filter((index) => index.unique).map((index) => index.columns), + ...context.uniques, + ].find((columns) => !columns.includes(physical)); + if (unique !== undefined) { + refuse( + `a unique index on (${unique.join(', ')}) does not hold "${physical}": Postgres requires it`, + `adding ${init.column} to that unique index or invariant`, + ); + } + return { + column: physical, + interval: init.interval, + ahead, + ...(init.retain === undefined ? {} : { retain: retainOf(entity, init.retain.after) }), + }; +} diff --git a/packages/entity/src/personal-data-kind.ts b/packages/entity/src/personal-data-kind.ts new file mode 100644 index 000000000..8cac18977 --- /dev/null +++ b/packages/entity/src/personal-data-kind.ts @@ -0,0 +1,15 @@ +// The closed set of personal-data kinds `.personalData(kind)` takes, and its membership test. +// Imports nothing, so `column.ts` (the chain) and `types.ts` (the meta) can both read it without +// a cycle through the readers in `personal-data.ts` — the shape `search.ts` keeps for `.searchable()`. + +/** + * What a marked value IS, so an export and an erase can treat each kind by its own rule: an + * identifier (a handle, a national id) ties rows to a person; a contact (email, phone, address) + * reaches one; free text (a bio, a message) may hold either and is read only by a human. + */ +export const PERSONAL_DATA_KINDS = ['identifier', 'contact', 'free-text'] as const; + +export type PersonalDataKind = (typeof PERSONAL_DATA_KINDS)[number]; + +export const isPersonalDataKind = (value: unknown): value is PersonalDataKind => + typeof value === 'string' && (PERSONAL_DATA_KINDS as readonly string[]).includes(value); diff --git a/packages/entity/src/personal-data.test.ts b/packages/entity/src/personal-data.test.ts new file mode 100644 index 000000000..b1a39f1d2 --- /dev/null +++ b/packages/entity/src/personal-data.test.ts @@ -0,0 +1,92 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { isUltimateError } from '@ultimat3/core'; +import { integer, money, text, uuid } from './columns'; +import { entity } from './entity'; +import { personalDataEntities, personalDataKindOf, personalDataOf } from './personal-data'; +import { PERSONAL_DATA_KINDS, type PersonalDataKind } from './personal-data-kind'; +import { clearRegistry } from './registry'; + +afterEach(() => clearRegistry()); + +const declareUsers = () => + entity('pd_users', { + columns: { + id: uuid().primaryKey(), + orgId: uuid().tenant(), + email: text().unique().personalData('contact'), + // Chained AFTER the marker too: the mark must survive every link, or the order of a chain + // decides whether a column is erased. + nickname: text().personalData('identifier').nullable(), + bio: text({ max: 400 }).personalData('free-text').nullable().column('about'), + balance: money(), + age: integer(), + }, + }); + +describe('.personalData()', () => { + test('marks a column with its kind and changes nothing else about it', () => { + const plain = text().unique(); + const marked = text().unique().personalData('contact'); + expect(personalDataKindOf(marked)).toBe('contact'); + expect(personalDataKindOf(plain)).toBeUndefined(); + expect(marked.$meta).toEqual({ ...plain.$meta, personalData: 'contact' }); + expect(marked.$parse('a@b.c')).toBe('a@b.c'); + }); + + test('every chain link after the marker keeps it', () => { + const users = declareUsers(); + expect(personalDataKindOf(users.nickname)).toBe('identifier'); + expect(users.nickname.$meta.notNull).toBe(false); + expect(personalDataKindOf(users.bio)).toBe('free-text'); + expect(users.bio.$meta.name).toBe('about'); + }); + + test('an unknown kind is refused where it was written', () => { + let code = 'answered'; + try { + text().personalData('secret' as PersonalDataKind); + } catch (error) { + code = isUltimateError(error) ? error.code : 'not coded'; + } + expect(code).toBe('X_INVARIANT_VIOLATED'); + expect(PERSONAL_DATA_KINDS).toEqual(['identifier', 'contact', 'free-text']); + }); +}); + +describe('personalDataOf()', () => { + test('lists every marked column of an entity, in declaration order, physical name resolved', () => { + const users = declareUsers(); + expect(personalDataOf(users)).toEqual([ + { property: 'email', column: 'email', kind: 'contact', columnKind: 'text', nullable: false }, + { + property: 'nickname', + column: 'nickname', + kind: 'identifier', + columnKind: 'text', + nullable: true, + }, + { property: 'bio', column: 'about', kind: 'free-text', columnKind: 'text', nullable: true }, + ]); + }); + + test('an entity with no marked column answers none', () => { + const plain = entity('pd_plain', { columns: { id: uuid().primaryKey(), title: text() } }); + expect(personalDataOf(plain)).toEqual([]); + }); +}); + +describe('personalDataEntities()', () => { + test('walks the registry: only entities holding a marked column, with their tenant', () => { + declareUsers(); + entity('pd_plain', { columns: { id: uuid().primaryKey(), title: text() } }); + const found = personalDataEntities(); + expect(found.map((each) => [each.entity.$name, each.tenantColumn])).toEqual([ + ['pd_users', 'orgId'], + ]); + expect(found[0]?.columns.map((column) => column.property)).toEqual([ + 'email', + 'nickname', + 'bio', + ]); + }); +}); diff --git a/packages/entity/src/personal-data.ts b/packages/entity/src/personal-data.ts new file mode 100644 index 000000000..c1572ba17 --- /dev/null +++ b/packages/entity/src/personal-data.ts @@ -0,0 +1,66 @@ +// The readers a GDPR export or erase job walks by: every column `.personalData(kind)` marked, per +// entity and across the registry. The mark is `$meta.personalData`, never on `$describe()` — being +// personal data changes no column, so it must not move the drift hash. + +import { columnName } from './column'; +import type { EntityCore } from './entity'; +import type { PersonalDataKind } from './personal-data-kind'; +import { registeredEntities } from './registry'; +import type { AnyColumn, ColumnKind } from './types'; + +// The kind set's home is `personal-data-kind.ts` (the chain reads it); the barrel takes it from here. +export type { PersonalDataKind } from './personal-data-kind'; +export { PERSONAL_DATA_KINDS } from './personal-data-kind'; + +/** The kind a column was marked with, or `undefined` for a column that holds no personal data. */ +export const personalDataKindOf = (column: AnyColumn): PersonalDataKind | undefined => + column.$meta.personalData; + +/** One marked column of one entity, resolved to where it landed. */ +export interface PersonalDataColumn { + /** The row property — what a repository write names. */ + readonly property: string; + /** The physical column (the base name, for a money column's parts). */ + readonly column: string; + readonly kind: PersonalDataKind; + readonly columnKind: ColumnKind; + /** Whether an erase may write `null`; otherwise it must write a replacement value. */ + readonly nullable: boolean; +} + +/** Every marked column of `entity`, in declaration order. */ +export const personalDataOf = ( + entity: Pick, '$columns'>, +): readonly PersonalDataColumn[] => + Object.entries(entity.$columns).flatMap(([property, column]) => { + const kind = column.$meta.personalData; + if (kind === undefined) return []; + return [ + { + property, + column: columnName(property, column.$meta), + kind, + columnKind: column.$meta.kind, + nullable: !column.$meta.notNull, + }, + ]; + }); + +/** A registered entity holding personal data, with the tenant a subject's rows are scoped by. */ +export interface PersonalDataEntity { + readonly entity: EntityCore; + /** The tenant column's property, or `null` for an untenanted entity. */ + readonly tenantColumn: string | null; + readonly columns: readonly PersonalDataColumn[]; +} + +/** Every registered entity carrying at least one marked column, in the registry's name order. */ +export const personalDataEntities = (): readonly PersonalDataEntity[] => + registeredEntities().flatMap((entry) => { + const core = entry.core; + if (core === undefined) return []; + const columns = personalDataOf(core); + return columns.length === 0 + ? [] + : [{ entity: core, tenantColumn: core.$tenantColumn, columns }]; + }); diff --git a/packages/entity/src/query.ts b/packages/entity/src/query.ts index 2031a5155..9b5c20bbf 100644 --- a/packages/entity/src/query.ts +++ b/packages/entity/src/query.ts @@ -18,6 +18,7 @@ import { type ReadAggregates, readAggregates } from './read-aggregates'; import type { Repo, RepoOptions, UpsertArgs } from './repo'; import { pickRow } from './sealed'; import { SEARCH_PROPERTY } from './search'; +import { searchedRepo } from './search-repo'; import { type TableLocks, tableLocks } from './table-locks'; import type { Predicate, QueryPlan, SortDirection, SortKey } from './tenancy'; import { stampedColumns, touch } from './touch'; @@ -74,6 +75,10 @@ export interface ReadBuilder * chain can serve: `ts_rank` is a computed value and the cursor carries columns, so the order * stays the one the caller declared. An entity with no searchable column is `X_SEARCH_UNDECLARED` * and the in-memory driver is `X_SEARCH_IN_MEMORY` — never a different answer from the two. + * + * The match is the INSTALLED `SearchIndex`'s (`searchIndex()`, `runtime.ts`'s `searchIndex`). + * The built-in Postgres one is this predicate in the statement, as above; any other index is + * asked for ids first (at most `MAX_PAGE_SIZE`), which the chain then reads (`search-repo.ts`). */ search(term: string): ReadBuilder; orderBy( @@ -379,10 +384,14 @@ export const tableFor = ( related?: RelatedTables, ): Table => ({ // One cast: `sealed-repo.ts` refuses at runtime, `SealedKeysOf` is the same rule as a type. - ...(builder(entity, repo, EMPTY, (row) => row, related) as ReadBuilder< - Row, - SealedKeysOf - >), + // Reads go through `searchedRepo`: a `.search()` asks the installed `SearchIndex` (`search-repo.ts`). + ...(builder( + entity, + searchedRepo(entity, repo), + EMPTY, + (row) => row, + related, + ) as ReadBuilder>), insert: async (values, options) => repo.insert(entity.$parse(values), options), insertAll: async (rows, options) => repo.insertAll( diff --git a/packages/entity/src/refuse.test.ts b/packages/entity/src/refuse.test.ts index fe6852009..5e1866d94 100644 --- a/packages/entity/src/refuse.test.ts +++ b/packages/entity/src/refuse.test.ts @@ -118,6 +118,8 @@ const SITES: readonly (readonly [string, () => unknown])[] = [ // A JS caller reaching the weight refusal — the union makes it unwritable in TypeScript, and // `.searchable(input.weight)` from parsed JSON is exactly how it arrives anyway. ['searchable weight', () => text().searchable('Z' as 'A')], + // Same arrival path as the weight: a JS caller, or a kind read from configuration. + ['personal data kind', () => text().personalData('secret' as 'contact')], ['transitions table', () => enumerated(['a', 'b']).transitions({ a: ['c'], b: [] } as never)], [ 'transitions nullable', diff --git a/packages/entity/src/registry.ts b/packages/entity/src/registry.ts index 3356f37c6..dd67870a3 100644 --- a/packages/entity/src/registry.ts +++ b/packages/entity/src/registry.ts @@ -12,6 +12,7 @@ import type { import type { EntityCore } from './entity'; import { entityDuplicate } from './entity-error'; import type { InvariantKind } from './invariants'; +import type { PartitionDescription } from './partition'; import type { RecordProjection } from './record-projection'; import type { ColumnDefault, OnDelete } from './types'; @@ -177,6 +178,8 @@ export interface EntityDescription { readonly retain?: RetainDescriptionLike; /** On a history entity: its parent's name. */ readonly historyOf?: string; + /** `entity({ partitionBy })`, checked (`partition.ts`). Absent otherwise. `x db gen` reads it. */ + readonly partition?: PartitionDescription; } export interface RegistryEntry { diff --git a/packages/entity/src/search-index-installed.ts b/packages/entity/src/search-index-installed.ts new file mode 100644 index 000000000..d8fd51ca0 --- /dev/null +++ b/packages/entity/src/search-index-installed.ts @@ -0,0 +1,31 @@ +// The installed `SearchIndex`: `runtime.ts`'s `searchIndex`, else `postgresSearchIndex()` over the +// ambient pool. One slot (`adapterSlot`), read per call, so a test's `installSearchIndex(memory)` +// and the boot's install both release only their own frame. + +import { adapterSlot } from '@ultimat3/core/adapter'; +import type { SearchIndex } from './search-index'; +import { postgresSearchIndex } from './search-index-postgres'; +import { syncSearchIndex } from './search-index-sync'; + +let builtIn: SearchIndex | undefined; + +const slot = /* @__PURE__ */ adapterSlot(() => { + builtIn ??= postgresSearchIndex(); + return builtIn; +}); + +/** + * On `./adapter` only. Answers the release: it drops this install and no other. An external index + * is subscribed to the write path (`search-index-sync.ts`) for as long as it is installed. + */ +export const installSearchIndex = (index: SearchIndex): (() => void) => { + const release = slot.install(index); + const stop = syncSearchIndex(index, () => slot.current()); + return () => { + stop(); + release(); + }; +}; + +/** The index every search through the port asks — what `reindex` writes and a read queries. */ +export const searchIndex = (): SearchIndex => slot.current(); diff --git a/packages/entity/src/search-index-memory.ts b/packages/entity/src/search-index-memory.ts new file mode 100644 index 000000000..3363f541f --- /dev/null +++ b/packages/entity/src/search-index-memory.ts @@ -0,0 +1,103 @@ +// `memorySearchIndex()`: the reference `SearchIndex` an adapter author copies, and the one a unit +// test installs. Words, case-folded, every query word required — NOT Postgres: no stemming, no +// stop words, no `websearch` operators — why `memoryDriver()` still refuses `.search()`. Newest +// version per id wins; tombstones live as long as the index (a reference and a test index only). + +import type { + SearchDocument, + SearchHit, + SearchIndex, + SearchQuery, + SearchScope, +} from './search-index'; +import { searchLimitOf } from './search-index'; + +/** Letters and digits in any script; everything else separates words. */ +const WORD = /[\p{L}\p{N}]+/gu; + +/** The distinct lower-cased words of `text`. */ +export const searchWords = (text: string): readonly string[] => [ + ...new Set(text.toLocaleLowerCase('und').match(WORD) ?? []), +]; + +/** One id's newest state: its words, or `null` — the tombstone a `remove` leaves. */ +interface Held { + readonly id: string; + readonly version: number; + readonly words: ReadonlyMap | null; +} + +const countWords = (document: SearchDocument): ReadonlyMap => { + const counts = new Map(); + for (const text of Object.values(document.fields)) { + for (const word of text.toLocaleLowerCase('und').match(WORD) ?? []) { + counts.set(word, (counts.get(word) ?? 0) + 1); + } + } + return counts; +}; + +/** One key per `(entity, tenant)`: JSON, so no entity name can be spelled into another scope. */ +const scopeKey = (scope: SearchScope): string => JSON.stringify([scope.entity, scope.tenant]); + +export const memorySearchIndex = (): SearchIndex => { + const scopes = new Map>(); + const scopeOf = (scope: SearchScope): Map => { + const key = scopeKey(scope); + const found = scopes.get(key); + if (found !== undefined) return found; + const created = new Map(); + scopes.set(key, created); + return created; + }; + + return { + name: 'memory', + index: async (documents) => { + for (const document of documents) { + const scope = scopeOf(document); + // Keep only newer: a reindex batch that read the row before a write — or before its + // delete — arrives here after it, and must not put the old text back. + if ((scope.get(document.id)?.version ?? Number.NEGATIVE_INFINITY) >= document.version) { + continue; + } + scope.set(document.id, { + id: document.id, + version: document.version, + words: countWords(document), + }); + } + }, + remove: async (scope, ids, version) => { + const held = scopeOf(scope); + for (const id of ids) { + // A remove older than what is held (a re-create after it) changes nothing. + if ((held.get(id)?.version ?? Number.NEGATIVE_INFINITY) > version) continue; + held.set(id, { id, version, words: null }); + } + }, + query: async (query: SearchQuery): Promise => { + const limit = searchLimitOf(query); + const words = searchWords(query.term); + if (words.length === 0) return []; + const held = scopes.get(scopeKey(query)) ?? new Map(); + const scored: { readonly id: string; readonly score: number }[] = []; + for (const each of held.values()) { + if (each.words === null) continue; + let score = 0; + for (const word of words) { + const count = each.words.get(word) ?? 0; + if (count === 0) { + score = -1; + break; + } + score += count; + } + if (score > 0) scored.push({ id: each.id, score }); + } + // Best first; the id breaks a tie so the same index answers the same order every time. + scored.sort((a, b) => b.score - a.score || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); + return scored.slice(0, limit).map(({ id }) => ({ id })); + }, + }; +}; diff --git a/packages/entity/src/search-index-postgres.ts b/packages/entity/src/search-index-postgres.ts new file mode 100644 index 000000000..e349a2a3c --- /dev/null +++ b/packages/entity/src/search-index-postgres.ts @@ -0,0 +1,85 @@ +// `postgresSearchIndex()`: the built-in `SearchIndex` — `.searchable()`'s generated `tsvector`, +// behind the port. Postgres keeps the vector itself on every write (`generated always as … stored`), +// so `index`/`remove` only check the entity and send nothing; `query` is the repository's own +// `matches` read, so tenancy, the actor check and soft delete are the ones `.search()` already has. + +import type { DbClient } from '@ultimat3/db'; +import type { EntityCore } from './entity'; +import { EntityError } from './errors'; +import { searchUndeclared } from './feature-errors'; +import { postgresRepo } from './pg-driver'; +import { getEntity } from './registry'; +import { rowKey } from './row-key'; +import { SEARCH_PROPERTY } from './search'; +import type { SearchHit, SearchIndex, SearchScope } from './search-index'; +import { searchLimitOf } from './search-index'; +import type { Predicate } from './tenancy'; + +export interface PostgresSearchIndexOptions { + /** Pin a client; left out, the ambient pool (or the open transaction) per statement. */ + readonly client?: DbClient; +} + +/** The registered entity a scope names, with a search vector — refused otherwise. */ +const searchableEntity = (name: string): EntityCore => { + const core = getEntity(name)?.core; + if (core === undefined || core.$search === null) throw searchUndeclared(name); + return core; +}; + +/** The tenant predicate a scope adds; a tenant named on an untenanted entity is refused. */ +const tenantPredicate = (core: EntityCore, scope: SearchScope): readonly Predicate[] => { + const column = core.$tenantColumn; + if (column === null) { + if (scope.tenant === null) return []; + throw new EntityError({ + code: 'X_INVARIANT_VIOLATED', + cause: `search of ${core.$name} named a tenant, but ${core.$name} has no tenant column — the tenant would scope nothing`, + fix: 'searchIndex().query({ entity, tenant: null, term }) — an untenanted entity is searched whole', + }); + } + // `null` on a tenanted entity adds no predicate: the repository then takes the acting actor's + // tenant, or refuses `X_TENANCY_UNSCOPED` — the rule every other read of it already follows. + return scope.tenant === null ? [] : [{ column, op: 'eq', value: scope.tenant }]; +}; + +/** + * `index` and `remove` send nothing, so the version contract holds by construction: the table IS + * the newest state, and a stale document never reaches it. + * + * The indexes whose match is the statement's own `matches` predicate — a chain's `.search()` then + * sends today's single statement (`pg-sql.ts`'s `searchSql`), never a round trip for ids. Only + * this file adds to it: an app's index is always asked through the port. + */ +const inStatement = new WeakSet(); + +export const answersInStatement = (index: SearchIndex): boolean => inStatement.has(index); + +export const postgresSearchIndex = (options: PostgresSearchIndexOptions = {}): SearchIndex => { + const repoOf = (core: EntityCore) => + postgresRepo(core, options.client === undefined ? {} : { client: options.client }); + const index: SearchIndex = { + name: 'postgres', + index: async (documents) => { + for (const document of documents) + tenantPredicate(searchableEntity(document.entity), document); + }, + remove: async (scope) => { + tenantPredicate(searchableEntity(scope.entity), scope); + }, + query: async (query): Promise => { + const limit = searchLimitOf(query); + const core = searchableEntity(query.entity); + const where: readonly Predicate[] = [ + ...tenantPredicate(core, query), + { column: SEARCH_PROPERTY, op: 'matches', value: query.term }, + ]; + // Primary-key order, as `.search()` with no `orderBy` answers: a rank is not a sort key the + // plan can seek on, so the built-in index returns matches, not a relevance order. + const page = await repoOf(core).findMany({ where, limit }); + return page.rows.map((row) => ({ id: rowKey(core, row) })); + }, + }; + inStatement.add(index); + return index; +}; diff --git a/packages/entity/src/search-index-sync.test.ts b/packages/entity/src/search-index-sync.test.ts new file mode 100644 index 000000000..e213ecb62 --- /dev/null +++ b/packages/entity/src/search-index-sync.test.ts @@ -0,0 +1,137 @@ +// The write path: with an external index installed, committed entity writes keep it current — an +// insert, update or upsert indexes the row, a delete (soft or hard) removes it — and a rollback +// indexes nothing. The built-in Postgres index subscribes to nothing. + +import { afterAll, afterEach, describe, expect, test } from 'bun:test'; +import type { Clock } from '@ultimat3/core'; +import { ctxOf, runWithContext, subscribed, userActor } from '@ultimat3/core'; +import { text, timestamp, uuid } from './columns'; +import { database, memoryDriver } from './database'; +import { entity } from './entity'; +import { clearRegistry } from './registry'; +import type { SearchDocument, SearchIndex } from './search-index'; +import { installSearchIndex } from './search-index-installed'; +import { memorySearchIndex } from './search-index-memory'; +import { postgresSearchIndex } from './search-index-postgres'; + +afterAll(() => { + clearRegistry(); +}); + +const ORG = '00000000-0000-7000-8000-00000000000a'; +const ID = '00000000-0000-7000-8000-000000000001'; + +const notes = entity('ssync_notes', { + columns: { + id: uuid().primaryKey(), + orgId: uuid().tenant(), + title: text().searchable(), + updatedAt: timestamp().defaultNow().onUpdateNow(), + deletedAt: timestamp().nullable(), + }, +}); + +const plain = entity('ssync_plain', { columns: { id: uuid().primaryKey(), note: text() } }); + +let release: (() => void) | undefined; +afterEach(() => { + release?.(); + release = undefined; +}); + +/** A clock each write moves on by a second: every write is a NEWER version than the last. */ +let at = Date.parse('2026-11-09T10:00:00.000Z'); +const ticking: Clock = { + now: () => { + at += 1_000; + return new Date(at); + }, + monotonic: () => at, +}; + +/** As a member of `ORG`, so a point write on the tenant-scoped entity is scoped. */ +const asMember = (fn: () => Promise): Promise => + runWithContext(ctxOf({ clock: ticking, actor: userActor({ id: 'member-1', orgId: ORG }) }), fn); + +/** Settles the fire-and-forget index calls a commit started. */ +const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); + +const hits = async (index: SearchIndex, term: string) => + (await index.query({ entity: notes.$name, tenant: ORG, term })).map((hit) => hit.id); + +describe('the search index write path', () => { + test('insert, update and delete keep an installed index current', async () => { + const index = memorySearchIndex(); + release = installSearchIndex(index); + const db = database({ notes }, { driver: memoryDriver() }); + await asMember(() => db.notes.insert({ id: ID, orgId: ORG, title: 'zebra' })); + await settle(); + expect(await hits(index, 'zebra')).toEqual([ID]); + await asMember(() => db.notes.update(ID, { title: 'quokka' })); + await settle(); + expect(await hits(index, 'zebra')).toEqual([]); + expect(await hits(index, 'quokka')).toEqual([ID]); + await asMember(() => db.notes.delete(ID)); + await settle(); + expect(await hits(index, 'quokka')).toEqual([]); + }); + + test('a rolled-back write indexes nothing', async () => { + const index = memorySearchIndex(); + release = installSearchIndex(index); + const driver = memoryDriver(); + const db = database({ notes }, { driver }); + await driver + .transactor() + .run(async () => { + await db.notes.insert({ id: ID, orgId: ORG, title: 'zebra' }); + // The rollback is the transactor's answer to a rejected unit of work: input, not a verdict. + return Promise.reject(new RangeError('rolled back on purpose')); + }) + .catch(() => undefined); + await settle(); + expect(await hits(index, 'zebra')).toEqual([]); + }); + + test("an index that fails does not fail the user's write", async () => { + const refusing: SearchIndex = { + name: 'refusing', + index: () => Promise.reject(new TypeError('index is down')), + remove: async () => undefined, + query: async () => [], + }; + release = installSearchIndex(refusing); + const db = database({ notes }, { driver: memoryDriver() }); + const stored = await db.notes.insert({ id: ID, orgId: ORG, title: 'zebra' }); + await settle(); + expect(stored.id).toBe(ID); + }); + + test('an entity with nothing searchable is never sent to the index', async () => { + const documents: SearchDocument[] = []; + release = installSearchIndex({ + name: 'recording', + index: async (batch) => { + documents.push(...batch); + }, + remove: async () => undefined, + query: async () => [], + }); + const db = database({ plain }, { driver: memoryDriver() }); + await db.plain.insert({ id: ID, note: 'zebra' }); + await settle(); + expect(documents).toEqual([]); + }); + + test('the built-in Postgres index subscribes to nothing: zero cost per write', () => { + expect(subscribed('entity.row')).toBe(false); + release = installSearchIndex(postgresSearchIndex()); + expect(subscribed('entity.row')).toBe(false); + release(); + release = installSearchIndex(memorySearchIndex()); + expect(subscribed('entity.row')).toBe(true); + release(); + release = undefined; + expect(subscribed('entity.row')).toBe(false); + }); +}); diff --git a/packages/entity/src/search-index-sync.ts b/packages/entity/src/search-index-sync.ts new file mode 100644 index 000000000..31652f6ab --- /dev/null +++ b/packages/entity/src/search-index-sync.ts @@ -0,0 +1,96 @@ +// The search index WRITE path: an installed external index hears every committed row change of a +// searchable entity through core's `entity.row` event — which `row-observer.ts` emits AFTER COMMIT, +// so a rollback indexes nothing. Insert/update/upsert/transition index the row; a delete removes it +// at the delete's instant. A failure is a log line, never the user's write; `reindex` repairs it. + +import { isUltimateError, logger, renderThrowable, subscribe } from '@ultimat3/core'; +import { entityNow } from './clock'; +import type { EntityCore } from './entity'; +import { getEntity } from './registry'; +import { rowKey } from './row-key'; +import type { SearchIndex } from './search-index'; +import { searchDocumentOf } from './search-index'; +import { answersInStatement } from './search-index-postgres'; + +/** The one log event a failed mirror writes — its fields name the entity, the op and the code. */ +export const SEARCH_INDEX_WRITE_FAILED = 'entity.search-index.write-failed'; + +const unmirrored = (entity: string, op: string, cause: string, code?: string): void => { + logger.warn(SEARCH_INDEX_WRITE_FAILED, { + entity, + op, + ...(code === undefined ? {} : { code }), + cause, + fix: `reindex({ name: 'reindex-${entity}', entity, source, tenant: 'none' }) # rebuild the index, which missed this ${op}`, + }); +}; + +const failed = (entity: string, op: string, error: unknown): void => + unmirrored(entity, op, renderThrowable(error), isUltimateError(error) ? error.code : undefined); + +/** The searchable entity an event names, or `undefined` for one nothing in the index reads. */ +const searchableNamed = (name: string): EntityCore | undefined => { + const core = getEntity(name)?.core; + return core === undefined || core.$search === null ? undefined : core; +}; + +/** One committed change, mirrored. Synchronous to the bus; the index call settles on its own. */ +const mirror = ( + index: SearchIndex, + entity: string, + op: string, + before: Readonly> | null, + after: Readonly> | null, +): void => { + const core = searchableNamed(entity); + if (core === undefined) return; + try { + if (after !== null) { + index.index([searchDocumentOf(core, after)]).catch((error) => failed(entity, op, error)); + return; + } + // A delete whose row this process could not read (`before` is `null` when the key is not + // `id`) names no id to remove — the gap `reindex` cannot close either, so it is logged. + if (before === null) { + unmirrored(entity, op, 'the deleted row was not read: its primary key is not id'); + return; + } + const tenantColumn = core.$tenantColumn; + const tenant = tenantColumn === null ? null : before[tenantColumn]; + index + .remove( + { entity, tenant: typeof tenant === 'string' ? tenant : null }, + [rowKey(core, before)], + entityNow().getTime(), + ) + .catch((error) => failed(entity, op, error)); + } catch (error) { + failed(entity, op, error); + } +}; + +/** + * Subscribe `index` to the write path, or nothing for the built-in Postgres index — its vector is + * the row, so it costs a write nothing, not even `row-observer.ts`'s `before` read. Only the index + * `searchIndex()` answers mirrors: a test's install over the boot's does not write to both. + */ +export const syncSearchIndex = (index: SearchIndex, current: () => SearchIndex): (() => void) => { + if (answersInStatement(index)) return () => {}; + const rows = subscribe('entity.row', ({ payload }) => { + if (current() !== index) return; + mirror(index, payload.entity, payload.op, payload.before, payload.after); + }); + // A filtered write names a count, not rows: nothing to mirror, so it is said out loud. + const bulk = subscribe('entity.bulk', ({ payload }) => { + if (current() !== index || searchableNamed(payload.entity) === undefined) return; + unmirrored( + payload.entity, + `${payload.op}Where`, + `${payload.rows} rows changed by a filtered write are not mirrored`, + ); + }); + return () => { + rows(); + bulk(); + }; +}; diff --git a/packages/entity/src/search-index-version.ts b/packages/entity/src/search-index-version.ts new file mode 100644 index 000000000..8fb4e07ec --- /dev/null +++ b/packages/entity/src/search-index-version.ts @@ -0,0 +1,42 @@ +// A search document's VERSION: the row's `onUpdateNow()` instant, epoch ms. The one monotonic +// per-row value the entity layer guarantees — `touch()` stamps it on every update, upsert and +// transition from `entityNow()` — so an index can keep only newer and refuse a stale batch. + +import type { EntityCore } from './entity'; +import { EntityError } from './errors'; + +/** + * The property a searchable entity is versioned by: its first `onUpdateNow()` timestamp. Refused + * when there is none — without it no index can tell a `reindex` batch's stale read from a write. + */ +export const searchVersionColumn = (entity: EntityCore): string => { + for (const [property, column] of Object.entries(entity.$columns)) { + if (column.$meta.kind === 'timestamptz' && column.$meta.onUpdate !== undefined) return property; + } + throw new EntityError({ + code: 'X_INVARIANT_VIOLATED', + cause: `${entity.$name} is searchable but declares no onUpdateNow() column, so a search document has no version and an index cannot refuse a stale one`, + fix: `updatedAt: timestamp().defaultNow().onUpdateNow() # in entity('${entity.$name}'), then: x db gen "version ${entity.$name} for search"`, + }); +}; + +const instantMs = (value: unknown): number | undefined => { + if (value instanceof Date) return value.getTime(); + if (typeof value === 'string') return Date.parse(value); + return undefined; +}; + +/** The version of one row of `entity`: its version column, epoch ms. */ +export const searchVersionOf = (entity: EntityCore, row: Row): number => { + const core = entity as EntityCore; + const property = searchVersionColumn(core); + const ms = instantMs((row as Readonly>)[property]); + if (ms === undefined || !Number.isFinite(ms)) { + throw new EntityError({ + code: 'X_INVARIANT_VIOLATED', + cause: `a ${entity.$name} row reached the search index without its ${property} instant, so its document has no version`, + fix: `searchDocumentOf(${entity.$name}, await db.${entity.$name}.findById(id)) # the whole row: a select() that drops ${property} has no version`, + }); + } + return ms; +}; diff --git a/packages/entity/src/search-index.test.ts b/packages/entity/src/search-index.test.ts new file mode 100644 index 000000000..907e0bf0b --- /dev/null +++ b/packages/entity/src/search-index.test.ts @@ -0,0 +1,193 @@ +import { afterAll, describe, expect, test } from 'bun:test'; +import { isUltimateError } from '@ultimat3/core'; +import { recordingClient } from '@ultimat3/db'; +import { text, timestamp, uuid } from './columns'; +import { entity } from './entity'; +import { clearRegistry } from './registry'; +import { searchDocumentOf, searchLimitOf } from './search-index'; +import { installSearchIndex, searchIndex } from './search-index-installed'; +import { memorySearchIndex, searchWords } from './search-index-memory'; +import { postgresSearchIndex } from './search-index-postgres'; + +afterAll(() => { + clearRegistry(); +}); + +const ORG = '00000000-0000-7000-8000-00000000000a'; +const ID = '00000000-0000-7000-8000-000000000001'; + +const posts = entity('sidx_posts', { + columns: { + id: uuid().primaryKey(), + orgId: uuid().tenant(), + title: text().searchable('A'), + body: text().nullable().searchable(), + secretNote: text(), + updatedAt: timestamp().defaultNow().onUpdateNow(), + }, +}); + +const shared = entity('sidx_shared', { + columns: { + id: uuid().primaryKey(), + title: text().searchable(), + updatedAt: timestamp().defaultNow().onUpdateNow(), + }, +}); + +const unversioned = entity('sidx_unversioned', { + columns: { id: uuid().primaryKey(), title: text().searchable() }, +}); + +const AT = new Date('2026-11-09T10:00:00.000Z'); + +const plain = entity('sidx_plain', { columns: { id: uuid().primaryKey(), title: text() } }); + +const codeOf = async (pending: Promise): Promise => + pending.then( + () => 'answered', + (error: unknown) => (isUltimateError(error) ? error.code : `not coded: ${String(error)}`), + ); + +describe('searchDocumentOf()', () => { + test('carries only the searchable columns, the tenant and the row key', () => { + const row = { + id: ID.toUpperCase(), + orgId: ORG, + title: 'Zebra', + body: null, + secretNote: 'x', + updatedAt: AT, + }; + expect(searchDocumentOf(posts, row)).toEqual({ + entity: 'sidx_posts', + tenant: ORG, + id: ID, + version: AT.getTime(), + fields: { title: 'Zebra' }, + }); + }); + + test('an untenanted entity is indexed under tenant null', () => { + expect(searchDocumentOf(shared, { id: ID, title: 'a', updatedAt: AT }).tenant).toBeNull(); + }); + + test('the version is the onUpdateNow() instant; an entity with none is refused', () => { + // A string instant (a raw row) reads as the same version as its Date. + expect( + searchDocumentOf(shared, { id: ID, title: 'a', updatedAt: AT.toISOString() as never }) + .version, + ).toBe(AT.getTime()); + expect(() => searchDocumentOf(unversioned, { id: ID, title: 'a' })).toThrow( + /declares no onUpdateNow\(\) column/, + ); + // A projection that dropped the instant cannot be indexed: no version, no document. + expect(() => searchDocumentOf(shared, { id: ID, title: 'a' } as never)).toThrow( + /without its updatedAt instant/, + ); + }); +}); + +describe('searchLimitOf()', () => { + test('defaults, and refuses a limit no page could hold', () => { + const query = { entity: 'sidx_posts', tenant: ORG, term: 'a' }; + expect(searchLimitOf(query)).toBe(20); + for (const limit of [0, -1, 1.5, Number.NaN, 10_001]) { + expect(() => searchLimitOf({ ...query, limit })).toThrow(/limit must be a whole number/); + } + }); +}); + +describe('memorySearchIndex()', () => { + test('words are letters and digits in any script, case-folded, once each', () => { + expect(searchWords('Ünïcode, ünïcode! 42 (x)')).toEqual(['ünïcode', '42', 'x']); + }); + + test('ranks the document holding a word more often first', async () => { + const index = memorySearchIndex(); + await index.index([ + { entity: 'e', tenant: null, id: 'b', version: 1, fields: { title: 'zebra' } }, + { entity: 'e', tenant: null, id: 'a', version: 1, fields: { title: 'zebra zebra' } }, + ]); + expect(await index.query({ entity: 'e', tenant: null, term: 'zebra' })).toEqual([ + { id: 'a' }, + { id: 'b' }, + ]); + }); + + test('an entity is its own scope', async () => { + const index = memorySearchIndex(); + await index.index([ + { entity: 'e', tenant: null, id: 'a', version: 1, fields: { title: 'zebra' } }, + ]); + expect(await index.query({ entity: 'f', tenant: null, term: 'zebra' })).toEqual([]); + }); +}); + +describe('postgresSearchIndex()', () => { + test("queries through the repository's own match, tenant predicate included", async () => { + const client = recordingClient(); + client.on(/websearch_to_tsquery/, { + rows: [{ id: ID, org_id: ORG, title: 'Zebra', body: null, secret_note: 'x' }], + }); + const hits = await postgresSearchIndex({ client }).query({ + entity: 'sidx_posts', + tenant: ORG, + term: 'zebra', + limit: 5, + }); + expect(hits).toEqual([{ id: ID }]); + const statement = client.statements.find((each) => each.text.includes('websearch_to_tsquery')); + expect(statement?.text).toContain(`"search_tsv" @@ websearch_to_tsquery('english'`); + expect(statement?.text).toContain('"org_id"'); + expect(statement?.values).toContain(ORG); + expect(statement?.values).toContain('zebra'); + }); + + test('index and remove send nothing: the server keeps the generated vector', async () => { + const client = recordingClient(); + const index = postgresSearchIndex({ client }); + await index.index([ + { entity: 'sidx_posts', tenant: ORG, id: ID, version: 1, fields: { title: 'a' } }, + ]); + await index.remove({ entity: 'sidx_posts', tenant: ORG }, [ID], 1); + expect(client.statements).toEqual([]); + }); + + test('an entity with no search vector, or none registered, is X_SEARCH_UNDECLARED', async () => { + const index = postgresSearchIndex({ client: recordingClient() }); + for (const name of [plain.$name, 'sidx_nowhere']) { + expect(await codeOf(index.query({ entity: name, tenant: null, term: 'a' }))).toBe( + 'X_SEARCH_UNDECLARED', + ); + expect( + await codeOf(index.index([{ entity: name, tenant: null, id: ID, version: 1, fields: {} }])), + ).toBe('X_SEARCH_UNDECLARED'); + } + }); + + test('a tenant named on an untenanted entity is refused, never ignored', async () => { + const index = postgresSearchIndex({ client: recordingClient() }); + expect(await codeOf(index.query({ entity: shared.$name, tenant: ORG, term: 'a' }))).toBe( + 'X_INVARIANT_VIOLATED', + ); + }); + + test('a tenanted entity searched with no tenant and no actor is X_TENANCY_UNSCOPED', async () => { + const index = postgresSearchIndex({ client: recordingClient() }); + expect(await codeOf(index.query({ entity: posts.$name, tenant: null, term: 'a' }))).toBe( + 'X_TENANCY_UNSCOPED', + ); + }); +}); + +describe('installSearchIndex()', () => { + test('replaces the built-in index until released, and releases only its own frame', () => { + expect(searchIndex().name).toBe('postgres'); + const first = memorySearchIndex(); + const release = installSearchIndex(first); + expect(searchIndex()).toBe(first); + release(); + expect(searchIndex().name).toBe('postgres'); + }); +}); diff --git a/packages/entity/src/search-index.ts b/packages/entity/src/search-index.ts new file mode 100644 index 000000000..bef5b266c --- /dev/null +++ b/packages/entity/src/search-index.ts @@ -0,0 +1,107 @@ +// The `searchIndex` port: where a full-text search over an entity's `.searchable()` columns is +// answered. Postgres' generated `tsvector` is the built-in one (`search-index-postgres.ts`); an app +// writes Meilisearch or Typesense against this interface. Tenant-scoped on every call: a document +// is indexed under its tenant, and a query or a removal names the tenant it may reach. + +import type { EntityCore } from './entity'; +import { EntityError } from './errors'; +import { MAX_PAGE_SIZE } from './plan'; +import { rowKey } from './row-key'; +import { searchVersionOf } from './search-index-version'; + +/** What a search answers: the entity's name and the rows' tenant, never one without the other. */ +export interface SearchScope { + /** The entity NAME (`entity('', …)`), not its table. */ + readonly entity: string; + /** The tenant column's value; `null` for an untenanted entity, and only for one. */ + readonly tenant: string | null; +} + +/** One row as the index sees it: its key and the text of every searchable column it holds. */ +export interface SearchDocument extends SearchScope { + /** `rowKey(entity, row)` — the id a hit hands back to `findById`. */ + readonly id: string; + /** + * The row's `onUpdateNow()` instant, epoch ms (`search-index-version.ts`). An index keeps the + * NEWEST version per id and refuses an older one — a `reindex` batch read before a write. + */ + readonly version: number; + /** Searchable property → its text. A NULL column is absent, never `''`. */ + readonly fields: Readonly>; +} + +export interface SearchQuery extends SearchScope { + /** + * What a search box sent. Every word must match (case-insensitively); punctuation is text and + * never an error; a term with no word in it answers no hits. + */ + readonly term: string; + /** At most this many hits. Default `DEFAULT_SEARCH_LIMIT`, at most `MAX_PAGE_SIZE`. */ + readonly limit?: number; +} + +/** A match: the row's id, best first in the index's own relevance order. */ +export interface SearchHit { + readonly id: string; +} + +/** + * The port. `index` and `remove` take batches because `reindex` (a `backfill()` job) walks a table + * page by page. Every method is tenant-scoped: an index that answers tenant A a document indexed + * under tenant B is broken, and `searchIndexConformance` says so. + * + * Keep only newer: a document is applied only when its `version` is newer than what the index + * holds for that id — a document, or the TOMBSTONE a `remove` leaves. A version equal to the one + * held is a replay and changes nothing. + */ +export interface SearchIndex { + readonly name: string; + /** Insert or replace each document under its `(entity, tenant, id)`, when it is newer. */ + index(documents: readonly SearchDocument[]): Promise; + /** + * Drop these ids of `scope` and leave a tombstone at `version` (the delete's instant, epoch ms), + * so a document no newer than the delete is refused. Another tenant's id is left where it is. + */ + remove(scope: SearchScope, ids: readonly string[], version: number): Promise; + query(query: SearchQuery): Promise; +} + +export const DEFAULT_SEARCH_LIMIT = 20; + +/** The limit a query asked for, checked once for every index. */ +export const searchLimitOf = (query: SearchQuery): number => { + const limit = query.limit ?? DEFAULT_SEARCH_LIMIT; + if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_PAGE_SIZE) { + throw new EntityError({ + code: 'X_INVARIANT_VIOLATED', + cause: `search of ${query.entity}: limit must be a whole number from 1 to ${MAX_PAGE_SIZE}, got ${String(limit)}`, + fix: `searchIndex().query({ entity, tenant, term, limit: ${DEFAULT_SEARCH_LIMIT} })`, + }); + } + return limit; +}; + +/** + * The document a row of `entity` is indexed as — the ONE builder, so a write hook and `reindex` + * cannot index the same row two ways. Only `.searchable()` columns travel; nothing else of the row + * (a sealed column is never searchable, `sealed-column.ts`) leaves for an external index. Its + * version is the row's `onUpdateNow()` instant; an entity declaring none is refused. + */ +export const searchDocumentOf = (entity: EntityCore, row: Row): SearchDocument => { + const record = row as Readonly>; + const fields: Record = {}; + for (const [property, column] of Object.entries(entity.$columns)) { + if (column.$meta.searchable === undefined) continue; + const value = record[property]; + if (typeof value === 'string') fields[property] = value; + } + const tenantColumn = entity.$tenantColumn; + const tenant = tenantColumn === null ? null : record[tenantColumn]; + return { + entity: entity.$name, + tenant: typeof tenant === 'string' ? tenant : null, + id: rowKey(entity, row), + version: searchVersionOf(entity, row), + fields, + }; +}; diff --git a/packages/entity/src/search-repo.test.ts b/packages/entity/src/search-repo.test.ts new file mode 100644 index 000000000..81d5a7cb5 --- /dev/null +++ b/packages/entity/src/search-repo.test.ts @@ -0,0 +1,87 @@ +// `.search(term)` asks the INSTALLED index: the built-in one stays the statement's own `matches` +// predicate (so the memory driver still refuses it), any other one answers ids the chain then reads. + +import { afterAll, afterEach, describe, expect, test } from 'bun:test'; +import { isUltimateError } from '@ultimat3/core'; +import { recordingClient } from '@ultimat3/db'; +import { text, timestamp, uuid } from './columns'; +import { entity } from './entity'; +import { memoryRepo } from './memory-repo'; +import { postgresRepo } from './pg-driver'; +import { tableFor } from './query'; +import { clearRegistry } from './registry'; +import { searchDocumentOf } from './search-index'; +import { installSearchIndex } from './search-index-installed'; +import { memorySearchIndex } from './search-index-memory'; + +afterAll(() => { + clearRegistry(); +}); + +const ORG_A = '00000000-0000-7000-8000-00000000000a'; +const ORG_B = '00000000-0000-7000-8000-00000000000b'; +const id = (n: number): string => `00000000-0000-7000-8000-${String(n).padStart(12, '0')}`; + +const notes = entity('srepo_notes', { + columns: { + id: uuid().primaryKey(), + orgId: uuid().tenant(), + title: text().searchable('A'), + updatedAt: timestamp().defaultNow().onUpdateNow(), + }, +}); + +const AT = new Date('2026-11-09T10:00:00.000Z'); +const ROWS = [ + { id: id(1), orgId: ORG_A, title: 'zebra crossing', updatedAt: AT }, + { id: id(2), orgId: ORG_A, title: 'quokka', updatedAt: AT }, + { id: id(3), orgId: ORG_B, title: 'zebra', updatedAt: AT }, +]; + +let release: (() => void) | undefined; +afterEach(() => { + release?.(); + release = undefined; +}); + +const codeOf = async (pending: Promise): Promise => + pending.then( + () => 'answered', + (error: unknown) => (isUltimateError(error) ? error.code : `not coded: ${String(error)}`), + ); + +const memoryTable = async () => { + const table = tableFor(notes, memoryRepo(notes)); + await table.insertAll(ROWS); + return table; +}; + +describe('.search() through the installed index', () => { + test('the built-in index is the statement: the memory driver still refuses it', async () => { + const table = await memoryTable(); + expect(await codeOf(table.where({ orgId: ORG_A }).search('zebra').all())).toBe( + 'X_SEARCH_IN_MEMORY', + ); + }); + + test('the built-in index keeps today’s SQL: one statement, the match inside it', async () => { + const client = recordingClient(); + await tableFor(notes, postgresRepo(notes, { client })) + .where({ orgId: ORG_A }) + .search('zebra') + .all(); + expect(client.texts.filter((text) => text.includes('websearch_to_tsquery')).length).toBe(1); + }); + + test("another index answers the ids, scoped to the chain's tenant", async () => { + const index = memorySearchIndex(); + release = installSearchIndex(index); + await index.index(ROWS.map((row) => searchDocumentOf(notes, row))); + const table = await memoryTable(); + const found = await table.where({ orgId: ORG_A }).search('zebra').all(); + expect(found.map((row) => row.id)).toEqual([id(1)]); + expect(await table.where({ orgId: ORG_A }).search('zebra').count()).toBe(1); + expect(await table.where({ orgId: ORG_B }).search('zebra').count()).toBe(1); + expect(await table.where({ orgId: ORG_A }).search('walrus').all()).toEqual([]); + }); +}); diff --git a/packages/entity/src/search-repo.ts b/packages/entity/src/search-repo.ts new file mode 100644 index 000000000..ecf8343ae --- /dev/null +++ b/packages/entity/src/search-repo.ts @@ -0,0 +1,91 @@ +// The ONE search path: a chain's `.search(term)` asks the INSTALLED `SearchIndex`. The built-in +// Postgres index answers inside the statement (its `matches` predicate, today's SQL, untouched); +// any other index is asked for ids first, and the chain then reads those rows with its own +// tenancy, soft delete, order and cursor — so a Meilisearch hit can never widen what the chain sees. + +import { tryUseContext } from '@ultimat3/core'; +import type { EntityCore } from './entity'; +import { EntityError } from './errors'; +import { MAX_PAGE_SIZE } from './plan'; +import type { FindManyArgs, Repo } from './repo'; +import { SEARCH_PROPERTY } from './search'; +import { searchIndex } from './search-index-installed'; +import { answersInStatement } from './search-index-postgres'; +import type { Predicate } from './tenancy'; + +const isMatch = (predicate: Predicate): boolean => + predicate.column === SEARCH_PROPERTY && predicate.op === 'matches'; + +/** The tenant the chain is scoped to: its own `eq` on the tenant column, else the actor's. */ +const tenantOf = (entity: EntityCore, where: readonly Predicate[]): string | null => { + const column = entity.$tenantColumn; + if (column === null) return null; + const named = where.find((each) => each.column === column && each.op === 'eq'); + if (typeof named?.value === 'string') return named.value; + return tryUseContext()?.actor.orgId ?? null; +}; + +/** The single key a hit's id is read back by; a composite key has no `in` an index id can name. */ +const keyOf = (entity: EntityCore): string => { + const [key, ...rest] = entity.$primaryKey; + if (key === undefined || rest.length > 0) { + throw new EntityError({ + code: 'X_INVARIANT_VIOLATED', + cause: `search of ${entity.$name} through an external index needs a single-column primary key; it has ${entity.$primaryKey.length}`, + fix: 'installSearchIndex(postgresSearchIndex()) — the built-in index matches inside the statement, whatever the key', + }); + } + return key; +}; + +/** `args` with every `matches` predicate replaced by the ids the installed index answers. */ +const resolved = async (entity: EntityCore, args: FindManyArgs): Promise => { + const where = args.where ?? []; + const index = searchIndex(); + if (answersInStatement(index)) return args; + const tenant = tenantOf(entity, where); + const swapped: Predicate[] = []; + for (const predicate of where) { + if (!isMatch(predicate)) { + swapped.push(predicate); + continue; + } + const hits = await index.query({ + entity: entity.$name, + tenant, + term: String(predicate.value ?? ''), + limit: MAX_PAGE_SIZE, + }); + swapped.push({ column: keyOf(entity), op: 'in', value: hits.map((hit) => hit.id) }); + } + return { ...args, where: swapped }; +}; + +const searching = (args: FindManyArgs | undefined): boolean => args?.where?.some(isMatch) === true; + +/** + * `repo` with every read that takes a chain's arguments routed through `resolved` — only when the + * chain searched, so a read with no `.search()` reaches the repository exactly as before. + */ +export const searchedRepo = (entity: EntityCore, repo: Repo): Repo => { + const core = entity as EntityCore; + const series = repo.series; + return { + ...repo, + findMany: async (args) => + repo.findMany(searching(args) ? await resolved(core, args ?? {}) : args), + count: async (args) => repo.count(searching(args) ? await resolved(core, args ?? {}) : args), + countBy: async (column, args) => + repo.countBy(column, searching(args) ? await resolved(core, args ?? {}) : args), + aggregate: async (fn, column, args) => + repo.aggregate(fn, column, searching(args) ? await resolved(core, args ?? {}) : args), + approximateCount: async (args) => + repo.approximateCount(searching(args) ? await resolved(core, args ?? {}) : args), + ...(series === undefined + ? {} + : { + series: async (spec, args) => + series.call(repo, spec, searching(args) ? await resolved(core, args ?? {}) : args), + }), + }; +}; diff --git a/packages/entity/src/types.ts b/packages/entity/src/types.ts index dbf84bbb3..fced60a0d 100644 --- a/packages/entity/src/types.ts +++ b/packages/entity/src/types.ts @@ -7,6 +7,7 @@ // from the column set instead of being written a second time as a hand-maintained schema. import type { IndexMethod } from '@ultimat3/db'; +import type { PersonalDataKind } from './personal-data-kind'; /** * Postgres types the builders emit. `money` expands to `bigint` + `char(3)` (+ a nullable @@ -163,6 +164,12 @@ export interface ColumnMeta { * one GIN index per column. */ readonly searchable?: SearchWeight; + /** + * Presence marks this column as personal data, and the value is what kind. Read by a subject + * export or erase (`personal-data.ts`); never on `$describe()` — it changes no DDL, so it must + * not move the drift hash. + */ + readonly personalData?: PersonalDataKind; /** * The state machine this column's values move through, when one was declared. Built once at * declaration (`stateMachineOf`) so an illegal table is a refusal where it was written, and held @@ -239,6 +246,11 @@ export interface Column { * only — every other kind is refused here, where the chain was written. */ searchable(weight?: SearchWeight): Column; + /** + * Marks the column as personal data of `kind` — what `accountExport`/`accountErase` walk. No DDL + * moves, so marking an existing column is not a migration. + */ + personalData(kind: PersonalDataKind): Column; } /** diff --git a/packages/http/src/error-map-tier-1.ts b/packages/http/src/error-map-tier-1.ts index 73fce1608..e8c109069 100644 --- a/packages/http/src/error-map-tier-1.ts +++ b/packages/http/src/error-map-tier-1.ts @@ -108,4 +108,8 @@ export const TIER_1_ERROR_STATUS = { X_SEO_OG_IMAGE_CONFLICT: 500, // @ultimat3/money — a fromDecimal input that is not a plain decimal string X_MONEY_NOT_DECIMAL: 422, + // @ultimat3/storage — the upload scanner found malware in the file + X_STORAGE_UPLOAD_INFECTED: 422, + // @ultimat3/storage — the upload scanner could not give a verdict + X_STORAGE_SCAN_UNAVAILABLE: 503, } satisfies Readonly>; diff --git a/packages/http/src/error-map-tier-2.ts b/packages/http/src/error-map-tier-2.ts index 111ea8150..dcf300e99 100644 --- a/packages/http/src/error-map-tier-2.ts +++ b/packages/http/src/error-map-tier-2.ts @@ -141,4 +141,10 @@ export const TIER_2_ERROR_STATUS = { X_DRIVER_UNSUPPORTED: 501, // @ultimat3/entity — a custom driver returned a row outside the plan it was given X_DRIVER_ROW_OUT_OF_SCOPE: 500, + // @ultimat3/auth — a saml response failed its signature, wrapping, audience, recipient, time, InResponseTo or replay check + X_SAML_RESPONSE_INVALID: 400, + // @ultimat3/auth — saml IdP metadata has no entity id, HTTP-Redirect sign-on endpoint or signing certificate + X_SAML_METADATA_INVALID: 500, + // @ultimat3/auth — the URL named an identity provider the sso routes were not given + X_IDENTITY_PROVIDER_UNKNOWN: 404, } satisfies Readonly>; diff --git a/packages/http/src/error-map-tier-3.ts b/packages/http/src/error-map-tier-3.ts index 145a36aa2..b57123ba8 100644 --- a/packages/http/src/error-map-tier-3.ts +++ b/packages/http/src/error-map-tier-3.ts @@ -129,4 +129,16 @@ export const TIER_3_ERROR_STATUS = { X_METER_INVALID: 500, // @ultimat3/realtime — a channel's catch-up read decides with another policy X_CHANNEL_CATCHUP_POLICY_MISMATCH: 500, + // @ultimat3/jobs — withLock could not take the lock within its wait bound + X_LOCK_NOT_ACQUIRED: 409, + // @ultimat3/jobs — a lock withLock held expired before its work finished + X_LOCK_LOST: 409, + // @ultimat3/jobs — withLock found no lease store to hold the lock in + X_LOCK_STORE_MISSING: 500, + // @ultimat3/jobs — a ledger posting does not sum to zero in every currency + X_LEDGER_UNBALANCED: 422, + // @ultimat3/jobs — a ledger entry is not a whole-minor Money amount on a named account + X_LEDGER_ENTRY_INVALID: 422, + // @ultimat3/jobs — a ledger posting key is already stored with other entries + X_LEDGER_POSTING_CONFLICT: 409, } satisfies Readonly>; diff --git a/packages/http/src/error-map-tier-4.ts b/packages/http/src/error-map-tier-4.ts index a80057e34..4c0c3fa71 100644 --- a/packages/http/src/error-map-tier-4.ts +++ b/packages/http/src/error-map-tier-4.ts @@ -180,4 +180,19 @@ export const TIER_4_ERROR_STATUS = { X_MCP_TENANT_REQUIRED: 400, // @ultimat3/mcp — the tenant of an MCP call could not be resolved X_MCP_TENANT_UNRESOLVED: 500, + // @ultimat3/scraping — pdf() has no origin to render the route on + X_SCRAPE_PDF_ORIGIN_MISSING: 500, + // @ultimat3/scraping — a pdf() route is not a path on this app + X_SCRAPE_PDF_ROUTE_INVALID: 500, + // @ultimat3/scraping — the route pdf() printed answered an error status + X_SCRAPE_PDF_ROUTE_FAILED: 502, + // @ultimat3/scraping — the raw-CDP browser (`@ultimat3/scraping/cdp`), reached by a request + // through pdf(): no browser on this host and one that never started are this server's 503; a + // refused call is the upstream browser's 502, an unanswered one its 504. + X_CDP_BROWSER_MISSING: 503, + X_CDP_LAUNCH_FAILED: 503, + X_CDP_CALL_FAILED: 502, + X_CDP_TIMEOUT: 504, + // @ultimat3/scraping — every pdf() render slot stayed busy past the queue deadline + X_SCRAPE_PDF_BUSY: 503, } satisfies Readonly>; diff --git a/packages/jobs/CLAUDE.md b/packages/jobs/CLAUDE.md index 0adc9e3eb..00733568a 100644 --- a/packages/jobs/CLAUDE.md +++ b/packages/jobs/CLAUDE.md @@ -124,6 +124,7 @@ Tier 3. The `job` + `task` primitives, durable steps, transactional outbox, queu - **The retry decision reads the ERROR, on core's backoff curve** (`retry-classification.ts`, `backoffDelayMs`): `docs/history/jobs.md`, "Moved 2026-10-01 — retry and bounds". - **The claim loop re-arms on the PASS, never the jobs**; `tick()` resolves with that pass's executions. +- **A FULL pass marks a backlog**: the next freed slot claims at once (`worker-backlog.test.ts`). - **The scheduler runs one dispatch round at a time**; `stop()` waits it out before `leader.release()`. Full rule: `docs/history/jobs.md`. - **`run-once` fires ONE catch-up**: the watermark rides the fire (`ScheduledFire.watermarkMs`), never a `markFired` behind it; `skip` fires the latest missed occurrence (`latestOccurrenceBy`). @@ -274,6 +275,11 @@ Long form: [`docs/history/jobs.md`](../../docs/history/jobs.md), "Moved 2026-10- | `import-format.ts` / `-source.ts` / `-errors.ts` | record boundaries on bytes; the `ImportSource` seam; the `X_IMPORT_*` classes | | `driver-redis-refusal.ts` | `typedRefusals`: a send refused for want of a connection → `X_DRIVER_UNAVAILABLE` | | `subject*.ts` | `subjectExport`/`subjectErase`; reach only via DELETED tables (README) | +| `gdpr-*.ts` | `.personalData()` coverage (thrown by `plannedSubject`, listed by `x exercise`); the erase's audit record | +| `ledger*.ts` | `ledger()`: balanced per currency, ONE step; a key is one posting per tenant, re-used with other lines `X_LEDGER_POSTING_CONFLICT` | +| `lock*.ts` | `withLock`: one `lock:` slot; token = the store's `fence`; renewal as `worker-fleet-slots.ts` | +| `reindex.ts` | `reindex()`: a `backfill()` into the `searchIndex` port via `searchDocumentOf` | +| `partitions.ts` | `partitions()` + `.schedule`: db's `partitionMaintenance` per partitioned entity, job clock | | `webhook-receiver.ts` / `-route.ts` / `-types.ts` / `-errors.ts` | `webhookReceiver()` and `webhookReceiverRoutes`; the route; `ReceivedEvent`, `WebhookInbound`; its three refusals | Commands: `bun test ./packages/jobs/` · `bun run --filter @ultimat3/jobs typecheck`. Test harnesses: diff --git a/packages/jobs/README.md b/packages/jobs/README.md index 726af4db7..cc1717b15 100644 --- a/packages/jobs/README.md +++ b/packages/jobs/README.md @@ -482,6 +482,59 @@ the new pods serve, puts the sweeps on the queue and exits, and a slow UPDATE ne open against a database still serving the previous build. `--all` isolates per name and continues past a failure, so one wedged cleanup cannot block every later one forever. +### Rebuilding a search index: `reindex()` + +`reindex({ name, entity, source, tenant, index? })` is a `backfill()` whose batch writes each row's +document to the `searchIndex` port — entity's one builder, `searchDocumentOf`, so only +`.searchable()` columns leave and each document keeps its tenant. Run it after moving from the +built-in Postgres index to Meilisearch or Typesense, or after an index lost data. An entity with no +searchable column is refused at declaration. The index keeps only the NEWER version of each id, +so neither a replayed batch nor a stale read can overwrite a newer write or bring a deleted row +back. + +```ts +import { database, entity, text, timestamp, uuid } from '@ultimat3/entity'; +import { reindex } from '@ultimat3/jobs'; + +const posts = entity('posts', { + columns: { + id: uuid().primaryKey(), + title: text().searchable(), + updatedAt: timestamp().defaultNow().onUpdateNow(), // each document's version + }, +}); +const db = database({ posts }); + +export const reindexPosts = reindex({ + name: 'reindex-posts', + entity: posts, + tenant: 'none', + source: () => db.posts, +}); +``` + +## Partition maintenance is a job too + +`partitions({ name?, cron?, tz?, clock? })` keeps every `entity({ partitionBy })` table ready: one +step per partitioned entity runs `@ultimat3/db`'s `partitionMaintenance(description, now)` — the +function the migration also calls — creating `ahead` partitions past the current one and +dropping those `retain` expired. It is a job plus a `.schedule` `task()` (`.schedule`, +default `DEFAULT_PARTITIONS_CRON`, 03:17 UTC daily), the shape `consumer()`'s `.supervisor` has. +`now` is the job's clock, never the server's, so a test moves it; a second run at one instant +changes nothing (`create … if not exists`). Entities are read per run, so one declared later is +maintained from its first tick. + +```ts +import { partitions } from '@ultimat3/jobs'; + +export const maintainPartitions = partitions(); // list it in defineApi({ jobs }) +``` + +Before a `backfill()` copies rows older than the first partition, enqueue it once with `{ from: +'' }`: that run creates every partition back to that row's period and drops +nothing. Every DDL statement runs under a `lock_timeout`, failing fast with `X_DB_LOCK_TIMEOUT` +(retryable — the job's retry absorbs it) rather than queueing behind a long transaction. + ## Consumers are jobs too `consumer()` reads a queue, a change stream or a vendor's cursor: a factory over `job()` plus a @@ -732,6 +785,29 @@ throws before a row is touched (`X_SUBJECT_TABLE_UNDECLARED`, `X_SUBJECT_TREATME `authSubjectTarget`, `auditSubjectTarget` and `storageSubjectTarget` satisfy it from their own packages. `{ target, retain: '' }` exports a target and never erases it. +### `personalData()` coverage + +`.personalData(kind)` on a column (`@ultimat3/entity`) marks it as a person's data. It adds no +second export/erase path: it is the proof that the declared subjects cover every marked column. + +| Check | Refuses | +|---|---| +| `unerasedPersonalData(plan)` | a reached table whose `{ anonymize }` names fewer than every marked column (`X_PERSONAL_DATA_UNERASED`). `'delete'` clears the row; `{ retain: '' }` is a stated reason | +| `unreachedPersonalData(plans)` | a marked entity no subject's walk reaches (`X_PERSONAL_DATA_UNREACHED`) | +| `personalDataProblems()` | both, over `registeredSubjects()` and every marked entity — what `x exercise` lists | + +Enforced twice: `plannedSubject()` throws the first `X_PERSONAL_DATA_UNERASED` when a run starts, so +`subjectExport`/`subjectErase` refuse before a row is touched, and `x exercise` lists both codes +as findings. **An erase is audited**: with an `auditSink` installed, `subjectErase` writes one +`allowed` record per completed erase and a `failed` record per attempt that threw: `name` the job, +`primitive: 'action'`, `surface: 'job'`, `input: { subject, run }` — the entity and the run id, +never the subject's key, which an app may make an email. At least once: a sink dedupes on +`idempotencyKey` (`:`). A sink that refuses fails the run; one refusing every attempt +can hold no record, and the dead-lettered run is the trail. + +`gdpr.job.test.ts` proves the property, not the shape: a covered erase leaves no marked value of the +person in any marked table, and a declaration missing one column is named and does leak it. + ## Outbound webhooks are jobs too `webhook()` is a factory over `job()` and delivers **one event to one endpoint**. Retry with @@ -865,6 +941,53 @@ export const vendorEvents = webhookReceiver({ | `.routes()` is set on the job handle in place, never on a copy | `isJobHandle` proves the object `job()` built, so `defineApi({ jobs })` registers it like any job | | the path is absolute and outside `/api` and `/_x` | `X_WEBHOOK_RECEIVER_INVALID` at declaration, beside a missing `verify`/`handle` or a bad `maxBytes` | +## Double-entry postings are jobs too + +`ledger({ name, tenant, book })` is a factory over `job()`. Its input is a posting — `{ key, +entries: [{ account, amount: Money }], orgId? }` — and its run refuses, before a line is written, +anything that is not balanced Money; then it writes every line in ONE durable step through the +app's `LedgerBook`. No provider, account plan or currency policy ships (axiom 8): the accounts +are your strings, the table is your entity. + +```ts +import { defaultDriver, entity, integer, money, text } from '@ultimat3/entity'; +import { entityLedgerBook, ledger } from '@ultimat3/jobs'; + +// The app's lines: keyed (postingKey, line) — (orgId, postingKey, line) when tenanted, where a key +// is one posting PER TENANT and the posting's `orgId` fills the tenant column. +export const ledgerLines = entity('ledger_lines', { + columns: { postingKey: text(), line: integer(), account: text(), amount: money() }, + primaryKey: ['postingKey', 'line'], +}); + +export const postPayment = ledger({ + name: 'payments.post', + tenant: (posting) => posting.orgId ?? 'none', + book: () => entityLedgerBook({ driver: defaultDriver(), entity: ledgerLines }), +}); + +await postPayment.enqueue({ + key: 'pay_123', + entries: [ + { account: 'cash', amount: { minor: 1299, currency: 'EUR' } }, + { account: 'revenue', amount: { minor: -1099, currency: 'EUR' } }, + { account: 'vat-payable', amount: { minor: -200, currency: 'EUR' } }, + ], +}); +``` + +| Rule | Why | +|---|---| +| every currency's entries sum to zero, each on its own (`X_LEDGER_UNBALANCED`) | an FX posting balances in EUR and in USD separately; netting across currencies is the error | +| `minor` a safe integer, the currency ISO 4217, one `scale` per currency, a non-blank account, at least two entries (`X_LEDGER_ENTRY_INVALID`) | no float money; a sum across two scales is not a sum | +| both refusals TERMINAL | the posting is the input: attempt five sees the same entries | +| idempotent by key: the job's live key is `:`; the book writes a key once per tenant, `(tenant, postingKey, line)` | a retry, a replay or a second enqueue of `pay_123` posts it once — `entityLedgerBook` is one `insert … on conflict do nothing` | +| `assertBalancedPosting(posting)` is exported | the action that enqueues refuses at the request, before a row is queued | +| a key already stored with other lines — another count, another line — is `X_LEDGER_POSTING_CONFLICT` (409, terminal), nothing written | a superset under one key would store only the new lines beside the old: an unbalanced book. A correction is a new key. The Postgres book inserts and reads the key back in ONE transaction | + +A payment provider's webhook is a `webhookReceiver()` whose `handle` enqueues the posting, keyed +by the provider's own id — the receiver dedupes the delivery, the ledger dedupes the posting. + ## The deadline cancels A job's `timeout` aborts `ctx.signal` **before** it fails the attempt, because the nack that @@ -1478,6 +1601,38 @@ Nor for a job being settled (`As of 2026-10-30`): the heartbeat stops when the w listing, and a renewal there matched no row — `jobs.lease.lost` for a job that had finished, with the hook's `ctx.signal` aborted under it. +### `withLock(key, fn, { ttl })` + +One holder fleet-wide for `fn`, held in the job driver's `LeaseStore` as one slot under +`lock:` — the rows `job.concurrency` already counts, so no table, no Redis, no second +mechanism. Not a primitive: a helper a job or action body calls. + +```ts +import { withLock } from '@ultimat3/jobs'; + +// The app's two halves: the slow read, and a write its store refuses under an older token. +declare function recompute(id: string, options: { signal: AbortSignal }): Promise; +declare function saveTotal(id: string, total: number, token: number): Promise; + +export const recount = (id: string): Promise => + withLock( + `invoice:${id}`, + async ({ token, signal }) => { + const total = await recompute(id, { signal }); + await saveTotal(id, total, token); + }, + { ttl: '30s', wait: '5s' }, + ); +``` + +| Option / fact | Meaning | +|---|---| +| `ttl` (required) | how long the key outlives a dead holder; renewed at a third of it while `fn` runs | +| `wait` (default one `ttl`, `0` = one try) | then `X_LOCK_NOT_ACQUIRED` (409, retryable), naming the holder | +| `token` | a NUMBER the lease store assigns per grant (`HeldLease.fence`; Postgres: `x_job_leases.fence` from one sequence), greater than every earlier holder's — never a client clock. Write it beside the data and refuse a smaller one. A store answering no fence is `X_LOCK_STORE_MISSING` | +| a renewal refused, or none landing within `ttl` — decided before each call, one in flight, a late `true` still late | `fn`'s `signal` aborts with `X_LOCK_LOST`, and the call rejects with it even if `fn` resolved — its work was not exclusive. A renew hung on a dead socket is caught by the next tick | +| `store` | default the installed driver's `leases`; none is `X_LOCK_STORE_MISSING` | + ## The operator surface Every operator capability is a member of `JobIntrospection` (`driver.introspect`), implemented by diff --git a/packages/jobs/src/driver-pg-ddl.ts b/packages/jobs/src/driver-pg-ddl.ts index 9d99ab7aa..d543b5f9e 100644 --- a/packages/jobs/src/driver-pg-ddl.ts +++ b/packages/jobs/src/driver-pg-ddl.ts @@ -180,6 +180,11 @@ create table if not exists x_job_leases ( create index if not exists x_job_leases_expiry_idx on x_job_leases (expires_at); +-- The fencing number every grant takes, from ONE sequence: increasing across holders and keys, +-- and a count the database keeps, never a client clock. A table created before it gains the column. +create sequence if not exists x_job_lease_fence; +alter table x_job_leases add column if not exists fence bigint; + -- Events step.waitForEvent consumes. Stored and not broadcast: the publisher is a web pod and -- the resumer is a worker pod, so an in-heap bus strands every waiting run in a real deployment. create table if not exists x_job_events ( diff --git a/packages/jobs/src/driver-pg-sql.ts b/packages/jobs/src/driver-pg-sql.ts index aed5238d2..6a82575fc 100644 --- a/packages/jobs/src/driver-pg-sql.ts +++ b/packages/jobs/src/driver-pg-sql.ts @@ -196,22 +196,27 @@ on conflict (task_name) do update * `(lease_key, slot)` primary key serialises two workers that picked the same slot, and the * loser's `do update` is guarded on the slot having expired — a live slot returns no row, so the * grant is never doubled. Under contention this can refuse a slot that is genuinely free; a - * refusal costs one poll interval and an over-grant costs the guarantee. + * refusal costs one poll interval and an over-grant costs the guarantee. `nextval` runs on the ONE + * candidate the inner `limit 1` kept, and a takeover takes the new number too: the fence. */ export const SQL_LEASE_ACQUIRE = ` -insert into x_job_leases (lease_key, slot, holder, expires_at) -select $1, s.slot, $2, now() + ($4::bigint * interval '1 millisecond') - from generate_series(0, $3::int - 1) as s(slot) - where not exists ( - select 1 from x_job_leases l - where l.lease_key = $1 and l.slot = s.slot and l.expires_at > now() - ) - order by s.slot - limit 1 +insert into x_job_leases (lease_key, slot, holder, expires_at, fence) +select f.lease_key, f.slot, f.holder, f.expires_at, nextval('x_job_lease_fence') + from ( + select $1::text as lease_key, s.slot, $2::text as holder, + now() + ($4::bigint * interval '1 millisecond') as expires_at + from generate_series(0, $3::int - 1) as s(slot) + where not exists ( + select 1 from x_job_leases l + where l.lease_key = $1 and l.slot = s.slot and l.expires_at > now() + ) + order by s.slot + limit 1 + ) as f on conflict (lease_key, slot) do update - set holder = excluded.holder, expires_at = excluded.expires_at + set holder = excluded.holder, expires_at = excluded.expires_at, fence = excluded.fence where x_job_leases.expires_at <= now() -returning slot +returning slot, fence `.trim(); /** diff --git a/packages/jobs/src/driver-pg-stores.test.ts b/packages/jobs/src/driver-pg-stores.test.ts index dc0cb0fbc..d65a55543 100644 --- a/packages/jobs/src/driver-pg-stores.test.ts +++ b/packages/jobs/src/driver-pg-stores.test.ts @@ -239,12 +239,14 @@ describe('the pg lease store — fleet-wide slots', () => { expect(executor.calls[0]?.params).toEqual(['sendInvite', 'w-1', 2, 30_000]); }); - test('a granted slot arrives as a NUMBER even when the client typed the column as text', async () => { - const executor = executorFor({ x_job_leases: [{ slot: '1' }] }); + test('a granted slot and its fence arrive as NUMBERS even when the client typed them as text', async () => { + // `fence` is a `bigint` column: the pg client hands it back as text. + const executor = executorFor({ x_job_leases: [{ slot: '1', fence: '42' }] }); expect(await driverWith(executor).leases?.acquire('sendInvite', 2, 30_000, 'w-1')).toEqual({ key: 'sendInvite', slot: 1, holder: 'w-1', + fence: 42, }); }); diff --git a/packages/jobs/src/driver-pg.ts b/packages/jobs/src/driver-pg.ts index a7676e3ea..98b865251 100644 --- a/packages/jobs/src/driver-pg.ts +++ b/packages/jobs/src/driver-pg.ts @@ -165,14 +165,15 @@ function postgresLeaseStore(exec: () => PgExecutor): LeaseStore { return { async acquire(key, limit, ttlMs, holder) { if (limit <= 0) return undefined; - const rows = await exec().query<{ slot: number | string }>(SQL_LEASE_ACQUIRE, [ - key, - holder, - limit, - ttlMs, - ]); + const rows = await exec().query<{ slot: number | string; fence: number | string }>( + SQL_LEASE_ACQUIRE, + [key, holder, limit, ttlMs], + ); const row = rows[0]; - return row === undefined ? undefined : { key, slot: Number(row.slot), holder }; + // `bigint` crosses as text; a sequence stays under 2^53 for any fleet's lifetime. + return row === undefined + ? undefined + : { key, slot: Number(row.slot), holder, fence: Number(row.fence) }; }, async renew(lease, ttlMs) { const rows = await exec().query<{ slot: number | string }>(SQL_LEASE_RENEW, [ diff --git a/packages/jobs/src/error-codes.ts b/packages/jobs/src/error-codes.ts index e7a770fa0..b9e79da58 100644 --- a/packages/jobs/src/error-codes.ts +++ b/packages/jobs/src/error-codes.ts @@ -67,6 +67,14 @@ export const JOB_OWNED_ERROR_CODES = [ 'X_SUBJECT_TABLE_UNDECLARED', 'X_SUBJECT_TREATMENT_INVALID', 'X_SUBJECT_CYCLE', + 'X_LOCK_NOT_ACQUIRED', + 'X_LOCK_LOST', + 'X_LOCK_STORE_MISSING', + 'X_LEDGER_UNBALANCED', + 'X_LEDGER_ENTRY_INVALID', + 'X_PERSONAL_DATA_UNERASED', + 'X_PERSONAL_DATA_UNREACHED', + 'X_LEDGER_POSTING_CONFLICT', ] as const; /** @@ -150,6 +158,15 @@ export const JOB_ERROR_TITLES: Readonly> = { 'a table reaches the declared subject and defineSubject gives it no treatment', X_SUBJECT_TREATMENT_INVALID: 'a defineSubject treatment cannot be carried out', X_SUBJECT_CYCLE: 'the subject erase has no deletion order', + X_LOCK_NOT_ACQUIRED: 'withLock could not take the lock within its wait bound', + X_LOCK_LOST: 'a lock withLock held expired before its work finished', + X_LOCK_STORE_MISSING: 'withLock found no lease store to hold the lock in', + X_LEDGER_UNBALANCED: 'a ledger posting does not sum to zero in every currency', + X_LEDGER_ENTRY_INVALID: 'a ledger entry is not a whole-minor Money amount on a named account', + X_PERSONAL_DATA_UNERASED: + 'a subject erase would leave a personalData() column of a reached table', + X_PERSONAL_DATA_UNREACHED: 'an entity carries personalData() columns no declared subject reaches', + X_LEDGER_POSTING_CONFLICT: 'a ledger posting key is already stored with other entries', }; // One unconditional call, so a second package claiming one of jobs' codes throws @@ -230,4 +247,14 @@ registerErrorRetry({ X_SUBJECT_TABLE_UNDECLARED: 'terminal', X_SUBJECT_TREATMENT_INVALID: 'terminal', X_SUBJECT_CYCLE: 'terminal', + // A posting is the job's input: the same entries are refused on every attempt. + X_LEDGER_UNBALANCED: 'terminal', + X_LEDGER_ENTRY_INVALID: 'terminal', + // A stored posting does not change between attempts: the repair is a new key. + X_LEDGER_POSTING_CONFLICT: 'terminal', + // No store is a wiring fact; contention (`X_LOCK_NOT_ACQUIRED`) and a lost lock stay retryable. + X_LOCK_STORE_MISSING: 'terminal', + // Read off the declarations, as the `X_SUBJECT_*` refusals are: the same every attempt. + X_PERSONAL_DATA_UNERASED: 'terminal', + X_PERSONAL_DATA_UNREACHED: 'terminal', }); diff --git a/packages/jobs/src/gdpr-audit.ts b/packages/jobs/src/gdpr-audit.ts new file mode 100644 index 000000000..088db0d66 --- /dev/null +++ b/packages/jobs/src/gdpr-audit.ts @@ -0,0 +1,75 @@ +// The erase of a person is a sensitive mutation, so `subjectErase()` writes it to the `auditSink` +// port when one is installed: an `allowed` record per completed erase and a `failed` one per +// attempt that threw. The input is the subject's entity and the run id — never the key, which an +// app may make an email. At least once: `idempotencyKey` (`:`) is what a sink dedupes on. + +import type { AuditRecord, Ctx } from '@ultimat3/core'; +import { AUDIT_SINK_FAILED_EVENT, auditFailureOf, getAuditSink, logger } from '@ultimat3/core'; +import type { StepApi } from './steps-api'; + +export interface EraseAuditInput { + /** The job's durable name: the record's `name`. */ + readonly job: string; + /** The subject's ENTITY name — which kind of person, never which one. */ + readonly subject: string; + /** The run: what an auditor joins to `x jobs show`, and the sink's dedupe key. */ + readonly runId: string; + readonly ctx: Ctx; +} + +/** + * `AuditRecord.primitive` is `'action' | 'query'` in core; an erase is a write, so `'action'`, + * and `surface: 'job'` says which projection ran it — the pair an `agentJob` write records too. + */ +const recordOf = (input: EraseAuditInput, error: unknown, failed: boolean): AuditRecord => ({ + at: input.ctx.now(), + name: input.job, + primitive: 'action', + mutator: false, + surface: 'job', + ctx: input.ctx, + input: { subject: input.subject, run: input.runId }, + idempotencyKey: `${input.job}:${input.runId}`, + replayed: false, + outcome: failed ? 'failed' : 'allowed', + failure: failed ? auditFailureOf(error) : null, +}); + +/** Run `erase`, then record it: `allowed` once it completed, `failed` for an attempt that threw. */ +export async function auditedErase( + input: EraseAuditInput, + step: StepApi, + erase: () => Promise, +): Promise { + let report: R; + try { + report = await erase(); + } catch (error: unknown) { + // The erase's own error wins; a sink that also refuses is logged, as action's gate logs it. + const record = recordOf(input, error, true); + await Promise.resolve() + .then(() => getAuditSink()?.write(record)) + .catch((sinkError: unknown) => { + logger.error(AUDIT_SINK_FAILED_EVENT, { + action: record.name, + outcome: record.outcome, + error: sinkError, + }); + }); + throw error; + } + const sink = getAuditSink(); + // Its own step, so a replay after it landed writes nothing; a write that landed and then lost its + // checkpoint is written again under the same `idempotencyKey`. A sink that refuses on EVERY + // attempt can hold no record at all — the dead-lettered run (`x jobs show`) is the trail, and the + // refusal is logged here as action's audit gate logs it. + if (sink !== null) { + await step + .run('audit', async () => sink.write(recordOf(input, null, false))) + .catch((error: unknown) => { + logger.error(AUDIT_SINK_FAILED_EVENT, { action: input.job, outcome: 'allowed', error }); + throw error; + }); + } + return report; +} diff --git a/packages/jobs/src/gdpr-coverage.ts b/packages/jobs/src/gdpr-coverage.ts new file mode 100644 index 000000000..5d723ba5d --- /dev/null +++ b/packages/jobs/src/gdpr-coverage.ts @@ -0,0 +1,94 @@ +// What `personalData()` (entity's marker) adds to `defineSubject()`: proof the declared subjects +// cover every marked column. A subject's plan says WHICH tables it reaches and how each is treated; +// this says whether that treatment clears the marked columns, and whether every marked entity is +// reached at all. Pure — plans and marked entities in, refusals out — so a run and a check agree. + +import type { PersonalDataEntity } from '@ultimat3/entity'; +import { personalDataEntities, registeredEntities } from '@ultimat3/entity'; +import { PersonalDataUnerasedError, PersonalDataUnreachedError } from './gdpr-errors'; +import type { Subject } from './subject'; +import { registeredSubjects } from './subject'; +import type { SubjectPlan } from './subject-plan'; +import { planSubject } from './subject-plan'; + +const markedOf = ( + marked: readonly PersonalDataEntity[], + table: string, +): PersonalDataEntity | undefined => marked.find((each) => each.entity.$name === table); + +/** + * Every reached table whose treatment would leave a marked column: `'delete'` clears the row, + * `{ retain }` is a stated reason (printed in the export's index), and `{ anonymize }` must name + * EVERY marked property — a value it overwrites is no longer the person's. + */ +export function unerasedPersonalData( + plan: SubjectPlan, + marked: readonly PersonalDataEntity[] = personalDataEntities(), +): readonly PersonalDataUnerasedError[] { + return plan.steps.flatMap((step) => { + const entry = markedOf(marked, step.entity); + const { treatment } = step; + if (entry === undefined || typeof treatment !== 'object' || !('anonymize' in treatment)) { + return []; + } + const left = entry.columns + .map((column) => column.property) + .filter((property) => !Object.hasOwn(treatment.anonymize, property)); + return left.length === 0 + ? [] + : [ + new PersonalDataUnerasedError({ + subject: plan.subject, + table: step.entity, + columns: left, + }), + ]; + }); +} + +/** Every marked entity no plan reaches, in the registry's name order. */ +export function unreachedPersonalData( + plans: readonly SubjectPlan[], + marked: readonly PersonalDataEntity[] = personalDataEntities(), +): readonly PersonalDataUnreachedError[] { + const reached = new Set(plans.flatMap((plan) => plan.steps.map((step) => step.entity))); + const subjects = plans.map((plan) => plan.subject); + return marked + .filter((entry) => !reached.has(entry.entity.$name)) + .map( + (entry) => + new PersonalDataUnreachedError({ + table: entry.entity.$name, + columns: entry.columns.map((column) => column.property), + subjects, + }), + ); +} + +/** + * The whole check over this process's declarations — what `x exercise` lists, once every entity + * module has loaded. A subject whose own plan is refused (`X_SUBJECT_*`) raises that refusal first: + * coverage of an unplannable subject is not a question with an answer. + */ +export function personalDataProblems( + subjects: readonly Subject[] = registeredSubjects(), + marked: readonly PersonalDataEntity[] = personalDataEntities(), +): readonly (PersonalDataUnerasedError | PersonalDataUnreachedError)[] { + // `planSubject` and not `subject-jobs.ts`'s `plannedSubject`: that one is where a run asks + // `unerasedPersonalData`, and importing it back would be a cycle. + const plans = subjects.map((subject) => { + const { plan, problems } = planSubject({ + subject: subject.entity, + tables: subject.init.tables, + columns: subject.init.columns, + entries: registeredEntities(), + }); + const [first] = problems; + if (first !== undefined) throw first; + return plan; + }); + return [ + ...plans.flatMap((plan) => unerasedPersonalData(plan, marked)), + ...unreachedPersonalData(plans, marked), + ]; +} diff --git a/packages/jobs/src/gdpr-errors.ts b/packages/jobs/src/gdpr-errors.ts new file mode 100644 index 000000000..3091c9e40 --- /dev/null +++ b/packages/jobs/src/gdpr-errors.ts @@ -0,0 +1,35 @@ +// The two `X_PERSONAL_DATA_*` refusals: a subject's erase that would leave a `personalData()` column +// standing, and a marked entity no subject reaches. Both are refusals of DECLARATIONS — the values +// printed are entity and column names, never a person's data. + +import { UltimateError } from '@ultimat3/core'; +import './error-codes'; + +/** A reached table's `{ anonymize }` does not name every marked column of it. */ +export class PersonalDataUnerasedError extends UltimateError { + constructor(input: { subject: string; table: string; columns: readonly string[] }) { + const named = input.columns.map((column) => `${column}: null`).join(', '); + super({ + code: 'X_PERSONAL_DATA_UNERASED', + cause: `defineSubject(${input.subject}) anonymizes ${input.table} without ${input.columns.join(', ')}, marked personalData() — an erase would leave the person's value in them`, + fix: `tables: { ${input.table}: { anonymize: { ${named} } } } # in defineSubject(${input.subject}, …), merged into the existing anonymize — or 'delete', or { retain: '' }`, + meta: { subject: input.subject, table: input.table, columns: [...input.columns] }, + }); + } +} + +/** No declared subject's walk reaches an entity carrying marked columns. */ +export class PersonalDataUnreachedError extends UltimateError { + constructor(input: { table: string; columns: readonly string[]; subjects: readonly string[] }) { + const by = + input.subjects.length === 0 + ? 'no subject is declared' + : `declared: ${input.subjects.join(', ')}`; + super({ + code: 'X_PERSONAL_DATA_UNREACHED', + cause: `${input.table} marks ${input.columns.join(', ')} personalData() and no defineSubject() walk reaches it (${by}) — no export or erase would ever see them`, + fix: `defineSubject(${input.table}, { driver, key, tables: { ${input.table}: 'delete' } }) # in the module declaring subjects — or a reference to a subject's entity, or columns: { ${input.table}: '' } on an existing defineSubject`, + meta: { table: input.table, columns: [...input.columns], subjects: [...input.subjects] }, + }); + } +} diff --git a/packages/jobs/src/gdpr.job.test.ts b/packages/jobs/src/gdpr.job.test.ts new file mode 100644 index 000000000..df64a9329 --- /dev/null +++ b/packages/jobs/src/gdpr.job.test.ts @@ -0,0 +1,239 @@ +// `personalData()` against the subject jobs: a declaration that covers every marked column erases +// every marked value of the person (proved by scanning each marked table after the run); one that +// misses a column, or a marked table no subject reaches, is named before any run. Memory driver. + +import { afterAll, beforeEach, describe, expect, test } from 'bun:test'; +import type { AuditRecord } from '@ultimat3/core'; +import { ctxOf, userActor } from '@ultimat3/core'; +import { installAuditSink } from '@ultimat3/core/adapter'; +import type { Driver } from '@ultimat3/entity'; +import { + clearRegistry, + database, + entity, + memoryDriver, + personalDataEntities, + text, + uuid, +} from '@ultimat3/entity'; +import { t } from '@ultimat3/schema'; +import type { ClaimedJob } from './driver'; +import { memoryJobDriver } from './driver-memory'; +import { executeJob } from './execute'; +import { personalDataProblems } from './gdpr-coverage'; +import type { AnyJobHandle } from './job'; +import { resetJobs } from './job'; +import type { SubjectTreatment } from './subject'; +import { defineSubject, resetSubjects } from './subject'; +import { subjectErase } from './subject-jobs'; + +const ORG = '00000000-0000-7000-8000-0000000000e1'; +const ME = '00000000-0000-7000-8000-000000000201'; +const YOU = '00000000-0000-7000-8000-000000000202'; + +const people = entity('gd_people', { + columns: { + id: uuid().primaryKey(), + orgId: uuid().tenant(), + email: text({ max: 80 }).personalData('contact'), + handle: text({ max: 40 }).personalData('identifier'), + plan: text({ max: 20 }), + }, +}); +const posts = entity('gd_posts', { + columns: { + id: uuid().primaryKey(), + orgId: uuid().tenant(), + authorId: uuid().references(() => people.id), + body: text({ max: 200 }).personalData('free-text'), + }, +}); +const leads = entity('gd_leads', { + columns: { id: uuid().primaryKey(), phone: text({ max: 20 }).personalData('contact') }, +}); + +const INPUT = t.object({ orgId: t.string, memberId: t.string }); +let rows: Driver; + +const subjectWith = (tables: Readonly>) => + defineSubject<{ readonly orgId: string; readonly memberId: string }>(people, { + driver: rows, + key: (input) => input.memberId, + tables, + }); + +const COVERED = { + gd_people: { anonymize: { email: 'erased@invalid', handle: 'erased' } }, + gd_posts: 'delete', +} as const; + +beforeEach(async () => { + resetJobs(); + resetSubjects(); + rows = memoryDriver(); + const db = database({ people, posts }, { driver: rows }); + const scope = { orgId: ORG }; + await db.people.insert( + { id: ME, orgId: ORG, email: 'me@x.test', handle: 'me', plan: 'pro' }, + scope, + ); + await db.people.insert( + { id: YOU, orgId: ORG, email: 'you@x.test', handle: 'you', plan: 'free' }, + scope, + ); + await db.posts.insert({ id: crypto.randomUUID(), orgId: ORG, authorId: ME, body: 'mine' }, scope); + await db.posts.insert( + { id: crypto.randomUUID(), orgId: ORG, authorId: YOU, body: 'yours' }, + scope, + ); +}); + +afterAll(() => { + resetJobs(); + resetSubjects(); + clearRegistry(); +}); + +const runErase = async (tables: Readonly>) => { + const handle = subjectErase({ + name: 'gd.erase', + input: INPUT, + tenant: (input) => input.orgId, + subject: subjectWith(tables), + }) as AnyJobHandle; + const driver = memoryJobDriver(); + await driver.enqueue({ + name: handle.name, + queue: handle.queue, + input: { orgId: ORG, memberId: ME }, + idempotencyKey: `gd.erase:${ME}`, + tenantId: ORG, + maxAttempts: 1, + }); + const [claimed] = await driver.claim({ + queues: [handle.queue], + limit: 1, + visibilityTimeoutMs: 30_000, + workerId: 'w1', + }); + const ctx = ctxOf({ role: 'worker', actor: userActor({ id: 'worker-1' }) }); + return executeJob({ driver, claimed: claimed as ClaimedJob, handle, ctx }); +}; + +/** Every marked value still stored in every marked table the subject reached, as text. */ +const markedValues = async (): Promise => { + const db = database({ people, posts }, { driver: rows }); + const found = [ + ...(await db.people.where({ orgId: ORG }).all()), + ...(await db.posts.where({ orgId: ORG }).all()), + ]; + const marked = personalDataEntities().flatMap((each) => each.columns.map((c) => c.property)); + return found.flatMap((row) => + marked.flatMap((property) => { + const value = (row as Readonly>)[property]; + return typeof value === 'string' ? [value] : []; + }), + ); +}; + +const codes = (problems: readonly { code: string; meta?: unknown }[]) => + problems.map((problem) => [problem.code, problem.meta]); + +describe('personalData() coverage of the declared subjects', () => { + test('covered: the erase leaves no marked value of the person, and all of everyone else’s', async () => { + subjectWith(COVERED); + const reached = personalDataEntities().filter((each) => each.entity.$name !== 'gd_leads'); + expect(personalDataProblems(undefined, reached)).toEqual([]); + expect((await runErase(COVERED)).outcome).toBe('completed'); + const left = await markedValues(); + for (const mine of ['me@x.test', 'me', 'mine']) expect(left).not.toContain(mine); + for (const yours of ['you@x.test', 'you', 'yours']) expect(left).toContain(yours); + }); + + test('an anonymize missing a marked column is named, and its erase refused before any row', async () => { + const leaky = { ...COVERED, gd_people: { anonymize: { email: 'erased@invalid' } } }; + subjectWith(leaky); + expect(codes(personalDataProblems())).toContainEqual([ + 'X_PERSONAL_DATA_UNERASED', + { subject: 'gd_people', table: 'gd_people', columns: ['handle'] }, + ]); + const refused = await runErase(leaky); + expect(refused.code).toBe('X_PERSONAL_DATA_UNERASED'); + // Planned before the first step: the posts it would have deleted are still there. + for (const kept of ['me@x.test', 'me', 'mine']) expect(await markedValues()).toContain(kept); + }); + + test('a marked entity no subject reaches is named, with the subjects that exist', () => { + subjectWith(COVERED); + expect(codes(personalDataProblems())).toEqual([ + [ + 'X_PERSONAL_DATA_UNREACHED', + { table: 'gd_leads', columns: ['phone'], subjects: ['gd_people'] }, + ], + ]); + }); + + test('retain is a stated reason, not a leak', () => { + subjectWith({ ...COVERED, gd_posts: { retain: 'kept for the moderation record' } }); + const reached = personalDataEntities().filter((each) => each.entity.$name !== 'gd_leads'); + expect(personalDataProblems(undefined, reached)).toEqual([]); + }); + + test('declares nothing about the unmarked plan column', () => { + expect(leads.$name).toBe('gd_leads'); + expect( + personalDataEntities() + .find((each) => each.entity.$name === 'gd_people') + ?.columns.map((c) => c.property), + ).toEqual(['email', 'handle']); + }); +}); + +describe('subjectErase() audits the erase', () => { + test('one allowed record per completed erase: the subject and the run — never the key', async () => { + const records: AuditRecord[] = []; + const release = installAuditSink({ write: (record) => void records.push(record) }); + let execution: Awaited> | undefined; + try { + execution = await runErase(COVERED); + } finally { + release(); + } + expect(execution?.outcome).toBe('completed'); + const runId = String((records[0]?.input as { run?: unknown } | undefined)?.run); + expect(records.map((r) => [r.name, r.primitive, r.surface, r.outcome, r.input])).toEqual([ + ['gd.erase', 'action', 'job', 'allowed', { subject: 'gd_people', run: runId }], + ]); + // The run id dedupes a replayed write: the step that writes it is at least once. + expect(records[0]?.idempotencyKey).toBe(`gd.erase:${runId}`); + // The key is the app's (an email, if it keys people by one): never in the sink. + const written = JSON.stringify(records.map((r) => [r.input, r.idempotencyKey])); + for (const value of [ME, 'me@x.test', 'me']) expect(written).not.toContain(`"${value}"`); + expect(written).not.toContain(ME); + }); + + test('a refused erase is a failed record with its code', async () => { + const records: AuditRecord[] = []; + const release = installAuditSink({ write: (record) => void records.push(record) }); + try { + await runErase({ ...COVERED, gd_people: { anonymize: { email: 'erased@invalid' } } }); + } finally { + release(); + } + expect(records.map((r) => [r.outcome, r.failure?.code])).toEqual([ + ['failed', 'X_PERSONAL_DATA_UNERASED'], + ]); + }); + + test('a sink that refuses fails the run: no erase is unrecorded', async () => { + const release = installAuditSink({ + // INPUT to the code under test: a sink whose store is down. + write: () => Promise.reject(new TypeError('audit store down')), + }); + try { + expect((await runErase(COVERED)).outcome).not.toBe('completed'); + } finally { + release(); + } + }); +}); diff --git a/packages/jobs/src/index.ts b/packages/jobs/src/index.ts index bbf302b01..6457d64ce 100644 --- a/packages/jobs/src/index.ts +++ b/packages/jobs/src/index.ts @@ -218,6 +218,8 @@ export { } from './export-sink'; export type { CallerFailure } from './failure-row'; export { callerFailure } from './failure-row'; +export { personalDataProblems, unerasedPersonalData, unreachedPersonalData } from './gdpr-coverage'; +export { PersonalDataUnerasedError, PersonalDataUnreachedError } from './gdpr-errors'; export { IDLE_POLL_CEILING_MS } from './idle-backoff'; export type { ImportDefinition, ImportReport } from './import'; export { @@ -313,6 +315,20 @@ export type { JobsRetentionOptions } from './jobs-retention'; export { jobsPurgeTarget } from './jobs-retention'; export type { HeldLease, LeaseStore, MemoryLeaseStore, MemoryLeaseStoreOptions } from './leases'; export { jobLeaseKey, memoryLeaseStore } from './leases'; +export type { LedgerDefinition, LedgerEntry, LedgerPosting, LedgerReport } from './ledger'; +export { assertBalancedPosting, ledger, ledgerLines, MIN_LEDGER_ENTRIES } from './ledger'; +export type { + EntityLedgerBookOptions, + LedgerBook, + LedgerLine, + MemoryLedgerBook, +} from './ledger-book'; +export { assertSamePosting, entityLedgerBook, memoryLedgerBook } from './ledger-book'; +export { + LedgerEntryInvalidError, + LedgerPostingConflictError, + LedgerUnbalancedError, +} from './ledger-errors'; export type { Lease, LimitConfig, @@ -323,6 +339,9 @@ export type { RateLimit, } from './limits'; export { concurrencyLimiter, NO_TENANT, tenantKeyFrom } from './limits'; +export type { HeldLock, WithLockOptions } from './lock'; +export { DEFAULT_LOCK_POLL_MS, lockLeaseKey, withLock } from './lock'; +export { LockLostError, LockNotAcquiredError, LockStoreMissingError } from './lock-errors'; export { queueDeadJobs, queueDemand, @@ -356,6 +375,18 @@ export type { OutboxRelay, RelayOptions } from './outbox-relay'; export { outboxRelay } from './outbox-relay'; export type { OutboxRetentionOptions } from './outbox-retention'; export { outboxPurgeTarget } from './outbox-retention'; +export type { + PartitionsDefinition, + PartitionsJob, + PartitionsReport, + PartitionsTableReport, +} from './partitions'; +export { + DEFAULT_PARTITIONS_CRON, + DEFAULT_PARTITIONS_NAME, + partitionedEntities, + partitions, +} from './partitions'; export type { ProgressFn } from './progress'; export type { PurgeDefinition, @@ -377,6 +408,8 @@ export type { PgListener, QueueWake, QueueWakeOptions } from './queue-wake'; export { startQueueWake } from './queue-wake'; export type { RefreshViewDefinition, RefreshViewReport } from './refresh-view'; export { refreshView } from './refresh-view'; +export type { ReindexDefinition } from './reindex'; +export { reindex } from './reindex'; export type { IntervalScheduler } from './renewal-timer'; export type { BackoffStrategy, Random, RetryDecision, RetryPolicy } from './retry'; export { diff --git a/packages/jobs/src/leases.test.ts b/packages/jobs/src/leases.test.ts index 0399d779a..7ce23bd6c 100644 --- a/packages/jobs/src/leases.test.ts +++ b/packages/jobs/src/leases.test.ts @@ -3,10 +3,11 @@ // agent reads off the docstring and ships against. import { afterEach, describe, expect, test } from 'bun:test'; -import { ctxOf } from '@ultimat3/core'; +import { ctxOf, frozenClock } from '@ultimat3/core'; import type { StandardSchemaV1 } from '@ultimat3/schema'; import { memoryJobDriver } from './driver-memory'; import { job, resetJobs } from './job'; +import type { HeldLease } from './leases'; import { jobLeaseKey, LEASE_SWEEP_INTERVAL_MS, memoryLeaseStore } from './leases'; import { jobWorker } from './worker'; @@ -219,3 +220,19 @@ describe('the lease store itself', () => { expect(store.retained()).toBe(1); }); }); + +describe('memoryLeaseStore · fences', () => { + test('every grant carries a fence greater than every earlier one, release or expiry between', async () => { + const clock = frozenClock('2026-11-09T00:00:00Z'); + const store = memoryLeaseStore({ clock }); + const a = await store.acquire('fence', 1, 1_000, 'a'); + await store.release(a as HeldLease); + const b = await store.acquire('fence', 1, 1_000, 'b'); + clock.advance(2_000); + const c = await store.acquire('fence', 1, 1_000, 'c'); + const fences = [a?.fence, b?.fence, c?.fence]; + expect(fences.every((fence) => typeof fence === 'number')).toBe(true); + expect([...fences].sort((x, y) => Number(x) - Number(y))).toEqual(fences); + expect(new Set(fences).size).toBe(3); + }); +}); diff --git a/packages/jobs/src/leases.ts b/packages/jobs/src/leases.ts index ab4fc5add..b3e9eec8d 100644 --- a/packages/jobs/src/leases.ts +++ b/packages/jobs/src/leases.ts @@ -13,6 +13,13 @@ export interface HeldLease { readonly key: string; readonly slot: number; readonly holder: string; + /** + * The grant's FENCING number, assigned by the store on every acquire (a takeover too) and + * greater than every one it assigned before — `withLock`'s token. The store's counter and never + * a client clock: a host whose clock runs fast would otherwise win "refuse a smaller token". + * Optional so a store written before it still holds `job.concurrency`; `withLock` refuses one. + */ + readonly fence?: number; } export interface LeaseStore { @@ -66,6 +73,8 @@ export function memoryLeaseStore(options: MemoryLeaseStoreOptions = {}): MemoryL const clock = options.clock ?? liveClock; type Slots = Map; const slots = new Map(); + // One counter per store, as the pg store's one sequence: increasing across every key. + let fences = 0; /** * The live slots of `key`, expired ones dropped — and the KEY dropped with its last slot. A @@ -101,7 +110,8 @@ export function memoryLeaseStore(options: MemoryLeaseStoreOptions = {}): MemoryL if (held.has(slot)) continue; held.set(slot, { holder, expiresAt: nowMs(clock) + ttlMs }); slots.set(key, held); - return Promise.resolve({ key, slot, holder }); + fences += 1; + return Promise.resolve({ key, slot, holder, fence: fences }); } return Promise.resolve(undefined); }, diff --git a/packages/jobs/src/ledger-book.ts b/packages/jobs/src/ledger-book.ts new file mode 100644 index 000000000..88363a7a3 --- /dev/null +++ b/packages/jobs/src/ledger-book.ts @@ -0,0 +1,168 @@ +// Where `ledger()` writes: the `LedgerBook` seam, the memory book a test reads, and the book over an +// app's own entity. The TABLE is the app's (axiom 8: its accounts, its retention, its tenancy); the +// mechanism is one idempotent statement keyed by `(postingKey, line)`, so a replay writes nothing. + +import type { Driver, EntityCore, RowWrite } from '@ultimat3/entity'; +import type { MoneyValue } from '@ultimat3/schema'; +import { LedgerEntryInvalidError, LedgerPostingConflictError } from './ledger-errors'; + +/** One stored leg. `(tenant, postingKey, line)` is unique: the identity a replay lands on. */ +export interface LedgerLine { + /** The posting's `orgId`: a key names one posting PER TENANT. Absent for an untenanted book. */ + readonly tenant?: string | undefined; + readonly postingKey: string; + readonly line: number; + readonly account: string; + readonly amount: MoneyValue; +} + +export interface LedgerBook { + /** + * Store one posting's lines, all or none, and answer how many THIS call wrote: every line, or + * `0` when the key is already stored with exactly these lines. A key stored with any other lines + * — another count, another line — is `X_LEDGER_POSTING_CONFLICT`, and nothing is written: the + * new lines beside the stored ones would unbalance the book. Atomic: never half a posting. + */ + write(lines: readonly LedgerLine[]): Promise; +} + +const sameAmount = (a: MoneyValue, b: MoneyValue): boolean => + a.minor === b.minor && a.currency === b.currency && (a.scale ?? null) === (b.scale ?? null); + +/** + * Refuse unless `stored` (the key's lines in the book) are exactly `incoming`. `fresh` is how many + * of `incoming` this write itself inserted: all of them is a new posting, none a replay — any + * other number means part of the key was already there, which is a conflict whatever it now reads. + */ +export function assertSamePosting( + incoming: readonly LedgerLine[], + stored: readonly Pick[], + fresh: number, +): void { + const key = incoming[0]?.postingKey ?? ''; + const conflict = (line: number | null): never => { + throw new LedgerPostingConflictError({ + key, + stored: stored.length - fresh, + incoming: incoming.length, + line, + }); + }; + if ((fresh !== 0 && fresh !== incoming.length) || stored.length !== incoming.length) { + conflict(null); + } + const byLine = new Map(stored.map((row) => [row.line, row])); + for (const line of incoming) { + const held = byLine.get(line.line); + if ( + held === undefined || + held.account !== line.account || + !sameAmount(held.amount, line.amount) + ) { + conflict(line.line); + } + } +} + +/** The memory book, plus what a test reads back. */ +export interface MemoryLedgerBook extends LedgerBook { + lines(): readonly LedgerLine[]; + /** One account's balance in one currency, in minor units. */ + balance(account: string, currency: string): number; +} + +const lineKey = (line: Pick): string => + `${line.tenant ?? ''}\u0000${line.postingKey}\u0000${line.line}`; + +/** Which posting a line belongs to: one per (tenant, key). A posting id, not a credential. */ +const postingOf = (line: Pick): string => + `${line.tenant ?? ''}\u0000${line.postingKey}`; + +export function memoryLedgerBook(): MemoryLedgerBook { + const stored = new Map(); + return { + async write(lines) { + const first = lines[0]; + const posting = first === undefined ? undefined : postingOf(first); + const held = [...stored.values()].filter((line) => postingOf(line) === posting); + // Judged before any write: a refused posting leaves the book as it was. + if (held.length > 0) { + assertSamePosting(lines, held, 0); + return 0; + } + for (const line of lines) stored.set(lineKey(line), line); + return lines.length; + }, + lines: () => [...stored.values()], + balance: (account, currency) => + [...stored.values()] + .filter((line) => line.account === account && line.amount.currency === currency) + .reduce((net, line) => net + line.amount.minor, 0), + }; +} + +export interface EntityLedgerBookOptions { + /** The driver the app's `database()` reads through, so tenancy and row observers hold. */ + readonly driver: Driver; + /** + * The app's lines entity: `postingKey` text, `line` integer, `account` text, `amount` money, a + * UNIQUE index (or the primary key) on `(postingKey, line)` — on `(, postingKey, line)` + * when the entity has a tenant column, which the posting's `orgId` fills — and whatever else it + * keeps defaulted. Without that unique Postgres refuses the statement (`42P10`). + */ + readonly entity: EntityCore; +} + +/** + * The book over an app entity: in ONE transaction, `insert … on conflict do nothing`, then the + * key's stored lines read back and judged (`assertSamePosting`) — a refusal rolls this write's + * lines back, and a concurrent write of the same key with other lines waits on the first's rows, + * then meets them committed and is refused. + */ +export function entityLedgerBook( + options: EntityLedgerBookOptions, +): LedgerBook { + return { + write(lines) { + const tenant = options.entity.$tenantColumn; + const first = lines[0]; + // A tenanted book holds one posting per (tenant, key): the org is the row's, never inferred. + if (tenant !== null && (first === undefined || first.tenant === undefined)) { + throw new LedgerEntryInvalidError({ + key: first?.postingKey ?? '', + index: null, + field: 'orgId', + got: first?.tenant, + }); + } + const scope: Record = tenant === null ? {} : { [tenant]: first?.tenant }; + // The lines ARE rows of `Row` minus its defaulted columns, which the statement fills. + const rows = lines.map(({ tenant: _, ...line }) => ({ + ...scope, + ...line, + })) as unknown as readonly RowWrite[]; + const target = ( + tenant === null ? ['postingKey', 'line'] : [tenant, 'postingKey', 'line'] + ) as (keyof Row & string)[]; + const repo = options.driver.repo(options.entity); + return options.driver.transactor().run(async (tx) => { + const written = await repo.upsertAll(rows, { onConflict: target, onMatch: 'nothing', tx }); + const held = await repo.findMany({ + // The tenant named, not left to the run's scope: outside a job no scope exists, and an + // unscoped tenanted read is refused (`X_TENANCY_UNSCOPED`). + where: [ + { column: 'postingKey', op: 'eq', value: first?.postingKey }, + ...(tenant === null + ? [] + : [{ column: tenant, op: 'eq' as const, value: first?.tenant }]), + ], + // One past what a match holds: a longer stored posting is seen as longer, never cut. + limit: lines.length + 1, + tx, + }); + assertSamePosting(lines, held.rows, written.length); + return written.length; + }); + }, + }; +} diff --git a/packages/jobs/src/ledger-errors.ts b/packages/jobs/src/ledger-errors.ts new file mode 100644 index 000000000..04938444a --- /dev/null +++ b/packages/jobs/src/ledger-errors.ts @@ -0,0 +1,48 @@ +// The two `X_LEDGER_*` refusals a posting can end on — both about the POSTING, raised before a line +// is written, so a refused posting leaves the book exactly as it was. Amounts and accounts are the +// app's own strings and integers; nothing `unknown` is interpolated (`describeValue` renders those). + +import { UltimateError } from '@ultimat3/core'; +import { describeValue } from '@ultimat3/schema'; +import './error-codes'; + +/** One entry, or the posting's shape, is not something a balanced book can hold. */ +export class LedgerEntryInvalidError extends UltimateError { + constructor(input: { key: string; index: number | null; field: string; got: unknown }) { + const where = input.index === null ? 'the posting' : `entries[${input.index}]`; + super({ + code: 'X_LEDGER_ENTRY_INVALID', + cause: `ledger posting ${JSON.stringify(input.key)}: ${where}.${input.field} is ${describeValue(input.got)}`, + fix: `{ account: 'cash', amount: { minor: 1299, currency: 'EUR' } } # ${where} in the enqueue of the ledger job: a named account, integer minor units (fromDecimal('12.99', 'EUR') from @ultimat3/money), at least two entries`, + meta: { key: input.key, index: input.index, field: input.field }, + }); + } +} + +/** The entries of one currency do not net to zero. `net` is that currency's sum, in minor units. */ +export class LedgerUnbalancedError extends UltimateError { + constructor(input: { key: string; currency: string; net: number }) { + super({ + code: 'X_LEDGER_UNBALANCED', + cause: `ledger posting ${JSON.stringify(input.key)} nets ${input.net} minor units of ${input.currency}, not zero — every currency's entries must sum to zero`, + fix: `{ account: '', amount: { minor: ${-input.net}, currency: '${input.currency}' } } # one more entry in the posting's entries, or correct the amounts`, + meta: { key: input.key, currency: input.currency, net: input.net }, + }); + } +} + +/** A key already stored with other lines: writing beside them would unbalance the book. */ +export class LedgerPostingConflictError extends UltimateError { + constructor(input: { key: string; stored: number; incoming: number; line: number | null }) { + const differs = + input.line === null + ? `holds ${input.stored} line(s) and this posting has ${input.incoming}` + : `holds another line ${input.line}`; + super({ + code: 'X_LEDGER_POSTING_CONFLICT', + cause: `ledger posting ${JSON.stringify(input.key)} is already in the book and ${differs} — a key names one posting, and its lines never change`, + fix: `{ key: ${JSON.stringify(`${input.key}:adjust-1`)}, entries: [ … ] } # the posting in the enqueue of the ledger job: a correction under a new key, or the entries exactly as first posted`, + meta: { key: input.key, stored: input.stored, incoming: input.incoming, line: input.line }, + }); + } +} diff --git a/packages/jobs/src/ledger-tenant.job.test.ts b/packages/jobs/src/ledger-tenant.job.test.ts new file mode 100644 index 000000000..6ba411b46 --- /dev/null +++ b/packages/jobs/src/ledger-tenant.job.test.ts @@ -0,0 +1,122 @@ +// `entityLedgerBook` over a TENANTED lines entity, run by `executeJob` under each posting's org: a +// key is one posting per tenant, so tenant B's `pay-1` is written beside tenant A's — never skipped +// as A's replay — and each tenant's lines carry its own org. Memory driver: tenancy and uniques +// are judged as Postgres judges them (`entityDriverConformance`). + +import { afterAll, afterEach, describe, expect, test } from 'bun:test'; +import { ctxOf, userActor } from '@ultimat3/core'; +import type { Driver } from '@ultimat3/entity'; +import { + clearRegistry, + database, + entity, + integer, + memoryDriver, + money, + text, + uuid, +} from '@ultimat3/entity'; +import type { ClaimedJob } from './driver'; +import { memoryJobDriver } from './driver-memory'; +import { executeJob } from './execute'; +import type { AnyJobHandle } from './job'; +import { resetJobs } from './job'; +import type { LedgerPosting } from './ledger'; +import { ledger } from './ledger'; +import { entityLedgerBook } from './ledger-book'; + +const ORG_A = '00000000-0000-7000-8000-0000000000a1'; +const ORG_B = '00000000-0000-7000-8000-0000000000b2'; + +const lines = entity('lt_lines', { + columns: { + orgId: uuid().tenant(), + postingKey: text(), + line: integer(), + account: text(), + amount: money(), + }, + primaryKey: ['orgId', 'postingKey', 'line'], +}); + +afterEach(() => { + resetJobs(); +}); +afterAll(() => { + clearRegistry(); +}); + +const run = async (handle: AnyJobHandle, posting: LedgerPosting) => { + const driver = memoryJobDriver(); + await driver.enqueue({ + name: handle.name, + queue: handle.queue, + input: posting, + idempotencyKey: posting.key, + ...(posting.orgId === undefined ? {} : { tenantId: posting.orgId }), + maxAttempts: 1, + }); + const [claimed] = await driver.claim({ + queues: [handle.queue], + limit: 1, + visibilityTimeoutMs: 30_000, + workerId: 'w1', + }); + const ctx = ctxOf({ role: 'worker', actor: userActor({ id: 'worker-1' }) }); + return executeJob({ driver, claimed: claimed as ClaimedJob, handle, ctx }); +}; + +const eur = (minor: number) => ({ minor, currency: 'EUR' }); +const posting = (orgId: string, minor: number): LedgerPosting => ({ + key: 'pay-1', + orgId, + entries: [ + { account: 'cash', amount: eur(minor) }, + { account: 'revenue', amount: eur(-minor) }, + ], +}); + +describe('entityLedgerBook · a tenanted lines entity', () => { + test('one key is one posting PER TENANT: B is written beside A, each under its own org', async () => { + const rows: Driver = memoryDriver(); + const post = ledger({ + name: 'lt.post', + tenant: (p) => p.orgId ?? 'none', + book: () => entityLedgerBook({ driver: rows, entity: lines }), + }) as AnyJobHandle; + const a = await run(post, posting(ORG_A, 1000)); + expect(a.outcome).toBe('completed'); + const b = await run(post, posting(ORG_B, 700)); + expect(b.outcome).toBe('completed'); + expect(b.result).toEqual({ key: 'pay-1', lines: 2, written: 2 }); + // A's replay is still a replay, and still its own. + expect((await run(post, posting(ORG_A, 1000))).result).toEqual({ + key: 'pay-1', + lines: 2, + written: 0, + }); + const db = database({ lines }, { driver: rows }); + const of = async (orgId: string) => + (await db.lines.where({ orgId }).all()).map((row) => [row.account, row.amount.minor]); + expect(await of(ORG_A)).toEqual([ + ['cash', 1000], + ['revenue', -1000], + ]); + expect(await of(ORG_B)).toEqual([ + ['cash', 700], + ['revenue', -700], + ]); + }); + + test('a tenanted lines entity with no org on the posting is refused before any write', async () => { + const rows: Driver = memoryDriver(); + const post = ledger({ + name: 'lt.untenanted', + tenant: 'none', + book: () => entityLedgerBook({ driver: rows, entity: lines }), + }) as AnyJobHandle; + const { orgId: _dropped, ...noOrg } = posting(ORG_A, 1000); + const refused = await run(post, noOrg); + expect(refused.code).toBe('X_LEDGER_ENTRY_INVALID'); + }); +}); diff --git a/packages/jobs/src/ledger.live.test.ts b/packages/jobs/src/ledger.live.test.ts new file mode 100644 index 000000000..fb0180477 --- /dev/null +++ b/packages/jobs/src/ledger.live.test.ts @@ -0,0 +1,154 @@ +// `ledger()` against a real Postgres through `entityLedgerBook`: the lines land in the app's own +// table in ONE statement, a second run of the same posting writes nothing, two concurrent runs of +// one key still store it once, and a refused posting leaves the table empty. Skips unless +// `TEST_DATABASE_URL`. + +import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; +import { ctxOf, probeDatabaseName, sweepProbeDatabases, userActor } from '@ultimat3/core'; +import type { PostgresClient } from '@ultimat3/db'; +import { dbExecutor, generateMigration, postgresClient, raw, statementsOf } from '@ultimat3/db'; +import { clearRegistry, entity, integer, money, postgresDriver, text } from '@ultimat3/entity'; +import type { ClaimedJob } from './driver'; +import { memoryJobDriver } from './driver-memory'; +import { executeJob } from './execute'; +import type { AnyJobHandle } from './job'; +import { resetJobs } from './job'; +import type { LedgerPosting } from './ledger'; +import { ledger } from './ledger'; +import { entityLedgerBook } from './ledger-book'; + +const url = Bun.env['TEST_DATABASE_URL']; +const hasPostgres = typeof url === 'string' && url.length > 0; +const PROBE_DB = probeDatabaseName('x_ledger_live'); + +const lines = entity('ledger_live_lines', { + columns: { postingKey: text(), line: integer(), account: text(), amount: money() }, + primaryKey: ['postingKey', 'line'], +}); + +const probeUrl = (): string => { + const parsed = new URL(url ?? 'postgres://localhost/postgres'); + parsed.pathname = `/${PROBE_DB}`; + return parsed.toString(); +}; + +afterAll(() => { + clearRegistry(); + resetJobs(); +}); + +describe.skipIf(!hasPostgres)('live · postgres · ledger()', () => { + let admin: PostgresClient; + let client: PostgresClient; + + beforeAll(async () => { + admin = postgresClient({ url: url ?? '', role: 'web', profile: { max: 1 } }); + await admin.execute(raw(`drop database if exists ${PROBE_DB} with (force)`)); + await sweepProbeDatabases( + dbExecutor(() => admin), + PROBE_DB, + ); + await admin.execute(raw(`create database ${PROBE_DB}`)); + client = postgresClient({ url: probeUrl(), role: 'web', profile: { max: 4 } }); + const { up } = generateMigration({ entities: [lines.$describe()], name: 'ledger lines' }); + for (const statement of statementsOf(up)) await client.execute(raw(statement)); + }); + + beforeEach(async () => { + await client.execute(raw('delete from ledger_live_lines')); + }); + + afterAll(async () => { + await client.close(); + await admin.execute(raw(`drop database if exists ${PROBE_DB} with (force)`)); + await admin.close(); + }); + + const run = async (handle: AnyJobHandle, posting: LedgerPosting) => { + const driver = memoryJobDriver(); + await driver.enqueue({ + name: handle.name, + queue: handle.queue, + input: posting, + idempotencyKey: posting.key, + maxAttempts: 1, + }); + const [claimed] = await driver.claim({ + queues: [handle.queue], + limit: 1, + visibilityTimeoutMs: 30_000, + workerId: 'w1', + }); + const ctx = ctxOf({ role: 'worker', actor: userActor({ id: 'worker-1' }) }); + return executeJob({ driver, claimed: claimed as ClaimedJob, handle, ctx }); + }; + + const stored = async () => + client.query<{ account: string; minor: string; currency: string }>( + raw( + 'select account, amount_minor::text as minor, amount_currency as currency from ledger_live_lines order by line', + ), + ); + + const posting: LedgerPosting = { + key: 'pay_live_1', + entries: [ + { account: 'cash', amount: { minor: 1299, currency: 'EUR' } }, + { account: 'revenue', amount: { minor: -1299, currency: 'EUR' } }, + ], + }; + + test('a posting lands once in the app table, however often or concurrently it runs', async () => { + const book = () => entityLedgerBook({ driver: postgresDriver({ client }), entity: lines }); + const post = ledger({ name: 'ledger-live', tenant: 'none', book }) as AnyJobHandle; + const [a, b] = await Promise.all([run(post, posting), run(post, posting)]); + expect([a.outcome, b.outcome]).toEqual(['completed', 'completed']); + const again = await run(post, posting); + expect(again.result).toEqual({ key: 'pay_live_1', lines: 2, written: 0 }); + expect(await stored()).toEqual([ + { account: 'cash', minor: '1299', currency: 'EUR' }, + { account: 'revenue', minor: '-1299', currency: 'EUR' }, + ]); + }); + + test('a key re-used with other entries is refused and writes no line', async () => { + const book = () => entityLedgerBook({ driver: postgresDriver({ client }), entity: lines }); + const post = ledger({ name: 'ledger-live-reused', tenant: 'none', book }) as AnyJobHandle; + expect((await run(post, posting)).outcome).toBe('completed'); + const longer = await run(post, { + key: posting.key, + entries: [ + ...posting.entries, + { account: 'vat', amount: { minor: 200, currency: 'EUR' } }, + { account: 'cash', amount: { minor: -200, currency: 'EUR' } }, + ], + }); + expect(longer.code).toBe('X_LEDGER_POSTING_CONFLICT'); + const moved = await run(post, { + key: posting.key, + entries: [ + { account: 'cash', amount: { minor: 1299, currency: 'EUR' } }, + { account: 'other', amount: { minor: -1299, currency: 'EUR' } }, + ], + }); + expect(moved.code).toBe('X_LEDGER_POSTING_CONFLICT'); + expect(await stored()).toEqual([ + { account: 'cash', minor: '1299', currency: 'EUR' }, + { account: 'revenue', minor: '-1299', currency: 'EUR' }, + ]); + }); + + test('an unbalanced posting writes no line', async () => { + const book = () => entityLedgerBook({ driver: postgresDriver({ client }), entity: lines }); + const post = ledger({ name: 'ledger-live-unbalanced', tenant: 'none', book }) as AnyJobHandle; + const execution = await run(post, { + key: 'pay_live_2', + entries: [ + { account: 'cash', amount: { minor: 1299, currency: 'EUR' } }, + { account: 'revenue', amount: { minor: -1200, currency: 'EUR' } }, + ], + }); + expect(execution.code).toBe('X_LEDGER_UNBALANCED'); + expect(await stored()).toEqual([]); + }); +}); diff --git a/packages/jobs/src/ledger.test.ts b/packages/jobs/src/ledger.test.ts new file mode 100644 index 000000000..3a1bee147 --- /dev/null +++ b/packages/jobs/src/ledger.test.ts @@ -0,0 +1,174 @@ +// `ledger()` run by `executeJob` on the memory driver and the memory book: a balanced posting +// lands once however often it runs; an unbalanced, float, scale-mixed or one-legged one is refused +// TERMINALLY before any line is written. + +import { afterEach, describe, expect, test } from 'bun:test'; +import { ctxOf, isUltimateError, userActor } from '@ultimat3/core'; +import type { ClaimedJob } from './driver'; +import { memoryJobDriver } from './driver-memory'; +import { executeJob } from './execute'; +import type { AnyJobHandle } from './job'; +import { resetJobs } from './job'; +import type { LedgerEntry, LedgerPosting } from './ledger'; +import { assertBalancedPosting, ledger } from './ledger'; +import { memoryLedgerBook } from './ledger-book'; + +const eur = (minor: number) => ({ minor, currency: 'EUR' }); +const usd = (minor: number) => ({ minor, currency: 'USD' }); + +afterEach(() => { + resetJobs(); +}); + +const run = async (handle: AnyJobHandle, posting: LedgerPosting, maxAttempts = 3) => { + const driver = memoryJobDriver(); + await driver.enqueue({ + name: handle.name, + queue: handle.queue, + input: posting, + idempotencyKey: `${handle.name}:${posting.key}`, + maxAttempts, + }); + const [claimed] = await driver.claim({ + queues: [handle.queue], + limit: 1, + visibilityTimeoutMs: 30_000, + workerId: 'w1', + }); + const ctx = ctxOf({ role: 'worker', actor: userActor({ id: 'worker-1' }) }); + return executeJob({ driver, claimed: claimed as ClaimedJob, handle, ctx }); +}; + +const codeOf = (fn: () => void): string | undefined => { + try { + fn(); + } catch (error: unknown) { + return isUltimateError(error) ? error.code : 'uncoded'; + } + return undefined; +}; + +describe('ledger()', () => { + test('a balanced posting is written once, however often it runs', async () => { + const book = memoryLedgerBook(); + const post = ledger({ name: 'ledger-post', tenant: 'none', book: () => book }); + const posting: LedgerPosting = { + key: 'pay_1', + entries: [ + { account: 'cash', amount: eur(1299) }, + { account: 'revenue', amount: eur(-1099) }, + { account: 'vat', amount: eur(-200) }, + ], + }; + const first = await run(post as AnyJobHandle, posting); + expect(first.outcome).toBe('completed'); + expect(first.result).toEqual({ key: 'pay_1', lines: 3, written: 3 }); + const again = await run(post as AnyJobHandle, posting); + expect(again.result).toEqual({ key: 'pay_1', lines: 3, written: 0 }); + expect(book.lines()).toHaveLength(3); + expect(book.balance('cash', 'EUR')).toBe(1299); + expect(book.balance('vat', 'EUR')).toBe(-200); + }); + + test('a key re-used with other entries is refused, the stored posting untouched', async () => { + const book = memoryLedgerBook(); + const post = ledger({ name: 'ledger-reused', tenant: 'none', book: () => book }); + const first: LedgerPosting = { + key: 'pay-1', + entries: [ + { account: 'cash', amount: eur(1000) }, + { account: 'revenue', amount: eur(-1000) }, + ], + }; + expect((await run(post as AnyJobHandle, first)).outcome).toBe('completed'); + // Balanced on its own, and a superset of the first: only line 2 would be new. + const longer = await run(post as AnyJobHandle, { + key: 'pay-1', + entries: [ + ...first.entries, + { account: 'vat', amount: eur(200) }, + { account: 'cash', amount: eur(-200) }, + ], + }); + expect(longer.code).toBe('X_LEDGER_POSTING_CONFLICT'); + // Same count, another account on one line. + const moved = await run(post as AnyJobHandle, { + key: 'pay-1', + entries: [first.entries[0] as LedgerEntry, { account: 'other', amount: eur(-1000) }], + }); + expect(moved.code).toBe('X_LEDGER_POSTING_CONFLICT'); + expect(book.lines()).toHaveLength(2); + expect(book.balance('cash', 'EUR') + book.balance('revenue', 'EUR')).toBe(0); + }); + + test('an unbalanced posting is refused terminally, nothing written', async () => { + const book = memoryLedgerBook(); + const post = ledger({ name: 'ledger-unbalanced', tenant: 'none', book: () => book }); + const execution = await run(post as AnyJobHandle, { + key: 'pay_2', + entries: [ + { account: 'cash', amount: eur(1000) }, + { account: 'revenue', amount: eur(-999) }, + ], + }); + expect(execution.code).toBe('X_LEDGER_UNBALANCED'); + expect(execution.attempt).toBe(1); + expect(execution.stopReason).toBeDefined(); + expect(book.lines()).toHaveLength(0); + }); + + test('every currency nets to zero on its own: EUR and USD legs balance separately', () => { + const fx = { + key: 'fx_1', + entries: [ + { account: 'eur-cash', amount: eur(-1000) }, + { account: 'fx', amount: eur(1000) }, + { account: 'fx', amount: usd(-1090) }, + { account: 'usd-cash', amount: usd(1090) }, + ], + }; + expect(codeOf(() => assertBalancedPosting(fx))).toBeUndefined(); + // Netting across currencies is exactly what the per-currency sum refuses. + const crossed = { + key: 'fx_2', + entries: [ + { account: 'eur-cash', amount: eur(-1000) }, + { account: 'usd-cash', amount: usd(1000) }, + ], + }; + expect(codeOf(() => assertBalancedPosting(crossed))).toBe('X_LEDGER_UNBALANCED'); + }); + + test.each([ + ['a float minor', { account: 'a', amount: eur(12.99) }], + ['a currency that is not ISO 4217', { account: 'a', amount: { minor: 1, currency: 'eur' } }], + ['an empty account', { account: ' ', amount: eur(1) }], + ['a second scale in one currency', { account: 'a', amount: { ...eur(1), scale: 6 } }], + ])('%s is X_LEDGER_ENTRY_INVALID', (_, entry) => { + const posting = { key: 'p', entries: [{ account: 'b', amount: eur(-1) }, entry] }; + expect(codeOf(() => assertBalancedPosting(posting))).toBe('X_LEDGER_ENTRY_INVALID'); + }); + + test('the input schema states what the posting check refuses, so a drawn sample can pass', async () => { + const post = ledger({ name: 'ledger-schema', tenant: 'none', book: () => memoryLedgerBook() }); + const issues = async (value: unknown) => { + const result = await post.input['~standard'].validate(value); + return 'issues' in result && result.issues !== undefined ? result.issues.length : 0; + }; + const legs = [ + { account: 'cash', amount: eur(1) }, + { account: 'revenue', amount: eur(-1) }, + ]; + expect(await issues({ key: 'p', entries: legs })).toBe(0); + expect(await issues({ key: 'p', entries: legs.slice(0, 1) })).toBeGreaterThan(0); + expect(await issues({ key: '', entries: legs })).toBeGreaterThan(0); + expect( + await issues({ key: 'p', entries: [{ account: '', amount: eur(1) }, legs[1]] }), + ).toBeGreaterThan(0); + }); + + test('a one-legged posting is refused even at zero', () => { + const posting = { key: 'p', entries: [{ account: 'a', amount: eur(0) }] }; + expect(codeOf(() => assertBalancedPosting(posting))).toBe('X_LEDGER_ENTRY_INVALID'); + }); +}); diff --git a/packages/jobs/src/ledger.ts b/packages/jobs/src/ledger.ts new file mode 100644 index 000000000..4abd1bd63 --- /dev/null +++ b/packages/jobs/src/ledger.ts @@ -0,0 +1,145 @@ +// `ledger()`: double-entry postings as a FACTORY over `job()`, never a ninth primitive. A posting +// is balanced Money entries under one key; the job refuses an unbalanced or float posting before a +// line is written, then writes every line in ONE durable step through the app's `LedgerBook` — +// idempotent by `(key, line)`, so a retry, a replay or a second enqueue of one key posts it once. + +import type { MoneyValue, StandardSchemaV1 } from '@ultimat3/schema'; +import { isCurrencyCode, isMoneyScale, t } from '@ultimat3/schema'; +import type { DurationInput } from './clock'; +import type { JobHandle } from './job'; +import { job } from './job'; +import type { LedgerBook, LedgerLine } from './ledger-book'; +import { LedgerEntryInvalidError, LedgerUnbalancedError } from './ledger-errors'; +import type { RetryPolicy } from './retry'; +import { DEFAULT_RETRY } from './retry'; +import type { JobTenant } from './tenant'; + +/** One leg: an account the app names and a signed amount. Debit or credit is the sign's, not ours. */ +export interface LedgerEntry { + readonly account: string; + readonly amount: MoneyValue; +} + +/** What the job is enqueued with. `key` is the posting's identity — a payment id, a transfer id. */ +export interface LedgerPosting { + readonly key: string; + readonly entries: readonly LedgerEntry[]; + /** The tenant the posting belongs to, read by `tenant: (p) => p.orgId ?? …` when the app has one. */ + readonly orgId?: string | undefined; +} + +export interface LedgerDefinition { + /** REQUIRED: a durable key — the queue row and the idempotency key carry it. */ + readonly name: string; + readonly tenant: JobTenant; + /** Where lines land — a thunk, read per run (`entityLedgerBook`, or the app's own). */ + readonly book: () => LedgerBook; + readonly queue?: string; + readonly retry?: RetryPolicy; + readonly timeout?: DurationInput; +} + +/** What one run answers: the lines the posting has, and how many THIS run wrote. */ +export interface LedgerReport { + readonly key: string; + readonly lines: number; + readonly written: number; +} + +/** A posting's legs: at least two, since one entry can only balance at zero, which moves nothing. */ +export const MIN_LEDGER_ENTRIES = 2; + +/** + * Every rule `assertBalancedPosting` can say without adding anything up: a key, at least + * `MIN_LEDGER_ENTRIES` legs, a named account, a Money amount. Stated in the schema so a sample an + * exercise pass or a client draws from it is a posting the run could accept — only the per-currency + * sum, which no schema can express, is left to the run. + */ +const postingSchema: StandardSchemaV1 = t.object({ + key: t.string.min(1), + entries: t.array(t.object({ account: t.string.min(1), amount: t.money }), { + min: MIN_LEDGER_ENTRIES, + }), + orgId: t.string.optional(), +}); + +function assertEntry(key: string, entry: LedgerEntry, index: number): void { + const refuse = (field: string, got: unknown): never => { + throw new LedgerEntryInvalidError({ key, index, field, got }); + }; + if (typeof entry.account !== 'string' || entry.account.trim() === '') { + refuse('account', entry.account); + } + const { minor, currency, scale } = entry.amount; + if (!Number.isSafeInteger(minor)) refuse('amount.minor', minor); + if (!isCurrencyCode(currency)) refuse('amount.currency', currency); + if (scale !== undefined && !isMoneyScale(scale)) refuse('amount.scale', scale); +} + +/** + * Refuse a posting that is not balanced Money, before anything is written. Exported so the action + * that enqueues one refuses it at the request — the job refuses it again, since a queue row is input. + * Within one currency every entry carries the same `scale`: a sum across two scales is not a sum. + */ +export function assertBalancedPosting(posting: LedgerPosting): void { + const { key } = posting; + if (typeof key !== 'string' || key === '') { + throw new LedgerEntryInvalidError({ key: String(key), index: null, field: 'key', got: key }); + } + if (posting.entries.length < MIN_LEDGER_ENTRIES) { + throw new LedgerEntryInvalidError({ + key, + index: null, + field: 'entries.length', + got: posting.entries.length, + }); + } + const sums = new Map(); + posting.entries.forEach((entry, index) => { + assertEntry(key, entry, index); + const { minor, currency, scale } = entry.amount; + const seen = sums.get(currency); + if (seen !== undefined && seen.scale !== scale) { + throw new LedgerEntryInvalidError({ key, index, field: 'amount.scale', got: scale }); + } + const net = (seen?.net ?? 0) + minor; + // Safe integers summed past 2^53 stop being exact: refused rather than rounded. + if (!Number.isSafeInteger(net)) { + throw new LedgerEntryInvalidError({ key, index, field: 'amount.minor', got: minor }); + } + sums.set(currency, { net, scale }); + }); + for (const [currency, { net }] of [...sums].sort(([a], [b]) => (a < b ? -1 : 1))) { + if (net !== 0) throw new LedgerUnbalancedError({ key, currency, net }); + } +} + +/** The posting as the lines a book stores: one per entry, numbered from 0 in the posting's order. */ +export const ledgerLines = (posting: LedgerPosting): readonly LedgerLine[] => + posting.entries.map((entry, line) => ({ + ...(posting.orgId === undefined ? {} : { tenant: posting.orgId }), + postingKey: posting.key, + line, + account: entry.account, + amount: entry.amount, + })); + +export function ledger(definition: LedgerDefinition): JobHandle { + return job({ + name: definition.name, + input: postingSchema, + tenant: definition.tenant, + // One live run per posting; the book's `(postingKey, line)` holds it after the run is gone. + idempotencyKey: (posting) => `${definition.name}:${posting.key}`, + retry: definition.retry ?? DEFAULT_RETRY, + ...(definition.queue === undefined ? {} : { queue: definition.queue }), + ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }), + run: async ({ step, input }): Promise => { + assertBalancedPosting(input); + const lines = ledgerLines(input); + // ONE step: every line or none, and a replay answers the persisted count without a write. + const written = await step.run('post', () => definition.book().write(lines)); + return { key: input.key, lines: lines.length, written }; + }, + }); +} diff --git a/packages/jobs/src/lock-errors.ts b/packages/jobs/src/lock-errors.ts new file mode 100644 index 000000000..cec7eb113 --- /dev/null +++ b/packages/jobs/src/lock-errors.ts @@ -0,0 +1,54 @@ +// The three `X_LOCK_*` refusals `withLock()` ends on: not taken within `wait`, lost mid-work, and +// nothing to hold it in. Codes, titles and retry classes are `error-codes.ts`'s; the key is printed +// JSON-quoted, since an app builds it from its own data (`invoice:${id}`). + +import { UltimateError } from '@ultimat3/core'; +import './error-codes'; + +/** Another holder kept the key for the whole `wait` bound. Retryable: contention ends. */ +export class LockNotAcquiredError extends UltimateError { + constructor(input: { key: string; waitMs: number; holders: readonly string[] }) { + super({ + code: 'X_LOCK_NOT_ACQUIRED', + cause: `withLock(${JSON.stringify(input.key)}) waited ${input.waitMs}ms and another holder (${input.holders.length === 0 ? 'released a moment ago' : input.holders.join(', ')}) kept the key throughout`, + fix: `withLock(${JSON.stringify(input.key)}, fn, { ttl, wait: '${Math.max(1, Math.ceil((input.waitMs * 2) / 1000))}s' }) # in the calling file: a longer wait:, or let the caller retry`, + meta: { key: input.key, waitMs: input.waitMs, holders: [...input.holders] }, + }); + } +} + +/** + * A renewal was refused, or none landed within one `ttl`: the key may be someone else's now. The + * reason `fn`'s signal aborts with — a write fenced on the token is what stops the stale holder. + */ +export class LockLostError extends UltimateError { + constructor(input: { key: string; token: number; ttlMs: number }) { + super({ + code: 'X_LOCK_LOST', + cause: `withLock(${JSON.stringify(input.key)}) lost the lock held as ${input.token}: no renewal landed within its ttl of ${input.ttlMs}ms, so another holder may have it`, + fix: `withLock(${JSON.stringify(input.key)}, fn, { ttl: '${Math.ceil((input.ttlMs * 4) / 1000)}s' }) # in the calling file: a longer ttl:, and fn hands lock.signal to every write it makes`, + meta: { key: input.key, token: input.token, ttlMs: input.ttlMs }, + }); + } +} + +/** + * No job driver installed, one without a `leases` store, or a store that answers no `fence`: no + * fleet-wide place to hold a key behind a token the database assigned. + */ +export class LockStoreMissingError extends UltimateError { + constructor(input: { key: string; driver: string | undefined; fence?: boolean }) { + const why = + input.fence === true + ? 'the lease store granted the key with no fence, so there is no token a write can be refused by' + : input.driver === undefined + ? 'no job driver is installed' + : `the installed job driver "${input.driver}" has no leases`; + super({ + code: 'X_LOCK_STORE_MISSING', + cause: `withLock(${JSON.stringify(input.key)}) has no lease store to lock in: ${why}`, + fix: 'jobs: postgresJobDriver() # in apps//runtime.ts — or withLock(key, fn, { ttl, store }) in the calling file', + meta: { key: input.key, driver: input.driver ?? null }, + }); + } +} diff --git a/packages/jobs/src/lock.live.test.ts b/packages/jobs/src/lock.live.test.ts new file mode 100644 index 000000000..ca8ceb32f --- /dev/null +++ b/packages/jobs/src/lock.live.test.ts @@ -0,0 +1,144 @@ +// `withLock()` over `postgresJobDriver`'s lease store on a real server: two drivers (two pods) race +// one key and exactly one runs at a time; a holder whose TTL lapses hands the key +// over (the stale holder learning of it is `lock.test.ts`). Skips unless `TEST_DATABASE_URL`. + +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import type { PgExecutor } from '@ultimat3/core'; +import { isUltimateError, probeDatabaseName, sweepProbeDatabases } from '@ultimat3/core'; +import type { PostgresClient } from '@ultimat3/db'; +import { dbExecutor, postgresClient, raw } from '@ultimat3/db'; +import { postgresJobDriver } from './driver-pg'; +import { SQL_JOBS_TABLE } from './driver-pg-sql'; +import type { LeaseStore } from './leases'; +import { lockLeaseKey, withLock } from './lock'; + +const url = Bun.env['TEST_DATABASE_URL']; +const hasPostgres = typeof url === 'string' && url.length > 0; +const PROBE_DB = probeDatabaseName('x_lock_live'); + +const probeUrl = (): string => { + const parsed = new URL(url ?? 'postgres://localhost/postgres'); + parsed.pathname = `/${PROBE_DB}`; + return parsed.toString(); +}; + +/** Event-loop turns until `condition` holds — bounded, so a lease that never lapses fails. */ +async function until(condition: () => boolean | Promise, budgetMs = 5_000): Promise { + const deadline = performance.now() + budgetMs; + while (!(await condition())) { + if (performance.now() > deadline) expect.unreachable('the condition never became true'); + await new Promise((done) => setImmediate(done)); + } +} + +describe.skipIf(!hasPostgres)('live · postgres · withLock()', () => { + let admin: PostgresClient; + const clients: PostgresClient[] = []; + const stores: LeaseStore[] = []; + + beforeAll(async () => { + admin = postgresClient({ url: url ?? '', role: 'web', profile: { max: 1 } }); + await admin.execute(raw(`drop database if exists ${PROBE_DB} with (force)`)); + await sweepProbeDatabases( + dbExecutor(() => admin), + PROBE_DB, + ); + await admin.execute(raw(`create database ${PROBE_DB}`)); + // Two pools, two drivers: what two pods are. + for (let pod = 0; pod < 2; pod += 1) { + const client = postgresClient({ url: probeUrl(), role: 'web', profile: { max: 2 } }); + clients.push(client); + const executor: PgExecutor = { + query: (text: string, values: readonly unknown[]): Promise => + client.query({ text, values }), + }; + const leases = postgresJobDriver({ executor }).leases; + expect(leases).toBeDefined(); + stores.push(leases as LeaseStore); + } + for (const statement of SQL_JOBS_TABLE.split(';')) { + if (statement.trim().length > 0) await clients[0]?.execute(raw(statement)); + } + }); + + afterAll(async () => { + for (const client of clients) await client.close(); + await admin.execute(raw(`drop database if exists ${PROBE_DB} with (force)`)); + await admin.close(); + }); + + const store = (pod: number): LeaseStore => stores[pod] as LeaseStore; + + test('two pods race one key: never two inside at once, both finish', async () => { + let inside = 0; + let most = 0; + let entered = 0; + let refused = 0; + // Counts a refused acquire: the evidence a holder waits for, instead of a fixed delay. + const counting = (pod: number): LeaseStore => { + const inner = store(pod); + return { + ...inner, + async acquire(...args: Parameters) { + const lease = await inner.acquire(...args); + if (lease === undefined) refused += 1; + return lease; + }, + }; + }; + const ATTEMPTS = 4; + const tokens: number[] = []; + const work = async ({ token }: { readonly token: number }): Promise => { + tokens.push(token); + inside += 1; + entered += 1; + most = Math.max(most, inside); + // Hold the key until another pod has been REFUSED it — the last holder has nobody to wait for. + const seen = refused; + if (entered < ATTEMPTS) await until(() => refused > seen); + inside -= 1; + }; + await Promise.all( + [0, 1, 0, 1].map((pod) => + withLock('live:race', work, { ttl: '5s', wait: '5s', poll: 5, store: counting(pod) }), + ), + ); + expect(most).toBe(1); + expect(entered).toBe(ATTEMPTS); + // The server's fence: each holder's token beats the one before, whichever pod it ran on. + expect(tokens).toEqual([...tokens].sort((a, b) => a - b)); + expect(new Set(tokens).size).toBe(ATTEMPTS); + }); + + test('a lapsed TTL hands the key to the other pod, refused before it lapses', async () => { + let release = (): void => {}; + const gate = new Promise((done) => { + release = done; + }); + // No renewal ever fires: the holder is a paused process. + const stale = withLock('live:ttl', () => gate, { + ttl: 300, + store: store(0), + schedule: () => () => {}, + }).catch((error: unknown) => error); + // The stale holder HOLDS it (asked of the server), not "has probably acquired by now". + await until(async () => (await store(1).holders(lockLeaseKey('live:ttl'))).length === 1); + const refused = await withLock('live:ttl', () => 'early', { + ttl: 300, + wait: 0, + store: store(1), + }).catch((error: unknown) => error); + expect(isUltimateError(refused) && refused.code).toBe('X_LOCK_NOT_ACQUIRED'); + // The lapse is on the SERVER's clock: poll the other pod's one-shot acquire until it lands. + let next: unknown; + await until(async () => { + next = await withLock('live:ttl', () => 'next', { ttl: 300, wait: 0, store: store(1) }).catch( + (error: unknown) => error, + ); + return next === 'next'; + }); + expect(next).toBe('next'); + release(); + await stale; + }); +}); diff --git a/packages/jobs/src/lock.test.ts b/packages/jobs/src/lock.test.ts new file mode 100644 index 000000000..83098baea --- /dev/null +++ b/packages/jobs/src/lock.test.ts @@ -0,0 +1,357 @@ +// `withLock()` on the memory lease store and a frozen clock: one winner, a bounded wait that +// refuses with the holder named, a TTL expiry that hands the key over and aborts the stale holder. + +import { afterEach, describe, expect, test } from 'bun:test'; +import { frozenClock, isUltimateError } from '@ultimat3/core'; +import { installJobDriver, resetJobDriver } from './driver'; +import { memoryJobDriver } from './driver-memory'; +import { memoryLeaseStore } from './leases'; +import type { HeldLock } from './lock'; +import { lockLeaseKey, withLock } from './lock'; +import type { IntervalScheduler } from './renewal-timer'; + +const codeOf = (error: unknown): string | undefined => + isUltimateError(error) ? error.code : undefined; + +/** A scheduler a test fires by hand: the renewal happens when the test says, never on the wall. */ +function manualSchedule(): { schedule: IntervalScheduler; tick(): Promise } { + const ticks: (() => void)[] = []; + return { + schedule: (tick) => { + ticks.push(tick); + return () => { + ticks.splice(ticks.indexOf(tick), 1); + }; + }, + async tick() { + for (const tick of [...ticks]) tick(); + // The renewal body is a promise chain: one macrotask lets every link of it settle. + await settle(); + }, + }; +} + +/** One macrotask: an acquire, its timer arming and a renewal chain all settle inside it. */ +const settle = (): Promise => new Promise((done) => setImmediate(done)); + +const deferred = (): { promise: Promise; resolve(): void } => { + let resolve = (): void => {}; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +}; + +afterEach(() => { + resetJobDriver(); +}); + +describe('withLock', () => { + test('two holders: the second is refused within wait, naming the first', async () => { + const clock = frozenClock('2026-11-09T00:00:00Z'); + const store = memoryLeaseStore({ clock }); + const gate = deferred(); + let first: HeldLock | undefined; + const holding = withLock( + 'invoice:1', + async (lock) => { + first = lock; + await gate.promise; + return 'a'; + }, + { ttl: '1h', store, clock, schedule: manualSchedule().schedule }, + ); + await settle(); + const refused = await withLock('invoice:1', () => 'b', { + ttl: '1h', + wait: 500, + store, + clock, + }).catch((error: unknown) => error); + expect(codeOf(refused)).toBe('X_LOCK_NOT_ACQUIRED'); + expect(isUltimateError(refused) && refused.meta?.['holders']).toHaveLength(1); + expect(typeof first?.token).toBe('number'); + gate.resolve(); + expect(await holding).toBe('a'); + expect(await store.held(lockLeaseKey('invoice:1'))).toBe(0); + }); + + test('a waiter takes the key once the holder releases, with a greater token', async () => { + const clock = frozenClock('2026-11-09T00:00:00Z'); + const store = memoryLeaseStore({ clock }); + const gate = deferred(); + let firstToken = 0; + const holding = withLock( + 'k', + async (lock) => { + firstToken = lock.token; + await gate.promise; + }, + { ttl: '1h', store, clock, schedule: manualSchedule().schedule }, + ); + await settle(); + // The frozen clock's sleep advances it, so release the holder on the waiter's first poll. + const waiting = withLock('k', (lock) => lock.token, { ttl: '1h', wait: '1m', store, clock }); + gate.resolve(); + await holding; + const secondToken = await waiting; + expect(secondToken > firstToken).toBe(true); + }); + + test('a TTL expiry hands the key over; the stale holder is aborted with X_LOCK_LOST', async () => { + const clock = frozenClock('2026-11-09T00:00:00Z'); + const store = memoryLeaseStore({ clock }); + const renewals = manualSchedule(); + const gate = deferred(); + let seen: unknown; + const stale = withLock( + 'k', + async (lock) => { + await gate.promise; + seen = lock.signal.reason; + return 'stale'; + }, + { ttl: 1_000, store, clock, schedule: renewals.schedule }, + ).catch((error: unknown) => error); + await settle(); + clock.advance(1_500); + expect(await withLock('k', () => 'next', { ttl: 1_000, wait: 0, store, clock })).toBe('next'); + // Re-take it so the stale holder's renewal meets a key that is somebody else's. + const third = withLock('k', () => gate.promise, { ttl: '1h', wait: 0, store, clock }); + await settle(); + await renewals.tick(); + gate.resolve(); + await third; + expect(codeOf(seen)).toBe('X_LOCK_LOST'); + expect(codeOf(await stale)).toBe('X_LOCK_LOST'); + }); + + test('a renewal that lands keeps the key past one ttl', async () => { + const clock = frozenClock('2026-11-09T00:00:00Z'); + const store = memoryLeaseStore({ clock }); + const renewals = manualSchedule(); + const gate = deferred(); + const holding = withLock('k', () => gate.promise, { + ttl: 900, + store, + clock, + schedule: renewals.schedule, + }); + await settle(); + for (let round = 0; round < 4; round += 1) { + clock.advance(300); + await renewals.tick(); + } + const refused = await withLock('k', () => 'b', { ttl: 900, wait: 0, store, clock }).catch( + (error: unknown) => error, + ); + expect(codeOf(refused)).toBe('X_LOCK_NOT_ACQUIRED'); + gate.resolve(); + await holding; + }); + + test('a renewal that throws is a loss only once a whole ttl passed without one landing', async () => { + const clock = frozenClock('2026-11-09T00:00:00Z'); + const inner = memoryLeaseStore({ clock }); + let failing = false; + const store = { + ...inner, + renew: (lease: Parameters[0], ttlMs: number) => + failing ? Promise.reject(new TypeError('pool closed')) : inner.renew(lease, ttlMs), + }; + const renewals = manualSchedule(); + const gate = deferred(); + let signal: AbortSignal | undefined; + const holding = withLock( + 'k', + async (lock) => { + signal = lock.signal; + await gate.promise; + }, + { ttl: 900, store, clock, schedule: renewals.schedule }, + ).catch((error: unknown) => error); + await settle(); + clock.advance(600); + await renewals.tick(); + failing = true; + clock.advance(600); + await renewals.tick(); + expect(signal?.aborted).toBe(false); + clock.advance(400); + await renewals.tick(); + expect(codeOf(signal?.reason)).toBe('X_LOCK_LOST'); + gate.resolve(); + expect(codeOf(await holding)).toBe('X_LOCK_LOST'); + }); + + test('an abort landing during the final acquire reaches fn’s signal', async () => { + const inner = memoryLeaseStore(); + const outer = new AbortController(); + const reason = new TypeError('caller gave up'); + // The acquire that SUCCEEDS is in flight when the caller aborts. + const store = { + ...inner, + acquire: async (...args: Parameters) => { + const lease = await inner.acquire(...args); + outer.abort(reason); + return lease; + }, + }; + let seen: AbortSignal | undefined; + await withLock( + 'k', + (lock) => { + seen = lock.signal; + }, + { ttl: '1m', store, signal: outer.signal, schedule: manualSchedule().schedule }, + ); + expect(seen?.aborted).toBe(true); + expect(seen?.reason).toBe(reason); + }); + + test('a renewal that never settles is a loss once the ttl lapses — decided before the call', async () => { + const clock = frozenClock('2026-11-09T00:00:00Z'); + const inner = memoryLeaseStore({ clock }); + let calls = 0; + // A dead socket: the renew is sent and never answers, neither way. + const store = { + ...inner, + renew: () => { + calls += 1; + return new Promise(() => {}); + }, + }; + const renewals = manualSchedule(); + const gate = deferred(); + let signal: AbortSignal | undefined; + const holding = withLock( + 'k', + async (lock) => { + signal = lock.signal; + await gate.promise; + }, + { ttl: 900, store, clock, schedule: renewals.schedule }, + ).catch((error: unknown) => error); + await settle(); + clock.advance(600); + await renewals.tick(); + clock.advance(600); + await renewals.tick(); + expect(codeOf(signal?.reason)).toBe('X_LOCK_LOST'); + // One renewal in flight at a time: the hung one is not joined by a second. + expect(calls).toBe(1); + gate.resolve(); + expect(codeOf(await holding)).toBe('X_LOCK_LOST'); + }); + + test('a renewal answering after the ttl lapsed is a loss, though it says true', async () => { + const clock = frozenClock('2026-11-09T00:00:00Z'); + const inner = memoryLeaseStore({ clock }); + let answer = (_held: boolean): void => {}; + const store = { + ...inner, + renew: () => + new Promise((done) => { + answer = done; + }), + }; + const renewals = manualSchedule(); + const gate = deferred(); + let signal: AbortSignal | undefined; + const holding = withLock( + 'k', + async (lock) => { + signal = lock.signal; + await gate.promise; + }, + { ttl: 900, store, clock, schedule: renewals.schedule }, + ).catch((error: unknown) => error); + await settle(); + clock.advance(300); + await renewals.tick(); + // Sent at 300, answered at 1,300: the store dated the expiry at 300 + 900 = 1,200. + clock.advance(1_000); + answer(true); + await settle(); + expect(codeOf(signal?.reason)).toBe('X_LOCK_LOST'); + gate.resolve(); + expect(codeOf(await holding)).toBe('X_LOCK_LOST'); + }); + + test('a throw between the acquire and fn hands the lease back', async () => { + const store = memoryLeaseStore(); + const boom = new TypeError('schedule refused'); + const thrown = await withLock('k', () => 1, { + ttl: '1m', + store, + // INPUT to the code under test: a renewal scheduler that throws when armed. + schedule: () => { + throw boom; + }, + }).catch((error: unknown) => error); + expect(thrown).toBe(boom); + expect(await store.held(lockLeaseKey('k'))).toBe(0); + }); + + test('fn throwing releases the key and rethrows', async () => { + const store = memoryLeaseStore(); + // INPUT to the code under test: the failure an app's body rejects with. + const boom = new TypeError('boom'); + const thrown = await withLock('k', () => Promise.reject(boom), { ttl: '1m', store }).catch( + (error: unknown) => error, + ); + expect(thrown).toBe(boom); + expect(await store.held(lockLeaseKey('k'))).toBe(0); + }); + + test('defaults to the installed driver’s lease store, and refuses with none', async () => { + const refused = await withLock('k', () => 1, { ttl: '1m' }).catch((error: unknown) => error); + expect(codeOf(refused)).toBe('X_LOCK_STORE_MISSING'); + installJobDriver(memoryJobDriver()); + expect(await withLock('k', () => 1, { ttl: '1m' })).toBe(1); + }); + + test('a ttl under 3ms is refused before anything is acquired', async () => { + const store = memoryLeaseStore(); + const refused = await withLock('k', () => 1, { ttl: 1, store }).catch((e: unknown) => e); + expect(codeOf(refused)).toBe('X_INVARIANT'); + expect(await store.held(lockLeaseKey('k'))).toBe(0); + }); + + test('the token is the STORE’s fence, never this process’s clock', async () => { + const store = memoryLeaseStore(); + // A clock far in the past must not make a later holder's token smaller. + const behind = frozenClock('2000-01-01T00:00:00Z'); + const first = await withLock('k', (lock) => lock.token, { ttl: '1m', store }); + const second = await withLock('k', (lock) => lock.token, { ttl: '1m', store, clock: behind }); + expect(second).toBeGreaterThan(first); + }); + + test('a store that answers no fence is refused, its lease handed back', async () => { + const inner = memoryLeaseStore(); + const fenceless = { + ...inner, + acquire: async (...args: Parameters) => { + const lease = await inner.acquire(...args); + return lease === undefined + ? undefined + : { key: lease.key, slot: lease.slot, holder: lease.holder }; + }, + }; + let ran = false; + const refused = await withLock( + 'k', + () => { + ran = true; + }, + { ttl: '1m', store: fenceless }, + ).catch((error: unknown) => error); + expect(codeOf(refused)).toBe('X_LOCK_STORE_MISSING'); + expect(ran).toBe(false); + expect(await inner.held(lockLeaseKey('k'))).toBe(0); + }); + + test('a lock never shares a slot table with a job cap', () => { + expect(lockLeaseKey('job:a')).toBe('lock:job%3Aa'); + }); +}); diff --git a/packages/jobs/src/lock.ts b/packages/jobs/src/lock.ts new file mode 100644 index 000000000..0bec32f48 --- /dev/null +++ b/packages/jobs/src/lock.ts @@ -0,0 +1,176 @@ +// `withLock(key, fn, { ttl })`: one holder fleet-wide for `fn`, held in the job driver's +// `LeaseStore` (one slot, `lock:`) — the same rows `job.concurrency` counts, so a lock needs +// no table, no Redis and no second mechanism. Not a primitive: a helper a job or action body calls. +// The holder renews at a third of `ttl`; a lost lease aborts `fn`'s signal with `X_LOCK_LOST`. + +import type { SleepClock } from '@ultimat3/core'; +import { ambientClock, assert, logger, renderThrowable, uuidV7 } from '@ultimat3/core'; +import type { DurationInput } from './clock'; +import { finiteDurationMs, nowMs } from './clock'; +import { jobDriver } from './driver'; +import type { HeldLease, LeaseStore } from './leases'; +import { LockLostError, LockNotAcquiredError, LockStoreMissingError } from './lock-errors'; +import type { IntervalScheduler, RenewalTimer } from './renewal-timer'; +import { startRenewalTimer } from './renewal-timer'; + +/** Between two acquire attempts while another holder has the key. Never above `wait`'s remainder. */ +export const DEFAULT_LOCK_POLL_MS = 100; + +export interface WithLockOptions { + /** REQUIRED. How long the key stays held if this process dies; renewed while `fn` runs. */ + readonly ttl: DurationInput; + /** How long to keep trying before `X_LOCK_NOT_ACQUIRED`. Default: one `ttl`. `0`: one attempt. */ + readonly wait?: DurationInput; + /** Between attempts. Default `DEFAULT_LOCK_POLL_MS`. */ + readonly poll?: DurationInput; + /** Default: the installed job driver's `leases` (`postgresJobDriver` over `x_job_leases`). */ + readonly store?: LeaseStore; + /** Default `ambientClock()`: the wait, the expiry bookkeeping and the token's timestamp. */ + readonly clock?: SleepClock; + /** Gives up the WAIT (rejects with its reason); once held, it aborts `fn`'s signal too. */ + readonly signal?: AbortSignal; + /** The renewal interval's seam — `renewal-timer.ts`'s, so a test renews when it says so. */ + readonly schedule?: IntervalScheduler; +} + +/** What `fn` holds: the key, the fencing token and the signal a lost lease aborts. */ +export interface HeldLock { + readonly key: string; + /** + * The FENCING token: the store's number for this grant (`HeldLease.fence`), greater than every + * token any earlier holder of any key got — the database's counter, never a clock. Write it + * beside whatever `fn` changes and refuse a write carrying a smaller one: that is what stops a + * holder that paused past its `ttl` (GC, a frozen VM) from writing after the next one took over. + */ + readonly token: number; + readonly signal: AbortSignal; +} + +/** The lease key: its own prefix, so no lock ever shares a slot table with a job's cap. */ +export const lockLeaseKey = (key: string): string => `lock:${encodeURIComponent(key)}`; + +const storeFor = (key: string, options: WithLockOptions): LeaseStore => { + if (options.store !== undefined) return options.store; + const driver = jobDriver(); + if (driver?.leases === undefined) throw new LockStoreMissingError({ key, driver: driver?.name }); + return driver.leases; +}; + +async function acquire( + store: LeaseStore, + key: string, + token: string, + bounds: { ttlMs: number; waitMs: number; pollMs: number }, + clock: SleepClock, + signal: AbortSignal | undefined, + sent: (atMs: number) => void, +): Promise { + const leaseKey = lockLeaseKey(key); + const deadline = nowMs(clock) + bounds.waitMs; + for (;;) { + signal?.throwIfAborted(); + const sentAt = nowMs(clock); + const lease = await store.acquire(leaseKey, 1, bounds.ttlMs, token); + if (lease !== undefined) { + sent(sentAt); + return lease; + } + const left = deadline - nowMs(clock); + if (left <= 0) { + const holders = await store.holders(leaseKey); + throw new LockNotAcquiredError({ key, waitMs: bounds.waitMs, holders }); + } + await clock.sleep(Math.min(bounds.pollMs, left), signal); + } +} + +/** + * Run `fn` while this process alone holds `key` across the fleet, then release it. Refuses with + * `X_LOCK_NOT_ACQUIRED` when `wait` passes first. If the lease is lost mid-run, `fn`'s signal + * aborts and the call rejects with `X_LOCK_LOST` even when `fn` resolved: its work was not exclusive. + */ +export async function withLock( + key: string, + fn: (lock: HeldLock) => Promise | T, + options: WithLockOptions, +): Promise { + const ttlMs = finiteDurationMs(options.ttl, 'withLock', 'ttl'); + assert( + Number.isSafeInteger(ttlMs) && ttlMs >= 3, + `withLock(${JSON.stringify(key)}) declares ttl: ${String(options.ttl)} — renewed at a third of it, so at least 3ms`, + `withLock(${JSON.stringify(key)}, fn, { ttl: '30s' }) # in the calling file`, + ); + const waitMs = Math.max(0, finiteDurationMs(options.wait ?? ttlMs, 'withLock', 'wait')); + const pollMs = Math.max( + 1, + finiteDurationMs(options.poll ?? DEFAULT_LOCK_POLL_MS, 'withLock', 'poll'), + ); + const clock = options.clock ?? ambientClock(); + const store = storeFor(key, options); + const holder = uuidV7(clock); + // Stamped BEFORE the acquire that won: the store dates the expiry at its statement. + let renewedAt = nowMs(clock); + const lease = await acquire( + store, + key, + holder, + { ttlMs, waitMs, pollMs }, + clock, + options.signal, + (sentAt) => { + renewedAt = sentAt; + }, + ); + + const controller = new AbortController(); + const onOuterAbort = (): void => controller.abort(options.signal?.reason); + let lost: LockLostError | undefined; + let timer: RenewalTimer | undefined; + // Everything after the acquire is inside the `try`: a throw anywhere still hands the lease back. + try { + const token = lease.fence; + if (token === undefined) + throw new LockStoreMissingError({ key, driver: undefined, fence: true }); + options.signal?.addEventListener('abort', onOuterAbort, { once: true }); + // An abort that landed while the acquire that SUCCEEDED was in flight fired no listener. + if (options.signal?.aborted === true) controller.abort(options.signal.reason); + const lapsed = (): boolean => nowMs(clock) - renewedAt >= ttlMs; + const lose = (): void => { + lost ??= new LockLostError({ key, token, ttlMs }); + if (!controller.signal.aborted) controller.abort(lost); + }; + let renewing = false; + // `worker-fleet-slots.ts`'s renewal: a renew hung on a dead socket never settles, so the + // lapse is decided BEFORE the call, one renewal is in flight at a time, and a reply that + // lands after the expiry it would have extended is still a loss. + const renew = async (): Promise => { + if (lost !== undefined || timer?.stopped() === true) return; + if (lapsed()) return lose(); + if (renewing) return; + renewing = true; + const sentAt = nowMs(clock); + try { + const held = await store.renew(lease, ttlMs); + if (timer?.stopped() === true) return; + if (held === false || lapsed()) return lose(); + renewedAt = sentAt; + } catch (error: unknown) { + logger.warn('jobs.lock.renew_failed', { key, error: renderThrowable(error) }); + if (timer?.stopped() !== true && lapsed()) lose(); + } finally { + renewing = false; + } + }; + timer = startRenewalTimer(Math.max(1, Math.floor(ttlMs / 3)), renew, options.schedule); + const result = await fn({ key, token, signal: controller.signal }); + if (lost !== undefined) throw lost; + return result; + } finally { + timer?.stop(); + options.signal?.removeEventListener('abort', onOuterAbort); + // Released only by this holder (the store checks `holder`); a failure leaves it to the TTL. + await store.release(lease).catch((error: unknown) => { + logger.warn('jobs.lock.release_failed', { key, error: renderThrowable(error) }); + }); + } +} diff --git a/packages/jobs/src/partitions.live.test.ts b/packages/jobs/src/partitions.live.test.ts new file mode 100644 index 000000000..35aba43f1 --- /dev/null +++ b/packages/jobs/src/partitions.live.test.ts @@ -0,0 +1,166 @@ +// `partitions()` on a real server: the migration makes this month and next; the job, its clock moved +// a month at a time, makes the next one ahead each time and drops the partition `retain` expired — +// and a second run at the same instant changes nothing. Skips unless `TEST_DATABASE_URL`. + +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { + ctxOf, + frozenClock, + probeDatabaseName, + sweepProbeDatabases, + userActor, +} from '@ultimat3/core'; +import type { PostgresClient } from '@ultimat3/db'; +import { + dbExecutor, + generateMigration, + postgresClient, + raw, + setDbClient, + statementsOf, +} from '@ultimat3/db'; +import { clearRegistry, entity, text, timestamp, uuid } from '@ultimat3/entity'; +import type { ClaimedJob } from './driver'; +import { memoryJobDriver } from './driver-memory'; +import { executeJob } from './execute'; +import type { AnyJobHandle } from './job'; +import { resetJobs } from './job'; +import { partitions } from './partitions'; +import { resetTasks } from './task'; + +const url = Bun.env['TEST_DATABASE_URL']; +const hasPostgres = typeof url === 'string' && url.length > 0; +const PROBE_DB = probeDatabaseName('x_partitions_live'); +const DAY_MS = 86_400_000; + +const events = entity('pl_events', { + columns: { id: uuid(), at: timestamp(), kind: text() }, + primaryKey: ['id', 'at'], + partitionBy: { column: 'at', interval: 'month', ahead: 1, retain: { after: '20d' } }, +}); + +/** The server's now: the migration judged its first partitions by it, not by the test's clock. */ +let serverNow = new Date(0); + +/** The first instant of the UTC month `offset` months from the server's (the migration's) now. */ +const monthStart = (offset: number): Date => + new Date(Date.UTC(serverNow.getUTCFullYear(), serverNow.getUTCMonth() + offset, 1)); +const suffix = (at: Date): string => + `pl_events_p${at.toISOString().slice(0, 10).replaceAll('-', '')}`; + +const probeUrl = (): string => { + const parsed = new URL(url ?? 'postgres://localhost/postgres'); + parsed.pathname = `/${PROBE_DB}`; + return parsed.toString(); +}; + +afterAll(() => { + clearRegistry(); + resetJobs(); + resetTasks(); +}); + +describe.skipIf(!hasPostgres)('live · postgres · partitions()', () => { + let admin: PostgresClient; + let client: PostgresClient; + + beforeAll(async () => { + admin = postgresClient({ url: url ?? '', role: 'web', profile: { max: 1 } }); + await admin.execute(raw(`drop database if exists ${PROBE_DB} with (force)`)); + await sweepProbeDatabases( + dbExecutor(() => admin), + PROBE_DB, + ); + await admin.execute(raw(`create database ${PROBE_DB}`)); + client = postgresClient({ url: probeUrl(), role: 'web', profile: { max: 2 } }); + setDbClient(client); + const [row] = await client.query<{ ms: string }>( + raw('select (extract(epoch from now()) * 1000)::bigint::text as ms'), + ); + serverNow = new Date(Number(row?.ms)); + const { up } = generateMigration({ entities: [events.$describe()], name: 'partitioned' }); + for (const statement of statementsOf(up)) await client.execute(raw(statement)); + }); + + afterAll(async () => { + setDbClient(undefined); + await client.close(); + await admin.execute(raw(`drop database if exists ${PROBE_DB} with (force)`)); + await admin.close(); + }); + + const children = async (): Promise => + ( + await client.query<{ name: string }>( + raw( + "select c.relname::text as name from pg_inherits h join pg_class c on c.oid = h.inhrelid where h.inhparent = 'pl_events'::regclass order by 1", + ), + ) + ).map((row) => row.name); + + test('the clock moving a month at a time makes the next partition and drops the expired', async () => { + const clock = frozenClock(new Date(monthStart(0).getTime() + 14 * DAY_MS)); + const maintain = partitions({ name: 'pl.partitions', clock }); + expect(maintain.schedule.name).toBe('pl.partitions.schedule'); + const run = async (input: { readonly from?: string } = {}) => { + const driver = memoryJobDriver(); + await driver.enqueue({ + name: maintain.name, + queue: maintain.queue, + input, + idempotencyKey: maintain.name, + maxAttempts: 1, + }); + const [claimed] = await driver.claim({ + queues: [maintain.queue], + limit: 1, + visibilityTimeoutMs: 30_000, + workerId: 'w1', + }); + const ctx = ctxOf({ role: 'worker', actor: userActor({ id: 'worker-1' }) }); + const execution = await executeJob({ + driver, + claimed: claimed as ClaimedJob, + handle: maintain as AnyJobHandle, + ctx, + }); + expect(execution.outcome).toBe('completed'); + return execution.result; + }; + + // The migration already made this month and the next: nothing to do. + expect(await run()).toEqual({ tables: [{ table: 'pl_events', created: [], dropped: [] }] }); + expect(await children()).toEqual([suffix(monthStart(0)), suffix(monthStart(1))]); + + clock.advance(31 * DAY_MS); + expect(await run()).toEqual({ + tables: [{ table: 'pl_events', created: [suffix(monthStart(2))], dropped: [] }], + }); + + clock.advance(31 * DAY_MS); + expect(await run()).toEqual({ + tables: [ + { table: 'pl_events', created: [suffix(monthStart(3))], dropped: [suffix(monthStart(0))] }, + ], + }); + expect(await run()).toEqual({ tables: [{ table: 'pl_events', created: [], dropped: [] }] }); + expect(await children()).toEqual([ + suffix(monthStart(1)), + suffix(monthStart(2)), + suffix(monthStart(3)), + ]); + + // Before a backfill copies old rows in: `from` creates every partition back to the oldest + // row's and sweeps nothing, so the month retention just dropped comes back for the copy. + const oldest = new Date(monthStart(-2).getTime() + 9 * DAY_MS); + expect(await run({ from: oldest.toISOString() })).toEqual({ + tables: [ + { + table: 'pl_events', + created: [suffix(monthStart(-2)), suffix(monthStart(-1)), suffix(monthStart(0))], + dropped: [], + }, + ], + }); + }); +}); diff --git a/packages/jobs/src/partitions.test.ts b/packages/jobs/src/partitions.test.ts new file mode 100644 index 000000000..33b9aee7a --- /dev/null +++ b/packages/jobs/src/partitions.test.ts @@ -0,0 +1,60 @@ +// `partitions()` as declared: a job plus its `.schedule` task, daily in UTC unless told otherwise, +// and a run with no partitioned entity registered touches no database at all. + +import { afterEach, describe, expect, test } from 'bun:test'; +import { ctxOf, userActor } from '@ultimat3/core'; +import { clearRegistry, entity, uuid } from '@ultimat3/entity'; +import type { ClaimedJob } from './driver'; +import { memoryJobDriver } from './driver-memory'; +import { executeJob } from './execute'; +import type { AnyJobHandle } from './job'; +import { getJob, resetJobs } from './job'; +import { DEFAULT_PARTITIONS_CRON, partitionedEntities, partitions } from './partitions'; +import { getTask, resetTasks } from './task'; + +afterEach(() => { + resetJobs(); + resetTasks(); + clearRegistry(); +}); + +describe('partitions()', () => { + test('a job and its daily UTC schedule, both registered under the declared name', () => { + const maintain = partitions(); + expect(getJob('partitions')).toBe(maintain); + expect(getTask('partitions.schedule')).toBe(maintain.schedule); + expect(maintain.schedule.cron).toBe(DEFAULT_PARTITIONS_CRON); + expect(maintain.schedule.tz).toBe('UTC'); + const hourly = partitions({ name: 'hourly-parts', cron: '5 * * * *', tz: 'Europe/Berlin' }); + expect(hourly.schedule.cron).toBe('5 * * * *'); + expect(hourly.schedule.tz).toBe('Europe/Berlin'); + }); + + test('only entities declaring partitionBy are maintained; none means an empty report', async () => { + entity('pt_plain', { columns: { id: uuid().primaryKey() } }); + expect(partitionedEntities()).toEqual([]); + const maintain = partitions({ name: 'pt.partitions' }) as AnyJobHandle; + const driver = memoryJobDriver(); + await driver.enqueue({ + name: maintain.name, + queue: maintain.queue, + input: {}, + idempotencyKey: maintain.name, + maxAttempts: 1, + }); + const [claimed] = await driver.claim({ + queues: [maintain.queue], + limit: 1, + visibilityTimeoutMs: 30_000, + workerId: 'w1', + }); + const ctx = ctxOf({ role: 'worker', actor: userActor({ id: 'worker-1' }) }); + const execution = await executeJob({ + driver, + claimed: claimed as ClaimedJob, + handle: maintain, + ctx, + }); + expect(execution.result).toEqual({ tables: [] }); + }); +}); diff --git a/packages/jobs/src/partitions.ts b/packages/jobs/src/partitions.ts new file mode 100644 index 000000000..b6cd2116a --- /dev/null +++ b/packages/jobs/src/partitions.ts @@ -0,0 +1,119 @@ +// `partitions()`: the maintenance every `entity({ partitionBy })` needs — partitions created ahead, +// expired ones dropped — as a FACTORY over `job()` plus a `.schedule` `task()` (default daily), the +// shape `consumer()`'s `.supervisor` has. Never a ninth primitive. The work is `@ultimat3/db`'s +// `maintainPartitions` (bounded lock waits, `detach … concurrently`), judged at THIS job's clock. + +import type { Clock } from '@ultimat3/core'; +import { assert, liveClock } from '@ultimat3/core'; +import { db, maintainPartitions } from '@ultimat3/db'; +import type { EntityDescription } from '@ultimat3/entity'; +import { registeredEntities } from '@ultimat3/entity'; +import { t } from '@ultimat3/schema'; +import type { DurationInput } from './clock'; +import { nowMs } from './clock'; +import type { JobHandle } from './job'; +import { job } from './job'; +import type { RetryPolicy } from './retry'; +import { DEFAULT_RETRY } from './retry'; +import type { TaskHandle } from './task'; +import { task } from './task'; + +/** 03:17 UTC daily: off the hour every other cron in a fleet picks, and well before month-end. */ +export const DEFAULT_PARTITIONS_CRON = '17 3 * * *'; + +export const DEFAULT_PARTITIONS_NAME = 'partitions'; + +export interface PartitionsDefinition { + /** The job's durable name; the task is `.schedule`. Default `DEFAULT_PARTITIONS_NAME`. */ + readonly name?: string; + /** Default `DEFAULT_PARTITIONS_CRON`. Daily is enough for any interval but `day` with `ahead: 0`. */ + readonly cron?: string; + /** Default `'UTC'`: partition bounds are UTC, so the schedule is too. */ + readonly tz?: string; + /** The instant partitions are judged at — the framework clock, so a test can move it. */ + readonly clock?: Clock; + readonly queue?: string; + readonly retry?: RetryPolicy; + readonly timeout?: DurationInput; +} + +/** One partitioned table's maintenance: the partitions this run created and dropped. */ +export interface PartitionsTableReport { + readonly table: string; + readonly created: readonly string[]; + readonly dropped: readonly string[]; +} + +export interface PartitionsReport { + readonly tables: readonly PartitionsTableReport[]; +} + +/** + * The job's input. The schedule enqueues `{}`. Before a `backfill()` copies rows older than the + * table's first partition, enqueue it once with `from`: the oldest row's instant (ISO). That run + * creates every partition back to it and sweeps nothing, so the copy routes instead of `23514`. + */ +export interface PartitionsInput { + readonly from?: string | undefined; +} + +/** The maintenance job, with the `task()` that enqueues it — as `consumer()` carries `.supervisor`. */ +export interface PartitionsJob extends JobHandle { + readonly schedule: TaskHandle; +} + +/** Every registered entity declaring `partitionBy`, described, in registry name order. */ +export const partitionedEntities = (): readonly EntityDescription[] => + registeredEntities() + .map((entry) => entry.describe()) + .filter((described) => described.partition != null); + +/** `from` as an instant, or refused where it was enqueued: a bad bound would create nothing. */ +function fromOf(name: string, from: string | undefined): Date | undefined { + if (from === undefined) return undefined; + const at = new Date(from); + assert( + !Number.isNaN(at.getTime()), + `${name}: from "${from}" is no ISO instant`, + `enqueue ${name} with { from: '2025-01-01T00:00:00.000Z' } — the oldest row's instant`, + ); + return at; +} + +export function partitions(definition: PartitionsDefinition = {}): PartitionsJob { + const name = definition.name ?? DEFAULT_PARTITIONS_NAME; + const clock = definition.clock ?? liveClock; + const handle = job({ + name, + input: t.object({ from: t.string.optional() }), + // Partitions are tables, not any tenant's rows. + tenant: 'none', + // One live pass: a second tick while one runs is the same pass, and the SQL is idempotent anyway. + // A `from` run is a different pass: it must not fold into the schedule's. + idempotencyKey: (input) => (input.from === undefined ? name : `${name}:from:${input.from}`), + retry: definition.retry ?? DEFAULT_RETRY, + ...(definition.queue === undefined ? {} : { queue: definition.queue }), + ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }), + async run({ input, step }): Promise { + const from = fromOf(name, input.from); + const tables: PartitionsTableReport[] = []; + // Read per run, never at declaration: an entity module may load after this one. + for (const described of partitionedEntities()) { + // One step per table: a retry resumes at the table that failed, never re-dropping. + const report = await step.run(`partitions:${described.table}`, async () => { + const run = await maintainPartitions(db(), described, new Date(nowMs(clock)), { from }); + return { table: described.table, created: run.created, dropped: run.dropped }; + }); + tables.push(report); + } + return { tables }; + }, + }); + const schedule = task({ + name: `${name}.schedule`, + cron: definition.cron ?? DEFAULT_PARTITIONS_CRON, + tz: definition.tz ?? 'UTC', + enqueue: () => [[handle, {}]], + }); + return Object.assign(handle, { schedule }); +} diff --git a/packages/jobs/src/reindex.test.ts b/packages/jobs/src/reindex.test.ts new file mode 100644 index 000000000..0ea70f25a --- /dev/null +++ b/packages/jobs/src/reindex.test.ts @@ -0,0 +1,143 @@ +// `reindex()` run by `executeJob` over a memory table: every row reaches the index as entity's own +// document (searchable columns only, its tenant kept), batch by batch; an entity with nothing +// searchable is refused where it was declared. + +import { afterEach, describe, expect, test } from 'bun:test'; +import { ctxOf, isUltimateError, userActor } from '@ultimat3/core'; +import type { SearchDocument, SearchIndex } from '@ultimat3/entity'; +import { + clearRegistry, + entity, + memoryRepo, + tableFor, + text, + timestamp, + uuid, +} from '@ultimat3/entity'; +import type { ClaimedJob } from './driver'; +import { memoryJobDriver } from './driver-memory'; +import { executeJob } from './execute'; +import type { AnyJobHandle } from './job'; +import { resetJobs } from './job'; +import { reindex } from './reindex'; + +const ORG_A = '00000000-0000-7000-8000-0000000000a1'; +const ORG_B = '00000000-0000-7000-8000-0000000000b2'; + +const posts = entity('reindex_posts', { + columns: { + id: uuid().primaryKey(), + orgId: uuid(), + title: text().searchable('A'), + secret: text(), + updatedAt: timestamp().defaultNow().onUpdateNow(), + }, +}); +const unversioned = entity('reindex_unversioned', { + columns: { id: uuid().primaryKey(), title: text().searchable() }, +}); + +const AT = new Date('2026-11-09T10:00:00.000Z'); +const plain = entity('reindex_plain', { columns: { id: uuid().primaryKey(), note: text() } }); + +const rows = [1, 2, 3].map((n) => ({ + id: `00000000-0000-7000-8000-00000000000${n}`, + orgId: n === 3 ? ORG_B : ORG_A, + title: `post ${n}`, + secret: 'never indexed', + updatedAt: new Date(AT.getTime() + n), +})); + +const recording = (): SearchIndex & { readonly calls: SearchDocument[][] } => { + const calls: SearchDocument[][] = []; + return { + name: 'recording', + calls, + index: async (documents) => { + calls.push([...documents]); + }, + remove: async () => undefined, + query: async () => [], + }; +}; + +afterEach(() => { + resetJobs(); + clearRegistry(); +}); + +const run = async (handle: AnyJobHandle) => { + const driver = memoryJobDriver(); + await driver.enqueue({ + name: handle.name, + queue: handle.queue, + input: {}, + idempotencyKey: handle.name, + maxAttempts: 1, + }); + const [claimed] = await driver.claim({ + queues: [handle.queue], + limit: 1, + visibilityTimeoutMs: 30_000, + workerId: 'w1', + }); + const ctx = ctxOf({ role: 'worker', actor: userActor({ id: 'worker-1' }) }); + return executeJob({ driver, claimed: claimed as ClaimedJob, handle, ctx }); +}; + +describe('reindex()', () => { + test('every row reaches the index as its document, one call per batch', async () => { + const index = recording(); + const handle = reindex({ + name: 'reindex-posts', + entity: posts, + tenant: 'none', + batch: 2, + rate: 1_000, + source: () => tableFor(posts, memoryRepo(posts, rows)), + index: () => index, + }); + const execution = await run(handle as AnyJobHandle); + expect(execution.outcome).toBe('completed'); + expect(index.calls.map((batch) => batch.length)).toEqual([2, 1]); + expect(index.calls.flat()).toEqual( + rows.map((row) => ({ + entity: 'reindex_posts', + tenant: row.orgId, + id: row.id, + version: row.updatedAt.getTime(), + fields: { title: row.title }, + })), + ); + }); + + test('an entity with no searchable column is refused at declaration', () => { + let thrown: unknown; + try { + reindex({ + name: 'reindex-plain', + entity: plain, + tenant: 'none', + source: () => tableFor(plain, memoryRepo(plain, [])), + }); + } catch (error: unknown) { + thrown = error; + } + expect(isUltimateError(thrown) ? thrown.code : undefined).toBe('X_INVARIANT'); + }); + + test('an entity with no onUpdateNow() version is refused at declaration', () => { + let thrown: unknown; + try { + reindex({ + name: 'reindex-unversioned', + entity: unversioned, + tenant: 'none', + source: () => tableFor(unversioned, memoryRepo(unversioned, [])), + }); + } catch (error: unknown) { + thrown = error; + } + expect(isUltimateError(thrown) ? thrown.code : undefined).toBe('X_INVARIANT_VIOLATED'); + }); +}); diff --git a/packages/jobs/src/reindex.ts b/packages/jobs/src/reindex.ts new file mode 100644 index 000000000..79b420f02 --- /dev/null +++ b/packages/jobs/src/reindex.ts @@ -0,0 +1,70 @@ +// `reindex()`: rebuild a `SearchIndex` from the table, as a `backfill()` — so a FACTORY over a +// factory over `job()`, never a ninth primitive. It is what an app runs after swapping the built-in +// Postgres index for Meilisearch or Typesense, or after the index lost data: one batch per durable +// step, the cursor checkpointed, each row through entity's ONE document builder (`searchDocumentOf`). + +import type { Ctx, Environment } from '@ultimat3/core'; +import { assert } from '@ultimat3/core'; +import type { EntityCore, ReadBuilder, SearchIndex } from '@ultimat3/entity'; +import { searchDocumentOf, searchIndex, searchVersionColumn } from '@ultimat3/entity'; +import type { BackfillInput } from './backfill'; +import { backfill } from './backfill'; +import type { DurationInput } from './clock'; +import type { JobHandle } from './job'; +import type { RetryPolicy } from './retry'; +import type { JobTenant } from './tenant'; + +export interface ReindexDefinition { + /** REQUIRED: the backfill's durable key — its ledger row and queue key carry it. */ + readonly name: string; + /** The entity whose `.searchable()` columns are indexed. */ + readonly entity: EntityCore; + /** The rows to index — usually the whole table: `() => db.posts`. Read once per attempt. */ + source(args: { readonly ctx: Ctx }): ReadBuilder; + /** As on `backfill()`: `'none'` for a sweep of every tenant, each document keeping its own. */ + readonly tenant: JobTenant; + /** Default: the installed index (`runtime.ts`'s `searchIndex`, else Postgres' built-in). */ + readonly index?: () => SearchIndex; + readonly batch?: number; + readonly rate?: number; + readonly queue?: string; + readonly retry?: RetryPolicy; + readonly timeout?: DurationInput; + readonly environments?: readonly Environment[]; +} + +/** True when at least one column of `entity` is declared `.searchable()`. */ +const isSearchable = (entity: EntityCore): boolean => + Object.values(entity.$columns).some((column) => column.$meta.searchable !== undefined); + +/** + * A full pass that writes every row's document to the index. At least once per batch, like every + * backfill: each document carries its row's `onUpdateNow()` version and the index keeps only + * newer, so a replayed batch is a no-op and a batch that read a row before a write (or before its + * delete's tombstone) cannot put the old text back. Rows deleted since the last pass are NOT + * removed — that is the write path's `remove(scope, ids, deletedAt)`, not a sweep's. + */ +export function reindex(definition: ReindexDefinition): JobHandle { + const { entity } = definition; + assert( + isSearchable(entity as EntityCore), + `reindex "${definition.name}" names entity ${entity.$name}, which has no .searchable() column — there is nothing to index`, + `title: text().searchable() # in entity('${entity.$name}'): mark the columns a search reads, or drop reindex "${definition.name}"`, + ); + // At declaration, not at the first batch: a document without a version is one an index cannot + // order against a write, so the entity must declare its `onUpdateNow()` column first. + searchVersionColumn(entity as EntityCore); + const index = definition.index ?? searchIndex; + return backfill({ + name: definition.name, + tenant: definition.tenant, + source: (args) => definition.source(args), + handle: ({ rows }) => index().index(rows.map((row) => searchDocumentOf(entity, row))), + ...(definition.batch === undefined ? {} : { batch: definition.batch }), + ...(definition.rate === undefined ? {} : { rate: definition.rate }), + ...(definition.queue === undefined ? {} : { queue: definition.queue }), + ...(definition.retry === undefined ? {} : { retry: definition.retry }), + ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }), + ...(definition.environments === undefined ? {} : { environments: definition.environments }), + }); +} diff --git a/packages/jobs/src/scheduler-failover.live.test.ts b/packages/jobs/src/scheduler-failover.live.test.ts new file mode 100644 index 000000000..d64056e11 --- /dev/null +++ b/packages/jobs/src/scheduler-failover.live.test.ts @@ -0,0 +1,243 @@ +// Scheduler single-leader failover on a real Postgres: the leader dies MID-TICK — frozen inside the +// fire of an occurrence it already owns, its lease never renewed — and a standby takes over on +// expiry. Every occurrence must be queued exactly once (no double fire) and none may be missing (no +// skip), including the one the dead leader was holding when it froze — and when that leader thaws +// and its stale fire lands late, the watermark fence must refuse it. Skips unless +// `TEST_DATABASE_URL` is set. + +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; +import type { Clock, PgExecutor } from '@ultimat3/core'; +import { probeDatabaseName, resetLifecycle, sweepProbeDatabases } from '@ultimat3/core'; +import type { PostgresClient } from '@ultimat3/db'; +import { dbExecutor, postgresClient, raw } from '@ultimat3/db'; +import type { StandardSchemaV1 } from '@ultimat3/schema'; +import { postgresJobDriver } from './driver-pg'; +import { SQL_SCHEDULER_FIRE } from './driver-pg-operator-sql'; +import { SQL_JOBS_TABLE } from './driver-pg-sql'; +import { job, resetJobs } from './job'; +import type { CronResolver, Scheduler } from './scheduler'; +import { jobScheduler } from './scheduler'; +import { postgresLeaseLeader, postgresSchedulerState } from './scheduler-pg'; +import type { TaskHandle } from './task'; +import { resetTasks, task } from './task'; + +const url = Bun.env['TEST_DATABASE_URL']; +const hasPostgres = typeof url === 'string' && url.length > 0; +const PROBE_DB = probeDatabaseName('x_jobs_failover_live'); +const TASK = 'heartbeatTask'; +/** Short enough that the standby's wait is a fraction of a second; the logic is the shipped one. */ +const TTL_MS = 400; +const SECOND = 1_000; +const T0 = Date.UTC(2026, 6, 26, 0, 0, 0); + +const probeUrl = (): string => { + const parsed = new URL(url ?? 'postgres://localhost/postgres'); + parsed.pathname = `/${PROBE_DB}`; + return parsed.toString(); +}; + +const executorFor = (client: PostgresClient): PgExecutor => ({ + query: (text: string, values: readonly unknown[]): Promise => + client.query({ text, values }), +}); + +/** One occurrence on every whole second — aligned, so both nodes agree on every instant. */ +const everySecond: CronResolver = (_cron, options) => + new Date(Math.floor(options.from.getTime() / SECOND) * SECOND + SECOND); + +/** The scheduling clock both nodes share, moved by hand; the LEASE expires on the server's clock. */ +function manualClock(start: number): Clock & { advance(ms: number): void; at(): number } { + let current = start; + return { + now: () => new Date(current), + monotonic: () => current, + advance(ms: number) { + current += ms; + }, + at: () => current, + }; +} + +interface Freezable { + readonly executor: PgExecutor; + /** The next fire statement blocks before it reaches the server: the process is frozen. */ + freezeNextFire(): void; + /** Settles once a fire is held — the leader is provably mid-tick, holding an occurrence. */ + frozen(): Promise; + /** Let the held fire through, late: the zombie the fence has to refuse. */ + thaw(): void; +} + +function freezable(base: PgExecutor): Freezable { + let armed = false; + let arrived = (): void => undefined; + const reached = new Promise((resolve) => { + arrived = resolve; + }); + let open = (): void => undefined; + const gate = new Promise((resolve) => { + open = resolve; + }); + return { + executor: { + async query(text: string, values: readonly unknown[]): Promise { + if (armed && text === SQL_SCHEDULER_FIRE) { + armed = false; + arrived(); + await gate; + } + return base.query(text, values); + }, + }, + freezeNextFire: () => { + armed = true; + }, + frozen: () => reached, + thaw: () => open(), + }; +} + +function passthrough(): StandardSchemaV1 { + return { + '~standard': { + version: 1, + vendor: 'ultimate-test', + validate: (value: unknown) => ({ value: value as T }), + }, + }; +} + +afterEach(() => { + resetJobs(); + resetTasks(); + resetLifecycle(); +}); + +describe.skipIf(!hasPostgres)('live · postgres · scheduler leader failover', () => { + let admin: PostgresClient; + let podA: PostgresClient; + let podB: PostgresClient; + let heartbeat: TaskHandle; + + beforeAll(async () => { + admin = postgresClient({ url: url ?? '', role: 'web', profile: { max: 1 } }); + await admin.execute(raw(`drop database if exists ${PROBE_DB} with (force)`)); + await sweepProbeDatabases( + dbExecutor(() => admin), + PROBE_DB, + ); + await admin.execute(raw(`create database ${PROBE_DB}`)); + podA = postgresClient({ url: probeUrl(), role: 'scheduler', profile: { max: 2 } }); + podB = postgresClient({ url: probeUrl(), role: 'scheduler', profile: { max: 2 } }); + for (const statement of SQL_JOBS_TABLE.split(';')) { + if (statement.trim().length > 0) await podA.execute(raw(statement)); + } + }); + + beforeEach(() => { + const beat = job<{ occurrenceMs: number }>({ + tenant: 'none', + name: 'beat', + input: passthrough<{ occurrenceMs: number }>(), + idempotencyKey: (input) => String(input.occurrenceMs), + retry: { attempts: 1 }, + run: () => Promise.resolve(), + }); + heartbeat = task({ + name: TASK, + cron: '* * * * * *', + tz: 'UTC', + catchUp: 'run-all', + maxCatchUp: 100, + enqueue: (occurrenceMs) => [[beat, { occurrenceMs }]], + }); + }); + + afterAll(async () => { + await podA.close(); + await podB.close(); + await admin.execute(raw(`drop database if exists ${PROBE_DB} with (force)`)); + await admin.close(); + }); + + const node = (executor: PgExecutor, holder: string, clock: Clock): Scheduler => + jobScheduler({ + driver: postgresJobDriver({ executor }), + state: postgresSchedulerState(executor), + leader: postgresLeaseLeader({ executor, holder, ttlMs: TTL_MS }), + clock, + cron: everySecond, + tasks: [heartbeat], + tickIntervalMs: 3_600_000, + drainOnShutdown: false, + }); + + /** Every occurrence the queue holds a job for, in order, duplicates kept. */ + const queuedOccurrences = async (): Promise => { + const rows = await podA.query<{ occurrence: string }>({ + text: `select (input->>'occurrenceMs') as occurrence from x_jobs where name = 'beat' + order by (input->>'occurrenceMs')::bigint`, + values: [], + }); + return rows.map((row) => Number(row.occurrence)); + }; + + test('the leader freezes mid-fire: the standby fires that occurrence once, and the late fire is refused', async () => { + const clock = manualClock(T0); + const frozenA = freezable(executorFor(podA)); + const a = node(frozenA.executor, 'pod-a', clock); + const b = node(executorFor(podB), 'pod-b', clock); + + // A wins the lease and arms the task; B is a standby. + expect(await a.tick()).toEqual([]); + expect(await b.tick()).toEqual([]); + clock.advance(SECOND); + expect((await a.tick()).map((fire) => fire.occurrenceMs)).toEqual([T0 + SECOND]); + + // A is mid-tick on T0+2s when it dies: the fire never reaches the server, the lease is never + // renewed and never released. + clock.advance(SECOND); + frozenA.freezeNextFire(); + const zombieRound = a.tick(); + await frozenA.frozen(); + + // B keeps asking on the cadence a standby does; it takes over only once the lease EXPIRES. + const takeoverStarted = performance.now(); + let takenOver: readonly number[] = []; + while (takenOver.length === 0) { + if (performance.now() - takeoverStarted > 10_000) { + expect.unreachable('the standby never took the lease over'); + } + // real-clock: the lease expires on the Postgres server's clock, which no test clock moves + await Bun.sleep(50); + clock.advance(SECOND / 4); + takenOver = (await b.tick()).map((fire) => fire.occurrenceMs); + } + // Not before the lease ran out: a standby that fires sooner is a second leader. + expect(performance.now() - takeoverStarted).toBeGreaterThanOrEqual(TTL_MS - 100); + // The first thing the new leader fires is the occurrence the dead one was holding. + expect(takenOver[0]).toBe(T0 + 2 * SECOND); + + // The frozen leader comes back and its stale fire lands: the watermark refuses it. + frozenA.thaw(); + expect(await zombieRound).toEqual([]); + // And it learns it is no longer the leader. + clock.advance(SECOND); + expect(await a.tick()).toEqual([]); + const lastFromB = await b.tick(); + expect(lastFromB.length).toBeGreaterThan(0); + + const queued = await queuedOccurrences(); + // No double fire: every occurrence once. + expect(new Set(queued).size).toBe(queued.length); + // No skip: every whole second from the first fire to the last, contiguous. + const first = queued[0] ?? expect.unreachable('nothing was queued'); + const last = queued[queued.length - 1] ?? first; + const expected = Array.from( + { length: (last - first) / SECOND + 1 }, + (_, index) => first + index * SECOND, + ); + expect(queued).toEqual(expected); + expect(last).toBe(Math.floor(clock.at() / SECOND) * SECOND); + }); +}); diff --git a/packages/jobs/src/subject-jobs.ts b/packages/jobs/src/subject-jobs.ts index 9314475e0..3eacebd3f 100644 --- a/packages/jobs/src/subject-jobs.ts +++ b/packages/jobs/src/subject-jobs.ts @@ -10,6 +10,8 @@ import type { DurationInput } from './clock'; import { nowMs } from './clock'; import type { ExportFormat } from './export-format'; import type { ExportSink } from './export-sink'; +import { auditedErase } from './gdpr-audit'; +import { unerasedPersonalData } from './gdpr-coverage'; import type { JobHandle } from './job'; import { job } from './job'; import type { RetryPolicy } from './retry'; @@ -57,6 +59,10 @@ export function plannedSubject(subject: Subject): SubjectPlan { }); const [first] = problems; if (first !== undefined) throw first; + // A reached table whose treatment would leave a `.personalData()` column standing: refused before + // a row is touched, as an untreated table is (`gdpr-coverage.ts`). + const [leak] = unerasedPersonalData(plan); + if (leak !== undefined) throw leak; return plan; } @@ -78,8 +84,10 @@ export function subjectErase(definition: SubjectJobDefinition): JobHandle< ...common(definition), // One live erase per person: a second request while one runs is the same run. idempotencyKey: (input) => `${definition.name}:${subject.init.key(input)}`, - run: ({ step, input }): Promise => - subjectErasePass(step, readerFor(subject, input), subject.init.targets ?? []), + run: ({ step, input, ctx, runId }): Promise => + auditedErase({ job: definition.name, subject: subject.entity, runId, ctx }, step, () => + subjectErasePass(step, readerFor(subject, input), subject.init.targets ?? []), + ), }); } diff --git a/packages/jobs/src/worker-backlog.test.ts b/packages/jobs/src/worker-backlog.test.ts new file mode 100644 index 000000000..624335a1b --- /dev/null +++ b/packages/jobs/src/worker-backlog.test.ts @@ -0,0 +1,133 @@ +// A backlog drains at the speed of its jobs, never one poll interval per batch: a pass that came +// back FULL means more is waiting, so a slot freeing behind it claims at once. The floor here is a +// minute, so a regression is not "slower" but a timed-out `until` (bench: 31 claims/s at 250 ms). + +import { afterEach, describe, expect, test } from 'bun:test'; +import { ctxOf } from '@ultimat3/core'; +import { resetJobDriver } from './adapter'; +import type { JobDriver } from './driver'; +import { memoryJobDriver } from './driver-memory'; +import { signalEnqueued } from './enqueue-signal'; +import { resetJobs } from './job'; +import { itemJob } from './operator-surface-fixture'; +import { jobWorker } from './worker'; + +afterEach(() => { + resetJobs(); + resetJobDriver(); +}); + +const context = () => ctxOf({ role: 'worker', buildId: 'test' }); + +/** Real time, bounded: fails rather than hangs when the loop never gets there. */ +async function until(condition: () => Promise, budgetMs: number): Promise { + const deadline = performance.now() + budgetMs; + while (!(await condition())) { + if (performance.now() > deadline) expect.unreachable('the backlog did not drain in time'); + await new Promise((done) => setImmediate(done)); + } +} + +const turns = async (count: number): Promise => { + for (let turn = 0; turn < count; turn += 1) { + await new Promise((done) => setImmediate(done)); + } +}; + +const BACKLOG = 40; +const SLOTS = 8; + +describe('the claim loop under a backlog', () => { + test('N no-op jobs drain in about N/slots passes, with no floor-long wait between them', async () => { + const inner = memoryJobDriver(); + let claims = 0; + const driver: JobDriver = { + ...inner, + claim(options) { + claims += 1; + return inner.claim(options); + }, + }; + const handle = itemJob({ run: () => Promise.resolve() }); + for (let n = 0; n < BACKLOG; n += 1) { + await driver.enqueue({ + name: handle.name, + queue: 'default', + input: { item: `b${n}` }, + idempotencyKey: `backlog:${n}`, + maxAttempts: 1, + }); + } + const worker = jobWorker({ + driver, + concurrency: SLOTS, + pollIntervalMs: 60_000, + context, + drainOnShutdown: false, + }); + const started = performance.now(); + worker.start(); + try { + // The first pass is armed at the floor too: kick it, as an enqueue's wake would. + await worker.tick(); + await until(async () => (await worker.stats()).processed >= BACKLOG, 3_000); + expect(performance.now() - started).toBeLessThan(3_000); + // Passes, not timers: one per freed slot at most, never a spin. + expect(claims).toBeLessThanOrEqual(BACKLOG + 2); + expect(claims).toBeGreaterThanOrEqual(Math.ceil(BACKLOG / SLOTS)); + } finally { + await worker.stop(); + } + }); + + test('a pass whose rows a cap sent back is not a backlog: a completion does not re-claim them', async () => { + const inner = memoryJobDriver(); + let claims = 0; + const driver: JobDriver = { + ...inner, + claim(options) { + claims += 1; + return inner.claim(options); + }, + }; + let release = (): void => {}; + const gate = new Promise((done) => { + release = done; + }); + // One at a time: a pass claims SLOTS rows, admits ONE, and sends the rest back. The admitted + // run is held until the pass is over, so its completion is the kick under test. + const handle = itemJob({ run: () => gate, concurrency: 1 }); + for (let n = 0; n < SLOTS; n += 1) { + await driver.enqueue({ + name: handle.name, + queue: 'default', + input: { item: `c${n}` }, + idempotencyKey: `capped:${n}`, + maxAttempts: 3, + }); + } + const worker = jobWorker({ + driver, + concurrency: SLOTS, + pollIntervalMs: 60_000, + context, + drainOnShutdown: false, + }); + worker.start(); + try { + // The loop's own pass, woken as an enqueue wakes it — `tick()` would wait for the run. + signalEnqueued('default'); + await until(async () => claims === 1 && (await worker.stats()).inFlight === 1, 3_000); + await turns(20); + release(); + await until(async () => (await worker.stats()).processed === 1, 3_000); + await turns(5); + // A kick sets the wait to 0 at once and its pass claims a moment later: either is the bug. + expect((await worker.stats()).pollDelayMs).toBe(60_000); + expect(claims).toBe(1); + } finally { + release(); + await worker.stop(); + } + }); +}); diff --git a/packages/jobs/src/worker-loop.ts b/packages/jobs/src/worker-loop.ts index f49d3613c..cf380da70 100644 --- a/packages/jobs/src/worker-loop.ts +++ b/packages/jobs/src/worker-loop.ts @@ -42,19 +42,34 @@ export interface ClaimLoop { stop(): void; /** True once a pass has come back empty. */ idle(): boolean; - /** A pass ended: `found` decides the next wait. */ - passed(found: boolean): void; + /** + * A pass ended: `found` decides the next wait. `full` — some ask came back with every row it + * asked for — means a backlog is likely, so a slot freeing before the next pass claims at once. + */ + passed(found: boolean, full?: boolean): void; /** The wait the loop will arm, or has armed. */ delayMs(): number; /** * A slot came free, or a handed-back row is due: one pass NOW if the wait in hand is longer - * than the floor — and no reset, so a pass that finds nothing costs one statement, not a ramp. + * than the floor, or the last pass came back full — and no reset, so a pass that finds nothing + * costs one statement, not a ramp. */ kick(): void; /** A row this worker settled back onto the queue is claimable again in `ms`. */ dueIn(ms: number): void; } +/** + * Whether one ask came back FULL — every row it asked for STARTED. Counted on what a pass admitted, + * never on what it claimed: rows a concurrency or rate cap sent back are not a backlog this worker + * can run, and kicking on them is a claim plus N nacks per completion. + */ +export function fillsAsk(admitted: ReadonlyMap, limit: number): boolean { + let ran = 0; + for (const count of admitted.values()) ran += count; + return ran > 0 && ran >= limit; +} + /** A due time further out than this is the poll's to find: no timer is held for a 3-day sleep. */ const DUE_HINT_MAX_MS = 60_000; /** After the due time, never on it: the row's `run_at` is the database's clock, not this one. */ @@ -74,6 +89,12 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop { /** The timer's own pass is in flight. A wake landing now asks for ONE more, never a second chain. */ let passing = false; let again = false; + /** + * The last pass came back full. A pass that found work waits the FLOOR, and a kick shorter than + * that did nothing — so a backlog drained at one batch per floor: 8 slots / 250 ms = 32 claims/s, + * which `scripts/bench/worker-throughput.bench.ts` measured exactly. Cleared by the next pass. + */ + let backlog = false; let unsubscribe: (() => void) | undefined; /** One timer per floor-wide bucket of due times, so a burst of retries holds a bounded few. */ const due = new Map(); @@ -126,7 +147,7 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop { }; const kick = (): void => { - if (delayMs > floor) now(); + if (delayMs > floor || backlog) now(); }; return { @@ -140,6 +161,7 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop { stop() { active = false; again = false; + backlog = false; if (timer !== undefined) timers.clear(timer); timer = undefined; for (const hint of due.values()) timers.clear(hint); @@ -148,7 +170,8 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop { unsubscribe = undefined; }, idle: () => backoff.idle(), - passed(found) { + passed(found, full = false) { + backlog = full; delayMs = backoff.next(found); }, delayMs: () => delayMs, diff --git a/packages/jobs/src/worker.ts b/packages/jobs/src/worker.ts index 94fd5a7d2..151f2e0ac 100644 --- a/packages/jobs/src/worker.ts +++ b/packages/jobs/src/worker.ts @@ -28,7 +28,7 @@ import { armWorkerCutoff, type DrainCutoff } from './worker-drain-cutoff'; import { assertConcurrencyEnforceable, createFleetSlots } from './worker-fleet-slots'; import { createHeldRuns } from './worker-held'; import type { ClaimLoop } from './worker-loop'; -import { createClaimLoop } from './worker-loop'; +import { createClaimLoop, fillsAsk } from './worker-loop'; import { resolveWorkerTimings } from './worker-options'; import { createProcessCap } from './worker-process-cap'; import { createQueueDepthPublisher } from './worker-queue-depth'; @@ -143,6 +143,7 @@ export function jobWorker(options: WorkerOptions): Worker { await publishQueueDepth(); const started: Promise[] = []; let found = false; + let full = false; // an ask whose every row STARTED: a backlog (`worker-loop.ts`) // One ask; WHICH queue, how many and in what order is `worker-claim-sweep.ts`'s (`maxInFlight` // and the turn). It re-reads `claiming()` per ask: a `stop()` mid-pass stops claiming now. @@ -218,9 +219,8 @@ export function jobWorker(options: WorkerOptions): Worker { .finally(async () => { lease.release(); returnCap(); - // The slot is free NOW. A full worker asks the queue for nothing, so its loop backs - // off like an idle one — and waiting that out with a backlog behind the job that - // just finished is throughput lost to a timer. + // The slot is free NOW: behind a FULL pass, or a backed-off loop, the next pass runs at + // once (`worker-loop.ts`) — a backlog never waits out a timer for its next row. loop.kick(); // AWAITED, never `void`: the slot is a row in `x_job_leases`, and a DELETE still on the // wire at `driver.close()` left a `concurrency: 1` job unclaimable by the replacement @@ -257,6 +257,7 @@ export function jobWorker(options: WorkerOptions): Worker { }, ); } + if (fillsAsk(admitted, ask.limit)) full = true; return admitted; }; @@ -269,9 +270,8 @@ export function jobWorker(options: WorkerOptions): Worker { claim: claimOne, }); - // The loop's next wait: the floor while passes find work, doubling to the ceiling while they - // do not (`idle-backoff.ts`). Reported by the pass itself, so `tick()` and the timer agree. - loop.passed(found); + // The next wait (`idle-backoff.ts`), reported by the pass itself: `tick()` and the timer agree. + loop.passed(found, full); return started; }; diff --git a/packages/notify/CLAUDE.md b/packages/notify/CLAUDE.md index 41f30aec3..2e65ef987 100644 --- a/packages/notify/CLAUDE.md +++ b/packages/notify/CLAUDE.md @@ -128,6 +128,7 @@ so an `open` checkpointed with duplicates by an earlier version is repaired on r | `push-device.ts` · `push-device-pg.ts` | the device seam, memory and Postgres (`x_push_devices`) | | `push-register.ts` | `pushRegister` / `pushUnregister`, factories over `action` | | `push-errors.ts` | `X_NOTIFY_PUSH_*`, registered in `errors.ts` | +| `sms-transport.ts` · `sms-channel.ts` · `sms-errors.ts` | the `smsTransport` port (an `ADAPTER_PORTS` row: interface + installer on `./adapter`, memory reference, E.164 check), the reference `smsChannel()`, and `X_NOTIFY_SMS_*`. A provider is the app's adapter, never shipped (axiom 8) | One entry point, deliberately: every module runs on the server, so there is no browser half to split off. diff --git a/packages/notify/README.md b/packages/notify/README.md index 7c50ec17d..f8ecf6361 100644 --- a/packages/notify/README.md +++ b/packages/notify/README.md @@ -319,6 +319,35 @@ export const approvalRequested = notifier({ | another 4xx | `X_NOTIFY_PUSH_PROVIDER_REFUSED`, logged; the next device is tried. When no device took it, the refusal fails the delivery | | 429, 5xx, timeout | rethrown after every other device was tried — the job's retry | +### SMS — the `smsTransport` port + +```ts +import { smsChannel } from '@ultimat3/notify'; +import type { SmsTransport } from '@ultimat3/notify/adapter'; + +interface LoginCode { + readonly code: string; +} + +// a notifier's channel: the body is a catalog key, rendered in the recipient's locale +export const loginCodeSms = smsChannel({ + message: ({ event }) => ({ key: 'sms.login_code', params: { code: event.params.code } }), + numberOf: (recipient) => recipient.to, // the default: E.164, `+4915112345678` +}); + +// apps//runtime.ts — the provider is the app's adapter (Twilio, Vonage, SNS) +declare function twilioSms(config: { readonly from: string }): SmsTransport; +export const runtime = { smsTransport: twilioSms({ from: '+15550100000' }) }; +``` + +| Rule | Detail | +|---|---| +| one port | `SmsTransport` (`send(message) → { transport, id, to, idempotencyKey }`) on `@ultimat3/notify/adapter`, installed from `smsTransport:` in `runtime.ts` (`installSmsTransport`, an `adapterSlot`). Proved by `smsTransportConformance` (`@ultimat3/testing/conformance`) | +| default | `memorySmsTransport()` in development/test; anywhere else the first send is `X_NOTIFY_SMS_TRANSPORT_MISSING` (terminal) — a heap that swallows a login code is not a default | +| the text is a catalog key | `message` answers `{ key, params }`, rendered in `recipient.locale` (else the app's fallback) by `translatorFor` — never text | +| E.164 or refused | `assertSmsNumber` before render or send: a local spelling is `X_NOTIFY_SMS_NUMBER_INVALID` (terminal), the number never in the cause. Every transport calls it too (the suite checks). No number is skipped and logged `notify.address_missing`, as mail | +| one key per delivery | `smsIdempotencyKey(notifier, key, channel, recipient)` — the same on every attempt, no number in it | + ## The inbox @@ -374,6 +403,8 @@ without bound and every replay would re-send the whole audience. | `X_NOTIFY_PUSH_DEVICE_GONE` | APNs 410 / FCM `UNREGISTERED`: the device row is deleted, nothing fails | | `X_NOTIFY_PUSH_PROVIDER_REFUSED` | APNs or FCM refused for good: a credential, a topic or project, a payload limit | | `X_NOTIFY_PUSH_MESSAGE_INVALID` | a data-only push carrying more than ids | +| `X_NOTIFY_SMS_TRANSPORT_MISSING` | an SMS sent outside development/test with no `smsTransport:` installed | +| `X_NOTIFY_SMS_NUMBER_INVALID` | an SMS number that is not E.164 | ### Error classes @@ -390,6 +421,8 @@ a job boundary the class is gone and the `code` is what survives — match on th | `NotifyPushDeviceGoneError` | `X_NOTIFY_PUSH_DEVICE_GONE` | `src/push-errors.ts` | | `NotifyPushMessageInvalidError` | `X_NOTIFY_PUSH_MESSAGE_INVALID` | `src/push-errors.ts` | | `NotifyPushProviderRefusedError` | `X_NOTIFY_PUSH_PROVIDER_REFUSED` | `src/push-errors.ts` | +| `NotifySmsNumberInvalidError` | `X_NOTIFY_SMS_NUMBER_INVALID` | `src/sms-errors.ts` | +| `NotifySmsTransportMissingError` | `X_NOTIFY_SMS_TRANSPORT_MISSING` | `src/sms-errors.ts` | | `NotifyStoreMissingError` | `X_NOTIFY_STORE_MISSING` | `src/errors.ts` | ## Boundary diff --git a/packages/notify/src/adapter.ts b/packages/notify/src/adapter.ts index e7e0e795e..66afba87d 100644 --- a/packages/notify/src/adapter.ts +++ b/packages/notify/src/adapter.ts @@ -1,5 +1,5 @@ -// `@ultimat3/notify/adapter` — what a notifier's stores are built and installed from: the port -// interfaces, the error a missing store raises, and the boot's installer with its reset. Never the +// `@ultimat3/notify/adapter` — what a notifier's stores and its SMS transport are built and +// installed from: the port interfaces, the errors a driver raises, and the boot's installers. Never the // `.` barrel: an app hands its stores over with `notifyStores:` in `apps//runtime.ts`, so the // only callers here are the cli boot, `@ultimat3/testing` and a store author. @@ -14,5 +14,8 @@ export type { DeliveryStatus, } from './ledger'; export type { PreferenceBatchQuery, PreferenceStore } from './preferences'; +export { NotifySmsNumberInvalidError, NotifySmsTransportMissingError } from './sms-errors'; +export type { SmsMessage, SmsSendResult, SmsTransport } from './sms-transport'; +export { installSmsTransport, resetSmsTransport } from './sms-transport'; export type { InstalledNotifyStores, NotifyStores } from './stores'; export { installNotifyStores, resetNotifyStores } from './stores'; diff --git a/packages/notify/src/errors.test.ts b/packages/notify/src/errors.test.ts index ebdc6bb81..80a1e2e24 100644 --- a/packages/notify/src/errors.test.ts +++ b/packages/notify/src/errors.test.ts @@ -148,6 +148,8 @@ describe('unit · @ultimat3/notify retry classification', () => { 'X_NOTIFY_DELIVERY_HELD', 'X_NOTIFY_PUSH_PROVIDER_REFUSED', 'X_NOTIFY_PUSH_MESSAGE_INVALID', + 'X_NOTIFY_SMS_TRANSPORT_MISSING', + 'X_NOTIFY_SMS_NUMBER_INVALID', ]; // Thrown by one device's send and caught by `nativePush()` before a send returns. const contained = ['X_NOTIFY_PUSH_DEVICE_GONE']; diff --git a/packages/notify/src/errors.ts b/packages/notify/src/errors.ts index c2b617add..c1c505f74 100644 --- a/packages/notify/src/errors.ts +++ b/packages/notify/src/errors.ts @@ -22,6 +22,8 @@ export const NOTIFY_ERROR_CODES = [ 'X_NOTIFY_PUSH_DEVICE_GONE', 'X_NOTIFY_PUSH_PROVIDER_REFUSED', 'X_NOTIFY_PUSH_MESSAGE_INVALID', + 'X_NOTIFY_SMS_TRANSPORT_MISSING', + 'X_NOTIFY_SMS_NUMBER_INVALID', ] as const; export type NotifyErrorCode = (typeof NOTIFY_ERROR_CODES)[number]; @@ -37,6 +39,8 @@ export const NOTIFY_ERROR_TITLES: Readonly> = { X_NOTIFY_PUSH_DEVICE_GONE: 'the push provider says a device token is dead', X_NOTIFY_PUSH_PROVIDER_REFUSED: 'APNs or FCM refused a push for good', X_NOTIFY_PUSH_MESSAGE_INVALID: 'a data-only push carries more than an id', + X_NOTIFY_SMS_TRANSPORT_MISSING: 'an SMS was sent with no SmsTransport installed', + X_NOTIFY_SMS_NUMBER_INVALID: 'an SMS number is not E.164', }; // One unconditional call, so a second package claiming one of notify's codes throws @@ -84,6 +88,10 @@ registerErrorRetry({ // it never leaves a send. X_NOTIFY_PUSH_PROVIDER_REFUSED: 'terminal', X_NOTIFY_PUSH_MESSAGE_INVALID: 'terminal', + // SMS (`sms-errors.ts`). A transport is installed at boot or it is not, and a stored number is + // the same number on attempt 2: neither is the moment. + X_NOTIFY_SMS_TRANSPORT_MISSING: 'terminal', + X_NOTIFY_SMS_NUMBER_INVALID: 'terminal', }); /** Refused where it is written: a notifier with no channel is a job that fans out to nobody. */ diff --git a/packages/notify/src/index.ts b/packages/notify/src/index.ts index d23ff52ff..e41fe485a 100644 --- a/packages/notify/src/index.ts +++ b/packages/notify/src/index.ts @@ -130,6 +130,13 @@ export { DATA_PUSH_MAX_KEYS, DATA_PUSH_MAX_VALUE } from './push-message'; export type { PushRegisterOptions } from './push-register'; export { PUSH_DEVICES_PER_ACTOR, pushRegister, pushUnregister } from './push-register'; export { purgeNotifyDeliveries, purgeNotifyDigests, purgeNotifyInbox } from './retention'; +// SMS: the reference channel over the `smsTransport` port. The port's interface and installer are +// on `./adapter`; a provider (Twilio, Vonage, SNS) is the app's adapter. +export type { NotifySmsMessage, SmsChannelOptions } from './sms-channel'; +export { SMS_CHANNEL, smsChannel, smsIdempotencyKey } from './sms-channel'; +export { NotifySmsNumberInvalidError, NotifySmsTransportMissingError } from './sms-errors'; +export type { MemorySmsTransport } from './sms-transport'; +export { assertSmsNumber, isSmsNumber, memorySmsTransport } from './sms-transport'; export type { InstalledNotifyStores, NotifyStores } from './stores'; export { notifyStores, diff --git a/packages/notify/src/sms-channel.test.ts b/packages/notify/src/sms-channel.test.ts new file mode 100644 index 000000000..1343e1f45 --- /dev/null +++ b/packages/notify/src/sms-channel.test.ts @@ -0,0 +1,109 @@ +// `smsChannel()` driven through a real notifier: per recipient, the number resolved, the key +// rendered in the recipient's locale, one idempotency key per delivery — and the two refusals +// (no number is skipped, a non-E.164 one is terminal) before any transport is asked. + +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { resetJobs } from '@ultimat3/jobs'; +import { t } from '@ultimat3/schema'; +import { installNotifyStores, resetNotifyStores } from './adapter'; +import { notifier } from './notifier'; +import type { TestParams } from './notify-fixture'; +import { driver } from './notify-fixture'; +import { smsChannel, smsIdempotencyKey } from './sms-channel'; +import type { SmsMessage, SmsTransport } from './sms-transport'; +import { memorySmsTransport } from './sms-transport'; + +const POST = '00000000-0000-7000-8000-00000000beef'; + +beforeEach(() => { + resetJobs(); + installNotifyStores({}); +}); + +afterEach(() => { + resetNotifyStores(); + resetJobs(); +}); + +/** Renders `::` so the test can see which locale and key reached the body. */ +const echoTranslate = + (locale: string) => (key: string, params?: Readonly>) => + `${locale}:${key}:${JSON.stringify(params ?? {})}`; + +function liked(transport: SmsTransport) { + const handle = notifier({ + name: 'post.liked', + input: t.object({ postId: t.uuid }), + tenant: 'none', + key: (input) => `like:${input.postId}`, + deliver: [ + { + channel: smsChannel({ + transport, + translate: echoTranslate, + message: ({ event }) => ({ key: 'sms.liked', params: { post: event.params.postId } }), + }), + }, + ], + }); + return handle; +} + +describe('unit · smsChannel', () => { + test('one SMS per recipient, rendered in its locale, keyed per delivery', async () => { + const transport = memorySmsTransport(); + const handle = liked(transport); + await driver().finish(handle, { + params: { postId: POST }, + recipients: [ + { id: 'ana', to: '+4915112345678', locale: 'de' }, + { id: 'ben', to: '+14155550123', locale: 'en' }, + ], + }); + expect(transport.sent).toEqual([ + { + to: '+4915112345678', + body: `de:sms.liked:{"post":"${POST}"}`, + idempotencyKey: smsIdempotencyKey('post.liked', `like:${POST}`, 'sms', 'ana'), + }, + { + to: '+14155550123', + body: `en:sms.liked:{"post":"${POST}"}`, + idempotencyKey: smsIdempotencyKey('post.liked', `like:${POST}`, 'sms', 'ben'), + }, + ]); + }); + + test('a recipient with no number is skipped, not failed', async () => { + const transport = memorySmsTransport(); + const handle = liked(transport); + await driver().finish(handle, { + params: { postId: POST }, + recipients: [{ id: 'ana' }, { id: 'ben', to: '+14155550123' }], + }); + expect(transport.sent.map((sms) => sms.to)).toEqual(['+14155550123']); + }); + + test('a non-E.164 number is refused, terminal, before the transport is asked', async () => { + // A transport that checks nothing, so the refusal seen is the CHANNEL's own. + const asked: SmsMessage[] = []; + const lax: SmsTransport = { + name: 'lax', + send: (message) => { + asked.push(message); + return Promise.resolve({ transport: 'lax', id: 'x', to: message.to, idempotencyKey: '' }); + }, + }; + const handle = liked(lax); + const refused = driver().finish(handle, { + params: { postId: POST }, + recipients: [{ id: 'ana', to: '0151 12345678' }], + }); + await expect(refused).rejects.toMatchObject({ retry: 'terminal' }); + expect(asked).toEqual([]); + }); + + test('the idempotency key holds no number', () => { + expect(smsIdempotencyKey('n', 'k', 'sms', 'ana')).not.toContain('+'); + }); +}); diff --git a/packages/notify/src/sms-channel.ts b/packages/notify/src/sms-channel.ts new file mode 100644 index 000000000..bb2380964 --- /dev/null +++ b/packages/notify/src/sms-channel.ts @@ -0,0 +1,72 @@ +// SMS as a channel, over the `smsTransport` port: per recipient, its E.164 number, the message as a +// catalog KEY rendered in the recipient's locale, one idempotency key per delivery. The provider is +// the app's adapter (`smsTransport:` in `apps//runtime.ts`); this is the reference channel. +// +// smsChannel({ message: ({ event }) => ({ key: 'sms.login_code', params: { code: event.params.code } }) }) + +import { currentLocale, translatorFor } from '@ultimat3/i18n'; +import type { DeliveryArgs, NotifyChannel } from './channel'; +import { deliveryChannel } from './channel'; +import type { Recipient } from './notification'; +import type { PushTranslate } from './push-message'; +import type { SmsTransport } from './sms-transport'; +import { assertSmsNumber, smsTransport } from './sms-transport'; + +export const SMS_CHANNEL = 'sms'; + +/** What one SMS says, as a catalog key and its params — never text (`t()`, house rule). */ +export interface NotifySmsMessage { + readonly key: string; + readonly params?: Readonly> | undefined; +} + +export interface SmsChannelOptions { + readonly message: (args: DeliveryArgs) => NotifySmsMessage; + /** + * Where the number comes from. Defaults to `recipient.to`. `undefined` back is a recipient with + * no phone — settled as sent and logged `notify.address_missing`, as `mailChannel` does. + */ + readonly numberOf?: ((recipient: Recipient) => string | undefined) | undefined; + readonly name?: string | undefined; + /** Defaults to the installed `smsTransport()`, read per delivery. A test may hand one in. */ + readonly transport?: SmsTransport | undefined; + /** Defaults to `translatorFor(locale)`. The recipient's locale, else the app's fallback. */ + readonly translate?: ((locale: string) => PushTranslate) | undefined; +} + +/** + * The key a provider dedupes on: the same for every attempt of one (notification, channel, + * recipient) — the delivery ledger's own key — and holding no number. + */ +export const smsIdempotencyKey = ( + notifier: string, + eventKey: string, + channel: string, + recipient: string, +): string => JSON.stringify([notifier, eventKey, channel, recipient]); + +export function smsChannel( + options: SmsChannelOptions, +): NotifyChannel { + const name = options.name ?? SMS_CHANNEL; + const numberOf = options.numberOf ?? ((recipient: Recipient) => recipient.to); + const translate = options.translate ?? ((locale: string) => translatorFor(locale)); + return deliveryChannel(name, async (args) => { + const { recipient, event, ctx, signal } = args; + const to = numberOf(recipient); + if (to === undefined || to === '') { + ctx.logger.warn('notify.address_missing', { recipient: recipient.id, channel: name }); + return; + } + // Refused before rendering or sending: a number that is wrong now is wrong on every retry. + assertSmsNumber(to, recipient.id); + const message = options.message(args); + const body = translate(recipient.locale ?? currentLocale())(message.key, message.params); + await (options.transport ?? smsTransport()).send({ + to, + body, + idempotencyKey: smsIdempotencyKey(event.notifier, event.key, name, recipient.id), + signal, + }); + }); +} diff --git a/packages/notify/src/sms-errors.ts b/packages/notify/src/sms-errors.ts new file mode 100644 index 000000000..5aa09722d --- /dev/null +++ b/packages/notify/src/sms-errors.ts @@ -0,0 +1,32 @@ +// Single responsibility: the SMS seam's two refusals. Registered and classified with the rest of the +// package's codes in `errors.ts`. No cause ever carries a phone number or a message body: both are +// a person's, and a cause lands in logs, traces and the dead-letter row. + +import { UltimateError } from '@ultimat3/core'; +// Bare: the titles these codes render with, and their retry classification, are registered there. +import './errors'; + +/** No transport installed, outside development and test — where the memory one stands in. */ +export class NotifySmsTransportMissingError extends UltimateError { + constructor() { + super({ + code: 'X_NOTIFY_SMS_TRANSPORT_MISSING', + cause: + 'an SMS was sent with no SmsTransport installed: outside development and test the memory transport would swallow the message, so the send is refused', + fix: 'smsTransport: yourTransport(), # in apps//runtime.ts — behavesLike(smsTransportConformance, () => yourTransport()) from @ultimat3/testing/conformance proves it', + }); + } +} + +/** A number that is not E.164. The number itself stays out of the cause: `meta.recipient` names whose. */ +export class NotifySmsNumberInvalidError extends UltimateError { + constructor(input: { readonly recipient: string | undefined }) { + super({ + code: 'X_NOTIFY_SMS_NUMBER_INVALID', + cause: + 'an SMS number is not E.164 (+, a country code, at most 15 digits in all): a local spelling is a number a provider guesses the country of', + fix: "smsChannel({ numberOf: (recipient) => recipient.to }) # answering E.164 ('+4915112345678'): store numbers that way", + ...(input.recipient === undefined ? {} : { meta: { recipient: input.recipient } }), + }); + } +} diff --git a/packages/notify/src/sms-transport.test.ts b/packages/notify/src/sms-transport.test.ts new file mode 100644 index 000000000..d6a446086 --- /dev/null +++ b/packages/notify/src/sms-transport.test.ts @@ -0,0 +1,78 @@ +// The `smsTransport` slot: memory in development and test, a coded refusal elsewhere, an installed +// transport winning until its own release — and the E.164 check every transport makes. + +import { afterEach, describe, expect, test } from 'bun:test'; +import { + assertSmsNumber, + installSmsTransport, + isSmsNumber, + memorySmsTransport, + resetSmsTransport, + smsTransport, +} from './sms-transport'; + +const saved = { ultimate: Bun.env['ULTIMATE_ENV'] }; + +afterEach(() => { + if (saved.ultimate === undefined) delete Bun.env['ULTIMATE_ENV']; + else Bun.env['ULTIMATE_ENV'] = saved.ultimate; + resetSmsTransport(); +}); + +const message = { to: '+4915112345678', body: 'hi', idempotencyKey: 'k1' }; + +describe('unit · the smsTransport slot', () => { + for (const environment of ['production', 'staging'] as const) { + test(`${environment}: the default refuses with X_NOTIFY_SMS_TRANSPORT_MISSING`, async () => { + Bun.env['ULTIMATE_ENV'] = environment; + const refused = await smsTransport() + .send(message) + .then( + () => undefined, + (thrown: unknown) => thrown, + ); + expect(refused).toBeUltimateError('X_NOTIFY_SMS_TRANSPORT_MISSING'); + expect(String((refused as { fix: unknown }).fix)).toContain('runtime.ts'); + }); + } + + test('test: the default is the memory transport, and answers', async () => { + Bun.env['ULTIMATE_ENV'] = 'test'; + expect(await smsTransport().send(message)).toMatchObject({ to: message.to }); + }); + + test('an installed transport wins until its own release', async () => { + Bun.env['ULTIMATE_ENV'] = 'production'; + const installed = memorySmsTransport('installed'); + const release = installSmsTransport(installed); + expect((await smsTransport().send(message)).transport).toBe('installed'); + expect(installed.sent).toHaveLength(1); + release(); + await expect(smsTransport().send(message)).rejects.toThrow(/X_NOTIFY_SMS_TRANSPORT_MISSING/); + }); +}); + +describe('unit · E.164', () => { + test('a number is +, a country code, at most 15 digits', () => { + expect(['+4915112345678', '+14155550123', '+999999999999999'].map(isSmsNumber)).toEqual([ + true, + true, + true, + ]); + for (const bad of ['015112345678', '+0151', '+49 151 123', '+1234567890123456', '', '+']) { + expect(isSmsNumber(bad)).toBe(false); + } + }); + + test('a refusal names the recipient and never the number', () => { + let thrown: unknown; + try { + assertSmsNumber('0151 9876543', 'ana'); + } catch (error) { + thrown = error; + } + expect(thrown).toBeUltimateError('X_NOTIFY_SMS_NUMBER_INVALID'); + expect(JSON.stringify(thrown)).not.toContain('9876543'); + expect((thrown as { meta?: unknown }).meta).toEqual({ recipient: 'ana' }); + }); +}); diff --git a/packages/notify/src/sms-transport.ts b/packages/notify/src/sms-transport.ts new file mode 100644 index 000000000..b3f62c0f7 --- /dev/null +++ b/packages/notify/src/sms-transport.ts @@ -0,0 +1,109 @@ +// The `smsTransport` port: what an SMS provider adapter implements, the memory reference, and the +// boot's slot. Twilio, Vonage, SNS are the APP's adapters (axiom 8) — this package ships the seam, +// the E.164 check every transport makes, and a transport that never leaves the process. + +import { isLocal } from '@ultimat3/core'; +import { adapterSlot } from '@ultimat3/core/adapter'; +import { NotifySmsNumberInvalidError, NotifySmsTransportMissingError } from './sms-errors'; + +/** One message, already rendered. The transport never sees catalog keys or a recipient row. */ +export interface SmsMessage { + /** E.164: `+`, a country code, at most 15 digits in all. `assertSmsNumber` is the check. */ + readonly to: string; + readonly body: string; + /** + * The same on every attempt of one delivery. A provider that dedupes reads it; one that does not + * still answers it back, so a caller can match a retry to its first send. + */ + readonly idempotencyKey: string; + /** The delivery's cancellation. Hand it to `fetch`. */ + readonly signal?: AbortSignal | undefined; +} + +export interface SmsSendResult { + /** The transport's own `name`. */ + readonly transport: string; + /** The provider's message id. Never the number, the body or the idempotency key: it is logged. */ + readonly id: string; + /** The number accepted — `message.to`. */ + readonly to: string; + readonly idempotencyKey: string; +} + +/** The port. A rejection is a failed delivery; the notifier's job decides whether it retries. */ +export interface SmsTransport { + readonly name: string; + send(message: SmsMessage): Promise; +} + +const E164 = /^\+[1-9]\d{1,14}$/; + +/** Whether `value` is an E.164 number — what every transport and `smsChannel` refuse otherwise. */ +export const isSmsNumber = (value: string): boolean => E164.test(value); + +/** Refuses a non-E.164 number with `X_NOTIFY_SMS_NUMBER_INVALID`. Every transport calls it first. */ +export function assertSmsNumber(value: string, recipient?: string): void { + if (!isSmsNumber(value)) throw new NotifySmsNumberInvalidError({ recipient }); +} + +export interface MemorySmsTransport extends SmsTransport { + /** Every accepted message, oldest first — what a test asserts on. */ + readonly sent: readonly SmsMessage[]; + clear(): void; +} + +/** + * The reference transport: keeps messages in the heap and answers a fresh id per send. Never a + * production default — `smsTransport()` refuses outside development and test instead. + */ +export function memorySmsTransport(name = 'memory'): MemorySmsTransport { + const sent: SmsMessage[] = []; + return { + name, + sent, + clear: () => { + sent.length = 0; + }, + send(message) { + assertSmsNumber(message.to); + sent.push({ to: message.to, body: message.body, idempotencyKey: message.idempotencyKey }); + return Promise.resolve({ + transport: name, + id: `sms_${crypto.randomUUID()}`, + to: message.to, + idempotencyKey: message.idempotencyKey, + }); + }, + }; +} + +let localMemory: MemorySmsTransport | undefined; + +/** The default: one memory transport per process in development and test, a refusal elsewhere. */ +const localTransport = (): SmsTransport => ({ + name: 'memory', + send(message) { + if (!isLocal()) return Promise.reject(new NotifySmsTransportMissingError()); + localMemory ??= memorySmsTransport(); + return localMemory.send(message); + }, +}); + +const slot = adapterSlot(localTransport); + +/** + * The boot's installer, on `@ultimat3/notify/adapter` only: an app names `smsTransport:` in + * `apps//runtime.ts`. Its release removes THIS install, in any order (`adapterSlot`). + */ +export function installSmsTransport(transport: SmsTransport): () => void { + return slot.install(transport); +} + +/** Back to the default, for a test that installed one and must hand the process back. */ +export function resetSmsTransport(): void { + slot.reset(); + localMemory = undefined; +} + +/** The installed transport, read per send — a channel is declared long before a boot installs. */ +export const smsTransport = (): SmsTransport => slot.current(); diff --git a/packages/realtime/CLAUDE.md b/packages/realtime/CLAUDE.md index 4655cbfc9..2a61f4bb1 100644 --- a/packages/realtime/CLAUDE.md +++ b/packages/realtime/CLAUDE.md @@ -190,7 +190,8 @@ Tier 3 package. Channels, live queries, local-first sync. One protocol for all t - **The pump has one way out, `#die`**: failure, timer, connection, then `onEnd`. **A walsender that stops answering is dead** — 3 intervals read-silent, or a confirm unsettled (`pg-replication-liveness.ts`): destroyed, no goodbye. A run is fenced by its connection. -- **A stream that ENDS, or a LOST lock, is restarted** from `lastLsn` on `retryDelayMs`, feed then +- **A stream that ENDS, or a LOST lock, is restarted** from `lastLsn` on `retryDelayMs`; a lost + connection (`connectionLost`) redials at once, then ≤ 2 s (`replicator-reconnect.ts`); feed then lock ABANDONED; `running` = pumping and no refused change owed (history, 2026-11-09). - **A slot is asked its `wal_status`**: `lost` is `X_REPLICATION_SLOT_INVALIDATED`; one created under a resume is `X_REPLICATION_SLOT_RECREATED`, drops `from`, and owes `x.change.-resync` (a node diff --git a/packages/realtime/src/replicator-reconnect.test.ts b/packages/realtime/src/replicator-reconnect.test.ts new file mode 100644 index 000000000..0351abd19 --- /dev/null +++ b/packages/realtime/src/replicator-reconnect.test.ts @@ -0,0 +1,177 @@ +// A database restart must cost the live windows seconds, not a minute. The demo took ~73 s to +// reconnect after one: every redial while the server was down grew the takeover backoff toward its +// 30 s cap, so the server came back and the replicator was still asleep. A connection the server +// ended (57P01/57P02/57P03) or the network reset is redialled at once, then on a curve capped low. + +import { describe, expect, test } from 'bun:test'; +import { InProcessTransport } from './fanout'; +import { postgresAdvisoryLock } from './pg-advisory-lock'; +import { + authOk, + commandComplete, + dataRow, + errorResponse, + FakeStream, + readyForQuery, +} from './pg-connection-fixture'; +import { ensurePostsEntity, FakeWalsender, feedOver } from './pg-replication-fixture'; +import { changeFeedReplicator, STOP_DEADLINE_MS } from './replicator'; +import { connectionLost, REPLICATOR_RECONNECT_MAX_MS } from './replicator-reconnect'; +import { manual, stubLock } from './replicator-rig-fixture'; + +const turns = async (count = 400): Promise => { + for (let tick = 0; tick < count; tick += 1) await Promise.resolve(); +}; + +/** A lock session past its handshake that answers the one lock query with granted. */ +const lockSession = (): FakeStream => { + const stream = new FakeStream(); + stream.push(authOk(), readyForQuery()); + stream.push(dataRow('t'), commandComplete('SELECT 1'), readyForQuery()); + return stream; +}; + +/** What Bun's `connect` rejects with while nothing listens: a plain Error carrying `code`. */ +const refused = (): Error => + Object.assign(new Error('Failed to connect'), { code: 'ECONNREFUSED' }); + +/** The top of every jitter range: the curve that waits longest is the one being measured. */ +const worst = (): number => 0.999_999; + +describe('the database restarts under a running replicator', () => { + test('the backend is terminated (57P01): the replicator dials again with no wait', async () => { + ensurePostsEntity(); + const sessions = [lockSession(), lockSession()]; + let lockDials = 0; + const walsenders: FakeWalsender[] = []; + const feed = feedOver(() => { + // The second walsender finds the slot still held by the backend on its way out: the + // preflight's slot-release wait stays in front of START_REPLICATION. + const server = new FakeWalsender(walsenders.length === 0 ? {} : { slotHeldFor: 2 }); + walsenders.push(server); + return Promise.resolve(server); + }); + const lock = postgresAdvisoryLock({ + url: 'postgres://repluser:hunter2@localhost:5432/app?sslmode=disable', + key: 'x:replicator:restart', + stream: () => { + const session = sessions[lockDials]; + lockDials += 1; + if (session === undefined) return expect.unreachable('an unscripted lock dial'); + return Promise.resolve(session); + }, + }); + // Timers held by hand: the claim is the WAIT the replicator asks for, never a wall-clock race. + const timers = manual(); + const replicator = changeFeedReplicator({ + feed, + lock, + transport: new InProcessTransport(), + schedule: timers.schedule, + rng: worst, + }); + expect(await replicator.start()).toBe(true); + + sessions[0]?.push( + errorResponse({ + S: 'FATAL', + C: '57P01', + M: 'terminating connection due to administrator command', + }), + ); + sessions[0]?.end(); + await turns(); + expect(replicator.running).toBe(false); + expect(replicator.stats().failure).toContain('57P01'); + // Once the redial is due, at once: the old curve asked for up to 500 ms here, then 30 s. + expect(timers.delays).toEqual([0]); + + timers.fire(); + await turns(2_000); + expect(replicator.running).toBe(true); + expect(walsenders).toHaveLength(2); + expect(walsenders[1]?.queries.some((sql) => sql.includes('pg_sleep'))).toBe(true); + const stopping = replicator.stop(); + await turns(); + sessions[1]?.push(dataRow('t'), commandComplete('SELECT 1'), readyForQuery()); + await stopping; + }); + + test('the database stays down: every redial waits seconds at most, and the first none', async () => { + ensurePostsEntity(); + let down = true; + let dials = 0; + const feed = feedOver(() => { + dials += 1; + return Promise.resolve(new FakeWalsender()); + }); + const lock = stubLock(); + const acquire = lock.tryAcquire; + // The lock is dialled first, so a server that is down refuses the lock session. + lock.tryAcquire = () => (down ? Promise.reject(refused()) : acquire()); + const timers = manual(); + const replicator = changeFeedReplicator({ + feed, + lock, + transport: new InProcessTransport(), + schedule: timers.schedule, + rng: worst, + }); + down = false; + expect(await replicator.start()).toBe(true); + down = true; + + lock.lose('postgres replication idle failed: 57P01 — terminating connection'); + await turns(); + for (let round = 0; round < 12; round += 1) { + timers.fire(); + await turns(); + } + expect(replicator.running).toBe(false); + const waits = timers.delays.filter((ms) => ms !== STOP_DEADLINE_MS); + expect(waits).toHaveLength(13); + expect(waits[0]).toBe(0); + expect(Math.max(...waits)).toBeLessThanOrEqual(REPLICATOR_RECONNECT_MAX_MS); + + down = false; + timers.fire(); + await turns(); + expect(replicator.running).toBe(true); + expect(dials).toBe(2); + await replicator.stop(); + }); +}); + +describe('connectionLost', () => { + test.each([ + ['57P01 admin shutdown', 'postgres replication stream failed: 57P01 — terminating connection'], + ['57P02 crash shutdown', 'postgres replication idle failed: 57P02 — crash of another server'], + [ + '57P03 starting up', + 'postgres replication startup failed: 57P03 — the database system is starting up', + ], + ['a reset socket', 'read ECONNRESET'], + ['a refused dial', 'connect ECONNREFUSED 127.0.0.1:5432'], + ['a lost lock session', 'the advisory lock was lost: the server closed the connection'], + [ + 'a dial that found no host', + 'postgres replication connect failed: Failed to connect — no route', + ], + ])('%s is a connection the server or the network ended', (_name, reason) => { + expect(connectionLost(reason)).toBe(true); + }); + + test('reads the code a socket error carries, not only its message', () => { + expect(connectionLost(refused())).toBe(true); + expect(connectionLost(Object.assign(new Error('socket'), { code: 'ECONNRESET' }))).toBe(true); + }); + + test.each([ + ['a refused publish', 'payload exceeds max_payload'], + ['an undecodable stream', 'X_REPLICATION_PROTOCOL: postgres replication decode: unknown tag'], + ['a privilege the role lacks', 'postgres replication query failed: 42501 — must be owner'], + ['a code that merely contains the digits', 'lsn 0/57P01AB'], + ])('%s keeps the long curve', (_name, reason) => { + expect(connectionLost(reason)).toBe(false); + }); +}); diff --git a/packages/realtime/src/replicator-reconnect.ts b/packages/realtime/src/replicator-reconnect.ts new file mode 100644 index 000000000..f8f34f5f1 --- /dev/null +++ b/packages/realtime/src/replicator-reconnect.ts @@ -0,0 +1,85 @@ +// Single responsibility: which replicator failures are the CONNECTION going away — a server shut +// down or restarting, a reset socket, a lock session that died — and the short curve they redial +// on. Measured on the demo: a database restart cost ~73 s of frozen live windows, because every +// refused redial while the server was down grew the takeover backoff toward its 30 s cap. + +import { renderThrowable } from '@ultimat3/core'; +import { type BackoffPolicy, policyDelay } from './thundering-herd'; + +/** The longest the replicator waits between two redials while the connection is what failed. */ +export const REPLICATOR_RECONNECT_MAX_MS = 2_000; + +/** + * The redial curve after a lost connection: the FIRST redial is immediate (`replicator.ts`), the + * rest climb from 250 ms to two seconds. `equal` jitter keeps a floor, so a server still starting + * is not asked again at once; there is one replicator per database, so there is no herd to spread. + */ +export const reconnectBackoff: BackoffPolicy = { + baseMs: 250, + maxMs: REPLICATOR_RECONNECT_MAX_MS, + factor: 2, + jitter: 'equal', +}; + +/** What a socket error carries in `code` (Bun's `connect` rejects with `{ code: 'ECONNREFUSED' }`). */ +const SOCKET_CODES: ReadonlySet = new Set(['ECONNRESET', 'ECONNREFUSED', 'EPIPE']); + +/** + * In the rendered failure: an admin, crash or startup shutdown (SQLSTATE 57P01/57P02/57P03, as + * `describeFields` leads with it), a socket code in a message, and the phrases this package's own + * Postgres client says when the far end went away — the dial found no listener + * (`pg-socket.ts`'s `connectFailure`), a write the dead socket refused, an idle session closed, and + * any lost advisory-lock session (it is lost only when its connection ends). + */ +const CONNECTION_TEXT = + /(? number; + readonly #rng: () => number; + #attempt = 0; + #reconnects = 0; + + constructor(long: (attempt: number) => number, rng: () => number) { + this.#long = long; + this.#rng = rng; + } + + /** The wait before the next redial, counted. */ + next(lost: boolean): number { + if (!lost) { + const wait = this.#long(this.#attempt); + this.#attempt += 1; + return wait; + } + const count = this.#reconnects; + this.#reconnects += 1; + return count === 0 ? 0 : policyDelay(reconnectBackoff, count, this.#rng); + } + + /** A change went out: the next death is a first one again, on either curve. */ + progress(): void { + this.#attempt = 0; + this.#reconnects = 0; + } +} diff --git a/packages/realtime/src/replicator-supervision.test.ts b/packages/realtime/src/replicator-supervision.test.ts index 2c6d1fbf2..721e26e5a 100644 --- a/packages/realtime/src/replicator-supervision.test.ts +++ b/packages/realtime/src/replicator-supervision.test.ts @@ -22,7 +22,8 @@ import { } from './pg-replication-fixture'; import { changeFeedReplicator, STOP_DEADLINE_MS } from './replicator'; import { parseChange } from './replicator-envelope'; -import type { Scheduler } from './thundering-herd'; +import { reconnectBackoff } from './replicator-reconnect'; +import { policyDelay, type Scheduler } from './thundering-herd'; /** A lock that counts, with the await a real session has between being asked and answering. */ const countingLock = (): AdvisoryLock & { @@ -190,7 +191,7 @@ describe('a replication stream that ends', () => { await replicator.stop(); }); - test('a restart that fails is retried on a longer delay, and stop ends the loop', async () => { + test('a restart that fails is retried, on the reconnect curve while no database answers', async () => { ensurePostsEntity(); let dials = 0; const servers: FakeWalsender[] = []; @@ -210,6 +211,7 @@ describe('a replication stream that ends', () => { transport: new InProcessTransport(), schedule: timers.schedule, backoff: { baseMs: 100, maxMs: 1_000, factor: 2, jitter: 'none' }, + rng: () => 0.5, }); await replicator.start(); servers[0]?.push(copyDone()); @@ -224,7 +226,8 @@ describe('a replication stream that ends', () => { timers.fire(); await turns(); - expect(timers.delays).toEqual([100, 200, 400]); + // A refused dial is the connection, not the change: at once, then on the seconds-capped curve. + expect(timers.delays).toEqual([100, 0, policyDelay(reconnectBackoff, 1, () => 0.5)]); timers.fire(); await turns(); @@ -235,7 +238,7 @@ describe('a replication stream that ends', () => { await turns(); // A stream that came back is not progress — only a published change is. Nothing was // published, so the wait keeps growing rather than redialling at the base forever. - expect(timers.delays).toEqual([100, 200, 400, 800]); + expect(timers.delays).toEqual([100, 0, policyDelay(reconnectBackoff, 1, () => 0.5), 200]); await replicator.stop(); timers.fire(); diff --git a/packages/realtime/src/replicator.ts b/packages/realtime/src/replicator.ts index 436cdcd35..e90cf88e6 100644 --- a/packages/realtime/src/replicator.ts +++ b/packages/realtime/src/replicator.ts @@ -19,6 +19,7 @@ import type { Transport } from './fanout'; import { encodeResync, RESYNC_SUBJECT } from './replication-resync'; import { withinStopDeadline } from './replicator-deadline'; import { encodeEnvelope } from './replicator-envelope'; +import { connectionLost, RedialCurve } from './replicator-reconnect'; import { ResyncDebt } from './replicator-resync'; import { changeSubject, normalize } from './replicator-subject'; import { @@ -151,12 +152,12 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator { unwatchLock = undefined; }; /** - * Deaths since a change was last PUBLISHED. "The stream started" is not progress: a change the - * bus refuses every time kills each new stream on its first delivery, and counting from the - * start made that a redial — two dials, a preflight, a `START_REPLICATION` — at the base delay - * forever, with the WAL retained behind it. + * Deaths since a change was last PUBLISHED, per curve. "The stream started" is not progress: a + * change the bus refuses every time kills each new stream on its first delivery, and counting + * from the start made that a redial at the base delay forever, with the WAL retained behind it. + * A lost CONNECTION (a database restart) redials at once, then on a seconds cap. */ - let attempt = 0; + const curve = new RedialCurve((count) => retryDelayMs(count), options.rng ?? Math.random); /** Counts the starts a `stop()` stopped waiting for — read by the `begin` it gave up on. */ let givenUp = 0; let refusal: Refusal | null = null; @@ -254,7 +255,7 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator { lastLsn = change.lsn; published += 1; // Progress: the next death waits the base delay again, and nothing is owed any more. - attempt = 0; + curve.progress(); if (refusal !== null) { refusal = null; if (pumping) failure = null; @@ -367,15 +368,14 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator { policyDelay(backoff, count + 1, options.rng ?? Math.random); /** - * The takeover loop. One timer at a time, each on a longer delay than the last until a change - * is PUBLISHED — which is what resets the count (`changeHandler`), not a stream coming up. - * `false` from `launch` is another process holding the lock, which is this loop working: that - * process replicates, and this one keeps asking in case it dies too. + * The takeover loop. One timer at a time, each longer than the last until a change is PUBLISHED + * (`RedialCurve`). `lost` — the connection went (`connectionLost`) — redials at once, then within + * seconds, unless a change is owed: that death is the bus's. `false` from `launch` is another + * process holding the lock: it replicates, and this one keeps asking in case it dies too. */ - const retry = (): void => { + const retry = (lost: boolean): void => { if (stopped) return; - const wait = retryDelayMs(attempt); - attempt += 1; + const wait = curve.next(lost && refusal === null); cancelRetry = schedule(() => { cancelRetry = undefined; // `pumping` already: the owner called `start()` itself while this timer was armed. @@ -384,12 +384,12 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator { void launch().then( (started) => { if (started) restarts += 1; - else retry(); + else retry(false); }, (thrown: unknown) => { failure = describeFailure(renderThrowable(thrown)); logger.warn('replicator.restart_failed', { key: options.lock.key, error: failure }); - retry(); + retry(connectionLost(thrown)); }, ); }, wait); @@ -415,7 +415,7 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator { // socket still held is refused by the run's own fence. options.feed.abandon(); options.lock.abandon(); - retry(); + retry(connectionLost(reason)); }; return { diff --git a/packages/render/src/module-loader.test.ts b/packages/render/src/module-loader.test.ts index 05810adbf..8605e2eb4 100644 --- a/packages/render/src/module-loader.test.ts +++ b/packages/render/src/module-loader.test.ts @@ -336,8 +336,11 @@ describe('an app served from /app', () => { // The order `x build` meets: a sheet can register before `loadApp` names the root, and a // classification frozen at registration would keep the answer the wrong root gave. test('naming the root reclassifies what registered before it, and moves the revision', () => { + // A sheet outside `apps//`, whose surface only the root can decide: under `apps/` the + // workspace segment already names it (`surfaces.ts`'s `WORKSPACE_SURFACE`), whatever the root. + const bare = '/app/site/page.module.scss'; setStylesheetRoot('/'); - loadStylesheet(SITE_SHEET, '.hero{color:red}'); + loadStylesheet(bare, '.hero{color:red}'); expect(stylesFor('site')).toBe(''); const before = stylesheetsRevision(); diff --git a/packages/render/src/surfaces.test.ts b/packages/render/src/surfaces.test.ts index 378426099..cf967e56f 100644 --- a/packages/render/src/surfaces.test.ts +++ b/packages/render/src/surfaces.test.ts @@ -1,9 +1,11 @@ import { describe, expect, test } from 'bun:test'; import { SurfaceBoundaryError } from './errors'; +import type { Surface } from './surfaces'; import { assertSurfaceBoundary, checkSurfaceBoundary, importGraph, + locateSurface, surfaceOf, surfaceUnder, } from './surfaces'; @@ -184,3 +186,28 @@ describe('checkSurfaceBoundary derives every edge from SURFACE_SPECS', () => { ).toEqual([]); }); }); + +// An app WORKSPACE may carry a surface's name: `apps/api/` is the natural name for an API app. +// The surface is the segment directly under `apps//`, never the workspace name itself — +// `apps/api/site/x/page.tsx` read as surface `api` was `X_ROUTE_FILE_INVALID` for every route. +/** A path, the surface it is in, and what follows the surface segment. */ +const WORKSPACE_CASES: [string, Surface, string][] = [ + ['apps/api/site/x/page.tsx', 'site', 'x/page.tsx'], + ['apps/app/app/dashboard/page.tsx', 'app', 'dashboard/page.tsx'], + ['apps/site/api/index.ts', 'api', 'index.ts'], + ['apps/shared/shared/theme.ts', 'shared', 'theme.ts'], + ['/srv/app/apps/web/site/page.tsx', 'site', 'page.tsx'], + // A checkout root that itself holds `apps///`: the app's own segment is the LAST. + ['/home/u/apps/acme/app/apps/web/site/x/page.tsx', 'site', 'x/page.tsx'], +]; + +describe('a workspace named like a surface', () => { + test.each(WORKSPACE_CASES)('%s is surface %s', (path, surface, rest) => { + expect(locateSurface(path)).toEqual({ surface, rest }); + }); + + test('a path outside apps/ is read as before', () => { + expect(surfaceOf('site/pricing/page.tsx')).toBe('site'); + expect(surfaceOf('packages/ui/src/card.tsx')).toBeNull(); + }); +}); diff --git a/packages/render/src/surfaces.ts b/packages/render/src/surfaces.ts index 0273d239e..9eb113572 100644 --- a/packages/render/src/surfaces.ts +++ b/packages/render/src/surfaces.ts @@ -70,6 +70,24 @@ export const SURFACE_SPECS = Object.freeze>({ }); const SURFACE_SEGMENT = /(?:^|\/)(site|app|api|shared)\//; +/** + * The surface directly under an app workspace, `apps///`. Read first, so a workspace + * named like a surface (`apps/api/site/…`) and a root that contains one (`/srv/app/apps/web/…`) + * resolve to the real surface; `SURFACE_SEGMENT` alone took the first matching segment, and + * `apps/api/site/x/page.tsx` was surface `api` — `X_ROUTE_FILE_INVALID` for every route there. + */ +const WORKSPACE_SURFACE = /(?<=^|\/)apps\/[^/]+\/(site|app|api|shared)\//g; + +/** + * The LAST `apps///`: an absolute path's checkout root may hold one of its own + * (`/home/u/apps/acme/app/apps/web/site/…` is surface `site`, never `app`). Lookbehind, not a + * consumed `/`, so two adjacent matches are both seen. + */ +const lastWorkspaceSurface = (path: string): RegExpExecArray | null => { + let last: RegExpExecArray | null = null; + for (const match of path.matchAll(WORKSPACE_SURFACE)) last = match as RegExpExecArray; + return last; +}; /** Where the surface segment is, and what follows it — the two halves of one match. */ export interface SurfaceLocation { @@ -86,7 +104,7 @@ export interface SurfaceLocation { */ export function locateSurface(file: string): SurfaceLocation | null { const normalized = normalize(file); - const match = SURFACE_SEGMENT.exec(normalized); + const match = lastWorkspaceSurface(normalized) ?? SURFACE_SEGMENT.exec(normalized); const found = match?.[1]; if (found === undefined || match === null) return null; return { surface: found as Surface, rest: normalized.slice(match.index + match[0].length) }; diff --git a/packages/scraping/CLAUDE.md b/packages/scraping/CLAUDE.md index 27455b44d..dd3afd707 100644 --- a/packages/scraping/CLAUDE.md +++ b/packages/scraping/CLAUDE.md @@ -7,7 +7,8 @@ Tier 4. May import tiers 0-3. Never sideways, never upward. | Exports | `src/index.ts`, explicit, no `export *` — and complete enough that a third party implements `ScrapeDriver` from it alone. A driver author needing a deep import means the seam is not a seam | | Errors | `src/errors.ts` owns the codes and their **retry classification**; `src/error-throws.ts` owns one constructor per failure mode, `src/error-throws-session.ts` the session ones (the first file is at its size ceiling) | | Files | one responsibility each, tests beside the source | -| Dependencies | `@ultimat3/core`, `@ultimat3/jobs`, `@ultimat3/schema`, `@ultimat3/storage`. **No third-party runtime dependency at all** | +| Dependencies | `@ultimat3/action` (`pdf()`, a factory over `action`), `@ultimat3/core`, `@ultimat3/jobs`, `@ultimat3/schema`, `@ultimat3/storage`; `@ultimat3/policy` for tests only. **No third-party runtime dependency at all** | +| Entry points | `.`, `./adapter` (the `scrapeDriver` port), `./cdp` (the raw-CDP launcher and connection — `cdp.ts`, imports `core` only, so `@ultimat3/testing`'s barrel and `x shot` load nothing else for it) | | Opt-in suite | `session.job.test.ts` — the pg driver and the stored event bus on a real Postgres (`TEST_DATABASE_URL`, its own database), a second OS process publishing the answer. Skips without the env. No `@ultimat3/db` here, so its `PgExecutor` is `Bun.SQL` with `prepare: false` and a hand-written array literal | Commands: `bun test ./packages/scraping/` from the repo root, `bunx tsc --noEmit -p tsconfig.json`. @@ -21,7 +22,15 @@ claims and that page does not carry is a capability an app author cannot find. reserve room for `recover: 'agent'` to import `@ultimat3/ai`; 26.0.0 deleted that variant (#709). A recover hook is a function — one that asks a model is the app's own, wrapping `llm()` (axiom 8). -Nothing in the framework imports it (`x shot` left in 22.0.0), so no sideways edge exists. +`@ultimat3/testing` (tier 5) imports `./cdp`, downward; nothing at tier 4 imports it, so no sideways edge exists. + +## The raw-CDP launcher lives here since 31.0.0 + +`cdp-launch*.ts`, `cdp-connection.ts`, `cdp-pipe.ts`, `cdp-wire-watch.ts`, `cdp-errors.ts` moved from +`@ultimat3/testing`: `pdf()` prints at RUNTIME and a runtime package cannot import a test one. The +four `X_CDP_*` codes are registered by `cdp-error-codes.ts`; testing lists them in `E2E_ERROR_CODES` +and registers only its own. These files speak raw CDP over Bun's own pipe and `WebSocket` — no +puppeteer type, so the puppeteer rule below does not reach them. ## puppeteer-core is NOT a dependency @@ -50,7 +59,8 @@ and a value cannot leak. `driver-parity.test.ts` runs the real driver's code pat | Rule | Enforced by | |---|---| -| no wait outside `clock.ts` | `clock-discipline.test.ts` scans `src/*.ts` for `setTimeout`/`setInterval`/`Bun.sleep`. `http.ts` and `robots-fetch.ts` are pinned as the exceptions — the two files that dial `fetch`, one `AbortSignal.timeout` each. The exemption covers `AbortSignal.timeout` only; a real timer in either still offends | +| no wait outside `clock.ts` | `clock-discipline.test.ts` scans `src/*.ts` for `setTimeout`/`setInterval`/`Bun.sleep`. `http.ts` and `robots-fetch.ts` are pinned as the exceptions — the two files that dial `fetch`, one `AbortSignal.timeout` each. The exemption covers `AbortSignal.timeout` only; a real timer in either still offends. The raw-CDP launcher's files are pinned BY NAME (`PROCESS_DEADLINE_FILES`): their deadlines are on a child Chrome, the OS's time, and a frozen clock would kill a starting browser | +| a PDF is the app's own route, never an error page | `pdf()` (`pdf-action.ts`) joins `route()` onto the origin and refuses anything that leaves it (`X_SCRAPE_PDF_ROUTE_INVALID`); `pdf-render.ts` reads the document's status after `load` + `document.fonts.ready` and refuses 4xx/5xx (`X_SCRAPE_PDF_ROUTE_FAILED`), and closes the browser it launched in a `finally` (`pdf-render.test.ts`, `pdf-action.test.ts`). At most `maxConcurrent` browsers live per renderer (`pdf-slots.ts`, queue bounded, `X_SCRAPE_PDF_BUSY`), the request's signal kills its own, and `pdf-fence.ts` sends headers to the route's origin only and blocks every other origin (`pdf-render-slots.test.ts`, `pdf-render-fence.test.ts`) | | the fake never drifts from the real driver | `driver-parity.test.ts` runs one suite against `fake`, `fixture` and the puppeteer path, and pins the one honest divergence (no layout engine offline, so no box and no hit-target) | | an unrecorded request throws | `html-target.ts` and `http-recorded.ts`. **One request escapes: the robots read.** Offline drivers under the default `robots: 'obey'` fire a real `fetch` for `/robots.txt` (the gate is built in `scrape-run.ts`, blind to the driver), and under `bun test` the sealed network refuses it: unreachable, so `X_SCRAPE_ROBOTS_DISALLOWED`. Declare `robots: { ignore: '' }` on an offline scrape → `wiki/Known-Gaps.md` | | `allowHosts` is enforced, never advisory | `intercept.ts` is the single decision, asked by every driver AND by the HTTP leg, before a byte leaves. The CDP driver asks it at the BROWSER too (`armBrowser`: `Fetch` on `browser.target()`), so a popup is screened (`cdp-arm-targets.test.ts`). A wildcard-admitted name is resolved and pinned on the HTTP leg and robots read (`pinned-host.ts`), and screened on the browser leg (`browser-dial.ts`). Open: Chrome's rebinding, WebSockets (`wiki/Known-Gaps.md`) | diff --git a/packages/scraping/README.md b/packages/scraping/README.md index 49ab79fe4..dd7982f0e 100644 --- a/packages/scraping/README.md +++ b/packages/scraping/README.md @@ -396,6 +396,61 @@ The offline drivers honour it, so the crop is provable with no Chrome — `fakeB different deterministic bytes per rectangle, which is what lets a component screenshot be tested on a machine with no browser — the case CI is. +## A route printed to PDF — `pdf()`, a factory over `action` + +```ts +import type { Ctx } from '@ultimat3/core'; +import { can } from '@ultimat3/policy'; +import { t } from '@ultimat3/schema'; +import { pdf } from '@ultimat3/scraping'; +import { disk } from '@ultimat3/storage'; + +/** The app's own short-lived signer for the print route — never the framework's. */ +declare function printToken(ctx: Ctx): string; + +export const invoicePdf = pdf({ + input: t.object({ id: t.uuid }), + policy: can('invoices:read'), + route: ({ input }) => `/invoices/${input.id}/print`, // a PATH on this app, never a URL + key: ({ input, ctx }) => `invoices/${ctx.actor.orgId}/${input.id}.pdf`, + storage: () => disk('documents'), // a thunk: read per call + layout: { format: 'a4', marginMm: { top: 12, bottom: 12 }, printBackground: true }, + headers: ({ ctx }) => ({ authorization: `Bearer ${printToken(ctx)}` }), // the app's own token +}); +// → { key, bytes, contentType: 'application/pdf' } — HTTP, OpenAPI, MCP, .job(), .as() included +``` + +| Rule | Detail | +|---|---| +| it is an `action` | input schema, policy, `rateLimit`, `meter`, `audit` are the action's: refused before a browser starts (`pdf-action.test.ts`). `PRIMITIVE_FACTORIES` row `pdf` | +| only the app's own origin | `route()` must answer a path; a URL, `//host` or a backslash is `X_SCRAPE_PDF_ROUTE_INVALID` (500, terminal). The origin is `APP_URL`, read per call, or `origin: () => …`; neither is `X_SCRAPE_PDF_ORIGIN_MISSING` (500, terminal) | +| an error page is never a document | the page's own status is read after `load` and `document.fonts.ready`: 4xx/5xx is `X_SCRAPE_PDF_ROUTE_FAILED` (502; a 4xx terminal, a 5xx/429 retryable) | +| the layout is checked where `pdf()` is written | `pdfLayoutOf`, as `page.pdf()`: `X_SCRAPE_PDF_OPTIONS_INVALID` at module load | +| one browser per render | `chromePdfRenderer()` launches Chrome through `@ultimat3/scraping/cdp`, prints over raw CDP (`Page.printToPDF`, `pdf-render.ts`, `pdf-cdp-params.ts`) and closes it in a `finally`; no browser outlives a request, none is shared between tenants. A pool is a measured next step, not shipped | +| a bounded number of browsers | at most `maxConcurrent` (default 2 — each Chrome is 150-300 MB) per renderer, and every `pdf()` naming no `renderer` shares ONE. The rest queue for `queueTimeoutMs` (default `timeoutMs`) on the clock, then `X_SCRAPE_PDF_BUSY` (503, retryable) — `pdf-slots.ts`, `pdf-render-slots.test.ts` | +| a dropped request ends its render | the action's `ctx.signal` reaches the renderer: aborted in the queue, it leaves without a slot; aborted mid-print, its browser is killed and the slot freed, and the request's own reason is thrown | +| headers reach the route's origin only | `Fetch.enable` on the page's session (`pdf-fence.ts`): a request to the route's exact origin (scheme, host, port) continues with `headers` added; every other — a user's ``, a redirect off-site, `169.254.169.254` — is failed `BlockedByClient` before it leaves. Never `Network.setExtraHTTPHeaders`, which sent the token everywhere (`pdf-render-fence.test.ts`) | +| the sandbox stays on | the default launch keeps Chrome's sandbox (`pdfLaunchOptions`): a print renders user content. The runtime image ships no Chrome; the app's image supplies it and runs it as non-root, which needs unprivileged user namespaces (or Chrome's setuid helper). A container that cannot grant them fails `X_CDP_LAUNCH_FAILED` quoting "No usable sandbox" — opt out there in one visible line, `chromePdfRenderer({ sandbox: false })`. The e2e driver and `x shot` keep `--no-sandbox` for CI runners | +| the bytes go to storage | `storage().put(key, bytes, { contentType: 'application/pdf' })`; the answer is the key, never the bytes. Serve it with a signed URL | +| tested with no Chrome | `renderer:` takes any `PdfRenderer`; `chromePdfRenderer({ launch })` takes any `{ connection, close }` — `pdf-render.test.ts` drives it over a fake CDP connection | + +Open (`As of 2026-11`): the route must be readable by the printing browser — a private one forwards +the app's own short-lived credential through `headers`; no session is minted for it. + +## The raw-CDP browser — `@ultimat3/scraping/cdp` + +The launcher and connection the e2e driver and `x shot` have used since 22.0.0, moved here from +`@ultimat3/testing` in 31.0.0 because `pdf()` needs them at RUNTIME and testing is a test package. +Its own entry, off `.`, so a caller that only launches loads no job or storage module. +This is every name's one home: `@ultimat3/testing` no longer re-exports them (31.0.0, `X_HELPER_COPY`); its `E2E_ERROR_CODES` still lists the four codes. + +| Export | What | +|---|---| +| `findChrome(env)` · `launchChrome({ executable, timeoutMs })` · `launchFoundChrome(env, timeoutMs)` | `CHROME_PATH`, then the platform's install paths; a throwaway profile, the pipe wire, one relaunch | +| `cdpConnect({ endpoint, timeoutMs })` · `cdpConnectOver(transport, timeoutMs)` | a CDP connection over a remote WebSocket or any whole-message transport | +| `CdpBrowserMissingError` · `CdpLaunchFailedError` · `CdpCallFailedError` · `CdpTimeoutError` | `X_CDP_BROWSER_MISSING` · `X_CDP_LAUNCH_FAILED` · `X_CDP_CALL_FAILED` · `X_CDP_TIMEOUT`, registered by `cdp-error-codes.ts` | +| `CHROME_CANDIDATES` · `CONTAINER_CHROME_ARGS` · `chromeLaunchFlags` · `LAUNCH_*` · `CLOSE_GRACE_MS` · `LAUNCHED_CLOSE_MS` | the flags and the deadlines, derived, never restated | + ## What it owns | Module | Owns | @@ -414,6 +469,8 @@ on a machine with no browser — the case CI is. | `secrets.ts` / `url-secrets.ts` / `browser-record.ts` | what may leave this package — declared secrets, values learned mid-run (`conceal`), the credential-bearing parts of a URL — and how a browser's own string map is read | | `expect.ts` | the silent-green alarm | | `watchdog.ts` | the wedge and zombie discipline | +| `pdf-action.ts` / `pdf-render.ts` / `pdf-cdp-params.ts` / `pdf-slots.ts` / `pdf-fence.ts` | `pdf()` over `action`; one page printed over raw CDP; a layout in the protocol's words; the bound on live browsers; the request fence | +| `cdp.ts` / `cdp-launch*.ts` / `cdp-connection.ts` / `cdp-pipe.ts` / `cdp-wire-watch.ts` / `cdp-errors.ts` | `@ultimat3/scraping/cdp`: the raw-CDP launcher and connection (moved from `@ultimat3/testing` in 31.0.0) | | `capture-clip.ts` | the one framing rule — crop, whole page, or a refusal — checked before any driver sees it | | `errors.ts` / `error-throws.ts` / `error-throws-session.ts` | this package's `X_*` codes and their retry classification | diff --git a/packages/scraping/package.json b/packages/scraping/package.json index cb70557f4..d2648be1d 100644 --- a/packages/scraping/package.json +++ b/packages/scraping/package.json @@ -25,7 +25,8 @@ }, "exports": { ".": "./src/index.ts", - "./adapter": "./src/adapter.ts" + "./adapter": "./src/adapter.ts", + "./cdp": "./src/cdp.ts" }, "files": [ "src", @@ -42,6 +43,7 @@ "test": "bun test" }, "dependencies": { + "@ultimat3/action": "30.9.0", "@ultimat3/core": "30.9.0", "@ultimat3/jobs": "30.9.0", "@ultimat3/schema": "30.9.0", diff --git a/packages/testing/src/cdp-connection.test.ts b/packages/scraping/src/cdp-connection.test.ts similarity index 100% rename from packages/testing/src/cdp-connection.test.ts rename to packages/scraping/src/cdp-connection.test.ts diff --git a/packages/testing/src/cdp-connection.ts b/packages/scraping/src/cdp-connection.ts similarity index 93% rename from packages/testing/src/cdp-connection.ts rename to packages/scraping/src/cdp-connection.ts index faa7502c3..471fc89af 100644 --- a/packages/testing/src/cdp-connection.ts +++ b/packages/scraping/src/cdp-connection.ts @@ -1,20 +1,19 @@ // One responsibility: a Chrome DevTools Protocol connection — request framing, response -// correlation, and the per-call deadline — over a TRANSPORT that moves whole messages. Two exist: -// the pipe a launched Chrome is driven over (`cdp-pipe.ts`, what the e2e driver uses) and a remote -// browser's WebSocket (`cdpConnect` below). Launching is `cdp-launch.ts`; the page surface is -// `cdp-e2e-page.ts`; this file knows nothing about either. -// -// **No library, and that is the point rather than an economy.** `packages/scraping/src/cdp-port.ts` -// declares a ~25-method port because `ScrapePage` is a full scraping surface, and its intended -// implementation is `puppeteer-core`. `E2eBrowserPage` is FIVE methods, and CDP's wire format is -// one JSON object with an `id` — so the whole thing an e2e driver needs is this file plus two -// small ones, on Bun's native `WebSocket`, with no dependency to add to a repo whose first -// non-negotiable is that Bun's natives replace most of them. +// correlation, the per-call deadline — over a TRANSPORT that moves whole messages: a launched +// Chrome's pipe (`cdp-pipe.ts`) or a remote browser's WebSocket (`cdpConnect`). It knows neither +// the launch (`cdp-launch.ts`) nor a page surface (testing's `cdp-e2e-page.ts`, `pdf-render.ts`). import { assert, stringField } from '@ultimat3/core'; import { CdpCallFailedError, CdpTimeoutError } from './cdp-errors'; import { wireWatch } from './cdp-wire-watch'; +// **No library, and that is the point rather than an economy.** `cdp-port.ts` declares a ~25-method +// port because `ScrapePage` is a full scraping surface, and its intended implementation is +// `puppeteer-core`. `E2eBrowserPage` is FIVE methods, and CDP's wire format is one JSON object with +// an `id` — so the whole thing an e2e driver needs is this file plus two small ones, on Bun's +// native `WebSocket`, with no dependency to add to a repo whose first non-negotiable is that Bun's +// natives replace most of them. + /** One CDP result. `unknown` because every payload here is somebody else's JSON. */ export interface CdpResult { readonly result?: Record | undefined; diff --git a/packages/scraping/src/cdp-error-codes.ts b/packages/scraping/src/cdp-error-codes.ts new file mode 100644 index 000000000..1850821c3 --- /dev/null +++ b/packages/scraping/src/cdp-error-codes.ts @@ -0,0 +1,28 @@ +// The registry for the raw-CDP browser's four `X_CDP_*` codes — the launcher, the connection and +// the calls over it. Owned here since 31.0.0, when the launcher moved from `@ultimat3/testing` to +// its runtime home; `@ultimat3/testing` re-reads these titles and registers only its own `X_E2E_*`. +import { registerErrorCodes } from '@ultimat3/core'; + +// Four and not one, because the four repairs differ: install a browser, read the browser's own +// stderr, look at the page, raise a deadline. +export const CDP_ERROR_CODES = [ + 'X_CDP_BROWSER_MISSING', + 'X_CDP_LAUNCH_FAILED', + 'X_CDP_CALL_FAILED', + 'X_CDP_TIMEOUT', +] as const; + +export type CdpErrorCode = (typeof CDP_ERROR_CODES)[number]; + +export const CDP_ERROR_TITLES = Object.freeze>({ + X_CDP_BROWSER_MISSING: 'no Chrome or Chromium is installed for the e2e driver to launch', + X_CDP_LAUNCH_FAILED: 'the browser started and never announced a DevTools endpoint', + X_CDP_CALL_FAILED: 'the browser refused a DevTools call', + X_CDP_TIMEOUT: 'a DevTools call did not answer inside its deadline', +}); + +// Owned here and borrowed by nobody, so unconditional: a second package claiming one must fail as +// X_ERROR_CODE_DUPLICATE rather than quietly keep whichever title registered first. +registerErrorCodes( + Object.fromEntries(Object.entries(CDP_ERROR_TITLES).map(([code, title]) => [code, { title }])), +); diff --git a/packages/testing/src/cdp-errors.ts b/packages/scraping/src/cdp-errors.ts similarity index 90% rename from packages/testing/src/cdp-errors.ts rename to packages/scraping/src/cdp-errors.ts index 479466350..85c01fbc1 100644 --- a/packages/testing/src/cdp-errors.ts +++ b/packages/scraping/src/cdp-errors.ts @@ -1,10 +1,10 @@ -// One constructor per way the raw-CDP e2e browser refuses. Every cause quotes a value that came -// out of a BROWSER or off a spawned process's stderr, so every one is rendered rather than -// interpolated — the rule `e2e-errors.ts` already states. +// One constructor per way the raw-CDP browser refuses — the e2e driver's, `x shot`'s and `pdf()`'s. +// Every cause quotes a value that came out of a BROWSER or off a spawned process's stderr, so +// every one is rendered rather than interpolated. import { isFixShellSafe, renderCauseValue, UltimateError } from '@ultimat3/core'; // Bare: the titles these constructors' codes render with are registered there. -import './e2e-error-codes'; +import './cdp-error-codes'; /** * No browser to drive. This is the one an author meets first, so its fix names the two ways out: @@ -15,7 +15,7 @@ export class CdpBrowserMissingError extends UltimateError { super({ code: 'X_CDP_BROWSER_MISSING', cause: `no Chrome or Chromium executable was found — tried ${renderCauseValue(input.tried)}`, - fix: 'set CHROME_PATH to a Chrome or Chromium binary (GitHub-hosted ubuntu runners ship one at /usr/bin/google-chrome), or skip the browser-backed e2e suite by leaving it unset', + fix: 'export CHROME_PATH=/usr/bin/google-chrome # any Chrome or Chromium binary (GitHub-hosted ubuntu runners ship this one); unset, a browser-backed e2e suite skips', }); } } @@ -90,7 +90,7 @@ export class CdpCallFailedError extends UltimateError { super({ code: 'X_CDP_CALL_FAILED', cause: `the browser refused ${renderCauseValue(input.method)}: ${renderCauseValue(input.detail)}`, - fix: 'print what the page had: await page.evaluate(() => document.body.innerHTML) — a call that refuses the same way means the browser itself is gone, so read the launch above it', + fix: 'await page.evaluate(() => document.body.innerHTML) # what the page had; a call that refuses the same way means the browser itself is gone, so read the launch above it', }); } } @@ -168,7 +168,7 @@ export class CdpTimeoutError extends UltimateError { : `${base} — ${READING[reading](observed)}`, fix: reading === undefined || reading === 'no-answer' - ? 'raise timeoutMs on installE2eDriver({ timeoutMs }), or find the request the page is still waiting on — a navigation that never settles is an app that never finishes its response' + ? 'installE2eDriver({ page, baseUrl, timeoutMs: 60_000 }) # a larger deadline — or find the request the page still waits on: a navigation that never settles is an app that never finishes its response' : // `--json`: the rerun's `meta.reading` and `meta.framesDropped` are only printed there. 'x test e2e --json', ...(observed === undefined diff --git a/packages/testing/src/cdp-launch-attempt.ts b/packages/scraping/src/cdp-launch-attempt.ts similarity index 94% rename from packages/testing/src/cdp-launch-attempt.ts rename to packages/scraping/src/cdp-launch-attempt.ts index b2df12af8..39b1c3bf7 100644 --- a/packages/testing/src/cdp-launch-attempt.ts +++ b/packages/scraping/src/cdp-launch-attempt.ts @@ -47,6 +47,12 @@ export interface LaunchAttemptOptions { readonly timeoutMs: number; /** The first call's deadline: how long a cold start may take. */ readonly launchTimeoutMs: number; + /** + * The longest the browser may live at all, from its spawn: past it the OS process is killed + * (`SIGKILL`) even if no `close()` ever ran — a caller that crashed, a hook Bun cut short. The + * normal end is still `close()`; this is the bound the spawn-timeout guard asks for. + */ + readonly lifetimeMs: number; } export type LaunchAttemptResult = @@ -220,5 +226,9 @@ function spawnBrowser(options: LaunchAttemptOptions, profileDir: string, platfor return Bun.spawn([options.executable, ...options.flags(profileDir)], { stdio: [stdin, stdout, stderr, ...extra], detached: platform !== 'win32', + // A backstop, never the close: `close()` ends a browser its caller is done with. This ends one + // whose caller vanished, so no Chrome outlives every reason it was started for. + timeout: options.lifetimeMs, + killSignal: 'SIGKILL', }); } diff --git a/packages/testing/src/cdp-launch-candidates.ts b/packages/scraping/src/cdp-launch-candidates.ts similarity index 100% rename from packages/testing/src/cdp-launch-candidates.ts rename to packages/scraping/src/cdp-launch-candidates.ts diff --git a/packages/testing/src/cdp-launch-probe.test.ts b/packages/scraping/src/cdp-launch-probe.test.ts similarity index 100% rename from packages/testing/src/cdp-launch-probe.test.ts rename to packages/scraping/src/cdp-launch-probe.test.ts diff --git a/packages/testing/src/cdp-launch-probe.ts b/packages/scraping/src/cdp-launch-probe.ts similarity index 100% rename from packages/testing/src/cdp-launch-probe.ts rename to packages/scraping/src/cdp-launch-probe.ts diff --git a/packages/testing/src/cdp-launch-profile.test.ts b/packages/scraping/src/cdp-launch-profile.test.ts similarity index 100% rename from packages/testing/src/cdp-launch-profile.test.ts rename to packages/scraping/src/cdp-launch-profile.test.ts diff --git a/packages/testing/src/cdp-launch-profile.ts b/packages/scraping/src/cdp-launch-profile.ts similarity index 100% rename from packages/testing/src/cdp-launch-profile.ts rename to packages/scraping/src/cdp-launch-profile.ts diff --git a/packages/testing/src/cdp-launch-reap.test.ts b/packages/scraping/src/cdp-launch-reap.test.ts similarity index 100% rename from packages/testing/src/cdp-launch-reap.test.ts rename to packages/scraping/src/cdp-launch-reap.test.ts diff --git a/packages/testing/src/cdp-launch-reap.ts b/packages/scraping/src/cdp-launch-reap.ts similarity index 100% rename from packages/testing/src/cdp-launch-reap.ts rename to packages/scraping/src/cdp-launch-reap.ts diff --git a/packages/testing/src/cdp-launch-relaunch.test.ts b/packages/scraping/src/cdp-launch-relaunch.test.ts similarity index 100% rename from packages/testing/src/cdp-launch-relaunch.test.ts rename to packages/scraping/src/cdp-launch-relaunch.test.ts diff --git a/packages/testing/src/cdp-launch-wire.test.ts b/packages/scraping/src/cdp-launch-wire.test.ts similarity index 100% rename from packages/testing/src/cdp-launch-wire.test.ts rename to packages/scraping/src/cdp-launch-wire.test.ts diff --git a/packages/testing/src/cdp-launch-wire.ts b/packages/scraping/src/cdp-launch-wire.ts similarity index 100% rename from packages/testing/src/cdp-launch-wire.ts rename to packages/scraping/src/cdp-launch-wire.ts diff --git a/packages/testing/src/cdp-launch.test.ts b/packages/scraping/src/cdp-launch.test.ts similarity index 92% rename from packages/testing/src/cdp-launch.test.ts rename to packages/scraping/src/cdp-launch.test.ts index d6d4d8872..6335b7260 100644 --- a/packages/testing/src/cdp-launch.test.ts +++ b/packages/scraping/src/cdp-launch.test.ts @@ -12,6 +12,7 @@ import { tmpdir } from 'node:os'; // why: joining the script and marker paths. import { join } from 'node:path'; import { + BROWSER_LIFETIME_MS, CHROME_CANDIDATES, CHROME_PATH_ENV, CLOSE_GRACE_MS, @@ -221,6 +222,11 @@ describe('chromeLaunchFlags', () => { // sat on the far side of that stall, which is the intermittent `X_CDP_TIMEOUT` a full e2e run // reported for suites that passed alone. `--use-mock-keychain` is the same question on macOS. const flags = chromeLaunchFlags('/tmp/profile'); + // The sandboxed launch (`pdf()`'s) drops the one switch and keeps the rest. + const sandboxed = chromeLaunchFlags('/tmp/profile', 'pipe', true); + expect(sandboxed).not.toContain('--no-sandbox'); + expect(sandboxed).toContain('--disable-dev-shm-usage'); + expect(flags).toContain('--no-sandbox'); expect(flags).toContain('--password-store=basic'); expect(flags).toContain('--use-mock-keychain'); @@ -304,6 +310,37 @@ describe.skipIf(WINDOWS)('launchChrome', () => { } }); + // The spawn's own bound (`lifetimeMs`, the spawn-timeout guard): a caller that crashed or a hook + // Bun cut short never calls `close()`, and the browser must not outlive every reason it ran for. + test('a browser nobody closes is killed by the OS at its lifetime, SIGTERM ignored or not', async () => { + const { fake, dir } = await fakeBrowser( + `trap '' TERM\n${ANSWER}while true; do sleep 1; done\n`, + ); + try { + const launched = await launchChrome({ executable: fake, timeoutMs: 5_000, lifetimeMs: 400 }); + const answer = await launched.connection + .send('Browser.getVersion', {}, undefined, 3_000) + .then( + () => 'answered', + (error: unknown) => (error as { code?: string }).code, + ); + // Never answered past the first call, and the wire closed on the kill: not the 3 s deadline. + expect(answer).toBe('X_CDP_CALL_FAILED'); + await launched.close(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, 10_000); + + test('a lifetime that is not a positive count is refused before a process starts', async () => { + const error = await launchChrome({ + executable: NOWHERE, + timeoutMs: 5_000, + lifetimeMs: Number.NaN, + }).catch((e: unknown) => e); + expect(error).toBeUltimateError('X_INVARIANT'); + }); + test('close() kills a browser that ignores SIGTERM, and still resolves — bounded', async () => { const { fake, dir } = await fakeBrowser( `trap '' TERM\n${ANSWER}while true; do sleep 1; done\n`, @@ -412,6 +449,7 @@ describe.skipIf(realChrome === undefined && !browserRequired)( wire, timeoutMs: 10_000, launchTimeoutMs: 60_000, + lifetimeMs: BROWSER_LIFETIME_MS, }); if (!started.ok) expect.unreachable(`no launch: ${JSON.stringify(started.failure)}`); try { diff --git a/packages/testing/src/cdp-launch.ts b/packages/scraping/src/cdp-launch.ts similarity index 84% rename from packages/testing/src/cdp-launch.ts rename to packages/scraping/src/cdp-launch.ts index 4caae0e78..946b0c239 100644 --- a/packages/testing/src/cdp-launch.ts +++ b/packages/scraping/src/cdp-launch.ts @@ -1,6 +1,7 @@ // One responsibility: WHICH Chrome, with which flags, and how many starts it gets — the candidate // list, the launch deadline and the one relaunch. One start of the process is -// `cdp-launch-attempt.ts`; the connection is `cdp-connection.ts`, the page `cdp-e2e-page.ts`. +// `cdp-launch-attempt.ts`; the connection is `cdp-connection.ts`. Its callers: `pdf()` here, and +// `@ultimat3/testing`'s e2e driver and `x shot` through `@ultimat3/scraping/cdp`. import { finiteCount } from '@ultimat3/core'; import type { CdpLaunchAttempt } from './cdp-errors'; @@ -78,11 +79,15 @@ export const CONTAINER_CHROME_ARGS: readonly string[] = ['--no-sandbox', '--disa export const chromeLaunchFlags = ( profileDir: string, wire: CdpWire = defaultWire(process.platform), + sandbox = false, ): readonly string[] => [ '--headless=new', wireFlag(wire), `--user-data-dir=${profileDir}`, - ...CONTAINER_CHROME_ARGS, + // `sandbox: true` drops `--no-sandbox` alone: `/dev/shm` is as small in a production container. + ...(sandbox + ? CONTAINER_CHROME_ARGS.filter((arg) => arg !== '--no-sandbox') + : CONTAINER_CHROME_ARGS), '--disable-gpu', // The cookie store's encryption key comes from the OS keyring, asked over D-Bus on the first // cookie access — which is the first navigation. With no keyring answering, Chrome waits out the @@ -129,6 +134,14 @@ export const LAUNCH_ATTEMPTS = 2; */ export const LAUNCH_BUDGET_MS = LAUNCH_ATTEMPTS * (LAUNCH_TIMEOUT_MS + LAUNCH_REAP_MS); +/** + * The longest a launched browser lives when its caller names no bound: one hour from its spawn, then + * `SIGKILL` from the OS. Never the way a browser ends — `close()` is — but the way one ends whose + * caller vanished. An hour, because an e2e lease holds ONE browser for a whole run; a caller with a + * shorter job (`pdf()`'s one render) passes its own `lifetimeMs`. + */ +export const BROWSER_LIFETIME_MS = 60 * 60_000; + export interface LaunchOptions { readonly executable: string; /** Every CDP call's deadline once the browser is up. */ @@ -137,8 +150,19 @@ export interface LaunchOptions { readonly launchTimeoutMs?: number | undefined; /** The wire. Defaults to this platform's (`defaultWire`): the pipe, and on Windows the port. */ readonly wire?: CdpWire | undefined; + /** From each start, the longest the process may live before the OS kills it. `BROWSER_LIFETIME_MS`. */ + readonly lifetimeMs?: number | undefined; + /** + * Keep Chrome's sandbox — no `--no-sandbox`. Off by default, for the e2e driver and `x shot` on CI + * runners that cannot grant it; ON for `pdf()`, which renders user content in production + * (`pdfLaunchOptions`, `pdf-render.ts`, says why). + */ + readonly sandbox?: boolean | undefined; } +/** What `launchFoundChrome` passes through to `launchChrome`, beside the deadline. */ +export type FoundChromeOptions = Pick; + /** * Start Chrome on a throwaway profile and answer once it has answered one CDP call. A browser that * dies or stays silent is started once more; twice is `X_CDP_LAUNCH_FAILED` with each start's own @@ -152,15 +176,22 @@ export async function launchChrome(options: LaunchOptions): Promise chromeLaunchFlags(profileDir, wire), + flags: (profileDir) => chromeLaunchFlags(profileDir, wire, options.sandbox === true), wire, timeoutMs: options.timeoutMs, launchTimeoutMs, + lifetimeMs, }); if (started.ok) return started.browser; failures.push(started.failure); @@ -175,10 +206,11 @@ export async function launchChrome(options: LaunchOptions): Promise>, timeoutMs: number, + options: FoundChromeOptions = {}, ): Promise { const executable = await findChrome(env); if (executable === undefined) { throw new CdpBrowserMissingError({ tried: chromeCandidates(process.platform, env) }); } - return launchChrome({ executable, timeoutMs }); + return launchChrome({ executable, timeoutMs, ...options }); } diff --git a/packages/testing/src/cdp-pipe.test.ts b/packages/scraping/src/cdp-pipe.test.ts similarity index 100% rename from packages/testing/src/cdp-pipe.test.ts rename to packages/scraping/src/cdp-pipe.test.ts diff --git a/packages/testing/src/cdp-pipe.ts b/packages/scraping/src/cdp-pipe.ts similarity index 100% rename from packages/testing/src/cdp-pipe.ts rename to packages/scraping/src/cdp-pipe.ts diff --git a/packages/testing/src/cdp-wire-watch.test.ts b/packages/scraping/src/cdp-wire-watch.test.ts similarity index 98% rename from packages/testing/src/cdp-wire-watch.test.ts rename to packages/scraping/src/cdp-wire-watch.test.ts index f884c9d0e..5c769c758 100644 --- a/packages/testing/src/cdp-wire-watch.test.ts +++ b/packages/scraping/src/cdp-wire-watch.test.ts @@ -67,7 +67,7 @@ describe('X_CDP_TIMEOUT says which of three things happened', () => { expect(error.cause).toContain('the target did not answer'); expect(error.cause).toContain('the page navigated 1 time(s) since the call'); // A silent target is not answered by a rerun: this reading keeps the timeoutMs instruction. - expect(error.fix).toContain('installE2eDriver({ timeoutMs })'); + expect(error.fix).toContain('timeoutMs: 60_000'); }); test('the reply was lost in transport: frames arrived after the call that nothing could parse', async () => { diff --git a/packages/testing/src/cdp-wire-watch.ts b/packages/scraping/src/cdp-wire-watch.ts similarity index 100% rename from packages/testing/src/cdp-wire-watch.ts rename to packages/scraping/src/cdp-wire-watch.ts diff --git a/packages/scraping/src/cdp.ts b/packages/scraping/src/cdp.ts new file mode 100644 index 000000000..4513af11d --- /dev/null +++ b/packages/scraping/src/cdp.ts @@ -0,0 +1,46 @@ +// `@ultimat3/scraping/cdp` — the raw-CDP browser as a runtime: find a Chrome, launch it over its +// pipe, talk to it, and the four `X_CDP_*` refusals. Its own entry, off the `.` barrel: a caller +// that only launches (`@ultimat3/testing`'s e2e driver, `x shot`) loads no job, storage or scrape +// module for it. `pdf()` on the barrel prints through the same launcher. + +export type { + CdpConnection, + CdpConnectionOptions, + CdpEventListener, + CdpResult, + CdpTransport, +} from './cdp-connection'; +export { cdpConnect, cdpConnectOver } from './cdp-connection'; +export type { CdpErrorCode } from './cdp-error-codes'; +export { CDP_ERROR_CODES, CDP_ERROR_TITLES } from './cdp-error-codes'; +export type { + CdpLaunchAttempt, + CdpTargetGone, + CdpTimeoutObservation, + CdpTimeoutReading, +} from './cdp-errors'; +export { + CdpBrowserMissingError, + CdpCallFailedError, + CdpLaunchFailedError, + CdpTimeoutError, +} from './cdp-errors'; +export type { CdpWire, FoundChromeOptions, LaunchedBrowser, LaunchOptions } from './cdp-launch'; +export { + BROWSER_LIFETIME_MS, + CHROME_CANDIDATES, + CHROME_PATH_ENV, + CLOSE_GRACE_MS, + CONTAINER_CHROME_ARGS, + chromeCandidates, + chromeLaunchFlags, + defaultWire, + findChrome, + LAUNCH_ATTEMPTS, + LAUNCH_BUDGET_MS, + LAUNCH_REAP_MS, + LAUNCH_TIMEOUT_MS, + launchChrome, + launchFoundChrome, +} from './cdp-launch'; +export { LAUNCHED_CLOSE_MS } from './cdp-launch-attempt'; diff --git a/packages/scraping/src/clock-discipline.test.ts b/packages/scraping/src/clock-discipline.test.ts index 1df6f9c58..074194b70 100644 --- a/packages/scraping/src/clock-discipline.test.ts +++ b/packages/scraping/src/clock-discipline.test.ts @@ -7,6 +7,22 @@ import { describe, expect, test } from 'bun:test'; /** Every file that dials the platform's own `fetch`, and no others. */ const DEADLINE_FILES = new Set(['http.ts', 'robots-fetch.ts']); +/** + * The raw-CDP launcher and its wire (moved from `@ultimat3/testing` in 31.0.0). Their deadlines are + * on a CHILD PROCESS — a Chrome that has not answered, a SIGTERM's grace before SIGKILL — and + * belong to the operating system's time, not the run's: under a frozen test clock they would fire + * on the next microtask and kill a browser that was starting. Pinned by name, so a new file is not. + */ +const PROCESS_DEADLINE_FILES = new Set([ + 'cdp-connection.ts', + 'cdp-launch.ts', + 'cdp-launch-attempt.ts', + 'cdp-launch-probe.ts', + 'cdp-launch-profile.ts', + 'cdp-launch-reap.ts', + 'cdp-launch-wire.ts', +]); + const TIMERS = /\b(?:setTimeout|setInterval|Bun\.sleep|AbortSignal\.timeout)\s*\(/; describe('unit · no file here may make time pass — core’s SleepClock does', () => { @@ -19,7 +35,7 @@ describe('unit · no file here may make time pass — core’s SleepClock does', // Comments quote the banned spellings on purpose (they explain why the rule exists), so the // scan reads code only — a rule that could be defeated by a comment is not a rule. const code = source.replaceAll(/\/\/[^\n]*|\/\*[\S\s]*?\*\//g, ''); - if (!TIMERS.test(code)) continue; + if (!TIMERS.test(code) || PROCESS_DEADLINE_FILES.has(path)) continue; // These hand ONE deadline each to the platform's own `fetch`, which is not a wait this // package performs — and under a test clock a slept deadline fires on the next microtask // and cancels every request. Pinned here, in one place, with the reason. The exemption is diff --git a/packages/scraping/src/error-throws-pdf.ts b/packages/scraping/src/error-throws-pdf.ts index b904d448d..59db68ada 100644 --- a/packages/scraping/src/error-throws-pdf.ts +++ b/packages/scraping/src/error-throws-pdf.ts @@ -1,6 +1,7 @@ // The failure mode of a PDF layout request. Split from `error-throws.ts`, which stands at its size // ceiling; the code itself stays declared in `errors.ts` — one registry, one place. +import { isRetryableStatus, renderCauseValue, renderFixShellArg } from '@ultimat3/core'; import { ScrapeError } from './errors'; /** @@ -14,3 +15,47 @@ export const pdfOptionsInvalid = (reason: string, meta: { readonly field: string fix: "page.pdf({ format: 'a4', marginMm: { top: 12, bottom: 12 } }) — exactly one size: a format, or widthMm with heightMm, or preferCssPageSize alone", meta: { field: meta.field }, }); + +/** No server to load the route from: `APP_URL` unset and no `origin` on the declaration. */ +export const pdfOriginMissing = (): ScrapeError => + new ScrapeError({ + code: 'X_SCRAPE_PDF_ORIGIN_MISSING', + cause: + 'pdf() has no origin to load its route from: APP_URL is unset in this process and the declaration names no origin', + fix: "export APP_URL=https://your.app # or pdf({ origin: () => 'http://127.0.0.1:3000' }) in the declaring file", + }); + +/** `route()` answered something the printing browser would load from another origin. */ +export const pdfRouteInvalid = (route: string): ScrapeError => + new ScrapeError({ + code: 'X_SCRAPE_PDF_ROUTE_INVALID', + cause: `pdf() route() answered ${renderCauseValue(route)}, which is not a path on this app's origin — the printing browser loads nothing else`, + fix: "pdf({ route: ({ input }) => '/invoices/' + input.id + '/print' }) # route() answers a path on this app, never a URL", + }); + +/** + * The page answered an error status. A 4xx that no retry can change (a missing record, a route the + * printing browser may not read) is terminal on the instance; a 5xx or 429 stays retryable. + */ +export const pdfRouteFailed = (url: string, status: number): ScrapeError => { + const path = new URL(url).pathname; + return new ScrapeError({ + code: 'X_SCRAPE_PDF_ROUTE_FAILED', + cause: `the route pdf() printed, ${renderCauseValue(path)}, answered HTTP ${String(status)}: printing it would store an error page as the document`, + fix: `curl -i "$APP_URL"${renderFixShellArg(path, '/')} # a private route needs pdf({ headers }); a missing one, a route() naming a record that exists`, + meta: { status }, + retry: status >= 400 && status < 500 && !isRetryableStatus(status) ? 'terminal' : 'retryable', + }); +}; + +/** Every render slot stayed busy for the whole queue deadline: a burst, not a broken page. */ +export const pdfBusy = (input: { + readonly maxConcurrent: number; + readonly queueTimeoutMs: number; +}): ScrapeError => + new ScrapeError({ + code: 'X_SCRAPE_PDF_BUSY', + cause: `all ${String(input.maxConcurrent)} pdf() render slot(s) stayed busy for ${String(input.queueTimeoutMs)}ms — each holds a live Chrome, so the renderer bounds them rather than let a burst exhaust this process's memory`, + fix: `pdf({ renderer: chromePdfRenderer({ maxConcurrent: ${String(input.maxConcurrent * 2)} }) }) # only on a host with the memory for it; otherwise the caller retries later`, + meta: { maxConcurrent: input.maxConcurrent, queueTimeoutMs: input.queueTimeoutMs }, + }); diff --git a/packages/scraping/src/errors.ts b/packages/scraping/src/errors.ts index 3f26e4e65..18b904092 100644 --- a/packages/scraping/src/errors.ts +++ b/packages/scraping/src/errors.ts @@ -43,6 +43,10 @@ export const SCRAPE_OWNED_ERROR_CODES = [ 'X_SCRAPE_EGRESS_IN_PAYLOAD', 'X_SCRAPE_LAUNCH_ARGS_INVALID', 'X_SCRAPE_PDF_OPTIONS_INVALID', + 'X_SCRAPE_PDF_ORIGIN_MISSING', + 'X_SCRAPE_PDF_ROUTE_INVALID', + 'X_SCRAPE_PDF_ROUTE_FAILED', + 'X_SCRAPE_PDF_BUSY', ] as const; /** @@ -101,6 +105,10 @@ export const SCRAPE_ERROR_TITLES: Readonly> 'the launch options carry arguments this driver cannot hand the browser', X_SCRAPE_PDF_OPTIONS_INVALID: 'the PDF request names no single page size a print engine can lay out', + X_SCRAPE_PDF_ORIGIN_MISSING: 'pdf() has no origin to render the route on', + X_SCRAPE_PDF_ROUTE_INVALID: 'a pdf() route is not a path on this app', + X_SCRAPE_PDF_ROUTE_FAILED: 'the route pdf() printed answered an error status', + X_SCRAPE_PDF_BUSY: 'every pdf() render slot stayed busy past the queue deadline', }; // One unconditional call, so a second package claiming one of these codes throws @@ -214,6 +222,14 @@ export const SCRAPE_ERROR_RETRY = { // The request is the caller's literal, refused before any driver sees it: attempt 2 asks for the // same two sizes and is refused identically. X_SCRAPE_PDF_OPTIONS_INVALID: 'terminal', + // The deploy's environment and the declaration's own `route()`: attempt 2 reads the same ones. + X_SCRAPE_PDF_ORIGIN_MISSING: 'terminal', + X_SCRAPE_PDF_ROUTE_INVALID: 'terminal', + // The page answered an error: a 503 mid-deploy is the moment, so a `.job()` run tries again. + // The throw site narrows a 4xx to terminal — a missing record is missing on attempt 2 too. + X_SCRAPE_PDF_ROUTE_FAILED: 'retryable', + // Every slot busy is the moment, never the request: the same render fits once one ends. + X_SCRAPE_PDF_BUSY: 'retryable', } as const satisfies Readonly>; registerErrorRetry(SCRAPE_ERROR_RETRY); diff --git a/packages/scraping/src/index.ts b/packages/scraping/src/index.ts index cdb717816..af2d67e9a 100644 --- a/packages/scraping/src/index.ts +++ b/packages/scraping/src/index.ts @@ -138,6 +138,15 @@ export type { } from './page'; export type { PageContext } from './page-over-target'; export { pageOverTarget } from './page-over-target'; +export type { PdfActionArgs, PdfActionOptions, PdfActionOutput } from './pdf-action'; +export { PDF_CONTENT_TYPE, pdf } from './pdf-action'; +export type { + ChromePdfRendererOptions, + PdfBrowser, + PdfRenderer, + PdfRenderRequest, +} from './pdf-render'; +export { chromePdfRenderer, DEFAULT_PDF_TIMEOUT_MS } from './pdf-render'; export type { PdfFormat, PdfLayout, PdfMarginMm, PdfRequest } from './pdf-request'; export { PDF_FORMATS } from './pdf-request'; export type { Pacer } from './rate'; diff --git a/packages/scraping/src/pdf-action.test.ts b/packages/scraping/src/pdf-action.test.ts new file mode 100644 index 000000000..d32164e95 --- /dev/null +++ b/packages/scraping/src/pdf-action.test.ts @@ -0,0 +1,121 @@ +// `pdf()` is an action: its input schema refuses before a browser starts, its policy refuses an +// actor before a byte is printed, and an allowed call renders the app's OWN route and writes the +// bytes to the disk under the key the app chose. + +import { afterEach, describe, expect, test } from 'bun:test'; +import { registerAction, resetActions } from '@ultimat3/action'; +import { userActor } from '@ultimat3/core'; +import { allow, deny } from '@ultimat3/policy'; +import { t } from '@ultimat3/schema'; +import { memoryStorageDriver } from '@ultimat3/storage'; +import { pdf } from './pdf-action'; +import type { PdfRenderer, PdfRenderRequest } from './pdf-render'; + +afterEach(() => { + resetActions(); +}); + +const ana = userActor({ id: 'ana', orgId: 'o1' }); +const BYTES = new TextEncoder().encode('%PDF-1.7 invoice'); + +function recordingRenderer(): PdfRenderer & { readonly seen: PdfRenderRequest[] } { + const seen: PdfRenderRequest[] = []; + return { + seen, + render(request) { + seen.push(request); + return Promise.resolve(BYTES); + }, + }; +} + +function invoicePdf( + overrides: { route?: string; origin?: string | undefined; allowed?: boolean } = {}, +) { + const renderer = recordingRenderer(); + const disk = memoryStorageDriver(); + const declared = pdf({ + input: t.object({ id: t.string.min(1) }), + policy: overrides.allowed === false ? deny('invoices.not-yours') : allow(), + route: ({ input }) => overrides.route ?? `/invoices/${input.id}/print`, + key: ({ input, ctx }) => `invoices/${ctx.actor.id}/${input.id}.pdf`, + storage: () => disk, + layout: { format: 'a4' }, + headers: () => ({ authorization: 'Bearer short-lived' }), + origin: () => ('origin' in overrides ? overrides.origin : 'https://app.test'), + renderer, + }); + resetActions(); + return { action: registerAction('invoicePdf', declared), renderer, disk }; +} + +const codeOf = async (run: () => Promise): Promise => { + try { + await run(); + } catch (error) { + return (error as { code?: string }).code; + } + return expect.unreachable('expected a refusal'); +}; + +describe('unit · pdf()', () => { + test('renders the app route and stores the bytes under the key it answers', async () => { + const { action, renderer, disk } = invoicePdf(); + const answer = await action.as(ana, { id: '42' }); + expect(answer).toEqual({ + key: 'invoices/ana/42.pdf', + bytes: BYTES.byteLength, + contentType: 'application/pdf', + }); + expect(renderer.seen).toMatchObject([ + { + url: 'https://app.test/invoices/42/print', + layout: { format: 'a4' }, + headers: { authorization: 'Bearer short-lived' }, + }, + ]); + // The call's own cancellation reaches the renderer, so a dropped request frees its browser. + expect(renderer.seen[0]?.signal).toBeInstanceOf(AbortSignal); + const stored = await disk.get('invoices/ana/42.pdf'); + expect(new TextDecoder().decode(stored.bytes)).toBe('%PDF-1.7 invoice'); + }); + + test('the input schema refuses before any browser is asked', async () => { + const { action, renderer } = invoicePdf(); + expect(await codeOf(() => action.as(ana, { id: '' }))).toBe('X_INPUT_INVALID'); + expect(renderer.seen).toEqual([]); + }); + + test('the policy refuses before any browser is asked', async () => { + const { action, renderer } = invoicePdf({ allowed: false }); + expect(await codeOf(() => action.as(ana, { id: '42' }))).toBe('X_FORBIDDEN'); + expect(renderer.seen).toEqual([]); + }); + + test('a route that leaves the origin is refused, never loaded', async () => { + for (const route of ['https://evil.test/x', '//evil.test/x', 'invoices/42', '/\\evil.test']) { + const { action, renderer } = invoicePdf({ route }); + expect(await codeOf(() => action.as(ana, { id: '42' }))).toBe('X_SCRAPE_PDF_ROUTE_INVALID'); + expect(renderer.seen).toEqual([]); + } + }); + + test('no origin is X_SCRAPE_PDF_ORIGIN_MISSING', async () => { + const { action } = invoicePdf({ origin: undefined }); + expect(await codeOf(() => action.as(ana, { id: '42' }))).toBe('X_SCRAPE_PDF_ORIGIN_MISSING'); + }); + + test('a layout no engine honours is refused where pdf() is written', () => { + expect(() => + pdf({ + input: t.object({}), + policy: allow(), + route: () => '/x', + key: () => 'x.pdf', + storage: () => memoryStorageDriver(), + layout: { format: 'a4', widthMm: 100, heightMm: 100 }, + renderer: recordingRenderer(), + }), + ).toThrow(expect.objectContaining({ code: 'X_SCRAPE_PDF_OPTIONS_INVALID' })); + }); +}); diff --git a/packages/scraping/src/pdf-action.ts b/packages/scraping/src/pdf-action.ts new file mode 100644 index 000000000..1ca5b0907 --- /dev/null +++ b/packages/scraping/src/pdf-action.ts @@ -0,0 +1,116 @@ +// `pdf()` — a factory over ACTION (`PRIMITIVE_FACTORIES`), never a ninth primitive: render one of +// the app's own routes to PDF through the CDP browser, write it to a disk, answer its key. Input +// schema, policy, rate limit, meter and audit are the action's own, so an invoice PDF is gated +// exactly like the invoice: HTTP, OpenAPI, MCP, `.job()` and `.as()` come with it. + +import type { Action, ActionMeter, ActionPolicy, ActionRateLimit } from '@ultimat3/action'; +import { action } from '@ultimat3/action'; +import type { Ctx } from '@ultimat3/core'; +import type { InferOutput, StandardSchemaV1 } from '@ultimat3/schema'; +import { t } from '@ultimat3/schema'; +import type { StorageDriver } from '@ultimat3/storage'; +import { pdfOriginMissing, pdfRouteInvalid } from './error-throws-pdf'; +import type { PdfRenderer } from './pdf-render'; +import { chromePdfRenderer } from './pdf-render'; +import type { PdfLayout } from './pdf-request'; +import { pdfLayoutOf } from './pdf-request'; + +export const PDF_CONTENT_TYPE = 'application/pdf'; + +/** What every `pdf()` answers: where the document is, and how big. Never the bytes themselves. */ +const pdfOutput = () => t.object({ key: t.string, bytes: t.number, contentType: t.string }); + +export type PdfActionOutput = ReturnType; + +/** What `route`, `key` and `headers` are handed: the parsed input and the call's context. */ +export interface PdfActionArgs { + readonly input: InferOutput; + readonly ctx: Ctx; +} + +export interface PdfActionOptions { + readonly input: TInput; + readonly policy: ActionPolicy; + /** A PATH on this app (`/invoices/42/print`). Another origin, or a `//` path, is refused. */ + route(args: PdfActionArgs): string; + /** Where the document lands on `storage`. Tenant-scoped by the app: the key is its to choose. */ + key(args: PdfActionArgs): string; + /** A THUNK, read per call: the app's disks exist after boot, the action at module evaluation. */ + readonly storage: () => StorageDriver; + /** Checked once, here, by `pdfLayoutOf` — a layout no engine honours never reaches a browser. */ + readonly layout?: PdfLayout | undefined; + /** + * Headers the browser sends with every request the page makes — how a private route learns who + * is printing it (a short-lived token the app signs). Absent: the route must be readable bare. + */ + readonly headers?: + | ((args: PdfActionArgs) => Readonly>) + | undefined; + /** The origin the route is served on. Defaults to `APP_URL`, read per call. */ + readonly origin?: (() => string | undefined) | undefined; + /** + * Defaults to one `chromePdfRenderer()` the whole process shares — its `maxConcurrent` bounds every + * `pdf()` that names none. A test hands in one over a fake CDP connection. + */ + readonly renderer?: PdfRenderer | undefined; + readonly row?: ((args: PdfActionArgs) => TRow | null | Promise) | undefined; + readonly rateLimit?: ActionRateLimit | undefined; + readonly meter?: ActionMeter | undefined; + readonly audit?: boolean | undefined; +} + +const appOrigin = (): string | undefined => process.env['APP_URL']; + +let shared: PdfRenderer | undefined; +/** + * ONE default renderer per process, built on first use: its `maxConcurrent` bounds every `pdf()` + * that names none, so ten declarations do not mean twenty live Chromes. + */ +const sharedRenderer = (): PdfRenderer => { + shared ??= chromePdfRenderer(); + return shared; +}; + +/** The route joined onto the origin, refused unless the result is still that origin. */ +function routeUrl(origin: string, path: string): string { + if (!path.startsWith('/') || path.startsWith('//') || path.includes('\\')) { + throw pdfRouteInvalid(path); + } + const base = new URL(origin); + const url = new URL(path, base); + if (url.origin !== base.origin) throw pdfRouteInvalid(path); + return url.href; +} + +export function pdf( + options: PdfActionOptions, +): Action { + const layout = pdfLayoutOf(options.layout ?? {}); + const renderer = options.renderer ?? sharedRenderer(); + const origin = options.origin ?? appOrigin; + const { row } = options; + return action({ + input: options.input, + output: pdfOutput(), + policy: options.policy, + ...(row === undefined ? {} : { row: (args) => row(args) }), + ...(options.rateLimit === undefined ? {} : { rateLimit: options.rateLimit }), + ...(options.meter === undefined ? {} : { meter: options.meter }), + ...(options.audit === undefined ? {} : { audit: options.audit }), + async handle(args) { + const served = origin()?.trim(); + if (served === undefined || served === '') throw pdfOriginMissing(); + const url = routeUrl(served, options.route(args)); + const bytes = await renderer.render({ + url, + layout, + // A client that went away, or a request past its deadline, frees its slot and its browser. + signal: args.ctx.signal, + ...(options.headers === undefined ? {} : { headers: options.headers(args) }), + }); + const key = options.key(args); + await options.storage().put(key, bytes, { contentType: PDF_CONTENT_TYPE }); + return { key, bytes: bytes.byteLength, contentType: PDF_CONTENT_TYPE }; + }, + }); +} diff --git a/packages/scraping/src/pdf-cdp-params.ts b/packages/scraping/src/pdf-cdp-params.ts new file mode 100644 index 000000000..f64a63736 --- /dev/null +++ b/packages/scraping/src/pdf-cdp-params.ts @@ -0,0 +1,65 @@ +// A checked PDF layout in the PROTOCOL's own words — `Page.printToPDF`'s parameters — for the raw-CDP +// printer `pdf()` drives. `pdf-print.ts` is the same layout in puppeteer's words; the two exist +// because the two callers speak to different things. Inches, because the protocol counts in them. + +import type { PdfFormat, PdfLayout } from './pdf-request'; + +const MM_PER_INCH = 25.4; + +/** Width × height in millimetres, portrait — ISO 216 and the ANSI sizes, as every engine has them. */ +const PAPER_MM: ReadonlyMap = new Map([ + ['letter', [215.9, 279.4]], + ['legal', [215.9, 355.6]], + ['tabloid', [279.4, 431.8]], + ['ledger', [431.8, 279.4]], + ['a0', [841, 1189]], + ['a1', [594, 841]], + ['a2', [420, 594]], + ['a3', [297, 420]], + ['a4', [210, 297]], + ['a5', [148, 210]], + ['a6', [105, 148]], +]); + +/** An empty band rather than an absent one, for `pdf-print.ts`'s reason: Chrome's own date and URL. */ +const BLANK_TEMPLATE = ''; + +const inches = (mm: number): number => mm / MM_PER_INCH; + +/** The paper, in millimetres, when the layout names one. `undefined` is the engine's letter. */ +const paperOf = (layout: PdfLayout): readonly [number, number] | undefined => { + if (layout.format !== undefined) return PAPER_MM.get(layout.format); + if (layout.widthMm !== undefined && layout.heightMm !== undefined) { + return [layout.widthMm, layout.heightMm]; + } + return undefined; +}; + +/** + * `Page.printToPDF`'s params for a layout `pdfLayoutOf` already checked. Only the fields the caller + * gave are sent; the bytes come back base64 in the reply (`transferMode: 'ReturnAsBase64'`). + */ +export function printToPdfParams(layout: PdfLayout): Record { + const paper = paperOf(layout); + const margin = layout.marginMm; + const band = layout.headerHtml !== undefined || layout.footerHtml !== undefined; + return { + transferMode: 'ReturnAsBase64', + ...(paper === undefined ? {} : { paperWidth: inches(paper[0]), paperHeight: inches(paper[1]) }), + ...(margin?.top === undefined ? {} : { marginTop: inches(margin.top) }), + ...(margin?.right === undefined ? {} : { marginRight: inches(margin.right) }), + ...(margin?.bottom === undefined ? {} : { marginBottom: inches(margin.bottom) }), + ...(margin?.left === undefined ? {} : { marginLeft: inches(margin.left) }), + ...(layout.printBackground === undefined ? {} : { printBackground: layout.printBackground }), + ...(layout.preferCssPageSize === undefined + ? {} + : { preferCSSPageSize: layout.preferCssPageSize }), + ...(band + ? { + displayHeaderFooter: true, + headerTemplate: layout.headerHtml ?? BLANK_TEMPLATE, + footerTemplate: layout.footerHtml ?? BLANK_TEMPLATE, + } + : {}), + }; +} diff --git a/packages/scraping/src/pdf-fence.ts b/packages/scraping/src/pdf-fence.ts new file mode 100644 index 000000000..0f9a7e275 --- /dev/null +++ b/packages/scraping/src/pdf-fence.ts @@ -0,0 +1,86 @@ +// The print's request fence, on the page's own session: every request pauses (`Fetch`); one to the +// route's ORIGIN continues with the app's headers, every other — an off-origin redirect too — is +// failed before it leaves. Never `Network.setExtraHTTPHeaders`: it sent the print token to any +// `` a post carried, and let the page dial internal hosts. + +import type { CdpConnection } from './cdp-connection'; + +export interface PdfFenceInit { + readonly connection: CdpConnection; + readonly sessionId: string; + /** `new URL(route).origin` — scheme, host and port, compared whole. */ + readonly origin: string; + readonly headers?: Readonly> | undefined; +} + +interface Paused { + readonly requestId: string; + readonly url: string; + readonly headers: Readonly>; +} + +const readPaused = (params: Record): Paused | undefined => { + const request = params['request']; + const requestId = params['requestId']; + if (typeof requestId !== 'string' || typeof request !== 'object' || request === null) { + return undefined; + } + const url = 'url' in request && typeof request.url === 'string' ? request.url : ''; + const headers = + 'headers' in request && typeof request.headers === 'object' && request.headers !== null + ? Object.fromEntries(Object.entries(request.headers)) + : {}; + return { requestId, url, headers }; +}; + +const originOf = (url: string): string | undefined => { + try { + return new URL(url).origin; + } catch { + return undefined; + } +}; + +/** The request's own headers with the app's laid over them, case-insensitively, as CDP's list. */ +function mergedHeaders( + own: Readonly>, + extra: Readonly>, +): { readonly name: string; readonly value: string }[] { + const overridden = new Set(Object.keys(extra).map((name) => name.toLowerCase())); + const kept = Object.entries(own) + .filter(([name, value]) => typeof value === 'string' && !overridden.has(name.toLowerCase())) + .map(([name, value]) => ({ name, value: String(value) })); + return [...kept, ...Object.entries(extra).map(([name, value]) => ({ name, value }))]; +} + +/** + * Raise the fence before the page navigates. Answers its removal. A decision the browser refuses + * (the target already gone) is dropped: the page's own load then fails, which is the error to read. + */ +export async function raisePdfFence(init: PdfFenceInit): Promise<() => void> { + const { connection, sessionId } = init; + const extra = init.headers ?? {}; + const decide = (paused: Paused): Promise => { + if (originOf(paused.url) !== init.origin) { + return connection.send( + 'Fetch.failRequest', + { requestId: paused.requestId, errorReason: 'BlockedByClient' }, + sessionId, + ); + } + return connection.send( + 'Fetch.continueRequest', + Object.keys(extra).length === 0 + ? { requestId: paused.requestId } + : { requestId: paused.requestId, headers: mergedHeaders(paused.headers, extra) }, + sessionId, + ); + }; + const off = connection.on('Fetch.requestPaused', (params, from) => { + if (from !== sessionId) return; + const paused = readPaused(params); + if (paused !== undefined) void decide(paused).catch(() => undefined); + }); + await connection.send('Fetch.enable', { patterns: [{ urlPattern: '*' }] }, sessionId); + return off; +} diff --git a/packages/scraping/src/pdf-render-fence.test.ts b/packages/scraping/src/pdf-render-fence.test.ts new file mode 100644 index 000000000..882bef545 --- /dev/null +++ b/packages/scraping/src/pdf-render-fence.test.ts @@ -0,0 +1,81 @@ +// The print's request fence: the app's headers reach the route's own origin and nowhere else, and +// every request to another origin — an `` in user content, a redirect off-site, an internal +// host — is failed by the browser before it leaves. + +import { describe, expect, test } from 'bun:test'; +import { chromePdfRenderer } from './pdf-render'; +import type { FakeCall } from './pdf-render-fixture'; +import { fakePdfBrowser } from './pdf-render-fixture'; + +const URL_ = 'https://app.test/invoices/42/print'; +const TOKEN = { authorization: 'Bearer print-token' }; + +const decided = (calls: readonly FakeCall[], requestId: string): FakeCall | undefined => + calls.find( + (call) => + (call.method === 'Fetch.continueRequest' || call.method === 'Fetch.failRequest') && + call.params?.['requestId'] === requestId, + ); + +const headerNames = (call: FakeCall | undefined): string[] => { + const headers = call?.params?.['headers']; + return Array.isArray(headers) + ? headers.map((header: { name?: unknown }) => String(header.name).toLowerCase()) + : []; +}; + +describe('unit · the pdf request fence', () => { + test('the fence is up before the page navigates, and no header is set browser-wide', async () => { + const fake = fakePdfBrowser(); + await chromePdfRenderer({ launch: () => Promise.resolve(fake.browser) }).render({ + url: URL_, + layout: {}, + headers: TOKEN, + }); + const methods = fake.calls.map((call) => call.method); + // Presence first: `indexOf` answers -1 for an absent call, and -1 is less than every index. + expect(methods).toContain('Fetch.enable'); + expect(methods).toContain('Page.navigate'); + expect(methods.indexOf('Fetch.enable')).toBeLessThan(methods.indexOf('Page.navigate')); + expect(methods).not.toContain('Network.setExtraHTTPHeaders'); + }); + + test('a same-origin request carries the headers; a cross-origin one is blocked without them', async () => { + const fake = fakePdfBrowser({ + subresources: [ + 'https://app.test/assets/logo.png', + 'https://evil.test/x.png', + 'http://169.254.169.254/latest/meta-data', + 'http://app.test/invoices/42/print', + ], + }); + await chromePdfRenderer({ launch: () => Promise.resolve(fake.browser) }).render({ + url: URL_, + layout: {}, + headers: TOKEN, + }); + // r0 the document, r1 its own asset: continued, with the token. + for (const id of ['r0', 'r1']) { + const call = decided(fake.calls, id); + expect(call?.method).toBe('Fetch.continueRequest'); + expect(headerNames(call)).toContain('authorization'); + expect(headerNames(call)).toContain('accept'); + } + // r2 another site, r3 the metadata service, r4 the same host on another SCHEME: failed. + for (const id of ['r2', 'r3', 'r4']) { + const call = decided(fake.calls, id); + expect(call?.method).toBe('Fetch.failRequest'); + expect(call?.params?.['errorReason']).toBe('BlockedByClient'); + expect(headerNames(call)).toEqual([]); + } + }); + + test('with no headers, a same-origin request is continued untouched', async () => { + const fake = fakePdfBrowser(); + await chromePdfRenderer({ launch: () => Promise.resolve(fake.browser) }).render({ + url: URL_, + layout: {}, + }); + expect(decided(fake.calls, 'r0')?.params).toEqual({ requestId: 'r0' }); + }); +}); diff --git a/packages/scraping/src/pdf-render-fixture.ts b/packages/scraping/src/pdf-render-fixture.ts new file mode 100644 index 000000000..8eff70516 --- /dev/null +++ b/packages/scraping/src/pdf-render-fixture.ts @@ -0,0 +1,103 @@ +// A fake browser for the raw-CDP printer: a connection that answers the calls `pdf-render.ts` makes, +// emits `Fetch.requestPaused` for the document and any subresources the page "loads", and can hold +// the print open so a test can watch the concurrency bound and an abort. No Chrome. + +import type { CdpConnection, CdpEventListener, CdpResult } from './cdp-connection'; +import { CdpCallFailedError } from './cdp-errors'; +import type { PdfBrowser } from './pdf-render'; + +export const PDF_BYTES = new TextEncoder().encode('%PDF-1.7 fake'); + +export interface FakePdfBrowserOptions { + readonly status?: number; + readonly loads?: boolean; + readonly navigateError?: string; + /** URLs the page requests after its document, each paused for the fence to decide. */ + readonly subresources?: readonly string[]; + /** Held until resolved: the print does not answer before it. */ + readonly printGate?: Promise; +} + +export interface FakeCall { + readonly method: string; + readonly params: Record | undefined; +} + +export interface FakePdfBrowser { + readonly browser: PdfBrowser; + readonly calls: FakeCall[]; + readonly closed: () => boolean; +} + +export function fakePdfBrowser(options: FakePdfBrowserOptions = {}): FakePdfBrowser { + const calls: FakeCall[] = []; + const listeners = new Map>(); + let closed = false; + let killed: () => void = () => undefined; + const dead = new Promise((_, reject) => { + killed = () => + reject(new CdpCallFailedError({ method: 'Page.printToPDF', detail: 'browser closed' })); + }); + dead.catch(() => undefined); + const emit = (method: string, params: Record): void => { + for (const listener of listeners.get(method) ?? []) listener(params, 'S1'); + }; + const pause = (url: string, index: number): void => + emit('Fetch.requestPaused', { + requestId: `r${String(index)}`, + request: { url, method: 'GET', headers: { accept: '*/*' } }, + }); + const answer = async (method: string, params?: Record): Promise => { + switch (method) { + case 'Target.createTarget': + return { result: { targetId: 'T1' } }; + case 'Target.attachToTarget': + return { result: { sessionId: 'S1' } }; + case 'Page.navigate': { + const url = typeof params?.['url'] === 'string' ? params['url'] : ''; + [url, ...(options.subresources ?? [])].forEach(pause); + return { result: { frameId: 'F', errorText: options.navigateError ?? '' } }; + } + case 'Runtime.evaluate': + return { result: { result: { type: 'number', value: options.status ?? 200 } } }; + case 'Page.printToPDF': + await Promise.race([options.printGate ?? Promise.resolve(), dead]); + return { result: { data: PDF_BYTES.toBase64() } }; + default: + return { result: {} }; + } + }; + const connection: CdpConnection = { + send: (method, params) => { + calls.push({ method, params }); + return answer(method, params); + }, + // A load that never comes answers `false` at its deadline — later than any reply, as in Chrome. + once: () => + options.loads === false + ? Promise.resolve() + .then(() => undefined) + .then(() => undefined) + .then(() => false) + : Promise.resolve(true), + on: (method, listener) => { + const set = listeners.get(method) ?? new Set(); + set.add(listener); + listeners.set(method, set); + return () => set.delete(listener); + }, + close: () => undefined, + }; + return { + browser: { + connection, + close: () => { + closed = true; + killed(); + return Promise.resolve(); + }, + }, + calls, + closed: () => closed, + }; +} diff --git a/packages/scraping/src/pdf-render-slots.test.ts b/packages/scraping/src/pdf-render-slots.test.ts new file mode 100644 index 000000000..ed33d608e --- /dev/null +++ b/packages/scraping/src/pdf-render-slots.test.ts @@ -0,0 +1,104 @@ +// The renderer's bound on live browsers: at most `maxConcurrent` launched at once, a queued render +// refused with a code past its deadline, and an aborted request releasing its slot and its browser. + +import { describe, expect, test } from 'bun:test'; +import { noWaitClock } from '@ultimat3/core'; +import { chromePdfRenderer, DEFAULT_PDF_MAX_CONCURRENT, pdfLaunchOptions } from './pdf-render'; +import type { FakePdfBrowser } from './pdf-render-fixture'; +import { fakePdfBrowser } from './pdf-render-fixture'; + +const URL_ = 'http://app.test/invoices/42/print'; + +/** Launches counted, each print held until the test opens its gate. */ +function gatedLaunches() { + const launched: FakePdfBrowser[] = []; + const gates: (() => void)[] = []; + return { + launched, + open: (index: number) => gates[index]?.(), + launch: () => { + let open: () => void = () => undefined; + const printGate = new Promise((resolve) => { + open = resolve; + }); + gates.push(open); + const fake = fakePdfBrowser({ printGate }); + launched.push(fake); + return Promise.resolve(fake.browser); + }, + }; +} + +const settle = async (): Promise => { + for (let hop = 0; hop < 20; hop += 1) await Promise.resolve(); +}; + +describe('unit · chromePdfRenderer concurrency', () => { + test('a small default, and the production launch keeps the sandbox', () => { + expect(DEFAULT_PDF_MAX_CONCURRENT).toBe(2); + expect(pdfLaunchOptions(30_000)).toMatchObject({ sandbox: true }); + }); + + test('never more than maxConcurrent browsers; a queued render launches when one ends', async () => { + const gated = gatedLaunches(); + const renderer = chromePdfRenderer({ launch: gated.launch, maxConcurrent: 2 }); + const renders = [0, 1, 2].map(() => renderer.render({ url: URL_, layout: {} })); + await settle(); + expect(gated.launched).toHaveLength(2); + gated.open(0); + await renders[0]; + await settle(); + expect(gated.launched).toHaveLength(3); + gated.open(1); + gated.open(2); + await Promise.all(renders); + expect(gated.launched.every((fake) => fake.closed())).toBe(true); + }); + + test('a render queued past its deadline is X_SCRAPE_PDF_BUSY, and never launches', async () => { + const gated = gatedLaunches(); + const renderer = chromePdfRenderer({ + launch: gated.launch, + maxConcurrent: 1, + clock: noWaitClock, + }); + const first = renderer.render({ url: URL_, layout: {} }); + await settle(); + await expect(renderer.render({ url: URL_, layout: {} })).rejects.toMatchObject({ + code: 'X_SCRAPE_PDF_BUSY', + }); + expect(gated.launched).toHaveLength(1); + gated.open(0); + await first; + }); + + test('a request aborted while queued leaves the queue, launching nothing', async () => { + const gated = gatedLaunches(); + const renderer = chromePdfRenderer({ launch: gated.launch, maxConcurrent: 1 }); + const first = renderer.render({ url: URL_, layout: {} }); + const cancel = new AbortController(); + const queued = renderer.render({ url: URL_, layout: {}, signal: cancel.signal }); + await settle(); + cancel.abort(new Error('client went away')); + await expect(queued).rejects.toThrow('client went away'); + gated.open(0); + await first; + expect(gated.launched).toHaveLength(1); + }); + + test('a request aborted mid-print kills its browser and frees its slot', async () => { + const gated = gatedLaunches(); + const renderer = chromePdfRenderer({ launch: gated.launch, maxConcurrent: 1 }); + const cancel = new AbortController(); + const printing = renderer.render({ url: URL_, layout: {}, signal: cancel.signal }); + await settle(); + cancel.abort(new Error('client went away')); + await expect(printing).rejects.toThrow('client went away'); + expect(gated.launched[0]?.closed()).toBe(true); + const next = renderer.render({ url: URL_, layout: {} }); + await settle(); + expect(gated.launched).toHaveLength(2); + gated.open(1); + await next; + }); +}); diff --git a/packages/scraping/src/pdf-render.test.ts b/packages/scraping/src/pdf-render.test.ts new file mode 100644 index 000000000..614eb1c16 --- /dev/null +++ b/packages/scraping/src/pdf-render.test.ts @@ -0,0 +1,72 @@ +// The raw-CDP printer over a FAKE connection: the calls it makes in order, the bytes it decodes, +// the error page it refuses, and the browser it always closes — no Chrome needed. + +import { describe, expect, test } from 'bun:test'; +import { chromePdfRenderer } from './pdf-render'; +import { fakePdfBrowser } from './pdf-render-fixture'; + +const fakeBrowser = fakePdfBrowser; + +const URL_ = 'http://app.test/invoices/42/print'; + +describe('unit · chromePdfRenderer', () => { + test('prints the loaded page and answers the decoded bytes', async () => { + const fake = fakeBrowser(); + const renderer = chromePdfRenderer({ launch: () => Promise.resolve(fake.browser) }); + const bytes = await renderer.render({ url: URL_, layout: { format: 'a4' } }); + expect(new TextDecoder().decode(bytes)).toBe('%PDF-1.7 fake'); + expect(fake.calls.map((call) => call.method)).toEqual([ + 'Target.createTarget', + 'Target.attachToTarget', + 'Page.enable', + 'Fetch.enable', + 'Page.navigate', + 'Fetch.continueRequest', + 'Runtime.evaluate', + 'Page.printToPDF', + 'Target.closeTarget', + ]); + const print = fake.calls.find((call) => call.method === 'Page.printToPDF'); + expect(print?.params).toMatchObject({ transferMode: 'ReturnAsBase64' }); + expect(print?.params?.['paperWidth']).toBeCloseTo(210 / 25.4, 5); + expect(fake.closed()).toBe(true); + }); + + test('an error page is refused, never printed, and the browser still closes', async () => { + const fake = fakeBrowser({ status: 404 }); + const renderer = chromePdfRenderer({ launch: () => Promise.resolve(fake.browser) }); + const refused = renderer.render({ url: URL_, layout: {} }); + await expect(refused).rejects.toMatchObject({ + code: 'X_SCRAPE_PDF_ROUTE_FAILED', + retry: 'terminal', + }); + expect(fake.calls.some((call) => call.method === 'Page.printToPDF')).toBe(false); + expect(fake.closed()).toBe(true); + }); + + test('a 503 is retryable — the moment, not the page', async () => { + const fake = fakeBrowser({ status: 503 }); + const renderer = chromePdfRenderer({ launch: () => Promise.resolve(fake.browser) }); + await expect(renderer.render({ url: URL_, layout: {} })).rejects.toMatchObject({ + code: 'X_SCRAPE_PDF_ROUTE_FAILED', + retry: 'retryable', + }); + }); + + test('a page that never loads is X_CDP_TIMEOUT', async () => { + const fake = fakeBrowser({ loads: false }); + const renderer = chromePdfRenderer({ launch: () => Promise.resolve(fake.browser) }); + await expect(renderer.render({ url: URL_, layout: {} })).rejects.toMatchObject({ + code: 'X_CDP_TIMEOUT', + }); + expect(fake.closed()).toBe(true); + }); + + test('a refused navigation is X_CDP_CALL_FAILED', async () => { + const fake = fakeBrowser({ navigateError: 'net::ERR_CONNECTION_REFUSED', loads: false }); + const renderer = chromePdfRenderer({ launch: () => Promise.resolve(fake.browser) }); + await expect(renderer.render({ url: URL_, layout: {} })).rejects.toMatchObject({ + code: 'X_CDP_CALL_FAILED', + }); + }); +}); diff --git a/packages/scraping/src/pdf-render.ts b/packages/scraping/src/pdf-render.ts new file mode 100644 index 000000000..8c338afcf --- /dev/null +++ b/packages/scraping/src/pdf-render.ts @@ -0,0 +1,241 @@ +// One page printed to PDF over raw CDP: open a target, fence its requests to the route's origin, +// navigate, wait for the load event and the fonts, refuse an error page, `Page.printToPDF`. The +// browser is `cdp-launch.ts`'s — one launch per render, at most `maxConcurrent` alive, killed when +// the request is aborted, closed in a `finally` — and injectable, so `bun test` needs no Chrome. + +import type { SleepClock } from '@ultimat3/core'; +import { ambientClock, finiteCount, stringField } from '@ultimat3/core'; +import type { CdpConnection } from './cdp-connection'; +import { CdpCallFailedError, CdpTimeoutError } from './cdp-errors'; +import type { FoundChromeOptions } from './cdp-launch'; +import { LAUNCH_TIMEOUT_MS, launchFoundChrome } from './cdp-launch'; +import { pdfRouteFailed } from './error-throws-pdf'; +import { printToPdfParams } from './pdf-cdp-params'; +import { raisePdfFence } from './pdf-fence'; +import type { PdfLayout } from './pdf-request'; +import { pdfSlots } from './pdf-slots'; + +/** One render's every deadline: the launch's first answer, each CDP call, the load event. */ +export const DEFAULT_PDF_TIMEOUT_MS = 30_000; + +/** + * Sequential deadlines one render spends after the launch: create, attach, `Page.enable`, the two + * header calls, the load, the settled read, the print, the close — and one spare. + */ +export const PDF_RENDER_DEADLINES = 10; + +/** The launched browser's OS-level bound: a cold start, then every deadline a render spends. */ +export const pdfBrowserLifetimeMs = (timeoutMs: number): number => + Math.max(timeoutMs, LAUNCH_TIMEOUT_MS) + PDF_RENDER_DEADLINES * timeoutMs; + +/** + * Live print browsers per renderer, by default. Each Chrome holds 150-300 MB; two keeps a burst of + * prints (or a retry storm) from taking the web role's memory with it, and queues the rest. + */ +export const DEFAULT_PDF_MAX_CONCURRENT = 2; + +/** + * How a production render launches. The SANDBOX STAYS ON: a print renders the app's own pages, and + * those carry user content — the sandbox is what stands between a renderer exploit and the web + * role. The test launcher's `--no-sandbox` is for CI runners; the framework's runtime image is + * distroless and ships no Chrome, so the app's own image supplies it and runs it as non-root, where + * Chrome's namespace sandbox needs unprivileged user namespaces (or its setuid helper). A container + * that cannot grant them fails the launch with `X_CDP_LAUNCH_FAILED` quoting Chrome's "No usable + * sandbox"; that operator opts out in one visible line, `chromePdfRenderer({ sandbox: false })`. + */ +export const pdfLaunchOptions = (timeoutMs: number, sandbox = true): FoundChromeOptions => ({ + lifetimeMs: pdfBrowserLifetimeMs(timeoutMs), + sandbox, +}); + +export interface PdfRenderRequest { + /** Absolute. `pdf()` builds it from the app's own origin and a route path, never a caller's URL. */ + readonly url: string; + readonly layout: PdfLayout; + /** + * Added to the requests the page makes to the ROUTE'S OWN ORIGIN — how a private route learns + * who is printing it. Never sent anywhere else: every other origin is blocked (`pdf-fence.ts`). + */ + readonly headers?: Readonly> | undefined; + /** The request's cancellation: aborted, a queued render leaves the queue, a running one its browser. */ + readonly signal?: AbortSignal | undefined; +} + +/** The seam `pdf()` prints through. `chromePdfRenderer()` is the one this package ships. */ +export interface PdfRenderer { + render(request: PdfRenderRequest): Promise; +} + +/** A browser one render owns: its connection, and the close that ends it. `LaunchedBrowser` fits. */ +export interface PdfBrowser { + readonly connection: CdpConnection; + close(): Promise; +} + +export interface ChromePdfRendererOptions { + /** Defaults to `launchFoundChrome` with `pdfLaunchOptions`: `CHROME_PATH`, then the known paths. */ + readonly launch?: (() => Promise) | undefined; + readonly timeoutMs?: number | undefined; + /** Browsers alive at once. Default `DEFAULT_PDF_MAX_CONCURRENT`. */ + readonly maxConcurrent?: number | undefined; + /** How long a render waits for a slot before `X_SCRAPE_PDF_BUSY`. Default `timeoutMs`. */ + readonly queueTimeoutMs?: number | undefined; + /** What the queue waits on. Default `ambientClock()`, read per wait. */ + readonly clock?: SleepClock | undefined; + /** Chrome's sandbox for the default launch — on unless the container cannot grant it. */ + readonly sandbox?: boolean | undefined; +} + +/** + * Read in the page once it loaded: the fonts settled (a PDF printed before a webfont arrives prints + * the fallback face), then the document's own HTTP status — `0` where the browser cannot say. + */ +const SETTLED_STATUS = `(async () => { + await document.fonts.ready; + const entry = performance.getEntriesByType('navigation')[0]; + return entry && typeof entry.responseStatus === 'number' ? entry.responseStatus : 0; +})()`; + +const required = (value: string | undefined, method: string, field: string): string => { + if (value === undefined || value === '') { + throw new CdpCallFailedError({ method, detail: `the browser answered no ${field}` }); + } + return value; +}; + +/** Navigate and wait for `load` — the event, not the reply, which Chrome drops on a process swap. */ +async function navigate( + connection: CdpConnection, + sessionId: string, + url: string, + timeoutMs: number, +): Promise { + const loaded = connection.once('Page.loadEventFired', sessionId, timeoutMs); + const refused = connection.send('Page.navigate', { url }, sessionId).then( + (answer) => { + const failed = answer.result?.['errorText']; + return typeof failed === 'string' && failed !== '' ? failed : undefined; + }, + () => undefined, + ); + const failed = await Promise.race([loaded.then(() => undefined), refused]); + if (failed !== undefined) { + throw new CdpCallFailedError({ method: `Page.navigate to ${url}`, detail: failed }); + } + if (!(await loaded)) throw new CdpTimeoutError({ method: 'Page.loadEventFired', timeoutMs }); +} + +async function printOnce( + connection: CdpConnection, + request: PdfRenderRequest, + timeoutMs: number, +): Promise { + const created = await connection.send('Target.createTarget', { url: 'about:blank' }); + const targetId = required( + stringField(created.result, 'targetId'), + 'Target.createTarget', + 'targetId', + ); + try { + const attached = await connection.send('Target.attachToTarget', { targetId, flatten: true }); + const sessionId = required( + stringField(attached.result, 'sessionId'), + 'Target.attachToTarget', + 'sessionId', + ); + await connection.send('Page.enable', {}, sessionId); + await raisePdfFence({ + connection, + sessionId, + origin: new URL(request.url).origin, + headers: request.headers, + }); + await navigate(connection, sessionId, request.url, timeoutMs); + const settled = await connection.send( + 'Runtime.evaluate', + { expression: SETTLED_STATUS, awaitPromise: true, returnByValue: true }, + sessionId, + ); + const inner = settled.result?.['result']; + const status = + typeof inner === 'object' && inner !== null && 'value' in inner ? inner.value : 0; + if (typeof status === 'number' && status >= 400) throw pdfRouteFailed(request.url, status); + const printed = await connection.send( + 'Page.printToPDF', + printToPdfParams(request.layout), + sessionId, + ); + return Uint8Array.fromBase64( + required(stringField(printed.result, 'data'), 'Page.printToPDF', 'data'), + ); + } finally { + await connection.send('Target.closeTarget', { targetId }).catch(() => undefined); + } +} + +/** One render on a browser this call owns: killed the moment the request is aborted. */ +async function renderOn( + browser: PdfBrowser, + request: PdfRenderRequest, + timeoutMs: number, +): Promise { + const { signal } = request; + const kill = (): void => { + void browser.close(); + }; + signal?.addEventListener('abort', kill, { once: true }); + try { + signal?.throwIfAborted(); + return await printOnce(browser.connection, request, timeoutMs); + } catch (error) { + // A call cut off by the kill reports the dead socket; the reason the request was dropped is + // the truth, and is what the caller's own catch expects. + throw signal?.aborted === true ? signal.reason : error; + } finally { + signal?.removeEventListener('abort', kill); + await browser.close(); + } +} + +/** + * Prints through a Chrome launched for the one render and closed after it, success or not: no + * browser outlives a request, and nothing is shared between two tenants' documents. At most + * `maxConcurrent` are alive; the rest queue, bounded. A launch costs a second or two; a pool is the + * measured next step, not a guess (axiom 9). + */ +export function chromePdfRenderer(options: ChromePdfRendererOptions = {}): PdfRenderer { + const timeoutMs = finiteCount( + 'chromePdfRenderer', + 'timeoutMs', + options.timeoutMs ?? DEFAULT_PDF_TIMEOUT_MS, + 1, + ); + const slots = pdfSlots({ + maxConcurrent: finiteCount( + 'chromePdfRenderer', + 'maxConcurrent', + options.maxConcurrent ?? DEFAULT_PDF_MAX_CONCURRENT, + 1, + ), + queueTimeoutMs: finiteCount( + 'chromePdfRenderer', + 'queueTimeoutMs', + options.queueTimeoutMs ?? timeoutMs, + 0, + ), + clock: () => options.clock ?? ambientClock(), + }); + const launch = + options.launch ?? + (() => launchFoundChrome(process.env, timeoutMs, pdfLaunchOptions(timeoutMs, options.sandbox))); + return { + async render(request) { + const release = await slots.acquire(request.signal); + try { + return await renderOn(await launch(), request, timeoutMs); + } finally { + release(); + } + }, + }; +} diff --git a/packages/scraping/src/pdf-slots.ts b/packages/scraping/src/pdf-slots.ts new file mode 100644 index 000000000..d270199b6 --- /dev/null +++ b/packages/scraping/src/pdf-slots.ts @@ -0,0 +1,68 @@ +// The bound on live print browsers: a counting semaphore, first come first served. A queued render +// waits on the CLOCK (never a timer of its own — `clock-discipline.test.ts`) for a bounded time and +// then refuses with a code; an aborted request leaves the queue at once and takes no slot. + +import type { SleepClock } from '@ultimat3/core'; +import { pdfBusy } from './error-throws-pdf'; + +export interface PdfSlotsOptions { + readonly maxConcurrent: number; + readonly queueTimeoutMs: number; + /** Read per wait, so an `installClock` made after the renderer still governs it. */ + readonly clock: () => SleepClock; +} + +export interface PdfSlots { + /** Answers the release — idempotent — once a slot is this caller's. */ + acquire(signal?: AbortSignal): Promise<() => void>; +} + +interface Waiter { + readonly grant: () => void; +} + +export function pdfSlots(options: PdfSlotsOptions): PdfSlots { + let busy = 0; + const queue: Waiter[] = []; + const releaser = (): (() => void) => { + let released = false; + return () => { + if (released) return; + released = true; + const next = queue.shift(); + // The slot passes straight to the next waiter: `busy` never dips, so no newcomer jumps it. + if (next === undefined) busy -= 1; + else next.grant(); + }; + }; + return { + async acquire(signal) { + signal?.throwIfAborted(); + if (busy < options.maxConcurrent) { + busy += 1; + return releaser(); + } + const granted = new AbortController(); + const waiter: Waiter = { grant: () => granted.abort() }; + queue.push(waiter); + const leave = (): void => { + const at = queue.indexOf(waiter); + if (at !== -1) queue.splice(at, 1); + }; + try { + const wake = + signal === undefined ? granted.signal : AbortSignal.any([granted.signal, signal]); + await options.clock().sleep(options.queueTimeoutMs, wake); + } catch (error) { + // Woken: either the slot is ours, or the caller gave up — the request's reason, verbatim. + if (granted.signal.aborted) return releaser(); + leave(); + throw signal?.aborted === true ? signal.reason : error; + } + // Slept the whole deadline. A grant racing the wake-up still owns the slot: hand it back. + if (granted.signal.aborted) return releaser(); + leave(); + throw pdfBusy(options); + }, + }; +} diff --git a/packages/scraping/tsconfig.json b/packages/scraping/tsconfig.json index 584e59960..d5a201e59 100644 --- a/packages/scraping/tsconfig.json +++ b/packages/scraping/tsconfig.json @@ -8,6 +8,9 @@ "include": ["src/**/*"], "exclude": ["src/**/*.test.ts"], "references": [ + { + "path": "../action" + }, { "path": "../core" }, diff --git a/packages/storage/CLAUDE.md b/packages/storage/CLAUDE.md index bcdaffe0f..426da75dd 100644 --- a/packages/storage/CLAUDE.md +++ b/packages/storage/CLAUDE.md @@ -34,7 +34,7 @@ Tier 1. Object storage: named disks, safe keys, signed URLs, sniffed uploads. | Keys | every driver method starts with `assertSafeKey()`. No exceptions, no sanitising. `META_DIR` (`.meta`) is a reserved first segment on every driver — see below | | Time | take a `Clock`; never `Date.now()` | | Bytes | `Uint8Array \| ReadableStream \| Blob`. Never a base64 string, never unbounded — `toBytes(body, limit)` | -| Exports | explicit in `src/index.ts`; no `export *`. `./adapter` (`adapter.ts`): `StorageDriver`, its shapes, the errors and helpers a driver raises with — the `storage` port; no installer (the boot reads `runtime.ts`'s `storage` driver as its default disk) | +| Exports | explicit in `src/index.ts`; no `export *`. `./adapter` (`adapter.ts`): `StorageDriver`, its shapes, the errors and helpers a driver raises with — the `storage` port; no installer (the boot reads `runtime.ts`'s `storage` driver as its default disk); the `uploadScanner` port + installer | | File | Owns | |---|---| @@ -67,6 +67,8 @@ Tier 1. Object storage: named disks, safe keys, signed URLs, sniffed uploads. | `image.ts` | deterministic variant keys; byte path over core's pipeline (png/jpeg encode only) | | | `VARIANT_FORMATS` — what a variant KEY can carry — and NOT `IMAGE_FORMATS`, which is core's and means what core can PROBE. See below | | | `variantKey` is the cache identity `@ultimat3/cli`'s `/media/*` route looks a variant up by — derived, never stored, so a request that misses transforms once and every later one is a disk read | +| `image-variant.ts` | `variant()`: one kept variant, made by an `images` driver PASSED IN (seo's, structurally) | +| `upload-scanner*.ts` / `upload-scan.ts` | the `uploadScanner` port + memory (EICAR) reference; `promoteAttachment` scans a per-attempt copy | | `image-warm.ts` | `warmVariants`: a stored image's variants written ahead of `/media`, under its keys, the revision off the bytes read; core's spec, so a warmed variant is the one a miss would store | | `storage.ts` | `defineStorage` + module-level `storage()` / `disk()` | | `storage-instrument.ts` | each resolved disk: `storage.` bus event + client span | @@ -168,7 +170,8 @@ Gotchas: never-scanned upload with no second prefix to walk, and `isPendingKey` stays true for it. `promoteAttachment` refuses a quarantined key (`X_STORAGE_QUARANTINED`); `releaseQuarantine` is the app's scan verdict, and returns the ordinary pending key promotion accepts. Ultimate ships - no scanner — axiom 8, and `application/zip` is an accepted OOXML container by design. + no scanner vendor (axiom 8); the inline seam is the `uploadScanner` port. **No verdict fails + closed in every environment** (`onScanUnavailable` absent = `'refuse'`, `As of 2026-11-09`). - **`StorageListEntry` vs `StorageObject`.** A listing's `contentType` is optional; a `get()`'s is not. `ListObjectsV2` returns no Content-Type, and no driver invents one. - **`META_DIR` lives in `path.ts`, not in `driver-local.ts`.** The sidecar namespace overlapped the diff --git a/packages/storage/README.md b/packages/storage/README.md index b69a9d1f3..721984f59 100644 --- a/packages/storage/README.md +++ b/packages/storage/README.md @@ -491,7 +491,39 @@ refused delete goes in `failed` with the disk's own words and the sweep keeps go array of "deleted" keys put every refusal in it, so an erasure sweep over a bucket whose policy had lost `s3:DeleteObject` reported 200 objects gone that were all still there. -### Quarantine is the mechanism; the scanner is your app's +### Scanning on confirm: the `uploadScanner` port + +`As of 2026-11-09`. An `uploadScanner` in `apps//runtime.ts` is asked by `promoteAttachment` +about every upload it attaches. It scans a per-attempt copy (`pending/scan/-`): server +named, so no presigned URL rewrites it after the scan, and never the row's key, so a refused second +confirm of a double-click cannot delete the file the first one attached. The clean attempt is what +is copied onto the row's key: + +| Verdict | `promoteAttachment` | +|---|---| +| `{ verdict: 'clean' }` | promotes, as without a scanner | +| `{ verdict: 'infected', signature }` | deletes the pending upload and the attempt, throws `X_STORAGE_UPLOAD_INFECTED` (422, `meta.signature`) | +| rejects (daemon down, timeout), or the attempt cannot be read | deletes the attempt, keeps the upload pending, throws `X_STORAGE_SCAN_UNAVAILABLE` (503) — **fail closed by default, in every environment**; `uploadPolicy({ onScanUnavailable: 'accept' })` promotes unscanned instead. An infected verdict is refused under either | + +```ts +// apps/web/runtime.ts +import type { RuntimeOverrides } from '@ultimat3/cli/serve'; +import { memoryUploadScanner } from '@ultimat3/storage'; +// In production, your own UploadScanner here: e.g. one streaming `body` to clamd's INSTREAM +// over Bun.connect, mapping `OK` → clean, ` FOUND` → infected, anything else → reject. +export const runtime = { + uploadScanner: memoryUploadScanner(), +} satisfies RuntimeOverrides; +``` + +`UploadScanner` (`@ultimat3/storage/adapter`) is `{ name, scan({ key, size, contentType, body }) }`, +`body` a `ReadableStream`. `memoryUploadScanner()` is the reference: it flags the EICAR test string +(`EICAR_TEST_STRING`) anywhere in the stream, across chunk boundaries. Hold your adapter to the +contract with `behavesLike(uploadScannerConformance, () => yours)` from +`@ultimat3/testing/conformance`. No scanner ships: ClamAV, a vendor API, its licence and its +latency budget are the app's (axiom 8). No scanner installed, nothing is scanned. + +### Quarantine: the deferred scan, for a scanner that runs as a job Magic-byte sniffing closes stored XSS. It is **not** malware scanning, and `application/zip` is accepted on purpose as the OOXML container, so a macro-laden `.docx` passes `validateUpload` @@ -538,6 +570,8 @@ name is not found. | `X_STORAGE_LIST_FAILED` | the disk REFUSED a listing — denied `s3:ListBucket`, a throttle, an unreadable root. An **empty** disk is still not an error | | `X_STORAGE_QUARANTINED` | `promoteAttachment` on a key nothing has released from `pending/quarantine/` | | `X_STORAGE_NOT_PENDING` | `promoteAttachment` on a key outside the org's `pending/` prefix — most often another row's attached key | +| `X_STORAGE_UPLOAD_INFECTED` | the installed `uploadScanner` found malware in an upload `promoteAttachment` was attaching; both keys are deleted, `meta.signature` names the detection | +| `X_STORAGE_SCAN_UNAVAILABLE` | the installed `uploadScanner` rejected instead of answering, under the default `onScanUnavailable: 'refuse'`; the upload stays pending for a retry | | `X_STORAGE_ALREADY_PROMOTED` | `promoteAttachment` on a pending key an earlier call already moved (a retried or doubled confirm); `meta.attachedKey` names where the bytes are | | `X_STORAGE_PUT_FAILED` | the disk REFUSED a `put()`/`copy()` — `EACCES`, `ENOSPC`, `EROFS`, a provider's refused PUT | | `X_STORAGE_OBJECT_LOCKED` | local, memory: a delete, overwrite or copy onto a key under retention or a legal hold; the cause names the key, the mode and `retainUntil` or the hold, and the bytes are unchanged. Never raised by s3, where the provider keeps the locked version | @@ -563,6 +597,24 @@ a job boundary the class is gone and the `code` is what survives — match on th `variantKey()`, `srcsetDescriptors()`, `fitDimensions()` are pure — `@ultimat3/seo` builds `srcset` from them without decoding a byte. +`variant(key, { disk, images, width, format? })` (`image-variant.ts`) is ONE variant, kept: made by +the `images` driver you pass — `runtime.ts`'s `images` port value, `@ultimat3/seo`'s +`builtinImageDriver({ read })` or a CDN's, any `ImageTransformDriver` (storage takes it structurally; +it is tier 1 beside seo and imports none of it) — and written under +`variantKey(key, { width, format, revision })`. A second call is a `stat`, never a decode +(`{ key, object, created }`); an overwritten source is a new revision, so a new key. `format` +omitted keeps a png/jpeg/webp source's encoding. Refused with core's `X_IMAGE_UNSUPPORTED`: a source +that is not a raster image (a PDF, an SVG), a gif/avif source with no `format`, and a driver that +answers another encoding than the key names. A source overwritten while the driver read it is +`X_STORAGE_OBJECT_CHANGED`, and nothing is stored. `As of 2026-11-09`. + +```ts +import { builtinImageDriver } from '@ultimat3/seo'; +import { disk, variant } from '@ultimat3/storage'; +const images = builtinImageDriver({ read: async (src) => (await disk().get(src)).bytes }); +const { key } = await variant('org/o1/post/p1/cover/u1.png', { disk: disk(), images, width: 640, format: 'webp' }); +``` + `warmVariants({ disk, key, widths, formats })` (`image-warm.ts`) writes every width x format of a stored image under the key `/media` reads — `variantKey(key, { width, format, revision })`, the revision off the etag of the bytes it read — so the job an upload enqueues pays the decode and no diff --git a/packages/storage/src/adapter.test.ts b/packages/storage/src/adapter.test.ts index 1d6054ca1..5f449bfba 100644 --- a/packages/storage/src/adapter.test.ts +++ b/packages/storage/src/adapter.test.ts @@ -1,5 +1,6 @@ // `@ultimat3/storage/adapter` resolves through the package's export map, and every value it carries is -// the `.` one by identity — an adapter author's import, never a copy of the package's own. +// the `.` one by identity — an adapter author's import, never a copy of the package's own — except +// the `uploadScanner` installer, which is here and NOT on `.`: `runtime.ts` is the one way to install. import { describe, expect, test } from 'bun:test'; @@ -22,6 +23,9 @@ const VALUES: readonly string[] = [ 'toBytes', ]; +/** On `./adapter` only: a `.` export would be a second way to pick a scanner. */ +const INSTALLERS: readonly string[] = ['installUploadScanner', 'resetUploadScanner']; + describe('unit · @ultimat3/storage/adapter', () => { test('is the subpath the export map names', () => { expect(import.meta.resolve('@ultimat3/storage/adapter')).toEndWith( @@ -33,7 +37,8 @@ describe('unit · @ultimat3/storage/adapter', () => { test('carries exactly its values, each the barrel export by identity', async () => { const adapter: Record = await import('@ultimat3/storage/adapter'); const barrel: Record = await import('@ultimat3/storage'); - expect(Object.keys(adapter).sort()).toEqual([...VALUES].sort()); + expect(Object.keys(adapter).sort()).toEqual([...VALUES, ...INSTALLERS].sort()); for (const name of VALUES) expect([name, adapter[name] === barrel[name]]).toEqual([name, true]); + for (const name of INSTALLERS) expect([name, name in barrel]).toEqual([name, false]); }); }); diff --git a/packages/storage/src/adapter.ts b/packages/storage/src/adapter.ts index ff521855f..a13e079d3 100644 --- a/packages/storage/src/adapter.ts +++ b/packages/storage/src/adapter.ts @@ -1,7 +1,7 @@ -// `@ultimat3/storage/adapter`: the `storage` port (`ADAPTER_PORTS`) as a driver author needs it — -// `StorageDriver` and the shapes it reads and answers, the errors a driver raises, and the helpers -// every driver must refuse with the same way. No installer: the boot READS `runtime.ts`'s `storage` -// driver as its default disk (`@ultimat3/cli`'s `runtime-storage-boot.ts`). +// `@ultimat3/storage/adapter`: two ports as an adapter author needs them. `storage`: `StorageDriver`, +// its shapes, errors and shared refusal helpers — no installer, the boot READS `runtime.ts`'s +// driver as its default disk (`@ultimat3/cli`'s `runtime-storage-boot.ts`). `uploadScanner`: +// `UploadScanner`, what it is asked and answers, and the boot's installer. export type { ByteLimit, @@ -39,3 +39,5 @@ export type { ObjectLock, ObjectLockOptions } from './object-lock'; export type { PutStreamOptions } from './put-stream'; export type { ByteRange, GetOptions } from './range'; export type { SignedPart, SignedUrlVerification } from './signed-url'; +export type { ScanInput, ScanVerdict, UploadScanner } from './upload-scanner'; +export { installUploadScanner, resetUploadScanner } from './upload-scanner'; diff --git a/packages/storage/src/attachment-scan-race.test.ts b/packages/storage/src/attachment-scan-race.test.ts new file mode 100644 index 000000000..8e5e2692d --- /dev/null +++ b/packages/storage/src/attachment-scan-race.test.ts @@ -0,0 +1,100 @@ +// `promoteAttachment`'s scan step under concurrency: a doubled confirm whose first scan fails after +// the second has attached must not delete the second's committed file, and a scanner that returns +// before reading the whole body must not leave the disk's stream (an S3 connection) open. + +import { afterEach, describe, expect, test } from 'bun:test'; +import { frozenClock, isUltimateError } from '@ultimat3/core'; +import { attachmentKey, pendingKey, pendingPrefix, promoteAttachment } from './attachment'; +import type { StorageDriver } from './driver'; +import { memoryStorageDriver } from './driver-memory'; +import { uploadPolicy } from './upload'; +import { installUploadScanner, resetUploadScanner, type ScanVerdict } from './upload-scanner'; + +const ORG = 'org-1'; +const TARGET = { entity: 'post', id: 'p-1', field: 'file' } as const; +const TEXT = uploadPolicy({ allowedContentTypes: ['text/plain'] }); +const disk = () => + memoryStorageDriver({ signingSecret: 's', clock: frozenClock('2026-11-09T00:00:00Z') }); +const encode = (text: string): Uint8Array => new TextEncoder().encode(text); + +afterEach(() => resetUploadScanner()); + +describe('promoteAttachment scanning under concurrency', () => { + test("a doubled confirm whose scanner fails late never deletes the sibling's attached file", async () => { + const store = disk(); + const key = pendingKey(ORG, 'u-1.txt'); + await store.put(key, encode('a plain note'), { contentType: 'text/plain' }); + let failFirst: (reason: unknown) => void = () => undefined; + let calls = 0; + installUploadScanner({ + name: 'clamd', + scan: async () => { + calls += 1; + if (calls === 1) { + return new Promise((_, reject) => { + failFirst = reject; + }); + } + return { verdict: 'clean' }; + }, + }); + const first = promoteAttachment({ disk: store, key, orgId: ORG, target: TARGET, policy: TEXT }); + await Bun.sleep(0); + const second = await promoteAttachment({ + disk: store, + key, + orgId: ORG, + target: TARGET, + policy: TEXT, + }); + failFirst(new TypeError('clamd timed out')); + const error = await first.then( + () => expect.unreachable('the first confirm was expected to be refused'), + (reason: unknown) => reason, + ); + expect(isUltimateError(error) ? error.code : error).toBe('X_STORAGE_SCAN_UNAVAILABLE'); + // The row the second confirm wrote still has its object. + expect(second.key).toBe(attachmentKey(ORG, TARGET, 'u-1.txt')); + expect(await store.exists(second.key)).toBe(true); + // Neither attempt left a staged copy behind. + expect((await store.list({ prefix: pendingPrefix(ORG) })).rows).toEqual([]); + }); + + test('a scanner that answers before reading the whole body has its stream cancelled', async () => { + const store = disk(); + const key = pendingKey(ORG, 'u-2.txt'); + await store.put(key, encode('a plain note, long enough to need several chunks'), { + contentType: 'text/plain', + }); + let cancelled = false; + const watched: StorageDriver = { + ...store, + async stream(streamed, options) { + const bytes = (await store.get(streamed)).bytes; + void options; + let at = 0; + return new ReadableStream({ + pull(controller) { + if (at >= bytes.byteLength) return controller.close(); + controller.enqueue(bytes.slice(at, at + 4)); + at += 4; + }, + cancel() { + cancelled = true; + }, + }); + }, + }; + installUploadScanner({ + name: 'head-only', + async scan({ body }) { + const reader = body.getReader(); + await reader.read(); + reader.releaseLock(); + return { verdict: 'clean' }; + }, + }); + await promoteAttachment({ disk: watched, key, orgId: ORG, target: TARGET, policy: TEXT }); + expect(cancelled).toBe(true); + }); +}); diff --git a/packages/storage/src/attachment-scan-read.test.ts b/packages/storage/src/attachment-scan-read.test.ts new file mode 100644 index 000000000..082c3a7fb --- /dev/null +++ b/packages/storage/src/attachment-scan-read.test.ts @@ -0,0 +1,93 @@ +// `promoteAttachment`'s scan step when the ATTACHED copy cannot be read for the scanner (an S3 GET +// reset, an etag-pinned read refused): that is "no verdict", exactly as a scanner that rejects — so +// the policy decides, the copy never stays unscanned on the row's key, and a cleanup that fails +// does not hide the refusal. + +import { afterEach, describe, expect, test } from 'bun:test'; +import { frozenClock, isUltimateError } from '@ultimat3/core'; +import { attachmentKey, pendingKey, promoteAttachment } from './attachment'; +import type { StorageDriver } from './driver'; +import { memoryStorageDriver } from './driver-memory'; +import { uploadPolicy } from './upload'; +import { installUploadScanner, resetUploadScanner } from './upload-scanner'; +import { memoryUploadScanner } from './upload-scanner-memory'; + +const ORG = 'org-1'; +const TARGET = { entity: 'post', id: 'p-1', field: 'file' } as const; +const REFUSE = uploadPolicy({ allowedContentTypes: ['text/plain'] }); +const ACCEPT = uploadPolicy({ allowedContentTypes: ['text/plain'], onScanUnavailable: 'accept' }); + +/** A memory disk whose `stream` fails — and, optionally, whose delete of an attempt copy does. */ +function unreadable(options: { readonly failDelete?: boolean } = {}): StorageDriver { + const store = memoryStorageDriver({ + signingSecret: 's', + clock: frozenClock('2026-11-09T00:00:00Z'), + }); + return { + ...store, + stream: () => Promise.reject(new TypeError('read ECONNRESET')), + delete: (key: string) => + options.failDelete === true && key.startsWith(`org/${ORG}/pending/scan/`) + ? Promise.reject(new TypeError('delete EACCES')) + : store.delete(key), + }; +} + +async function caught(run: () => Promise): Promise { + try { + await run(); + } catch (error) { + return error; + } + return expect.unreachable('the promotion was expected to be refused'); +} + +async function pending(store: StorageDriver, name: string): Promise { + const key = pendingKey(ORG, name); + await store.put(key, new TextEncoder().encode('a plain note'), { contentType: 'text/plain' }); + return key; +} + +afterEach(() => resetUploadScanner()); + +describe('promoteAttachment when the attached copy cannot be read for the scanner', () => { + test('is X_STORAGE_SCAN_UNAVAILABLE, the copy is deleted and the upload stays pending', async () => { + installUploadScanner(memoryUploadScanner()); + const store = unreadable(); + const key = await pending(store, 'u-1.txt'); + const error = await caught(() => + promoteAttachment({ disk: store, key, orgId: ORG, target: TARGET, policy: REFUSE }), + ); + expect(isUltimateError(error) ? error.code : error).toBe('X_STORAGE_SCAN_UNAVAILABLE'); + expect(isUltimateError(error) ? error.cause : '').toContain('ECONNRESET'); + expect(await store.exists(attachmentKey(ORG, TARGET, 'u-1.txt'))).toBe(false); + // Only the upload itself is left under pending/: the attempt copy was deleted. + expect( + (await store.list({ prefix: `org/${ORG}/pending/` })).rows.map((row) => row.key), + ).toEqual([key]); + }); + + test("onScanUnavailable: 'accept' promotes it unscanned", async () => { + installUploadScanner(memoryUploadScanner()); + const store = unreadable(); + const key = await pending(store, 'u-2.txt'); + const object = await promoteAttachment({ + disk: store, + key, + orgId: ORG, + target: TARGET, + policy: ACCEPT, + }); + expect(object.key).toBe(attachmentKey(ORG, TARGET, 'u-2.txt')); + }); + + test('a delete of the copy that fails does not hide the refusal', async () => { + installUploadScanner(memoryUploadScanner()); + const store = unreadable({ failDelete: true }); + const key = await pending(store, 'u-3.txt'); + const error = await caught(() => + promoteAttachment({ disk: store, key, orgId: ORG, target: TARGET, policy: REFUSE }), + ); + expect(isUltimateError(error) ? error.code : error).toBe('X_STORAGE_SCAN_UNAVAILABLE'); + }); +}); diff --git a/packages/storage/src/attachment-scan.test.ts b/packages/storage/src/attachment-scan.test.ts new file mode 100644 index 000000000..1b06a7003 --- /dev/null +++ b/packages/storage/src/attachment-scan.test.ts @@ -0,0 +1,150 @@ +// `promoteAttachment` with an `uploadScanner` installed: the bytes that land on the row's key are +// scanned there, an infected upload is deleted and refused, and a scanner that cannot answer +// refuses the promotion unless the upload's policy says the app accepts unscanned files. + +import { afterEach, describe, expect, test } from 'bun:test'; +import { frozenClock, isUltimateError } from '@ultimat3/core'; +import { attachmentKey, pendingKey, promoteAttachment } from './attachment'; +import { memoryStorageDriver } from './driver-memory'; +import { uploadPolicy } from './upload'; +import { installUploadScanner, resetUploadScanner, type UploadScanner } from './upload-scanner'; +import { EICAR_TEST_STRING, memoryUploadScanner } from './upload-scanner-memory'; + +const ORG = 'org-1'; +const TARGET = { entity: 'post', id: 'p-1', field: 'file' } as const; +const TEXT = uploadPolicy({ allowedContentTypes: ['text/plain'] }); +const disk = () => + memoryStorageDriver({ signingSecret: 's', clock: frozenClock('2026-11-09T00:00:00Z') }); +const encode = (text: string): Uint8Array => new TextEncoder().encode(text); + +async function caught(run: () => Promise): Promise { + try { + await run(); + } catch (error) { + return error; + } + return expect.unreachable('the promotion was expected to be refused'); +} + +const down: UploadScanner = { + name: 'clamd', + scan: () => Promise.reject(new TypeError('connect ECONNREFUSED 127.0.0.1:3310')), +}; + +afterEach(() => resetUploadScanner()); + +describe('promoteAttachment with an uploadScanner installed', () => { + test('a clean upload is scanned and promoted', async () => { + const seen: string[] = []; + installUploadScanner({ + name: 'spy', + scan: async (input) => { + seen.push(input.key); + return memoryUploadScanner().scan(input); + }, + }); + const store = disk(); + const key = pendingKey(ORG, 'u-1.txt'); + await store.put(key, encode('a plain note'), { contentType: 'text/plain' }); + const object = await promoteAttachment({ + disk: store, + key, + orgId: ORG, + target: TARGET, + policy: TEXT, + }); + expect(object.key).toBe(attachmentKey(ORG, TARGET, 'u-1.txt')); + // A server-named attempt copy is what is scanned — no presigned URL reaches it, and it is never + // the row's key — and it is gone once its bytes are attached. + expect(seen).toHaveLength(1); + expect(seen[0]).toStartWith(`org/${ORG}/pending/scan/`); + expect(seen[0]).toEndWith('-u-1.txt'); + expect(await store.exists(seen[0] ?? '')).toBe(false); + }); + + test('an infected upload is refused, and neither the pending nor the attached key keeps it', async () => { + installUploadScanner(memoryUploadScanner()); + const store = disk(); + const key = pendingKey(ORG, 'u-2.txt'); + await store.put(key, encode(EICAR_TEST_STRING), { contentType: 'text/plain' }); + const error = await caught(() => + promoteAttachment({ disk: store, key, orgId: ORG, target: TARGET, policy: TEXT }), + ); + expect(isUltimateError(error) ? error.code : error).toBe('X_STORAGE_UPLOAD_INFECTED'); + expect(isUltimateError(error) ? error.meta : {}).toMatchObject({ + signature: 'EICAR-Test-File', + scanner: 'memory', + }); + expect(await store.exists(key)).toBe(false); + expect(await store.exists(attachmentKey(ORG, TARGET, 'u-2.txt'))).toBe(false); + }); + + test('a scanner that cannot answer refuses by default, and the upload stays pending', async () => { + installUploadScanner(down); + const store = disk(); + const key = pendingKey(ORG, 'u-3.txt'); + await store.put(key, encode('a plain note'), { contentType: 'text/plain' }); + const error = await caught(() => + promoteAttachment({ disk: store, key, orgId: ORG, target: TARGET, policy: TEXT }), + ); + expect(isUltimateError(error) ? error.code : error).toBe('X_STORAGE_SCAN_UNAVAILABLE'); + expect(isUltimateError(error) ? error.cause : '').toContain('ECONNREFUSED'); + // Retryable: the pending upload is still there, the half-made attached copy is not. + expect(await store.exists(key)).toBe(true); + expect(await store.exists(attachmentKey(ORG, TARGET, 'u-3.txt'))).toBe(false); + }); + + test("onScanUnavailable: 'accept' promotes unscanned when the scanner cannot answer", async () => { + installUploadScanner(down); + const store = disk(); + const key = pendingKey(ORG, 'u-4.txt'); + await store.put(key, encode('a plain note'), { contentType: 'text/plain' }); + const policy = uploadPolicy({ + allowedContentTypes: ['text/plain'], + onScanUnavailable: 'accept', + }); + const object = await promoteAttachment({ + disk: store, + key, + orgId: ORG, + target: TARGET, + policy, + }); + expect(object.key).toBe(attachmentKey(ORG, TARGET, 'u-4.txt')); + }); + + test('an infected verdict is never overridden by onScanUnavailable', async () => { + installUploadScanner(memoryUploadScanner()); + const store = disk(); + const key = pendingKey(ORG, 'u-5.txt'); + await store.put(key, encode(EICAR_TEST_STRING), { contentType: 'text/plain' }); + const policy = uploadPolicy({ + allowedContentTypes: ['text/plain'], + onScanUnavailable: 'accept', + }); + const error = await caught(() => + promoteAttachment({ disk: store, key, orgId: ORG, target: TARGET, policy }), + ); + expect(isUltimateError(error) ? error.code : error).toBe('X_STORAGE_UPLOAD_INFECTED'); + }); + + test('with no scanner installed, an EICAR upload promotes exactly as before', async () => { + const store = disk(); + const key = pendingKey(ORG, 'u-6.txt'); + await store.put(key, encode(EICAR_TEST_STRING), { contentType: 'text/plain' }); + const object = await promoteAttachment({ + disk: store, + key, + orgId: ORG, + target: TARGET, + policy: TEXT, + }); + expect(await store.exists(object.key)).toBe(true); + }); + + test('uploadPolicy refuses an onScanUnavailable that is neither refuse nor accept', () => { + expect(() => uploadPolicy({ onScanUnavailable: 'ignore' as unknown as 'accept' })).toThrow( + /onScanUnavailable/, + ); + }); +}); diff --git a/packages/storage/src/attachment.ts b/packages/storage/src/attachment.ts index 092b97f70..11aba5bce 100644 --- a/packages/storage/src/attachment.ts +++ b/packages/storage/src/attachment.ts @@ -5,25 +5,26 @@ // so the tenant prefix is a construction, not a check somebody remembered to write. import type { Clock } from '@ultimat3/core'; -import { assert, finiteCount, renderThrowable, systemClock } from '@ultimat3/core'; +import { assert, finiteCount, nanoid, renderThrowable, systemClock } from '@ultimat3/core'; import type { StorageDriver, StorageListEntry, StorageObject } from './driver'; import { objectNotFound, orgMismatch, tooLarge } from './errors'; import { alreadyPromoted, notPending, objectChanged, quarantined } from './errors-promote'; import { assertSafeKey, isWithinOrg, orgPrefix, scopedKey } from './path'; import type { UploadPolicy } from './upload'; import { SNIFF_WINDOW_BYTES, validateStoredHead } from './upload'; +import { SCAN_SEGMENT, scanAttempt, scanning } from './upload-scan'; /** The one segment an unattached upload lives under. `sweepOrphans` reads only this prefix. */ export const PENDING_SEGMENT = 'pending'; /** - * One more segment inside `pending/`, and the whole of Ultimate's content-scanning mechanism. + * One more segment inside `pending/`: the app's OWN scan workflow, for a scanner that runs as a job. * * Magic-byte sniffing closes stored XSS; it is not malware scanning, and `application/zip` is * deliberately accepted as the OOXML container, so a macro-laden `.docx` passes `validateUpload` - * exactly as a clean one does. Scanning is the APP's — a scanner is a business decision with a - * vendor, a licence and a latency budget (axiom 8) — so what ships is the place to put one: an - * upload granted with `quarantine: true` lands under this prefix, `promoteAttachment` REFUSES a + * exactly as a clean one does. The scanner is the APP's — a vendor, a licence and a latency budget + * (axiom 8) — so two seams ship. Inline: an `uploadScanner` in `runtime.ts`, which + * `promoteAttachment` asks about a per-attempt copy (`upload-scan.ts`). Deferred: an upload granted with `quarantine: true` lands under this prefix, `promoteAttachment` REFUSES a * key that is still there (`X_STORAGE_QUARANTINED`), and the app's scan job calls * `releaseQuarantine` on a verdict of clean. Inside `pending/` on purpose: an upload nobody ever * scanned is still an orphan, so `sweepOrphans` collects it with no second prefix to walk. @@ -197,6 +198,12 @@ async function assertCopyIsWhatWasJudged( * pending too. A ranged read because `maxGetBytes` bounds the range, never the object. An overwrite * after that read is `X_STORAGE_OBJECT_CHANGED`, or the copy itself is judged, and never lands. * + * SCANNED before it attaches, when `runtime.ts` installs an `uploadScanner`: on a per-attempt copy + * no presigned URL can rewrite (`scannedAttempt`), which is then what is attached. Infected is + * `X_STORAGE_UPLOAD_INFECTED` with attempt and upload deleted; no verdict is + * `X_STORAGE_SCAN_UNAVAILABLE` with the attempt deleted and the upload still pending, unless + * `policy.onScanUnavailable` accepts. Neither ever touches the row's key. + * * A source that is gone with the destination already there is `X_STORAGE_ALREADY_PROMOTED` * (409, `meta.attachedKey`), never the destination answered as if this call had moved it. 27.1.0 * answered it: a confirm sent twice then inserted a second row for one key, that insert failed, @@ -235,16 +242,43 @@ export async function promoteAttachment(input: PromoteAttachmentInput): Promise< throw tooLarge(input.key, pending.size, input.policy.maxBytes); } await judge(disk, pending, input.key, input.policy); + const scanned = scanning() ? await scannedAttempt(disk, pending, input, name) : undefined; + const source = scanned ?? pending; const object = await disk.copy( - input.key, + scanned?.key ?? input.key, attached, - pending.etag === '' ? undefined : { ifMatch: pending.etag }, + source.etag === '' ? undefined : { ifMatch: source.etag }, ); - await assertCopyIsWhatWasJudged(disk, pending, object, input.policy); + await assertCopyIsWhatWasJudged(disk, source, object, input.policy); + // The attempt is this call's scratch: a delete that fails leaves an orphan the sweep collects, + // never a refusal of a promotion that already happened. + if (scanned !== undefined) await disk.delete(scanned.key).catch(() => undefined); await disk.delete(input.key); return object; } +/** + * With an `uploadScanner` installed, what is attached is a copy THIS call made and scanned: under + * `pending/scan/`, server-named (no presigned URL rewrites it after the scan) and its own (a + * refusal deletes it and nothing a doubled confirm attached). Judged like any copy first. + */ +async function scannedAttempt( + disk: StorageDriver, + pending: StorageObject, + input: PromoteAttachmentInput, + name: string, +): Promise { + const key = scopedKey(input.orgId, PENDING_SEGMENT, SCAN_SEGMENT, `${nanoid()}-${name}`); + const attempt = await disk.copy( + input.key, + key, + pending.etag === '' ? undefined : { ifMatch: pending.etag }, + ); + await assertCopyIsWhatWasJudged(disk, pending, attempt, input.policy); + await scanAttempt({ disk, pendingKey: input.key, attempt, policy: input.policy }); + return attempt; +} + export interface SweepOrphansInput { readonly disk: StorageDriver; readonly orgId: string; diff --git a/packages/storage/src/driver-local-list.ts b/packages/storage/src/driver-local-list.ts index f67d26d3b..39ebdfb77 100644 --- a/packages/storage/src/driver-local-list.ts +++ b/packages/storage/src/driver-local-list.ts @@ -57,7 +57,7 @@ export async function listLocalPage(input: LocalListInput): Promise'")}`, + `ls -ld ${renderFixShellArg(root, "''")} # make the disk root readable by this process, then retry`, ); } keys.sort(); diff --git a/packages/storage/src/errors-scan.ts b/packages/storage/src/errors-scan.ts new file mode 100644 index 000000000..6c0fc2e2e --- /dev/null +++ b/packages/storage/src/errors-scan.ts @@ -0,0 +1,35 @@ +// The two refusals of a scanned promotion (`upload-scan.ts`): the scanner said infected, or the +// scanner could not say. Their codes are registered with the rest of the package's in `errors.ts`. + +import { renderFixLiteral } from '@ultimat3/core'; +import { StorageError } from './errors'; + +/** Never retryable with the same bytes, and never overridable by a policy: they are deleted. */ +export const uploadInfected = ( + key: string, + scanner: string, + signature: string, + removed: boolean, +): StorageError => + new StorageError({ + code: 'X_STORAGE_UPLOAD_INFECTED', + cause: `the ${scanner} upload scanner found "${signature}" in "${key}"; ${ + removed + ? 'the pending upload and its scan copy were deleted' + : 'deleting it failed, so sweepOrphans will collect what is left under pending/' + }`, + fix: 'grantUpload({ disk, orgId, request }) for a new, clean file — this one is refused for good', + meta: { key, scanner, signature, removed }, + }); + +/** + * The scanner rejected rather than answered. `reason` is already rendered (`renderThrowable`): a + * daemon's own words are the most useful thing in the cause, and an `unknown` is never interpolated. + */ +export const scanUnavailable = (key: string, scanner: string, reason: string): StorageError => + new StorageError({ + code: 'X_STORAGE_SCAN_UNAVAILABLE', + cause: `the ${scanner} upload scanner gave no verdict for "${key}": ${reason}`, + fix: `promoteAttachment({ disk, key: ${renderFixLiteral(key, "''")}, orgId, target, policy }) # again once the ${scanner} scanner answers — the upload is still pending; or uploadPolicy({ onScanUnavailable: 'accept' }) to promote unscanned files`, + meta: { key, scanner }, + }); diff --git a/packages/storage/src/errors-variant.ts b/packages/storage/src/errors-variant.ts new file mode 100644 index 000000000..17822338d --- /dev/null +++ b/packages/storage/src/errors-variant.ts @@ -0,0 +1,40 @@ +// The refusals of `variant()` (`image-variant.ts`). A source that is no raster and a driver that +// answered another encoding are core's `X_IMAGE_UNSUPPORTED`, as every other image refusal here is; +// a source overwritten mid-transform is storage's `X_STORAGE_OBJECT_CHANGED`. + +import { type ImageUnsupportedError, imageUnsupported, renderFixLiteral } from '@ultimat3/core'; +import { StorageError } from './errors'; + +export const notARaster = (key: string, contentType: string): ImageUnsupportedError => + imageUnsupported( + `"${key}" is stored as ${contentType}, which is not a raster image, so it has no width to resize to`, + `await disk.stat(${renderFixLiteral(key, "''")}) # variant() takes a png, jpeg, webp, gif or avif source: check its contentType, or serve it as-is`, + { key, contentType }, + ); + +export const variantFormatUnknown = (key: string, contentType: string): ImageUnsupportedError => + imageUnsupported( + `"${key}" is a ${contentType} source and no variant format was named, so no key can say what the transform will answer`, + `variant(${renderFixLiteral(key, "''")}, { disk, images, width, format: 'webp' })`, + { key, contentType }, + ); + +export const driverEncodingMismatch = ( + key: string, + driver: string, + expected: string, + actual: string, +): ImageUnsupportedError => + imageUnsupported( + `the ${driver} image driver answered ${actual} for "${key}", whose key names ${expected}, so the bytes would be served under the wrong type`, + `variant(${renderFixLiteral(key, "''")}, { disk, images, width, format: '' }) # or make its transform() honour request.format`, + { key, driver, expected, actual }, + ); + +export const variantSourceChanged = (key: string, expected: string, actual: string): StorageError => + new StorageError({ + code: 'X_STORAGE_OBJECT_CHANGED', + cause: `"${key}" was overwritten while its variant was being made (etag ${expected} → ${actual}), so the bytes cannot be filed under either revision`, + fix: `variant(${renderFixLiteral(key, "''")}, { disk, images, width }) again — it keys the variant by the new revision`, + meta: { key, expected, actual }, + }); diff --git a/packages/storage/src/errors.ts b/packages/storage/src/errors.ts index cb65edd6c..cb32b834c 100644 --- a/packages/storage/src/errors.ts +++ b/packages/storage/src/errors.ts @@ -39,6 +39,8 @@ export const STORAGE_OWNED_ERROR_CODES = [ 'X_STORAGE_MULTIPART_UNSUPPORTED', 'X_STORAGE_UPLOAD_UNKNOWN', 'X_STORAGE_PARTS_INVALID', + 'X_STORAGE_UPLOAD_INFECTED', + 'X_STORAGE_SCAN_UNAVAILABLE', ] as const; /** @@ -89,6 +91,8 @@ export const STORAGE_ERROR_TITLES: Readonly + memoryStorageDriver({ signingSecret: 's', clock: frozenClock('2026-11-09T00:00:00Z') }); + +/** `builtinImageDriver`'s behaviour, over the disk: what `@ultimat3/seo` hands `variant()`. */ +function transformer(store: StorageDriver): VariantTransformer & { calls: number } { + const self = { + name: 'builtin-like', + calls: 0, + async transform(request: { src: string; width: number; format?: string }) { + self.calls += 1; + const source = await store.get(request.src); + const format = + request.format === 'jpeg' || request.format === 'png' ? request.format : 'webp'; + const bytes = await transformImageBytes(source.bytes, { width: request.width, format }); + const info = probeImage(bytes); + return { + bytes, + contentType: IMAGE_MIME_TYPES[info.format], + width: info.width, + height: info.height, + }; + }, + }; + return self; +} + +async function caught(run: () => Promise): Promise { + try { + await run(); + } catch (error) { + return error; + } + return expect.unreachable('variant() was expected to refuse'); +} + +describe('variant', () => { + test('stores the transform under the derived key, then answers it without transforming again', async () => { + const store = disk(); + const source = await store.put('p/hero.png', png(40, 20), { contentType: 'image/png' }); + const images = transformer(store); + const first = await variant('p/hero.png', { disk: store, images, width: 10, format: 'webp' }); + const revision = variantRevision(source.etag); + expect(first.key).toBe(variantKey('p/hero.png', { width: 10, format: 'webp', revision })); + expect(first.created).toBe(true); + expect(first.object.contentType).toBe('image/webp'); + expect(probeImage((await store.get(first.key)).bytes).width).toBe(10); + + const again = await variant('p/hero.png', { disk: store, images, width: 10, format: 'webp' }); + expect(again).toMatchObject({ key: first.key, created: false }); + expect(images.calls).toBe(1); + }); + + test('no format keeps the source encoding, and an overwritten source misses the old variant', async () => { + const store = disk(); + await store.put('p/a.png', png(40, 20), { contentType: 'image/png' }); + const images = transformer(store); + const before = await variant('p/a.png', { disk: store, images, width: 8 }); + expect(before.key.endsWith('.png')).toBe(true); + await store.put('p/a.png', png(30, 30), { contentType: 'image/png' }); + const after = await variant('p/a.png', { disk: store, images, width: 8 }); + expect(after.key).not.toBe(before.key); + expect(after.created).toBe(true); + }); + + test('a non-image source is refused with X_IMAGE_UNSUPPORTED and nothing is transformed', async () => { + const store = disk(); + await store.put('d/report.pdf', new TextEncoder().encode('%PDF-1.7'), { + contentType: 'application/pdf', + }); + const images = transformer(store); + // A format named, so the refusal is the source's type and not the missing format. + const error = await caught(() => + variant('d/report.pdf', { disk: store, images, width: 10, format: 'webp' }), + ); + expect(isUltimateError(error) ? error.code : error).toBe('X_IMAGE_UNSUPPORTED'); + expect(images.calls).toBe(0); + }); + + test('an svg source is refused: a vector has no width to resample to', async () => { + const store = disk(); + await store.put( + 'd/logo.svg', + new TextEncoder().encode(''), + { + contentType: 'image/svg+xml', + }, + ); + const images = transformer(store); + const error = await caught(() => + variant('d/logo.svg', { disk: store, images, width: 10, format: 'png' }), + ); + expect(isUltimateError(error) ? error.code : error).toBe('X_IMAGE_UNSUPPORTED'); + expect(images.calls).toBe(0); + }); + + test('a missing source is X_STORAGE_NOT_FOUND', async () => { + const store = disk(); + const error = await caught(() => + variant('p/none.png', { disk: store, images: transformer(store), width: 10 }), + ); + expect(isUltimateError(error) ? error.code : error).toBe('X_STORAGE_NOT_FOUND'); + }); + + test('a driver answering another encoding than the key names is refused, and nothing is stored', async () => { + const store = disk(); + await store.put('p/b.png', png(40, 20), { contentType: 'image/png' }); + const liar = transformer(store); + const images: VariantTransformer = { + name: 'liar', + transform: async (request) => ({ + ...(await liar.transform(request)), + contentType: 'image/jpeg', + }), + }; + const error = await caught(() => + variant('p/b.png', { disk: store, images, width: 10, format: 'webp' }), + ); + expect(isUltimateError(error) ? error.code : error).toBe('X_IMAGE_UNSUPPORTED'); + expect((await store.list({ prefix: 'p/b.png@' })).rows).toEqual([]); + }); + + test('a source overwritten while the driver read it is X_STORAGE_OBJECT_CHANGED, and nothing is stored', async () => { + const store = disk(); + await store.put('p/d.png', png(40, 20), { contentType: 'image/png' }); + const inner = transformer(store); + const images: VariantTransformer = { + name: 'racing', + async transform(request) { + const made = await inner.transform(request); + await store.put('p/d.png', png(20, 20), { contentType: 'image/png' }); + return made; + }, + }; + const error = await caught(() => variant('p/d.png', { disk: store, images, width: 10 })); + expect(isUltimateError(error) ? error.code : error).toBe('X_STORAGE_OBJECT_CHANGED'); + expect((await store.list({ prefix: 'p/d.png@' })).rows).toEqual([]); + }); + + test('a width that is not a whole positive pixel count is refused before any read', async () => { + const store = disk(); + await store.put('p/c.png', png(4, 4), { contentType: 'image/png' }); + const images = transformer(store); + await expect(variant('p/c.png', { disk: store, images, width: Number.NaN })).rejects.toThrow(); + expect(images.calls).toBe(0); + }); +}); diff --git a/packages/storage/src/image-variant.ts b/packages/storage/src/image-variant.ts new file mode 100644 index 000000000..883eb4372 --- /dev/null +++ b/packages/storage/src/image-variant.ts @@ -0,0 +1,106 @@ +// Single responsibility: `variant()` — one resized image, made by the app's image driver and kept on +// the disk under `variantKey(…, { revision })`, so the second ask is a `stat` and never a decode. +// The driver is an ARGUMENT, structurally `@ultimat3/seo`'s `ImageTransformDriver`: storage is tier +// 1 beside seo and imports none of it; the `images` port value is what an app passes. + +import { assert, finiteCount, IMAGE_MIME_TYPES } from '@ultimat3/core'; +import type { StorageDriver, StorageObject } from './driver'; +import { objectNotFound } from './errors'; +import { + driverEncodingMismatch, + notARaster, + variantFormatUnknown, + variantSourceChanged, +} from './errors-variant'; +import { type VariantFormat, variantFormatOf, variantKey, variantRevision } from './image'; +import { normalizeContentType } from './upload'; + +/** `@ultimat3/seo`'s `TransformRequest`, field for field: `src` is the SOURCE storage key. */ +export interface VariantRequest { + src: string; + width: number; + format?: string; + quality?: number; +} + +/** `@ultimat3/seo`'s `TransformedImage`, field for field. */ +export interface VariantBytes { + bytes: Uint8Array; + contentType: string; + width: number; + height: number; +} + +/** + * The half of `ImageTransformDriver` this needs — so `runtime.ts`'s `images` driver, a + * `builtinImageDriver({ read: (src) => disk.get(src).then((r) => r.bytes) })` or a CDN's, is passed + * as it is. Its `read` must resolve `src` against the SAME disk handed to `variant()`. + */ +export interface VariantTransformer { + readonly name: string; + transform(request: VariantRequest): Promise; +} + +export interface VariantOptions { + readonly disk: StorageDriver; + readonly images: VariantTransformer; + /** Whole pixels, at least one. A width above the source's is the driver's to clamp. */ + readonly width: number; + /** Omitted: the source's own encoding (png, jpeg, webp); a gif or avif source must name one. */ + readonly format?: VariantFormat | undefined; +} + +export interface StoredVariant { + readonly key: string; + readonly object: StorageObject; + /** True when THIS call transformed and wrote it; false when it was already on the disk. */ + readonly created: boolean; +} + +/** What a driver can resample: every raster core can probe. An svg has no pixels to resize. */ +const RASTER_TYPES: ReadonlySet = new Set( + Object.entries(IMAGE_MIME_TYPES) + .filter(([format]) => format !== 'svg') + .map(([, type]) => type), +); + +/** + * The variant of `key` at `width` (and `format`), from the disk if it is there, else transformed by + * `images` and written. Idempotent: the key is derived from the source's revision, so a repeat is a + * hit and an overwritten source is a different key. Concurrent first calls both write the same key + * with the same bytes — harmless, and cheaper than a lock. + */ +export async function variant(key: string, options: VariantOptions): Promise { + const width = finiteCount('variant', 'width', options.width, 1); + const { disk, images } = options; + const source = await disk.stat(key); + if (source === undefined) throw objectNotFound(disk.name, key); + const sourceType = normalizeContentType(source.contentType); + if (!RASTER_TYPES.has(sourceType)) throw notARaster(key, sourceType); + const format = options.format === undefined ? variantFormatOf(sourceType) : options.format; + if (format === undefined) throw variantFormatUnknown(key, sourceType); + const revision = variantRevision(source.etag); + assert( + revision !== undefined, + `variant('${key}'): the source etag "${source.etag}" carries no 8-hex revision, so its variant has no key that an overwrite would miss`, + `x doctor --json # the disk driver answers no content etag; serve ${key} through /media/* instead, which transforms without storing`, + ); + const target = variantKey(key, { width, format, revision }); + + const stored = await disk.stat(target); + if (stored !== undefined) return { key: target, object: stored, created: false }; + + const made = await images.transform({ src: key, width, format }); + const expected = IMAGE_MIME_TYPES[format]; + if (normalizeContentType(made.contentType) !== expected) { + throw driverEncodingMismatch(key, images.name, expected, made.contentType); + } + // The driver read the source itself: bracketing it with two stats is how these bytes are known to + // be the revision the key names, and not an overwrite that landed while it read. + const after = await disk.stat(key); + if (after?.etag !== source.etag) { + throw variantSourceChanged(key, source.etag, after?.etag ?? 'deleted'); + } + const object = await disk.put(target, made.bytes, { contentType: expected }); + return { key: target, object, created: true }; +} diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index 37c5259fe..5c682a252 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -110,6 +110,7 @@ export { export { multipartUnsupported, partsInvalid, uploadUnknown } from './errors-multipart'; export { alreadyPromoted, quarantined } from './errors-promote'; export { rangeTooLarge, rangeUnsatisfiable } from './errors-range'; +export { scanUnavailable, uploadInfected } from './errors-scan'; export type { AbortResult, AbortStaleUploadsInput, @@ -159,6 +160,14 @@ export { variantRevision, variantSourceOf, } from './image'; +export type { + StoredVariant, + VariantBytes, + VariantOptions, + VariantRequest, + VariantTransformer, +} from './image-variant'; +export { variant } from './image-variant'; export type { WarmFailure, WarmVariantsInput, WarmVariantsResult } from './image-warm'; export { warmVariants } from './image-warm'; export type { @@ -235,6 +244,7 @@ export type { } from './subject-target'; export { STORAGE_SUBJECT_COLUMNS, storageSubjectTarget } from './subject-target'; export type { + ScanUnavailablePolicy, StoredHeadCandidate, UploadCandidate, UploadPolicy, @@ -279,3 +289,5 @@ export { PART_CONCURRENCY, uploadInParts, } from './upload-client-multipart'; +// The `uploadScanner` port's reference: the interface and its installer are on `./adapter`. +export { EICAR_TEST_STRING, memoryUploadScanner } from './upload-scanner-memory'; diff --git a/packages/storage/src/upload-scan.ts b/packages/storage/src/upload-scan.ts new file mode 100644 index 000000000..cfd653c7e --- /dev/null +++ b/packages/storage/src/upload-scan.ts @@ -0,0 +1,78 @@ +// Single responsibility: the scan step of `promoteAttachment` — the installed `UploadScanner` asked +// about a per-ATTEMPT copy of the upload, and its three outcomes: clean (the attempt is what gets +// attached), infected (attempt and upload deleted, refused) and no verdict (attempt deleted, +// refused and retryable, unless the policy accepts unscanned). + +import { renderThrowable } from '@ultimat3/core'; +import type { StorageDriver, StorageObject } from './driver'; +import { scanUnavailable, uploadInfected } from './errors-scan'; +import type { UploadPolicy } from './upload'; +import { uploadScanner } from './upload-scanner'; + +/** + * `pending/scan/-`: a key of THIS confirm's own (`attachment.ts` builds it). + * Server-named, so no presigned URL reaches it and one scan of it is final; never the row's key, so + * a refusal deletes nothing a doubled confirm attached; inside `pending/`, so an attempt a crash + * left behind is an orphan `sweepOrphans` collects. + */ +export const SCAN_SEGMENT = 'scan'; + +export interface ScanAttemptInput { + readonly disk: StorageDriver; + /** The upload as it was confirmed: what a refusal names and a retry promotes again. */ + readonly pendingKey: string; + /** The attempt copy under `SCAN_SEGMENT`, already sniffed. */ + readonly attempt: StorageObject; + readonly policy: UploadPolicy; +} + +/** Deleted, or reported as not: a cleanup that fails never replaces the refusal it follows. */ +async function removeAll(disk: StorageDriver, keys: readonly string[]): Promise { + let removed = true; + for (const key of keys) { + try { + await disk.delete(key); + } catch { + removed = false; + } + } + return removed; +} + +/** True when a scanner is installed — `promoteAttachment` stages an attempt copy only then. */ +export const scanning = (): boolean => uploadScanner() !== undefined; + +/** + * Stream the attempt — pinned to its etag where it has one — to the installed scanner. Returns on + * clean (or no verdict under `onScanUnavailable: 'accept'`); every refusal deletes the attempt, and + * infected deletes the upload too. The stream is cancelled however the scanner returns: one that + * answers off the first chunk would otherwise hold the disk's connection open. + */ +export async function scanAttempt(input: ScanAttemptInput): Promise { + const scanner = uploadScanner(); + if (scanner === undefined) return; + const { disk, attempt } = input; + let body: ReadableStream | undefined; + let verdict: Awaited>; + try { + // Inside the `try`: an attempt that cannot be READ for the scanner (an S3 GET reset, an + // etag-pinned read refused) is no verdict too, never a raw error past the cleanup. + body = await disk.stream(attempt.key, attempt.etag === '' ? undefined : { etag: attempt.etag }); + verdict = await scanner.scan({ + key: attempt.key, + size: attempt.size, + contentType: attempt.contentType, + body, + }); + } catch (error) { + if (input.policy.onScanUnavailable === 'accept') return; + // The attempt goes, the upload stays: a retry once the scanner is back is a fresh confirm. + await removeAll(disk, [attempt.key]); + throw scanUnavailable(input.pendingKey, scanner.name, renderThrowable(error)); + } finally { + await body?.cancel().catch(() => undefined); + } + if (verdict.verdict === 'clean') return; + const removed = await removeAll(disk, [attempt.key, input.pendingKey]); + throw uploadInfected(input.pendingKey, scanner.name, verdict.signature, removed); +} diff --git a/packages/storage/src/upload-scanner-memory.ts b/packages/storage/src/upload-scanner-memory.ts new file mode 100644 index 000000000..70c645667 --- /dev/null +++ b/packages/storage/src/upload-scanner-memory.ts @@ -0,0 +1,62 @@ +// The reference `UploadScanner`: finds the EICAR test file, the one byte string every antivirus +// agrees is a virus and no file legitimately contains. A test's scanner and the shape an app's +// ClamAV adapter copies — streamed, matched across chunk boundaries, no state between calls. + +import type { ScanInput, ScanVerdict, UploadScanner } from './upload-scanner'; + +/** + * The 68-byte EICAR anti-malware test string. Assembled from two halves so this source file is not + * itself a detection on a developer's machine or in a registry scan. + */ +export const EICAR_TEST_STRING = `${'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR'}${'-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*'}`; + +const SIGNATURE = 'EICAR-Test-File'; +const NEEDLE = new TextEncoder().encode(EICAR_TEST_STRING); + +function indexOf(haystack: Uint8Array, needle: Uint8Array): number { + outer: for (let at = 0; at + needle.byteLength <= haystack.byteLength; at += 1) { + for (let i = 0; i < needle.byteLength; i += 1) { + if (haystack[at + i] !== needle[i]) continue outer; + } + return at; + } + return -1; +} + +/** + * Reads the body to the end (or to the first match, then cancels it), keeping only the last + * `NEEDLE.length - 1` bytes between chunks: a match split by the stream is still a match, and the + * heap never holds more than one chunk plus that tail. + */ +async function containsEicar(body: ReadableStream): Promise { + const reader = body.getReader(); + let tail = new Uint8Array(); + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) return false; + const window = new Uint8Array(tail.byteLength + value.byteLength); + window.set(tail); + window.set(value, tail.byteLength); + if (indexOf(window, NEEDLE) !== -1) { + await reader.cancel(); + return true; + } + tail = window.slice(Math.max(0, window.byteLength - (NEEDLE.byteLength - 1))); + } + } finally { + reader.releaseLock(); + } +} + +/** Infected exactly when the bytes contain the EICAR string, anywhere. Never unavailable. */ +export function memoryUploadScanner(): UploadScanner { + return { + name: 'memory', + async scan(input: ScanInput): Promise { + return (await containsEicar(input.body)) + ? { verdict: 'infected', signature: SIGNATURE } + : { verdict: 'clean' }; + }, + }; +} diff --git a/packages/storage/src/upload-scanner.test.ts b/packages/storage/src/upload-scanner.test.ts new file mode 100644 index 000000000..00249058c --- /dev/null +++ b/packages/storage/src/upload-scanner.test.ts @@ -0,0 +1,68 @@ +// The `uploadScanner` slot and its memory reference: EICAR is found wherever the stream cuts it, +// clean bytes are clean, and an install is undone by its own release only. + +import { afterEach, describe, expect, test } from 'bun:test'; +import { + installUploadScanner, + resetUploadScanner, + type UploadScanner, + uploadScanner, +} from './upload-scanner'; +import { EICAR_TEST_STRING, memoryUploadScanner } from './upload-scanner-memory'; + +const encode = (text: string): Uint8Array => new TextEncoder().encode(text); + +/** A body cut into `size`-byte chunks — a scanner that matched per chunk misses EICAR across one. */ +function chunked(bytes: Uint8Array, size: number): ReadableStream { + let at = 0; + return new ReadableStream({ + pull(controller) { + if (at >= bytes.byteLength) return controller.close(); + controller.enqueue(bytes.subarray(at, at + size)); + at += size; + }, + }); +} + +const scan = (scanner: UploadScanner, bytes: Uint8Array, size = bytes.byteLength) => + scanner.scan({ + key: 'org/o/pending/u.txt', + size: bytes.byteLength, + contentType: 'text/plain', + body: chunked(bytes, Math.max(1, size)), + }); + +afterEach(() => resetUploadScanner()); + +describe('memoryUploadScanner', () => { + test('the EICAR test file is infected, with a signature naming it', async () => { + const verdict = await scan(memoryUploadScanner(), encode(EICAR_TEST_STRING)); + expect(verdict).toEqual({ verdict: 'infected', signature: 'EICAR-Test-File' }); + }); + + test('EICAR split across one-byte chunks is still found', async () => { + const verdict = await scan(memoryUploadScanner(), encode(`hello ${EICAR_TEST_STRING}`), 1); + expect(verdict.verdict).toBe('infected'); + }); + + test('clean and empty bodies are clean', async () => { + expect(await scan(memoryUploadScanner(), encode('a plain note'))).toEqual({ verdict: 'clean' }); + expect(await scan(memoryUploadScanner(), new Uint8Array())).toEqual({ verdict: 'clean' }); + }); + + test('the literal is the 68-byte EICAR string', () => { + expect(EICAR_TEST_STRING).toHaveLength(68); + expect(EICAR_TEST_STRING.startsWith('X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR')).toBe(true); + }); +}); + +describe('the uploadScanner slot', () => { + test('answers nothing until a scanner is installed, and nothing after its release', () => { + expect(uploadScanner()).toBeUndefined(); + const scanner = memoryUploadScanner(); + const release = installUploadScanner(scanner); + expect(uploadScanner()).toBe(scanner); + release(); + expect(uploadScanner()).toBeUndefined(); + }); +}); diff --git a/packages/storage/src/upload-scanner.ts b/packages/storage/src/upload-scanner.ts new file mode 100644 index 000000000..29de154d6 --- /dev/null +++ b/packages/storage/src/upload-scanner.ts @@ -0,0 +1,52 @@ +// The `uploadScanner` port (`ADAPTER_PORTS`): what a content scanner is asked, what it answers, and +// the one slot the boot installs `runtime.ts`'s scanner into. `promoteAttachment` asks it; the +// framework ships the seam and a memory reference, never a vendor (axiom 8 — ClamAV is the app's). + +import { adapterSlot } from '@ultimat3/core/adapter'; + +/** One stored object, handed over as a stream: a scanner never forces the whole file into a heap. */ +export interface ScanInput { + /** Where the scanned bytes live: this confirm's own copy under `pending/scan/`, no URL rewrites. */ + readonly key: string; + readonly size: number; + /** The type the object is stored under, already sniffed against its bytes. */ + readonly contentType: string; + /** Every byte of the object. Read it to the end or cancel it; it is not reused. */ + readonly body: ReadableStream; +} + +/** + * A verdict about the bytes, never about the scanner. A scanner that cannot decide — its daemon is + * down, it timed out, the file is past its own limit — REJECTS instead: `promoteAttachment` turns + * that into `X_STORAGE_SCAN_UNAVAILABLE` (or a pass, under `onScanUnavailable: 'accept'`), and an + * infected verdict is never something a policy can wave through. + */ +export type ScanVerdict = + | { readonly verdict: 'clean' } + | { readonly verdict: 'infected'; readonly signature: string }; + +export interface UploadScanner { + /** Named in every refusal's `meta.scanner` — `clamd`, `memory`. */ + readonly name: string; + scan(input: ScanInput): Promise; +} + +const ambient = /* @__PURE__ */ adapterSlot(() => undefined); + +/** + * The boot's install, from `uploadScanner` in `apps//runtime.ts` — exported from + * `@ultimat3/storage/adapter` only, so that file is the one place an app chooses a scanner. Answers + * the release that removes THIS install and no other. + */ +export function installUploadScanner(scanner: UploadScanner): () => void { + return ambient.install(scanner); +} + +/** The installed scanner, or `undefined`: no scanner installed means promotion scans nothing. */ +export function uploadScanner(): UploadScanner | undefined { + return ambient.current(); +} + +export function resetUploadScanner(): void { + ambient.reset(); +} diff --git a/packages/storage/src/upload.ts b/packages/storage/src/upload.ts index c9c6aa999..f656e3c11 100644 --- a/packages/storage/src/upload.ts +++ b/packages/storage/src/upload.ts @@ -7,7 +7,7 @@ // confirmed. A type no rule can ever confirm (`text/csv`, `application/json`) is still accepted // on the client's word: there is nothing to compare it against. -import { finiteCount } from '@ultimat3/core'; +import { assert, finiteCount } from '@ultimat3/core'; import { sha256Base64 } from './driver'; import { checksumMismatch, @@ -55,12 +55,33 @@ export interface UploadPolicy { readonly allowedContentTypes: readonly string[]; /** When true a candidate without a `checksum` is rejected, not silently trusted. */ readonly requireChecksum: boolean; + /** + * What `promoteAttachment` does when the installed `uploadScanner` gives no verdict (its daemon is + * down, it timed out). `'refuse'` — fail closed, `X_STORAGE_SCAN_UNAVAILABLE`, the upload stays + * pending for a retry — is the default and what an absent field means. `'accept'` promotes the + * file unscanned. An INFECTED verdict is refused under either: this decides availability only. + */ + readonly onScanUnavailable?: ScanUnavailablePolicy | undefined; } +export type ScanUnavailablePolicy = 'refuse' | 'accept'; + export interface UploadPolicyInit { readonly maxBytes?: number | undefined; readonly allowedContentTypes?: readonly string[] | undefined; readonly requireChecksum?: boolean | undefined; + readonly onScanUnavailable?: ScanUnavailablePolicy | undefined; +} + +/** Screened where it is declared: a typo (`'allow'`) must not read as the fail-open answer. */ +function scanUnavailablePolicy(value: ScanUnavailablePolicy | undefined): ScanUnavailablePolicy { + const policy = value === undefined ? 'refuse' : value; + assert( + policy === 'refuse' || policy === 'accept', + `uploadPolicy was given onScanUnavailable ${JSON.stringify(policy)}, which is neither 'refuse' nor 'accept'`, + "uploadPolicy({ onScanUnavailable: 'refuse' }) — or 'accept' to promote unscanned files while the scanner is down", + ); + return policy; } export function uploadPolicy(init: UploadPolicyInit = {}): UploadPolicy { @@ -84,6 +105,7 @@ export function uploadPolicy(init: UploadPolicyInit = {}): UploadPolicy { normalizeContentType, ), requireChecksum: init.requireChecksum ?? false, + onScanUnavailable: scanUnavailablePolicy(init.onScanUnavailable), }; } diff --git a/packages/testing/CLAUDE.md b/packages/testing/CLAUDE.md index bc5db1647..0d5f9510b 100644 --- a/packages/testing/CLAUDE.md +++ b/packages/testing/CLAUDE.md @@ -3,7 +3,7 @@ Tier 5. May import tiers 0–4. Imported by every package's tests and by generated apps. Deps (21): `core` `schema`; `cache` `db` `i18n` `seo` `time`; `auth` `entity` `http` `policy`; `action` -`jobs` `query` `realtime`; `ai` `mail` `notify` `pwa` `render` `scraping` (`ai`, `auth`, `notify`, `scraping` for `swapAdapter`, dynamic-only). Static in `.`: `cache` +`jobs` `query` `realtime`; `ai` `mail` `notify` `pwa` `render` `scraping` (`ai`, `auth`, `notify`, `scraping` for `swapAdapter`, dynamic-only). Static in `.`: `scraping/cdp`, `cache` (`registry-leak-guard.ts:7`), `i18n`, `jobs`, `policy` (`registry-snapshot.ts:29-38`), `db` (`reusable-database.ts:20`), `realtime/server` (`fixture-subscribe.ts:13`). Dynamic-only: `action`, `http`, `mail`, `pwa`, `query`, `render`, `schema` (`exercise-input.ts:16`), `time` — no `mail` destructure, no mail @@ -132,7 +132,8 @@ The island fixture's cost and its whole-chain proof: `docs/history/testing.md`. ## The browser-backed e2e driver (moved here from `@ultimat3/cli` in 22.0.0) -Raw CDP: `@ultimat3/core` and Bun, nothing else; `cli` imports it over `cli -> testing`. One job per +Raw CDP; `cli` imports it over `cli -> testing`. Launcher + connection: `@ultimat3/scraping/cdp` +(31.0.0), never re-exported. One job per file (`e2e-*.ts`, `cdp-*.ts`); the table of which file does what is [`docs/history/testing-e2e-driver.md`](../../docs/history/testing-e2e-driver.md). The ONE call an app's test preload makes is `installE2eDriver({ page, baseUrl })` (`e2e-driver.ts`). diff --git a/packages/testing/README.md b/packages/testing/README.md index 3c21f3110..b1e72f965 100644 --- a/packages/testing/README.md +++ b/packages/testing/README.md @@ -524,10 +524,11 @@ is meant to FAIL it. The subject is called once per test. | `FailureStore` (core) | `behavesLike(failureStoreConformance, () => ({ store, find, client }))` | | `Gateway` (ai) | `behavesLike(gatewayConformance, () => ({ gateway, model }))` | | `IdempotencyStore` (action) | `behavesLike(idempotencyStoreConformance, () => ({ store, client, reclaimAfterMs, … }))` | +| `IdentityProvider` (auth) | `behavesLike(identityProviderConformance, () => ({ provider, callbackUrl, clock, answer, tamper }))` — the harness plays the IdP: `answer(redirect)` is a consenting user's callback params, `tamper(params)` alters their proof. Not an `ADAPTER_PORTS` row: an app holds several providers at once | | `ImageTransformDriver` (seo) | `behavesLike(imageTransformDriverConformance, () => ({ driver, src }))` — `src` reads as `IMAGE_CONFORMANCE_SOURCE` (64×32 PNG) | | `IsrStore` (render) | `behavesLike(isrStoreConformance, () => store)` — the tag-fence check runs only when `store.tagFence` exists | | `JobDriver` (jobs) | `behavesLike(jobDriverConformance, () => driver)` | -| `LeaseStore` (jobs) | `behavesLike(leaseStoreConformance, () => ({ store, elapse }))` | +| `LeaseStore` (jobs) | `behavesLike(leaseStoreConformance, () => ({ store, elapse }))` — every grant needs a `fence` greater than every earlier one, takeovers included (`withLock`'s token) | | `MailDriver` (mail) | `behavesLike(mailDriverConformance, () => driver)` | | `McpConfirmationStore` (mcp) | `behavesLike(mcpConfirmationStoreConformance, () => store)` | | `MeterStore` (action) | `behavesLike(meterStoreConformance, () => store)` | @@ -540,6 +541,7 @@ is meant to FAIL it. The subject is called once per test. | `RateLimitStore` (http) | `behavesLike(rateLimitStoreConformance, () => store)` | | `ScrapeDriver` (scraping) | `behavesLike(scrapeDriverConformance, () => driver)` — serving `SCRAPE_CONFORMANCE_PAGES` on `shop.test` | | `SecretSource` (core) | `behavesLike(secretSourceConformance, () => ({ source, holding, empty }))` | +| `SmsTransport` (notify) | `behavesLike(smsTransportConformance, () => transport)` — `memorySmsTransport()` passes; a provider adapter (Twilio, Vonage, SNS) proves itself here | | `SleepClock` (core) | `behavesLike(sleepClockConformance, () => clock)` | | `StepStore` (jobs) | `behavesLike(stepStoreConformance, () => driver)` — any `JobDriver`; a lone store: `memoryJobDriver({ steps })` | | `StorageDriver` (storage) | `behavesLike(storageDriverConformance, () => disk)` — checks write under `conformance//` | @@ -1119,6 +1121,10 @@ declared edge and the reverse is a `bun run boundaries` failure. ## Launching Chrome +**Imported from `@ultimat3/scraping/cdp` since 31.0.0** — its one home, which `pdf()` prints +through: `findChrome`, `launchChrome`, `CHROME_*`, `LAUNCH_*`, `LAUNCHED_CLOSE_MS`, `cdpConnect*` and +the `Cdp*Error`s. Never re-exported here (`flight-copies`, `X_HELPER_COPY`). + `launchChrome({ executable, timeoutMs, launchTimeoutMs?, wire? })` — what the `e2e` step and `x shot` both start a browser with. Driven over `--remote-debugging-pipe`, and on Windows — where the parent cannot hold Chrome's fds 3 and 4 — over `--remote-debugging-port=0` and the profile's @@ -1134,7 +1140,7 @@ on Linux, the app bundles on macOS, Chrome under `%ProgramFiles%` / `%ProgramFil | `LAUNCH_ATTEMPTS` | `2` — an unanswered start is reaped and started once more on a fresh profile. Never more | | `LAUNCH_BUDGET_MS` | derived: `LAUNCH_ATTEMPTS × (LAUNCH_TIMEOUT_MS + LAUNCH_REAP_MS)` — the longest a launch is designed to take, both starts and their bounded reaps | | `E2E_BROWSER_OPEN_MS` | derived: `LAUNCH_BUDGET_MS` + the session's two setup calls + the first tab, at `DEFAULT_CDP_TIMEOUT_MS` each. THE deadline of every `beforeAll` that opens a browser — one below it is killed by Bun before the relaunch, and `X_CDP_LAUNCH_FAILED` never names the slow step | -| `E2E_BROWSER_CLOSE_MS` | derived: `4 × CLOSE_GRACE_MS` — SIGTERM's grace, SIGKILL's, the group reap, the profile's removal. THE deadline of an `afterAll` that closes a browser it holds (one that releases a lease takes `E2E_BROWSER_OPEN_MS`: it may await the open). At Bun's 5 s default a close cut short by the run's end left the profile half-removed in the temp root | +| `LAUNCHED_CLOSE_MS` (`@ultimat3/scraping/cdp`; was `E2E_BROWSER_CLOSE_MS`) | derived: `4 × CLOSE_GRACE_MS` — SIGTERM's grace, SIGKILL's, the group reap, the profile's removal. THE deadline of an `afterAll` that closes a browser it holds (one that releases a lease takes `E2E_BROWSER_OPEN_MS`: it may await the open). At Bun's 5 s default a close cut short by the run's end left the profile half-removed in the temp root | | `E2E_TAB_OPEN_MS` | derived: one more `session.newTab()` — `Target.createTarget`, then its attach | | `E2E_GOTO_MS` | derived: one `tab.goto()` — the navigate raced by the load event, then two reads of the document | | `E2E_APP_START_MS` | derived: `startE2eApp()` at its default deadline — `x db reset`, `x db seed` and `/readyz` (wall-clock), each its whole 90 s, then a refused boot's stop | diff --git a/packages/testing/e2e/cdp-browser.e2e.test.ts b/packages/testing/e2e/cdp-browser.e2e.test.ts index a43523737..e13e22405 100644 --- a/packages/testing/e2e/cdp-browser.e2e.test.ts +++ b/packages/testing/e2e/cdp-browser.e2e.test.ts @@ -17,10 +17,10 @@ // refuses to skip when the extension is missing, for the same reason. import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { findChrome } from '@ultimat3/scraping/cdp'; import type { PageLike } from '@ultimat3/testing'; import type { E2eBrowser } from '../src/cdp-browser'; import { E2E_BROWSER_OPEN_MS, openE2eBrowser, openE2eBrowserIfAvailable } from '../src/cdp-browser'; -import { findChrome } from '../src/cdp-launch'; import { e2ePage } from '../src/e2e-page'; // The page under test. Deliberately hand-written HTML rather than a rendered app: what is being diff --git a/packages/testing/e2e/cdp-launch-targets.e2e.test.ts b/packages/testing/e2e/cdp-launch-targets.e2e.test.ts index 3eb3f7cfc..2adadcbc1 100644 --- a/packages/testing/e2e/cdp-launch-targets.e2e.test.ts +++ b/packages/testing/e2e/cdp-launch-targets.e2e.test.ts @@ -5,8 +5,8 @@ // `E2E_BROWSER_REQUIRED=1`. import { describe, expect, test } from 'bun:test'; +import { findChrome, launchChrome } from '@ultimat3/scraping/cdp'; import { E2E_BROWSER_OPEN_MS } from '../src/cdp-browser'; -import { findChrome, launchChrome } from '../src/cdp-launch'; const chrome = await findChrome(process.env); const required = process.env['E2E_BROWSER_REQUIRED'] === '1'; diff --git a/packages/testing/e2e/cdp-session.e2e.test.ts b/packages/testing/e2e/cdp-session.e2e.test.ts index f22dbf1b1..ed98e4e1a 100644 --- a/packages/testing/e2e/cdp-session.e2e.test.ts +++ b/packages/testing/e2e/cdp-session.e2e.test.ts @@ -4,9 +4,9 @@ // Chrome, and refuses to skip under `E2E_BROWSER_REQUIRED=1`, like `cdp-browser.e2e.test.ts`. import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { findChrome } from '@ultimat3/scraping/cdp'; import type { E2eBrowser } from '../src/cdp-browser'; import { E2E_BROWSER_OPEN_MS, openE2eBrowser, openE2eBrowserIfAvailable } from '../src/cdp-browser'; -import { findChrome } from '../src/cdp-launch'; /** The worker every tab connects to: one socket, opened at start-up, for all of them. */ const WORKER = ` diff --git a/packages/testing/e2e/service-worker-handover.e2e.test.ts b/packages/testing/e2e/service-worker-handover.e2e.test.ts index 24713ebca..2d9846307 100644 --- a/packages/testing/e2e/service-worker-handover.e2e.test.ts +++ b/packages/testing/e2e/service-worker-handover.e2e.test.ts @@ -9,10 +9,10 @@ // states why. import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { findChrome } from '@ultimat3/scraping/cdp'; import type { PageLike } from '@ultimat3/testing'; import type { E2eBrowser } from '../src/cdp-browser'; import { E2E_BROWSER_OPEN_MS, openE2eBrowser, openE2eBrowserIfAvailable } from '../src/cdp-browser'; -import { findChrome } from '../src/cdp-launch'; import { e2ePage } from '../src/e2e-page'; // Registered on `load`, as the framework's own `x-sw-register.js` does — so the document is diff --git a/packages/testing/src/cdp-browser-lease.test.ts b/packages/testing/src/cdp-browser-lease.test.ts index 1684796d7..5b0abc8a1 100644 --- a/packages/testing/src/cdp-browser-lease.test.ts +++ b/packages/testing/src/cdp-browser-lease.test.ts @@ -2,11 +2,11 @@ // closes is the assertion. The real Chrome under it is `packages/cli/e2e/client-navigation-*`. import { describe, expect, test } from 'bun:test'; +import { CdpLaunchFailedError } from '@ultimat3/scraping/cdp'; import type { E2eBrowser } from './cdp-browser'; import { E2E_BROWSER_OPEN_MS } from './cdp-browser'; import type { RunEndHook } from './cdp-browser-lease'; import { createE2eBrowserLeases } from './cdp-browser-lease'; -import { CdpLaunchFailedError } from './cdp-errors'; /** A browser that counts its closes; nothing else of it is touched here. */ const counted = () => { diff --git a/packages/testing/src/cdp-browser.test.ts b/packages/testing/src/cdp-browser.test.ts index a66779e74..4bee23570 100644 --- a/packages/testing/src/cdp-browser.test.ts +++ b/packages/testing/src/cdp-browser.test.ts @@ -3,9 +3,17 @@ // the rule a CI box with no browser depends on. import { describe, expect, test } from 'bun:test'; +import type { CdpConnection, CdpEventListener, CdpResult } from '@ultimat3/scraping/cdp'; +import { + CHROME_PATH_ENV, + CLOSE_GRACE_MS, + LAUNCH_ATTEMPTS, + LAUNCH_BUDGET_MS, + LAUNCH_TIMEOUT_MS, + LAUNCHED_CLOSE_MS, +} from '@ultimat3/scraping/cdp'; import { DEFAULT_CDP_TIMEOUT_MS, - E2E_BROWSER_CLOSE_MS, E2E_BROWSER_OPEN_MS, E2E_GOTO_MS, E2E_TAB_OPEN_MS, @@ -15,17 +23,8 @@ import { SESSION_SETUP_DEADLINES, TAB_OPEN_DEADLINES, } from './cdp-browser'; -import type { CdpConnection, CdpEventListener, CdpResult } from './cdp-connection'; import { FULL_AX_TREE } from './cdp-e2e-page'; import { cdpE2eSession } from './cdp-e2e-session'; -import { - CHROME_PATH_ENV, - LAUNCH_ATTEMPTS, - LAUNCH_BUDGET_MS, - LAUNCH_TIMEOUT_MS, -} from './cdp-launch'; -import { LAUNCHED_CLOSE_MS } from './cdp-launch-attempt'; -import { CLOSE_GRACE_MS } from './cdp-launch-reap'; const NOWHERE = { [CHROME_PATH_ENV]: '/nonexistent/definitely-not-a-browser' }; @@ -110,7 +109,7 @@ describe('the open budget a browser hook is given', () => { }); test('the close budget is the launched browser close, every step of it bounded by a grace', () => { - expect(E2E_BROWSER_CLOSE_MS).toBe(LAUNCHED_CLOSE_MS); + expect(LAUNCHED_CLOSE_MS).toBe(LAUNCHED_CLOSE_MS); // SIGTERM's grace, SIGKILL's, the group reap, the profile's removal. expect(LAUNCHED_CLOSE_MS).toBe(4 * CLOSE_GRACE_MS); }); diff --git a/packages/testing/src/cdp-browser.ts b/packages/testing/src/cdp-browser.ts index e4dde1a3e..488f905fd 100644 --- a/packages/testing/src/cdp-browser.ts +++ b/packages/testing/src/cdp-browser.ts @@ -8,13 +8,17 @@ // already decided a browser is required. import { finiteCount } from '@ultimat3/core'; +import type { LaunchedBrowser } from '@ultimat3/scraping/cdp'; +import { + CdpBrowserMissingError, + CHROME_CANDIDATES, + findChrome, + LAUNCH_BUDGET_MS, + launchChrome, +} from '@ultimat3/scraping/cdp'; import type { E2eTab } from './cdp-e2e-page'; import type { E2eSession } from './cdp-e2e-session'; import { cdpE2eSession } from './cdp-e2e-session'; -import { CdpBrowserMissingError } from './cdp-errors'; -import type { LaunchedBrowser } from './cdp-launch'; -import { CHROME_CANDIDATES, findChrome, LAUNCH_BUDGET_MS, launchChrome } from './cdp-launch'; -import { LAUNCHED_CLOSE_MS } from './cdp-launch-attempt'; /** How long a launch, a connect or a single CDP call may take. One number, three deadlines. */ export const DEFAULT_CDP_TIMEOUT_MS = 30_000; @@ -41,13 +45,8 @@ export const E2E_TAB_OPEN_MS = TAB_OPEN_DEADLINES * DEFAULT_CDP_TIMEOUT_MS; export const E2E_BROWSER_OPEN_MS = LAUNCH_BUDGET_MS + SESSION_SETUP_DEADLINES * DEFAULT_CDP_TIMEOUT_MS + E2E_TAB_OPEN_MS; -/** - * The longest `E2eBrowser.close()` is DESIGNED to take — the launched browser's close, every step - * bounded (`LAUNCHED_CLOSE_MS`). THE deadline for a hook that closes a browser it already holds: a - * hook left at Bun's 5 s default is killed mid-close, and when it is the run's last the process - * exits with the profile half-removed — the `x-e2e-chrome-*` directories found leaked in `/tmp`. - */ -export const E2E_BROWSER_CLOSE_MS = LAUNCHED_CLOSE_MS; +// The longest `E2eBrowser.close()` takes is `LAUNCHED_CLOSE_MS` (`@ultimat3/scraping/cdp`), the +// launched browser's own close: THE deadline for a hook that closes a browser, one name since 31.0.0. /** * Sequential deadlines one `tab.goto()` spends: the load event raced against `Page.navigate`'s diff --git a/packages/testing/src/cdp-e2e-page.test.ts b/packages/testing/src/cdp-e2e-page.test.ts index 75f4804fe..a7826d803 100644 --- a/packages/testing/src/cdp-e2e-page.test.ts +++ b/packages/testing/src/cdp-e2e-page.test.ts @@ -4,10 +4,10 @@ // (`cdp-e2e-session.test.ts`). import { describe, expect, test } from 'bun:test'; -import type { CdpConnection, CdpResult } from './cdp-connection'; +import type { CdpConnection, CdpResult } from '@ultimat3/scraping/cdp'; +import { CdpCallFailedError } from '@ultimat3/scraping/cdp'; import type { E2eTab } from './cdp-e2e-page'; import { cdpE2eTab, FULL_AX_TREE } from './cdp-e2e-page'; -import { CdpCallFailedError } from './cdp-errors'; interface Call { readonly method: string; diff --git a/packages/testing/src/cdp-e2e-page.ts b/packages/testing/src/cdp-e2e-page.ts index 43d7ac8ae..bac47f365 100644 --- a/packages/testing/src/cdp-e2e-page.ts +++ b/packages/testing/src/cdp-e2e-page.ts @@ -7,10 +7,10 @@ // through it: `ScrapePage` is a full scraping surface whose intended implementation is // `puppeteer-core`, and an e2e driver needs none of it. +import type { CdpConnection } from '@ultimat3/scraping/cdp'; +import { CdpCallFailedError, CdpTimeoutError } from '@ultimat3/scraping/cdp'; import type { AxTreeNode } from './ax-node'; import { axRecord, toAxTreeNodes } from './ax-node'; -import type { CdpConnection } from './cdp-connection'; -import { CdpCallFailedError, CdpTimeoutError } from './cdp-errors'; import type { AxTreeRequest, E2eBrowserPage } from './e2e-page'; /** What the page threw, when `Runtime.evaluate` answered with an exception rather than a value. */ diff --git a/packages/testing/src/cdp-e2e-session-edges.test.ts b/packages/testing/src/cdp-e2e-session-edges.test.ts index 72402a113..eea8196d7 100644 --- a/packages/testing/src/cdp-e2e-session-edges.test.ts +++ b/packages/testing/src/cdp-e2e-session-edges.test.ts @@ -4,7 +4,7 @@ // applies the cut is a bounded timeout, and a tab gone by the switch is dropped from it. import { describe, expect, test } from 'bun:test'; -import type { CdpConnection } from './cdp-connection'; +import type { CdpConnection } from '@ultimat3/scraping/cdp'; import { cdpE2eSession } from './cdp-e2e-session'; import { fake } from './cdp-e2e-session-fake-fixture'; import { NETWORK_PROBE } from './cdp-network-probe'; diff --git a/packages/testing/src/cdp-e2e-session-fake-fixture.ts b/packages/testing/src/cdp-e2e-session-fake-fixture.ts index 603424624..cd19be9fd 100644 --- a/packages/testing/src/cdp-e2e-session-fake-fixture.ts +++ b/packages/testing/src/cdp-e2e-session-fake-fixture.ts @@ -2,8 +2,8 @@ // call, holds a paused target's answers, and models when a page APPLIES a network condition — its // `navigator.onLine` and, separately, what a request from it meets. -import type { CdpConnection, CdpEventListener, CdpResult } from './cdp-connection'; -import { CdpCallFailedError, CdpTimeoutError } from './cdp-errors'; +import type { CdpConnection, CdpEventListener, CdpResult } from '@ultimat3/scraping/cdp'; +import { CdpCallFailedError, CdpTimeoutError } from '@ultimat3/scraping/cdp'; import { NETWORK_PROBE, NETWORK_PROBE_PATH } from './cdp-network-probe'; export interface Call { diff --git a/packages/testing/src/cdp-e2e-session.test.ts b/packages/testing/src/cdp-e2e-session.test.ts index 96149627f..69a2d070f 100644 --- a/packages/testing/src/cdp-e2e-session.test.ts +++ b/packages/testing/src/cdp-e2e-session.test.ts @@ -3,7 +3,7 @@ // browser run is `e2e/cdp-session.e2e.test.ts`; this is the part a fake can pin exactly. import { describe, expect, test } from 'bun:test'; -import type { CdpConnection } from './cdp-connection'; +import type { CdpConnection } from '@ultimat3/scraping/cdp'; import { cdpE2eSession } from './cdp-e2e-session'; import { type Call, confirming, fake } from './cdp-e2e-session-fake-fixture'; import { NETWORK_PROBE } from './cdp-network-probe'; diff --git a/packages/testing/src/cdp-e2e-session.ts b/packages/testing/src/cdp-e2e-session.ts index 6c4ef9a90..06f648860 100644 --- a/packages/testing/src/cdp-e2e-session.ts +++ b/packages/testing/src/cdp-e2e-session.ts @@ -3,10 +3,10 @@ // from their first byte; the offline switch, the init scripts, and the log of every WebSocket and // request the browser made. A tab is `cdp-e2e-page.ts`'s; this file decides what every tab shares. -import type { CdpConnection } from './cdp-connection'; +import type { CdpConnection } from '@ultimat3/scraping/cdp'; +import { CdpCallFailedError, CdpTimeoutError } from '@ultimat3/scraping/cdp'; import type { E2eTab } from './cdp-e2e-page'; import { cdpE2eTab } from './cdp-e2e-page'; -import { CdpCallFailedError, CdpTimeoutError } from './cdp-errors'; import { isNetworkProbe, networkAgrees } from './cdp-network-probe'; import { offlineScripts } from './cdp-offline-script'; import type { AuditedTab } from './e2e-a11y-tabs'; diff --git a/packages/testing/src/cdp-network-probe.ts b/packages/testing/src/cdp-network-probe.ts index 58010c219..e7f639996 100644 --- a/packages/testing/src/cdp-network-probe.ts +++ b/packages/testing/src/cdp-network-probe.ts @@ -3,7 +3,7 @@ // the condition (two pipes, no order between them), so a `fetch` on the next line went through under // a cut the page already read as offline — `browser-harness.e2e.test.ts`, CI run 39223207154. -import type { CdpResult } from './cdp-connection'; +import type { CdpResult } from '@ultimat3/scraping/cdp'; type Send = ( method: string, diff --git a/packages/testing/src/cdp-offline-script.test.ts b/packages/testing/src/cdp-offline-script.test.ts index f6089f00a..b17b9392a 100644 --- a/packages/testing/src/cdp-offline-script.test.ts +++ b/packages/testing/src/cdp-offline-script.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'bun:test'; -import { CdpCallFailedError } from './cdp-errors'; +import { CdpCallFailedError } from '@ultimat3/scraping/cdp'; import { offlineScripts } from './cdp-offline-script'; // Row aa: `offline(true)` twice registered the script twice and kept one id, so `online()` removed diff --git a/packages/testing/src/cdp-offline-script.ts b/packages/testing/src/cdp-offline-script.ts index 413fb7cc1..a9bbaf284 100644 --- a/packages/testing/src/cdp-offline-script.ts +++ b/packages/testing/src/cdp-offline-script.ts @@ -12,8 +12,8 @@ // restore; one it never told gets one from `RESTORE_ONLINE`, so a page reconnects exactly once. import { stringField } from '@ultimat3/core'; -import type { CdpResult } from './cdp-connection'; -import { CdpCallFailedError } from './cdp-errors'; +import type { CdpResult } from '@ultimat3/scraping/cdp'; +import { CdpCallFailedError } from '@ultimat3/scraping/cdp'; type Send = ( method: string, diff --git a/packages/testing/src/conformance.ts b/packages/testing/src/conformance.ts index 9ca1713dd..44dd52e2f 100644 --- a/packages/testing/src/conformance.ts +++ b/packages/testing/src/conformance.ts @@ -3,7 +3,8 @@ // off the `.` barrel: these suites statically import the packages whose ports they hold to. // Grouped by the package that owns the interface; names match `ADAPTER_PORTS[].conformance`. -// @ultimat3/ai — Gateway (aiGateway), BudgetStore; Provider, VectorStore (no rows) +// @ultimat3/ai — Gateway (aiGateway), Embedder (embedder), SemanticCacheFactory (semanticCache), +// BudgetStore; Provider, VectorStore (no rows) export type { GatewayCheck, GatewaySubject } from './ai-gateway-conformance'; export { GATEWAY_CHECKS, gatewayConformance } from './ai-gateway-conformance'; export type { ProviderCheck, ProviderSubject } from './ai-provider-conformance'; @@ -39,6 +40,8 @@ export type { CacheTierCheck } from './cache-tier-conformance'; export { CACHE_TIER_CHECKS, cacheTierConformance } from './cache-tier-conformance'; export type { SleepClockCheck } from './clock-conformance'; export { SLEEP_CLOCK_CHECKS, sleepClockConformance } from './clock-conformance'; +export type { EmbedderCheck } from './embedder-conformance'; +export { EMBEDDER_CHECKS, embedderConformance } from './embedder-conformance'; // @ultimat3/entity — the Driver behind `database()` (no row: chosen per `database()` call) export { ENTITY_DRIVER_CHECKS } from './entity-conformance-checks'; export type { @@ -85,6 +88,15 @@ export { idempotencyKeyOf, idempotencyStoreConformance, } from './idempotency-store-conformance'; +// @ultimat3/auth — IdentityProvider (no row: an app holds several at once, by id) +export type { + IdentityProviderCheck, + IdentityProviderHarness, +} from './identity-provider-conformance'; +export { + IDENTITY_PROVIDER_CHECKS, + identityProviderConformance, +} from './identity-provider-conformance'; // @ultimat3/seo — ImageTransformDriver (images) export type { ImageTransformCheck, ImageTransformSubject } from './image-transform-conformance'; export { @@ -107,7 +119,6 @@ export { MAIL_DRIVER_CHECKS, mailDriverConformance, } from './mail-driver-conformance'; - // @ultimat3/mcp — McpConfirmationStore export type { McpConfirmationStoreCheck } from './mcp-confirmation-store-conformance'; export { @@ -160,8 +171,38 @@ export { SCRAPE_CONFORMANCE_PAGES, SCRAPE_CONFORMANCE_URLS, } from './scrape-driver-conformance-site'; +// @ultimat3/entity — SearchIndex (searchIndex) +export { + SEARCH_INDEX_ALL_CHECKS, + SEARCH_INDEX_CHECKS, + searchIndexConformance, +} from './search-index-conformance'; +export type { + SearchDocOptions, + SearchIndexCheck, + SearchIndexSubject, +} from './search-index-conformance-support'; +export { + SEARCH_TENANT_A, + SEARCH_TENANT_B, + searchDoc, + searchDocId, +} from './search-index-conformance-support'; +export { SEARCH_INDEX_VERSION_CHECKS } from './search-index-conformance-versions'; export type { SecretSourceCheck, SecretSourceSubject } from './secret-source-conformance'; export { SECRET_SOURCE_CHECKS, secretSourceConformance } from './secret-source-conformance'; +export type { SemanticCacheFactoryCheck } from './semantic-cache-factory-conformance'; +export { + SEMANTIC_CACHE_FACTORY_CHECKS, + semanticCacheFactoryConformance, +} from './semantic-cache-factory-conformance'; +// @ultimat3/notify — SmsTransport (smsTransport) +export type { SmsTransportCheck } from './sms-transport-conformance'; +export { + conformanceSms, + SMS_TRANSPORT_CHECKS, + smsTransportConformance, +} from './sms-transport-conformance'; export type { StepStoreCheck, StepStoreSubject } from './step-store-conformance'; export { STEP_STORE_CHECKS, stepStoreConformance } from './step-store-conformance'; // @ultimat3/storage — StorageDriver @@ -172,10 +213,20 @@ export { TOTP_REPLAY_GUARD_CHECKS, totpReplayGuardConformance, } from './totp-replay-guard-conformance'; - // @ultimat3/realtime — Transport export type { TransportCheck } from './transport-conformance'; export { TRANSPORT_CHECKS, transportConformance } from './transport-conformance'; +// @ultimat3/storage — UploadScanner (uploadScanner), structural +export type { + ScanVerdictLike, + UploadScannerCheck, + UploadScannerLike, +} from './upload-scanner-conformance'; +export { + UPLOAD_SCANNER_CHECKS, + UPLOAD_SCANNER_EICAR, + uploadScannerConformance, +} from './upload-scanner-conformance'; export { vectorStoreConformance } from './vector-store-conformance'; export type { VectorStoreCheck } from './vector-store-conformance-checks'; export { diff --git a/packages/testing/src/e2e-a11y-run.test.ts b/packages/testing/src/e2e-a11y-run.test.ts index 4ca87cf44..c03d4d32f 100644 --- a/packages/testing/src/e2e-a11y-run.test.ts +++ b/packages/testing/src/e2e-a11y-run.test.ts @@ -3,8 +3,8 @@ // `allow` with no reason. The real browser half is `e2e-a11y.e2e.test.ts`. import { describe, expect, test } from 'bun:test'; +import { CdpCallFailedError } from '@ultimat3/scraping/cdp'; import type { AxTreeNode } from './ax-node'; -import { CdpCallFailedError } from './cdp-errors'; import type { A11yAllow } from './e2e-a11y-run'; import { checkAllow, E2eA11yViolationError, startA11yRun } from './e2e-a11y-run'; import type { AxTreeRequest, E2eBrowserPage } from './e2e-page'; diff --git a/packages/testing/src/e2e-a11y.e2e.test.ts b/packages/testing/src/e2e-a11y.e2e.test.ts index 014eb1d4e..d1056baf3 100644 --- a/packages/testing/src/e2e-a11y.e2e.test.ts +++ b/packages/testing/src/e2e-a11y.e2e.test.ts @@ -8,9 +8,9 @@ // Skips with no browser on the machine, and refuses to where `E2E_BROWSER_REQUIRED=1` promised one. import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { findChrome, LAUNCHED_CLOSE_MS } from '@ultimat3/scraping/cdp'; import type { E2eBrowser } from './cdp-browser'; -import { E2E_BROWSER_CLOSE_MS, E2E_BROWSER_OPEN_MS, openE2eBrowser } from './cdp-browser'; -import { findChrome } from './cdp-launch'; +import { E2E_BROWSER_OPEN_MS, openE2eBrowser } from './cdp-browser'; import { endTabA11y } from './e2e-a11y-tabs'; import { e2eRunner } from './e2e-driver'; import type { E2eBody } from './test-types'; @@ -66,7 +66,7 @@ describe.skipIf(noBrowser)('every e2eTest audits the browser’s accessibility t afterAll(async () => { await browser?.close(); server.stop(true); - }, E2E_BROWSER_CLOSE_MS); + }, LAUNCHED_CLOSE_MS); const visit = (path: string): E2eBody => diff --git a/packages/testing/src/e2e-browser-handle.ts b/packages/testing/src/e2e-browser-handle.ts index 2ed6ef0ed..dc9347076 100644 --- a/packages/testing/src/e2e-browser-handle.ts +++ b/packages/testing/src/e2e-browser-handle.ts @@ -2,9 +2,8 @@ // `e2eApp()`. On `globalThis` under one `Symbol.for` key, because the preload and a test may each // hold their own copy of this module — the page-client handle's reason, one runtime over. +import { CdpBrowserMissingError, CHROME_CANDIDATES } from '@ultimat3/scraping/cdp'; import type { E2eBrowser } from './cdp-browser'; -import { CdpBrowserMissingError } from './cdp-errors'; -import { CHROME_CANDIDATES } from './cdp-launch'; import type { E2eApp } from './e2e-app'; /** Set by the e2e step to the app root; the preload spawns that app and opens a browser. */ diff --git a/packages/testing/src/e2e-browser-hooks.test.ts b/packages/testing/src/e2e-browser-hooks.test.ts index c0cd1932b..1c639b8d5 100644 --- a/packages/testing/src/e2e-browser-hooks.test.ts +++ b/packages/testing/src/e2e-browser-hooks.test.ts @@ -1,5 +1,5 @@ // Every hook that opens or closes a browser has a deadline derived from what it does: an open -// starts at `E2E_BROWSER_OPEN_MS`, a close at `E2E_BROWSER_CLOSE_MS`, an app boot adds +// starts at `E2E_BROWSER_OPEN_MS`, a close at `LAUNCHED_CLOSE_MS`, an app boot adds // `E2E_APP_START_MS` and an app stop `E2E_APP_STOP_MS`. A literal there was the shape of four CI // flakes — Bun killed the hook before the relaunch — and Bun's 5 s default around a close was a // leaked Chrome profile per run cut short. Read from the suites' source, through their helpers. @@ -30,7 +30,7 @@ const OPENERS = [ const STARTERS = ['startE2eApp']; const OPEN = 'E2E_BROWSER_OPEN_MS'; -const CLOSE = 'E2E_BROWSER_CLOSE_MS'; +const CLOSE = 'LAUNCHED_CLOSE_MS'; const APP_START = 'E2E_APP_START_MS'; const APP_STOP = 'E2E_APP_STOP_MS'; diff --git a/packages/testing/src/e2e-error-codes.test.ts b/packages/testing/src/e2e-error-codes.test.ts index 820dbb55a..2fdfc83b7 100644 --- a/packages/testing/src/e2e-error-codes.test.ts +++ b/packages/testing/src/e2e-error-codes.test.ts @@ -2,7 +2,7 @@ // that throws them — the two ways a process reaches a CDP refusal. import { expect, test } from 'bun:test'; import { describeErrorCode, hasErrorCode } from '@ultimat3/core'; -import { CdpTimeoutError } from './cdp-errors'; +import { CdpTimeoutError } from '@ultimat3/scraping/cdp'; import { E2E_ERROR_CODES, E2E_ERROR_TITLES } from './index'; test('every e2e and CDP code is registered with its own title', () => { diff --git a/packages/testing/src/e2e-error-codes.ts b/packages/testing/src/e2e-error-codes.ts index 74693f8c7..b16424636 100644 --- a/packages/testing/src/e2e-error-codes.ts +++ b/packages/testing/src/e2e-error-codes.ts @@ -1,9 +1,10 @@ -// The registry for the browser-backed e2e driver's codes: the seven `X_E2E_*` a page refuses with -// and the four `X_CDP_*` its raw-CDP browser refuses with. Registered here, not in `errors.ts`, -// because that catalogue sits at its ceiling; anchored by the barrel and by both constructor files. +// The registry for the browser-backed e2e driver's codes: the seven `X_E2E_*` a page refuses with, +// registered here (not in `errors.ts`, which sits at its ceiling). The four `X_CDP_*` its raw-CDP +// browser refuses with are `@ultimat3/scraping/cdp`'s since 31.0.0 — listed here, registered there. import { registerErrorCodes } from '@ultimat3/core'; +import { CDP_ERROR_CODES, CDP_ERROR_TITLES } from '@ultimat3/scraping/cdp'; -export const E2E_ERROR_CODES = [ +const OWN_E2E_CODES = [ 'X_E2E_EVALUATE_UNSUPPORTED', 'X_E2E_EVALUATE_CAPTURED', 'X_E2E_EVALUATE_THREW', @@ -11,17 +12,16 @@ export const E2E_ERROR_CODES = [ 'X_E2E_LOCATOR_AMBIGUOUS', 'X_E2E_SERVICE_WORKER_ABSENT', 'X_E2E_APP_FAILED', - // Four and not one, because the four repairs differ: install a browser, read the browser's own - // stderr, look at the page, raise a deadline. - 'X_CDP_BROWSER_MISSING', - 'X_CDP_LAUNCH_FAILED', - 'X_CDP_CALL_FAILED', - 'X_CDP_TIMEOUT', ] as const; +type OwnE2eCode = (typeof OWN_E2E_CODES)[number]; + +/** Every code an e2e run can refuse with: this package's seven, then the browser's four. */ +export const E2E_ERROR_CODES = [...OWN_E2E_CODES, ...CDP_ERROR_CODES] as const; + export type E2eErrorCode = (typeof E2E_ERROR_CODES)[number]; -export const E2E_ERROR_TITLES = Object.freeze>({ +const OWN_E2E_TITLES = Object.freeze>({ X_E2E_EVALUATE_UNSUPPORTED: 'a page.evaluate() closure cannot be sent into the browser', X_E2E_EVALUATE_CAPTURED: 'a page.evaluate() closure named a binding the page does not have', X_E2E_EVALUATE_THREW: 'an expression an e2e page ran threw inside the browser', @@ -29,14 +29,15 @@ export const E2E_ERROR_TITLES = Object.freeze>({ X_E2E_LOCATOR_AMBIGUOUS: 'an e2e locator matched more than one element and was asked to click', X_E2E_SERVICE_WORKER_ABSENT: 'no service worker took control of the page within the budget', X_E2E_APP_FAILED: 'the app an e2e run spawned did not come up', - X_CDP_BROWSER_MISSING: 'no Chrome or Chromium is installed for the e2e driver to launch', - X_CDP_LAUNCH_FAILED: 'the browser started and never announced a DevTools endpoint', - X_CDP_CALL_FAILED: 'the browser refused a DevTools call', - X_CDP_TIMEOUT: 'a DevTools call did not answer inside its deadline', +}); + +export const E2E_ERROR_TITLES = Object.freeze>({ + ...OWN_E2E_TITLES, + ...CDP_ERROR_TITLES, }); // Owned here and borrowed by nobody, so unconditional: a second package claiming one must fail as // X_ERROR_CODE_DUPLICATE rather than quietly keep whichever title registered first. registerErrorCodes( - Object.fromEntries(Object.entries(E2E_ERROR_TITLES).map(([code, title]) => [code, { title }])), + Object.fromEntries(Object.entries(OWN_E2E_TITLES).map(([code, title]) => [code, { title }])), ); diff --git a/packages/testing/src/e2e-run.test.ts b/packages/testing/src/e2e-run.test.ts index d6eff2d0a..0419c9325 100644 --- a/packages/testing/src/e2e-run.test.ts +++ b/packages/testing/src/e2e-run.test.ts @@ -2,8 +2,9 @@ // is asserted is the run's own contract — the page it installs follows the CURRENT browser, a // deploy or a hung browser is relaunched before the next test, and everything is released after. import { afterEach, describe, expect, test } from 'bun:test'; +import { LAUNCHED_CLOSE_MS } from '@ultimat3/scraping/cdp'; import type { E2eBrowser } from './cdp-browser'; -import { E2E_BROWSER_CLOSE_MS, E2E_BROWSER_OPEN_MS } from './cdp-browser'; +import { E2E_BROWSER_OPEN_MS } from './cdp-browser'; import type { E2eApp } from './e2e-app'; import { e2eApp, e2eBaseUrl, e2eBrowser } from './e2e-browser-handle'; import type { E2eDriverOptions } from './e2e-driver'; @@ -113,10 +114,8 @@ describe('unit · one e2e run', () => { const run = harness([fakeBrowser('first')]); await startE2eRun(run.deps); - expect(run.hooks.beforeMs).toBeGreaterThanOrEqual( - 20 + E2E_BROWSER_CLOSE_MS + E2E_BROWSER_OPEN_MS, - ); - expect(run.hooks.afterMs).toBeGreaterThanOrEqual(E2E_BROWSER_CLOSE_MS + E2E_APP_STOP_MS); + expect(run.hooks.beforeMs).toBeGreaterThanOrEqual(20 + LAUNCHED_CLOSE_MS + E2E_BROWSER_OPEN_MS); + expect(run.hooks.afterMs).toBeGreaterThanOrEqual(LAUNCHED_CLOSE_MS + E2E_APP_STOP_MS); }); test('publishes the app and the browser, and installs a page over the app’s origin', async () => { diff --git a/packages/testing/src/e2e-run.ts b/packages/testing/src/e2e-run.ts index d664d7bb5..e03cdbb19 100644 --- a/packages/testing/src/e2e-run.ts +++ b/packages/testing/src/e2e-run.ts @@ -4,8 +4,9 @@ // hooks and the real app and browser; a test is this with doubles, which is how it is measured. import { finiteCount } from '@ultimat3/core'; +import { LAUNCHED_CLOSE_MS } from '@ultimat3/scraping/cdp'; import type { E2eBrowser } from './cdp-browser'; -import { E2E_BROWSER_CLOSE_MS, E2E_BROWSER_OPEN_MS } from './cdp-browser'; +import { E2E_BROWSER_OPEN_MS } from './cdp-browser'; import type { E2eApp } from './e2e-app'; import { E2E_APP_STOP_MS } from './e2e-app'; import { e2eBrowser, publishE2eRun, republishE2eBrowser } from './e2e-browser-handle'; @@ -91,7 +92,7 @@ export async function startE2eRun(deps: E2eRunDeps): Promise { browser = await deps.openBrowser(); republishE2eBrowser(browser); }, - probeMs + E2E_BROWSER_CLOSE_MS + E2E_BROWSER_OPEN_MS, + probeMs + LAUNCHED_CLOSE_MS + E2E_BROWSER_OPEN_MS, ); deps.afterAll(async () => { @@ -99,5 +100,5 @@ export async function startE2eRun(deps: E2eRunDeps): Promise { uninstall(); await browser.close(); await app.stop(); - }, E2E_BROWSER_CLOSE_MS + E2E_APP_STOP_MS); + }, LAUNCHED_CLOSE_MS + E2E_APP_STOP_MS); } diff --git a/packages/testing/src/e2e-teardown.test.ts b/packages/testing/src/e2e-teardown.test.ts index a0811c1be..8856b0964 100644 --- a/packages/testing/src/e2e-teardown.test.ts +++ b/packages/testing/src/e2e-teardown.test.ts @@ -2,7 +2,7 @@ // handshake that timed out left its socket dialling. import { afterEach, describe, expect, test } from 'bun:test'; -import { cdpConnect } from './cdp-connection'; +import { cdpConnect } from '@ultimat3/scraping/cdp'; import { openOrStop } from './e2e-run'; const RealWebSocket = globalThis.WebSocket; diff --git a/packages/testing/src/embedder-conformance.test.ts b/packages/testing/src/embedder-conformance.test.ts new file mode 100644 index 000000000..53170c03c --- /dev/null +++ b/packages/testing/src/embedder-conformance.test.ts @@ -0,0 +1,51 @@ +// The embedder suite over `@ultimat3/ai`'s hash embedder, and the proof it can fail: an embedder +// that answers its batch in reverse, and one that hands an empty text no direction at all. + +import { describe, expect, test } from 'bun:test'; +import { hashEmbedder } from '@ultimat3/ai'; +import type { Embedder } from '@ultimat3/ai/adapter'; +import { EMBEDDER_CHECKS, embedderConformance } from './embedder-conformance'; +import { behavesLike } from './shared-examples'; +import { testName } from './test-types'; + +describe(testName('unit', 'the hash embedder'), () => { + behavesLike(embedderConformance, () => hashEmbedder()); +}); + +const real = hashEmbedder({ dimension: 16 }); + +/** Every vector right, the batch answered backwards: the cache would key one prompt on another. */ +const reversing: Embedder = { + name: 'reversing', + dimension: real.dimension, + embed: async (texts) => [...(await real.embed(texts))].reverse(), +}; + +/** One zero vector per text: every cosine against it is NaN, and nothing ever ranks. */ +const directionless: Embedder = { + name: 'directionless', + dimension: 4, + embed: async (texts) => texts.map(() => new Float32Array(4)), +}; + +const check = (prefix: string) => { + const found = EMBEDDER_CHECKS.find((each) => each.name.startsWith(prefix)); + if (found === undefined) return expect.unreachable(`no embedder check named ${prefix}`); + return found; +}; + +describe(testName('unit', 'an embedder that breaks the contract fails conformance'), () => { + test('a reversed batch fails the order check, and only the checks that read order', async () => { + await expect(check('the batch keeps the order').run(reversing)).rejects.toThrow(); + await check('one vector per text').run(reversing); + }); + + test('a zero vector fails the direction check', async () => { + await expect(check('a text has a direction').run(directionless)).rejects.toThrow(); + }); + + test('every check has a name of its own', () => { + const names = EMBEDDER_CHECKS.map((each) => each.name); + expect(new Set(names).size).toBe(names.length); + }); +}); diff --git a/packages/testing/src/embedder-conformance.ts b/packages/testing/src/embedder-conformance.ts new file mode 100644 index 000000000..a68026ce1 --- /dev/null +++ b/packages/testing/src/embedder-conformance.ts @@ -0,0 +1,101 @@ +// The `embedder` port's contract as shared examples: `behavesLike(embedderConformance, () => yours)`. +// What `@ultimat3/ai`'s semantic cache and vector store assume of an `Embedder` and never re-check +// per call: one vector per text, in order, of the declared dimension, finite, with a direction. + +import { expect, test } from 'bun:test'; +import type { Embedder } from '@ultimat3/ai/adapter'; +import type { SharedExamples } from './shared-examples'; +import { sharedExamples } from './shared-examples'; + +export interface EmbedderCheck { + readonly name: string; + run(embedder: Embedder): Promise; +} + +/** Near-equal, not equal: a remote model may differ in the last bits between two calls. */ +const SAME_DIRECTION = 0.999; + +const TEXTS = ['list my orders', 'the invoice for March', 'reset my password'] as const; + +function cosine(a: Float32Array, b: Float32Array): number { + let dot = 0; + let normA = 0; + let normB = 0; + for (let i = 0; i < a.length; i += 1) { + const x = a[i] ?? 0; + const y = b[i] ?? 0; + dot += x * y; + normA += x * x; + normB += y * y; + } + return dot / (Math.sqrt(normA) * Math.sqrt(normB)); +} + +export const EMBEDDER_CHECKS: readonly EmbedderCheck[] = [ + { + name: 'it names itself and declares a whole, positive dimension', + async run(embedder) { + expect(embedder.name.length).toBeGreaterThan(0); + expect(Number.isSafeInteger(embedder.dimension) && embedder.dimension > 0).toBe(true); + }, + }, + { + name: 'one vector per text, of the declared dimension, every component finite', + async run(embedder) { + const vectors = await embedder.embed(TEXTS); + expect(vectors.length).toBe(TEXTS.length); + for (const vector of vectors) { + expect(vector).toBeInstanceOf(Float32Array); + expect(vector.length).toBe(embedder.dimension); + expect(vector.every((component) => Number.isFinite(component))).toBe(true); + } + }, + }, + { + name: 'an empty batch answers no vectors, never one', + async run(embedder) { + expect((await embedder.embed([])).length).toBe(0); + }, + }, + { + name: 'a text has a direction: no all-zero vector, which every cosine answers NaN against', + async run(embedder) { + for (const vector of await embedder.embed(TEXTS)) { + expect(vector.some((component) => component !== 0)).toBe(true); + } + }, + }, + { + name: 'the batch keeps the order the texts arrived in', + async run(embedder) { + // Against each text alone: a one-text batch has no order to get wrong. + const batch = await embedder.embed(TEXTS); + for (let at = 0; at < TEXTS.length; at += 1) { + const [alone] = await embedder.embed([TEXTS[at] ?? '']); + const inBatch = batch[at]; + if (alone === undefined || inBatch === undefined) return expect.unreachable(); + expect(cosine(inBatch, alone)).toBeGreaterThan(SAME_DIRECTION); + } + }, + }, + { + name: 'the same text twice is the same direction: a cache keyed on it can hit', + async run(embedder) { + const [first] = await embedder.embed([TEXTS[0]]); + const [again] = await embedder.embed([TEXTS[0]]); + if (first === undefined || again === undefined) return expect.unreachable(); + expect(cosine(first, again)).toBeGreaterThan(SAME_DIRECTION); + }, + }, +]; + +export const embedderConformance: SharedExamples = sharedExamples( + 'an embedder', + (subject) => { + for (const check of EMBEDDER_CHECKS) { + test(check.name, async () => { + await check.run(subject()); + }); + } + }, +); diff --git a/packages/testing/src/identity-provider-conformance.test.ts b/packages/testing/src/identity-provider-conformance.test.ts new file mode 100644 index 000000000..b33690f9c --- /dev/null +++ b/packages/testing/src/identity-provider-conformance.test.ts @@ -0,0 +1,141 @@ +// The IdentityProvider conformance suite over both adapters `@ultimat3/auth` ships — SAML against +// its in-process IdP, OIDC against a stubbed GitHub — and the proof it can fail: a provider that +// accepts any answer, and one that never spends an answer. + +import { describe, expect, test } from 'bun:test'; +import { type IdentityProvider, oidcIdentityProvider, samlIdentityProvider } from '@ultimat3/auth'; +import { frozenClock } from '@ultimat3/core'; +import { encodeResponse, readAuthnRequest, samlTestIdp } from '../../auth/src/saml-idp-fixture'; +import { + IDENTITY_PROVIDER_CHECKS, + type IdentityProviderHarness, + identityProviderConformance, +} from './identity-provider-conformance'; +import { behavesLike } from './shared-examples'; +import { testName } from './test-types'; + +const NOW = new Date('2026-11-09T12:00:00.000Z'); +const SP = 'https://app.example.test/saml/metadata'; +const ACS = 'https://app.example.test/auth/sso/corp/callback'; + +async function samlHarness(): Promise { + const idp = await samlTestIdp(); + return { + provider: samlIdentityProvider({ + id: 'corp', + idp: idp.idp, + spEntityId: SP, + trustEmail: true, + emailDomains: ['corp.example'], + }), + callbackUrl: ACS, + clock: frozenClock(NOW), + async answer(redirect) { + const request = readAuthnRequest(redirect.location); + const xml = await idp.responseXml({ + requestId: request.id, + acsUrl: ACS, + audience: SP, + now: NOW, + }); + return { SAMLResponse: encodeResponse(xml), RelayState: request.relayState }; + }, + tamper(params) { + const xml = new TextDecoder().decode( + Uint8Array.from(atob(params['SAMLResponse'] ?? ''), (c) => c.charCodeAt(0)), + ); + return { ...params, SAMLResponse: encodeResponse(xml.replace('>u-1001<', '>root<')) }; + }, + }; +} + +const json = (body: unknown): Response => + new Response(JSON.stringify(body), { headers: { 'content-type': 'application/json' } }); + +function oidcHarness(): IdentityProviderHarness { + const callbackUrl = 'https://app.example.test/auth/sso/github/callback'; + return { + provider: oidcIdentityProvider({ + provider: 'github', + credentials: { clientId: 'client-id', clientSecret: 'client-secret' }, + fetch: async (url) => { + if (url.endsWith('/access_token')) + return json({ access_token: 'gho_x', token_type: 'bearer' }); + if (url.endsWith('/user')) return json({ id: 583231, login: 'octocat' }); + return json([{ email: 'octo@example.com', primary: true, verified: true }]); + }, + }), + callbackUrl, + clock: frozenClock(NOW), + async answer(redirect) { + return { state: new URL(redirect.location).searchParams.get('state') ?? '', code: 'c0de' }; + }, + tamper: (params) => ({ ...params, state: 'forged-state' }), + }; +} + +describe(testName('unit', 'the SAML identity provider'), () => { + behavesLike(identityProviderConformance, samlHarness); +}); + +describe(testName('unit', 'the OIDC identity provider'), () => { + behavesLike(identityProviderConformance, oidcHarness); +}); + +/** Believes whatever comes back, under whatever state. */ +const credulous = (): IdentityProvider => ({ + id: 'credulous', + protocol: 'oidc', + callbackMethod: 'GET', + begin: async () => ({ + location: 'https://idp.example.test/authorize', + state: crypto.randomUUID(), + }), + answers: () => true, + complete: async ({ params }) => ({ + profile: { + provider: 'credulous', + providerAccountId: params['sub'] ?? 'anyone', + email: null, + emailVerified: false, + name: null, + }, + expiresAt: null, + claims: {}, + }), +}); + +const brokenHarness = (provider: IdentityProvider): IdentityProviderHarness => ({ + provider, + callbackUrl: 'https://app.example.test/auth/sso/credulous/callback', + clock: frozenClock(NOW), + answer: async () => ({ sub: 'u-1' }), + tamper: (params) => ({ ...params, sub: 'root' }), +}); + +const check = (name: string) => + IDENTITY_PROVIDER_CHECKS.find((each) => each.name.includes(name)) ?? + expect.unreachable(`no conformance check named ${name}`); + +describe(testName('unit', 'a provider that breaks the contract fails conformance'), () => { + test('one that accepts a tampered answer', async () => { + await expect(check('tampered').run(brokenHarness(credulous()))).rejects.toThrow(); + }); + + test('one that completes the same answer twice', async () => { + await expect(check('completes once').run(brokenHarness(credulous()))).rejects.toThrow(); + }); + + test('one that ignores whose state it was handed', async () => { + await expect(check('another begin').run(brokenHarness(credulous()))).rejects.toThrow(); + }); + + test('one that calls any answer its own', async () => { + await expect(check('answers()').run(brokenHarness(credulous()))).rejects.toThrow(); + }); + + test('every check has a name of its own', () => { + const names = IDENTITY_PROVIDER_CHECKS.map((each) => each.name); + expect(new Set(names).size).toBe(names.length); + }); +}); diff --git a/packages/testing/src/identity-provider-conformance.ts b/packages/testing/src/identity-provider-conformance.ts new file mode 100644 index 000000000..8d7070575 --- /dev/null +++ b/packages/testing/src/identity-provider-conformance.ts @@ -0,0 +1,144 @@ +// The `IdentityProvider` contract of `@ultimat3/auth` as shared examples: +// `behavesLike(identityProviderConformance, () => harness)`. The harness plays the IdP (answers a +// redirect, tampers with an answer); every adapter must issue a fresh state per begin, prove one +// profile from an honest answer, and refuse a replayed, tampered or cross-wired one with a code. + +import { expect, test } from 'bun:test'; +import type { IdentityProvider, IdentityRedirect, VerifiedIdentity } from '@ultimat3/auth'; +import { memoryTotpReplayGuard, type TotpReplayGuard } from '@ultimat3/auth'; +import { type Clock, isUltimateError } from '@ultimat3/core'; +import type { SharedExamples } from './shared-examples'; +import { sharedExamples } from './shared-examples'; + +export interface IdentityProviderHarness { + readonly provider: IdentityProvider; + /** The URL `begin` and `complete` are given: what the IdP is registered to answer. */ + readonly callbackUrl: string; + readonly clock: Clock; + /** Play the IdP: the callback params a consenting user's browser would bring back. */ + answer(redirect: IdentityRedirect): Promise>>; + /** The same answer with its proof altered — a changed signed value, a forged state. */ + tamper(params: Readonly>): Readonly>; +} + +export interface IdentityProviderCheck { + readonly name: string; + run(harness: IdentityProviderHarness): Promise; +} + +const complete = ( + harness: IdentityProviderHarness, + state: string, + params: Readonly>, + replay: TotpReplayGuard, +): Promise => + harness.provider.complete({ + state, + params, + callbackUrl: harness.callbackUrl, + clock: harness.clock, + replay, + }); + +/** A refusal must be coded: an `X_*` an operator can look up, never a bare throw. */ +async function expectCodedRefusal(call: Promise): Promise { + const outcome = await call.then( + () => 'accepted', + (error: unknown) => + isUltimateError(error) && /^X_[A-Z0-9_]+$/.test(error.code) ? 'coded' : 'uncoded', + ); + expect(outcome).toBe('coded'); +} + +const begin = (harness: IdentityProviderHarness): Promise => + harness.provider.begin({ callbackUrl: harness.callbackUrl, clock: harness.clock }); + +export const IDENTITY_PROVIDER_CHECKS: readonly IdentityProviderCheck[] = [ + { + name: 'declares a cookie-safe id and a callback method matching its protocol', + async run({ provider }) { + expect(provider.id).toMatch(/^[A-Za-z0-9_-]+$/); + expect(provider.callbackMethod).toBe(provider.protocol === 'saml' ? 'POST' : 'GET'); + }, + }, + { + name: 'begin redirects to an absolute URL with a fresh state every time', + async run(harness) { + const [a, b] = [await begin(harness), await begin(harness)]; + expect(URL.canParse(a.location)).toBe(true); + expect(a.state).not.toBe(''); + expect(a.state).not.toBe(b.state); + }, + }, + { + name: 'an honest answer completes to one profile under the provider id', + async run(harness) { + const redirect = await begin(harness); + const identity = await complete( + harness, + redirect.state, + await harness.answer(redirect), + memoryTotpReplayGuard(), + ); + expect(identity.profile.provider).toBe(harness.provider.id); + expect(identity.profile.providerAccountId).not.toBe(''); + expect(identity.profile.emailVerified && identity.profile.email === null).toBe(false); + }, + }, + { + name: 'the same answer completes once; the second is a coded refusal', + async run(harness) { + const replay = memoryTotpReplayGuard(); + const redirect = await begin(harness); + const params = await harness.answer(redirect); + await complete(harness, redirect.state, params, replay); + await expectCodedRefusal(complete(harness, redirect.state, params, replay)); + }, + }, + { + name: 'a tampered answer is a coded refusal', + async run(harness) { + const redirect = await begin(harness); + const params = harness.tamper(await harness.answer(redirect)); + await expectCodedRefusal(complete(harness, redirect.state, params, memoryTotpReplayGuard())); + }, + }, + { + name: "an answer to one begin, completed with another begin's state, is a coded refusal", + async run(harness) { + const [first, second] = [await begin(harness), await begin(harness)]; + const params = await harness.answer(first); + await expectCodedRefusal(complete(harness, second.state, params, memoryTotpReplayGuard())); + }, + }, + { + name: 'answers() binds an answer to the state it was begun with, and to no other', + async run(harness) { + const [first, second] = [await begin(harness), await begin(harness)]; + const params = await harness.answer(first); + expect(harness.provider.answers(first.state, params)).toBe(true); + expect(harness.provider.answers(second.state, params)).toBe(false); + expect(harness.provider.answers('{"forged":true}', params)).toBe(false); + }, + }, + { + name: 'a state the provider never issued is a coded refusal', + async run(harness) { + const redirect = await begin(harness); + const params = await harness.answer(redirect); + await expectCodedRefusal( + complete(harness, '{"forged":true}', params, memoryTotpReplayGuard()), + ); + }, + }, +]; + +export const identityProviderConformance: SharedExamples< + IdentityProviderHarness | Promise +> = sharedExamples('an identity provider', (subject) => { + for (const check of IDENTITY_PROVIDER_CHECKS) { + test(check.name, async () => { + await check.run(await subject()); + }); + } +}); diff --git a/packages/testing/src/index.ts b/packages/testing/src/index.ts index f388fbb8f..b4039d7ca 100644 --- a/packages/testing/src/index.ts +++ b/packages/testing/src/index.ts @@ -47,11 +47,11 @@ export type { CassetteEntry, CassetteFile } from './cassette-format'; // `@ultimat3/cli` in 22.0.0. `installE2eDriver` is the ONE entry point an app's test preload calls; // `openE2eBrowserIfAvailable()` answers `undefined` on a machine with no Chrome, so a // browser-backed suite SKIPS rather than turning a gate red. RAW CDP over Bun's own `WebSocket`: -// no dependency, and no `@ultimat3/scraping` / `puppeteer-core` behind it. +// no dependency. The launcher, the connection, their errors and `LAUNCHED_CLOSE_MS` are imported +// from `@ultimat3/scraping/cdp` since 31.0.0 — their one home, which `pdf()` prints through. export type { E2eBrowser, OpenE2eBrowserOptions } from './cdp-browser'; export { DEFAULT_CDP_TIMEOUT_MS, - E2E_BROWSER_CLOSE_MS, E2E_BROWSER_OPEN_MS, E2E_GOTO_MS, E2E_TAB_OPEN_MS, @@ -60,42 +60,10 @@ export { } from './cdp-browser'; export type { E2eBrowserLease, RunEndHook } from './cdp-browser-lease'; export { closeE2eBrowsersAtRunEnd, leaseE2eBrowser } from './cdp-browser-lease'; -export type { - CdpConnection, - CdpConnectionOptions, - CdpEventListener, - CdpResult, - CdpTransport, -} from './cdp-connection'; -export { cdpConnect, cdpConnectOver } from './cdp-connection'; export type { CdpE2eTabOptions, E2eTab } from './cdp-e2e-page'; export { cdpE2eTab } from './cdp-e2e-page'; export type { CdpE2eSessionOptions, E2eSession } from './cdp-e2e-session'; export { cdpE2eSession } from './cdp-e2e-session'; -export type { - CdpLaunchAttempt, - CdpTargetGone, - CdpTimeoutObservation, - CdpTimeoutReading, -} from './cdp-errors'; -export { - CdpBrowserMissingError, - CdpCallFailedError, - CdpLaunchFailedError, - CdpTimeoutError, -} from './cdp-errors'; -export type { LaunchedBrowser, LaunchOptions } from './cdp-launch'; -export { - CHROME_CANDIDATES, - CHROME_PATH_ENV, - CONTAINER_CHROME_ARGS, - findChrome, - LAUNCH_ATTEMPTS, - LAUNCH_BUDGET_MS, - LAUNCH_TIMEOUT_MS, - launchChrome, - launchFoundChrome, -} from './cdp-launch'; // The exercise pass (plan 101 / 40, phase 2): an app's per-primitive overrides — 39's // `.contract({ exercise })` option, one shape — and the pass over every action, query and job. export type { ExerciseAppOptions, ExerciseOverride, ExerciseOverrides } from './define-exercise'; diff --git a/packages/testing/src/lease-store-conformance.test.ts b/packages/testing/src/lease-store-conformance.test.ts index 81eb1a5b1..c02c49bae 100644 --- a/packages/testing/src/lease-store-conformance.test.ts +++ b/packages/testing/src/lease-store-conformance.test.ts @@ -1,6 +1,7 @@ // The lease-store suite over `memoryLeaseStore()` on a frozen clock, and the proof it can fail: a // store that reads the count, awaits, then grants (over-grants under a race), and one whose renew -// revives a lapsed slot (the defect `SQL_LEASE_RENEW`'s expiry fence closed in Postgres). +// revives a lapsed slot (the defect `SQL_LEASE_RENEW`'s expiry fence closed in Postgres), and one +// that numbers its fences per slot, so a later holder shows an earlier holder's token. import { describe, expect, test } from 'bun:test'; import { frozenClock } from '@ultimat3/core'; @@ -42,6 +43,15 @@ const reviving = (inner: LeaseStore): LeaseStore => ({ }, }); +/** A fence per SLOT, not per grant: the next holder of a slot shows the token its last one had. */ +const slotFenced = (inner: LeaseStore): LeaseStore => ({ + ...inner, + async acquire(key, limit, ttlMs, holder) { + const lease = await inner.acquire(key, limit, ttlMs, holder); + return lease === undefined ? undefined : { ...lease, fence: lease.slot + 1 }; + }, +}); + const check = (prefix: string) => LEASE_STORE_CHECKS.find((each) => each.name.startsWith(prefix)) ?? expect.unreachable(`no lease-store check named ${prefix}`); @@ -57,6 +67,13 @@ describe(testName('unit', 'a store that breaks the lease contract fails conforma await expect(check('a lapsed slot').run(onFrozenClock(reviving))).rejects.toThrow(); }); + test('a fence reused by a later grant of the slot', async () => { + await check('slots are granted').run(onFrozenClock(slotFenced)); + await expect( + check('every grant carries a fence').run(onFrozenClock(slotFenced)), + ).rejects.toThrow(); + }); + test('every check has a name of its own', () => { const names = LEASE_STORE_CHECKS.map((each) => each.name); expect(new Set(names).size).toBe(names.length); diff --git a/packages/testing/src/lease-store-conformance.ts b/packages/testing/src/lease-store-conformance.ts index ee16ec6a0..11e1afdee 100644 --- a/packages/testing/src/lease-store-conformance.ts +++ b/packages/testing/src/lease-store-conformance.ts @@ -139,6 +139,36 @@ export const LEASE_STORE_CHECKS: readonly LeaseStoreCheck[] = [ granted(await store.acquire(key, 1, LONG_TTL_MS, 'b'), 'after release'); }, }, + { + // `withLock`'s token: a write fenced on it refuses a smaller one, so a later holder must + // never show an earlier holder's number — not after a release, not after a takeover. + name: 'every grant carries a fence greater than every earlier one, takeovers included', + async run(subject) { + const { store } = subject; + const key = keyOf(); + const fences: number[] = []; + const fenceOf = (lease: HeldLease | undefined, what: string): number => { + const { fence } = granted(lease, what); + if (typeof fence !== 'number' || !Number.isSafeInteger(fence)) { + return expect.unreachable(`${what}: the grant carries no whole-number fence`); + } + fences.push(fence); + return fence; + }; + fenceOf(await store.acquire(key, 2, LONG_TTL_MS, 'a'), 'slot 0'); + const second = granted(await store.acquire(key, 2, SHORT_TTL_MS, 'b'), 'slot 1'); + fenceOf(second, 'slot 1'); + // The same slot, released and granted again. + await store.release(second); + fenceOf(await store.acquire(key, 2, SHORT_TTL_MS, 'c'), 'slot 1 again'); + // The same slot, lapsed and TAKEN OVER without a release. + await elapse(subject, PAST_SHORT_TTL_MS); + fenceOf(await store.acquire(key, 2, LONG_TTL_MS, 'd'), 'slot 1 taken over'); + for (const [index, fence] of fences.entries()) { + if (index > 0) expect(fence).toBeGreaterThan(fences[index - 1] ?? Number.NaN); + } + }, + }, { name: 'holders names the live slots of one key in slot order, and nothing expired', async run(subject) { diff --git a/packages/testing/src/search-index-conformance-postgres.live.test.ts b/packages/testing/src/search-index-conformance-postgres.live.test.ts new file mode 100644 index 000000000..4d49b5b6b --- /dev/null +++ b/packages/testing/src/search-index-conformance-postgres.live.test.ts @@ -0,0 +1,88 @@ +// The search-index suite over the built-in `postgresSearchIndex` and a real Postgres +// (`TEST_DATABASE_URL`): a table the framework's own migration created, its generated `tsvector` +// kept by the server on every `store`, every query the repository's own `matches` read. + +import { afterAll, beforeAll, describe } from 'bun:test'; +import type { PostgresClient } from '@ultimat3/db'; +import { generateMigration, postgresClient, raw, sql, statementsOf } from '@ultimat3/db'; +import { + clearRegistry, + entity, + postgresSearchIndex, + text, + timestamp, + uuid, +} from '@ultimat3/entity'; +import { searchIndexConformance } from './search-index-conformance'; +import type { SearchIndexSubject } from './search-index-conformance-support'; +import { behavesLike } from './shared-examples'; +import type { WorkerDatabase } from './template-db'; +import { acquireWorkerDatabase } from './template-db'; +import { testName } from './test-types'; + +const adminUrl = Bun.env['TEST_DATABASE_URL'] ?? ''; + +afterAll(() => { + clearRegistry(); +}); + +const posts = entity('search_conformance_posts', { + columns: { + id: uuid().primaryKey(), + orgId: uuid().tenant(), + title: text().searchable('A'), + body: text().nullable().searchable(), + updatedAt: timestamp().defaultNow().onUpdateNow(), + }, +}); + +describe.skipIf(adminUrl === '')(testName('live', 'the built-in postgres search index'), () => { + let db: WorkerDatabase | undefined; + let client: PostgresClient | undefined; + + beforeAll(async () => { + db = await acquireWorkerDatabase({ adminUrl, templateName: 'ultimate_search_conformance' }); + const pool = postgresClient({ url: db.url, profile: { max: 2 } }); + client = pool; + const migration = generateMigration({ + entities: [posts.$describe()], + name: 'search conformance', + now: new Date('2026-11-09T00:00:00.000Z'), + }); + await pool.execute(raw(`drop table if exists "${posts.$table}" cascade`)); + for (const statement of statementsOf(migration.up)) await pool.execute(raw(statement)); + }); + + afterAll(async () => { + const failures: unknown[] = []; + await Promise.resolve(client?.close()).catch((error: unknown) => failures.push(error)); + await Promise.resolve(db?.drop()).catch((error: unknown) => failures.push(error)); + if (failures.length > 0) throw failures[0]; + }); + + const subject = async (): Promise => { + const pool = client as PostgresClient; + await pool.execute(raw(`truncate "${posts.$table}"`)); + return { + index: postgresSearchIndex({ client: pool }), + entity: posts.$name, + store: async (documents) => { + for (const each of documents) { + await pool.execute(sql` + insert into search_conformance_posts (id, org_id, title, body) + values (${each.id}, ${each.tenant}, ${each.fields['title'] ?? ''}, ${each.fields['body'] ?? null}) + on conflict (id) do update set title = excluded.title, body = excluded.body`); + } + }, + drop: async (scope, ids) => { + for (const id of ids) { + await pool.execute( + sql`delete from search_conformance_posts where id = ${id} and org_id = ${scope.tenant}`, + ); + } + }, + }; + }; + + behavesLike(searchIndexConformance, subject); +}); diff --git a/packages/testing/src/search-index-conformance-support.ts b/packages/testing/src/search-index-conformance-support.ts new file mode 100644 index 000000000..49781992a --- /dev/null +++ b/packages/testing/src/search-index-conformance-support.ts @@ -0,0 +1,64 @@ +// What every search-index check shares: the subject, the suite's ids and tenants, one document +// builder (version 1 unless a check says otherwise), stored-then-indexed, and the sorted hit ids. + +import { finiteOption } from '@ultimat3/core'; +import type { SearchDocument, SearchIndex, SearchScope } from '@ultimat3/entity/adapter'; + +export interface SearchIndexSubject { + readonly index: SearchIndex; + /** A searchable entity with `title` and `body` searchable columns and a tenant column. */ + readonly entity: string; + /** Writes (upserts) each document's source row; a no-op for an index holding its own copy. */ + store(documents: readonly SearchDocument[]): Promise; + /** Deletes the source rows of `ids` under `scope`; a no-op for an index holding its own copy. */ + drop(scope: SearchScope, ids: readonly string[]): Promise; +} + +export interface SearchIndexCheck { + readonly name: string; + run(subject: SearchIndexSubject): Promise; +} + +export const SEARCH_TENANT_A = '00000000-0000-7000-8000-00000000a0a0'; +export const SEARCH_TENANT_B = '00000000-0000-7000-8000-00000000b0b0'; + +/** A uuid-shaped id, so a uuid-keyed table holds the suite's rows. */ +export const searchDocId = (n: number): string => + `00000000-0000-7000-8000-${String(n).padStart(12, '0')}`; + +export interface SearchDocOptions { + readonly body?: string; + readonly tenant?: string; + /** The row's version (its `onUpdateNow()` instant, epoch ms). Default 1. */ + readonly version?: number; +} + +/** One suite document: tenant A, version 1, unless the check says otherwise. */ +export const searchDoc = ( + entity: string, + n: number, + title: string, + options: SearchDocOptions = {}, +): SearchDocument => ({ + entity, + tenant: options.tenant ?? SEARCH_TENANT_A, + id: searchDocId(n), + version: finiteOption('searchDoc', 'version', options.version ?? 1), + fields: options.body === undefined ? { title } : { title, body: options.body }, +}); + +/** Stored then indexed — the order a write hook and `reindex` both keep. */ +export const putSearchDocs = async ( + subject: SearchIndexSubject, + documents: readonly SearchDocument[], +) => { + await subject.store(documents); + await subject.index.index(documents); +}; + +export const searchIds = async ( + subject: SearchIndexSubject, + term: string, + tenant = SEARCH_TENANT_A, +) => + (await subject.index.query({ entity: subject.entity, tenant, term })).map((hit) => hit.id).sort(); diff --git a/packages/testing/src/search-index-conformance-versions.ts b/packages/testing/src/search-index-conformance-versions.ts new file mode 100644 index 000000000..873ca3b60 --- /dev/null +++ b/packages/testing/src/search-index-conformance-versions.ts @@ -0,0 +1,49 @@ +// The search index's ordering contract: keep only NEWER. A `reindex` batch that read a row at v1 +// may call `index` after the write path indexed v2 or removed the row; the index must answer v2, or +// nothing. Each stale call here is `index` ALONE — the source row already holds what won, exactly +// as when a batch's read raced a write — so the built-in Postgres index passes by construction. + +import { expect } from 'bun:test'; +import type { SearchScope } from '@ultimat3/entity/adapter'; +import type { SearchIndexCheck } from './search-index-conformance-support'; +import { + searchDoc as doc, + putSearchDocs, + SEARCH_TENANT_A, + searchDocId, + searchIds, +} from './search-index-conformance-support'; + +export const SEARCH_INDEX_VERSION_CHECKS: readonly SearchIndexCheck[] = [ + { + name: 'an older version indexed after a newer one leaves the newer one', + async run(subject) { + await putSearchDocs(subject, [doc(subject.entity, 1, 'lantern', { version: 20 })]); + await subject.index.index([doc(subject.entity, 1, 'zebra', { version: 10 })]); + expect(await searchIds(subject, 'zebra')).toEqual([]); + expect(await searchIds(subject, 'lantern')).toEqual([searchDocId(1)]); + }, + }, + { + name: 'a version indexed after the id was removed at a later version stays removed', + async run(subject) { + await putSearchDocs(subject, [doc(subject.entity, 1, 'zebra', { version: 10 })]); + const own: SearchScope = { entity: subject.entity, tenant: SEARCH_TENANT_A }; + await subject.drop(own, [searchDocId(1)]); + await subject.index.remove(own, [searchDocId(1)], 30); + await subject.index.index([doc(subject.entity, 1, 'zebra', { version: 20 })]); + expect(await searchIds(subject, 'zebra')).toEqual([]); + }, + }, + { + name: 'an id written again after its removal, at a newer version, is found', + async run(subject) { + await putSearchDocs(subject, [doc(subject.entity, 1, 'zebra', { version: 10 })]); + const own: SearchScope = { entity: subject.entity, tenant: SEARCH_TENANT_A }; + await subject.drop(own, [searchDocId(1)]); + await subject.index.remove(own, [searchDocId(1)], 30); + await putSearchDocs(subject, [doc(subject.entity, 1, 'quokka', { version: 40 })]); + expect(await searchIds(subject, 'quokka')).toEqual([searchDocId(1)]); + }, + }, +]; diff --git a/packages/testing/src/search-index-conformance.test.ts b/packages/testing/src/search-index-conformance.test.ts new file mode 100644 index 000000000..e23946428 --- /dev/null +++ b/packages/testing/src/search-index-conformance.test.ts @@ -0,0 +1,112 @@ +// The search-index suite over `@ultimat3/entity`'s memory index, and the proof it can fail: an +// index that drops the tenant from its key, and one that appends instead of replacing. + +import { describe, expect, test } from 'bun:test'; +import { memorySearchIndex } from '@ultimat3/entity'; +import type { SearchIndex } from '@ultimat3/entity/adapter'; +import { SEARCH_INDEX_ALL_CHECKS, searchIndexConformance } from './search-index-conformance'; +import type { SearchIndexSubject } from './search-index-conformance-support'; +import { searchDoc } from './search-index-conformance-support'; +import { behavesLike } from './shared-examples'; +import { testName } from './test-types'; + +const subjectOver = (index: SearchIndex): SearchIndexSubject => ({ + index, + entity: 'conformance_posts', + store: async () => {}, + drop: async () => {}, +}); + +describe(testName('unit', 'the memory search index'), () => { + behavesLike(searchIndexConformance, () => subjectOver(memorySearchIndex())); +}); + +/** Indexes every document under tenant `null`: one scope for the whole app. */ +const tenantBlind = (): SearchIndex => { + const inner = memorySearchIndex(); + return { + name: 'tenant-blind', + index: (documents) => inner.index(documents.map((each) => ({ ...each, tenant: null }))), + remove: (scope, ids, version) => inner.remove({ ...scope, tenant: null }, ids, version), + query: (query) => inner.query({ ...query, tenant: null }), + }; +}; + +/** Keeps every text an id was ever indexed with: a re-index appends. */ +const appending = (): SearchIndex => { + const inner = memorySearchIndex(); + const seen = new Map>(); + return { + name: 'appending', + index: (documents) => + inner.index( + documents.map((each) => { + const before = seen.get(each.id) ?? {}; + const merged: Record = { ...before }; + for (const [key, value] of Object.entries(each.fields)) { + merged[key] = `${before[key] ?? ''} ${value}`; + } + seen.set(each.id, merged); + return { ...each, fields: merged }; + }), + ), + remove: (scope, ids, version) => inner.remove(scope, ids, version), + query: (query) => inner.query(query), + }; +}; + +/** Stamps every call with its own arrival order: the last call wins, whatever version it carried. */ +const versionBlind = (): SearchIndex => { + const inner = memorySearchIndex(); + let arrival = 0; + return { + name: 'version-blind', + index: (documents) => { + arrival += 1; + const stamp = arrival; + return inner.index(documents.map((each) => ({ ...each, version: stamp }))); + }, + remove: (scope, ids) => { + arrival += 1; + return inner.remove(scope, ids, arrival); + }, + query: (query) => inner.query(query), + }; +}; + +const check = (prefix: string) => { + const found = SEARCH_INDEX_ALL_CHECKS.find((each) => each.name.startsWith(prefix)); + if (found === undefined) return expect.unreachable(`no search-index check named ${prefix}`); + return found; +}; + +describe(testName('unit', 'an index that breaks the contract fails conformance'), () => { + test('a tenant-blind index fails the tenant check, passes the word one', async () => { + await check('an indexed document is found').run(subjectOver(tenantBlind())); + await expect( + check('a tenant is never answered').run(subjectOver(tenantBlind())), + ).rejects.toThrow(); + }); + + test('an appending index fails the replace check', async () => { + await expect(check('indexing a newer version').run(subjectOver(appending()))).rejects.toThrow(); + }); + + test('a version-blind index fails both stale-write checks', async () => { + await expect( + check('an older version indexed').run(subjectOver(versionBlind())), + ).rejects.toThrow(); + await expect( + check('a version indexed after the id was removed').run(subjectOver(versionBlind())), + ).rejects.toThrow(); + }); + + test('a non-finite version is refused where the suite builds its document', () => { + expect(() => searchDoc('e', 1, 'zebra', { version: Number.NaN })).toThrow(/version is NaN/); + }); + + test('every check has a name of its own', () => { + const names = SEARCH_INDEX_ALL_CHECKS.map((each) => each.name); + expect(new Set(names).size).toBe(names.length); + }); +}); diff --git a/packages/testing/src/search-index-conformance.ts b/packages/testing/src/search-index-conformance.ts new file mode 100644 index 000000000..7ba0356a9 --- /dev/null +++ b/packages/testing/src/search-index-conformance.ts @@ -0,0 +1,124 @@ +// The `searchIndex` port's contract as shared examples: `behavesLike(searchIndexConformance, () => +// ({ index, entity, store, drop }))`. `store`/`drop` write the SOURCE rows (the built-in Postgres +// index reads its table; an external one ignores them), then the suite calls `index`/`remove` as +// `reindex` and a write hook do. Words are chosen no English stemmer rewrites, so every index agrees. + +import { expect, test } from 'bun:test'; +import type { SearchScope } from '@ultimat3/entity/adapter'; +import type { SearchIndexCheck, SearchIndexSubject } from './search-index-conformance-support'; +import { + searchDoc as doc, + putSearchDocs, + SEARCH_TENANT_A, + SEARCH_TENANT_B, + searchDocId, + searchIds, +} from './search-index-conformance-support'; +import { SEARCH_INDEX_VERSION_CHECKS } from './search-index-conformance-versions'; +import type { SharedExamples } from './shared-examples'; +import { sharedExamples } from './shared-examples'; + +export const SEARCH_INDEX_CHECKS: readonly SearchIndexCheck[] = [ + { + name: 'an indexed document is found by a word of any field, whatever its case', + async run(subject) { + expect(subject.index.name.length).toBeGreaterThan(0); + await putSearchDocs(subject, [ + doc(subject.entity, 1, 'Zebra crossing', { body: 'a lantern harbour' }), + ]); + expect(await searchIds(subject, 'zebra')).toEqual([searchDocId(1)]); + expect(await searchIds(subject, 'HARBOUR')).toEqual([searchDocId(1)]); + }, + }, + { + name: 'every word of the term must match', + async run(subject) { + await putSearchDocs(subject, [ + doc(subject.entity, 1, 'zebra quokka'), + doc(subject.entity, 2, 'zebra lantern'), + ]); + expect(await searchIds(subject, 'zebra quokka')).toEqual([searchDocId(1)]); + expect(await searchIds(subject, 'zebra')).toEqual([searchDocId(1), searchDocId(2)]); + }, + }, + { + name: 'no match, an empty term and bare punctuation answer no hits, never an error', + async run(subject) { + await putSearchDocs(subject, [doc(subject.entity, 1, 'zebra')]); + for (const term of ['walrus', '', ' ', '|||', '(((']) { + expect(await searchIds(subject, term), term).toEqual([]); + } + }, + }, + { + name: "a tenant is never answered another tenant's document", + async run(subject) { + await putSearchDocs(subject, [ + doc(subject.entity, 1, 'zebra'), + doc(subject.entity, 2, 'zebra', { tenant: SEARCH_TENANT_B }), + ]); + expect(await searchIds(subject, 'zebra')).toEqual([searchDocId(1)]); + expect(await searchIds(subject, 'zebra', SEARCH_TENANT_B)).toEqual([searchDocId(2)]); + }, + }, + { + name: 'indexing a newer version of an id replaces its text, and it is one hit', + async run(subject) { + await putSearchDocs(subject, [doc(subject.entity, 1, 'zebra')]); + await putSearchDocs(subject, [doc(subject.entity, 1, 'quokka zebra', { version: 2 })]); + expect(await searchIds(subject, 'zebra')).toEqual([searchDocId(1)]); + await putSearchDocs(subject, [doc(subject.entity, 1, 'lantern', { version: 3 })]); + expect(await searchIds(subject, 'zebra')).toEqual([]); + expect(await searchIds(subject, 'lantern')).toEqual([searchDocId(1)]); + }, + }, + { + name: "a removed id is gone, and a removal under another tenant leaves this one's", + async run(subject) { + await putSearchDocs(subject, [ + doc(subject.entity, 1, 'zebra'), + doc(subject.entity, 2, 'zebra'), + ]); + const other: SearchScope = { entity: subject.entity, tenant: SEARCH_TENANT_B }; + await subject.drop(other, [searchDocId(1)]); + await subject.index.remove(other, [searchDocId(1)], 2); + expect(await searchIds(subject, 'zebra')).toEqual([searchDocId(1), searchDocId(2)]); + const own: SearchScope = { entity: subject.entity, tenant: SEARCH_TENANT_A }; + await subject.drop(own, [searchDocId(1)]); + await subject.index.remove(own, [searchDocId(1)], 2); + expect(await searchIds(subject, 'zebra')).toEqual([searchDocId(2)]); + }, + }, + { + name: 'the limit bounds the hits, each id at most once', + async run(subject) { + await putSearchDocs( + subject, + [1, 2, 3].map((n) => doc(subject.entity, n, `zebra ${'quokka '.repeat(n)}`)), + ); + const hits = await subject.index.query({ + entity: subject.entity, + tenant: SEARCH_TENANT_A, + term: 'zebra', + limit: 2, + }); + expect(hits.length).toBe(2); + expect(new Set(hits.map((hit) => hit.id)).size).toBe(2); + }, + }, +]; + +export const SEARCH_INDEX_ALL_CHECKS: readonly SearchIndexCheck[] = [ + ...SEARCH_INDEX_CHECKS, + ...SEARCH_INDEX_VERSION_CHECKS, +]; + +export const searchIndexConformance: SharedExamples< + SearchIndexSubject | Promise +> = sharedExamples('a search index', (subject) => { + for (const check of SEARCH_INDEX_ALL_CHECKS) { + test(check.name, async () => { + await check.run(await subject()); + }); + } +}); diff --git a/packages/testing/src/semantic-cache-factory-conformance.test.ts b/packages/testing/src/semantic-cache-factory-conformance.test.ts new file mode 100644 index 000000000..0eac6a382 --- /dev/null +++ b/packages/testing/src/semantic-cache-factory-conformance.test.ts @@ -0,0 +1,35 @@ +// The semantic-cache suite over `@ultimat3/cache`'s memory driver, one per scope, and the proof it +// can fail: a factory that hands every scope the SAME cache — one tenant reading another's answer. + +import { describe, expect, test } from 'bun:test'; +import type { SemanticCacheFactory } from '@ultimat3/ai/adapter'; +import { memorySemanticCache } from '@ultimat3/cache'; +import { + SEMANTIC_CACHE_FACTORY_CHECKS, + semanticCacheFactoryConformance, +} from './semantic-cache-factory-conformance'; +import { behavesLike } from './shared-examples'; +import { testName } from './test-types'; + +describe(testName('unit', 'the memory semantic cache, one per scope'), () => { + behavesLike(semanticCacheFactoryConformance, () => () => memorySemanticCache()); +}); + +const check = (prefix: string) => { + const found = SEMANTIC_CACHE_FACTORY_CHECKS.find((each) => each.name.startsWith(prefix)); + if (found === undefined) return expect.unreachable(`no semantic-cache check named ${prefix}`); + return found; +}; + +describe(testName('unit', 'a factory that breaks the contract fails conformance'), () => { + test('one cache shared by every scope fails the isolation check', async () => { + const shared = memorySemanticCache(); + const sharing: SemanticCacheFactory = () => shared; + await expect(check('two scopes are two caches').run(sharing)).rejects.toThrow(); + }); + + test('every check has a name of its own', () => { + const names = SEMANTIC_CACHE_FACTORY_CHECKS.map((each) => each.name); + expect(new Set(names).size).toBe(names.length); + }); +}); diff --git a/packages/testing/src/semantic-cache-factory-conformance.ts b/packages/testing/src/semantic-cache-factory-conformance.ts new file mode 100644 index 000000000..b5ada495d --- /dev/null +++ b/packages/testing/src/semantic-cache-factory-conformance.ts @@ -0,0 +1,86 @@ +// The `semanticCache` port's contract as shared examples: +// `behavesLike(semanticCacheFactoryConformance, () => (scope) => yours(scope))`. The port is a +// FACTORY, one cache per scope, because two tenants asking near-identical questions of one shared +// cache is one tenant reading the other's answer — so scope isolation is the first check. + +import { expect, test } from 'bun:test'; +import type { SemanticCacheFactory } from '@ultimat3/ai/adapter'; +import { uuidV7 } from '@ultimat3/core'; +import type { SharedExamples } from './shared-examples'; +import { sharedExamples } from './shared-examples'; + +export interface SemanticCacheFactoryCheck { + readonly name: string; + run(factory: SemanticCacheFactory): Promise; +} + +/** A fresh scope per check, so a durable store's rows from an earlier check never answer. */ +const scope = (label: string): string => `conformance-${label}-${uuidV7()}`; + +const NORTH = [1, 0, 0] as const; +const EAST = [0, 1, 0] as const; + +export const SEMANTIC_CACHE_FACTORY_CHECKS: readonly SemanticCacheFactoryCheck[] = [ + { + name: 'two scopes are two caches: what one remembers the other never answers', + async run(factory) { + const a = factory(scope('a')); + const b = factory(scope('b')); + await a.remember('k', NORTH, 'org-a answer'); + expect(await b.lookup(NORTH, 0.9)).toBeUndefined(); + expect((await a.lookup(NORTH, 0.9))?.value).toBe('org-a answer'); + }, + }, + { + name: 'a remembered embedding answers its own lookup, with its key and a similarity near 1', + async run(factory) { + const cache = factory(scope('hit')); + expect(cache.name.length).toBeGreaterThan(0); + await cache.remember('the-key', NORTH, { answer: 42 }); + const hit = await cache.lookup<{ answer: number }>(NORTH, 0.9); + expect(hit?.key).toBe('the-key'); + expect(hit?.value).toEqual({ answer: 42 }); + expect(hit?.similarity).toBeGreaterThan(0.999); + }, + }, + { + name: 'below the threshold is a miss, never the nearest entry anyway', + async run(factory) { + const cache = factory(scope('miss')); + await cache.remember('north', NORTH, 'north'); + expect(await cache.lookup(EAST, 0.9)).toBeUndefined(); + }, + }, + { + name: 'an embedding that is not a number is not a match', + async run(factory) { + const cache = factory(scope('nan')); + await cache.remember('north', NORTH, 'north'); + expect(await cache.lookup([1, Number.NaN, 0], 0.5)).toBeUndefined(); + }, + }, + { + name: 'invalidateTags drops the tagged entries, names their keys, and keeps the rest', + async run(factory) { + const cache = factory(scope('tags')); + await cache.remember('tagged', NORTH, 'tagged', { tags: [{ entity: 'post', id: '1' }] }); + await cache.remember('kept', EAST, 'kept', { tags: [{ entity: 'post', id: '2' }] }); + expect(await cache.size()).toBe(2); + expect(await cache.invalidateTags([{ entity: 'post', id: '1' }])).toEqual(['tagged']); + expect(await cache.lookup(NORTH, 0.9)).toBeUndefined(); + expect((await cache.lookup(EAST, 0.9))?.value).toBe('kept'); + expect(await cache.size()).toBe(1); + }, + }, +]; + +export const semanticCacheFactoryConformance: SharedExamples = sharedExamples( + 'a semantic cache factory', + (subject) => { + for (const check of SEMANTIC_CACHE_FACTORY_CHECKS) { + test(check.name, async () => { + await check.run(subject()); + }); + } + }, +); diff --git a/packages/testing/src/sms-transport-conformance.test.ts b/packages/testing/src/sms-transport-conformance.test.ts new file mode 100644 index 000000000..433fb6f7a --- /dev/null +++ b/packages/testing/src/sms-transport-conformance.test.ts @@ -0,0 +1,53 @@ +// The reference transport passes its own port's suite, and the suite is not vacuously green: a +// transport that skips the E.164 check, or leaks the number into its id, fails a named check. + +import { describe, expect, test } from 'bun:test'; +import { memorySmsTransport } from '@ultimat3/notify'; +import type { SmsTransport } from '@ultimat3/notify/adapter'; +import { behavesLike } from './shared-examples'; +import { SMS_TRANSPORT_CHECKS, smsTransportConformance } from './sms-transport-conformance'; + +describe('memorySmsTransport', () => { + behavesLike(smsTransportConformance, () => memorySmsTransport()); +}); + +const failing = async (transport: SmsTransport): Promise => { + const failed: string[] = []; + for (const check of SMS_TRANSPORT_CHECKS) { + await check.run(transport).catch(() => failed.push(check.name)); + } + return failed; +}; + +describe('unit · the SMS suite can fail', () => { + test('a transport that checks no number fails the E.164 check alone', async () => { + const memory = memorySmsTransport(); + const lax: SmsTransport = { + name: 'lax', + send: (message) => + Promise.resolve({ + transport: 'lax', + id: crypto.randomUUID(), + to: message.to, + idempotencyKey: message.idempotencyKey, + }), + }; + expect(await failing(memory)).toEqual([]); + expect(await failing(lax)).toEqual([ + 'a number that is not E.164 is refused with X_NOTIFY_SMS_NUMBER_INVALID', + ]); + }); + + test('a transport that puts the number in its id fails the leak check', async () => { + const memory = memorySmsTransport(); + const leaky: SmsTransport = { + name: 'leaky', + send: async (message) => ({ + ...(await memory.send(message)), + transport: 'leaky', + id: `sent-${message.to}`, + }), + }; + expect(await failing(leaky)).toEqual(['the result id carries neither the number nor the body']); + }); +}); diff --git a/packages/testing/src/sms-transport-conformance.ts b/packages/testing/src/sms-transport-conformance.ts new file mode 100644 index 000000000..7f3675233 --- /dev/null +++ b/packages/testing/src/sms-transport-conformance.ts @@ -0,0 +1,102 @@ +// The `SmsTransport` contract of `@ultimat3/notify` as shared examples, for a provider adapter of +// your own (Twilio, Vonage, SNS): `behavesLike(smsTransportConformance, () => transport)`. Asked +// through `send` alone — what a transport answers about a message, never how its wire looks. + +import { expect, test } from 'bun:test'; +import type { SmsMessage, SmsTransport } from '@ultimat3/notify/adapter'; +import type { SharedExamples } from './shared-examples'; +import { sharedExamples } from './shared-examples'; + +export interface SmsTransportCheck { + readonly name: string; + run(transport: SmsTransport): Promise; +} + +const TO = '+4915112345678'; +const CONCURRENT_SENDS = 8; + +/** A rendered message, by hand: the port receives text already in the recipient's language. */ +export const conformanceSms = (overrides: Partial = {}): SmsMessage => ({ + to: TO, + body: 'Your code is 424242', + idempotencyKey: 'conformance-sms-1', + ...overrides, +}); + +const refusedCode = async (run: () => Promise): Promise => { + try { + await run(); + } catch (error) { + return (error as { code?: unknown }).code; + } + return expect.unreachable('expected the transport to refuse'); +}; + +export const SMS_TRANSPORT_CHECKS: readonly SmsTransportCheck[] = [ + { + name: 'a send resolves a result naming this transport and the number it accepted', + async run(transport) { + expect(transport.name.length).toBeGreaterThan(0); + const result = await transport.send(conformanceSms()); + expect([result.transport, result.to]).toEqual([transport.name, TO]); + expect(result.id.length).toBeGreaterThan(0); + }, + }, + { + // A caller matching a retry to its first send reads this key back. + name: 'the idempotency key is answered back unchanged, the same on every attempt', + async run(transport) { + const message = conformanceSms(); + const first = await transport.send(message); + const second = await transport.send(message); + expect([first.idempotencyKey, second.idempotencyKey]).toEqual([ + message.idempotencyKey, + message.idempotencyKey, + ]); + }, + }, + { + // The id lands in logs and the delivery ledger; the number and the body are a person's. + name: 'the result id carries neither the number nor the body', + async run(transport) { + const message = conformanceSms(); + const result = await transport.send(message); + for (const leak of [TO, TO.slice(1), message.body]) expect(result.id).not.toContain(leak); + }, + }, + { + // A local spelling is a number the provider guesses the country of: a message to a stranger. + name: 'a number that is not E.164 is refused with X_NOTIFY_SMS_NUMBER_INVALID', + async run(transport) { + for (const to of ['015112345678', '+49 151 12345678', '']) { + expect(await refusedCode(() => transport.send(conformanceSms({ to })))).toBe( + 'X_NOTIFY_SMS_NUMBER_INVALID', + ); + } + }, + }, + { + name: `${CONCURRENT_SENDS} concurrent sends each answer for their own message`, + async run(transport) { + const messages = Array.from({ length: CONCURRENT_SENDS }, (_, index) => + conformanceSms({ + to: `+1415555${String(index).padStart(4, '0')}`, + idempotencyKey: `conformance-sms-${String(index)}`, + }), + ); + const results = await Promise.all(messages.map((message) => transport.send(message))); + expect(results.map((result) => [result.to, result.idempotencyKey])).toEqual( + messages.map((message) => [message.to, message.idempotencyKey]), + ); + }, + }, +]; + +export const smsTransportConformance: SharedExamples> = + sharedExamples('an SMS transport', (subject) => { + for (const check of SMS_TRANSPORT_CHECKS) { + test(check.name, async () => { + await check.run(await subject()); + }); + } + }); diff --git a/packages/testing/src/swap-adapter.ts b/packages/testing/src/swap-adapter.ts index f322d3519..171cd09b9 100644 --- a/packages/testing/src/swap-adapter.ts +++ b/packages/testing/src/swap-adapter.ts @@ -4,7 +4,7 @@ // `UNSWAPPED` with what a test does instead — a new row in core's table is a compile error here. import type { IdempotencyStore, MeterStore } from '@ultimat3/action/adapter'; -import type { Gateway } from '@ultimat3/ai/adapter'; +import type { Embedder, Gateway, SemanticCacheFactory } from '@ultimat3/ai/adapter'; import type { AuthLimiterFactory, TotpReplayGuard } from '@ultimat3/auth/adapter'; import type { AdapterPortName } from '@ultimat3/core'; import type { @@ -14,10 +14,11 @@ import type { SecretSource, SleepClock, } from '@ultimat3/core/adapter'; +import type { SearchIndex } from '@ultimat3/entity/adapter'; import type { RateLimitStore } from '@ultimat3/http/adapter'; import type { EventBus, JobDriver } from '@ultimat3/jobs/adapter'; import type { MailDriver } from '@ultimat3/mail/adapter'; -import type { NotifyStores } from '@ultimat3/notify/adapter'; +import type { NotifyStores, SmsTransport } from '@ultimat3/notify/adapter'; import type { Transport } from '@ultimat3/realtime/adapter'; import type { ScrapeDriver } from '@ultimat3/scraping/adapter'; import { TestAdapterPortUnswappableError } from './stub-errors'; @@ -30,6 +31,7 @@ export interface SwappableAdapters { readonly auditSink: AuditSink; readonly authLimiter: AuthLimiterFactory; readonly clock: SleepClock; + readonly embedder: Embedder; readonly errorReporter: ErrorReporter; readonly eventBus: EventBus; readonly failureStore: FailureStore; @@ -40,7 +42,10 @@ export interface SwappableAdapters { readonly notifyStores: NotifyStores; readonly rateLimitStore: RateLimitStore; readonly scrapeDriver: ScrapeDriver; + readonly searchIndex: SearchIndex; readonly secretSource: SecretSource; + readonly semanticCache: SemanticCacheFactory; + readonly smsTransport: SmsTransport; readonly totpReplay: TotpReplayGuard; readonly transport: Transport; } @@ -56,6 +61,7 @@ const BINDINGS: Bindings = { auditSink: async (impl) => (await import('@ultimat3/core/adapter')).installAuditSink(impl), authLimiter: async (impl) => (await import('@ultimat3/auth/adapter')).installAuthLimiters(impl), clock: async (impl) => (await import('@ultimat3/core/adapter')).installClock(impl), + embedder: async (impl) => (await import('@ultimat3/ai/adapter')).installAiEmbedder(impl), errorReporter: async (impl) => (await import('@ultimat3/core/adapter')).installErrorReporter(impl), eventBus: async (impl) => (await import('@ultimat3/jobs/adapter')).installEventBus(impl), @@ -71,7 +77,11 @@ const BINDINGS: Bindings = { (await import('@ultimat3/http/adapter')).installRateLimitStore(impl), scrapeDriver: async (impl) => (await import('@ultimat3/scraping/adapter')).installScrapeDriver(impl), + searchIndex: async (impl) => (await import('@ultimat3/entity/adapter')).installSearchIndex(impl), secretSource: async (impl) => (await import('@ultimat3/core/adapter')).installSecretSource(impl), + semanticCache: async (impl) => (await import('@ultimat3/ai/adapter')).installSemanticCache(impl), + smsTransport: async (impl) => + (await import('@ultimat3/notify/adapter')).installSmsTransport(impl), // The slot takes a factory; the port carries one guard, which every call hands back. totpReplay: async (impl) => (await import('@ultimat3/auth/adapter')).installTotpReplay(() => impl), @@ -98,6 +108,8 @@ export const UNSWAPPED: Readonly, purge: "handed to the cache's CDN purge at boot: pass your driver to the bust under test", pushSubscriptionStore: 'the push fixture owns one: destructure { push } in the test', storage: 'disks are built by the boot: build the disk over your driver in the test', + uploadScanner: + '@ultimat3/testing takes no dependency on @ultimat3/storage: call installUploadScanner(scanner) from @ultimat3/storage/adapter in the test and run the release it answers in afterEach', }; const isSwappable = (port: string): port is SwappablePort => Object.hasOwn(BINDINGS, port); diff --git a/packages/testing/src/upload-scanner-conformance.test.ts b/packages/testing/src/upload-scanner-conformance.test.ts new file mode 100644 index 000000000..36cf5dc7c --- /dev/null +++ b/packages/testing/src/upload-scanner-conformance.test.ts @@ -0,0 +1,86 @@ +// The upload-scanner suite over storage's memory reference, and the proof it can fail: a scanner +// that matches per chunk, one that keeps its last verdict, and one that calls everything infected. + +import { describe, expect, test } from 'bun:test'; +import { EICAR_TEST_STRING, memoryUploadScanner } from '@ultimat3/storage'; +import type { UploadScanner } from '@ultimat3/storage/adapter'; +import { behavesLike } from './shared-examples'; +import { testName } from './test-types'; +import type { UploadScannerLike } from './upload-scanner-conformance'; +import { + UPLOAD_SCANNER_CHECKS, + UPLOAD_SCANNER_EICAR, + uploadScannerConformance, +} from './upload-scanner-conformance'; + +/** Compile-time: storage's port satisfies the structural subject the suite takes. */ +const _assignable = (scanner: UploadScanner): UploadScannerLike => scanner; +void _assignable; + +describe(testName('unit', 'the memory upload scanner'), () => { + behavesLike(uploadScannerConformance, memoryUploadScanner); +}); + +const SIGNATURE = new TextEncoder().encode('EICAR-STANDARD'); + +/** Looks inside each chunk alone — what a scanner forwarding chunks to a per-buffer API does. */ +const perChunk = (): UploadScannerLike => ({ + name: 'per-chunk', + async scan({ body }) { + for await (const chunk of body) { + if (Buffer.from(chunk).includes(SIGNATURE)) return { verdict: 'infected', signature: 'x' }; + } + return { verdict: 'clean' }; + }, +}); + +/** Remembers the first infected verdict for every later file — a cache keyed on nothing. */ +function sticky(): UploadScannerLike { + const inner = memoryUploadScanner(); + let last: Awaited> | undefined; + return { + name: 'sticky', + async scan(input) { + const verdict = await inner.scan(input); + if (last?.verdict === 'infected') return last; + last = verdict; + return verdict; + }, + }; +} + +const paranoid = (): UploadScannerLike => ({ + name: 'paranoid', + scan: async () => ({ verdict: 'infected', signature: 'everything' }), +}); + +const check = (prefix: string) => { + const found = UPLOAD_SCANNER_CHECKS.find((each) => each.name.startsWith(prefix)); + if (found === undefined) return expect.unreachable(`no upload-scanner check named ${prefix}`); + return found; +}; + +describe(testName('unit', 'a scanner that breaks the contract fails conformance'), () => { + test('the suite and storage agree on the EICAR string', () => { + expect(UPLOAD_SCANNER_EICAR).toBe(EICAR_TEST_STRING); + }); + + test('a per-chunk matcher fails the chunked check', async () => { + await check('the EICAR test file is infected').run(perChunk()); + await expect(check('the EICAR test file delivered in').run(perChunk())).rejects.toThrow(); + }); + + test('a scanner that keeps its verdict fails the bytes-only check', async () => { + await expect(check('a verdict is about these bytes only').run(sticky())).rejects.toThrow(); + }); + + test('a scanner that calls everything infected fails the clean checks', async () => { + await expect(check('a plain text file is clean').run(paranoid())).rejects.toThrow(); + await expect(check('an empty file is clean').run(paranoid())).rejects.toThrow(); + }); + + test('every check has a name of its own', () => { + const names = UPLOAD_SCANNER_CHECKS.map((each) => each.name); + expect(new Set(names).size).toBe(names.length); + }); +}); diff --git a/packages/testing/src/upload-scanner-conformance.ts b/packages/testing/src/upload-scanner-conformance.ts new file mode 100644 index 000000000..95626c8bd --- /dev/null +++ b/packages/testing/src/upload-scanner-conformance.ts @@ -0,0 +1,108 @@ +// The `uploadScanner` port's contract as shared examples: `behavesLike(uploadScannerConformance, () +// => scanner)`. Structural over `@ultimat3/storage`'s `UploadScanner`, as the storage suite is over +// its driver, so this package takes no dependency on storage. Bodies arrive as streams, cut small. + +import { expect, test } from 'bun:test'; +import type { SharedExamples } from './shared-examples'; +import { sharedExamples } from './shared-examples'; + +/** `@ultimat3/storage`'s `ScanVerdict`, restated structurally. */ +export type ScanVerdictLike = + | { readonly verdict: 'clean' } + | { readonly verdict: 'infected'; readonly signature: string }; + +/** `@ultimat3/storage`'s `UploadScanner`: a ClamAV adapter, a vendor API, the memory reference. */ +export interface UploadScannerLike { + readonly name: string; + scan(input: { + readonly key: string; + readonly size: number; + readonly contentType: string; + readonly body: ReadableStream; + }): Promise; +} + +export interface UploadScannerCheck { + readonly name: string; + run(scanner: UploadScannerLike): Promise; +} + +/** The EICAR test string, in two halves so this file is no detection itself. */ +export const UPLOAD_SCANNER_EICAR = `${'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR'}${'-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*'}`; + +const encode = (text: string): Uint8Array => new TextEncoder().encode(text); + +/** `bytes` as a stream of `size`-byte chunks — EICAR cut by the transport is still EICAR. */ +function streamOf(bytes: Uint8Array, size: number): ReadableStream { + let at = 0; + return new ReadableStream({ + pull(controller) { + if (at >= bytes.byteLength) return controller.close(); + controller.enqueue(bytes.slice(at, at + size)); + at += size; + }, + }); +} + +const scan = (scanner: UploadScannerLike, bytes: Uint8Array, chunk = 64 * 1024) => + scanner.scan({ + key: 'org/conformance/post/p-1/file/u-1.txt', + size: bytes.byteLength, + contentType: 'text/plain', + body: streamOf(bytes, chunk), + }); + +const PLAIN = encode('A plain note: nothing in here is a virus.\n'); + +export const UPLOAD_SCANNER_CHECKS: readonly UploadScannerCheck[] = [ + { + name: 'has a name, for the refusal it ends up in', + async run(scanner) { + expect(scanner.name.length).toBeGreaterThan(0); + }, + }, + { + name: 'the EICAR test file is infected, with a non-empty signature', + async run(scanner) { + const verdict = await scan(scanner, encode(UPLOAD_SCANNER_EICAR)); + expect(verdict.verdict).toBe('infected'); + expect(verdict.verdict === 'infected' ? verdict.signature.length : 0).toBeGreaterThan(0); + }, + }, + { + name: 'the EICAR test file delivered in 7-byte chunks is still infected', + async run(scanner) { + expect((await scan(scanner, encode(UPLOAD_SCANNER_EICAR), 7)).verdict).toBe('infected'); + }, + }, + { + name: 'a plain text file is clean', + async run(scanner) { + expect(await scan(scanner, PLAIN)).toEqual({ verdict: 'clean' }); + }, + }, + { + name: 'an empty file is clean', + async run(scanner) { + expect(await scan(scanner, new Uint8Array())).toEqual({ verdict: 'clean' }); + }, + }, + { + name: 'a verdict is about these bytes only: infected, then clean, from one scanner', + async run(scanner) { + expect((await scan(scanner, encode(UPLOAD_SCANNER_EICAR))).verdict).toBe('infected'); + expect(await scan(scanner, PLAIN)).toEqual({ verdict: 'clean' }); + }, + }, +]; + +export const uploadScannerConformance: SharedExamples = sharedExamples( + 'an upload scanner', + (subject) => { + for (const check of UPLOAD_SCANNER_CHECKS) { + test(check.name, async () => { + await check.run(subject()); + }); + } + }, +); diff --git a/scripts/bench-claims.test.ts b/scripts/bench-claims.test.ts index 0ec166fa4..da3389e9a 100644 --- a/scripts/bench-claims.test.ts +++ b/scripts/bench-claims.test.ts @@ -195,9 +195,9 @@ describe('unit · this repo', () => { ); test('checks every committed run, so none can drift unwatched', () => { - // The two realtime runs, the first hour, http-hello's two targets, the big-page render and the - // span overhead. - expect(new Set(CLAIMS.map((one) => one.file)).size).toBe(7); + // The two realtime runs, the first hour, http-hello's two targets, the big-page render, the + // span overhead, and the scale ladder's four (workers, fan-out, boot, the production image). + expect(new Set(CLAIMS.map((one) => one.file)).size).toBe(11); expect(CLAIMS.length).toBeGreaterThan(9); }); }); @@ -215,7 +215,7 @@ describe('unit · a figure stated on another page (README.md, wiki/Compared.md)' test('off by one is refused, and the finding names that page', () => { const found = check(PROSE, 1_666_883); expect(found.map((finding) => finding.at)).toEqual(['README.md']); - expect(found[0]?.fix).toContain('edit README.md: write 1,666,883'); + expect(found[0]?.fix).toContain('after editing README.md: write 1,666,883'); }); test('agrees when that page states the measured figure — the claims page is not read for it', () => { diff --git a/scripts/bench-claims.ts b/scripts/bench-claims.ts index 907a128c4..9310d8573 100644 --- a/scripts/bench-claims.ts +++ b/scripts/bench-claims.ts @@ -122,7 +122,7 @@ const pageOf = (claim: BenchClaim): string => claim.doc ?? CLAIMS_FILE; const mismatchFinding = (gap: BenchGap): Finding => ({ code: 'X_BENCH_CLAIM_STALE', cause: `${pageOf(gap.claim)} states ${gap.stated ?? ''} for ${gap.claim.label} and ${gap.claim.file} measured ${gap.measured ?? ''}`, - fix: `edit ${pageOf(gap.claim)}: write ${gap.measured ?? ''} where it says ${gap.stated ?? ''} for ${gap.claim.label}, then bun run scripts/bench-claims.ts --json`, + fix: `bun run scripts/bench-claims.ts --json # after editing ${pageOf(gap.claim)}: write ${gap.measured ?? ''} where it says ${gap.stated ?? ''} for ${gap.claim.label}`, at: pageOf(gap.claim), }); diff --git a/scripts/bench/boot-vs-files-child.ts b/scripts/bench/boot-vs-files-child.ts new file mode 100644 index 000000000..149700b8c --- /dev/null +++ b/scripts/bench/boot-vs-files-child.ts @@ -0,0 +1,38 @@ +// One cold boot for `boot-vs-files.bench.ts`: a fresh process imports every module of a synthetic +// app directory, either one `await import()` after another (how `x` booted before plan 101 B1) or +// through the CLI's batched loader (`importInOrder`), and prints `FINAL ` with the wall time. +// +// bun run scripts/bench/boot-vs-files-child.ts --dir --mode serial|batched + +import { importInOrder } from '../../packages/cli/src/app-load-batch'; +import { parseScriptArgs } from '../lib/args'; +import { ScriptError } from '../lib/script-error'; + +const args = parseScriptArgs(Bun.argv.slice(2)); +const dir = args.flags.get('dir'); +const mode = args.flags.get('mode'); +if (typeof dir !== 'string' || (mode !== 'serial' && mode !== 'batched')) { + throw new ScriptError({ + code: 'X_CLI_BAD_FLAG', + cause: 'boot-vs-files-child is spawned by the bench with --dir and --mode serial|batched', + fix: 'bun run scripts/bench/boot-vs-files.bench.ts --files 500', + }); +} + +const files = [...new Bun.Glob('*.ts').scanSync({ cwd: dir, absolute: true })].sort(); +const started = performance.now(); +if (mode === 'serial') { + for (const file of files) await import(file); +} else { + const imported = await importInOrder(files); + const failed = imported.find((entry) => !entry.ok); + if (failed !== undefined && !failed.ok) { + throw new ScriptError({ + code: 'X_BENCH_RUN_FAILED', + cause: 'a synthetic app module failed to import, so the boot time measures a broken app', + fix: 'bun run scripts/bench/boot-vs-files.bench.ts --files 500 --json', + }); + } +} +const ms = performance.now() - started; +console.log(`FINAL ${JSON.stringify({ modules: files.length, ms: Math.round(ms) })}`); diff --git a/scripts/bench/boot-vs-files-shape.test.ts b/scripts/bench/boot-vs-files-shape.test.ts new file mode 100644 index 000000000..537ca5dfb --- /dev/null +++ b/scripts/bench/boot-vs-files-shape.test.ts @@ -0,0 +1,28 @@ +// The boot bench's fixture: names that sort the way the scan sorts, modules that transpile and +// form a graph, and the median it reports. + +import { describe, expect, test } from 'bun:test'; +import { median, moduleName, moduleSource } from './boot-vs-files-shape'; + +describe('boot-vs-files shape', () => { + test('a lexical sort of the names is the numeric order', () => { + const names = [moduleName(10), moduleName(9), moduleName(100)]; + expect([...names].sort()).toEqual([moduleName(9), moduleName(10), moduleName(100)]); + }); + + test('every tenth module imports the one before it, and every module transpiles', () => { + const transpiler = new Bun.Transpiler({ loader: 'ts' }); + expect(transpiler.scanImports(moduleSource(10)).map((entry) => entry.path)).toEqual([ + '@ultimat3/entity', + './mod-00009', + ]); + expect(transpiler.scanImports(moduleSource(11)).map((entry) => entry.path)).toEqual([ + '@ultimat3/entity', + ]); + }); + + test('the median of three is the middle one', () => { + expect(median([30, 10, 20])).toBe(20); + expect(median([])).toBe(0); + }); +}); diff --git a/scripts/bench/boot-vs-files-shape.ts b/scripts/bench/boot-vs-files-shape.ts new file mode 100644 index 000000000..1fbd74b13 --- /dev/null +++ b/scripts/bench/boot-vs-files-shape.ts @@ -0,0 +1,43 @@ +// The pure half of `boot-vs-files.bench.ts`: the synthetic app module it writes N of, and the +// median it reports. Split from the harness so the fixture's shape is tested without importing it. + +/** The file counts measured: the reference app's order of magnitude, 4×, and the plan's 5k app. */ +export const DEFAULT_FILE_COUNTS: readonly number[] = [500, 2_000, 5_000]; + +/** `mod-00042.ts` — zero-padded, so a lexical sort is the numeric one the scan uses. */ +export const moduleName = (index: number): string => `mod-${String(index).padStart(5, '0')}.ts`; + +/** + * One app module: an `entity()` with five columns (what most files in an app register), and every + * tenth one referencing the module before it — an app's files are a graph, not a flat list. + */ +export function moduleSource(index: number): string { + const table = `bench_t${String(index)}`; + const parent = index % 10 === 0 && index > 0 ? index - 1 : undefined; + const lines = [ + "import { entity, text, timestamp, uuid } from '@ultimat3/entity';", + ...(parent === undefined + ? [] + : [`import { t${String(parent)} } from './${moduleName(parent).replace(/\.ts$/, '')}';`]), + `export const t${String(index)} = entity('${table}', {`, + ' columns: {', + ' id: uuid().primaryKey(),', + parent === undefined + ? ' parentId: uuid().nullable(),' + : ` parentId: uuid().references(() => t${String(parent)}.id).nullable(),`, + ' title: text(),', + ' body: text().nullable(),', + ' createdAt: timestamp().defaultNow(),', + ' },', + '});', + `export const label${String(index)} = (title: string): string => \`${table}:\${title}\`;`, + '', + ]; + return lines.join('\n'); +} + +/** The middle value of an odd-length list of timings, or the lower middle of an even one. */ +export function median(values: readonly number[]): number { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor((sorted.length - 1) / 2)] ?? 0; +} diff --git a/scripts/bench/boot-vs-files.bench.ts b/scripts/bench/boot-vs-files.bench.ts new file mode 100644 index 000000000..1a2c5bf80 --- /dev/null +++ b/scripts/bench/boot-vs-files.bench.ts @@ -0,0 +1,129 @@ +#!/usr/bin/env bun +// App-module load time against file count (plan 101 slice 22, B1): a synthetic app of N modules, +// each an `entity()` file, imported cold in a fresh process — serially, as `x` booted before B1, +// and through the CLI's batched loader (`importInOrder`). The median of `--runs` cold boots each. +// Module load only: no server, no database — the part of a boot that grew with the app. +// +// bun run scripts/bench/boot-vs-files.bench.ts [--files 500,2000,5000] [--runs 3] +// [--out ] [--json] +// +// The fixture is written under `packages/cli/node_modules/.cache/bench-boot/` — the root has no +// `@ultimat3/*` links, the CLI's install does — so imports resolve to the workspace and nothing +// lands in the tree. + +// why: Bun ships no directory create or recursive delete of its own. +import { mkdir, rm } from 'node:fs/promises'; +import { parseScriptArgs } from '../lib/args'; +import { repoRoot } from '../lib/run'; +import { ScriptError } from '../lib/script-error'; +import { DEFAULT_FILE_COUNTS, median, moduleName, moduleSource } from './boot-vs-files-shape'; +import { boxFacts, readCount } from './http-hello'; + +const CHILD = Bun.fileURLToPath(new URL('./boot-vs-files-child.ts', import.meta.url)); + +export interface BootRun { + readonly files: number; + readonly serialMs: number; + readonly batchedMs: number; + /** Per thousand files, serial — the slope the plan measured at ~5 ms a file. */ + readonly serialMsPer1k: number; + readonly batchedMsPer1k: number; +} + +const fixtureRoot = (): string => `${repoRoot()}/packages/cli/node_modules/.cache/bench-boot`; + +async function writeFixture(files: number): Promise { + const dir = `${fixtureRoot()}/${String(files)}`; + await rm(dir, { recursive: true, force: true }); + await mkdir(dir, { recursive: true }); + await Promise.all( + Array.from({ length: files }, (_, index) => + Bun.write(`${dir}/${moduleName(index)}`, moduleSource(index)), + ), + ); + return dir; +} + +async function coldBoot(dir: string, mode: 'serial' | 'batched'): Promise { + const child = Bun.spawn(['bun', CHILD, '--dir', dir, '--mode', mode], { + stdout: 'pipe', + stderr: 'pipe', + // A 5,000-module import is seconds; minutes is a hung loader. + timeout: 300_000, + }); + const [out, err, code] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + const line = out.split('\n').find((entry) => entry.startsWith('FINAL ')); + if (code !== 0 || line === undefined) { + throw new ScriptError({ + code: 'X_BENCH_RUN_FAILED', + cause: `a ${mode} cold boot exited ${String(code)} without a figure: ${err.slice(0, 400)}`, + fix: 'bun run scripts/bench/boot-vs-files.bench.ts --files 500 --runs 1', + }); + } + const parsed: unknown = JSON.parse(line.slice('FINAL '.length)); + const ms = (parsed as { ms?: unknown }).ms; + return typeof ms === 'number' ? ms : Number.NaN; +} + +export async function benchBoot(counts: readonly number[], runs: number): Promise { + const out: BootRun[] = []; + for (const files of counts) { + const dir = await writeFixture(files); + // One warm-up of each, discarded: the first import of a fresh file pays Bun's transpile cache. + await coldBoot(dir, 'serial'); + await coldBoot(dir, 'batched'); + const serial: number[] = []; + const batched: number[] = []; + for (let run = 0; run < runs; run += 1) { + serial.push(await coldBoot(dir, 'serial')); + batched.push(await coldBoot(dir, 'batched')); + } + const serialMs = median(serial); + const batchedMs = median(batched); + out.push({ + files, + serialMs, + batchedMs, + serialMsPer1k: Math.round((serialMs * 1_000) / files), + batchedMsPer1k: Math.round((batchedMs * 1_000) / files), + }); + await rm(dir, { recursive: true, force: true }); + } + await rm(fixtureRoot(), { recursive: true, force: true }); + return out; +} + +if (import.meta.main) { + const args = parseScriptArgs(Bun.argv.slice(2)); + const rawFiles = args.flags.get('files'); + const counts = + typeof rawFiles === 'string' + ? rawFiles.split(',').map((part) => Number(part)) + : DEFAULT_FILE_COUNTS; + const runs = readCount(args.flags.get('runs'), 3, { flag: 'runs', min: 1, max: 15 }); + if ( + typeof runs !== 'number' || + !counts.every((n) => Number.isInteger(n) && n >= 1 && n <= 50_000) + ) { + throw new ScriptError({ + code: 'X_CLI_BAD_FLAG', + cause: + 'boot-vs-files takes --files as whole numbers 1..50000, comma separated, and --runs 1..15', + fix: 'bun run scripts/bench/boot-vs-files.bench.ts --files 500,2000,5000 --runs 3', + }); + } + const results = await benchBoot(counts, runs); + const data = { box: boxFacts(Bun.env), runs, results }; + const out = args.flags.get('out'); + if (typeof out === 'string') await Bun.write(out, `${JSON.stringify(data, null, 2)}\n`); + if (args.json) console.log(JSON.stringify(data, null, 2)); + else { + for (const r of results) { + console.log(`${r.files} files: serial ${r.serialMs} ms, batched ${r.batchedMs} ms`); + } + } +} diff --git a/scripts/bench/fanout-3-nodes.bench.ts b/scripts/bench/fanout-3-nodes.bench.ts new file mode 100644 index 000000000..36fc03435 --- /dev/null +++ b/scripts/bench/fanout-3-nodes.bench.ts @@ -0,0 +1,176 @@ +#!/usr/bin/env bun +// Cross-node fan-out latency over a real nats-server: three `sync` node PROCESSES (each the shipped +// `ChannelHub` + `SocketRegistry` on its own NATS connection) hold `--subscribers` sockets between +// them on one events channel topic; a fourth process — a worker's half, the bus and nothing else — +// publishes `--events` probes. Latency is publish to the socket's `send`, per socket per probe. +// +// The sockets are in-process `WsLike`s, not real WebSockets: this measures the bus and the node's +// fan-out, not the kernel's TCP stack or a browser — the restart bench covers real sockets. +// +// bun run scripts/bench/fanout-3-nodes.bench.ts [--nats nats://…] [--subscribers 10000] +// [--events 20] [--interval-ms 250] [--out ] [--json] +// +// `--nats` defaults to `TEST_NATS_URL` (`docker/test-services.env`). + +import { + installChannelTransport, + publishChannelEvent, +} from '../../packages/realtime/src/channel-publish'; +import { NatsTransport } from '../../packages/realtime/src/nats-transport'; +import { parseScriptArgs } from '../lib/args'; +import { ScriptError } from '../lib/script-error'; +import { + FANOUT_BUCKET, + fanoutChannel, + type LatencySummary, + nowMs, + splitAcross, + summarise, +} from './fanout-shape'; +import { boxFacts, readCount } from './http-hello'; + +const CHILD = Bun.fileURLToPath(new URL('./fanout-node-child.ts', import.meta.url)); +const NODES = 3; +/** Long enough for the three nodes' NATS subscriptions to reach the server before the first probe. */ +const SETTLE_MS = 1_000; +/** A node's whole life: its own window closes well before this; past it the node hung. */ +const CHILD_DEADLINE_MS = 10 * 60_000; + +const failed = (cause: string): ScriptError => + new ScriptError({ + code: 'X_BENCH_RUN_FAILED', + cause, + fix: 'docker compose -f docker/docker-compose.test.yml up -d --wait nats && bun run scripts/bench/fanout-3-nodes.bench.ts --subscribers 3000', + }); + +/** Lines of a child's stdout, until one starts with `prefix`; returns the rest of that line. */ +async function lineStarting( + reader: ReadableStreamDefaultReader, + carry: { buffer: string }, + prefix: string, +): Promise { + const decoder = new TextDecoder(); + for (;;) { + let newline = carry.buffer.indexOf('\n'); + while (newline >= 0) { + const line = carry.buffer.slice(0, newline); + carry.buffer = carry.buffer.slice(newline + 1); + if (line.startsWith(prefix)) return line.slice(prefix.length); + newline = carry.buffer.indexOf('\n'); + } + const { value, done } = await reader.read(); + if (done) throw failed(`a node process exited before it printed ${prefix.trim()}`); + carry.buffer += decoder.decode(value, { stream: true }); + } +} + +export interface FanoutRun { + readonly nodes: number; + readonly subscribers: number; + readonly events: number; + readonly delivered: number; + readonly expected: number; + readonly overall: LatencySummary; + readonly perNode: readonly LatencySummary[]; +} + +export async function benchFanout3( + nats: string, + subscribers: number, + events: number, + intervalMs: number, +): Promise { + const org = crypto.randomUUID(); + const children = splitAcross(subscribers, NODES).map((count) => + Bun.spawn( + [ + 'bun', + CHILD, + '--nats', + nats, + '--org', + org, + '--subscribers', + String(count), + '--events', + String(events), + ], + { stdout: 'pipe', stderr: 'inherit', timeout: CHILD_DEADLINE_MS }, + ), + ); + const readers = children.map((child) => ({ + reader: (child.stdout as ReadableStream).getReader(), + carry: { buffer: '' }, + })); + const publisher = new NatsTransport({ url: nats, bucket: FANOUT_BUCKET }); + try { + await Promise.all(readers.map(({ reader, carry }) => lineStarting(reader, carry, 'READY'))); + await publisher.connect(); + installChannelTransport(publisher); + const channel = fanoutChannel(); + await Bun.sleep(SETTLE_MS); + for (let index = 0; index < events; index += 1) { + await publishChannelEvent(channel, { org }, { probeIndex: index, probeSentAt: nowMs() }); + await Bun.sleep(intervalMs); + } + const finals = await Promise.all( + readers.map(async ({ reader, carry }) => { + const body: unknown = JSON.parse(await lineStarting(reader, carry, 'FINAL ')); + return body as { received: number; expected: number; latencies: number[][] }; + }), + ); + const perNode = finals.map((final) => summarise(final.latencies.flat())); + return { + nodes: NODES, + subscribers, + events, + delivered: finals.reduce((sum, final) => sum + final.received, 0), + expected: subscribers * events, + overall: summarise(finals.flatMap((final) => final.latencies.flat())), + perNode, + }; + } finally { + for (const child of children) child.kill(); + await Promise.all(children.map((child) => child.exited)); + await publisher.close(); + } +} + +if (import.meta.main) { + const args = parseScriptArgs(Bun.argv.slice(2)); + const nats = args.flags.get('nats') ?? Bun.env['TEST_NATS_URL']; + const subscribers = readCount(args.flags.get('subscribers'), 10_000, { + flag: 'subscribers', + min: NODES, + max: 300_000, + }); + const events = readCount(args.flags.get('events'), 20, { flag: 'events', min: 1, max: 1_000 }); + const intervalMs = readCount(args.flags.get('interval-ms'), 250, { + flag: 'interval-ms', + min: 10, + max: 10_000, + }); + if ( + typeof nats !== 'string' || + typeof subscribers !== 'number' || + typeof events !== 'number' || + typeof intervalMs !== 'number' + ) { + throw new ScriptError({ + code: 'X_CLI_BAD_FLAG', + cause: + 'fanout-3-nodes needs a NATS url (--nats or TEST_NATS_URL) and whole --subscribers, --events and --interval-ms', + fix: 'bun run scripts/bench/fanout-3-nodes.bench.ts --nats nats://127.0.0.1:4222 --subscribers 10000', + }); + } + const run = await benchFanout3(nats, subscribers, events, intervalMs); + const data = { box: boxFacts(Bun.env), intervalMs, run }; + const out = args.flags.get('out'); + if (typeof out === 'string') await Bun.write(out, `${JSON.stringify(data, null, 2)}\n`); + if (args.json) console.log(JSON.stringify(data, null, 2)); + else { + console.log( + `${run.subscribers} subscribers on ${run.nodes} nodes, ${run.events} probes: ${run.delivered}/${run.expected} delivered, p50 ${run.overall.p50Ms} ms, p99 ${run.overall.p99Ms} ms, max ${run.overall.maxMs} ms`, + ); + } +} diff --git a/scripts/bench/fanout-node-child.ts b/scripts/bench/fanout-node-child.ts new file mode 100644 index 000000000..b216b2020 --- /dev/null +++ b/scripts/bench/fanout-node-child.ts @@ -0,0 +1,78 @@ +// One `sync` NODE for `fanout-3-nodes.bench.ts`: its own process, its own NATS connection, the +// shipped `ChannelHub` + `SocketRegistry`, and `--subscribers` in-process sockets on one events +// channel topic. Prints `READY` once every socket holds the topic, then `FINAL ` with each +// probe's delivery latencies once `--events` probes arrived (or the window closed). +// +// bun run scripts/bench/fanout-node-child.ts --nats --org --subscribers --events + +import { ChannelHub } from '../../packages/realtime/src/channel'; +import { clearChannels } from '../../packages/realtime/src/channel-registry'; +import { NatsTransport } from '../../packages/realtime/src/nats-transport'; +import { SocketRegistry, SyncSocket, type WsLike } from '../../packages/realtime/src/socket'; +import { parseScriptArgs } from '../lib/args'; +import { ScriptError } from '../lib/script-error'; +import { FANOUT_BUCKET, fanoutChannel, latencyOf, nowMs } from './fanout-shape'; + +const args = parseScriptArgs(Bun.argv.slice(2)); +const nats = args.flags.get('nats'); +const org = args.flags.get('org'); +const subscribers = Number(args.flags.get('subscribers')); +const events = Number(args.flags.get('events')); +if (typeof nats !== 'string' || typeof org !== 'string' || !(subscribers > 0) || !(events > 0)) { + throw new ScriptError({ + code: 'X_CLI_BAD_FLAG', + cause: + 'fanout-node-child is spawned by the bench with --nats, --org, --subscribers and --events', + fix: 'bun run scripts/bench/fanout-3-nodes.bench.ts --subscribers 3000', + }); +} + +/** Per probe index, every socket's latency in ms. Read off the frame text: no decode on the clock. */ +const latencies: number[][] = Array.from({ length: events }, () => []); +let received = 0; +const window = Promise.withResolvers(); + +class ProbeWs implements WsLike { + send(data: string): number { + const probe = latencyOf(data, nowMs()); + if (probe !== undefined) { + latencies[probe.index]?.push(probe.ms); + received += 1; + if (received === subscribers * events) window.resolve(); + } + return data.length; + } + close(): void {} + subscribe(): void {} + unsubscribe(): void {} + getBufferedAmount(): number { + return 0; + } +} + +const channel = fanoutChannel(); +const transport = new NatsTransport({ url: nats, bucket: FANOUT_BUCKET }); +await transport.connect(); +const sockets = new SocketRegistry(); +const hub = new ChannelHub({ transport, sockets }); +for (let i = 0; i < subscribers; i += 1) { + const socket = new SyncSocket({ + ws: new ProbeWs(), + clientBuildId: 'bench', + serverBuildId: 'bench', + actor: null, + }); + sockets.add(socket); + await hub.subscribeChannel(socket, { kind: 'channel', channel: channel.name, params: { org } }); +} +console.log('READY'); + +// The window: every probe the publisher sends plus a generous tail, never forever. +const timeout = setTimeout(() => window.resolve(), 30_000 + events * 2_000); +await window.promise; +clearTimeout(timeout); +console.log(`FINAL ${JSON.stringify({ received, expected: subscribers * events, latencies })}`); +await hub.close(); +await transport.close(); +clearChannels(); +process.exit(0); diff --git a/scripts/bench/fanout-shape.test.ts b/scripts/bench/fanout-shape.test.ts new file mode 100644 index 000000000..8232d3243 --- /dev/null +++ b/scripts/bench/fanout-shape.test.ts @@ -0,0 +1,33 @@ +// The fan-out bench's arithmetic: a probe read off the frame the node REALLY sends (the shipped +// encoder, so a wire change that hides the probe fails here rather than reporting zero deliveries), +// nearest-rank percentiles, and the split of subscribers across nodes. + +import { describe, expect, test } from 'bun:test'; +import { encode, PROTOCOL_VERSION } from '../../packages/realtime/src/sync-protocol'; +import { latencyOf, splitAcross, summarise } from './fanout-shape'; + +describe('fan-out bench shape', () => { + test('a probe is read off an encoded events frame', () => { + const frame = encode({ + type: 'events', + v: PROTOCOL_VERSION, + channel: 'benchfanout:org', + event: { probeIndex: 7, probeSentAt: 1_000.5 }, + }); + expect(latencyOf(frame, 1_004)).toEqual({ index: 7, ms: 3.5 }); + }); + + test('a frame without a probe is not counted', () => { + expect(latencyOf('{"type":"hello"}', 1)).toBeUndefined(); + }); + + test('percentiles are nearest-rank, and empty is zeros', () => { + const samples = Array.from({ length: 100 }, (_, index) => index + 1); + expect(summarise(samples)).toEqual({ deliveries: 100, p50Ms: 50, p99Ms: 99, maxMs: 100 }); + expect(summarise([])).toEqual({ deliveries: 0, p50Ms: 0, p99Ms: 0, maxMs: 0 }); + }); + + test('subscribers split across nodes, whole sockets, summing to the total', () => { + expect(splitAcross(10_000, 3)).toEqual([3_334, 3_333, 3_333]); + }); +}); diff --git a/scripts/bench/fanout-shape.ts b/scripts/bench/fanout-shape.ts new file mode 100644 index 000000000..9828915a3 --- /dev/null +++ b/scripts/bench/fanout-shape.ts @@ -0,0 +1,60 @@ +// The pure half of `fanout-3-nodes.bench.ts`: the channel every process declares, the clock all of +// them read, the probe a frame carries and the percentiles a run reports. Split from the harness +// so the arithmetic is tested without a NATS server. + +import { allow } from '../../packages/policy/src/index'; +import { channel } from '../../packages/realtime/src/channel-decl'; + +/** The presence bucket this bench's transports use — never the `x_presence` an app uses. */ +export const FANOUT_BUCKET = 'xbenchfanout'; + +/** The declaration every process makes: a channel is code, loaded by every role. */ +export const fanoutChannel = () => + channel('benchfanout', { + params: ['org'], + catchUp: { name: 'benchFanoutRead' }, + events: true, + policy: allow('public'), + }); + +/** + * Epoch ms with sub-ms resolution, comparable ACROSS processes on one box: `timeOrigin` is each + * process's wall-clock start and `now()` the monotonic offset from it. Never across machines. + */ +export const nowMs = (): number => performance.timeOrigin + performance.now(); + +const PROBE = /"probeIndex":(\d+),"probeSentAt":([\d.]+)/; + +/** The probe a frame carries and how long it took to reach this socket, or `undefined`. */ +export function latencyOf( + frame: string, + receivedAt: number, +): { index: number; ms: number } | undefined { + const match = PROBE.exec(frame); + if (match === null) return undefined; + return { index: Number(match[1]), ms: receivedAt - Number(match[2]) }; +} + +export interface LatencySummary { + readonly deliveries: number; + readonly p50Ms: number; + readonly p99Ms: number; + readonly maxMs: number; +} + +/** Nearest-rank percentiles, two decimals. An empty list is all zeros, never `NaN`. */ +export function summarise(samples: readonly number[]): LatencySummary { + const sorted = [...samples].sort((a, b) => a - b); + const rank = (p: number): number => { + if (sorted.length === 0) return 0; + const at = Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1)); + return Math.round((sorted[at] ?? 0) * 100) / 100; + }; + return { deliveries: sorted.length, p50Ms: rank(0.5), p99Ms: rank(0.99), maxMs: rank(1) }; +} + +/** Subscribers per node: the total split as evenly as whole sockets allow. */ +export function splitAcross(total: number, nodes: number): readonly number[] { + const base = Math.floor(total / nodes); + return Array.from({ length: nodes }, (_, index) => base + (index < total % nodes ? 1 : 0)); +} diff --git a/scripts/bench/http-prod-image.bench.ts b/scripts/bench/http-prod-image.bench.ts new file mode 100644 index 000000000..294f4d9d0 --- /dev/null +++ b/scripts/bench/http-prod-image.bench.ts @@ -0,0 +1,168 @@ +#!/usr/bin/env bun +// Requests per second against the PRODUCTION image — `x build --target docker`, deployed by +// `x deploy` on the app's own `docker/docker-compose.prod.yml` — at 1 and at N `web` pods. Pod 1 is +// the Compose `web` service; pods 2..N are `docker compose run --no-deps` copies of it (same image, +// env, network and database) each on its own host port, because Compose publishes a port to one +// binder (`17-scale-ladder.md`, rung 2) and a proxy in front would be the proxy's figure. The +// fleet's figure is the SUM of one load loop per pod, run at once from this process. +// +// KEEP_STACK=1 WORK= bash docker/deploy-proof/compose.sh # the app and its stack +// bun run scripts/bench/http-prod-image.bench.ts --app /proofapp [--pods 1,3] +// [--path /] [--seconds 30] [--concurrency 16] [--out ] [--json] +// +// The scaffold's read bucket must be raised before the image is built (bench.yml's step does the +// same for `x dev`): one client past a 120-request burst is a 429, and a 429 is not a figure. + +import { parseScriptArgs } from '../lib/args'; +import { ScriptError } from '../lib/script-error'; +import { boxFacts, MIN_SECONDS, readCount } from './http-hello'; +import { type LoadResult, loadVerdict, runLoad } from './http-hello-load'; + +const BASE_PORT = 3000; +const READY_TIMEOUT_MS = 60_000; +const COMPOSE = [ + 'docker', + 'compose', + '--env-file', + '.env.production', + '-f', + 'docker/docker-compose.prod.yml', +]; + +const failed = (cause: string): ScriptError => + new ScriptError({ + code: 'X_BENCH_RUN_FAILED', + cause, + fix: 'KEEP_STACK=1 bash docker/deploy-proof/compose.sh # then re-run this bench with --app on its proofapp', + }); + +const answerFetch = async (url: string) => { + const response = await fetch(url, { signal: AbortSignal.timeout(10_000) }); + return { status: response.status, bytes: (await response.arrayBuffer()).byteLength }; +}; + +async function ready(port: number): Promise { + const started = performance.now(); + while (performance.now() - started < READY_TIMEOUT_MS) { + const status = await answerFetch(`http://127.0.0.1:${String(port)}/readyz`).then( + (answer) => answer.status, + () => 0, + ); + if (status === 200) return; + await Bun.sleep(250); + } + throw failed(`the web pod on :${String(port)} never answered /readyz 200`); +} + +/** Pods 2..n: the Compose `web` service run again, detached, on the next host ports. */ +function extraPods(app: string, pods: number): readonly string[] { + return Array.from({ length: pods - 1 }, (_, index) => { + const port = BASE_PORT + index + 1; + const run = Bun.spawnSync( + [...COMPOSE, 'run', '-d', '--no-deps', '-p', `127.0.0.1:${String(port)}:3000`, 'web'], + { cwd: app, timeout: 120_000 }, + ); + const id = run.stdout.toString().trim().split('\n').pop() ?? ''; + if (run.exitCode !== 0 || id.length === 0) { + throw failed( + `docker compose run web on :${String(port)} failed: ${run.stderr.toString().slice(0, 300)}`, + ); + } + return id; + }); +} + +export interface PodsRun { + readonly pods: number; + readonly rps: number; + readonly worstP50Ms: number; + readonly worstP99Ms: number; + readonly perPod: readonly LoadResult[]; +} + +export async function benchPods( + app: string, + pods: number, + path: string, + seconds: number, + concurrency: number, +): Promise { + const extra = extraPods(app, pods); + try { + const ports = Array.from({ length: pods }, (_, index) => BASE_PORT + index); + await Promise.all(ports.map(ready)); + const perPod = await Promise.all( + ports.map((port) => + runLoad({ + url: `http://127.0.0.1:${String(port)}${path}`, + concurrency, + durationMs: seconds * 1_000, + fetch: answerFetch, + now: () => performance.now(), + }), + ), + ); + const refused = perPod.flatMap(loadVerdict); + if (refused.length > 0) throw failed(`${String(pods)} pods: ${refused.join('; ')}`); + return { + pods, + rps: Math.round(perPod.reduce((sum, pod) => sum + pod.rps, 0)), + worstP50Ms: Math.max(...perPod.map((pod) => pod.p50Ms)), + worstP99Ms: Math.max(...perPod.map((pod) => pod.p99Ms)), + perPod, + }; + } finally { + if (extra.length > 0) Bun.spawnSync(['docker', 'rm', '-f', ...extra], { timeout: 60_000 }); + } +} + +if (import.meta.main) { + const args = parseScriptArgs(Bun.argv.slice(2)); + const app = args.flags.get('app'); + const rawPods = args.flags.get('pods'); + const podCounts = typeof rawPods === 'string' ? rawPods.split(',').map(Number) : [1, 3]; + const path = args.flags.get('path'); + const seconds = readCount(args.flags.get('seconds'), 30, { + flag: 'seconds', + min: MIN_SECONDS, + max: 600, + }); + const concurrency = readCount(args.flags.get('concurrency'), 16, { + flag: 'concurrency', + min: 1, + max: 256, + }); + if ( + typeof app !== 'string' || + typeof seconds !== 'number' || + typeof concurrency !== 'number' || + !podCounts.every((n) => Number.isInteger(n) && n >= 1 && n <= 8) + ) { + throw new ScriptError({ + code: 'X_CLI_BAD_FLAG', + cause: + 'http-prod-image needs --app (a deployed scaffold), --pods as whole numbers 1..8 and --seconds ≥ 10', + fix: 'bun run scripts/bench/http-prod-image.bench.ts --app /proofapp --pods 1,3', + }); + } + const target = typeof path === 'string' ? path : '/'; + const runs: PodsRun[] = []; + for (const pods of podCounts) runs.push(await benchPods(app, pods, target, seconds, concurrency)); + const data = { + box: boxFacts(Bun.env), + path: target, + seconds, + concurrencyPerPod: concurrency, + runs, + }; + const out = args.flags.get('out'); + if (typeof out === 'string') await Bun.write(out, `${JSON.stringify(data, null, 2)}\n`); + if (args.json) console.log(JSON.stringify(data, null, 2)); + else { + for (const run of runs) { + console.log( + `${run.pods} pod(s): ${run.rps} req/s, worst p50 ${run.worstP50Ms} ms, worst p99 ${run.worstP99Ms} ms`, + ); + } + } +} diff --git a/scripts/bench/results/README.md b/scripts/bench/results/README.md index ea287b245..d1ad73f40 100644 --- a/scripts/bench/results/README.md +++ b/scripts/bench/results/README.md @@ -101,3 +101,45 @@ reads no clock and mints its span id only if something reads it; a root's trace pool filled by one `crypto.getRandomValues` call, each byte handed out once. Before: 1,976 ns, 3,336 ns and 3,712 ns. The regression test holds the COUNTS (clock reads, random draws, dispatches), never a wall-clock figure: [`span-overhead.bench.test.ts`](../span-overhead.bench.test.ts). + +## The scale ladder, measured — `As of 2026-11-09` + +Four benches for [`docs/idea/17-scale-ladder.md`](../../../docs/idea/17-scale-ladder.md) (plan 101 +slice 22). One local run each on a 12-core box (Intel Haswell, 46 GB, Bun 1.4.4, `box.runner: +local`) that eleven other agents were loading at the same time — load average ~11.6 on 12 cores — +so read every figure as one busy machine's order of magnitude, never as a ceiling. Not yet in +`bench.yml`: each needs a service (Postgres, NATS, a Docker daemon) the weekly job does not start. + +**Worker throughput** ([`worker-throughput.bench.ts`](../worker-throughput.bench.ts), +[`worker-throughput.json`](worker-throughput.json)): no-op jobs on the test Postgres, each worker +its own `bun` process running the stock `jobWorker`, elapsed on the server's clock from the first +enqueue to the last ack. With the shipped defaults (8 slots, a pass every 250 ms): 799 claims/s at +1 worker, 1,618 at 4, 2,277 at 16 (As of 2026-11-09). Until that day the defaults read 31 / 124 / +492 — exactly 8 jobs a pass, 4 passes a second: a pass that found work waited the whole floor and a +freed slot could not cut it short. A pass whose every claimed row STARTED now marks a backlog, and the next +freed slot claims at once (`worker-loop.ts`); the floor is the idle poll only. With the tuned +profile (32 slots, a pass every 10 ms): 1,709 claims/s at 1 worker, 3,724 at 4, 3,042 at 16 — the +queue, one Postgres on a shared box, stops scaling between 4 and 16 +workers. That plateau is the figure to compare against a Redis or NATS queue, not the defaults. + +**Cross-node fan-out** ([`fanout-3-nodes.bench.ts`](../fanout-3-nodes.bench.ts), +[`fanout-3-nodes.json`](fanout-3-nodes.json)): **10,000 subscribers across 3 `sync` nodes**, each +node its own process on its own connection to a nats-server 2.15, one events-channel topic, a +fourth process publishing 20 probes 250 ms apart: **200,000 of 200,000 deliveries**, p50 4.32 ms, p99 23.26 ms, max 31.45 ms, publish to the socket's `send`. The sockets are in-process +`WsLike`s, not WebSockets: this times the bus and the node's fan-out, and the restart bench above +is the real-socket figure. 10,000 is the plan's count, not this box's ceiling. + +**Module load against file count** ([`boot-vs-files.bench.ts`](../boot-vs-files.bench.ts), +[`boot-vs-files.json`](boot-vs-files.json)): a synthetic app of `entity()` modules, every tenth +importing the one before, imported cold in a fresh process, median of 3. 500 modules: serial 804 +ms, batched 393 ms. 5,000 modules: serial 3,258 ms, batched 1,259 ms — `importInOrder`'s 256-module +batches against one `await import()` per file. Entity files only: the plan's 5,073-file app also +carried routes, islands and actions, and measured ~5 ms a file serially. + +**The production image** ([`http-prod-image.bench.ts`](../http-prod-image.bench.ts), +[`http-prod-image.json`](http-prod-image.json)): a fresh scaffold built by `x build --target +docker` and deployed by `x deploy` on its own `docker-compose.prod.yml` +(`docker/deploy-proof/compose.sh` with `KEEP_STACK=1`), the read bucket raised as `bench.yml` does, +`/` for 30 s at concurrency 16 per pod, every answer 2xx. One `web` pod: **1,439 req/s**, p50 9.4 ms, p99 38.09 ms — 5.4× the 267 req/s `x dev` served above. Three pods on the same host: 1,196 req/s in sum, p99 203.89 ms: on one saturated box three pods and the load generator +share the same cores, so this is **not** a horizontal-scaling figure, and none is claimed. A clean +1-vs-3 comparison needs pods on separate hosts. diff --git a/scripts/bench/results/boot-vs-files.json b/scripts/bench/results/boot-vs-files.json new file mode 100644 index 000000000..acff6a84d --- /dev/null +++ b/scripts/bench/results/boot-vs-files.json @@ -0,0 +1,34 @@ +{ + "box": { + "os": "linux-x64", + "cpu": "Intel Core Processor (Haswell, no TSX)", + "cores": 12, + "memoryGb": 46, + "bun": "1.4.4", + "runner": "local" + }, + "runs": 3, + "results": [ + { + "files": 500, + "serialMs": 804, + "batchedMs": 393, + "serialMsPer1k": 1608, + "batchedMsPer1k": 786 + }, + { + "files": 2000, + "serialMs": 1133, + "batchedMs": 645, + "serialMsPer1k": 567, + "batchedMsPer1k": 323 + }, + { + "files": 5000, + "serialMs": 3258, + "batchedMs": 1259, + "serialMsPer1k": 652, + "batchedMsPer1k": 252 + } + ] +} diff --git a/scripts/bench/results/fanout-3-nodes.json b/scripts/bench/results/fanout-3-nodes.json new file mode 100644 index 000000000..6cd844555 --- /dev/null +++ b/scripts/bench/results/fanout-3-nodes.json @@ -0,0 +1,44 @@ +{ + "box": { + "os": "linux-x64", + "cpu": "Intel Core Processor (Haswell, no TSX)", + "cores": 12, + "memoryGb": 46, + "bun": "1.4.4", + "runner": "local" + }, + "intervalMs": 250, + "run": { + "nodes": 3, + "subscribers": 10000, + "events": 20, + "delivered": 200000, + "expected": 200000, + "overall": { + "deliveries": 200000, + "p50Ms": 4.32, + "p99Ms": 23.26, + "maxMs": 31.45 + }, + "perNode": [ + { + "deliveries": 66680, + "p50Ms": 4.04, + "p99Ms": 31.1, + "maxMs": 31.45 + }, + { + "deliveries": 66660, + "p50Ms": 4.88, + "p99Ms": 22.5, + "maxMs": 22.96 + }, + { + "deliveries": 66660, + "p50Ms": 4.17, + "p99Ms": 23.11, + "maxMs": 23.66 + } + ] + } +} diff --git a/scripts/bench/results/http-prod-image.json b/scripts/bench/results/http-prod-image.json new file mode 100644 index 000000000..a2121af32 --- /dev/null +++ b/scripts/bench/results/http-prod-image.json @@ -0,0 +1,87 @@ +{ + "box": { + "os": "linux-x64", + "cpu": "Intel Core Processor (Haswell, no TSX)", + "cores": 12, + "memoryGb": 46, + "bun": "1.4.4", + "runner": "local" + }, + "path": "/", + "seconds": 30, + "concurrencyPerPod": 16, + "runs": [ + { + "pods": 1, + "rps": 1439, + "worstP50Ms": 9.4, + "worstP99Ms": 38.09, + "perPod": [ + { + "requests": 43176, + "ok": 43176, + "non2xx": 0, + "errors": 0, + "statuses": { + "200": 43176 + }, + "durationMs": 30006.77, + "rps": 1438.88, + "p50Ms": 9.4, + "p99Ms": 38.09, + "bytes": 171970008 + } + ] + }, + { + "pods": 3, + "rps": 1196, + "worstP50Ms": 27.46, + "worstP99Ms": 203.89, + "perPod": [ + { + "requests": 11929, + "ok": 11929, + "non2xx": 0, + "errors": 0, + "statuses": { + "200": 11929 + }, + "durationMs": 30027.74, + "rps": 397.27, + "p50Ms": 26.78, + "p99Ms": 203.89, + "bytes": 47513207 + }, + { + "requests": 12025, + "ok": 12025, + "non2xx": 0, + "errors": 0, + "statuses": { + "200": 12025 + }, + "durationMs": 30033.94, + "rps": 400.38, + "p50Ms": 26.63, + "p99Ms": 194.21, + "bytes": 47895575 + }, + { + "requests": 11977, + "ok": 11977, + "non2xx": 0, + "errors": 0, + "statuses": { + "200": 11977 + }, + "durationMs": 30030.26, + "rps": 398.83, + "p50Ms": 27.46, + "p99Ms": 187.77, + "bytes": 47704391 + } + ] + } + ] +} diff --git a/scripts/bench/results/worker-throughput.json b/scripts/bench/results/worker-throughput.json new file mode 100644 index 000000000..f0269219a --- /dev/null +++ b/scripts/bench/results/worker-throughput.json @@ -0,0 +1,70 @@ +{ + "box": { + "os": "linux-x64", + "cpu": "Intel Core Processor (Haswell, no TSX)", + "cores": 12, + "memoryGb": 46, + "bun": "1.4.4", + "runner": "local" + }, + "profiles": [ + { + "name": "shipped", + "concurrency": 8, + "pollIntervalMs": 250, + "jobsPerWorker": 400, + "runs": [ + { + "workers": 1, + "jobs": 1000, + "elapsedMs": 1252, + "claimsPerSecond": 799, + "perWorker": 799 + }, + { + "workers": 4, + "jobs": 1600, + "elapsedMs": 989, + "claimsPerSecond": 1618, + "perWorker": 405 + }, + { + "workers": 16, + "jobs": 6400, + "elapsedMs": 2811, + "claimsPerSecond": 2277, + "perWorker": 142 + } + ] + }, + { + "name": "tuned", + "concurrency": 32, + "pollIntervalMs": 10, + "jobsPerWorker": 2000, + "runs": [ + { + "workers": 1, + "jobs": 2000, + "elapsedMs": 1170, + "claimsPerSecond": 1709, + "perWorker": 1709 + }, + { + "workers": 4, + "jobs": 8000, + "elapsedMs": 2148, + "claimsPerSecond": 3724, + "perWorker": 931 + }, + { + "workers": 16, + "jobs": 32000, + "elapsedMs": 10521, + "claimsPerSecond": 3042, + "perWorker": 190 + } + ] + } + ] +} diff --git a/scripts/bench/scale-claims.ts b/scripts/bench/scale-claims.ts new file mode 100644 index 000000000..df1c7bbf1 --- /dev/null +++ b/scripts/bench/scale-claims.ts @@ -0,0 +1,143 @@ +// The scale ladder's bench claims (plan 101 slice 22): worker throughput, cross-node fan-out, boot +// against file count and the production image's req/s — each figure stated on the claims page and +// the committed result it must equal. Data only; `scripts/lib/bench-claims-table.ts` spreads it +// into `CLAIMS`, so `bun run scripts/bench-claims.ts` holds these like every other figure. + +import type { BenchClaim } from '../lib/bench-claims-table'; + +export const RESULTS_WORKERS = 'scripts/bench/results/worker-throughput.json'; +export const RESULTS_FANOUT_3 = 'scripts/bench/results/fanout-3-nodes.json'; +export const RESULTS_BOOT = 'scripts/bench/results/boot-vs-files.json'; +export const RESULTS_HTTP_IMAGE = 'scripts/bench/results/http-prod-image.json'; +export const SCALE_LADDER = 'docs/idea/17-scale-ladder.md'; + +const workers = (profile: 0 | 1, run: 0 | 1 | 2, label: string, pattern: RegExp): BenchClaim => ({ + label, + file: RESULTS_WORKERS, + path: `profiles.${String(profile)}.runs.${String(run)}.claimsPerSecond`, + format: 'count', + pattern, +}); + +export const SCALE_CLAIMS: readonly BenchClaim[] = [ + workers( + 0, + 0, + 'shipped-default claims/s at 1 worker', + /defaults[^:]*: ([\d,]+) claims\/s at 1 worker/, + ), + workers( + 0, + 1, + 'shipped-default claims/s at 4 workers', + /defaults[^:]*: [\d,]+ claims\/s at 1 worker, ([\d,]+) at 4/, + ), + workers( + 0, + 2, + 'shipped-default claims/s at 16 workers', + /defaults[^:]*: [\d,]+ claims\/s at 1 worker, [\d,]+ at 4, ([\d,]+) at 16/, + ), + workers(1, 0, 'tuned claims/s at 1 worker', /tuned[^:]*: ([\d,]+) claims\/s at 1 worker/), + workers( + 1, + 1, + 'tuned claims/s at 4 workers', + /tuned[^:]*: [\d,]+ claims\/s at 1 worker, ([\d,]+) at 4/, + ), + workers( + 1, + 2, + 'tuned claims/s at 16 workers', + /tuned[^:]*: [\d,]+ claims\/s at 1 worker, [\d,]+ at 4, ([\d,]+) at 16/, + ), + { + label: 'fan-out subscribers across three nodes', + file: RESULTS_FANOUT_3, + path: 'run.subscribers', + format: 'count', + pattern: /\*\*([\d,]+) subscribers across 3 `sync` nodes/, + }, + { + label: 'fan-out deliveries', + file: RESULTS_FANOUT_3, + path: 'run.delivered', + format: 'count', + pattern: /apart: \*\*([\d,]+) of [\d,]+ deliveries/, + }, + { + label: 'fan-out p50', + file: RESULTS_FANOUT_3, + path: 'run.overall.p50Ms', + format: 'plain', + pattern: /deliveries\*\*, p50 ([\d.]+) ms/, + }, + { + label: 'fan-out p99', + file: RESULTS_FANOUT_3, + path: 'run.overall.p99Ms', + format: 'plain', + pattern: /deliveries\*\*, p50 [\d.]+ ms, p99 ([\d.]+) ms/, + }, + { + label: 'serial module load at 5,000 files', + file: RESULTS_BOOT, + path: 'results.2.serialMs', + format: 'count', + pattern: /5,000 modules: serial ([\d,]+) ms/, + }, + { + label: 'batched module load at 5,000 files', + file: RESULTS_BOOT, + path: 'results.2.batchedMs', + format: 'count', + pattern: /5,000 modules: serial [\d,]+ ms, batched ([\d,]+) ms/, + }, + { + label: 'production image req/s, one web pod', + file: RESULTS_HTTP_IMAGE, + path: 'runs.0.rps', + format: 'count', + pattern: /One `web` pod: \*\*([\d,]+) req\/s\*\*/, + }, + { + label: 'production image p99, one web pod', + file: RESULTS_HTTP_IMAGE, + path: 'runs.0.worstP99Ms', + format: 'plain', + pattern: /req\/s\*\*, p50 [\d.]+ ms, p99 ([\d.]+) ms/, + }, + { + label: 'production image req/s, three web pods', + file: RESULTS_HTTP_IMAGE, + path: 'runs.1.rps', + format: 'count', + pattern: /Three pods on the same host: ([\d,]+) req\/s/, + }, + // The ladder page cites the headline of each, so a re-run that moves one cannot leave the rung + // describing a run that never happened. + { + label: 'the ladder’s fan-out p99', + doc: SCALE_LADDER, + file: RESULTS_FANOUT_3, + path: 'run.overall.p99Ms', + format: 'plain', + pattern: /3 `sync` nodes over NATS: p99 ([\d.]+) ms/, + }, + { + label: 'the ladder’s tuned 4-worker claims/s', + doc: SCALE_LADDER, + file: RESULTS_WORKERS, + path: 'profiles.1.runs.1.claimsPerSecond', + format: 'count', + pattern: /the queue itself plateaus near ([\d,]+) claims\/s/, + }, + { + label: 'the ladder’s one-pod req/s', + doc: SCALE_LADDER, + file: RESULTS_HTTP_IMAGE, + path: 'runs.0.rps', + format: 'count', + pattern: /production image, one `web` pod: ([\d,]+) req\/s/, + }, +]; diff --git a/scripts/bench/worker-throughput-child.ts b/scripts/bench/worker-throughput-child.ts new file mode 100644 index 000000000..044cb22e6 --- /dev/null +++ b/scripts/bench/worker-throughput-child.ts @@ -0,0 +1,67 @@ +// One worker POD for `worker-throughput.bench.ts`: its own process, its own pool, the shipped +// `jobWorker` over `postgresJobDriver` (slots and poll floor from the bench's profile), a no-op job. Prints `ready` once it claims, and drains on +// SIGTERM. Never run by hand; the bench spawns one per worker it measures. +// +// bun run scripts/bench/worker-throughput-child.ts --url --concurrency --poll-ms + +import { ctxOf } from '../../packages/core/src/index'; +import { postgresClient } from '../../packages/db/src/index'; +import { postgresJobDriver } from '../../packages/jobs/src/driver-pg'; +import { job } from '../../packages/jobs/src/job'; +import { jobWorker } from '../../packages/jobs/src/worker'; +import { parseScriptArgs } from '../lib/args'; +import { ScriptError } from '../lib/script-error'; +import { benchJobInput, NOOP_JOB } from './worker-throughput-shape'; + +const args = parseScriptArgs(Bun.argv.slice(2)); +const url = args.flags.get('url'); +const concurrency = Number(args.flags.get('concurrency')); +const pollIntervalMs = Number(args.flags.get('poll-ms')); +if (typeof url !== 'string' || !(concurrency >= 1) || !(pollIntervalMs >= 1)) { + throw new ScriptError({ + code: 'X_CLI_BAD_FLAG', + cause: + 'worker-throughput-child is spawned by the bench with --url, --concurrency and --poll-ms, and was started without them', + fix: 'bun run scripts/bench/worker-throughput.bench.ts --workers 1', + }); +} + +// Named, so the worker resolves it from the registry `job()` writes to — the bench has no app. +job<{ n: number }>({ + name: NOOP_JOB, + tenant: 'none', + input: benchJobInput, + idempotencyKey: (input) => String(input.n), + retry: { attempts: 1 }, + run: () => Promise.resolve(), +}); + +const client = postgresClient({ url, role: 'worker' }); +const worker = jobWorker({ + driver: postgresJobDriver({ + executor: { + query: (text: string, values: readonly unknown[]): Promise => + client.query({ text, values }), + }, + }), + context: () => ctxOf({ role: 'worker', buildId: 'bench' }), + concurrency, + pollIntervalMs, + // An idle worker re-polls every 250 ms at most rather than backing off to 2 s, so the first claim + // after the batch lands is not a timer the bench would be measuring instead. + idlePollMaxMs: Math.max(pollIntervalMs, 250), + drainOnShutdown: false, +}); +worker.start(); +// The worker's timers are unrefed (a drained process must be free to exit), so a bare script would +// exit right here: this interval is the pod staying up until SIGTERM. +const alive = setInterval(() => undefined, 60_000); +console.log('ready'); + +process.on('SIGTERM', () => { + clearInterval(alive); + void worker + .stop() + .then(() => client.close()) + .finally(() => process.exit(0)); +}); diff --git a/scripts/bench/worker-throughput-shape.test.ts b/scripts/bench/worker-throughput-shape.test.ts new file mode 100644 index 000000000..ecb92565a --- /dev/null +++ b/scripts/bench/worker-throughput-shape.test.ts @@ -0,0 +1,46 @@ +// The worker bench's pure half: the worker counts a run accepts, claims per second, and the input +// schema both processes agree on. + +import { describe, expect, test } from 'bun:test'; +import { + benchJobInput, + DEFAULT_WORKER_COUNTS, + parseWorkerCounts, + throughputOf, + WORKER_PROFILES, +} from './worker-throughput-shape'; + +describe('worker throughput shape', () => { + test('the plan’s counts by default; whole numbers 1..64 only', () => { + expect(parseWorkerCounts(undefined)).toEqual(DEFAULT_WORKER_COUNTS); + expect(parseWorkerCounts('1, 8')).toEqual([1, 8]); + expect(parseWorkerCounts('0')).toBeUndefined(); + expect(parseWorkerCounts('2.5')).toBeUndefined(); + expect(parseWorkerCounts(true)).toBeUndefined(); + }); + + test('claims per second from jobs and server-side elapsed ms', () => { + expect(throughputOf(4, 2_000, 1_000)).toEqual({ + workers: 4, + jobs: 2_000, + elapsedMs: 1_000, + claimsPerSecond: 2_000, + perWorker: 500, + }); + expect(throughputOf(1, 10, 0).claimsPerSecond).toBe(0); + }); + + test('the shipped profile is jobWorker’s defaults: 8 slots, 250 ms', () => { + expect(WORKER_PROFILES[0]).toMatchObject({ + name: 'shipped', + concurrency: 8, + pollIntervalMs: 250, + }); + }); + + test('the job input is checked, never cast', async () => { + const validate = benchJobInput['~standard'].validate; + expect(await validate({ n: 3 })).toEqual({ value: { n: 3 } }); + expect('issues' in (await validate({ n: '3' }))).toBe(true); + }); +}); diff --git a/scripts/bench/worker-throughput-shape.ts b/scripts/bench/worker-throughput-shape.ts new file mode 100644 index 000000000..aa2335040 --- /dev/null +++ b/scripts/bench/worker-throughput-shape.ts @@ -0,0 +1,78 @@ +// The pure half of `worker-throughput.bench.ts`: the job both processes agree on, the worker counts +// a run may ask for, and claims per second from the server's own timestamps. Split from the +// harness so the arithmetic is tested without a database. + +import type { StandardSchemaV1 } from '../../packages/schema/src/index'; + +/** The one job the bench queues: no body, so a claim and its ack are all that is timed. */ +export const NOOP_JOB = 'benchNoop'; + +/** `{ n }`, checked rather than cast — the input crosses a process boundary as JSON. */ +export const benchJobInput: StandardSchemaV1 = { + '~standard': { + version: 1, + vendor: 'ultimate-bench', + validate: (value: unknown) => { + const n = + typeof value === 'object' && value !== null ? (value as { n?: unknown }).n : undefined; + return typeof n === 'number' && Number.isInteger(n) + ? { value: { n } } + : { issues: [{ message: 'expected { n: integer }' }] }; + }, + }, +}; + +/** + * How a worker process is configured for a run. `shipped` is `jobWorker`'s defaults — 8 slots, + * a pass every 250 ms while there is work — and on no-op jobs that IS the ceiling: 8 per pass, 4 + * passes a second. `tuned` raises both, to find where the QUEUE (Postgres) stops scaling instead. + */ +export interface WorkerProfile { + readonly name: 'shipped' | 'tuned'; + readonly concurrency: number; + readonly pollIntervalMs: number; + /** Jobs queued per worker process, so every run lasts seconds rather than minutes or ms. */ + readonly jobsPerWorker: number; +} + +export const WORKER_PROFILES: readonly WorkerProfile[] = [ + { name: 'shipped', concurrency: 8, pollIntervalMs: 250, jobsPerWorker: 400 }, + { name: 'tuned', concurrency: 32, pollIntervalMs: 10, jobsPerWorker: 2_000 }, +]; + +/** The worker counts the plan names: one pod, four, sixteen. */ +export const DEFAULT_WORKER_COUNTS: readonly number[] = [1, 4, 16]; + +/** `'1,4,16'` -> `[1, 4, 16]`; anything but whole numbers from 1 to 64 is `undefined`. */ +export function parseWorkerCounts( + raw: string | boolean | undefined, +): readonly number[] | undefined { + if (raw === undefined) return DEFAULT_WORKER_COUNTS; + if (typeof raw !== 'string') return undefined; + const counts = raw.split(',').map((part) => Number(part.trim())); + return counts.length > 0 && counts.every((n) => Number.isInteger(n) && n >= 1 && n <= 64) + ? counts + : undefined; +} + +export interface ThroughputRun { + readonly workers: number; + readonly jobs: number; + /** Server time from the first enqueue to the last ack, ms. */ + readonly elapsedMs: number; + readonly claimsPerSecond: number; + /** Throughput per worker process: how far from linear the fleet scales. */ + readonly perWorker: number; +} + +/** One run's figure. Whole claims per second: a fraction of a claim is noise this bench cannot see. */ +export function throughputOf(workers: number, jobs: number, elapsedMs: number): ThroughputRun { + const claimsPerSecond = elapsedMs > 0 ? Math.round((jobs * 1_000) / elapsedMs) : 0; + return { + workers, + jobs, + elapsedMs: Math.round(elapsedMs), + claimsPerSecond, + perWorker: Math.round(claimsPerSecond / workers), + }; +} diff --git a/scripts/bench/worker-throughput.bench.ts b/scripts/bench/worker-throughput.bench.ts new file mode 100644 index 000000000..31bdc453a --- /dev/null +++ b/scripts/bench/worker-throughput.bench.ts @@ -0,0 +1,196 @@ +#!/usr/bin/env bun +// Claims per second the shipped Postgres queue sustains at 1, 4 and 16 worker PROCESSES — each its +// own `bun` process, its own pool, the stock `jobWorker` — over no-op jobs, so what is timed is +// claim + ack and nothing an app's handler adds. Two profiles (`WORKER_PROFILES`): the defaults, +// whose poll floor is the ceiling, and a tuned one that finds the queue's. Elapsed is the SERVER's: from +// `now()` before the first enqueue to the last ack's `updated_at`. A fresh probe database per run. +// +// bun run scripts/bench/worker-throughput.bench.ts [--url postgres://…] [--workers 1,4,16] +// [--out ] [--json] +// +// `--url` defaults to `TEST_DATABASE_URL` (`docker/test-services.env`). + +import { probeDatabaseName } from '../../packages/core/src/index'; +import type { PostgresClient } from '../../packages/db/src/index'; +import { postgresClient, raw } from '../../packages/db/src/index'; +import { postgresJobDriver } from '../../packages/jobs/src/driver-pg'; +import { SQL_JOBS_TABLE } from '../../packages/jobs/src/driver-pg-sql'; +import { parseScriptArgs } from '../lib/args'; +import { ScriptError } from '../lib/script-error'; +import { boxFacts } from './http-hello'; +import { + NOOP_JOB, + parseWorkerCounts, + type ThroughputRun, + throughputOf, + WORKER_PROFILES, + type WorkerProfile, +} from './worker-throughput-shape'; + +const CHILD = Bun.fileURLToPath(new URL('./worker-throughput-child.ts', import.meta.url)); +const RUN_TIMEOUT_MS = 300_000; + +const failed = (cause: string): ScriptError => + new ScriptError({ + code: 'X_BENCH_RUN_FAILED', + cause, + fix: 'bun run scripts/bench/worker-throughput.bench.ts --workers 1', + }); + +const dbUrl = (base: string, name: string): string => { + const parsed = new URL(base); + parsed.pathname = `/${name}`; + return parsed.toString(); +}; + +/** Spawn `count` worker processes and resolve once every one has printed `ready`. */ +async function spawnWorkers( + count: number, + url: string, + profile: WorkerProfile, +): Promise { + const argv = [ + '--url', + url, + '--concurrency', + String(profile.concurrency), + '--poll-ms', + String(profile.pollIntervalMs), + ]; + const children = Array.from({ length: count }, () => + Bun.spawn(['bun', CHILD, ...argv], { + stdout: 'pipe', + stderr: 'inherit', + // Every run SIGTERMs its workers; this ends one a hung run left behind. + timeout: RUN_TIMEOUT_MS + 60_000, + }), + ); + await Promise.all( + children.map(async (child) => { + const reader = (child.stdout as ReadableStream).getReader(); + let seen = ''; + while (!seen.includes('ready')) { + const { value, done } = await reader.read(); + if (done) throw failed('a worker process exited before it printed ready'); + seen += new TextDecoder().decode(value); + } + // Keep draining: a worker logs a line per job, and a full pipe would stall it mid-run — the + // bench would then be timing its own back-pressure. + void (async () => { + while (!(await reader.read()).done) {} + })(); + }), + ); + return children; +} + +async function oneRun( + admin: PostgresClient, + base: string, + workers: number, + profile: WorkerProfile, +): Promise { + const jobs = Math.max(1_000, profile.jobsPerWorker * workers); + const name = probeDatabaseName('x_bench_workers'); + await admin.execute(raw(`create database ${name}`)); + const url = dbUrl(base, name); + const client = postgresClient({ url, role: 'web', profile: { max: 2 } }); + const children: Bun.Subprocess[] = []; + try { + for (const statement of SQL_JOBS_TABLE.split(';')) { + if (statement.trim().length > 0) await client.execute(raw(statement)); + } + children.push(...(await spawnWorkers(workers, url, profile))); + const driver = postgresJobDriver({ + executor: { + query: (text: string, values: readonly unknown[]): Promise => + client.query({ text, values }), + }, + }); + const [start] = await client.query<{ at: string }>({ + text: 'select (extract(epoch from clock_timestamp()) * 1000)::bigint as at', + values: [], + }); + const requests = Array.from({ length: jobs }, (_, n) => ({ + name: NOOP_JOB, + queue: 'default', + input: { n }, + idempotencyKey: String(n), + maxAttempts: 1, + runAt: 1, + })); + for (let at = 0; at < requests.length; at += 1_000) { + await driver.enqueueMany(requests.slice(at, at + 1_000)); + } + const deadline = performance.now() + RUN_TIMEOUT_MS; + for (;;) { + const [row] = await client.query<{ done: number; last: string | null }>({ + text: `select count(*)::int as done, + (extract(epoch from max(updated_at)) * 1000)::bigint as last + from x_jobs where state = 'done'`, + values: [], + }); + if ((row?.done ?? 0) >= jobs) { + return throughputOf(workers, jobs, Number(row?.last) - Number(start?.at)); + } + if (performance.now() > deadline) throw failed(`${workers} workers never drained the queue`); + await Bun.sleep(50); + } + } finally { + for (const child of children) child.kill('SIGTERM'); + await Promise.all(children.map((child) => child.exited)); + await client.close(); + await admin.execute(raw(`drop database if exists ${name} with (force)`)); + } +} + +export interface ProfileRuns extends WorkerProfile { + readonly runs: readonly ThroughputRun[]; +} + +export async function benchWorkers( + base: string, + counts: readonly number[], + profiles: readonly WorkerProfile[] = WORKER_PROFILES, +): Promise { + const admin = postgresClient({ url: base, role: 'web', profile: { max: 1 } }); + try { + const out: ProfileRuns[] = []; + for (const profile of profiles) { + const runs: ThroughputRun[] = []; + for (const workers of counts) runs.push(await oneRun(admin, base, workers, profile)); + out.push({ ...profile, runs }); + } + return out; + } finally { + await admin.close(); + } +} + +if (import.meta.main) { + const args = parseScriptArgs(Bun.argv.slice(2)); + const base = args.flags.get('url') ?? Bun.env['TEST_DATABASE_URL']; + const counts = parseWorkerCounts(args.flags.get('workers')); + if (typeof base !== 'string' || counts === undefined) { + throw new ScriptError({ + code: 'X_CLI_BAD_FLAG', + cause: + 'worker-throughput needs a Postgres url (--url or TEST_DATABASE_URL), --workers as whole numbers 1..64', + fix: 'bun run scripts/bench/worker-throughput.bench.ts --url postgres://ultimate:ultimate@127.0.0.1:5432/postgres --workers 1,4,16', + }); + } + const profiles = await benchWorkers(base, counts); + const data = { box: boxFacts(Bun.env), profiles }; + const out = args.flags.get('out'); + if (typeof out === 'string') await Bun.write(out, `${JSON.stringify(data, null, 2)}\n`); + if (args.json) console.log(JSON.stringify(data, null, 2)); + else { + for (const profile of profiles) { + for (const run of profile.runs) { + console.log( + `${profile.name}, ${run.workers} workers: ${run.claimsPerSecond} claims/s (${run.perWorker}/worker, ${run.jobs} jobs in ${run.elapsedMs} ms)`, + ); + } + } + } +} diff --git a/scripts/ci-workflow-shape.test.ts b/scripts/ci-workflow-shape.test.ts index ba1c9f47f..8ca2f7ab4 100644 --- a/scripts/ci-workflow-shape.test.ts +++ b/scripts/ci-workflow-shape.test.ts @@ -280,11 +280,16 @@ describe('unit · ci.yml · scaffold-smoke opens the app it scaffolded', () => { // A shape `x new` accepts is one a stranger can pick, so it gates on a runner of its own: a // name in `NEW_SHAPES` with no row here is a scaffold nothing has ever run `bun run verify` on. - test('the matrix is the default, --no-example, and one row per x new --shape', () => { - const rows = listOf(job('scaffold-smoke').strategy?.matrix?.['include']).map((row) => - text(record(row)['flags']), - ); - expect(rows).toEqual(['', '--no-example', ...NEW_SHAPES.map((shape) => `--shape ${shape}`)]); + test('the matrix is the default, --no-example, one row per x new --shape, and the monorepo', () => { + const include = listOf(job('scaffold-smoke').strategy?.matrix?.['include']).map(record); + expect(include.map((row) => text(row['flags']))).toEqual([ + '', + '--no-example', + ...NEW_SHAPES.map((shape) => `--shape ${shape}`), + // `multiapp`: the default scaffold grown by `x g app` and `x g package` (plan 101, 22 M1). + '', + ]); + expect(text(include.at(-1)?.['app'])).toBe('multiapp'); }); }); diff --git a/scripts/ci-workflow-windows.test.ts b/scripts/ci-workflow-windows.test.ts index fde9d6b65..b893084c4 100644 --- a/scripts/ci-workflow-windows.test.ts +++ b/scripts/ci-workflow-windows.test.ts @@ -27,7 +27,7 @@ describe('unit · ci.yml · the windows job runs in PowerShell, from a default-a './packages/cli/src/app-boundaries.test.ts', './packages/cli/src/path-segments.test.ts', './packages/cli/src/templates/scaffold-gitattributes.test.ts', - './packages/testing/src/cdp-launch.test.ts', + './packages/scraping/src/cdp-launch.test.ts', ]; const windows = job('windows'); const steps = windows.steps ?? []; diff --git a/scripts/error-map-backlog.ts b/scripts/error-map-backlog.ts index 6b73e0536..f44b85963 100644 --- a/scripts/error-map-backlog.ts +++ b/scripts/error-map-backlog.ts @@ -75,6 +75,7 @@ export const OFF_SOCKET: Pins = { 'X_MIGRATION_ROLLBACK_DESTRUCTIVE', 'X_ENTITY_RETAIN_INVALID', 'X_LIFECYCLE_TRIGGER_MISSING', + 'X_MIGRATION_PARTITION_CHANGE', ], // tier 1 — a flag declared twice, at registration. flags: ['X_FLAG_DUPLICATE'], @@ -111,6 +112,7 @@ export const OFF_SOCKET: Pins = { 'X_ENTITY_RETAIN_UNINDEXED', 'X_ENTITY_COUNTS_INVALID', 'X_DRIVER_CAPABILITY_MISSING', + 'X_ENTITY_PARTITION_INVALID', ], // tier 2 — a role declared twice at registration. policy: ['X_ROLE_REDEFINED'], @@ -154,6 +156,8 @@ export const OFF_SOCKET: Pins = { 'X_SUBJECT_TABLE_UNDECLARED', 'X_SUBJECT_TREATMENT_INVALID', 'X_SUBJECT_CYCLE', + 'X_PERSONAL_DATA_UNERASED', + 'X_PERSONAL_DATA_UNREACHED', ], // tier 3 — judged at `query()`, where the declaration is written, or on a live read: a // WebSocket subscription carrying a `kind`, not a request carrying a status (`live.ts`). @@ -319,6 +323,8 @@ export const OFF_SOCKET: Pins = { 'X_NOTIFY_PUSH_DEVICE_GONE', 'X_NOTIFY_PUSH_PROVIDER_REFUSED', 'X_NOTIFY_PUSH_MESSAGE_INVALID', + 'X_NOTIFY_SMS_TRANSPORT_MISSING', + 'X_NOTIFY_SMS_NUMBER_INVALID', ], }; diff --git a/scripts/identity-route-status.test.ts b/scripts/identity-route-status.test.ts new file mode 100644 index 000000000..a05011ba1 --- /dev/null +++ b/scripts/identity-route-status.test.ts @@ -0,0 +1,26 @@ +// The pin under the codes only the SSO routes throw: `oauth-route-status.test.ts` holds every ROW of +// `OAUTH_ROUTE_STATUS` equal to http's `statusFor`, but a missing row answers the descriptor's 502 +// standalone and its real status behind the pipeline. This asks that every SSO code IS a row, and +// agrees. Imports both tier-2 packages, which neither may do itself. + +import { describe, expect, test } from 'bun:test'; +import { OAUTH_ROUTE_STATUS } from '@ultimat3/auth'; +import { statusFor } from '@ultimat3/http'; + +/** Every code `identityProviderRoutes` and the SAML adapter raise that OAuth never did. */ +const SSO_CODES = [ + 'X_SAML_RESPONSE_INVALID', + 'X_SAML_METADATA_INVALID', + 'X_IDENTITY_PROVIDER_UNKNOWN', +] as const; + +describe('the sign-in descriptors and the HTTP status table, for the SSO codes', () => { + test('every SSO code is a row of the descriptors table', () => { + expect(SSO_CODES.filter((code) => !Object.hasOwn(OAUTH_ROUTE_STATUS, code))).toEqual([]); + }); + + test('and each row answers what statusFor answers', () => { + const rows = SSO_CODES.map((code) => [code, OAUTH_ROUTE_STATUS[code], statusFor(code)]); + expect(rows.filter(([, row, http]) => row !== http)).toEqual([]); + }); +}); diff --git a/scripts/identity-routes-csrf.test.ts b/scripts/identity-routes-csrf.test.ts new file mode 100644 index 000000000..327a6a0be --- /dev/null +++ b/scripts/identity-routes-csrf.test.ts @@ -0,0 +1,146 @@ +// The sign-in routes behind the REAL `@ultimat3/http` pipeline. A SAML IdP answers with an +// auto-submitted cross-site form, which the csrf stage refuses on any route that does not declare +// otherwise — every SAML login was a `403 X_CSRF_BLOCKED` once mounted. auth and http are both +// tier 2 and cannot import each other; this file can import both. + +import { describe, expect, test } from 'bun:test'; +import { + defineAuth, + identityProviderRoutes, + memoryAuthAdapter, + oidcIdentityProvider, + samlIdentityProvider, +} from '@ultimat3/auth'; +import { frozenClock } from '@ultimat3/core'; +import { defineHttpConfig, httpPipeline, httpRouter, type Route } from '@ultimat3/http'; +import { + encodeResponse, + readAuthnRequest, + samlTestIdp, +} from '../packages/auth/src/saml-idp-fixture'; + +const BASE = 'https://app.example.test'; +const SP = 'https://app.example.test/saml/metadata'; +const NOW = new Date('2026-11-09T12:00:00.000Z'); + +const json = (body: unknown): Response => + new Response(JSON.stringify(body), { headers: { 'content-type': 'application/json' } }); + +async function mounted() { + const idp = await samlTestIdp(); + const auth = defineAuth({ + adapter: memoryAuthAdapter(), + clock: frozenClock(NOW), + providers: ['corp', 'github'], + }); + const legs = identityProviderRoutes(auth, { + baseUrl: BASE, + secret: 's'.repeat(40), + providers: [ + samlIdentityProvider({ + id: 'corp', + idp: idp.idp, + spEntityId: SP, + trustEmail: true, + emailDomains: ['corp.example'], + }), + oidcIdentityProvider({ + provider: 'github', + credentials: { clientId: 'client-id', clientSecret: 'client-secret' }, + fetch: async (url) => { + if (url.endsWith('/access_token')) + return json({ access_token: 'g', token_type: 'bearer' }); + if (url.endsWith('/user')) return json({ id: 583231, login: 'octocat' }); + return json([{ email: 'octo@example.com', primary: true, verified: true }]); + }, + }), + ], + }); + // The documented mount: the pipeline's request carries the bare one as `raw`, and the + // descriptor's `meta` is spread as the route's — the csrf and authenticate claims included. + const routes: Route[] = [legs.start, legs.callback, legs.acs].map((leg) => ({ + method: leg.method, + path: leg.path, + handler: (request) => leg.handle(request.raw), + meta: leg.meta, + })); + const pipeline = httpPipeline({ + table: httpRouter(routes), + config: defineHttpConfig({ dev: false, buildId: null, rateLimit: { scope: 'process' } }), + }); + const send = (path: string, init: RequestInit = {}, cookie = '') => + pipeline.handle( + new Request(`${BASE}${path}`, { + ...init, + headers: { + ...(init.headers as Record | undefined), + ...(cookie === '' ? {} : { cookie }), + }, + }), + { role: 'web', ip: '203.0.113.9' }, + ); + return { idp, send }; +} + +const cookiePair = (response: Response): string => + (response.headers.getSetCookie()[0] ?? '').split(';')[0] ?? ''; + +describe('the sign-in routes behind the http pipeline', () => { + test("the SAML ACS takes the IdP's cross-site form POST", async () => { + const { idp, send } = await mounted(); + const start = await send('/auth/sso/corp'); + expect(start.status).toBe(302); + const request = readAuthnRequest(start.headers.get('location') ?? ''); + const xml = await idp.responseXml({ + requestId: request.id, + acsUrl: `${BASE}/auth/sso/corp/callback`, + audience: SP, + now: NOW, + }); + const answer = await send( + '/auth/sso/corp/callback', + { + method: 'POST', + headers: { + 'content-type': 'application/x-www-form-urlencoded', + origin: 'https://idp.example.test', + 'sec-fetch-site': 'cross-site', + 'sec-fetch-mode': 'navigate', + }, + body: new URLSearchParams({ + SAMLResponse: encodeResponse(xml), + RelayState: request.relayState, + }), + }, + cookiePair(start), + ); + expect(answer.status).toBe(303); + }); + + test("the OIDC callback takes the IdP's cross-site top-level redirect", async () => { + const { send } = await mounted(); + const start = await send('/auth/sso/github'); + const state = new URL(start.headers.get('location') ?? '').searchParams.get('state') ?? ''; + const answer = await send( + `/auth/sso/github/callback?code=c0de&state=${state}`, + { headers: { 'sec-fetch-site': 'cross-site', 'sec-fetch-mode': 'navigate' } }, + cookiePair(start), + ); + expect(answer.status).toBe(303); + }); + + test('a cross-site POST to the ACS without the state cookie is still refused, by the state', async () => { + const { send } = await mounted(); + const answer = await send('/auth/sso/corp/callback', { + method: 'POST', + headers: { + 'content-type': 'application/x-www-form-urlencoded', + origin: 'https://evil.test', + 'sec-fetch-site': 'cross-site', + }, + body: 'SAMLResponse=x&RelayState=y', + }); + expect(answer.status).toBe(400); + expect(await answer.json()).toMatchObject({ code: 'X_OAUTH_STATE_INVALID' }); + }); +}); diff --git a/scripts/lib/adapter-port-pins.ts b/scripts/lib/adapter-port-pins.ts index 33b92ed30..a90c32138 100644 --- a/scripts/lib/adapter-port-pins.ts +++ b/scripts/lib/adapter-port-pins.ts @@ -5,7 +5,7 @@ // // why: these tables are new with the guard (plan 101 slice 21); each row's VALUE is its reason, and // every BARREL_INSTALLERS row is a name a barrel exports today — one that selects no backend, or a -// move scheduled for wave F (`pending —`). PENDING_PARTS is empty: every port ships whole. +// move still scheduled (`pending —`). PENDING_PARTS is empty: every port ships whole. /** The five parts every port owes (plan 101 slice 21): one key per part, in this order. */ export const PORT_PARTS = ['field', 'binding', 'adapter', 'conformance', 'docs'] as const; @@ -27,14 +27,14 @@ export const BARREL_INSTALLERS = Object.freeze>({ 'why: declares the scope and ceiling the store is checked against; the store is the idempotencyStore port', '@ultimat3/action#configureActionPathStyle': 'why: the app pathStyle, configuration', '@ultimat3/ai#configureAi': - 'why: pending — wave F: takes the embedder and the semantic-cache factory, two backends with no ADAPTER_PORTS row; each becomes a row with a runtime.ts field, and configureAi keeps redact only', + 'why: the redactor only, configuration; the gateway, embedder and semantic cache are the aiGateway, embedder and semanticCache ports', '@ultimat3/core#configureErrorReporting': 'why: clock, release, environment and enabled only (reporter?: never): the reporter is the errorReporter port', '@ultimat3/core#setLogSink': 'why: pending — wave F: installLogSink sweep across ~47 test files + scaffold-auth template', '@ultimat3/core#configureCursorSigning': 'why: installs a secret, not a backend', '@ultimat3/core#configureMetrics': - 'why: pending — wave F: takes a MetricExporter, a backend slot beside the instrumentation bus; it becomes a subscribe() on the bus (slice 20) or an ADAPTER_PORTS row, and configureMetrics keeps clock/enabled only', + "why: clock and enabled only, configuration; an exporter is a subscribe('metric.export', …) on the instrumentation bus, as spans and logs are", '@ultimat3/core#configureTelemetry': 'why: service name, sampler and clock: configuration', '@ultimat3/core#setLoggerContextFields': 'why: internal wiring, context.ts hands the logger its field provider', diff --git a/scripts/lib/adapter-port-references.ts b/scripts/lib/adapter-port-references.ts index 4f833e71e..1a21c6a79 100644 --- a/scripts/lib/adapter-port-references.ts +++ b/scripts/lib/adapter-port-references.ts @@ -35,6 +35,11 @@ export const PORT_REFERENCES: Readonly> = symbol: 'systemClock', note: '`noWaitClock` beside it is the one a test hands a run that must not wait', }, + embedder: { + file: 'packages/ai/src/embeddings.ts', + symbol: 'hashEmbedder', + note: 'a network embedder is `remoteEmbedder` (`packages/ai/src/remote-embedder.ts`)', + }, errorReporter: { file: 'packages/core/src/error-reporter.ts', symbol: 'memoryErrorReporter' }, eventBus: { file: 'packages/jobs/src/events.ts', symbol: 'memoryEventBus' }, failureStore: { @@ -68,10 +73,25 @@ export const PORT_REFERENCES: Readonly> = }, rateLimitStore: { file: 'packages/http/src/rate-limit.ts', symbol: 'memoryRateLimitStore' }, scrapeDriver: { file: 'packages/scraping/src/driver-fake.ts', symbol: 'fakeBrowser' }, + searchIndex: { + file: 'packages/entity/src/search-index-memory.ts', + symbol: 'memorySearchIndex', + note: "the built-in `postgresSearchIndex` (`search-index-postgres.ts`) reads `.searchable()`'s generated tsvector and writes nothing on `index`", + }, + semanticCache: { + file: 'packages/cache/src/semantic.ts', + symbol: 'memorySemanticCache', + note: 'the port is the factory: `(scope) => memorySemanticCache()`, one cache per scope', + }, secretSource: { file: 'packages/core/src/secret-source.ts', symbol: 'envSecretSource' }, + smsTransport: { file: 'packages/notify/src/sms-transport.ts', symbol: 'memorySmsTransport' }, storage: { file: 'packages/storage/src/driver-memory.ts', symbol: 'memoryStorageDriver' }, totpReplay: { file: 'packages/auth/src/mfa.ts', symbol: 'memoryTotpReplayGuard' }, transport: { file: 'packages/realtime/src/fanout.ts', symbol: 'InProcessTransport' }, + uploadScanner: { + file: 'packages/storage/src/upload-scanner-memory.ts', + symbol: 'memoryUploadScanner', + }, }); /** @@ -86,6 +106,7 @@ export const CONFORMANCE_SUBJECTS: Readonly> = O authServerStore: 'store', cacheTiers: 'tier', clock: 'clock', + embedder: 'embedder', errorReporter: 'reporter', eventBus: '({ bus, elapse })', failureStore: '({ store, find, client })', @@ -102,8 +123,12 @@ export const CONFORMANCE_SUBJECTS: Readonly> = O pushSubscriptionStore: 'store', rateLimitStore: 'store', scrapeDriver: 'driver', + searchIndex: '({ index, entity, store, drop })', + semanticCache: 'factory', secretSource: '({ source, holding, empty })', + smsTransport: 'transport', storage: 'driver', totpReplay: 'guard', transport: 'transport', + uploadScanner: 'scanner', }); diff --git a/scripts/lib/bench-claims-table.ts b/scripts/lib/bench-claims-table.ts index 428bcac4b..1e8726167 100644 --- a/scripts/lib/bench-claims-table.ts +++ b/scripts/lib/bench-claims-table.ts @@ -2,6 +2,8 @@ // carries it, the dot path into that result, and the pattern that reads it off the prose. Data only, // split from the checker at its file ceiling; the checker re-exports these names. +import { SCALE_CLAIMS } from '../bench/scale-claims'; + export const CLAIMS_FILE = 'scripts/bench/results/README.md'; export const RESULTS_50K = 'scripts/bench/results/50k-restart.json'; export const RESULTS_10K = 'scripts/bench/results/10k-restart-seq.json'; @@ -345,4 +347,5 @@ export const CLAIMS: readonly BenchClaim[] = [ ...HTTP_CLAIMS, ...RENDER_CLAIMS, ...SPAN_CLAIMS, + ...SCALE_CLAIMS, ]; diff --git a/scripts/lib/fix-prose-pins.ts b/scripts/lib/fix-prose-pins.ts index 87718cd07..f5671269d 100644 --- a/scripts/lib/fix-prose-pins.ts +++ b/scripts/lib/fix-prose-pins.ts @@ -16,7 +16,7 @@ export const FIX_PROSE_PINS: Readonly> = { action: 25, admin: 30, ai: 21, - auth: 47, + auth: 46, cache: 10, cli: 78, core: 51, @@ -41,10 +41,10 @@ export const FIX_PROSE_PINS: Readonly> = { // why: the edit it names is to write release notes into CHANGELOG.md, which no command in this // repo performs, so the line cannot open with a command — the seven sibling findings that edit // that same file are already inside this number. - scripts: 173, + scripts: 172, seo: 13, - storage: 26, - testing: 28, + storage: 25, + testing: 25, time: 8, ui: 13, }; diff --git a/scripts/lib/proto-index-pins.ts b/scripts/lib/proto-index-pins.ts index 64f0c9b78..005018a63 100644 --- a/scripts/lib/proto-index-pins.ts +++ b/scripts/lib/proto-index-pins.ts @@ -37,7 +37,7 @@ export const PROTO_INDEX_PINS: Readonly> = { '`verify.ts` indexes by a `VerificationPurpose` and `tokens.ts` by a base64url character its own regex produced. Three of the thirteen were here.', }, cli: { - count: 7, + count: 6, reason: '`cmd-build.ts`, `generate-files.ts`, `verify-tests.ts`, `static-report.ts`, `app-boundaries.ts` and `mcp-errors.ts` index by a target, a surface, a test type, a boundary rule and an `X_*` code — each a union the CLI parsed and narrowed before the read.', }, diff --git a/scripts/lib/source-header-pins.ts b/scripts/lib/source-header-pins.ts index 645e68036..16dc11de8 100644 --- a/scripts/lib/source-header-pins.ts +++ b/scripts/lib/source-header-pins.ts @@ -89,7 +89,6 @@ export const SOURCE_HEADER_PINS: Readonly> = { 'packages/testing/src/island-scratch.ts': { count: 1, reason: TOO_LONG }, 'packages/testing/src/failure-context.ts': { count: 1, reason: TOO_LONG }, 'packages/testing/src/fixture-push.ts': { count: 1, reason: TOO_LONG }, - 'packages/testing/src/cdp-connection.ts': { count: 1, reason: TOO_LONG }, 'packages/testing/src/cdp-offline-script.ts': { count: 1, reason: TOO_LONG }, 'packages/jobs/src/purge.ts': { count: 1, reason: TOO_LONG }, 'packages/jobs/src/driver-pg-wake-sql.ts': { count: 1, reason: TOO_LONG }, diff --git a/scripts/lib/spawn-timeout-pins.ts b/scripts/lib/spawn-timeout-pins.ts index 52ffe12ca..b335f3d35 100644 --- a/scripts/lib/spawn-timeout-pins.ts +++ b/scripts/lib/spawn-timeout-pins.ts @@ -26,11 +26,12 @@ export const SPAWN_TIMEOUT_PINS: Readonly> = { reason: "why: `exec.ts` is the CLI subprocess boundary — its deadline is the caller's: `x verify` ends a step's whole tree in `verify-deadline.ts` (`X_VERIFY_STEP_TIMEOUT`). `local-cli-handoff.ts` spawns the app's own CLI with the user's argv, forwards its stop signals and answers its exit code, and `bin.ts` exits with that code, so its lifetime is the command the user typed. `dev-supervisor.ts` runs `x dev`'s server until Ctrl-C. `cmd-secrets.ts` opens `$EDITOR` on a terminal and waits for the human to close it.", }, - // why: three long-lived children each with its own stop path. + // why: two long-lived children each with its own stop path. Chrome's launch left this row in + // 31.0.0: it moved to `@ultimat3/scraping/cdp`, and its spawn carries `timeout: lifetimeMs`. testing: { - count: 3, + count: 2, reason: - "why: `e2e-spawn.ts` starts the app under test, which serves until `stop()` kills it; readiness is bounded by `DEFAULT_READY_TIMEOUT_MS`. `cdp-launch-attempt.ts` starts Chrome, which exits when its command pipe (fd 3) closes, and the launch is bounded by the attempt's own deadline. `drill-fleet.ts`'s `spawnRole` starts each role of a drill's fleet, which serves until the fleet's `stop()` SIGKILLs every child it holds (on a failed start too); readiness and the `ROLE=migrate` run are bounded by `readyTimeoutMs`, a drain by `exitWithin`.", + "why: `e2e-spawn.ts` starts the app under test, which serves until `stop()` kills it; readiness is bounded by `DEFAULT_READY_TIMEOUT_MS`. `drill-fleet.ts`'s `spawnRole` starts each role of a drill's fleet, which serves until the fleet's `stop()` SIGKILLs every child it holds (on a failed start too); readiness and the `ROLE=migrate` run are bounded by `readyTimeoutMs`, a drain by `exitWithin`.", }, // why: four children killed by the script that started them, and one bounded by test timeouts. // `lib/run.ts` left this row in #780: its own deadline (`DEFAULT_RUN_TIMEOUT_MS`) now ends a child. diff --git a/scripts/lib/test-real-sleep-pins.ts b/scripts/lib/test-real-sleep-pins.ts index 7f53c4f4e..4ab129689 100644 --- a/scripts/lib/test-real-sleep-pins.ts +++ b/scripts/lib/test-real-sleep-pins.ts @@ -30,8 +30,10 @@ export const REAL_SLEEP_PINS: Readonly> = { pwa: 6, realtime: 57, render: 7, - scraping: 2, + // why: moved from testing with the CDP launcher (testing −4): the same four waits on a real + // child process, socket or pipe that the launcher's own deadlines run on, no new one (31.0.0). + scraping: 6, scripts: 2, - testing: 7, + testing: 3, ui: 3, }; diff --git a/scripts/primitive-factories-used.test.ts b/scripts/primitive-factories-used.test.ts index 76d8b1792..b296d853f 100644 --- a/scripts/primitive-factories-used.test.ts +++ b/scripts/primitive-factories-used.test.ts @@ -25,7 +25,10 @@ const REFERENCE_APP = 'examples/dummy'; * ratchet, never an allow-list: a name here that the app now uses is a failure ("delete the row"), * so the list only shrinks. Adding a row is a reviewable diff with its reason beside it. */ -export const PENDING_IN_REFERENCE_APP: Readonly> = {}; +export const PENDING_IN_REFERENCE_APP: Readonly> = { + partitions: + "needs an append-only, time-keyed table that is NEW: Postly's (run_events, webhook_deliveries, daily_likes) exist unpartitioned, Postgres cannot partition one in place (X_MIGRATION_PARTITION_CHANGE), and a partitioned table nothing writes would be weight without function — the first time-series table Postly adds is declared with partitionBy and maintained by partitions()", +}; /** Shipped app source: no tests (a test calling a factory proves nothing about the app), no deps. */ const appSources = async (): Promise => { diff --git a/scripts/primitive-factories.test.ts b/scripts/primitive-factories.test.ts index 160b63bcf..eb401f80f 100644 --- a/scripts/primitive-factories.test.ts +++ b/scripts/primitive-factories.test.ts @@ -214,8 +214,11 @@ describe('unit · every primitive factory in the tree has a row in PRIMITIVE_FAC '@ultimat3/jobs.consumer:job', '@ultimat3/jobs.exportRows:job', '@ultimat3/jobs.importRows:job', + '@ultimat3/jobs.ledger:job', + '@ultimat3/jobs.partitions:job', '@ultimat3/jobs.purge:job', '@ultimat3/jobs.refreshView:job', + '@ultimat3/jobs.reindex:job', '@ultimat3/jobs.rollup:job', '@ultimat3/jobs.subjectErase:job', '@ultimat3/jobs.subjectExport:job', @@ -228,6 +231,7 @@ describe('unit · every primitive factory in the tree has a row in PRIMITIVE_FAC '@ultimat3/pwa.pushSubscribe:action', '@ultimat3/pwa.pushUnsubscribe:action', '@ultimat3/realtime.deviceSocket:route', + '@ultimat3/scraping.pdf:action', '@ultimat3/scraping.scrape:job', ]); expect(declared).toHaveLength(PRIMITIVE_FACTORIES.length); diff --git a/scripts/set-cookie-literals.ts b/scripts/set-cookie-literals.ts index 13227ece4..70a0b0bee 100644 --- a/scripts/set-cookie-literals.ts +++ b/scripts/set-cookie-literals.ts @@ -66,9 +66,9 @@ export const SET_COOKIE_PINS: Readonly> = {}; * its own serializer-built lines on the `Headers` it returns. May fall, never rise. */ export const SET_COOKIE_RELAY_PINS: Readonly> = { - 'packages/auth/src/oauth-route.ts': { + 'packages/auth/src/identity-provider-route-core.ts': { sites: 4, - why: 'appends handshakeCookie(), clearHandshakeCookie() and sessionCookie() output — each a serializeSetCookie line in oauth-cookie.ts / session.ts — onto the redirect and problem Responses this route builds itself', + why: 'appends identityStateCookie(), clearIdentityStateCookie() and sessionCookie() output — each a serializeSetCookie line in identity-provider-cookie.ts / session.ts — onto the redirect and problem Responses the sign-in routes build themselves (oauthLogin moved onto these routes in 31.0.0)', }, 'packages/auth/src/sign-out.ts': { sites: 1, diff --git a/scripts/wiki-fences-backlog.ts b/scripts/wiki-fences-backlog.ts index 60285822e..dad6987f3 100644 --- a/scripts/wiki-fences-backlog.ts +++ b/scripts/wiki-fences-backlog.ts @@ -109,7 +109,6 @@ export const WIKI_FENCE_BACKLOG: Readonly> = { 'N-Plus-One-Detection: // one statement for the page': 1, "N-Plus-One-Detection: test('the feed reads its authors once', async ({ statements }) => {": 1, "Notify: import { inAppChannel, mailChannel, notifier, t } from '@ultimat3/notify';": 1, - "Observability: import { collectMetrics, configureMetrics, exportMetrics, startMetricExport } from '@ultimat3/core';": 1, 'PWA-And-Offline: // app.config.ts — every member optional; text is a string or one per locale': 1, 'PWA-And-Offline: // app.config.ts — omitting `offline.fallback` is a compile error': 1, 'PWA-And-Offline: // app.config.ts: the two capabilities an app can switch on, both booleans': 1, diff --git a/tsconfig.base.json b/tsconfig.base.json index 0b25959b2..38f73102e 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -64,6 +64,7 @@ "@ultimat3/testing/e2e-preload": ["./packages/testing/src/e2e-preload.ts"], "@ultimat3/testing/registry-isolation": ["./packages/testing/src/registry-isolation.ts"], "@ultimat3/ui/icons/*": ["./packages/ui/src/icons/glyphs/*"], + "@ultimat3/scraping/cdp": ["./packages/scraping/src/cdp.ts"], "@ultimat3/*": ["./packages/*/src"] }, diff --git a/wiki/Adapters.md b/wiki/Adapters.md index 745b26a77..c879fc302 100644 --- a/wiki/Adapters.md +++ b/wiki/Adapters.md @@ -32,6 +32,7 @@ export const runtime = { | [`authServerStore`](#authserverstore) | `AuthServerStore` | `@ultimat3/auth` | | [`cacheTiers`](#cachetiers) | `CacheTier` | `@ultimat3/cache` | | [`clock`](#clock) | `SleepClock` | `@ultimat3/core` | +| [`embedder`](#embedder) | `Embedder` | `@ultimat3/ai` | | [`errorReporter`](#errorreporter) | `ErrorReporter` | `@ultimat3/core` | | [`eventBus`](#eventbus) | `EventBus` | `@ultimat3/jobs` | | [`failureStore`](#failurestore) | `FailureStore` | `@ultimat3/core` | @@ -48,10 +49,14 @@ export const runtime = { | [`pushSubscriptionStore`](#pushsubscriptionstore) | `PushSubscriptionStore` | `@ultimat3/pwa` | | [`rateLimitStore`](#ratelimitstore) | `RateLimitStore` | `@ultimat3/http` | | [`scrapeDriver`](#scrapedriver) | `ScrapeDriver` | `@ultimat3/scraping` | +| [`searchIndex`](#searchindex) | `SearchIndex` | `@ultimat3/entity` | | [`secretSource`](#secretsource) | `SecretSource` | `@ultimat3/core` | +| [`semanticCache`](#semanticcache) | `SemanticCacheFactory` | `@ultimat3/ai` | +| [`smsTransport`](#smstransport) | `SmsTransport` | `@ultimat3/notify` | | [`storage`](#storage) | `StorageDriver` | `@ultimat3/storage` | | [`totpReplay`](#totpreplay) | `TotpReplayGuard` | `@ultimat3/auth` | | [`transport`](#transport) | `Transport` | `@ultimat3/realtime` | +| [`uploadScanner`](#uploadscanner) | `UploadScanner` | `@ultimat3/storage` | ## aiGateway @@ -119,6 +124,17 @@ export const runtime = { | Wire | `clock: …` in `apps/web/runtime.ts`, typed `SleepClock` | | Boot | installed by `installClock` (boot phase) | +## embedder + +| Part | Value | +|---|---| +| Interface | `Embedder` from `@ultimat3/ai/adapter` | +| Default | hashEmbedder: deterministic, offline and lexical, never a network call | +| Copy | [`hashEmbedder`](https://github.com/developerz-ai/ultimate/blob/main/packages/ai/src/embeddings.ts) in `packages/ai/src/embeddings.ts` — a network embedder is `remoteEmbedder` (`packages/ai/src/remote-embedder.ts`) | +| Prove | `behavesLike(embedderConformance, () => embedder)` | +| Wire | `embedder: …` in `apps/web/runtime.ts`, typed `Embedder` | +| Boot | installed by `installAiEmbedder` (services phase) | + ## errorReporter | Part | Value | @@ -295,6 +311,17 @@ export const runtime = { | Wire | `scrapeDriver: …` in `apps/web/runtime.ts`, typed `ScrapeDriver` | | Boot | installed by `installScrapeDriver` (services phase) | +## searchIndex + +| Part | Value | +|---|---| +| Interface | `SearchIndex` from `@ultimat3/entity/adapter` | +| Default | postgresSearchIndex: .searchable()'s generated tsvector, over the app's pool | +| Copy | [`memorySearchIndex`](https://github.com/developerz-ai/ultimate/blob/main/packages/entity/src/search-index-memory.ts) in `packages/entity/src/search-index-memory.ts` — the built-in `postgresSearchIndex` (`search-index-postgres.ts`) reads `.searchable()`'s generated tsvector and writes nothing on `index` | +| Prove | `behavesLike(searchIndexConformance, () => ({ index, entity, store, drop }))` | +| Wire | `searchIndex: …` in `apps/web/runtime.ts`, typed `SearchIndex` | +| Boot | installed by `installSearchIndex` (services phase) | + ## secretSource | Part | Value | @@ -306,6 +333,28 @@ export const runtime = { | Wire | `secretSource: …` in `apps/web/runtime.ts`, typed `SecretSource` | | Boot | installed by `installSecretSource` (boot phase) | +## semanticCache + +| Part | Value | +|---|---| +| Interface | `SemanticCacheFactory` from `@ultimat3/ai/adapter` | +| Default | a memorySemanticCache per scope, per process | +| Copy | [`memorySemanticCache`](https://github.com/developerz-ai/ultimate/blob/main/packages/cache/src/semantic.ts) in `packages/cache/src/semantic.ts` — the port is the factory: `(scope) => memorySemanticCache()`, one cache per scope | +| Prove | `behavesLike(semanticCacheFactoryConformance, () => factory)` | +| Wire | `semanticCache: …` in `apps/web/runtime.ts`, typed `SemanticCacheFactory` | +| Boot | installed by `installSemanticCache` (services phase) | + +## smsTransport + +| Part | Value | +|---|---| +| Interface | `SmsTransport` from `@ultimat3/notify/adapter` | +| Default | memory in development/test; refused at the first send elsewhere (X_NOTIFY_SMS_TRANSPORT_MISSING) | +| Copy | [`memorySmsTransport`](https://github.com/developerz-ai/ultimate/blob/main/packages/notify/src/sms-transport.ts) in `packages/notify/src/sms-transport.ts` | +| Prove | `behavesLike(smsTransportConformance, () => transport)` | +| Wire | `smsTransport: …` in `apps/web/runtime.ts`, typed `SmsTransport` | +| Boot | installed by `installSmsTransport` (services phase) | + ## storage | Part | Value | @@ -338,3 +387,14 @@ export const runtime = { | Prove | `behavesLike(transportConformance, () => transport)` | | Wire | `transport: …` in `apps/web/runtime.ts`, typed `Transport` | | Boot | read as a value by `runtime-bus.ts, which installs the channel transport it dials` | + +## uploadScanner + +| Part | Value | +|---|---| +| Interface | `UploadScanner` from `@ultimat3/storage/adapter` | +| Default | none: promoteAttachment scans nothing | +| Copy | [`memoryUploadScanner`](https://github.com/developerz-ai/ultimate/blob/main/packages/storage/src/upload-scanner-memory.ts) in `packages/storage/src/upload-scanner-memory.ts` | +| Prove | `behavesLike(uploadScannerConformance, () => scanner)` | +| Wire | `uploadScanner: …` in `apps/web/runtime.ts`, typed `UploadScanner` | +| Boot | installed by `installUploadScanner` (services phase) | diff --git a/wiki/Auth.md b/wiki/Auth.md index 4d3ed78ad..9540b0dc9 100644 --- a/wiki/Auth.md +++ b/wiki/Auth.md @@ -19,7 +19,7 @@ export const auth = defineAuth({ }); const { actor, token, cookie } = await login(auth, { email, password, ip }); -const { start, callback } = oauthLogin(auth); // /auth/oauth/:provider and its callback +const { start, callback } = oauthLogin(auth); // /auth/sso/:provider and its callback ``` The tables (`x_users`, `x_sessions`, `x_accounts`, `x_verifications`, `x_api_keys`, and the @@ -80,6 +80,82 @@ bound to its provider. An id token is verified against the provider's JWKS. An i | a public client (`As of 2026-10-18`) | `tokenEndpointAuth: 'none'` on the provider, `discoverOAuthProvider` or `credentials`: no `_CLIENT_SECRET` is read and no `client_secret` is sent; PKCE still is. `'client_secret_basic'` sends the pair in `Authorization: Basic`. Absent: `'client_secret_post'`, unchanged | | `apple` | not a built-in since 26.0.0 ([#709](https://github.com/developerz-ai/ultimate/issues/709)): Apple POSTs its callback and both callback routes are GET, so it could never complete a sign-in. An app that needs it registers its own with `registerOAuthProvider` | +## SSO: OIDC and SAML behind one `IdentityProvider` port + +Rule: every federated sign-in is an `IdentityProvider` (`@ultimat3/auth`), and `signInWithIdentity(auth, identity)` is the one path from it to a session — the same account-link, MFA and grant rules as [OAuth](#oauth). `As of 2026-11-09`. + +| Member | Shape | +|---|---| +| `begin({ callbackUrl, clock })` | `{ location, state }` | +| `answers(state, params)` | whether `params` answer THIS browser's state — RelayState for SAML, `state` for OIDC — in constant time, nothing verified beyond it. The routes clear the state cookie only on an answer bound to it, so a junk cross-site form cannot end a sign-in in flight | +| `complete({ state, params, callbackUrl, clock, replay })` | `{ profile, expiresAt, claims }` | +| adapters | `samlIdentityProvider(config)`, `oidcIdentityProvider({ provider })` | +| your own | `behavesLike(identityProviderConformance, () => harness)` from `@ultimat3/testing/conformance` | + +`identityProviderRoutes(auth, { providers })` mounts both kinds: + +| Route | Method | Purpose | +|---|---|---| +| `/auth/sso/:provider` | GET | `begin`, seal the state in `__Host-x_sso_`, 302 to the IdP | +| `/auth/sso/:provider/callback` | POST | the SAML ACS — register this URL at the IdP. The state cookie is `SameSite=None`: the IdP posts cross-site | +| `/auth/sso/:provider/callback` | GET | the OIDC `redirect_uri` | + +```ts +import type { Auth } from '@ultimat3/auth'; +import { + identityProviderRoutes, + oidcIdentityProvider, + parseSamlMetadata, + samlIdentityProvider, +} from '@ultimat3/auth'; + +declare const auth: Auth; +declare const corpMetadataXml: string; // the IdP's metadata document, read at boot + +export const sso = identityProviderRoutes(auth, { + providers: [ + samlIdentityProvider({ + id: 'corp', + idp: parseSamlMetadata(corpMetadataXml), + spEntityId: 'https://app.example/saml/metadata', + trustEmail: true, + emailDomains: ['corp.example'], // the exact domains this IdP owns + }), + oidcIdentityProvider({ provider: 'github' }), + ], + successPath: '/app', +}); +``` + +`defineAuth({ providers })` lists both ids; an unknown or not-enabled id is `X_IDENTITY_PROVIDER_UNKNOWN` (404). + +**OIDC replay.** `oidcIdentityProvider` claims the handshake's `state` once in the shared replay store before it spends the code, keyed on when the handshake was issued: a callback lifted from a browser is `X_OAUTH_STATE_INVALID` even inside the cookie's lifetime, and so is any answer more than 10 minutes after `begin` (`DEFAULT_HANDSHAKE_TTL_MS`). + +Mount each leg with its `meta` verbatim — `{ method: leg.method, path: leg.path, handler: (request) => leg.handle(request.raw), meta: leg.meta }`. The ACS (POST) declares `csrf: 'cookieless'` and `authenticate: () => null`: the IdP answers with an auto-submitted cross-site form, which the csrf stage refuses on any other route (`403 X_CSRF_BLOCKED`). CSRF is held there by the protocol instead — the answer must carry this browser's RelayState, checked against the HMAC-sealed state cookie, and its assertion must answer this request's ID. + +**SAML scope.** SP-initiated only: an AuthnRequest over HTTP-Redirect with RelayState, the response over HTTP-POST. Signed assertions are required; encrypted assertions, SHA-1 and RSA under 2,048 bits are refused; `trustEmail` is required, because SAML has no `email_verified`, and `trustEmail: true` requires `emailDomains` — an IdP's admin can assert any address, so only an address in a listed domain (exact; list each subdomain) is verified, and one outside it can neither link to an existing account nor create one (`trustEmail: true` with no domains is a type error and `X_CONFIG_INVALID` at construction); clock skew is ±60 s (`skewMs`). `parseSamlMetadata(xml)` → `{ entityId, ssoUrl, certificates }`, refused as `X_SAML_METADATA_INVALID` with no HTTP-Redirect SSO, no signing certificate, or an RSA key under 2,048 bits. Verification is in-package, no dependency ([build vs wrap](https://github.com/developerz-ai/ultimate/blob/main/docs/idea/18-build-vs-wrap.md)). Every refusal is `X_SAML_RESPONSE_INVALID` with `meta.check`: + +| Check | `meta.check` | +|---|---| +| a DOCTYPE, an entity, a processing instruction, over 512 KiB, more than 64 namespace declarations, one attribute twice | `xml` | +| more than one Assertion, one not a direct child, a `ds:Object` in a Signature | `structure` | +| one ID on two elements | `id-duplicate` | +| an EncryptedAssertion or EncryptedID | `encrypted` | +| the assertion unsigned (a signed response alone is refused) | `signature-missing` | +| not RSA-SHA256 / SHA-256 / exc-c14n + enveloped | `algorithm` | +| the Reference URI is not the assertion's own ID | `reference` | +| no metadata certificate verifies (KeyInfo is never read) | `signature` | +| content altered | `digest` | +| Status is not Success | `status` | +| Issuer is not the metadata's entityID | `issuer` | +| Destination / Recipient is not the ACS URL | `destination` / `recipient` | +| InResponseTo is not our AuthnRequest's ID | `in-response-to` | +| Audience is not `spEntityId` | `audience` | +| NotBefore / NotOnOrAfter | `not-before` / `expired` | +| a transient or empty NameID | `subject` | +| RelayState is not the browser's | `relay-state` | +| an assertion already consumed (`auth.totpReplay`, shared) | `replay` | + ## Passkeys **WebAuthn, verified by hand on WebCrypto — no dependency** (`As of 2026-11`). Two uses, one diff --git a/wiki/CLI-Reference.md b/wiki/CLI-Reference.md index 3032f45ad..56f342feb 100644 --- a/wiki/CLI-Reference.md +++ b/wiki/CLI-Reference.md @@ -33,7 +33,7 @@ x version # CLI version | `x stop [--timeout 30s]` | stop this checkout's `x dev` (the pid its `.x/dev.lock` names, nothing else) and return once the next `x dev` can boot | shipped | | `x g ` | scaffold a primitive with its test | shipped | | `x db ` | gen, migrate, reset, seed, studio, branch, backfill, diagnostics | shipped | -| `x verify [--only [,…] [--shard i/n [--timings f]]] [--workers N] [--isolate]` · `x verify merge ` | the gate — 21 steps, in this order: typecheck, lint, boundaries, filesize, package-shape, errors, unit, contract, live, job, exercise, e2e, eval, drift, contract-diff, budgets, seo, i18n, policy, manifest, roadmap. `--only [,…]` runs the named steps, in one process and in gate order, for an iteration loop and announces `NOT A GATE RUN` in the summary and in `--json` (`data.notAGateRun`, `data.only` — always a list), writing no floor file; an unknown name is refused with every valid name and the nearest match. **The gate is this command with no flag.** There is no `--skip` | shipped | +| `x verify [--only [,…] [--shard i/n [--timings f]] \| --changed [--base ] [--dirty]] [--workers N] [--isolate]` · `x verify merge ` | the gate — 21 steps, in this order: typecheck, lint, boundaries, filesize, package-shape, errors, unit, contract, live, job, exercise, e2e, eval, drift, contract-diff, budgets, seo, i18n, policy, manifest, roadmap. `--only [,…]` runs the named steps, in one process and in gate order, for an iteration loop and announces `NOT A GATE RUN` in the summary and in `--json` (`data.notAGateRun`, `data.only` — always a list), writing no floor file; an unknown name is refused with every valid name and the nearest match. **The gate is this command with no flag.** There is no `--skip` | shipped | | `x env [check\|example\|init]` | validate the process env against `envSchema`, regenerate `.env.example` from it, or write `.env.production` (`init --production`) | shipped | | `x secrets ` | the committed encrypted secrets file: show, init, edit, set, rotate | shipped | | `x auth seal-mfa` | one-shot: seal every second-factor secret still stored in the clear | shipped | @@ -320,14 +320,18 @@ x g admin:page --permission [--at ] # a custom admin screen x g guard # a convention this app enforces, as a build error x g sign-in --password | --oidc # the app's one sign-in: pages, actions, boot guard x g chart-alerts [--print] # the chart's ServiceMonitor + alert rules, backfilled +x g app # one more app workspace: apps// +x g package # one more shared package: packages// ``` +**`x g app ` / `x g package `** (`As of 2026-11`) add a workspace to the monorepo: `apps//` (`package.json` + `tsconfig.json`) or `packages//` (manifest, README, `src/index.ts` and its test), named `@/` with the scope read off the root `package.json` (`@acme/root` → `acme`). Nothing outside the new directory is edited: the root `workspaces` globs and tsconfig alias already reach it. A name the other kind already holds is `X_WORKSPACE_NAME_TAKEN` (both would install as `@/`); one the same kind holds is `X_GENERATE_CONFLICT`. The run prints `bun install` then `bunx x verify --changed --dirty` (`data.next`). + **`x g sign-in`** (`As of 2026-11`) is what a deployed scaffold needs to issue a session: before it, a production `web` role answered every policy-protected route `401` (`packages/cli/e2e/sign-in.e2e.test.ts` proves the after). Exactly one of the two flags, and no `` — an app has one sign-in; anything else is `X_CLI_BAD_FLAG`, and so is any other `x g` flag it would ignore (`--feature`, `--surface`, `--at`, `--permission`, `--live`, `--admin`), and either flag on another generator. Its keys land in every catalog the app ships, or the ones `--locales` names, as every other generator's do. The auth slice lands under `apps/web/app/auth/` beside `x new`'s `auth.ts`, plus `apps/web/runtime.ts` and the edits in the last column. A dry run lists the deletions in `data.removed`, apart from `data.files`; a real run reports them under the same key: | Shape | Writes | Also edits | |---|---|---| | `--password` | `sign-in/page.tsx` (a native form, 0 kB) posting to `createSession` over `login()`; `sign-in/second-factor/page.tsx` posting to `confirmSession` over `completeMfa()` (the sealed challenge travels in an HttpOnly cookie); `destroySession` over `logout()` + `signOutHeaders()`; `sign-in-guard.ts`; `apps/web/runtime.ts` — 22 files on a dry run in the default scaffold, 24 after setup | `apps/web/api/index.ts` (the three actions), `app.config.ts` (`auth: { signInPath: '/auth/sign-in' }`), the catalogs | -| `--oidc ` | `oidc.ts` — `discoverOAuthProvider` on the FIRST sign-in, never at import — and `sign-in-routes.ts`, the `oauthLogin` legs on `runtime.routes`; the sign-in page (one link to `/auth/oauth/`); `destroySession`; the guard — 20 files on a dry run (plus the two in `data.removed`), 22 after setup | as above, plus `_ISSUER`, `_CLIENT_ID`, `_CLIENT_SECRET` in `envSchema`; and **the authenticator swap**: `authenticator.ts` loses the development fallback and `dev-actor.ts` is deleted, only while all four files are byte-for-byte what `x new` wrote — an edited one is an `X_GENERATE_CONFLICT` finding naming the line to delete by hand | +| `--oidc ` | `oidc.ts` — `discoverOAuthProvider` on the FIRST sign-in, never at import — and `sign-in-routes.ts`, the `oauthLogin` legs on `runtime.routes`; the sign-in page (one link to `/auth/sso/`); `destroySession`; the guard — 20 files on a dry run (plus the two in `data.removed`), 22 after setup | as above, plus `_ISSUER`, `_CLIENT_ID`, `_CLIENT_SECRET` in `envSchema`; and **the authenticator swap**: `authenticator.ts` loses the development fallback and `dev-actor.ts` is deleted, only while all four files are byte-for-byte what `x new` wrote — an edited one is an `X_GENERATE_CONFLICT` finding naming the line to delete by hand | The boot guard is generated, app-owned: `apps/web/runtime.ts` calls `assertSignInConfigured(process.env)` before the boot binds a port, and outside `development`/`test` a `web` or `sync` role with a sign-in key unset refuses to start — `X_AUTH_SIGN_IN_UNCONFIGURED`, fix `export =""` in the role's deployed environment, then restart it (`x env example` lists the keys). The keys: `ULTIMATE_SECRETS_KEY` for `--password` (it seals the second-factor challenge); the provider's three plus `SESSION_SECRET` and `APP_URL` for `--oidc`. A second `x g sign-in` is `X_GENERATE_CONFLICT` and writes nothing. `--password` keeps the development viewer: it ships no sign-up page, so the first user is `register(appAuth(), { email, password, orgId, roles })`. An app that already has an `apps/web/runtime.ts` is that conflict too — merge the two `runtime` objects by hand. @@ -346,6 +350,7 @@ is a word (`a-z`, `0-9`, `-`, `_`) or `[param]` / `[...rest]`; `..`, `.` and emp |---|---|---|---| | `--feature` | string | derived from the name | feature slice to write into. Not for `resource`: its name IS the slice (`app//`, page at `app//`), and a `--feature` naming anything else is `X_CLI_BAD_FLAG` | | `--surface` | string | `app` | `site` or `app` | +| `--app` | string | `web` | `route`, `island`, `query` only: the app workspace under `apps/` the files land in (`x g route reports --app admin` → `apps/admin/app/reports/page.tsx`). An app that is not there is `X_CLI_BAD_FLAG`, fix `x g app `; any other generator registers in files only `apps/web` holds (API index, roles, exercise file, db handle) and refuses it, fix the same run without the flag | | `--permission` | string | `:read` | `admin:page` only: the permission the page's own work needs, on top of the frame's `admin:read`. A `:`, or namespaced with more segments (`admin:blog:write`, `As of 27.6.2` — one colon exactly was required, which refused every `admin::` a staff console declares); an empty segment, a space or a quote is `X_CLI_BAD_FLAG` before a file is written — the value reaches the emitted source three times and `Permission` is `${string}:${string}` | | `--live` | boolean | `false` | `query` only: make it subscribable. On any other generator it is `X_CLI_BAD_FLAG` naming the `x g query … --live` to run instead | | `--admin` | boolean | `false` | `resource` only: also emit the per-entity admin override **and list it** under `resources:` in `apps/admin/app/admin/admin.ts`. The screen itself needs no flag — an entity in the handle is an admin screen | @@ -593,10 +598,29 @@ Errors, backfill: `X_BACKFILL_PENDING` (`--pending`, and the only one that is no ## x verify ```bash -x verify [--only [,…] [--shard i/n [--timings file]]] [--workers N] [--isolate] [--json] +x verify [--only [,…] [--shard i/n [--timings file]] | --changed [--base ] [--dirty]] [--workers N] [--isolate] [--json] x verify merge [--json] ``` +**`--changed`**, `As of 31.0` — the steps whose inputs the diff touches, NOT A GATE RUN like `--only` +and never beside it (`X_CLI_BAD_FLAG`). The diff is `x affected`'s: `...HEAD`, `--base` default +`main`, `--dirty` adds uncommitted and untracked work. Each file gets one rule, each rule selects +steps; `data.changed` carries `{ base, dirty, files: [{ path, rule }], steps: [{ name, files }] }` and +the human render one `step ← files` line per step. Fail-open: a path no rule speaks for runs the gate. +Each suite but `eval` then runs only the test files the diff's import closure reaches — `x test +--affected`'s selection, `data.changed.tests` — a suite none of them belongs to is dropped, and the +`unit` coverage floor is not judged on the subset; a file the import graph cannot speak for (a root +config, a catalog) runs every test. + +| rule | files | steps | +|---|---|---| +| `doc` | `*.md`, `docs/`, `wiki/` | package-shape, errors, unit, manifest, roadmap | +| `test` | `*.test.ts(x)`, `e2e/` | typecheck, lint, filesize, package-shape, errors + the suite that owns the file | +| `generated` | `x.manifest.json` | lint, contract-diff, manifest | +| `source` | `apps/`, `packages/`, `guards/`, `scripts/`, `types/`, `bin/` | every step but `drift` | +| `drift-input` | `packages/db/**`, `apps/*/clients/**`, `openapi.json`, any module importing `@ultimat3/entity` and everything it imports | every step | +| `root-config` · `deleted` · `unknown` · `outside` | a root file, any `app.config.ts`, a vanished module, another directory, a path outside the app | every step | + **`As of 22.7`** — the width is a memory budget (`max(2.75 GiB, 25% of RAM)` at 1.25 GiB a worker — no fixed 4 GiB cap since 2026-11-07, at most one per core, printed on the step line; `ULTIMATE_TEST_MEMORY_BUDGET` / `ULTIMATE_TEST_MAX_WORKERS` override, `--workers` wins), shared machine-wide across concurrent runs, diff --git a/wiki/Caching-And-Invalidation.md b/wiki/Caching-And-Invalidation.md index 85971bbfa..11e821595 100644 --- a/wiki/Caching-And-Invalidation.md +++ b/wiki/Caching-And-Invalidation.md @@ -91,6 +91,16 @@ Fan-out runs **after the handler resolves, in the same call** — `bustAfterComm > > **As of 31.0.0 a bust runs no script at all**, and every command names one key. Tag buckets are sorted sets (`t2:`/`e2:`) scored by the member's expiry plus 60 s; every write prunes members past their score, so a bucket holds live keys only. Before, only a bust removed members: an entity written constantly and never invalidated collected every key it ever wrote, and its first bust read millions in one reply. A value over `redisTier({ maxValueBytes })` (1 MiB) is refused with `X_CACHE_TOO_LARGE`. +**Across replicas without NATS.** `As of 2026-11-09`. The tier-2 message rides whichever bus reaches the other processes, decided once at boot ([`runtime-cache-bus.ts`](https://github.com/developerz-ai/ultimate/blob/main/packages/cli/src/runtime-cache-bus.ts)): + +| Configured | Invalidations ride | `cache.broadcast` log line | +|---|---|---| +| `NATS_URL`, or a `transport` in `runtime.ts` | that transport | `bus: 'nats'` (the transport's name) | +| no NATS, `REDIS_URL` (else `VALKEY_URL`) | Redis pub/sub (`redisPubSubAt`) | `bus: 'redis', source: 'REDIS_URL'` | +| neither | the in-process bus: this replica only | `bus: 'in-process'` | + +Never both: a second hop beside NATS would deliver every bust twice through two systems that fail independently. A replica whose Redis subscription dropped flushes its own tiers on reconnect rather than serve what it may have missed. This is what makes rung 1 of the [scale ladder](https://github.com/developerz-ai/ultimate/blob/main/docs/idea/17-scale-ladder.md) — roles split, Redis, no NATS — safe with two web replicas. + There is exactly one fan-out entry point in the implementation (`invalidateTags()`); no caller reaches a tier directly. Tier failures are collected into an invalidation report — **a cache tier may never fail a business write.** The write side holds the same rule one layer up: a fan-out that refuses outright — a tag no entity declared, `X_CACHE_TAG_UNKNOWN` — is absorbed rather than raised: one `action.invalidate.failed` error line, entries live until their TTL, and the caller keeps the write it already made. A replayed idempotent call (`idempotent: true` + a repeated `Idempotency-Key`) busts nothing at all: no handler ran, and the first call already did. diff --git a/wiki/Configuration.md b/wiki/Configuration.md index 828c61201..349aab4ea 100644 --- a/wiki/Configuration.md +++ b/wiki/Configuration.md @@ -117,7 +117,7 @@ await withReplicaReads(async () => { | `limiter` / `orgLimiter` | `AuthLimiter` | omitted means one process' worth of state, i.e. `maxAttempts × N` for N replicas. An attempt is reserved before the KDF (`reserve` / `refund`), never checked and recorded after it | | `mfa` | `Partial` | `required` is typed `false` and cannot be set true: both credential paths branch on `user.mfaSecret`, so a user who never enrolled would be locked out for good | | `totpReplay` | `TotpReplayGuard` | which TOTP steps are spent, one atomic `claim(subject, step, at)`, called by `completeMfa`. Omitted means the guard the host installed: a framework boot's is `postgresTotpReplayGuard` (`x_totp_steps`), so a code is single-use across the fleet. With no host, `memoryTotpReplayGuard()`, single-use per replica. A guard with no `claim` is `X_CONFIG_INVALID` | -| `providers` | `OAuthProviderId[]` | **defaults to `[]`**, `As of 2026-08-23` — an empty list is "no OAuth", and every `/auth/oauth/` answers `X_OAUTH_PROVIDER_UNKNOWN`. Never the live registry: that would let any dependency that calls `registerOAuthProvider` turn on a login route this app never enabled | +| `providers` | `OAuthProviderId[]` | **defaults to `[]`**, `As of 2026-08-23` — an empty list is "no OAuth", and every `/auth/sso/` answers `X_OAUTH_PROVIDER_UNKNOWN`. Never the live registry: that would let any dependency that calls `registerOAuthProvider` turn on a login route this app never enabled | | `link` | `OAuthLinkPolicy` | defaults to `'verified-email'` | | `passkeys` | `PasskeyPolicyInput` | `{ rpId, origins, rpName?, challengeTtlMs? }` (`As of 2026-11`): the relying party every WebAuthn ceremony verifies against. `rpId` a bare lowercase host, never a URL or an IP; each origin exact (`URL.origin` round-trips it), https or `http://localhost`, on `rpId` or a subdomain; `challengeTtlMs` 5 min. Anything else is `X_CONFIG_INVALID` at `defineAuth`. Absent means no passkeys: `passkeyLogin(auth)` / `passkeyStepUp(auth)` refuse at definition ([Auth](Auth#passkeys)) | diff --git a/wiki/Deployment.md b/wiki/Deployment.md index d466406c7..0e7ac0f37 100644 --- a/wiki/Deployment.md +++ b/wiki/Deployment.md @@ -31,6 +31,22 @@ ROLE=replicator myapp `PORT` selects the bind port, default `3000`. Empty or whitespace falls back to the default; anything else must be an integer in 0–65535 or the process refuses with `X_PORT_INVALID` rather than quietly binding 3000 and failing the platform's health probe with nothing in the log that names the cause. The production entry is the scaffolded `apps/web/server.ts` → [CLI reference](CLI-Reference). +## One repository, one image + +Rule: one repository is one app and one image. `As of 2026-11-09`. Every `apps/*` workspace (`web`, `admin`, `api`, …) boots into the one registry the root `app.config.ts` configures, and one image serves them all; `ROLE` chooses the process, never the app. + +| Question | Answer | +|---|---| +| URLs across apps | merged into one route table; a clash is `X_ROUTE_DUPLICATE` | +| migrations | once, in `packages/db/migrations`, applied by `ROLE=migrate` | +| an image per app | not a shape: an `app.config.ts` inside a workspace is `X_APP_ROOT_NESTED` (the `drift` step) | +| shared code | `packages/` (`x g package `), which never imports an app (`X_BOUNDARY_PACKAGE_TO_APP`) | +| a new app | `x g app `, reached as `@//…` through the root tsconfig's one alias (`@/*` → `packages/*/src`, then `apps/*`) | +| one app importing a sibling | allowed when its `package.json` declares it (`X_WORKSPACE_DEP_UNDECLARED`) — `apps/admin` projects `apps/web`'s API this way | +| the surface rules | hold across apps: a `site/` page importing another app's `app/` is `X_BOUNDARY_SITE_TO_APP`. The surface is the directory directly under `apps//`, so an app workspace may itself be named `api`, `site`, `app` or `shared` | + +`x verify --changed` gates only what a diff reaches: a docs-only or test-only diff runs a handful of steps; `drift` runs only when an entity's import closure, `packages/db/**`, a generated client or `openapi.json` moved; each suite runs only the tests the diff's import closure reaches ([CLI reference](CLI-Reference)). + ## Health endpoints **Every serving role answers both.** `web` and `sync` on their HTTP server; `worker`, `scheduler` and `replicator` construct none and answer them on the metrics listener (`METRICS_PORT`, default 9090, `packages/cli/src/metrics-endpoint.ts`) with the verdict only — `{ state, ready, role }`, never the check names. There `/readyz` is 503 until the role has started and from the first instant of a drain, which is what the chart's readiness probe on those roles reads. Both endpoints return a body, never a bare `200 OK`. @@ -197,6 +213,17 @@ the milliseconds and the line above. Measured on the demo app's image, `ROLE=web **An app scaffolded before 23.0.0 does not have the line** — its `docker/Dockerfile` is the app's own file. Add it; the boot log names it until you do ([Upgrading](Upgrading)). +**The prebuilt module registry.** `As of 2026-11-09`. The same `RUN` also writes `node_modules/.cache/ultimate/registry/`: every module the app scan imports, in scan order, as chunk modules of 256 static imports each, plus `registry.json` (format, Bun version, file list). The `web`, `sync` and `replicator` roles boot from it instead of scanning; `worker` and `scheduler` keep their document-graph scan. + +| Case | What the boot does | +|---|---| +| the registry's file list matches the app's files on disk (a directory listing, no import) | imports the registry | +| it does not | logs `X_PREBUILT_REGISTRY_STALE` (warn) and scans — move the prebuilt `RUN` after the last `COPY` and rebuild | +| the registry came from another Bun | ignores it and scans | +| a module would not import at build time | no registry was written; the boot scans and refuses with that module's cause (`X_APP_MODULE_LOAD_FAILED`) | + +Every scan without a registry — tools, `x dev`'s first load, `x build` — imports in batches of 256 through one generated entry module per batch, in the serial scan's evaluation order (`packages/cli/src/app-load-order.test.ts`). Measured cold module load, 5,000 synthetic modules: serial 3,258 ms, batched 1,259 ms ([`boot-vs-files.json`](https://github.com/developerz-ai/ultimate/blob/main/scripts/bench/results/boot-vs-files.json)). + All targets share one build ID (content hash), stamped into the image, the HTML, the assets, `x.manifest.json` — **and `sw.js`**, `As of 2026-08`: the worker's cache names carry the build id, so a deploy retires the previous build's caches instead of serving them ([#390](https://github.com/developerz-ai/ultimate/issues/390)). Every target that serves or writes a document emits `manifest.webmanifest`, `sw.js` and `x-sw-register.js` alike ([PWA and offline](PWA-And-Offline)). ``` diff --git a/wiki/Entities-And-Migrations.md b/wiki/Entities-And-Migrations.md index dcd2b5cb3..2f748ebb7 100644 --- a/wiki/Entities-And-Migrations.md +++ b/wiki/Entities-And-Migrations.md @@ -872,6 +872,23 @@ vocabulary cannot express, said precisely rather than approximated: | Not authz | an invariant is about data shape. Permission belongs in [Policies and authz](Policies-And-Authz) | | Duplicate entity | two entities on one table name is `X_ENTITY_DUPLICATE` — rename or merge | +## Full-text search — the `searchIndex` port + +Rule: `.search(term)` on a chain asks the INSTALLED `SearchIndex` — one search path. `As of 2026-11-09`. Canonical wording: `packages/entity/README.md`, "Search index". + +| Index | How `.search(term)` answers | +|---|---| +| the built-in Postgres one (default) | inside the statement, its `matches` predicate over the `.searchable()` columns — the same SQL as before the port; costs a write nothing | +| an external one (`searchIndex` in `apps//runtime.ts`: Meilisearch, Typesense, the app's adapter) | ids first, then the chain reads those rows — tenancy, soft delete, order and cursor still apply, so a hit can never widen what the chain sees | + +| Fact | Detail | +|---|---| +| the version | every `SearchDocument` carries `version`, the row's `onUpdateNow()` instant in epoch ms. A searchable entity declares `updatedAt: timestamp().defaultNow().onUpdateNow()`; without one `searchDocumentOf` and `reindex()` refuse (`X_INVARIANT_VIOLATED`) | +| keep only newer | an adapter applies a document only when its version is newer than the document or tombstone it holds for that id; `remove(scope, ids, version)` leaves a tombstone at the delete's instant; an equal version is a replay and changes nothing. `searchIndexConformance` (`@ultimat3/testing/conformance`) checks a stale write after a newer one, a stale write after a remove, and a re-create after a remove | +| the write path | with an external index installed, a committed insert, upsert, update or transition of a searchable entity indexes the row, and a delete (soft or hard) removes it — fed by the post-commit `entity.row` event, so a rolled-back write indexes nothing | +| a failed index write | never fails the business write: one `entity.search-index.write-failed` log line; repair with `reindex()` ([Jobs](Jobs-And-Workflows#rebuilding-a-search-index--reindex)) | +| not mirrored | `updateWhere` / `deleteWhere` (an `entity.bulk`, no rows), and a delete on an entity whose primary key is not `id` (the row is not read). Each is logged the same way. `reindex()` re-indexes what an `updateWhere` changed; it removes nothing, so ids a `deleteWhere` or a non-`id` delete left behind stay in the index until the app removes them | + ## Embeddings ```ts diff --git a/wiki/Error-Codes.md b/wiki/Error-Codes.md index f0afc1234..45a220484 100644 --- a/wiki/Error-Codes.md +++ b/wiki/Error-Codes.md @@ -244,12 +244,12 @@ A denial is `X_FORBIDDEN`, above — `@ultimat3/policy` owns it and every surfac | `X_SESSION_EXPIRED` | session passed its idle or absolute expiry | `cause` names which of the two clocks ran out | sign in again, or raise `session.absoluteTtlMs` / `session.idleTtlMs` in `defineAuth({ session })`. The wiki used to name `POST /auth/sign-in` here; the framework mounts no such route, and the password half is the app's until it ships one | | `X_MFA_REQUIRED` | password proven, second factor outstanding | the user has TOTP enrolled, and `login()` / `completeOAuthLogin()` throw before any session is written. `meta.challenge` is a sealed, five-minute credential for the second leg — `meta`, never `cause`: both public surfaces publish `cause` and drop `meta`, so the sign-in handler decides how it travels | catch `X_MFA_REQUIRED` in your sign-in handler, hand `error.meta.challenge` to the caller, and finish with `await completeMfa(auth, challenge, code)`. No route ships. A wrong code is `X_UNAUTHENTICATED` and stays counted against the account, address and tenant buckets; past `rateLimit.maxAttempts` it is `X_ACCOUNT_LOCKED`. Sealing needs the app's master key — `X_SEAL_KEY_MISSING` without one | | `X_MFA_SECRET_INVALID` | a TOTP secret that is not RFC 4648 base32, so it decodes to zero bytes | `enrolTotp` is handed an imported secret outside the alphabet, or `totpCode` is called with one. `verifyTotp` does **not** throw it — an unreadable stored secret reads as a non-verdict there, the same rule `verifyAgainst` follows for a password hash Bun cannot read, so a login checking a broken row gets a credential refusal rather than a 500 | issue a fresh one and store it: `const { secret } = enrolTotp(auth, { account: user.email })`. An imported secret must be base32 (`A-Z` and `2-7`; padding, spaces and dashes are ignored) and must decode to at least one byte. Zero bytes is not a decode that failed closed: an HMAC keyed with zero bytes is a valid HMAC, so every unreadable secret in the table used to verify against one shared code stream that needed no secret to compute | -| `X_OAUTH_STATE_INVALID` | state, nonce or PKCE verifier did not match | a replayed callback URL, a handshake that expired, or a token minted for another browser | restart the flow at `GET /auth/oauth/` — a callback URL is single-use | -| `X_OAUTH_EXCHANGE_FAILED` | the provider refused the exchange or returned no usable identity | wrong client secret, an unregistered `redirect_uri`, a spent code, or a missing scope. Also a discovery document that declares a different `issuer` than the one asked for, and a key set with no importable key (the cached keys are kept). **Not** the user pressing Cancel — that is `X_OAUTH_DENIED` | `meta.stage` is `token` or `userinfo`: for `token`, match `_CLIENT_SECRET` and the registered `redirect_uri`, then `x doctor --json`; for `userinfo`, restart at `GET /auth/oauth/` | -| `X_OAUTH_TOKEN_INVALID` | the id token failed its signature, issuer, audience or expiry check | `As of 2026-08` the signature is verified too, against the issuer's published key set through a JWKS client — until this cycle the token was decoded and its claims read, so a token minted by whoever posted it to the callback passed every check it faced. Also: a token that is not a three-segment JWT, a payload with no `iss`/`sub`/`exp`/`aud`, a client id in `.env` that is not the one the authorize URL was built with, or a skewed host clock | the `fix` branches on which check failed — for a signature, confirm the token came from the provider's token URL and was not minted by the caller; for the rest, match `_CLIENT_ID` to the id `beginOAuth()` used, then restart at `GET /auth/oauth/` | +| `X_OAUTH_STATE_INVALID` | state, nonce or PKCE verifier did not match | a replayed callback URL, a handshake that expired, or a token minted for another browser | restart the flow at `GET /auth/sso/` — a callback URL is single-use | +| `X_OAUTH_EXCHANGE_FAILED` | the provider refused the exchange or returned no usable identity | wrong client secret, an unregistered `redirect_uri`, a spent code, or a missing scope. Also a discovery document that declares a different `issuer` than the one asked for, and a key set with no importable key (the cached keys are kept). **Not** the user pressing Cancel — that is `X_OAUTH_DENIED` | `meta.stage` is `token` or `userinfo`: for `token`, match `_CLIENT_SECRET` and the registered `redirect_uri`, then `x doctor --json`; for `userinfo`, restart at `GET /auth/sso/` | +| `X_OAUTH_TOKEN_INVALID` | the id token failed its signature, issuer, audience or expiry check | `As of 2026-08` the signature is verified too, against the issuer's published key set through a JWKS client — until this cycle the token was decoded and its claims read, so a token minted by whoever posted it to the callback passed every check it faced. Also: a token that is not a three-segment JWT, a payload with no `iss`/`sub`/`exp`/`aud`, a client id in `.env` that is not the one the authorize URL was built with, or a skewed host clock | the `fix` branches on which check failed — for a signature, confirm the token came from the provider's token URL and was not minted by the caller; for the rest, match `_CLIENT_ID` to the id `beginOAuth()` used, then restart at `GET /auth/sso/` | | `X_OAUTH_PROVIDER_DUPLICATE` | two oauth providers were registered under one id | two `registerOAuthProvider` calls claiming one id. Refused at boot rather than replacing silently, because the winner would be whichever module imported second — and it decides where every login for that id goes, **including which `issuers` an id token may claim** | give one of them a different id, or delete the duplicate `registerOAuthProvider({ id: '' })` call | | `X_OAUTH_PROVIDER_UNKNOWN` | the URL named a provider this app has not enabled | the `:provider` segment is not a known provider, **or** is one but was left out of `defineAuth({ providers })`; also `providerFor()` reaching for an id nothing registered. One refusal for all of them: distinguishing them would describe the app's configuration to an anonymous caller | `registerOAuthProvider({ id: '' })` if it is not built in, then add that id to `defineAuth({ providers: [...] })` — the registration comes **first** because a segment nothing registered cannot be added to `providers` at all, so "add it" alone was an instruction that could not be followed. The list the `fix` prints is what is **known here**, never the subset this app enabled, and `meta` carries only `{ provider }` | -| `X_OAUTH_DENIED` | the user or the provider declined the authorization | `error=` on the callback redirect — nearly always the consent screen's Cancel. Split out of `X_OAUTH_EXCHANGE_FAILED` so the commonest non-success outcome of a login stops paging on-call as a 502 provider outage | restart at `GET /auth/oauth/` and approve the consent screen; nothing is wrong with the app | +| `X_OAUTH_DENIED` | the user or the provider declined the authorization | `error=` on the callback redirect — nearly always the consent screen's Cancel. Split out of `X_OAUTH_EXCHANGE_FAILED` so the commonest non-success outcome of a login stops paging on-call as a 502 provider outage | restart at `GET /auth/sso/` and approve the consent screen; nothing is wrong with the app | | `X_PASSWORD_WEAK` | strength check rejected the password | too short, or a known-common password | choose a longer, uncommon password — or relax `defineAuth({ password: { minLength } })` | | `X_ACCOUNT_LOCKED` | per-ip, per-account or per-org bucket is inside its lockout | repeated failed attempts — password or second-factor code. `cause` names the caller's own key (an address or an email) and the seconds left; a locked **org** is never named, because its id was looked up from the address the caller typed. The wiki used to name `x auth unlock `, a command that has never existed | wait out the seconds `cause` names — or clear this one bucket: `await auth.limiter.recordSuccess('')`, `await auth.orgLimiter.recordSuccess(orgKey(user.orgId))` for a tenant — or raise `defineAuth({ rateLimit })`. Never `reset()`: it drops every bucket in the table, including the spray this lockout is holding | | `X_AUTH_LIMITER_NOT_SHARED` | the app declared a fleet-wide credential throttle and the limiter keeps its counters per process | `defineAuth({ rateLimit: { scope: 'shared' } })` against the built-in limiter, whose `scope` is `'process'` — account lockout would be `maxAttempts × replicas`, and a lockout on one replica invisible to the others. Refused at `defineAuth`, never at a request | pass a shared `AuthLimiter` to `defineAuth({ limiter })`, or drop to `scope: 'process'` | @@ -266,8 +266,11 @@ A denial is `X_FORBIDDEN`, above — `@ultimat3/policy` owns it and every surfac | `X_PASSKEY_INVALID` | a passkey ceremony response failed its signature, origin, rp id, flag or counter check | the assertion or attestation was forged, tampered with, made for another origin or rp id, made without user verification, or came from a cloned authenticator whose signature counter went backwards | `x errors explain X_PASSKEY_INVALID --json, then start the ceremony again: await passkeys.loginOptions()` | | `X_PASSKEY_CHALLENGE_EXPIRED` | a passkey ceremony named a challenge that is unknown, already used or past its ttl | the response was replayed, the options were issued by another ceremony, or the person took longer than the challenge ttl | `x errors explain X_PASSKEY_CHALLENGE_EXPIRED --json, then fetch fresh options and run the ceremony once: await passkeys.loginOptions()` | | `X_PASSKEY_STEP_UP_REQUIRED` | a sensitive action ran without a fresh user-verified passkey proof for its digest | the call carried no assertion, or the assertion approved other bytes than the ones this call would act on | `x errors explain X_PASSKEY_STEP_UP_REQUIRED --json, then fetch stepUp.options({ userId, input }), run getPasskey(options) in the browser and resend the call with the response` | +| `X_SAML_RESPONSE_INVALID` | a saml response failed its signature, wrapping, audience, recipient, time, InResponseTo or replay check | the IdP signing certificate rotated, the SP entity id or ACS URL differs from what the IdP has configured, the clocks are skewed, or the response was tampered with or replayed | `edit the samlIdentityProvider({ metadata, spEntityId, acsUrl }) call: paste the IdP current metadata XML and the entity id and ACS URL the IdP application is configured with` | +| `X_SAML_METADATA_INVALID` | saml IdP metadata has no entity id, HTTP-Redirect sign-on endpoint or signing certificate | the metadata XML was truncated, is SP metadata rather than IdP metadata, or names only an HTTP-POST sign-on endpoint | `download the IdP metadata again from its federation metadata URL and pass the whole XML to samlIdentityProvider({ metadata })` | +| `X_IDENTITY_PROVIDER_UNKNOWN` | the URL named an identity provider the sso routes were not given | a sign-in link or the IdP ACS URL carries an id that no entry in identityProviderRoutes({ providers }) declares | `add the provider to identityProviderRoutes({ providers: [...] }), or correct the id in the link and in the IdP ACS URL` | -`GET /auth/oauth/` is the app's route to mount, not one the framework serves. Every OAuth fix above is reachable only where the host routes matching requests to `oauthLogin(auth).start.handle`; `oauthLogin()` supplies the descriptors and nothing else mounts them. The thrown `fix:` and the descriptor's `path` both read the one declaration in `oauth-paths.ts`, so the string cannot go stale — but `As of 2026-08` `serve.ts` composes a fixed set of route tables with no seam for a raw `Route`, so nothing `x new` scaffolds mounts the pair and an app that has not mounted it itself answers that path with a 404. +`GET /auth/sso/` is the app's route to mount, not one the framework serves. Every OAuth fix above is reachable only where the host routes matching requests to `oauthLogin(auth).start.handle`; `oauthLogin()` supplies the descriptors and nothing else mounts them. The thrown `fix:` and the descriptor's `path` both read the one declaration in `oauth-paths.ts`, so the string cannot go stale — but `As of 2026-08` `serve.ts` composes a fixed set of route tables with no seam for a raw `Route`, so nothing `x new` scaffolds mounts the pair and an app that has not mounted it itself answers that path with a 404. ## Entity and database @@ -349,6 +352,8 @@ A denial is `X_FORBIDDEN`, above — `@ultimat3/policy` owns it and every surfac | `X_DRIVER_UNSUPPORTED` | a custom driver was asked for a capability it does not declare | defineDriver({ capabilities }) omits the capability the call needs (meta.capability): transactions, lock, series, upsert, matches or aggregate: | `implement the method on the raw repo and add its name to capabilities in defineDriver({ … }), or run that call on postgresDriver()` | | `X_DRIVER_CAPABILITY_MISSING` | a custom driver declares a capability its raw repo does not implement | defineDriver({ capabilities }) names a capability (meta.capability) whose method the repo() factory does not return, or transactions with no transactor() | `add the method the cause names to the object repo() returns in defineDriver({ … }), or drop the capability from capabilities` | | `X_DRIVER_ROW_OUT_OF_SCOPE` | a custom driver returned a row outside the plan it was given | the raw repo's select() ignored a predicate the framework scoped the read with — another tenant's row, a soft-deleted row, or another key — and the kit refused it before the caller saw it | `evaluate every plan.where predicate in the driver's select(): selectRows(entity, rows, read) from @ultimat3/entity/adapter does it for an in-process store` | +| `X_MIGRATION_PARTITION_CHANGE` | an existing table gained or lost entity({ partitionBy }) | Postgres cannot partition a table in place, nor turn a partitioned one back into a plain table: both are a new table and a copy of every row | `put partitionBy back as the table was created, or declare a new entity with partitionBy, backfill() its rows from the old one, then drop the old one in a later release (wiki/Migrations-And-Backfills.md#partitioned-tables)` | +| `X_ENTITY_PARTITION_INVALID` | entity({ partitionBy }) declares a partitioning Postgres cannot build | the primary key or a unique index does not hold the partition column, the column is no timestamp() or date(), the interval is unknown, or retain/view/history sits beside it | `edit the entity's partitionBy as the error's cause says — e.g. primaryKey: ['id', 'at'] — then x db gen "partition"` | ## Actions @@ -466,6 +471,14 @@ link at. `As of 2026-08-23`. | `X_SUBJECT_TABLE_UNDECLARED` | a table reaches the declared subject and defineSubject gives it no treatment | an entity holds a foreign key into the subject (or into a table the subject erase deletes) and tables: in defineSubject names no treatment for it — usually a table added after the subject was declared. The cause prints the reference path | `tables: {
: 'delete' } in defineSubject — or { anonymize: { : } }, or { retain: '' }` | | `X_SUBJECT_TREATMENT_INVALID` | a defineSubject treatment cannot be carried out | a blank retain reason, an anonymize naming a column the entity does not have (or its key), 'delete' on an append-only or soft-delete entity, a treatment for a table nothing reaches, or a kept row whose parent the erase deletes. The cause names the table and the field | `tables: {
: { retain: '' } } in defineSubject — the cause names the field to change` | | `X_SUBJECT_CYCLE` | the subject erase has no deletion order | two tables the erase deletes reference each other (or one references itself) and no foreign key in the loop is nullable, so neither side can go first | `x db gen "nullable " after making one key in the loop .nullable(): the erase clears a nullable key before deleting either side` | +| `X_LOCK_NOT_ACQUIRED` | withLock could not take the lock within its wait bound | another holder kept the key past wait:, or wait: is shorter than the work the holder does under it | `withLock(key, fn, { ttl: '30s', wait: '10s' })` — in the calling file: raise wait:, or retry the caller later | +| `X_LOCK_LOST` | a lock withLock held expired before its work finished | a renewal was refused or did not land within ttl:, so another holder may hold the key now; the signal handed to fn aborted | `withLock(key, fn, { ttl: '2m' })` — in the calling file: a ttl: longer than one renewal round-trip, and fn passes lock.signal to every write | +| `X_LOCK_STORE_MISSING` | withLock found no lease store to hold the lock in | no job driver is installed, or the installed driver has no leases store | `jobs: postgresJobDriver()` — in apps//runtime.ts — or pass { store } to withLock in the calling file | +| `X_LEDGER_UNBALANCED` | a ledger posting does not sum to zero in every currency | the debits and credits of one posting differ in at least one currency | `entries: [{ account: 'cash', amount: { minor: -500, currency: 'EUR' } }, { account: 'revenue', amount: { minor: 500, currency: 'EUR' } }]` — in the enqueue of the ledger job: every currency nets to zero | +| `X_LEDGER_ENTRY_INVALID` | a ledger entry is not a whole-minor Money amount on a named account | an entry has a fractional or unsafe minor, a currency that is not an ISO 4217 code, a scale differing within one currency, an empty account, or the posting has fewer than two entries | `{ account: 'cash', amount: { minor: 1299, currency: 'EUR' } }` — in the enqueue of the ledger job: integer minor units (fromDecimal('12.99', 'EUR') from @ultimat3/money), never a float | +| `X_PERSONAL_DATA_UNERASED` | a subject erase would leave a personalData() column of a reached table | defineSubject gives a table it reaches { anonymize } without naming every personalData() column, so the erase leaves the person's value there | `tables: {
: { anonymize: { : null } } }` — in defineSubject(, …): name every personalData() column, or 'delete', or { retain: '' } | +| `X_PERSONAL_DATA_UNREACHED` | an entity carries personalData() columns no declared subject reaches | no defineSubject() walk reaches the entity, so neither an export nor an erase would ever see its marked columns | `defineSubject(, { … })` — in the module declaring the subject: a subject of its own, or a reference / columns: entry so an existing subject reaches it | +| `X_LEDGER_POSTING_CONFLICT` | a ledger posting key is already stored with other entries | the key was posted before and this posting differs in its line count or in a line, so writing the new lines beside the stored ones would leave the book unbalanced | `entries: [ … the posting as first stored … ]` — in the enqueue of the ledger job — or a new key (e.g. ':adjust-1') for a correcting posting | ### Backfills @@ -605,6 +618,8 @@ No secret and no signature reaches a `cause`, a `fix` or a `meta`, and a URL is | `X_STORAGE_MULTIPART_UNSUPPORTED` | this disk cannot take an upload in parts | a custom StorageDriver declares no multipart capability, and grantMultipart was asked to open a session on it | `implement StorageDriver.multipart on the driver, or grant one PUT with grantUpload({ disk, orgId, request })` | | `X_STORAGE_UPLOAD_UNKNOWN` | no open multipart upload has that id | the uploadId was never opened on this disk for that key, or it was already completed or aborted | `grantMultipart({ disk, orgId, request }) again, and upload the parts under the new uploadId` | | `X_STORAGE_PARTS_INVALID` | the parts named do not match the parts stored | completeUpload was handed a part the disk never received, parts out of ascending order, or an etag that is not the stored one | `listParts({ disk, orgId, key, uploadId }) to see what landed, resend the missing part, then completeUpload again` | +| `X_STORAGE_UPLOAD_INFECTED` | the upload scanner found malware in the file | promoteAttachment ran the uploadScanner installed in runtime.ts over its per-attempt copy (pending/scan/) and it answered infected; the pending upload and that copy are deleted, and the row's key is never written | `grantUpload({ disk, orgId, request }) for a new, clean file (this one is gone; meta.signature names the detection)` | +| `X_STORAGE_SCAN_UNAVAILABLE` | the upload scanner could not give a verdict | the uploadScanner installed in runtime.ts rejected (daemon down, timeout, file past its limit), or the per-attempt copy could not be read for it, and the upload policy fails closed; the attempt copy is deleted and the upload stays pending | `promoteAttachment({ disk, key, orgId, target, policy }) # again once the scanner answers — or uploadPolicy({ onScanUnavailable: 'accept' }) to promote unscanned files` | ## Flags @@ -761,6 +776,8 @@ Three of them are raised at `notifier(...)`, while the app's modules load, so th | `X_NOTIFY_PUSH_DEVICE_GONE` | the push provider says a device token is dead | APNs answered 410 or FCM answered UNREGISTERED for a device token: the app was uninstalled, or the device rotated its token | `x jobs list --json` — nothing to repair: the dead device row is already deleted, and the app stores its new token with its next pushRegister() call | | `X_NOTIFY_PUSH_PROVIDER_REFUSED` | APNs or FCM refused a push for good | the provider refused the request and will refuse it again: a bad or revoked credential, a wrong topic or project, or a payload over its limit | `x jobs list --json` — read meta.provider, meta.status and meta.reason on the failed step, then fix the credential (x secrets set ULTIMATE_APNS_PRIVATE_KEY) or the message | | `X_NOTIFY_PUSH_MESSAGE_INVALID` | a data-only push carries more than an id | the data of a data-only push names a reserved key, holds more than 8 keys, or a value longer than 256 characters | `pushChannel({ pusher, data: ({ event }) => ({ id: event.params.id }) })` — in the notifier: one id per push, the app fetches the rest over its API | +| `X_NOTIFY_SMS_TRANSPORT_MISSING` | an SMS was sent with no SmsTransport installed | smsChannel() delivered outside development and test, and apps//runtime.ts names no smsTransport — the memory transport would swallow the message, so the send is refused instead | `add smsTransport: yourTransport() to apps//runtime.ts — behavesLike(smsTransportConformance, () => yourTransport()) proves it` | +| `X_NOTIFY_SMS_NUMBER_INVALID` | an SMS number is not E.164 | the number smsChannel() resolved for a recipient is not +, a country code and up to 15 digits in all (E.164) — a local spelling a provider guesses the country of is a message sent to a stranger | `store numbers as E.164 (+4915112345678) and return that from smsChannel({ numberOf })` | ## MCP and AI @@ -920,6 +937,14 @@ Two sets override the table, in `failures.ts`: | `X_SCRAPE_EGRESS_IN_PAYLOAD` | the run exit carries a credential that is also in the job payload | scrape({ egress }) answered a proxy URL whose password is readable out of the run's own input, so it rode the queue payload and is stored in x_jobs in the clear. Refused before the browser opens; terminal, because every remaining attempt reads the same row | `look the exit up by id inside the worker — egress: async ({ connectionId }) => (await repo.connectionById(connectionId))?.exit — and drop the exit field from the scrape input` | | `X_SCRAPE_LAUNCH_ARGS_INVALID` | the launch options carry arguments this driver cannot hand the browser | **terminal.** Refused before anything is launched: `localBrowser({ options: { args } })` is not an array of strings, or it sets a proxy switch (`--proxy-server`, `--proxy-bypass-list`, `--proxy-pac-url`, `--proxy-auto-detect`, `--no-proxy-server`, either prefix, any case). The exit is the driver's `proxy` or the run's `egress`, appended after the caller's args; a second one in the args would let the browser leave by a route the session does not report. The fix lists every `localBrowser(` call and proxy switch; the cause names the edit — delete the switch and pass the exit as `localBrowser({ proxy })` or `scrape({ egress })`, or pass `args` as a list of strings | `grep -rnE --include='*.ts' -e 'localBrowser\(' -e '--(no-)?proxy-' .` | | `X_SCRAPE_PDF_OPTIONS_INVALID` | the PDF request names no single page size a print engine can lay out | page.pdf() was given format beside widthMm/heightMm, one of widthMm/heightMm alone, a size beside preferCssPageSize, or a measure that is not a finite number | `page.pdf({ format: 'a4' }) — exactly one size: a format, or widthMm with heightMm, or preferCssPageSize alone` | +| `X_SCRAPE_PDF_ORIGIN_MISSING` | pdf() has no origin to render the route on | APP_URL is unset in this process and the pdf() declaration names no origin, so there is no server for the browser to load the route from | `set APP_URL=https://your.app in the environment, or pdf({ origin: () => 'http://127.0.0.1:3000' }) in the declaring file` | +| `X_SCRAPE_PDF_ROUTE_INVALID` | a pdf() route is not a path on this app | route() answered something other than a path beginning with one slash — a full URL, a protocol-relative //host path or a backslash — and the printing browser only ever loads the app own origin | `return a path from route(): pdf({ route: ({ input }) => '/invoices/' + input.id + '/print' })` | +| `X_SCRAPE_PDF_ROUTE_FAILED` | the route pdf() printed answered an error status | the page the browser loaded answered 4xx or 5xx — a missing record, a route the printing browser is not authorised for (no headers: given), or a server error — and printing it would store an error page as the document | `curl -i "$APP_URL" — then pass pdf({ headers }) for a private route, or fix the page` | +| `X_CDP_BROWSER_MISSING` | no Chrome or Chromium is installed for the e2e driver to launch | the driver launches a real browser over raw CDP and carries **no dependency** to do it, so what it needs is a binary on disk — `CHROME_PATH` first, then the four `/usr/bin` names GitHub-hosted `ubuntu-latest` ships. Reached only by a caller that has already decided a browser is required: `openE2eBrowserIfAvailable()` answers `undefined` instead, so a machine with no browser SKIPS rather than failing | `export CHROME_PATH=/usr/bin/google-chrome` — or leave it unset and let the browser-backed suite skip | +| `X_CDP_LAUNCH_FAILED` | the browser started and never announced a DevTools endpoint | Chrome is driven over `--remote-debugging-pipe`, so its first answer IS the announcement. It gets the launch deadline (60 s, or `timeoutMs` when that is larger) and, unanswered, is reaped and started ONCE more on a fresh profile. The cause carries each launch: whether it exited (with its code) or was still silent at the deadline, and its last stderr lines — D-Bus lines alone are start-up noise, so a deadline beside them is a machine too slow, an exit code a crash, a rejected flag or a sandbox refusal | `/usr/bin/google-chrome --headless=new --no-sandbox --disable-dev-shm-usage --disable-gpu --remote-debugging-port=0 about:blank` — the binary the cause names; a healthy one prints `DevTools listening on ws://…` | +| `X_CDP_CALL_FAILED` | the browser refused a DevTools call | the call reached the browser and came back an error frame, or the connection died under it — every in-flight call is settled on close, so a dead browser reports this rather than one deadline per call | print what the page had: `await page.evaluate(() => document.body.innerHTML)` — a call that refuses the same way means the browser itself is gone, so read the launch above it | +| `X_CDP_TIMEOUT` | a DevTools call did not answer inside its deadline | its own code rather than `X_TIMEOUT`, because the actionable half is WHICH call: a hung `Page.navigate` is an app that never finishes responding, a hung `Runtime.evaluate` is an expression that never settles. A call the connection timed carries what the wire did meanwhile — `meta.reading` is `target-gone` (its session detached, or its target was destroyed or crashed), `lost-in-transport` (`meta.framesDropped` frames arrived unparseable after the call) or `no-answer` (the target was there and silent; `meta.framesArrived`, `meta.lastFrames`, `meta.navigations`, `meta.transportOpen` say what else happened) | `no-answer`: raise `timeoutMs` on `installE2eDriver({ timeoutMs })`, or find the request the page is still waiting on. The other two: `x test e2e --json` again — the cause names what went | +| `X_SCRAPE_PDF_BUSY` | every pdf() render slot stayed busy past the queue deadline | more renders arrived at once than the renderer launches browsers for (maxConcurrent, default 2), and this one waited out queueTimeoutMs — each Chrome holds 150-300 MB, so the bound keeps a burst from exhausting the web role memory | `pdf({ renderer: chromePdfRenderer({ maxConcurrent: 4 }) })` — only on a host with the memory for it; a caller retries after the Retry-After it was answered | `X_NOT_IMPLEMENTED` and `X_ENV_MISSING` are `@ultimat3/core`'s and are documented above: the first is the honest stub the agent recovery seam throws, the second is what a declared secret with no value in the environment is — a missing environment variable, not a scraping concept needing its own code. @@ -973,10 +998,6 @@ Two sets override the table, in `failures.ts`: | `X_E2E_LOCATOR_EMPTY` | an e2e locator matched no element | resolved lazily, so this is raised at the question, not at construction — the selector, role or text is quoted back as the call that was written | print what the page actually rendered: `await page.evaluate(() => document.body.innerHTML)`, then narrow or correct the locator call the cause quotes back | | `X_E2E_LOCATOR_AMBIGUOUS` | an e2e locator matched more than one element and was asked to click | an assertion handed three matches has an answer; a click does not, so `isVisible()`/`count()` answer and `click()` refuses — the "wrong receiver throws, wrong value returns" rule | add `.first()`, or narrow the locator | | `X_E2E_SERVICE_WORKER_ABSENT` | no service worker took control of the page within the budget | bounded **in the page** rather than by a loop in the harness, so there is exactly one retry mechanism; an unbounded wait is a test that hangs rather than fails | confirm the page registers one: `await page.evaluate(() => navigator.serviceWorker?.controller !== undefined)` — if it does, pass a longer budget to `waitForServiceWorker({ timeoutMs })` | -| `X_CDP_BROWSER_MISSING` | no Chrome or Chromium is installed for the e2e driver to launch | the driver launches a real browser over raw CDP and carries **no dependency** to do it, so what it needs is a binary on disk — `CHROME_PATH` first, then the four `/usr/bin` names GitHub-hosted `ubuntu-latest` ships. Reached only by a caller that has already decided a browser is required: `openE2eBrowserIfAvailable()` answers `undefined` instead, so a machine with no browser SKIPS rather than failing | `export CHROME_PATH=/usr/bin/google-chrome` — or leave it unset and let the browser-backed suite skip | -| `X_CDP_LAUNCH_FAILED` | the browser started and never announced a DevTools endpoint | Chrome is driven over `--remote-debugging-pipe`, so its first answer IS the announcement. It gets the launch deadline (60 s, or `timeoutMs` when that is larger) and, unanswered, is reaped and started ONCE more on a fresh profile. The cause carries each launch: whether it exited (with its code) or was still silent at the deadline, and its last stderr lines — D-Bus lines alone are start-up noise, so a deadline beside them is a machine too slow, an exit code a crash, a rejected flag or a sandbox refusal | `/usr/bin/google-chrome --headless=new --no-sandbox --disable-dev-shm-usage --disable-gpu --remote-debugging-port=0 about:blank` — the binary the cause names; a healthy one prints `DevTools listening on ws://…` | -| `X_CDP_CALL_FAILED` | the browser refused a DevTools call | the call reached the browser and came back an error frame, or the connection died under it — every in-flight call is settled on close, so a dead browser reports this rather than one deadline per call | print what the page had: `await page.evaluate(() => document.body.innerHTML)` — a call that refuses the same way means the browser itself is gone, so read the launch above it | -| `X_CDP_TIMEOUT` | a DevTools call did not answer inside its deadline | its own code rather than `X_TIMEOUT`, because the actionable half is WHICH call: a hung `Page.navigate` is an app that never finishes responding, a hung `Runtime.evaluate` is an expression that never settles. A call the connection timed carries what the wire did meanwhile — `meta.reading` is `target-gone` (its session detached, or its target was destroyed or crashed), `lost-in-transport` (`meta.framesDropped` frames arrived unparseable after the call) or `no-answer` (the target was there and silent; `meta.framesArrived`, `meta.lastFrames`, `meta.navigations`, `meta.transportOpen` say what else happened) | `no-answer`: raise `timeoutMs` on `installE2eDriver({ timeoutMs })`, or find the request the page is still waiting on. The other two: `x test e2e --json` again — the cause names what went | | `X_TEST_JOB_EXPECTED` | a matcher expected a job declaration and got something else | `toEmitSteps`/`recordSteps` called on `job.run` or an unrelated value | `call toEmitSteps(myJob)` with the job export, not `toEmitSteps(myJob.run)` | | `X_TEST_NETWORK_RACE` | a request raced `unsealNetwork()` and lost the patched fetch | `unsealNetwork()` called while a request from the same seal was still in flight | do not call `unsealNetwork()` while a request from the same test is still pending | | `X_TEST_FACTORY_TRAIT_UNKNOWN` | a factory was asked for a trait it does not declare | a typo at the call site, or a trait never added — the same symptom, so `cause` lists every declared trait to tell them apart | `declare it: defineFactory(
, { traits: { : { /* columns */ } } })` | @@ -1451,6 +1472,10 @@ Two sets override the table, in `failures.ts`: | `X_EXERCISE_FILE_UNEDITABLE` | x g could not add an entry to apps/web/exercise.ts | the app keeps apps/web/exercise.ts in a shape the generator cannot edit safely: no single export const exercise = defineExercise({ … }) with an object literal | `edit apps/web/exercise.ts — paste the entry the finding names into export const exercise = defineExercise({ … })` | | `X_ERROR_REPORTER_DSN_UNCLAIMED` | SENTRY_DSN is set, and the app installed a reporter marked neither as its Sentry transport nor as a local sink | the app installs an ErrorReporter of its own (a wrapper around sentryErrorReporter, a hand-written sink) without sentryTransport() or localErrorSink(), and the container also sets SENTRY_DSN; it might already be a transport, so the boot adds none and the DSN is unused | `edit the file that calls configureErrorReporting: wrap your Sentry wrapper in sentryTransport(wrapper), or a reporter that keeps reports on the box in localErrorSink(sink)` | | `X_BOUNDARY_ADAPTER_IMPORT` | an app module imports an adapter installer | a site/app/api/shared module imports @ultimat3//adapter at runtime | `export const runtime = { jobs: yourDriver } satisfies RuntimeOverrides; # in apps/web/runtime.ts — name the port your import installed` | +| `X_PREBUILT_REGISTRY_STALE` | the image's prebuilt module registry does not list the app's modules, so the role scanned instead | a module added, moved or deleted after x build --target prebuilt ran — the registry is written inside the image build, after COPY . . | `edit docker/Dockerfile: move RUN bun node_modules/@ultimat3/cli/src/bin.ts build --target prebuilt after the last COPY in the runtime stage, then rebuild the image` | +| `X_APP_ROOT_NESTED` | a second app.config.ts inside the app | An app.config.ts under apps/ or packages/ makes that workspace a second app root — its own registry, its own migrations, its own image. One repository is one app: every apps/* workspace boots in the one image, ROLE picks the process, and packages/db/migrations is the one migration owner | `x verify --only drift --json` — after deleting the nested app.config.ts and moving what it configured into the root app.config.ts | +| `X_WORKSPACE_NAME_TAKEN` | x g app or x g package was given a name the other workspace kind already holds | apps/ and packages/ are both named @/, so a second workspace under one name is two packages bun install cannot tell apart | `x g app admin-app --dry-run --json` | +| `X_PARTITIONS_UNMAINTAINED` | a partitioned entity with no partitions() job | An entity declares partitionBy and no partitions() job from @ultimat3/jobs is registered: no partition is ever created ahead, so the first insert past the last migrated partition fails in production, and retention never drops one | `x verify --only manifest --json` — after exporting partitions() from @ultimat3/jobs in a module listed under jobs in apps/web/api/index.ts, and its .schedule under tasks | ## Reserved codes diff --git a/wiki/Jobs-And-Workflows.md b/wiki/Jobs-And-Workflows.md index e6d3af493..f76203369 100644 --- a/wiki/Jobs-And-Workflows.md +++ b/wiki/Jobs-And-Workflows.md @@ -211,6 +211,30 @@ export const syncCrm = job({ A job over a tenant, queue or global cap, or over a `'wait'` concurrency cap, is **deferred, never dropped** — it stays queued with a later `runAt`. `whenBusy: 'fail'` is the one declared exception. `'wait'` holds the cap and states no order: a later enqueue can start first. +### A lock — `withLock()` + +`withLock(key, fn, { ttl, wait?, poll?, store?, signal? })`: one holder fleet-wide, held as one slot `lock:` in the job driver's `LeaseStore` (`x_job_leases` on Postgres) — no table of its own, no Redis. `fn` receives `{ key, token, signal }`. `As of 2026-11-09`. + +| Fact | Meaning | +|---|---| +| `ttl` (required) | how long a dead holder keeps the key; renewed at ttl/3 while `fn` runs | +| `wait` (default one ttl; `0` = one try) | then `X_LOCK_NOT_ACQUIRED` (409, retryable), naming the holder | +| `token` | the FENCING token: a number the store hands this grant (the database's counter, never a clock), greater than every token any earlier holder of any key got. Write it beside whatever `fn` changes and refuse a write carrying a smaller one — that stops a holder that paused past its `ttl` (GC, a frozen VM) from writing after the next one took over | +| no renewal landing within the ttl, or one refused | `signal` aborts with `X_LOCK_LOST`; the call rejects with it even if `fn` resolved | +| no store | `X_LOCK_STORE_MISSING` — `jobs: postgresJobDriver()` in `runtime.ts`, or `{ store }` | + +```ts +import { withLock } from '@ultimat3/jobs'; + +declare function sendDigest(orgId: string, options: { signal: AbortSignal }): Promise; +declare const orgId: string; + +await withLock(`digest:${orgId}`, ({ signal }) => sendDigest(orgId, { signal }), { + ttl: '2m', + wait: 0, +}); +``` + ### One run per key | Case under `whenBusy: 'fail'` | Answer | @@ -499,6 +523,18 @@ export const exportMember = subjectExport({ generated person, and searches every stored row. Their value outside a `retain` is `X_SUBJECT_ERASE_INCOMPLETE`, naming the table and column. +### `personalData()` coverage + +Rule: `.personalData(kind)` on a column (`@ultimat3/entity`) adds NO second export or erase path — `subjectExport`/`subjectErase` stay the one. It is the proof that the declared subjects cover every marked column. `As of 2026-11-09`. + +| Check (`@ultimat3/jobs`) | Refuses | +|---|---| +| `unerasedPersonalData(plan)` | a reached table whose `{ anonymize }` names fewer than every marked column — `X_PERSONAL_DATA_UNERASED`. `'delete'` clears the row; `{ retain: '' }` is a stated reason | +| `unreachedPersonalData(plans)` | a marked entity no subject's walk reaches — `X_PERSONAL_DATA_UNREACHED` | +| `personalDataProblems()` | both, over every `defineSubject()` and every marked entity | + +Enforced twice: a subject run refuses `X_PERSONAL_DATA_UNERASED` before it touches a row, and `x exercise` lists `X_PERSONAL_DATA_UNERASED` / `X_PERSONAL_DATA_UNREACHED` as findings. With an `auditSink` installed, `subjectErase` writes one `allowed` record per completed erase (`primitive: 'action'`, `surface: 'job'`, `input: { subject, run }` — the subject's entity and the run id, never the key, which an app may make an email) and one `failed` record per attempt that threw; a sink that refuses fails the run. At least once: a sink dedupes on the record's `idempotencyKey`, `:`. + ## Rollups and materialized views Two factories over `job()`, two jobs, `As of 2026-11`. A time window over append-mostly rows is @@ -566,6 +602,20 @@ export const refreshPostTotals = refreshView({ | `concurrently` | readers keep the old rows meanwhile; needs a unique index on the view (`.unique()`) | | Refused at declaration | a table, a plain view, `concurrently` with no unique index — `X_JOB_REFRESH_VIEW_INVALID` | +## Rebuilding a search index — `reindex()` + +`reindex({ name, entity, source, tenant, index? })` is a `backfill()` writing each row's `searchDocumentOf(entity, row)` to the `searchIndex` port (default: the installed one). Run it after swapping Postgres' built-in index for Meilisearch or Typesense, or after an index lost data. An entity with no `.searchable()` column is refused at declaration (`X_INVARIANT`). A deleted row is the write path's `remove`, never a sweep's. `As of 2026-11-09`. + +## Partition maintenance — `partitions()` + +`partitions({ name?, cron?, tz?, clock? })` keeps every `entity({ partitionBy })` table ready: one durable step per partitioned entity runs `@ultimat3/db`'s `maintainPartitions` — creation under a 5 s `lock_timeout` (`PARTITION_LOCK_TIMEOUT_MS`; a `55P03` is `X_DB_LOCK_TIMEOUT`, retryable, so a long report query delays the run rather than queueing every write behind ACCESS EXCLUSIVE on the parent), then each expired partition retired by `detach … concurrently` (outside any transaction, as Postgres requires) and dropped. It creates `ahead` partitions past the current one and retires those `retain` expired. A job plus a `.schedule` `task()` (`.schedule`, default `'17 3 * * *'` UTC, daily), as `consumer()` carries `.supervisor`. `now` is the job's clock, so a test moves it; a second run at one instant changes nothing. Report: `{ tables: [{ table, created, dropped }] }`. Moving older rows in: enqueue it once with `{ from: '' }` — that run creates every partition back to it and retires nothing, and its idempotency key is its own, so it never folds into the schedule's — then run the `backfill()`. `As of 2026-11-09`. + +```ts +import { partitions } from '@ultimat3/jobs'; + +export const maintainPartitions = partitions(); // list it in defineApi({ jobs }) +``` + ## Outbound webhooks — `webhook()` **An outbound webhook is a job: `webhook()` is a factory over `job()` that delivers one event to @@ -666,6 +716,56 @@ Codes: `X_WEBHOOK_ENDPOINT_UNKNOWN`, `X_WEBHOOK_ENDPOINT_INVALID`, `X_WEBHOOK_EN `X_WEBHOOK_EVENT_UNKNOWN`, `X_WEBHOOK_EVENT_INVALID`, `X_WEBHOOK_DELIVERY_FAILED`, `X_WEBHOOK_DELIVERY_THROTTLED`, `X_WEBHOOK_DELIVERY_REJECTED` ([Error codes](Error-Codes)). +## Double-entry postings — `ledger()` + +`ledger({ name, tenant, book })` is a factory over `job()`. Input: `{ key, entries: [{ account, amount: Money }], orgId? }`. The run refuses anything that is not balanced Money before a line is written, then writes every line in ONE durable step through the app's `LedgerBook`. No provider, account plan or currency policy ships (axiom 8). `As of 2026-11-09`. + +| Rule | Code | +|---|---| +| each currency's entries sum to zero on their own (an FX posting balances its EUR legs and its USD legs separately) | `X_LEDGER_UNBALANCED` (422, terminal) | +| integer `minor`, an ISO 4217 currency, one `scale` per currency, a non-blank account, two entries or more | `X_LEDGER_ENTRY_INVALID` (422, terminal) | +| idempotent by key: the live key is `:`; `entityLedgerBook` is one `insert … on conflict (posting_key, line) do nothing`. A replay of the same lines writes 0 | — | +| the key is already stored with OTHER lines — another count, another line | `X_LEDGER_POSTING_CONFLICT` (409, terminal); nothing is written, because the new lines beside the stored ones would unbalance the book | +| `assertBalancedPosting(posting)` | the enqueuing action refuses at the request | + +A provider's payment webhook is a `webhookReceiver()` whose `handle` enqueues the posting keyed by the provider's id: the receiver dedupes the delivery, the ledger dedupes the posting. + +```ts +import type { LedgerBook } from '@ultimat3/jobs'; +import { ledger, webhookReceiver } from '@ultimat3/jobs'; + +declare const book: LedgerBook; // entityLedgerBook({ driver, entity: ledgerLines }) in an app +declare function verifyPayment(bytes: Uint8Array, signature: string | null): { + id: string; + amountMinor: number; + currency: string; +}; + +export const postPayment = ledger({ name: 'payments.post', tenant: 'none', book: () => book }); + +export const paymentEvents = webhookReceiver({ + name: 'payments.events', + path: '/hooks/payments', + tenant: 'none', + verify: (inbound) => { + const event = verifyPayment(inbound.bytes, inbound.headers.get('x-signature')); + return { eventId: event.id, payload: event }; + }, + handle: async ({ eventId, payload, step }) => { + const { amountMinor, currency } = payload as { amountMinor: number; currency: string }; + await step.run('post', () => + postPayment.enqueue({ + key: eventId, + entries: [ + { account: 'cash', amount: { minor: amountMinor, currency } }, + { account: 'revenue', amount: { minor: -amountMinor, currency } }, + ], + }), + ); + }, +}); +``` + ## Driver interface One interface. **Job code never changes.** Step persistence hangs off the same object (`steps`), so it is identical on every implementation. It is the `jobs` adapter port: the interface is on `@ultimat3/jobs/adapter`, `memoryJobDriver` is the driver to copy, `behavesLike(jobDriverConformance, () => yourDriver())` proves one, and `jobs:` in `apps//runtime.ts` wires it — [Adapters](Adapters#jobs). The bus `step.waitForEvent` resumes from (`eventBus`) and the store behind the outbox (`outboxStore`) are two more ports, swapped the same way and no other: [Adapters](Adapters#eventbus), [Adapters](Adapters#outboxstore). diff --git a/wiki/Known-Gaps.md b/wiki/Known-Gaps.md index 9740b9099..5ac3adcdb 100644 --- a/wiki/Known-Gaps.md +++ b/wiki/Known-Gaps.md @@ -31,6 +31,7 @@ CHANGELOG.md` finds its bounds, and it changes on any commit. | Gap | Symptom | Work around it by | |---|---|---| +| `pdf()` prints with no session of its own | `As of 2026-11-09`. The Chrome that `pdf()` drives opens the app's route as nobody: no cookie jar is handed across, so a route behind a session answers exactly as it would a signed-out visitor — its 4xx is `X_SCRAPE_PDF_ROUTE_FAILED`, a sign-in page it renders is what gets printed. pin: none — the gap is an absence no assertion can hold (nothing hands the browser a cookie); the workaround is held by `packages/scraping/src/pdf-render-fence.test.ts`, 'a same-origin request carries the headers; a cross-origin one is blocked without them' | issue a short-lived token the print route accepts and send it through `pdf({ headers })` — [Scraping](Scraping#a-route-printed-to-pdf--pdf) | | A purge that a replica never hears is bounded by `ttl`, not by the next request | `As of 2026-10-10`. The purge reaches the other replicas through the invalidation broadcast. With the bus down, a replica with its own store answers its copy as a fresh hit until that copy's `ttl` (and tells an edge the ttl's remainder), then renders. `'purge'` therefore requires a `ttl` | choose the `ttl` as the longest a missed purge may last; or share one `runtime.isrStore` with a `tagFence`, where the delete is every replica's at once | | A bust purges by the **route's** tags | `As of 2026-10-10`. A stored `isr` page records no tags of its own, so `post:1` reaches every stored page of a route tagged `post` — one render too many per page, never one too few. Each bust reads the store's keys once; the route of each key is remembered, not re-matched [pin](https://github.com/developerz-ai/ultimate/blob/main/packages/render/src/render-isr-purge.test.ts#every%20stored%20key%20of%20the%20route%20goes%3A%20both%20locales%2C%20and%20every%20keyed%20query) | tag a route as narrowly as its data allows; a very large shared store pays one `paths()` per bust per replica | | A declared `isr` query parameter's **values** are unbounded | `As of 2026-10-10`. `query: ['page']` puts `page` in the key, and `?page=1` … `?page=999999` are each a key: the framework cannot know which values name a page. The store is bounded (least recently generated out first), so it thrashes rather than grows [pin](https://github.com/developerz-ai/ultimate/blob/main/packages/render/src/render-isr-query.test.ts#only%20a%20declared%20parameter%20is%20part%20of%20the%20key%2C%20and%20of%20the%20URL%20load%20is%20given) | answer `withStatus(404, …)` from `load` for a value that names nothing, and rate-limit the route at the ingress; put a small closed set in the path instead (`/blog/page/[n]` with `prerender()`) | diff --git a/wiki/Migrations-And-Backfills.md b/wiki/Migrations-And-Backfills.md index dd194e216..b377beed1 100644 --- a/wiki/Migrations-And-Backfills.md +++ b/wiki/Migrations-And-Backfills.md @@ -140,16 +140,41 @@ A hand-written migration with `create index concurrently`, `drop index concurren ## Partitioned tables -Rule: a partitioned table is a table. Declare it as an entity, `x db gen`, then hand-edit the -generated `create table`'s closing `);` to `) partition by range ("at");`. `As of 2026-11-06`. -No `entity({ partition })`: attach, default partition and retention are the app's choices. +Rule: declare it on the entity — `partitionBy: { column, interval, ahead?, retain? }` — and +`x db gen` writes the partitioned table. `As of 2026-11-09`. The hand edit at the end of this +section is the pre-31 recipe, still valid for a table `partitionBy` cannot describe (list or hash +partitioning, a non-time key). + +```ts +import { entity, text, timestamp, uuid } from '@ultimat3/entity'; + +export const events = entity('events', { + columns: { id: uuid(), at: timestamp().defaultNow(), orgId: uuid().tenant(), kind: text() }, + primaryKey: ['id', 'at'], + partitionBy: { column: 'at', interval: 'month', ahead: 3, retain: { after: '400d' } }, +}); +``` + +| Question | Answer | +|---|---| +| Refused at declaration | `X_ENTITY_PARTITION_INVALID`: the key or a unique index/invariant lacks the column; the column is no `timestamp()`/`date()`; an interval outside `day` · `week` · `month` · `year`; a row `retain` beside it; a view | +| What `x db gen` writes | `create table … partition by range ("at")`, the function `ultimate_partitions`, and a call creating the current partition plus `ahead` more AT APPLY TIME | +| Names | `
_pYYYYMMDD` (the lower bound, UTC); `week` is ISO, Monday first | +| Creating ahead, dropping old | `partitions()` from `@ultimat3/jobs`, daily ([Jobs](Jobs-And-Workflows#partition-maintenance--partitions)), runs `@ultimat3/db`'s `maintainPartitions`: creation under a 5 s `lock_timeout` (`X_DB_LOCK_TIMEOUT`, retried), then `retain.after` retires whole partitions whose upper bound it passed by `detach … concurrently`, then `drop` — never an ACCESS EXCLUSIVE queue on the parent. Idempotent; only `_pYYYYMMDD` children are touched | +| A row past every partition | refused by Postgres (`23514`). No default partition, by decision: one forbids `detach … concurrently`, and creating the month it caught moves rows under ACCESS EXCLUSIVE. Size `ahead × interval` to at least the scheduler outage you tolerate | +| Moving older rows in | enqueue `partitions()` once with `{ from: '' }` — it creates every partition back to it and retires nothing — then `backfill()` | +| Adopting on an existing plain table | refused at migrate time, `X_MIGRATION_PARTITION_CHANGE` (Postgres cannot partition in place: a new entity plus a `backfill()`); changing the interval or removing `partitionBy` is refused by `x db gen`, same code. A table the hand recipe already partitioned adopts cleanly | + +The hand recipe: declare it as an entity, `x db gen`, then hand-edit the generated `create table`'s +closing `);` to `) partition by range ("at");`. Attach, default partition and retention are then the +app's choices. | Question | Answer | |---|---| | Primary key | must hold the partition key — `primaryKey: ['id', 'at']`. Postgres refuses any unique index that does not (`0A000`) | | Drift | the PARENT is judged: columns, key, indexes, checks, triggers. A partition is not app schema (`pg_class.relispartition`), so next month's is no `unexpected-table` and its `fix:` is never `drop table`. A key Postgres clones per partition (`conparentid <> 0`) is not counted. `introspectSchema()` marks the parent `partitioned: true` | | A reference INTO one | no `.references()`. A one-column key is `42830` (no unique index on `id` alone can exist) and a same-tenant key's `(org_id, id)` index is `0A000`. Hand-write the composite key into the migration — `alter table "notes" add constraint "notes_event_fkey" foreign key ("event_id", "event_at") references "events" ("id", "at");` — drift judges declared keys only, so it is no finding. The repository's tenant filter still holds on reads; the database does not enforce the same tenant | -| Retention | `detach`, then `drop`, in the app's own job — never `retain`, never a `delete` | +| Retention | `partitionBy.retain` — never row `retain`, never a `delete`. By hand: `detach`, then `drop`, in the app's own job | | Next month | an app `task` on a monthly cron (`tz: 'UTC'`) enqueuing a job that runs the statement below, `if not exists` so a double fire is a no-op | | `create index concurrently` | refused on a parent (`0A000`): `X_MIGRATION_CONCURRENT_PARTITIONED`, whose fix is the sequence — each partition concurrently, the parent `on only`, then `alter index … attach partition`, in a `-- transaction: false` migration | | Schema dump | rendered: the parent with `partition by`, each partition `partition of … for values` in the parent's file. A partition's attached indexes, cloned keys and cloned triggers are not written: the parent's recreate them | diff --git a/wiki/Notify.md b/wiki/Notify.md index d25675a07..63d9843f9 100644 --- a/wiki/Notify.md +++ b/wiki/Notify.md @@ -71,6 +71,27 @@ Codes: `X_NOTIFY_CHANNELS_EMPTY`, `X_NOTIFY_CHANNEL_DUPLICATE`, `X_NOTIFY_DIGEST `X_NOTIFY_DELIVERY_FAILED`, and native push's `X_NOTIFY_PUSH_DEVICE_GONE`, `X_NOTIFY_PUSH_PROVIDER_REFUSED`, `X_NOTIFY_PUSH_MESSAGE_INVALID` — [Error codes](Error-Codes). +## SMS + +Rule: an SMS is a channel over the `smsTransport` port; no provider ships (axiom 8). `As of 2026-11-09`. + +```ts +import { smsChannel } from '@ultimat3/notify'; + +export const loginCodeSms = smsChannel<{ code: string }>({ + message: ({ event }) => ({ key: 'sms.login_code', params: { code: event.params.code } }), +}); +``` + +| Fact | Detail | +|---|---| +| the port | `smsTransport` in `apps//runtime.ts` — `SmsTransport` on `@ultimat3/notify/adapter`: `send({ to, body, idempotencyKey, signal })` → `{ transport, id, to, idempotencyKey }` | +| default | memory in development and test; elsewhere `X_NOTIFY_SMS_TRANSPORT_MISSING` (terminal) | +| prove yours | `behavesLike(smsTransportConformance, () => yours)`; a transport calls `assertSmsNumber(to)` first, and the suite checks it does | +| the body | the catalog key, rendered in `recipient.locale`, else the app's fallback | +| the number | `numberOf(recipient)`, default `recipient.to`, E.164. Not E.164 → `X_NOTIFY_SMS_NUMBER_INVALID` (terminal; the number is never in the cause). None → skipped, logged `notify.address_missing` | +| retries | `smsIdempotencyKey(notifier, key, channel, recipient)` is the same on every attempt | + ## Push `pushChannel` delivers one notification to **every device the recipient subscribed**, each in the diff --git a/wiki/Observability.md b/wiki/Observability.md index ae95f2b0d..2aa2708cf 100644 --- a/wiki/Observability.md +++ b/wiki/Observability.md @@ -99,7 +99,7 @@ The container boot (`startOtlpExport`) installs one exporter per signal an endpo |---|---|---| | traces | `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, else `OTEL_EXPORTER_OTLP_ENDPOINT` + `/v1/traces` | `subscribe('span.end', (e) => otlpSpanExporter().export(e.payload))` | | logs | `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`, else `…_ENDPOINT` + `/v1/logs` | `subscribe('log.record', (e) => otlpLogExporter().export(e.payload))` — the process logger's lines, redacted, joined to the active trace | -| metrics | `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`, else `…_ENDPOINT` + `/v1/metrics` | `configureMetrics({ exporter: otlpMetricExporter() })`, pushed every 60 s | +| metrics | `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`, else `…_ENDPOINT` + `/v1/metrics` | `subscribe('metric.export', (e) => otlpMetricExporter().export(e.payload))`, pushed every 60 s | Every boot, collector or not, also subscribes `subscribeSemconvMetrics()`: `db_client_operation_duration_seconds` (from `db.statement`), `job_duration_seconds` (from `job.perform`) and `gen_ai_client_token_usage` (from `ai.llm`) — on `/metrics` and over OTLP alike. OTLP/HTTP JSON over `fetch`, no dependency; `_HEADERS` and `_PROTOCOL` are read too. **gRPC on `:4317` is refused** with `X_OTLP_PROTOCOL_UNSUPPORTED` and will stay refused — HTTP/2 plus protobuf is a dependency and a second wire format, so point the collector's HTTP receiver at `:4318`. Scraping `metricsText()` remains equally supported. @@ -147,12 +147,15 @@ Dotted OTel names survive OTLP but not a Prometheus scrape, and the autoscaler r ## Reading them ```ts -import { collectMetrics, configureMetrics, exportMetrics, startMetricExport } from '@ultimat3/core'; +import type { MetricExporter } from '@ultimat3/core'; +import { collectMetrics, exportMetrics, startMetricExport, subscribe } from '@ultimat3/core'; -collectMetrics(); // a MetricCollection: { at, resource, metrics } -configureMetrics({ exporter: myExporter }); // install the seam -exportMetrics(); // collect + hand to the exporter, once -const stop = startMetricExport(60_000); // periodic; the timer is unref'd. Default 60s +declare const myExporter: MetricExporter; + +collectMetrics(); // a MetricCollection: { at, resource, metrics } +const release = subscribe('metric.export', (event) => myExporter.export(event.payload)); // the seam +exportMetrics(); // one `metric.export` tick; nothing is collected while nobody subscribes +const stop = startMetricExport(60_000); // periodic; the timer is unref'd. Default 60s ``` | Type | Shape | @@ -164,7 +167,7 @@ const stop = startMetricExport(60_000); // periodic; the timer is unref'd. | `MetricPoint` | `{ attributes, value }` | | `HistogramPoint` | adds `count`, `min`, `max`, `bounds`, `buckets` — `buckets` is one longer than `bounds`, the last being the `+Inf` overflow | -Temporality is **cumulative**. Shipped exporters: `noopMetricExporter` (the default) and `memoryMetricExporter()` for tests. `resetMetrics()` restores the defaults and clears the series while keeping declarations — test-only. +Temporality is **cumulative**. An exporter is a `metric.export` subscriber on the instrumentation bus, like the span and log exporters, released by its own unsubscribe; `configureMetrics({ clock, enabled })` is configuration only — its `exporter` option left in 31.0.0 ([Upgrading](Upgrading)). `memoryMetricExporter()` is the test double. `resetMetrics()` restores the defaults and clears the series while keeping declarations — test-only. ## The Prometheus body diff --git a/wiki/Scraping.md b/wiki/Scraping.md index af28952e8..63343cd7c 100644 --- a/wiki/Scraping.md +++ b/wiki/Scraping.md @@ -491,6 +491,47 @@ export const printInvoice = scrape({ }); ``` +### A route printed to PDF — `pdf()` + +`pdf()` is a factory over **action** (`@ultimat3/scraping`, a `PRIMITIVE_FACTORIES` row). It renders one of the app's OWN routes through Chrome over raw CDP (`Page.printToPDF`), writes the bytes to a disk and answers `{ key, bytes, contentType: 'application/pdf' }`. Schema, policy, rate limit, meter and audit are the action's; HTTP, OpenAPI, MCP, `.job()` and `.as()` come with it. `As of 2026-11-09`. + +```ts +import { can } from '@ultimat3/policy'; +import { t } from '@ultimat3/schema'; +import { pdf } from '@ultimat3/scraping'; +import { disk } from '@ultimat3/storage'; + +declare function printToken(orgId: string | undefined): string; // the app's short-lived token + +export const invoicePdf = pdf({ + input: t.object({ id: t.uuid }), + policy: can('invoices:read'), + route: ({ input }) => `/invoices/${input.id}/print`, // a PATH on the app's origin, never a URL + key: ({ input, ctx }) => `invoices/${ctx.actor?.orgId ?? 'none'}/${input.id}.pdf`, + storage: () => disk('documents'), + layout: { format: 'a4', marginMm: { top: 12, bottom: 12 }, printBackground: true }, + headers: ({ ctx }) => ({ authorization: `Bearer ${printToken(ctx.actor?.orgId)}` }), +}); +``` + +| Rule | Refusal | +|---|---| +| the origin is `APP_URL`, read per call, or `origin: () => …` | none → `X_SCRAPE_PDF_ORIGIN_MISSING` (500) | +| the route is a path on that origin | else `X_SCRAPE_PDF_ROUTE_INVALID` (500) | +| the page's own status, read after `load` and `document.fonts.ready` | 4xx/5xx → `X_SCRAPE_PDF_ROUTE_FAILED` (502; 4xx terminal, 5xx and 429 retryable) | +| the layout, checked at declaration | `X_SCRAPE_PDF_OPTIONS_INVALID` | + +| Bound | Detail | +|---|---| +| browsers at once | one Chrome per render, closed in a `finally`, at most `DEFAULT_PDF_MAX_CONCURRENT` (2) alive per process — `chromePdfRenderer({ maxConcurrent })` | +| a render with no slot | waits up to `queueTimeoutMs` (default: the render's own timeout), then `X_SCRAPE_PDF_BUSY` (503, retryable) | +| what the page may load | every request is paused on the page's session; one to the route's ORIGIN (scheme, host, port, compared whole) continues with `headers` added, every other is failed before it leaves — so the print token never reaches an `` a post carried, and a page cannot dial an internal host. A redirect off-origin is failed the same way | +| Chrome's sandbox | ON: a print renders pages that carry user content. A container that cannot grant it fails the launch with `X_CDP_LAUNCH_FAILED` quoting Chrome's "No usable sandbox"; opt out in one visible line, `chromePdfRenderer({ sandbox: false })` | + +A test passes `renderer:` (any `PdfRenderer`); `chromePdfRenderer({ launch })` takes any `{ connection, close }`. The printing browser has no session of its own: a private route needs the app's short-lived token through `headers` ([Known gaps](Known-Gaps)). + +**The CDP launcher moved.** `findChrome`, `launchChrome`, `launchFoundChrome`, `cdpConnect`, `cdpConnectOver`, the `Cdp*Error` classes and the `CHROME_*` / `LAUNCH_*` constants live in `@ultimat3/scraping/cdp` since 31.0.0 (its own entry, importing core only), and `X_CDP_*` are registered by scraping. It is their one import path: `@ultimat3/testing` no longer re-exports them, and `E2E_BROWSER_CLOSE_MS` is `LAUNCHED_CLOSE_MS` here ([Upgrading](Upgrading) #53). + A social card for a row published at runtime is the same shape: `page.screenshot()` of the row's 1200 x 630 card page, `disk().put()`, then `og: { image }` — [SEO](SEO#a-card-the-build-photographs-ogcapture) covers the cards a static build takes itself. `examples/dummy`'s `runs/jobs/print-statement.ts` is diff --git a/wiki/Storage-And-Uploads.md b/wiki/Storage-And-Uploads.md index 508d7fec6..45e06e073 100644 --- a/wiki/Storage-And-Uploads.md +++ b/wiki/Storage-And-Uploads.md @@ -343,6 +343,36 @@ does not pass. Bytes past the window are not sniffed on any path through promoti A grant with a `target` has no promotion step: [Known gaps](Known-Gaps). +## Scanning uploads — the `uploadScanner` port + +Rule: an `uploadScanner` returned from `apps//runtime.ts` is asked about every upload `promoteAttachment` attaches. `As of 2026-11-09`. It scans a copy THIS call made: under `pending/scan/`, server-named — so no presigned URL can rewrite it after the scan — and its own, so a doubled confirm never scans or deletes another attempt's bytes. That copy is what is attached; one scan is final. + +| Verdict | `promoteAttachment` | +|---|---| +| `{ verdict: 'clean' }` | promotes | +| `{ verdict: 'infected', signature }` | deletes the pending upload and the scan copy; `X_STORAGE_UPLOAD_INFECTED` (422, `meta.signature`) | +| the scanner rejects, or the copy cannot be read for it (daemon down, timeout, a reset read) | deletes the scan copy, the upload stays pending — a retry once the scanner is back is a fresh confirm; `X_STORAGE_SCAN_UNAVAILABLE` (503). Fail closed in every environment; `uploadPolicy({ onScanUnavailable: 'accept' })` promotes unscanned instead. Infected is refused under either | + +| Fact | Detail | +|---|---| +| the port | `UploadScanner = { name, scan({ key, size, contentType, body }) → Promise }` on `@ultimat3/storage/adapter`. "Cannot decide" is a rejection, never a verdict | +| the reference | `memoryUploadScanner()` flags the EICAR test string (`EICAR_TEST_STRING`), across chunk boundaries | +| prove yours | `behavesLike(uploadScannerConformance, () => yours)` from `@ultimat3/testing/conformance` | +| ClamAV | the app's adapter (axiom 8): stream `body` to clamd's `INSTREAM` over `Bun.connect`; `stream: OK` → clean, `stream: FOUND` → infected, anything else → reject | +| no scanner installed | nothing is scanned; the deferred `quarantine: true` + `releaseQuarantine` flow below is unchanged | + +```ts +// apps/web/runtime.ts +import type { RuntimeOverrides } from '@ultimat3/cli/serve'; +import type { UploadScanner } from '@ultimat3/storage/adapter'; + +declare function clamdScanner(options: { host: string; port: number }): UploadScanner; + +export const runtime = { + uploadScanner: clamdScanner({ host: 'clamav', port: 3310 }), +} satisfies Partial; +``` + ## Images A variant's key is the whole source key plus the transform — `photos/hero.png` at width 640 is @@ -363,6 +393,26 @@ encode `png`, `jpeg` and `webp`; `avif` is key and `srcset` math only. `variantK `srcsetDescriptors()` and `fitDimensions()` are pure, which is how [SEO](SEO) builds `srcset` without decoding a byte. +### One kept variant — `variant()` + +`variant(key, { disk, images, width, format? })` → `{ key, object, created }`. `As of 2026-11-09`. Made by the `images` driver you pass (any `@ultimat3/seo` `ImageTransformDriver` — `runtime.ts`'s `images`, `builtinImageDriver({ read })`, a CDN's), stored under `variantKey(key, { width, format, revision })`. A second call is a `stat`, never a decode; an overwritten source is a new revision, hence a new key. `format` omitted keeps a png/jpeg/webp source's encoding. + +| Refusal | Code | +|---|---| +| source missing | `X_STORAGE_NOT_FOUND` | +| source not a raster image (PDF, SVG), a gif/avif source with no `format`, a driver that answered another encoding | `X_IMAGE_UNSUPPORTED` | +| source overwritten while the driver read it (nothing stored) | `X_STORAGE_OBJECT_CHANGED` | + +```ts +import { builtinImageDriver } from '@ultimat3/seo'; +import { disk, variant } from '@ultimat3/storage'; + +declare const coverKey: string; + +const images = builtinImageDriver({ read: async (src) => (await disk().get(src)).bytes }); +export const thumbnail = await variant(coverKey, { disk: disk(), images, width: 640, format: 'webp' }); +``` + ### Warm the variants after the upload — `warmVariants` The first request for each width otherwise pays the decode on the request path. A job the upload diff --git a/wiki/Upgrading.md b/wiki/Upgrading.md index 5c84a00ae..49dd42bf3 100644 --- a/wiki/Upgrading.md +++ b/wiki/Upgrading.md @@ -8,7 +8,7 @@ | From → to | Breaking entries | Read | |---|---|---| -| 30.x → 31.0.0 | **49** so far, and **unreleased** — plan 101 final hardening: an `AuthServerStore` with a required `getCode` and a `pollDevice` that takes the client; a drain that abandoned work exits 70; validation issues capped at 100 and causes at 4 KiB; storage keys, header values and revision-pinned reads; one policy gate for action and query; a paged read capped at 500 rows over the wire; a `PreferenceStore` asked once per 500 recipients; remote MCP tools not re-exposed unless opted in; **a `(tenant, pk)` index on every tenant table that needs one `x db gen`**; an `IsrStore` that may answer with a promise; `localhost` no longer admitted by a wildcard `allowHosts`; span exporters installed by subscribing to `span.end` on the one instrumentation bus; an app's own Sentry transport replacing the `SENTRY_DSN` one, local sinks marked; `jobs.queues` defaulting to `['default']`; OpenTelemetry's stable http, messaging and gen_ai span attributes; a `defineStorage` disk that reports on the bus; OTLP logs and the semconv histograms on every boot; db, entity, policy and cache observer slots replaced by bus events; a database span for every statement in production; a bus installed from `runtime.ts`, never a realtime setter; one `ADAPTER_PORTS` table and core installers on `@ultimat3/core/adapter`; a `SecretSource` port for the master key; `realtime.transport`/`urlEnv` removed from `app.config.ts`; job, event-bus, mail, idempotency, meter and notify installers on each package's `./adapter`, swapped from `runtime.ts`; the rate-limit store, auth limiters, TOTP guard and scrape driver installed the same way, `httpServer` taking no store; one core clock for scraping and flags; admin audit sinks typed by core's `AuditSink`; the AI gateway installed only as `aiGateway`, never through `configureAi`; factories with `belongsTo`/`hasMany`, `buildList`/`createList` and a `create()` that answers the stored row; `runtime.syncAuthenticate` removed, every port a `runtime.ts` field and `/packages/db/factories.ts` linted on `unit`; the audit sink, error reporter and KDF gate installed on `./adapter`, `runtime.storage` a `StorageDriver` and the last four ports `runtime.ts` fields; port suites on `@ultimat3/testing/conformance` only; a malformed uuid refused before the statement; a push subscription that always names its person; no memory delivery ledger outside development | the `30.x → 31.0.0` section below. Its entries sit under `[Unreleased]` in `CHANGELOG.md` until the tag | +| 30.x → 31.0.0 | **53** so far, and **unreleased** — plan 101 final hardening: an `AuthServerStore` with a required `getCode` and a `pollDevice` that takes the client; a drain that abandoned work exits 70; validation issues capped at 100 and causes at 4 KiB; storage keys, header values and revision-pinned reads; one policy gate for action and query; a paged read capped at 500 rows over the wire; a `PreferenceStore` asked once per 500 recipients; remote MCP tools not re-exposed unless opted in; **a `(tenant, pk)` index on every tenant table that needs one `x db gen`**; an `IsrStore` that may answer with a promise; `localhost` no longer admitted by a wildcard `allowHosts`; span exporters installed by subscribing to `span.end` on the one instrumentation bus; an app's own Sentry transport replacing the `SENTRY_DSN` one, local sinks marked; `jobs.queues` defaulting to `['default']`; OpenTelemetry's stable http, messaging and gen_ai span attributes; a `defineStorage` disk that reports on the bus; OTLP logs and the semconv histograms on every boot; db, entity, policy and cache observer slots replaced by bus events; a database span for every statement in production; a bus installed from `runtime.ts`, never a realtime setter; one `ADAPTER_PORTS` table and core installers on `@ultimat3/core/adapter`; a `SecretSource` port for the master key; `realtime.transport`/`urlEnv` removed from `app.config.ts`; job, event-bus, mail, idempotency, meter and notify installers on each package's `./adapter`, swapped from `runtime.ts`; the rate-limit store, auth limiters, TOTP guard and scrape driver installed the same way, `httpServer` taking no store; one core clock for scraping and flags; admin audit sinks typed by core's `AuditSink`; the AI gateway installed only as `aiGateway`, never through `configureAi`; factories with `belongsTo`/`hasMany`, `buildList`/`createList` and a `create()` that answers the stored row; `runtime.syncAuthenticate` removed, every port a `runtime.ts` field and `/packages/db/factories.ts` linted on `unit`; the audit sink, error reporter and KDF gate installed on `./adapter`, `runtime.storage` a `StorageDriver` and the last four ports `runtime.ts` fields; port suites on `@ultimat3/testing/conformance` only; a malformed uuid refused before the statement; a push subscription that always names its person; no memory delivery ledger outside development; the embedder and semantic cache as `runtime.ts` ports, `configureAi` keeping `redact` only; metric exporters subscribing to `metric.export` on the bus; sign-in routes at `/auth/sso/:provider` for OAuth and SAML alike; the raw-CDP launcher imported from `@ultimat3/scraping/cdp` only | the `30.x → 31.0.0` section below. Its entries sit under `[Unreleased]` in `CHANGELOG.md` until the tag | | 29.x → 30.0.0 | **143** — plan 101 (ten waves), issues #990–#999 and two find-and-fix sweeps: a write past its deadline that rolls back, a closed-transaction refusal, a two-release column drop, **same-tenant composite keys that need one `x db gen`**, `upsertAll` overwriting only what it names, every agent and MCP call held by `mcpConfirmations`, `'null'` refused in `cors.origins`, `isr` and `static` that render as nobody, a query-free `isr` key by default, a pool that refuses past `acquireTimeoutMs`, an upload grant bound to its declared size, mail that refuses an unresolved key, a problem document with `retry`, a worker retire on by default, required `StorageDriver`, `JobDriver` and `PoolProfile` members, a `budgets` step that builds before it weighs, no `bin/check` in a new app, `ui`'s Standard Schema copies removed, no vendor link on the default error page, the render brands at `@ultimat3/core/render-node` only, a required `enqueueMany` on every `JobHandle`, `JobsFacade` and `JobDriver`, an outbox relay that claims 500 rows a pass, `'browser'` in `ERROR_SOURCES`, `assets` and `clientErrors` on `AppConfig`, `pwa`'s `retentionPlan` and `retainBuildIds` removed, a `DefineEvalInput` union, `MCP_PROTOCOL_VERSION` deleted, an MCP revision decided per request with `2026-07-28` the newest, a cross-origin `POST /mcp` refused with `'origin-refused'` in `McpAuthRefusal`, `x mcp serve` answering only a loopback Host, a raw `WorkloadToken` payload, a role that refuses a half-loaded app, a query's own MCP tool wording, no `QueryPoll.ms`, an exercise refused outside `bun test`, an accessibility audit on every `e2eTest`, factory values read off each column's declaration, a memory transaction that holds every write inside it, a 21st gate step that runs every primitive with no test written, a finding for an untenanted child of a tenant-owned table, a memory transaction that refuses a write after it ended and runs nested effects at the root's commit, required `failureCapture` and `time` on `AppConfig`, an `x_failures` table in the schema dump, a description on every page `/llms.txt` lists, a required `TimeZoneConfig.detectedCookie`, an MCP resource read handed its caller, a required `DevCapabilities.queryGraph` and `DevSources.mailTemplates`, an exercise step that plans every statement and reds an unindexed read or a loop over rows, an `OpenGraph` that is a hosted `image` or a captured page and never both, a `postgresAuthAdapter()` that reads the ambient client per call, a `refetch()` that returns a promise, 48 more `UI_KEYS` in a module namespace, charts that draw `null` and negative values and refuse a series that does not match its keys, a two-mode `TabsProps`, a command palette that ranks, two more contrast pairs on `bg-soft`, every `mock.module` put back at its file boundary, a static export that writes image variants as files, `/assets` answering only the variants it declares, an island `mount` that must return its disposer, passkey storage on every `AuthAdapter`, a per-call rate limit split from the fixed one, a signed vote in `McpConfirmationStore.decide`, a meter in the manifest and its contract diff, a hand-rolled websocket refused, bad-token counting on every `PushDeviceStore`, exact history images on Postgres, a policy on every multipart part and completion, `LiveHandle.retry`, a Bearer-only CSRF exemption, no `enqueueInTx`, and an event id from `EventLookup.find` | [the `30.0.0` section](#29x--3000-entry-by-entry), in order | | 28.x → 29.0.0 | **2** — **Bun 1.4.4 as the floor**; a `WORKER_QUEUES` entry the app does not know refuses the worker's boot | [the `29.0.0` section](#28x--2900-entry-by-entry), in order | | 27.x → 28.0.0 | **2** — **Bun 1.4.3 as the floor**; `@ultimat3/ui/jsx-probe` and its `probe`/`unprobe` removed | [the `28.0.0` section](#27x--2800-entry-by-entry), in order | @@ -70,15 +70,15 @@ Each entry changes a surface the table below covers. | # | Do | What you see until you do | Entries | |---|---|---|---| | 1 | commit, then `bunx @ultimat3/cli@latest upgrade --json`: it pins every `@ultimat3/*`, installs and runs the gate | `X_UPGRADE_DIRTY_TREE` on a dirty tree | — | -| 2 | `x verify --only typecheck` and close every error: a hand-written `AuthServerStore` or `PreferenceStore`, an import of a removed Redis script or of `MAX_WIRE_ISSUES`, a hand-built `ActionDeniedError`/`QueryDeniedError`, a `set*` job/event-bus/mail/idempotency/meter/notify setter (a `runtime.ts` field), a factory's `associate()` or `buildMany`, a `setAuditSink`/`configureKdfGate`/`configureErrorReporting({ reporter })` call, a `runtime.storage` built with `defineStorage`, a `PushSubscriptionRecord` with `actorId: null` | TS2741, TS2554, TS2305, TS2322 | 1, 6, 10, 14, 18, 29, 36, 40, 43–45, 48 | +| 2 | `x verify --only typecheck` and close every error: a hand-written `AuthServerStore` or `PreferenceStore`, an import of a removed Redis script or of `MAX_WIRE_ISSUES`, a hand-built `ActionDeniedError`/`QueryDeniedError`, a `set*` job/event-bus/mail/idempotency/meter/notify setter (a `runtime.ts` field), a factory's `associate()` or `buildMany`, a `setAuditSink`/`configureKdfGate`/`configureErrorReporting({ reporter })` call, a `runtime.storage` built with `defineStorage`, a `PushSubscriptionRecord` with `actorId: null`, a `configureAi({ embedder, semanticCache })` or `configureMetrics({ exporter })` call | TS2741, TS2554, TS2305, TS2322 | 1, 6, 10, 14, 18, 29, 36, 40, 43–45, 48, 50, 51 | | 3 | `x verify --only unit,contract`: shorten a name over 63 bytes, write each `cors.origins` entry as its own `URL.origin`, rename an unsafe storage key or header value, switch a `fromDecimal` branch to `X_MONEY_NOT_DECIMAL`, read a truncated validation through `issuesTruncated`, pass `expose: true` to an `mcpTools()` whose tools your `/mcp` serves, name `localhost` exactly in an `allowHosts` that reaches it through `'*'`, compare a resolved disk by `.name` | `X_SQL_UNSAFE`, `X_CORS_CONFIG_INVALID` at boot, `X_STORAGE_PATH_UNSAFE`, `X_INVARIANT`, `X_MONEY_NOT_DECIMAL`, a remote tool missing from your `tools/list`, `X_SCRAPE_HOST_BLOCKED` for a `localhost` URL | 2, 4, 7, 9, 11, 12, 19, 22, 27 | -| 4 | **before the deploy:** list stored objects whose keys the new rule refuses and copy them; cache a projection where a value passes 1 MiB; keep `redisTier`'s default `buildId` or plan the one-time bust; on `docker/helm`, move a pod that reads the metadata endpoint to a workload identity; name a Postgres `notifyStores.ledger` in `runtime.ts` if a notifier runs | a read of an old key refused; a cache miss logged `X_CACHE_TOO_LARGE`; stale tagged keys until their TTL; a metadata-endpoint timeout; `X_NOTIFY_STORE_MISSING` on the first notification | 5, 6, 11, 16, 49 | +| 4 | **before the deploy:** list stored objects whose keys the new rule refuses and copy them; cache a projection where a value passes 1 MiB; keep `redisTier`'s default `buildId` or plan the one-time bust; on `docker/helm`, move a pod that reads the metadata endpoint to a workload identity; name a Postgres `notifyStores.ledger` in `runtime.ts` if a notifier runs; add `/auth/sso//callback` as a redirect URI at every OAuth IdP and point sign-in links at `/auth/sso/` | a read of an old key refused; a cache miss logged `X_CACHE_TOO_LARGE`; stale tagged keys until their TTL; a metadata-endpoint timeout; `X_NOTIFY_STORE_MISSING` on the first notification | 5, 6, 11, 16, 49, 52 | | 5 | **after the deploy:** declare `maxPage` on a read a client pages past 500 rows (`X_INPUT_INVALID` until you do); treat exit `70` as "drained, work abandoned" in alerts; read rate-limit keys through `addressNetwork`; switch on `denial.surface === 'mcp'` for an MCP read; honour `{ etag }` in a hand-written storage driver; give the collector a logs pipeline or set `OTEL_LOGS_EXPORTER=none` | nothing fails: an alert, a dashboard filter or an ETag answers differently | 3, 8, 13, 14, 17, 28, 30 | | 6 | `x db gen "tenant indexes"`, commit `packages/db/migrations` and `packages/db/schema/`; the new index builds `concurrently`, so a hot table keeps taking writes | `X_DB_DRIFT` from `x verify --only drift` | 20 | ### Entry by entry -Tier 0 — `@ultimat3/core` (3), (23), (25), (31), (33–35), (43), `@ultimat3/core` with `@ultimat3/cli` (24), `@ultimat3/core` with `@ultimat3/scraping` (22), `@ultimat3/schema` with `@ultimat3/core` (4). Tier 1 — `@ultimat3/db` (2), (30), `@ultimat3/db` with `@ultimat3/entity`, `@ultimat3/policy` and `@ultimat3/cache` (29), `@ultimat3/cache` (5–6), `@ultimat3/money` (7), `@ultimat3/storage` (11–13), (27), `@ultimat3/flags` with `@ultimat3/scraping` (38). Tier 2 — `@ultimat3/auth` (1), (44), `@ultimat3/http` (8–9), `@ultimat3/entity` (20), (47), `@ultimat3/http` with `@ultimat3/jobs` and `@ultimat3/ai` (26), `@ultimat3/http` with `@ultimat3/auth` and `@ultimat3/scraping` (37). Tier 3 — `@ultimat3/action` (10), `@ultimat3/action` with `@ultimat3/query` (14), `@ultimat3/realtime` (15), (32), `@ultimat3/query` (17), `@ultimat3/jobs` with `@ultimat3/action`, `@ultimat3/mail` and `@ultimat3/notify` (36). Tier 4 — `@ultimat3/notify` (18), `@ultimat3/mcp` (19), `@ultimat3/render` (21), `@ultimat3/ai` (41), `@ultimat3/pwa` (48), `@ultimat3/notify` (49). Tier 5 — `@ultimat3/cli` (28), (42), (45), `@ultimat3/admin` (39), `@ultimat3/testing` (40), (46). The framework chart — `docker/helm` (16). +Tier 0 — `@ultimat3/core` (3), (23), (25), (31), (33–35), (43), (51), `@ultimat3/core` with `@ultimat3/cli` (24), `@ultimat3/core` with `@ultimat3/scraping` (22), `@ultimat3/schema` with `@ultimat3/core` (4). Tier 1 — `@ultimat3/db` (2), (30), `@ultimat3/db` with `@ultimat3/entity`, `@ultimat3/policy` and `@ultimat3/cache` (29), `@ultimat3/cache` (5–6), `@ultimat3/money` (7), `@ultimat3/storage` (11–13), (27), `@ultimat3/flags` with `@ultimat3/scraping` (38). Tier 2 — `@ultimat3/auth` (1), (44), (52), `@ultimat3/http` (8–9), `@ultimat3/entity` (20), (47), `@ultimat3/http` with `@ultimat3/jobs` and `@ultimat3/ai` (26), `@ultimat3/http` with `@ultimat3/auth` and `@ultimat3/scraping` (37). Tier 3 — `@ultimat3/action` (10), `@ultimat3/action` with `@ultimat3/query` (14), `@ultimat3/realtime` (15), (32), `@ultimat3/query` (17), `@ultimat3/jobs` with `@ultimat3/action`, `@ultimat3/mail` and `@ultimat3/notify` (36). Tier 4 — `@ultimat3/notify` (18), `@ultimat3/mcp` (19), `@ultimat3/render` (21), `@ultimat3/ai` (41), (50), `@ultimat3/pwa` (48), `@ultimat3/notify` (49). Tier 5 — `@ultimat3/cli` (28), (42), (45), `@ultimat3/admin` (39), `@ultimat3/testing` (40), (46), `@ultimat3/testing` with `@ultimat3/scraping` (53). The framework chart — `docker/helm` (16). | # | Surface | Costs you an edit if | |---|---|---| @@ -131,10 +131,14 @@ Tier 0 — `@ultimat3/core` (3), (23), (25), (31), (33–35), (43), `@ultimat3/c | 47 | a malformed uuid in a repository write (`@ultimat3/entity`) | a test that expected `X_DB_STATEMENT_FAILED` (Postgres 22P02) for `repo.insert({ id: 'not-a-uuid' })` now sees `X_INVARIANT_VIOLATED`, raised before the statement in both drivers — and memory no longer stores the row: expect the new code | | 48 | `PushSubscriptionRecord.actorId` (`@ultimat3/pwa`) | TS2322 where code builds a record with `actorId: null` — store it for `ctx.actor.id`, as `pushSubscribe()` does; an anonymous visitor has no subscription. At runtime both stores now refuse a missing or empty `actorId` with `X_PWA_PUSH_SUBSCRIPTION_INVALID` before writing (Postgres answered a not-null violation; memory stored it). A custom `PushSubscriptionStore` proves the same with `pushSubscriptionStoreConformance` | | 49 | the default delivery ledger (`@ultimat3/notify`) | you deploy (staging or production) with no `notifyStores.ledger` in `runtime.ts` and a notifier runs: its first claim is `X_NOTIFY_STORE_MISSING`, where a memory ledger answered — one per process, so a replay on another replica or after a restart sent twice. Set `notifyStores: { ledger: postgresDeliveryLedger({ executor: dbExecutor() }) }` in `export const runtime = { … }` (`apps//runtime.ts`); the boot already applies `x_notify_deliveries`. Name every store you keep in that one object — it replaces, never merges. Development and test keep the memory default | +| 50 | `configureAi({ embedder, semanticCache })` (`@ultimat3/ai`) | TS2353 on `embedder` / `semanticCache` — set them in `export const runtime = { embedder, semanticCache }` (`apps//runtime.ts`); `semanticCache` is the same `(scope) => SemanticCache` factory. `configureAi` keeps `redact` only. A test: `const release = installAiEmbedder(embedder)` / `installSemanticCache(factory)` from `@ultimat3/ai/adapter`, `release()` after — or `swapAdapter('embedder', …)` from `@ultimat3/testing`. Prove one with `embedderConformance` / `semanticCacheFactoryConformance` from `@ultimat3/testing/conformance` | +| 51 | `configureMetrics({ exporter })` (`@ultimat3/core`) | TS2353 on `exporter` — subscribe it: `const release = subscribe('metric.export', (event) => exporter.export(event.payload))`, `release()` to uninstall; `exportMetrics()` / `startMetricExport(ms)` publish the tick as before. `configureMetrics` keeps `clock` and `enabled`. With `OTEL_EXPORTER_OTLP_ENDPOINT` set the boot already subscribes `otlpMetricExporter()` | +| 52 | `oauthLogin` (`@ultimat3/auth`) — its paths, cookie and `restartAt` | the routes moved from `/auth/oauth/:provider[/callback]` to `/auth/sso/:provider[/callback]` (`oauthStartPath`/`oauthCallbackPath`/`OAUTH_*_ROUTE_PATH` answer the new values; the call and its options are unchanged). **Before deploying, add `https:///auth/sso//callback` as an allowed redirect URI at each IdP** (GitHub, Google, your OP) — keep the old one until the deploy is out — and point every sign-in link at `/auth/sso/`. A login started before the deploy fails its callback once (`X_OAUTH_STATE_INVALID`: the state cookie is now `__Host-x_sso_`); the user starts again. `restartAt(provider)` and every OAuth `fix:` name `/auth/sso/`. `oauthDenied` and `OAUTH_ROUTE_STATUS` keep their names on `@ultimat3/auth`; a deep import from `./oauth-route` moves to the barrel | +| 53 | the raw-CDP launcher (`@ultimat3/testing` → `@ultimat3/scraping/cdp`) | an e2e suite imports `findChrome`, `launchChrome`, `CHROME_PATH_ENV`, a `Cdp*Error` or `E2E_BROWSER_CLOSE_MS` from `@ultimat3/testing` (TS2305, no exported member): `bun add -d @ultimat3/scraping`, then `import { findChrome, LAUNCHED_CLOSE_MS } from '@ultimat3/scraping/cdp';` — `E2E_BROWSER_CLOSE_MS` is `LAUNCHED_CLOSE_MS`, the same value. Everything e2e-shaped (`openE2eBrowser`, `E2E_BROWSER_OPEN_MS`, `installE2eDriver`, …) stays on `@ultimat3/testing` | ### Every removed setter, and the `runtime.ts` field that replaced it -One lookup for entries 32–38, 41 and 43. A backend is swapped in ONE place, `export const runtime = { … }` in `apps//runtime.ts`; the installer beside each field is what the boot calls, and what a harness of your own imports — a test uses `await swapAdapter(port, impl)` from `@ultimat3/testing`. Each port's interface, reference implementation and conformance suite: [Adapters](Adapters). +One lookup for entries 32–38, 41, 43 and 50. A backend is swapped in ONE place, `export const runtime = { … }` in `apps//runtime.ts`; the installer beside each field is what the boot calls, and what a harness of your own imports — a test uses `await swapAdapter(port, impl)` from `@ultimat3/testing`. Each port's interface, reference implementation and conformance suite: [Adapters](Adapters). | Removed in 31.0.0 | `runtime.ts` field | The installer, on the package's `./adapter` | Entry | |---|---|---|---| @@ -155,6 +159,8 @@ One lookup for entries 32–38, 41 and 43. A backend is swapped in ONE place, `e | `configureAi({ gateway })` (`@ultimat3/ai`) | `aiGateway` | `installAiGateway` (`@ultimat3/ai/adapter`) | 41 | | `setAuditSink` (`@ultimat3/core`) | `auditSink` | `installAuditSink` (`@ultimat3/core/adapter`) | 43 | | `configureErrorReporting({ reporter })` (`@ultimat3/core`) | `errorReporter` | `installErrorReporter` (`@ultimat3/core/adapter`) | 43 | +| `configureAi({ embedder })` (`@ultimat3/ai`) | `embedder` | `installAiEmbedder` (`@ultimat3/ai/adapter`) | 50 | +| `configureAi({ semanticCache })` (`@ultimat3/ai`) | `semanticCache` | `installSemanticCache` (`@ultimat3/ai/adapter`) | 50 | A setter still on a package's `.` barrel is `X_ADAPTER_PORT_INCOMPLETE` on the `boundaries` step unless `scripts/lib/adapter-port-pins.ts` names why it selects no backend. From 43f79a48a47ea15b5a027a36f338768e22def91c Mon Sep 17 00:00:00 2001 From: sebi Date: Tue, 10 Nov 2026 01:31:32 -0500 Subject: [PATCH 2/2] =?UTF-8?q?fix(v31):=20wave=20F=20CI=20=E2=80=94=20iso?= =?UTF-8?q?late=20batched-scan=20temp=20paths=20per=20load,=20cover=20benc?= =?UTF-8?q?h=20entry=20points,=20coverage=20scanner=20reads=20arrow=20type?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- packages/cli/src/app-load-batch.test.ts | 65 ++++++++ packages/cli/src/app-load-batch.ts | 17 +- packages/cli/src/build-registry.test.ts | 27 ++-- packages/cli/src/build-registry.ts | 13 +- packages/cli/src/coverage-source.test.ts | 12 ++ packages/cli/src/coverage-source.ts | 5 +- packages/cli/src/document-options.ts | 5 +- .../src/runtime-cache-redis-peer-fixture.ts | 21 --- .../cli/src/runtime-cache-redis.live.test.ts | 31 +++- scripts/bench/boot-vs-files-child.ts | 61 ++++--- scripts/bench/boot-vs-files.bench.test.ts | 87 ++++++++++ scripts/bench/boot-vs-files.bench.ts | 31 ++-- scripts/bench/fanout-3-nodes.bench.test.ts | 108 +++++++++++++ scripts/bench/fanout-3-nodes.bench.ts | 151 +++++++++++++----- scripts/bench/fanout-node-child.ts | 151 +++++++++++------- scripts/bench/fanout-shape.ts | 7 +- scripts/bench/http-prod-image.bench.test.ts | 72 +++++++++ scripts/bench/http-prod-image.bench.ts | 81 ++++++---- scripts/bench/worker-throughput-child.ts | 130 +++++++++------ scripts/bench/worker-throughput.bench.test.ts | 100 ++++++++++++ scripts/bench/worker-throughput.bench.ts | 75 +++++---- 21 files changed, 963 insertions(+), 287 deletions(-) create mode 100644 packages/cli/src/app-load-batch.test.ts delete mode 100644 packages/cli/src/runtime-cache-redis-peer-fixture.ts create mode 100644 scripts/bench/boot-vs-files.bench.test.ts create mode 100644 scripts/bench/fanout-3-nodes.bench.test.ts create mode 100644 scripts/bench/http-prod-image.bench.test.ts create mode 100644 scripts/bench/worker-throughput.bench.test.ts diff --git a/packages/cli/src/app-load-batch.test.ts b/packages/cli/src/app-load-batch.test.ts new file mode 100644 index 000000000..07c66c6d8 --- /dev/null +++ b/packages/cli/src/app-load-batch.test.ts @@ -0,0 +1,65 @@ +// Two loads in one process never share a batch's files. `bun test --parallel` runs each file +// isolated in a long-lived worker: this module is evaluated afresh per file, Bun's module cache is +// not, and a batch directory named by a per-instance counter was written twice under one path — +// the second load imported the first load's hooks, which handed it the first app's modules. + +import { afterEach, describe, expect, test } from 'bun:test'; +// why: Bun has no API for a temporary directory or a recursive delete. +import { mkdtempSync, rmSync } from 'node:fs'; +// why: Bun exposes no tmpdir(), so only node:os answers the platform temp root. +import { tmpdir } from 'node:os'; +// why: Bun exposes no path-join primitive; Bun.file and import() take one already joined. +import { join } from 'node:path'; + +let root = ''; + +afterEach(() => { + if (root !== '') rmSync(root, { recursive: true, force: true }); + root = ''; +}); + +const BATCH = JSON.stringify(join(import.meta.dir, 'app-load-batch.ts')); + +/** A test file that imports its own three modules and asserts each hook handed back its own. */ +const loader = (app: string): string => `import { expect, test } from 'bun:test'; +import { importInOrder } from ${BATCH}; +test('${app} gets its own modules', async () => { + const dir = import.meta.dir + '/${app}'; + const files = ['1', '2', '3'].map((name) => dir + '/' + name + '.ts'); + const heard: string[] = []; + const imported = await importInOrder(files, new Set(), (_index, module) => { + heard.push(String(module.who)); + }); + expect(heard).toEqual(['${app}1', '${app}2', '${app}3']); + expect(imported.map((one) => (one.ok ? one.module.who : 'failed'))).toEqual(heard); +}); +`; + +describe('unit · a batch is never written where an earlier load in the process wrote one', () => { + test('two isolated files, one worker: each load hears its own modules', async () => { + root = mkdtempSync(join(tmpdir(), 'x-load-batch-')); + for (const app of ['a', 'b']) { + for (const name of ['1', '2', '3']) { + await Bun.write(join(root, app, `${name}.ts`), `export const who = '${app}${name}';\n`); + } + await Bun.write(join(root, `${app}.test.ts`), loader(app)); + } + // `--isolate` in one process is what `--parallel` does per worker: the files run in turn, each + // with a fresh copy of this package's modules and the one module cache between them. + const child = Bun.spawn(['bun', 'test', '--isolate', './a.test.ts', './b.test.ts'], { + cwd: root, + stdout: 'pipe', + stderr: 'pipe', + }); + const [out, err, code] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + expect({ code, failed: `${out}${err}`.match(/\(fail\)[^\n]*/g) ?? [] }).toEqual({ + code: 0, + failed: [], + }); + expect(`${out}${err}`).toContain('2 pass'); + }, 30_000); +}); diff --git a/packages/cli/src/app-load-batch.ts b/packages/cli/src/app-load-batch.ts index 90e3193fc..3b89b3ca9 100644 --- a/packages/cli/src/app-load-batch.ts +++ b/packages/cli/src/app-load-batch.ts @@ -84,13 +84,14 @@ export async function receiving( } } -let batches = 0; - -/** A directory no earlier batch of this process used: a module is cached by its path. */ -const batchDir = (): string => { - batches += 1; - return join(tmpdir(), `x-scan-${String(process.pid)}-${String(batches)}`); -}; +/** + * A directory no other batch has used — in this process or any copy of this module in it. Bun's + * module cache is keyed by path and outlives a module instance: `bun test --parallel` evaluates + * this file afresh per test file in one worker, and a per-instance counter named the same + * directory twice, so the second load imported the first one's cached hooks and was handed the + * first app's modules (2026-11-10, `app-load-batch.test.ts`). + */ +export const uniqueBatchName = (): string => `x-scan-${crypto.randomUUID()}`; /** The serial import, for one module: what the batch falls back to. */ async function importOne(absolute: string): Promise { @@ -133,7 +134,7 @@ async function importBatch( ): Promise { const slots: Slots = batch.map(() => undefined); if (batch.length > 1) { - const dir = batchDir(); + const dir = join(tmpdir(), uniqueBatchName()); const { entry, hooks } = batchSources( batch.map((absolute) => Bun.pathToFileURL(absolute).href), first, diff --git a/packages/cli/src/build-registry.test.ts b/packages/cli/src/build-registry.test.ts index 1c0ba82af..c96a78a8b 100644 --- a/packages/cli/src/build-registry.test.ts +++ b/packages/cli/src/build-registry.test.ts @@ -129,6 +129,9 @@ console.log(JSON.stringify({ boot: { return { ...answered, logged }; } +/** A chunk's file name, without the directory of the write that made it. */ +const chunkName = (chunk: string): string => chunk.split('/').at(-1) ?? ''; + const write = async (dir: string) => writePrebuiltRegistry(dir, await scannedModulePaths(dir)); describe('unit · the prebuilt module registry', () => { @@ -141,13 +144,14 @@ describe('unit · the prebuilt module registry', () => { 'apps/web/shared/rows.ts', 'packages/domain/src/other.ts', ]); - expect(registry.chunks).toEqual(['chunk-0.ts']); + expect(registry.chunks.map(chunkName)).toEqual(['chunk-0.ts']); expect(await readPrebuiltRegistry(dir)).toEqual(registry); // Relative: the image is the same tree at whatever path it is mounted. - const chunk = await Bun.file(join(registryDir(dir), 'chunk-0.ts')).text(); + const [own] = (registry.chunks[0] ?? '').split('/'); + const chunk = await Bun.file(join(registryDir(dir), registry.chunks[0] ?? '')).text(); expect(chunk.split('\n')[0]).toBe('import "./m0.ts";'); - const hook = await Bun.file(join(registryDir(dir), 'm0.ts')).text(); - expect(hook).toContain('from "../../../../apps/web/app/a/page.tsx"'); + const hook = await Bun.file(join(registryDir(dir), own ?? '', 'm0.ts')).text(); + expect(hook).toContain('from "../../../../../apps/web/app/a/page.tsx"'); expect(`${chunk}${hook}`).not.toContain(dir); }); @@ -175,7 +179,7 @@ ${log('apps/web/app/b/actions.ts:end')}export {}; `, }); const registry = await write(dir); - expect(registry.chunks).toEqual(['chunk-0.ts', 'chunk-1.ts', 'chunk-2.ts']); + expect(registry.chunks.map(chunkName)).toEqual(['chunk-0.ts', 'chunk-1.ts', 'chunk-2.ts']); const [prebuilt, scanned] = await Promise.all([boot(dir, true), boot(dir, false)]); expect(prebuilt.source).toBe('registry'); const end = scanned.order.indexOf('apps/web/app/b/actions.ts:end'); @@ -212,14 +216,19 @@ export function Page() { return
C
; } expect(await readPrebuiltRegistry(dir)).toBeUndefined(); }); - test('a rewrite removes every chunk of the registry before it', async () => { + // …and never writes where it did: Bun caches a module by its path for the life of the process, + // so a second write at the first one's paths would boot the first write's hooks. + test('a rewrite removes every chunk of the registry before it, and writes under new paths', async () => { const dir = await fixtureApp(); const many = Array.from({ length: BATCH_SIZE + 1 }, (_unused, index) => join(dir, `packages/domain/src/m${String(index)}.ts`), ); - expect((await writePrebuiltRegistry(dir, many)).chunks).toEqual(['chunk-0.ts', 'chunk-1.ts']); - await write(dir); - expect(await Bun.file(join(registryDir(dir), 'chunk-1.ts')).exists()).toBe(false); + const before = (await writePrebuiltRegistry(dir, many)).chunks; + expect(before.map(chunkName)).toEqual(['chunk-0.ts', 'chunk-1.ts']); + const after = (await write(dir)).chunks; + expect(await Bun.file(join(registryDir(dir), before[1] ?? '')).exists()).toBe(false); + expect(after.map(chunkName)).toEqual(['chunk-0.ts']); + expect(before).not.toContain(after[0]); }); test('the image build writes none over a load with a finding, and removes the last one', async () => { diff --git a/packages/cli/src/build-registry.ts b/packages/cli/src/build-registry.ts index c844e1f65..7369d6726 100644 --- a/packages/cli/src/build-registry.ts +++ b/packages/cli/src/build-registry.ts @@ -11,7 +11,7 @@ import { isJsonObject } from '@ultimat3/core'; import type { AppScan } from './app-load'; import { scannedModulePaths } from './app-load'; import { asyncModules } from './app-load-async'; -import { batchSources, planBatches } from './app-load-batch'; +import { batchSources, planBatches, uniqueBatchName } from './app-load-batch'; import { toPosix } from './posix-path'; import { PREBUILT_REGISTRY_DIR } from './serve-prebuilt-paths'; @@ -58,8 +58,11 @@ export async function writePrebuiltRegistry( root: string, modules: readonly string[], ): Promise { - const dir = registryDir(root); - await rm(dir, { recursive: true, force: true }); + await rm(registryDir(root), { recursive: true, force: true }); + // Under a directory of this write's own: a process that writes a registry twice and boots from + // it (a test, `x build` beside `x serve`) would otherwise import the first write's cached hooks. + const own = uniqueBatchName(); + const dir = join(registryDir(root), own); const chunks: string[] = []; // Cut as the scan cuts: a module that awaits at its top level, or reaches one, is alone. let first = 0; @@ -71,7 +74,7 @@ export async function writePrebuiltRegistry( ); await Promise.all(hooks.map(([hook, source]) => Bun.write(join(dir, hook), source))); await Bun.write(join(dir, name), entry); - chunks.push(name); + chunks.push(`${own}/${name}`); first += batch.length; } const registry: PrebuiltRegistry = { @@ -80,7 +83,7 @@ export async function writePrebuiltRegistry( files: modules.map((path) => toPosix(relative(root, path))), chunks, }; - await Bun.write(join(dir, INDEX_FILE), `${JSON.stringify(registry, null, 2)}\n`); + await Bun.write(join(registryDir(root), INDEX_FILE), `${JSON.stringify(registry, null, 2)}\n`); return registry; } diff --git a/packages/cli/src/coverage-source.test.ts b/packages/cli/src/coverage-source.test.ts index 4e1614366..b2859cfaf 100644 --- a/packages/cli/src/coverage-source.test.ts +++ b/packages/cli/src/coverage-source.test.ts @@ -74,6 +74,18 @@ export const y = 2; ).toBe(false); }); + test("an arrow's `=>` inside an alias is not a closing bracket", () => { + // The `>` of `=>` once counted as a close: depth went below zero, the scan stopped at the `;` + // inside the braces, and the `}` left behind read as code — `document-options.ts` was reported + // as an unimported module (`X_COVERAGE_UNMEASURED`) for an alias and nothing else. + expect( + hasExecutableCode( + 'export type F = ((a: string) => string) & { readonly g?: () => void; };\n', + ), + ).toBe(false); + expect(hasExecutableCode('type G = { run: () => void; };\nexport const h = 1;\n')).toBe(true); + }); + test('a types-only module and a barrel are both invisible for the RIGHT reason', () => { expect(hasExecutableCode('export interface Counter {\n add(n: number): void;\n}\n')).toBe( false, diff --git a/packages/cli/src/coverage-source.ts b/packages/cli/src/coverage-source.ts index 8e5002f1c..bae82b377 100644 --- a/packages/cli/src/coverage-source.ts +++ b/packages/cli/src/coverage-source.ts @@ -36,7 +36,10 @@ function endOfTypeAlias(text: string, from: number): number { for (let i = from; i < text.length; i += 1) { const ch = text[i]; if (ch === '{' || ch === '[' || ch === '(' || ch === '<') depth += 1; - else if (ch === '}' || ch === ']' || ch === ')' || ch === '>') depth -= 1; + // The `>` of `=>` closes nothing: counted, an arrow type drove depth below zero and the alias + // "ended" at a `;` inside its own braces, leaving the rest to read as code. + else if (ch === '}' || ch === ']' || ch === ')' || (ch === '>' && text[i - 1] !== '=')) + depth -= 1; else if (ch === ';' && depth <= 0) return i + 1; } return text.length; diff --git a/packages/cli/src/document-options.ts b/packages/cli/src/document-options.ts index 919ba9512..4a66f426b 100644 --- a/packages/cli/src/document-options.ts +++ b/packages/cli/src/document-options.ts @@ -20,9 +20,10 @@ export type IslandResolver = (routeFile: string) => IslandResolve; * rendered (`route-islands-preload.ts`). Optional so a resolver that is only a URL map (a test * double) still type-checks; it then earns no preload, never a wrong one. */ -export type IslandResolve = ((src: string) => string) & { +export interface IslandResolve { + (src: string): string; readonly importsOf?: (entryUrl: string) => readonly string[]; -}; +} export interface DocumentOptions { readonly resolveIsland?: IslandResolver; diff --git a/packages/cli/src/runtime-cache-redis-peer-fixture.ts b/packages/cli/src/runtime-cache-redis-peer-fixture.ts deleted file mode 100644 index 901bc6d2a..000000000 --- a/packages/cli/src/runtime-cache-redis-peer-fixture.ts +++ /dev/null @@ -1,21 +0,0 @@ -// Tests only: replica A, as its own process — boots the cache hop on Redis with no NATS, busts one -// tag, prints the report's errors as one JSON line, and exits. `runtime-cache-redis.live.test.ts` -// is replica B. Argv: . - -import { declareTags, invalidateTags, noopPurgeDriver, parseTag } from '@ultimat3/cache'; -import { InProcessTransport } from '@ultimat3/realtime/server'; -import { startCacheTiers } from './runtime-cache'; - -const [url = '', wire = ''] = Bun.argv.slice(2); -const bust = parseTag(wire); -declareTags([bust.entity]); -const release = startCacheTiers({ - env: { REDIS_URL: url }, - purge: noopPurgeDriver(), - transport: new InProcessTransport(), - tiers: ['lru'], -}); -const report = await invalidateTags([bust]); -await release(); -console.log(JSON.stringify(report.errors)); -process.exit(0); diff --git a/packages/cli/src/runtime-cache-redis.live.test.ts b/packages/cli/src/runtime-cache-redis.live.test.ts index 6a5d6cd8d..7f6e657df 100644 --- a/packages/cli/src/runtime-cache-redis.live.test.ts +++ b/packages/cli/src/runtime-cache-redis.live.test.ts @@ -24,7 +24,29 @@ import { CACHE_INVALIDATE_SUBJECT, startCacheTiers } from './runtime-cache'; const url = Bun.env['TEST_REDIS_URL']; const hasRedis = typeof url === 'string' && url.length > 0; -const PEER = `${import.meta.dir}/runtime-cache-redis-peer-fixture.ts`; +/** + * Replica A, run as its own process (`bun -e`, inline as `registry-lazy.test.ts` does: a fixture + * FILE only this live suite reaches is `X_COVERAGE_UNMEASURED` on a coverage run with no Redis). + * It boots the hop on Redis with no NATS, busts one tag, and prints the report's errors. + */ +const PEER = ` +const { declareTags, invalidateTags, noopPurgeDriver, parseTag } = await import('@ultimat3/cache'); +const { InProcessTransport } = await import('@ultimat3/realtime/server'); +const { startCacheTiers } = await import(${JSON.stringify(`${import.meta.dir}/runtime-cache.ts`)}); +const [url, wire] = Bun.argv.slice(-2); +const bust = parseTag(wire); +declareTags([bust.entity]); +const release = startCacheTiers({ + env: { REDIS_URL: url }, + purge: noopPurgeDriver(), + transport: new InProcessTransport(), + tiers: ['lru'], +}); +const report = await invalidateTags([bust]); +await release(); +console.log(JSON.stringify(report.errors)); +process.exit(0); +`; const KEY = 'live:post:1'; const PAGE = '/posts/1'; @@ -83,7 +105,12 @@ async function replicaB(): Promise<{ lru: CacheTier; revalidated: string[] }> { /** Replica A: a separate process busting `post:1`, as an action's `invalidates` would. */ async function writeOnA(): Promise { - const child = Bun.spawn(['bun', PEER, url ?? '', 'post:1'], { stdout: 'pipe', stderr: 'pipe' }); + const child = Bun.spawn(['bun', '-e', PEER, url ?? '', 'post:1'], { + // The cli workspace, where `@ultimat3/cache` and `@ultimat3/realtime` resolve; the root has neither. + cwd: `${import.meta.dir}/..`, + stdout: 'pipe', + stderr: 'pipe', + }); const out = await new Response(child.stdout).text(); expect(await child.exited).toBe(0); // No `broadcast` error: A's publish was accepted by Redis. diff --git a/scripts/bench/boot-vs-files-child.ts b/scripts/bench/boot-vs-files-child.ts index 149700b8c..e2d85c07c 100644 --- a/scripts/bench/boot-vs-files-child.ts +++ b/scripts/bench/boot-vs-files-child.ts @@ -8,31 +8,46 @@ import { importInOrder } from '../../packages/cli/src/app-load-batch'; import { parseScriptArgs } from '../lib/args'; import { ScriptError } from '../lib/script-error'; -const args = parseScriptArgs(Bun.argv.slice(2)); -const dir = args.flags.get('dir'); -const mode = args.flags.get('mode'); -if (typeof dir !== 'string' || (mode !== 'serial' && mode !== 'batched')) { - throw new ScriptError({ - code: 'X_CLI_BAD_FLAG', - cause: 'boot-vs-files-child is spawned by the bench with --dir and --mode serial|batched', - fix: 'bun run scripts/bench/boot-vs-files.bench.ts --files 500', - }); -} +export type BootMode = 'serial' | 'batched'; -const files = [...new Bun.Glob('*.ts').scanSync({ cwd: dir, absolute: true })].sort(); -const started = performance.now(); -if (mode === 'serial') { - for (const file of files) await import(file); -} else { - const imported = await importInOrder(files); - const failed = imported.find((entry) => !entry.ok); - if (failed !== undefined && !failed.ok) { +/** `--dir` and `--mode`, or the refusal naming both. */ +export function bootChildArgs(argv: readonly string[]): { dir: string; mode: BootMode } { + const args = parseScriptArgs(argv); + const dir = args.flags.get('dir'); + const mode = args.flags.get('mode'); + if (typeof dir !== 'string' || (mode !== 'serial' && mode !== 'batched')) { throw new ScriptError({ - code: 'X_BENCH_RUN_FAILED', - cause: 'a synthetic app module failed to import, so the boot time measures a broken app', - fix: 'bun run scripts/bench/boot-vs-files.bench.ts --files 500 --json', + code: 'X_CLI_BAD_FLAG', + cause: 'boot-vs-files-child is spawned by the bench with --dir and --mode serial|batched', + fix: 'bun run scripts/bench/boot-vs-files.bench.ts --files 500', }); } + return { dir, mode }; +} + +/** Import every `*.ts` under `dir` in sorted order, the way `mode` boots, and time it. */ +export async function bootModules( + dir: string, + mode: BootMode, +): Promise<{ modules: number; ms: number }> { + const files = [...new Bun.Glob('*.ts').scanSync({ cwd: dir, absolute: true })].sort(); + const started = performance.now(); + if (mode === 'serial') { + for (const file of files) await import(file); + } else { + const imported = await importInOrder(files); + if (imported.some((entry) => !entry.ok)) { + throw new ScriptError({ + code: 'X_BENCH_RUN_FAILED', + cause: 'a synthetic app module failed to import, so the boot time measures a broken app', + fix: 'bun run scripts/bench/boot-vs-files.bench.ts --files 500 --json', + }); + } + } + return { modules: files.length, ms: Math.round(performance.now() - started) }; +} + +if (import.meta.main) { + const { dir, mode } = bootChildArgs(Bun.argv.slice(2)); + console.log(`FINAL ${JSON.stringify(await bootModules(dir, mode))}`); } -const ms = performance.now() - started; -console.log(`FINAL ${JSON.stringify({ modules: files.length, ms: Math.round(ms) })}`); diff --git a/scripts/bench/boot-vs-files.bench.test.ts b/scripts/bench/boot-vs-files.bench.test.ts new file mode 100644 index 000000000..fd2d9eca1 --- /dev/null +++ b/scripts/bench/boot-vs-files.bench.test.ts @@ -0,0 +1,87 @@ +// The boot bench end to end on a tiny fixture: real cold boots in child processes, both modes, the +// figure parsed back — and the flags it refuses before any process starts. The child's own import +// loop is driven in process over plain modules, so no entity registers into this test's registry. + +import { afterAll, describe, expect, test } from 'bun:test'; +// why: Bun ships no temp directory or recursive delete of its own. +import { mkdtemp, rm } from 'node:fs/promises'; +// why: as above — the platform temp root is only node:os's to name. +import { tmpdir } from 'node:os'; +import { ScriptError } from '../lib/script-error'; +import { bootArgs, runBootBench } from './boot-vs-files.bench'; +import { bootChildArgs, bootModules } from './boot-vs-files-child'; + +const scratch: string[] = []; +afterAll(async () => { + for (const dir of scratch) await rm(dir, { recursive: true, force: true }); +}); + +const codeOf = (work: () => unknown): string | undefined => { + try { + work(); + } catch (error) { + return error instanceof ScriptError ? error.code : 'not a ScriptError'; + } + return undefined; +}; + +describe('boot-vs-files bench', () => { + test('two cold boots per mode, a figure each, written where --out says', async () => { + const dir = await mkdtemp(`${tmpdir()}/boot-bench-`); + scratch.push(dir); + const lines: string[] = []; + const results = await runBootBench( + ['--files', '3', '--runs', '1', '--out', `${dir}/out.json`], + (line) => lines.push(line), + ); + expect(results.map((run) => run.files)).toEqual([3]); + expect(results[0]?.serialMs).toBeGreaterThanOrEqual(0); + expect(results[0]?.batchedMs).toBeGreaterThanOrEqual(0); + expect(lines).toEqual([expect.stringMatching(/^3 files: serial \d+ ms, batched \d+ ms$/)]); + const written = (await Bun.file(`${dir}/out.json`).json()) as { results: unknown[] }; + expect(written.results).toHaveLength(1); + }, 60_000); + + test('--json prints the whole result instead of the lines', async () => { + const lines: string[] = []; + await runBootBench(['--files', '2', '--runs', '1', '--json'], (line) => lines.push(line)); + expect(JSON.parse(lines.join('\n'))).toMatchObject({ runs: 1, results: [{ files: 2 }] }); + }, 60_000); + + test('a file count or run count out of range is refused before anything runs', () => { + expect(codeOf(() => bootArgs(['--files', '0']))).toBe('X_CLI_BAD_FLAG'); + expect(codeOf(() => bootArgs(['--files', '2.5']))).toBe('X_CLI_BAD_FLAG'); + expect(codeOf(() => bootArgs(['--runs', '99']))).toBe('X_CLI_BAD_FLAG'); + expect(bootArgs([]).counts).toEqual([500, 2_000, 5_000]); + }); +}); + +describe('boot-vs-files child', () => { + test('imports every module in both modes and counts them', async () => { + const dir = await mkdtemp(`${tmpdir()}/boot-child-`); + scratch.push(dir); + await Bun.write(`${dir}/a.ts`, 'export const a = 1;\n'); + await Bun.write(`${dir}/b.ts`, "import { a } from './a';\nexport const b = a + 1;\n"); + expect((await bootModules(dir, 'serial')).modules).toBe(2); + expect((await bootModules(dir, 'batched')).modules).toBe(2); + }); + + test('a module that will not import fails the batched boot, never times it', async () => { + const dir = await mkdtemp(`${tmpdir()}/boot-broken-`); + scratch.push(dir); + await Bun.write(`${dir}/a.ts`, "throw new TypeError('broken on purpose');\n"); + const refused = await bootModules(dir, 'batched').then( + () => 'resolved', + (error: unknown) => (error instanceof ScriptError ? error.code : 'not a ScriptError'), + ); + expect(refused).toBe('X_BENCH_RUN_FAILED'); + }); + + test('--dir and --mode are required', () => { + expect(codeOf(() => bootChildArgs(['--dir', '/x']))).toBe('X_CLI_BAD_FLAG'); + expect(bootChildArgs(['--dir', '/x', '--mode', 'serial'])).toEqual({ + dir: '/x', + mode: 'serial', + }); + }); +}); diff --git a/scripts/bench/boot-vs-files.bench.ts b/scripts/bench/boot-vs-files.bench.ts index 1a2c5bf80..6e3bb90ae 100644 --- a/scripts/bench/boot-vs-files.bench.ts +++ b/scripts/bench/boot-vs-files.bench.ts @@ -32,7 +32,8 @@ export interface BootRun { const fixtureRoot = (): string => `${repoRoot()}/packages/cli/node_modules/.cache/bench-boot`; -async function writeFixture(files: number): Promise { +/** N synthetic modules under the CLI's install, written fresh; answers the directory. */ +export async function writeFixture(files: number): Promise { const dir = `${fixtureRoot()}/${String(files)}`; await rm(dir, { recursive: true, force: true }); await mkdir(dir, { recursive: true }); @@ -97,8 +98,9 @@ export async function benchBoot(counts: readonly number[], runs: number): Promis return out; } -if (import.meta.main) { - const args = parseScriptArgs(Bun.argv.slice(2)); +/** `--files` and `--runs`, or the refusal naming both. */ +export function bootArgs(argv: readonly string[]) { + const args = parseScriptArgs(argv); const rawFiles = args.flags.get('files'); const counts = typeof rawFiles === 'string' @@ -116,14 +118,25 @@ if (import.meta.main) { fix: 'bun run scripts/bench/boot-vs-files.bench.ts --files 500,2000,5000 --runs 3', }); } + const out = args.flags.get('out'); + return { counts, runs, out: typeof out === 'string' ? out : undefined, json: args.json }; +} + +/** The whole command: measure, write `--out`, print. `log` is the test's seam onto stdout. */ +export async function runBootBench( + argv: readonly string[], + log: (line: string) => void = console.log, +): Promise { + const { counts, runs, out, json } = bootArgs(argv); const results = await benchBoot(counts, runs); const data = { box: boxFacts(Bun.env), runs, results }; - const out = args.flags.get('out'); - if (typeof out === 'string') await Bun.write(out, `${JSON.stringify(data, null, 2)}\n`); - if (args.json) console.log(JSON.stringify(data, null, 2)); + if (out !== undefined) await Bun.write(out, `${JSON.stringify(data, null, 2)}\n`); + if (json) log(JSON.stringify(data, null, 2)); else { - for (const r of results) { - console.log(`${r.files} files: serial ${r.serialMs} ms, batched ${r.batchedMs} ms`); - } + for (const r of results) + log(`${r.files} files: serial ${r.serialMs} ms, batched ${r.batchedMs} ms`); } + return results; } + +if (import.meta.main) await runBootBench(Bun.argv.slice(2)); diff --git a/scripts/bench/fanout-3-nodes.bench.test.ts b/scripts/bench/fanout-3-nodes.bench.test.ts new file mode 100644 index 000000000..7f33bd0d7 --- /dev/null +++ b/scripts/bench/fanout-3-nodes.bench.test.ts @@ -0,0 +1,108 @@ +// The fan-out bench with no NATS: three nodes in this process over one in-process bus, the real +// probe loop and the real aggregation — what the published run does, minus the processes. Plus the +// child's stdout reader and the flags refused before anything starts. + +import { afterAll, describe, expect, test } from 'bun:test'; +import { clearChannels } from '../../packages/realtime/src/channel-registry'; +import { InProcessTransport } from '../../packages/realtime/src/fanout'; +import { ScriptError } from '../lib/script-error'; +import { + benchFanout3, + type FanoutRig, + fanoutArgs, + fanoutReport, + lineStarting, +} from './fanout-3-nodes.bench'; +import { fanoutNode, fanoutNodeArgs } from './fanout-node-child'; + +afterAll(() => { + clearChannels(); +}); + +/** Three nodes and the publisher on one in-process bus: the rig `natsRig` is, without processes. */ +const inProcessRig = async (org: string, counts: readonly number[], events: number) => { + const bus = new InProcessTransport(); + const nodes = await Promise.all( + counts.map((subscribers) => fanoutNode(bus, { org, subscribers, events, windowMs: 5_000 })), + ); + const rig: FanoutRig = { + publisher: bus, + finals: () => Promise.all(nodes.map((node) => node.final)), + close: async () => { + for (const node of nodes) await node.close(); + }, + }; + return rig; +}; + +const codeOf = (work: () => unknown): string | undefined => { + try { + work(); + } catch (error) { + return error instanceof ScriptError ? error.code : 'not a ScriptError'; + } + return undefined; +}; + +/** A stream that hands over `chunks`, then ends. */ +const streamOf = (chunks: readonly string[]) => + new ReadableStream({ + start(controller) { + for (const chunk of chunks) controller.enqueue(new TextEncoder().encode(chunk)); + controller.close(); + }, + }).getReader(); + +describe('fan-out bench', () => { + test('every probe reaches every socket on every node, and the figure counts them', async () => { + const run = await benchFanout3( + { subscribers: 7, events: 3, intervalMs: 10, settleMs: 0 }, + (org, counts) => inProcessRig(org, counts, 3), + ); + expect({ nodes: run.nodes, delivered: run.delivered, expected: run.expected }).toEqual({ + nodes: 3, + delivered: 21, + expected: 21, + }); + expect(run.perNode.map((node) => node.deliveries)).toEqual([9, 6, 6]); + expect(run.overall.maxMs).toBeGreaterThanOrEqual(run.overall.p50Ms); + expect(fanoutReport(run, 10, false)).toStartWith('7 subscribers on 3 nodes, 3 probes: 21/21'); + expect(JSON.parse(fanoutReport(run, 10, true))).toMatchObject({ intervalMs: 10 }); + }); + + test('a node whose window closes first reports what it got, never hangs', async () => { + const bus = new InProcessTransport(); + const node = await fanoutNode(bus, { org: 'quiet', subscribers: 2, events: 1, windowMs: 20 }); + expect(await node.final).toMatchObject({ received: 0, expected: 2 }); + await node.close(); + }); + + test('the stdout reader finds its line across chunks, and refuses a stream that ended first', async () => { + const carry = { buffer: '' }; + const reader = streamOf(['noise\nRE', 'ADY\nFINAL {"a":1}\n']); + expect(await lineStarting(reader, carry, 'READY')).toBe(''); + expect(await lineStarting(reader, carry, 'FINAL ')).toBe('{"a":1}'); + const ended = await lineStarting(streamOf(['nothing\n']), { buffer: '' }, 'READY').then( + () => 'resolved', + (error: unknown) => (error instanceof ScriptError ? error.code : 'not a ScriptError'), + ); + expect(ended).toBe('X_BENCH_RUN_FAILED'); + }); + + test('flags: a NATS url is required, counts are bounded', () => { + expect(codeOf(() => fanoutArgs([], {}))).toBe('X_CLI_BAD_FLAG'); + expect(codeOf(() => fanoutArgs(['--nats', 'nats://x:4222', '--subscribers', '1'], {}))).toBe( + 'X_CLI_BAD_FLAG', + ); + expect(fanoutArgs([], { TEST_NATS_URL: 'nats://x:4222' })).toMatchObject({ + nats: 'nats://x:4222', + subscribers: 10_000, + events: 20, + intervalMs: 250, + }); + expect(codeOf(() => fanoutNodeArgs(['--nats', 'n', '--org', 'o']))).toBe('X_CLI_BAD_FLAG'); + expect( + fanoutNodeArgs(['--nats', 'n', '--org', 'o', '--subscribers', '2', '--events', '3']), + ).toEqual({ nats: 'n', org: 'o', subscribers: 2, events: 3 }); + }); +}); diff --git a/scripts/bench/fanout-3-nodes.bench.ts b/scripts/bench/fanout-3-nodes.bench.ts index 36fc03435..a1558a9b3 100644 --- a/scripts/bench/fanout-3-nodes.bench.ts +++ b/scripts/bench/fanout-3-nodes.bench.ts @@ -16,9 +16,11 @@ import { installChannelTransport, publishChannelEvent, } from '../../packages/realtime/src/channel-publish'; +import type { Transport } from '../../packages/realtime/src/fanout'; import { NatsTransport } from '../../packages/realtime/src/nats-transport'; import { parseScriptArgs } from '../lib/args'; import { ScriptError } from '../lib/script-error'; +import type { FanoutFinal } from './fanout-node-child'; import { FANOUT_BUCKET, fanoutChannel, @@ -44,7 +46,7 @@ const failed = (cause: string): ScriptError => }); /** Lines of a child's stdout, until one starts with `prefix`; returns the rest of that line. */ -async function lineStarting( +export async function lineStarting( reader: ReadableStreamDefaultReader, carry: { buffer: string }, prefix: string, @@ -74,14 +76,22 @@ export interface FanoutRun { readonly perNode: readonly LatencySummary[]; } -export async function benchFanout3( +/** What a run stands on: the nodes, already subscribed, and a connected publisher. */ +export interface FanoutRig { + /** Each node's final, once its window closes. */ + readonly finals: () => Promise; + readonly publisher: Transport; + close(): Promise; +} + +/** The production rig: one child process per node over NATS, and a fourth connection publishing. */ +export async function natsRig( nats: string, - subscribers: number, + org: string, + counts: readonly number[], events: number, - intervalMs: number, -): Promise { - const org = crypto.randomUUID(); - const children = splitAcross(subscribers, NODES).map((count) => +): Promise { + const children = counts.map((count) => Bun.spawn( [ 'bun', @@ -103,42 +113,84 @@ export async function benchFanout3( carry: { buffer: '' }, })); const publisher = new NatsTransport({ url: nats, bucket: FANOUT_BUCKET }); + const close = async (): Promise => { + for (const child of children) child.kill(); + await Promise.all(children.map((child) => child.exited)); + await publisher.close(); + }; try { await Promise.all(readers.map(({ reader, carry }) => lineStarting(reader, carry, 'READY'))); await publisher.connect(); - installChannelTransport(publisher); + } catch (error) { + await close(); + throw error; + } + return { + publisher, + close, + finals: () => + Promise.all( + readers.map(async ({ reader, carry }) => { + const body: unknown = JSON.parse(await lineStarting(reader, carry, 'FINAL ')); + return body as FanoutFinal; + }), + ), + }; +} + +/** The figure a run reports, from every node's final. */ +export function fanoutRunOf( + finals: readonly FanoutFinal[], + subscribers: number, + events: number, +): FanoutRun { + return { + nodes: finals.length, + subscribers, + events, + delivered: finals.reduce((sum, final) => sum + final.received, 0), + expected: subscribers * events, + overall: summarise(finals.flatMap((final) => final.latencies.flat())), + perNode: finals.map((final) => summarise(final.latencies.flat())), + }; +} + +export interface FanoutInput { + readonly subscribers: number; + readonly events: number; + readonly intervalMs: number; + /** How long the nodes' subscriptions get to reach the bus before the first probe. */ + readonly settleMs?: number; +} + +/** `events` probes through `rig.publisher`, `intervalMs` apart, then every node's latencies. */ +export async function benchFanout3( + input: FanoutInput, + rig: (org: string, counts: readonly number[]) => Promise, +): Promise { + const org = crypto.randomUUID(); + const built = await rig(org, splitAcross(input.subscribers, NODES)); + try { + installChannelTransport(built.publisher); const channel = fanoutChannel(); - await Bun.sleep(SETTLE_MS); - for (let index = 0; index < events; index += 1) { + await Bun.sleep(input.settleMs ?? SETTLE_MS); + for (let index = 0; index < input.events; index += 1) { await publishChannelEvent(channel, { org }, { probeIndex: index, probeSentAt: nowMs() }); - await Bun.sleep(intervalMs); + await Bun.sleep(input.intervalMs); } - const finals = await Promise.all( - readers.map(async ({ reader, carry }) => { - const body: unknown = JSON.parse(await lineStarting(reader, carry, 'FINAL ')); - return body as { received: number; expected: number; latencies: number[][] }; - }), - ); - const perNode = finals.map((final) => summarise(final.latencies.flat())); - return { - nodes: NODES, - subscribers, - events, - delivered: finals.reduce((sum, final) => sum + final.received, 0), - expected: subscribers * events, - overall: summarise(finals.flatMap((final) => final.latencies.flat())), - perNode, - }; + return fanoutRunOf(await built.finals(), input.subscribers, input.events); } finally { - for (const child of children) child.kill(); - await Promise.all(children.map((child) => child.exited)); - await publisher.close(); + await built.close(); } } -if (import.meta.main) { - const args = parseScriptArgs(Bun.argv.slice(2)); - const nats = args.flags.get('nats') ?? Bun.env['TEST_NATS_URL']; +/** The flags, or the refusal naming them. `--nats` falls back to `TEST_NATS_URL`. */ +export function fanoutArgs( + argv: readonly string[], + env: Readonly>, +) { + const args = parseScriptArgs(argv); + const nats = args.flags.get('nats') ?? env['TEST_NATS_URL']; const subscribers = readCount(args.flags.get('subscribers'), 10_000, { flag: 'subscribers', min: NODES, @@ -163,14 +215,29 @@ if (import.meta.main) { fix: 'bun run scripts/bench/fanout-3-nodes.bench.ts --nats nats://127.0.0.1:4222 --subscribers 10000', }); } - const run = await benchFanout3(nats, subscribers, events, intervalMs); - const data = { box: boxFacts(Bun.env), intervalMs, run }; const out = args.flags.get('out'); - if (typeof out === 'string') await Bun.write(out, `${JSON.stringify(data, null, 2)}\n`); - if (args.json) console.log(JSON.stringify(data, null, 2)); - else { - console.log( - `${run.subscribers} subscribers on ${run.nodes} nodes, ${run.events} probes: ${run.delivered}/${run.expected} delivered, p50 ${run.overall.p50Ms} ms, p99 ${run.overall.p99Ms} ms, max ${run.overall.maxMs} ms`, - ); - } + return { + nats, + subscribers, + events, + intervalMs, + out: typeof out === 'string' ? out : undefined, + json: args.json, + }; +} + +/** The report a run prints: the whole JSON, or one line. */ +export function fanoutReport(run: FanoutRun, intervalMs: number, json: boolean): string { + if (json) return JSON.stringify({ box: boxFacts(Bun.env), intervalMs, run }, null, 2); + return `${run.subscribers} subscribers on ${run.nodes} nodes, ${run.events} probes: ${run.delivered}/${run.expected} delivered, p50 ${run.overall.p50Ms} ms, p99 ${run.overall.p99Ms} ms, max ${run.overall.maxMs} ms`; +} + +if (import.meta.main) { + const args = fanoutArgs(Bun.argv.slice(2), Bun.env); + const run = await benchFanout3(args, (org, counts) => + natsRig(args.nats, org, counts, args.events), + ); + const data = { box: boxFacts(Bun.env), intervalMs: args.intervalMs, run }; + if (args.out !== undefined) await Bun.write(args.out, `${JSON.stringify(data, null, 2)}\n`); + console.log(fanoutReport(run, args.intervalMs, args.json)); } diff --git a/scripts/bench/fanout-node-child.ts b/scripts/bench/fanout-node-child.ts index b216b2020..ab1ace085 100644 --- a/scripts/bench/fanout-node-child.ts +++ b/scripts/bench/fanout-node-child.ts @@ -7,72 +7,113 @@ import { ChannelHub } from '../../packages/realtime/src/channel'; import { clearChannels } from '../../packages/realtime/src/channel-registry'; +import type { Transport } from '../../packages/realtime/src/fanout'; import { NatsTransport } from '../../packages/realtime/src/nats-transport'; import { SocketRegistry, SyncSocket, type WsLike } from '../../packages/realtime/src/socket'; import { parseScriptArgs } from '../lib/args'; import { ScriptError } from '../lib/script-error'; import { FANOUT_BUCKET, fanoutChannel, latencyOf, nowMs } from './fanout-shape'; -const args = parseScriptArgs(Bun.argv.slice(2)); -const nats = args.flags.get('nats'); -const org = args.flags.get('org'); -const subscribers = Number(args.flags.get('subscribers')); -const events = Number(args.flags.get('events')); -if (typeof nats !== 'string' || typeof org !== 'string' || !(subscribers > 0) || !(events > 0)) { - throw new ScriptError({ - code: 'X_CLI_BAD_FLAG', - cause: - 'fanout-node-child is spawned by the bench with --nats, --org, --subscribers and --events', - fix: 'bun run scripts/bench/fanout-3-nodes.bench.ts --subscribers 3000', - }); +export interface FanoutNodeArgs { + readonly nats: string; + readonly org: string; + readonly subscribers: number; + readonly events: number; } -/** Per probe index, every socket's latency in ms. Read off the frame text: no decode on the clock. */ -const latencies: number[][] = Array.from({ length: events }, () => []); -let received = 0; -const window = Promise.withResolvers(); - -class ProbeWs implements WsLike { - send(data: string): number { - const probe = latencyOf(data, nowMs()); - if (probe !== undefined) { - latencies[probe.index]?.push(probe.ms); - received += 1; - if (received === subscribers * events) window.resolve(); - } - return data.length; - } - close(): void {} - subscribe(): void {} - unsubscribe(): void {} - getBufferedAmount(): number { - return 0; +/** The four flags the bench spawns a node with, or the refusal naming them. */ +export function fanoutNodeArgs(argv: readonly string[]): FanoutNodeArgs { + const args = parseScriptArgs(argv); + const nats = args.flags.get('nats'); + const org = args.flags.get('org'); + const subscribers = Number(args.flags.get('subscribers')); + const events = Number(args.flags.get('events')); + if (typeof nats !== 'string' || typeof org !== 'string' || !(subscribers > 0) || !(events > 0)) { + throw new ScriptError({ + code: 'X_CLI_BAD_FLAG', + cause: + 'fanout-node-child is spawned by the bench with --nats, --org, --subscribers and --events', + fix: 'bun run scripts/bench/fanout-3-nodes.bench.ts --subscribers 3000', + }); } + return { nats, org, subscribers, events }; +} + +export interface FanoutFinal { + readonly received: number; + readonly expected: number; + /** Per probe index, every socket's latency in ms. */ + readonly latencies: readonly (readonly number[])[]; } -const channel = fanoutChannel(); -const transport = new NatsTransport({ url: nats, bucket: FANOUT_BUCKET }); -await transport.connect(); -const sockets = new SocketRegistry(); -const hub = new ChannelHub({ transport, sockets }); -for (let i = 0; i < subscribers; i += 1) { - const socket = new SyncSocket({ - ws: new ProbeWs(), - clientBuildId: 'bench', - serverBuildId: 'bench', - actor: null, +export interface FanoutNode { + /** Settles once every probe reached every socket, or the window closed. */ + readonly final: Promise; + close(): Promise; +} + +/** + * A node over `transport` holding `subscribers` sockets on the bench topic, every one subscribed + * before this resolves. Latency is read off the frame TEXT at `send`: no decode on the clock. + */ +export async function fanoutNode( + transport: Transport, + input: Omit & { readonly windowMs: number }, +): Promise { + const latencies: number[][] = Array.from({ length: input.events }, () => []); + const expected = input.subscribers * input.events; + let received = 0; + const window = Promise.withResolvers(); + const probe: WsLike = { + send(data: string): number { + const seen = latencyOf(data, nowMs()); + if (seen !== undefined) { + latencies[seen.index]?.push(seen.ms); + received += 1; + if (received === expected) window.resolve(); + } + return data.length; + }, + close: () => undefined, + subscribe: () => undefined, + unsubscribe: () => undefined, + getBufferedAmount: () => 0, + }; + const channel = fanoutChannel(); + const sockets = new SocketRegistry(); + const hub = new ChannelHub({ transport, sockets }); + for (let i = 0; i < input.subscribers; i += 1) { + const socket = new SyncSocket({ + ws: probe, + clientBuildId: 'bench', + serverBuildId: 'bench', + actor: null, + }); + sockets.add(socket); + await hub.subscribeChannel(socket, { + kind: 'channel', + channel: channel.name, + params: { org: input.org }, + }); + } + // The window: every probe the publisher sends plus a tail, never forever. + const timeout = setTimeout(() => window.resolve(), input.windowMs); + const final = window.promise.then(() => { + clearTimeout(timeout); + return { received, expected, latencies }; }); - sockets.add(socket); - await hub.subscribeChannel(socket, { kind: 'channel', channel: channel.name, params: { org } }); + return { final, close: () => hub.close() }; } -console.log('READY'); -// The window: every probe the publisher sends plus a generous tail, never forever. -const timeout = setTimeout(() => window.resolve(), 30_000 + events * 2_000); -await window.promise; -clearTimeout(timeout); -console.log(`FINAL ${JSON.stringify({ received, expected: subscribers * events, latencies })}`); -await hub.close(); -await transport.close(); -clearChannels(); -process.exit(0); +if (import.meta.main) { + const args = fanoutNodeArgs(Bun.argv.slice(2)); + const transport = new NatsTransport({ url: args.nats, bucket: FANOUT_BUCKET }); + await transport.connect(); + const node = await fanoutNode(transport, { ...args, windowMs: 30_000 + args.events * 2_000 }); + console.log('READY'); + console.log(`FINAL ${JSON.stringify(await node.final)}`); + await node.close(); + await transport.close(); + clearChannels(); + process.exit(0); +} diff --git a/scripts/bench/fanout-shape.ts b/scripts/bench/fanout-shape.ts index 9828915a3..d0a198446 100644 --- a/scripts/bench/fanout-shape.ts +++ b/scripts/bench/fanout-shape.ts @@ -4,12 +4,17 @@ import { allow } from '../../packages/policy/src/index'; import { channel } from '../../packages/realtime/src/channel-decl'; +import { getChannel } from '../../packages/realtime/src/channel-registry'; /** The presence bucket this bench's transports use — never the `x_presence` an app uses. */ export const FANOUT_BUCKET = 'xbenchfanout'; -/** The declaration every process makes: a channel is code, loaded by every role. */ +/** + * The declaration every process makes: a channel is code, loaded by every role. Declared once per + * process — the publisher and a node share one when a test runs both in it. + */ export const fanoutChannel = () => + getChannel('benchfanout') ?? channel('benchfanout', { params: ['org'], catchUp: { name: 'benchFanoutRead' }, diff --git a/scripts/bench/http-prod-image.bench.test.ts b/scripts/bench/http-prod-image.bench.test.ts new file mode 100644 index 000000000..647ccfa5d --- /dev/null +++ b/scripts/bench/http-prod-image.bench.test.ts @@ -0,0 +1,72 @@ +// The production-image bench against a local server standing in for one `web` pod: the readiness +// wait, the load loop and the verdict — what `--pods 1` does against the Compose `web`. Pods 2..n +// are `docker compose run`, which no unit test starts; the flags are refused before any of it. + +import { afterAll, describe, expect, test } from 'bun:test'; +import { allowHost } from '../../packages/testing/src/sealed-network'; +import { ScriptError } from '../lib/script-error'; +import { benchPods, podsArgs, podsReport } from './http-prod-image.bench'; + +const servers: { stop(force?: boolean): unknown }[] = []; +afterAll(() => { + for (const server of servers) server.stop(true); +}); + +/** A pod on a free port: `/readyz` answers `ready`, `/` answers `status`. */ +const pod = (status: number, readyStatus = 200): number => { + const server = Bun.serve({ + port: 0, + fetch: (request) => + new URL(request.url).pathname === '/readyz' + ? new Response('ok', { status: readyStatus }) + : new Response('hello', { status }), + }); + servers.push(server); + // The test preload seals the network; this one loopback pod is let through, by host and port. + allowHost(`127.0.0.1:${String(server.port)}`); + return server.port ?? 0; +}; + +const refusalOf = (work: Promise) => + work.then( + () => 'resolved', + (error: unknown) => (error instanceof ScriptError ? error.code : 'not a ScriptError'), + ); + +const run = { app: '/no/app', pods: 1, path: '/', seconds: 0.2, concurrency: 2 }; + +describe('http-prod-image bench', () => { + test('one ready pod is loaded and its answers summed', async () => { + const result = await benchPods({ ...run, basePort: pod(200) }); + expect(result.pods).toBe(1); + expect(result.rps).toBeGreaterThan(0); + expect(result.perPod[0]?.non2xx).toBe(0); + expect(podsReport([result])[0]).toMatch(/^1 pod\(s\): \d+ req\/s, worst p50 /); + }); + + test('a pod answering 500 is a failed run, never a figure', async () => { + expect(await refusalOf(benchPods({ ...run, basePort: pod(500) }))).toBe('X_BENCH_RUN_FAILED'); + }); + + test('a pod that never reports ready is a failed run', async () => { + const port = pod(200, 503); + expect(await refusalOf(benchPods({ ...run, basePort: port, readyTimeoutMs: 50 }))).toBe( + 'X_BENCH_RUN_FAILED', + ); + }); + + test('flags: --app is required, pods and seconds are bounded', () => { + const codeOf = (argv: readonly string[]) => { + try { + podsArgs(argv); + } catch (error) { + return error instanceof ScriptError ? error.code : 'not a ScriptError'; + } + return undefined; + }; + expect(codeOf([])).toBe('X_CLI_BAD_FLAG'); + expect(codeOf(['--app', 'a', '--pods', '9'])).toBe('X_CLI_BAD_FLAG'); + expect(codeOf(['--app', 'a', '--seconds', '5'])).toBe('X_CLI_BAD_FLAG'); + expect(podsArgs(['--app', 'a'])).toMatchObject({ podCounts: [1, 3], path: '/', seconds: 30 }); + }); +}); diff --git a/scripts/bench/http-prod-image.bench.ts b/scripts/bench/http-prod-image.bench.ts index 294f4d9d0..475321d18 100644 --- a/scripts/bench/http-prod-image.bench.ts +++ b/scripts/bench/http-prod-image.bench.ts @@ -41,9 +41,9 @@ const answerFetch = async (url: string) => { return { status: response.status, bytes: (await response.arrayBuffer()).byteLength }; }; -async function ready(port: number): Promise { +async function ready(port: number, timeoutMs: number): Promise { const started = performance.now(); - while (performance.now() - started < READY_TIMEOUT_MS) { + while (performance.now() - started < timeoutMs) { const status = await answerFetch(`http://127.0.0.1:${String(port)}/readyz`).then( (answer) => answer.status, () => 0, @@ -55,9 +55,9 @@ async function ready(port: number): Promise { } /** Pods 2..n: the Compose `web` service run again, detached, on the next host ports. */ -function extraPods(app: string, pods: number): readonly string[] { +function extraPods(app: string, pods: number, basePort: number): readonly string[] { return Array.from({ length: pods - 1 }, (_, index) => { - const port = BASE_PORT + index + 1; + const port = basePort + index + 1; const run = Bun.spawnSync( [...COMPOSE, 'run', '-d', '--no-deps', '-p', `127.0.0.1:${String(port)}:3000`, 'web'], { cwd: app, timeout: 120_000 }, @@ -80,17 +80,24 @@ export interface PodsRun { readonly perPod: readonly LoadResult[]; } -export async function benchPods( - app: string, - pods: number, - path: string, - seconds: number, - concurrency: number, -): Promise { - const extra = extraPods(app, pods); +export interface PodsInput { + readonly app: string; + readonly pods: number; + readonly path: string; + readonly seconds: number; + readonly concurrency: number; + /** Pod 1's host port; pods 2..n take the ports after it. Default 3000, the Compose `web`'s. */ + readonly basePort?: number; + readonly readyTimeoutMs?: number; +} + +export async function benchPods(input: PodsInput): Promise { + const { pods, path, seconds, concurrency } = input; + const basePort = input.basePort ?? BASE_PORT; + const extra = extraPods(input.app, pods, basePort); try { - const ports = Array.from({ length: pods }, (_, index) => BASE_PORT + index); - await Promise.all(ports.map(ready)); + const ports = Array.from({ length: pods }, (_, index) => basePort + index); + await Promise.all(ports.map((port) => ready(port, input.readyTimeoutMs ?? READY_TIMEOUT_MS))); const perPod = await Promise.all( ports.map((port) => runLoad({ @@ -116,8 +123,9 @@ export async function benchPods( } } -if (import.meta.main) { - const args = parseScriptArgs(Bun.argv.slice(2)); +/** The flags, or the refusal naming them. */ +export function podsArgs(argv: readonly string[]) { + const args = parseScriptArgs(argv); const app = args.flags.get('app'); const rawPods = args.flags.get('pods'); const podCounts = typeof rawPods === 'string' ? rawPods.split(',').map(Number) : [1, 3]; @@ -145,24 +153,37 @@ if (import.meta.main) { fix: 'bun run scripts/bench/http-prod-image.bench.ts --app /proofapp --pods 1,3', }); } - const target = typeof path === 'string' ? path : '/'; + const out = args.flags.get('out'); + return { + app, + podCounts, + path: typeof path === 'string' ? path : '/', + seconds, + concurrency, + out: typeof out === 'string' ? out : undefined, + json: args.json, + }; +} + +/** One line per pod count. */ +export const podsReport = (runs: readonly PodsRun[]): readonly string[] => + runs.map( + (run) => + `${run.pods} pod(s): ${run.rps} req/s, worst p50 ${run.worstP50Ms} ms, worst p99 ${run.worstP99Ms} ms`, + ); + +if (import.meta.main) { + const args = podsArgs(Bun.argv.slice(2)); const runs: PodsRun[] = []; - for (const pods of podCounts) runs.push(await benchPods(app, pods, target, seconds, concurrency)); + for (const pods of args.podCounts) runs.push(await benchPods({ ...args, pods })); const data = { box: boxFacts(Bun.env), - path: target, - seconds, - concurrencyPerPod: concurrency, + path: args.path, + seconds: args.seconds, + concurrencyPerPod: args.concurrency, runs, }; - const out = args.flags.get('out'); - if (typeof out === 'string') await Bun.write(out, `${JSON.stringify(data, null, 2)}\n`); + if (args.out !== undefined) await Bun.write(args.out, `${JSON.stringify(data, null, 2)}\n`); if (args.json) console.log(JSON.stringify(data, null, 2)); - else { - for (const run of runs) { - console.log( - `${run.pods} pod(s): ${run.rps} req/s, worst p50 ${run.worstP50Ms} ms, worst p99 ${run.worstP99Ms} ms`, - ); - } - } + else for (const line of podsReport(runs)) console.log(line); } diff --git a/scripts/bench/worker-throughput-child.ts b/scripts/bench/worker-throughput-child.ts index 044cb22e6..7633b4602 100644 --- a/scripts/bench/worker-throughput-child.ts +++ b/scripts/bench/worker-throughput-child.ts @@ -6,62 +6,94 @@ import { ctxOf } from '../../packages/core/src/index'; import { postgresClient } from '../../packages/db/src/index'; +import type { JobDriver } from '../../packages/jobs/src/driver'; import { postgresJobDriver } from '../../packages/jobs/src/driver-pg'; -import { job } from '../../packages/jobs/src/job'; +import { getJob, job } from '../../packages/jobs/src/job'; import { jobWorker } from '../../packages/jobs/src/worker'; +import type { Worker } from '../../packages/jobs/src/worker-types'; import { parseScriptArgs } from '../lib/args'; import { ScriptError } from '../lib/script-error'; import { benchJobInput, NOOP_JOB } from './worker-throughput-shape'; -const args = parseScriptArgs(Bun.argv.slice(2)); -const url = args.flags.get('url'); -const concurrency = Number(args.flags.get('concurrency')); -const pollIntervalMs = Number(args.flags.get('poll-ms')); -if (typeof url !== 'string' || !(concurrency >= 1) || !(pollIntervalMs >= 1)) { - throw new ScriptError({ - code: 'X_CLI_BAD_FLAG', - cause: - 'worker-throughput-child is spawned by the bench with --url, --concurrency and --poll-ms, and was started without them', - fix: 'bun run scripts/bench/worker-throughput.bench.ts --workers 1', - }); +export interface WorkerPodArgs { + readonly url: string; + readonly concurrency: number; + readonly pollIntervalMs: number; } -// Named, so the worker resolves it from the registry `job()` writes to — the bench has no app. -job<{ n: number }>({ - name: NOOP_JOB, - tenant: 'none', - input: benchJobInput, - idempotencyKey: (input) => String(input.n), - retry: { attempts: 1 }, - run: () => Promise.resolve(), -}); +/** The three flags the bench spawns a pod with, or the refusal naming them. */ +export function workerPodArgs(argv: readonly string[]): WorkerPodArgs { + const args = parseScriptArgs(argv); + const url = args.flags.get('url'); + const concurrency = Number(args.flags.get('concurrency')); + const pollIntervalMs = Number(args.flags.get('poll-ms')); + if (typeof url !== 'string' || !(concurrency >= 1) || !(pollIntervalMs >= 1)) { + throw new ScriptError({ + code: 'X_CLI_BAD_FLAG', + cause: + 'worker-throughput-child is spawned by the bench with --url, --concurrency and --poll-ms, and was started without them', + fix: 'bun run scripts/bench/worker-throughput.bench.ts --workers 1', + }); + } + return { url, concurrency, pollIntervalMs }; +} -const client = postgresClient({ url, role: 'worker' }); -const worker = jobWorker({ - driver: postgresJobDriver({ - executor: { - query: (text: string, values: readonly unknown[]): Promise => - client.query({ text, values }), - }, - }), - context: () => ctxOf({ role: 'worker', buildId: 'bench' }), - concurrency, - pollIntervalMs, - // An idle worker re-polls every 250 ms at most rather than backing off to 2 s, so the first claim - // after the batch lands is not a timer the bench would be measuring instead. - idlePollMaxMs: Math.max(pollIntervalMs, 250), - drainOnShutdown: false, -}); -worker.start(); -// The worker's timers are unrefed (a drained process must be free to exit), so a bare script would -// exit right here: this interval is the pod staying up until SIGTERM. -const alive = setInterval(() => undefined, 60_000); -console.log('ready'); +/** + * The no-op job, named so the worker resolves it from the registry `job()` writes to — the bench + * has no app. Declared once per process. + */ +export function registerNoopJob(): void { + if (getJob(NOOP_JOB) !== undefined) return; + job<{ n: number }>({ + name: NOOP_JOB, + tenant: 'none', + input: benchJobInput, + idempotencyKey: (input) => String(input.n), + retry: { attempts: 1 }, + run: () => Promise.resolve(), + }); +} -process.on('SIGTERM', () => { - clearInterval(alive); - void worker - .stop() - .then(() => client.close()) - .finally(() => process.exit(0)); -}); +/** One pod's worker over `driver`: the stock `jobWorker`, the profile's slots and poll floor. */ +export function workerPod( + driver: JobDriver, + profile: Pick, +): Worker { + registerNoopJob(); + return jobWorker({ + driver, + context: () => ctxOf({ role: 'worker', buildId: 'bench' }), + concurrency: profile.concurrency, + pollIntervalMs: profile.pollIntervalMs, + // An idle worker re-polls every 250 ms at most rather than backing off to 2 s, so the first + // claim after the batch lands is not a timer the bench would be measuring instead. + idlePollMaxMs: Math.max(profile.pollIntervalMs, 250), + drainOnShutdown: false, + }); +} + +if (import.meta.main) { + const args = workerPodArgs(Bun.argv.slice(2)); + const client = postgresClient({ url: args.url, role: 'worker' }); + const worker = workerPod( + postgresJobDriver({ + executor: { + query: (text: string, values: readonly unknown[]): Promise => + client.query({ text, values }), + }, + }), + args, + ); + worker.start(); + // The worker's timers are unrefed (a drained process must be free to exit), so a bare script + // would exit right here: this interval is the pod staying up until SIGTERM. + const alive = setInterval(() => undefined, 60_000); + console.log('ready'); + process.on('SIGTERM', () => { + clearInterval(alive); + void worker + .stop() + .then(() => client.close()) + .finally(() => process.exit(0)); + }); +} diff --git a/scripts/bench/worker-throughput.bench.test.ts b/scripts/bench/worker-throughput.bench.test.ts new file mode 100644 index 000000000..b8a4a1b1f --- /dev/null +++ b/scripts/bench/worker-throughput.bench.test.ts @@ -0,0 +1,100 @@ +// The worker bench's process-free halves: one pod's worker draining no-op jobs off the memory +// driver (the same `workerPod` the spawned child runs over Postgres), the `ready` handshake its +// harness reads, and the flags and report around them. + +import { afterEach, describe, expect, test } from 'bun:test'; +import { memoryJobDriver } from '../../packages/jobs/src/driver-memory'; +import { resetJobs } from '../../packages/jobs/src/job'; +import { ScriptError } from '../lib/script-error'; +import { awaitReady, workerArgs, workerReport } from './worker-throughput.bench'; +import { registerNoopJob, workerPod, workerPodArgs } from './worker-throughput-child'; +import { NOOP_JOB, throughputOf, WORKER_PROFILES } from './worker-throughput-shape'; + +afterEach(() => { + resetJobs(); +}); + +const codeOf = (work: () => unknown): string | undefined => { + try { + work(); + } catch (error) { + return error instanceof ScriptError ? error.code : 'not a ScriptError'; + } + return undefined; +}; + +const streamOf = (chunks: readonly string[]) => + new ReadableStream({ + start(controller) { + for (const chunk of chunks) controller.enqueue(new TextEncoder().encode(chunk)); + controller.close(); + }, + }); + +describe('worker throughput bench', () => { + test('one pod claims and acks every queued no-op job', async () => { + const driver = memoryJobDriver(); + registerNoopJob(); + registerNoopJob(); // a second call in one process declares nothing twice + await driver.enqueueMany( + Array.from({ length: 5 }, (_, n) => ({ + name: NOOP_JOB, + queue: 'default', + input: { n }, + idempotencyKey: String(n), + maxAttempts: 1, + runAt: 0, + })), + ); + const worker = workerPod(driver, { concurrency: 8, pollIntervalMs: 1 }); + await worker.tick(); + const stats = await worker.stats(); + expect(stats.processed).toBe(5); + await worker.stop(); + }); + + test('ready is read across chunks; a stream that ends first is a failed run', async () => { + await awaitReady(streamOf(['{"log":1}\nrea', 'dy\nmore\n'])); + const ended = await awaitReady(streamOf(['nothing\n'])).then( + () => 'resolved', + (error: unknown) => (error instanceof ScriptError ? error.code : 'not a ScriptError'), + ); + expect(ended).toBe('X_BENCH_RUN_FAILED'); + }); + + test('flags: a Postgres url is required; a pod needs all three', () => { + expect(codeOf(() => workerArgs([], {}))).toBe('X_CLI_BAD_FLAG'); + expect(codeOf(() => workerArgs(['--url', 'postgres://x/y', '--workers', '0'], {}))).toBe( + 'X_CLI_BAD_FLAG', + ); + expect(workerArgs([], { TEST_DATABASE_URL: 'postgres://x/y' })).toMatchObject({ + base: 'postgres://x/y', + counts: [1, 4, 16], + }); + expect(codeOf(() => workerPodArgs(['--url', 'u']))).toBe('X_CLI_BAD_FLAG'); + expect(workerPodArgs(['--url', 'u', '--concurrency', '8', '--poll-ms', '250'])).toEqual({ + url: 'u', + concurrency: 8, + pollIntervalMs: 250, + }); + }); + + test('the report: a line per profile and worker count, or the whole JSON', () => { + const profiles = [ + { + ...WORKER_PROFILES[0], + name: 'shipped' as const, + concurrency: 8, + pollIntervalMs: 250, + jobsPerWorker: 400, + runs: [throughputOf(4, 1_000, 500)], + }, + ]; + expect(workerReport(profiles, false)).toEqual([ + 'shipped, 4 workers: 2000 claims/s (500/worker, 1000 jobs in 500 ms)', + ]); + expect(JSON.parse(workerReport(profiles, true)[0] ?? '')).toMatchObject({ + profiles: [{ name: 'shipped' }], + }); + }); +}); diff --git a/scripts/bench/worker-throughput.bench.ts b/scripts/bench/worker-throughput.bench.ts index 31bdc453a..64529a8a2 100644 --- a/scripts/bench/worker-throughput.bench.ts +++ b/scripts/bench/worker-throughput.bench.ts @@ -43,6 +43,23 @@ const dbUrl = (base: string, name: string): string => { return parsed.toString(); }; +/** + * Resolve once `stdout` printed `ready`, then keep draining it: a worker logs a line per job, and a + * full pipe would stall it mid-run — the bench would then be timing its own back-pressure. + */ +export async function awaitReady(stdout: ReadableStream): Promise { + const reader = stdout.getReader(); + let seen = ''; + while (!seen.includes('ready')) { + const { value, done } = await reader.read(); + if (done) throw failed('a worker process exited before it printed ready'); + seen += new TextDecoder().decode(value); + } + void (async () => { + while (!(await reader.read()).done) {} + })(); +} + /** Spawn `count` worker processes and resolve once every one has printed `ready`. */ async function spawnWorkers( count: number, @@ -66,20 +83,7 @@ async function spawnWorkers( }), ); await Promise.all( - children.map(async (child) => { - const reader = (child.stdout as ReadableStream).getReader(); - let seen = ''; - while (!seen.includes('ready')) { - const { value, done } = await reader.read(); - if (done) throw failed('a worker process exited before it printed ready'); - seen += new TextDecoder().decode(value); - } - // Keep draining: a worker logs a line per job, and a full pipe would stall it mid-run — the - // bench would then be timing its own back-pressure. - void (async () => { - while (!(await reader.read()).done) {} - })(); - }), + children.map((child) => awaitReady(child.stdout as ReadableStream)), ); return children; } @@ -167,9 +171,13 @@ export async function benchWorkers( } } -if (import.meta.main) { - const args = parseScriptArgs(Bun.argv.slice(2)); - const base = args.flags.get('url') ?? Bun.env['TEST_DATABASE_URL']; +/** The Postgres url and worker counts, or the refusal naming them. */ +export function workerArgs( + argv: readonly string[], + env: Readonly>, +) { + const args = parseScriptArgs(argv); + const base = args.flags.get('url') ?? env['TEST_DATABASE_URL']; const counts = parseWorkerCounts(args.flags.get('workers')); if (typeof base !== 'string' || counts === undefined) { throw new ScriptError({ @@ -179,18 +187,25 @@ if (import.meta.main) { fix: 'bun run scripts/bench/worker-throughput.bench.ts --url postgres://ultimate:ultimate@127.0.0.1:5432/postgres --workers 1,4,16', }); } - const profiles = await benchWorkers(base, counts); - const data = { box: boxFacts(Bun.env), profiles }; const out = args.flags.get('out'); - if (typeof out === 'string') await Bun.write(out, `${JSON.stringify(data, null, 2)}\n`); - if (args.json) console.log(JSON.stringify(data, null, 2)); - else { - for (const profile of profiles) { - for (const run of profile.runs) { - console.log( - `${profile.name}, ${run.workers} workers: ${run.claimsPerSecond} claims/s (${run.perWorker}/worker, ${run.jobs} jobs in ${run.elapsedMs} ms)`, - ); - } - } - } + return { base, counts, out: typeof out === 'string' ? out : undefined, json: args.json }; +} + +/** What a run prints: the whole JSON, or one line per profile and worker count. */ +export function workerReport(profiles: readonly ProfileRuns[], json: boolean): readonly string[] { + if (json) return [JSON.stringify({ box: boxFacts(Bun.env), profiles }, null, 2)]; + return profiles.flatMap((profile) => + profile.runs.map( + (run) => + `${profile.name}, ${run.workers} workers: ${run.claimsPerSecond} claims/s (${run.perWorker}/worker, ${run.jobs} jobs in ${run.elapsedMs} ms)`, + ), + ); +} + +if (import.meta.main) { + const args = workerArgs(Bun.argv.slice(2), Bun.env); + const profiles = await benchWorkers(args.base, args.counts); + const data = { box: boxFacts(Bun.env), profiles }; + if (args.out !== undefined) await Bun.write(args.out, `${JSON.stringify(data, null, 2)}\n`); + for (const line of workerReport(profiles, args.json)) console.log(line); }