PoC: feat/agent-e2e-verify

src/commands/deployments/deploy.ts

@@ -53,6 +53,9 @@ const EXCLUDE_PATTERNS = [
   '.cache',
   'skills',
   'coverage',
+  'test-results',
+  'playwright-report',
+  '.playwright-mcp',
   IGNORE_FILE_NAME,
 ];
 

src/commands/verify/finding.ts

@@ -0,0 +1,48 @@
+import type { Command } from 'commander';
+import { CLIError, getRootOpts, handleError } from '../../lib/errors.js';
+import { outputJson, outputInfo } from '../../lib/output.js';
+import { shutdownAnalytics, trackVerifyFinding } from '../../lib/analytics.js';
+import { getProjectConfig } from '../../lib/config.js';
+
+// Record a "loud" error the browser surfaced during the drive — a 4xx/5xx, a
+// `column does not exist`, a console exception — that the agent saw via
+// `browser_console_messages` / `browser_network_requests`. The rls/truth probes
+// only cover the *silent* findings; this is how the loud ones reach PostHog too.
+export function registerVerifyFindingCommand(verify: Command): void {
+  verify
+    .command('finding')
+    .description('Record a loud error surfaced during the drive (4xx/5xx, column-not-found, console) as a finding (experimental)')
+    .requiredOption('--kind <kind>', 'short error kind, e.g. pgrst_column_not_found, http_500, console_error')
+    .option('--type <type>', 'finding type', 'error')
+    .option('--status <n>', 'HTTP status, if any', (v) => parseInt(v, 10))
+    .option('--endpoint <path>', 'the endpoint/URL that errored')
+    .option('--message <text>', 'the error message the page showed')
+    .option('--table <name>', 'related table, if known')
+    .action(async (opts, cmd) => {
+      const { json } = getRootOpts(cmd);
+      try {
+        const config = getProjectConfig();
+        if (!config) throw new CLIError('No linked project found — run `insforge link` first.');
+        const finding = {
+          type: opts.type as string,
+          kind: opts.kind as string,
+          status: Number.isNaN(opts.status) ? undefined : (opts.status as number | undefined),
+          endpoint: opts.endpoint as string | undefined,
+          message: opts.message as string | undefined,
+          table: opts.table as string | undefined,
+        };
+        trackVerifyFinding(finding, config);
+        await shutdownAnalytics(); // flush the PostHog event before exit
+
+        if (json) {
+          outputJson({ recorded: true, finding });
+        } else {
+          outputInfo(
+            `📝 recorded ${finding.type} finding: ${finding.kind}${finding.status ? ` (${finding.status})` : ''}${finding.message ? ` — ${finding.message}` : ''}`,
+          );
+        }
+      } catch (e) {
+        handleError(e, json);
+      }
+    });
+}

src/commands/verify/index.ts

@@ -0,0 +1,14 @@
+// src/commands/verify/index.ts
+import type { Command } from 'commander';
+import { registerVerifyRlsCommand } from './rls.js';
+import { registerVerifyTruthCommand } from './truth.js';
+import { registerVerifyFindingCommand } from './finding.js';
+
+export function registerVerifyCommands(program: Command): void {
+  const verify = program
+    .command('verify', { hidden: true })
+    .description('[experimental] Backend-truth & RLS probes + loud-error recording for insforge-verify');
+  registerVerifyRlsCommand(verify);
+  registerVerifyTruthCommand(verify);
+  registerVerifyFindingCommand(verify);
+}

src/commands/verify/rls.test.ts

@@ -0,0 +1,77 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { Command } from 'commander';
+import type * as VerifyProbe from '../../lib/verify-probe.js';
+import { registerVerifyRlsCommand } from './rls.js';
+
+vi.mock('../../lib/config.js', () => ({
+  getProjectConfig: vi.fn(() => ({
+    project_id: 'p1', project_name: 'n', org_id: 'o1', region: 'us-east',
+    api_key: 'key', oss_host: 'https://h',
+  })),
+}));
+vi.mock('../../lib/api/oss.js', () => ({
+  getAnonKey: vi.fn(async () => 'anon'),
+  runRawSql: vi.fn(async () => ({ rows: [{ id: 'aid' }] })),
+}));
+vi.mock('../../lib/analytics.js', () => ({
+  trackVerifyFinding: vi.fn(),
+  shutdownAnalytics: vi.fn(async () => {}),
+}));
+// Keep the pure helpers (classifyRls / isSafeIdentifier / isLikelyEmail) real; mock the
+// two network calls.
+vi.mock('../../lib/verify-probe.js', async (importOriginal) => {
+  const actual = await importOriginal<typeof VerifyProbe>();
+  return { ...actual, login: vi.fn(async () => 'token'), recordsCount: vi.fn(async () => 0) };
+});
+
+function makeProgram() {
+  const program = new Command().exitOverride();
+  program.option('--json');
+  registerVerifyRlsCommand(program.command('verify'));
+  return program;
+}
+
+describe('verify rls (command)', () => {
+  let exitSpy: ReturnType<typeof vi.spyOn>;
+  beforeEach(() => {
+    vi.clearAllMocks();
+    process.exitCode = undefined;
+    exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => {
+      throw new Error(`exit:${code}`);
+    }) as never);
+    vi.spyOn(console, 'error').mockImplementation(() => {});
+    vi.spyOn(console, 'log').mockImplementation(() => {});
+  });
+  afterEach(() => {
+    exitSpy.mockRestore();
+    vi.restoreAllMocks();
+    process.exitCode = undefined;
+  });
+
+  it('rejects an --owner that smuggles PostgREST params, before any login', async () => {
+    const { login } = await import('../../lib/verify-probe.js');
+    await expect(
+      makeProgram().parseAsync(['verify', 'rls', '--table', 'orders', '--owner', 'user_id&select=secret', '--json'], { from: 'user' }),
+    ).rejects.toThrow(/exit:/);
+    expect(login).not.toHaveBeenCalled();
+  });
+
+  it('rejects a non-email --user-a, before any login', async () => {
+    const { login } = await import('../../lib/verify-probe.js');
+    await expect(
+      makeProgram().parseAsync(['verify', 'rls', '--table', 'orders', '--owner', 'user_id', '--user-a', 'not-an-email', '--json'], { from: 'user' }),
+    ).rejects.toThrow(/exit:/);
+    expect(login).not.toHaveBeenCalled();
+  });
+
+  it('scopes the anonymous control to A\'s owner filter (not the whole table)', async () => {
+    const { recordsCount } = await import('../../lib/verify-probe.js');
+    await makeProgram().parseAsync(['verify', 'rls', '--table', 'orders', '--owner', 'user_id', '--json'], { from: 'user' });
+    // 3 probes: B-of-A, A-own, anon — all must use the same owner-scoped filter.
+    expect(recordsCount).toHaveBeenCalledTimes(3);
+    // The anon probe (3rd call) must pass the filter + no token, NOT undefined for the filter.
+    expect(recordsCount).toHaveBeenNthCalledWith(
+      3, 'https://h', 'orders', expect.stringContaining('user_id=eq.'), undefined, 'anon',
+    );
+  });
+});

src/commands/verify/rls.ts

@@ -0,0 +1,87 @@
+import type { Command } from 'commander';
+import { CLIError, getRootOpts, handleError } from '../../lib/errors.js';
+import { getProjectConfig } from '../../lib/config.js';
+import { outputJson, outputInfo } from '../../lib/output.js';
+import { shutdownAnalytics, trackVerifyFinding } from '../../lib/analytics.js';
+import {
+  classifyRls,
+  isLikelyEmail,
+  isSafeIdentifier,
+  login,
+  recordsCount,
+} from '../../lib/verify-probe.js';
+import { getAnonKey, runRawSql } from '../../lib/api/oss.js';
+
+export function registerVerifyRlsCommand(verify: Command): void {
+  verify
+    .command('rls')
+    .description('Cross-user RLS isolation probe — checks B cannot read A, A can read own (experimental)')
+    .requiredOption('--table <name>', 'user-scoped table to probe')
+    .requiredOption('--owner <column>', 'owner column on the table (e.g. user_id)')
+    .option('--user-a <email>', 'seeded user A email', 'verify-a@example.com')
+    .option('--user-b <email>', 'seeded user B email', 'verify-b@example.com')
+    .option('--password <pw>', 'seeded users password', 'Test1234!pass')
+    .action(async (opts, cmd) => {
+      const { json } = getRootOpts(cmd);
+      try {
+        const config = getProjectConfig();
+        if (!config) throw new CLIError('No linked project found — run `insforge link` first.');
+        const baseUrl = config.oss_host;
+
+        // --table/--owner are interpolated into a PostgREST resource path and filter; keep
+        // them to bare identifiers so a value like `user_id&select=secret` can't inject extra
+        // params. --user-a/-b go into a raw SQL lookup; require an email shape (the single-
+        // quote escaping below already blocks string-literal injection — this removes the rest).
+        if (!isSafeIdentifier(String(opts.table))) {
+          throw new CLIError(`--table must be a bare table name (got ${JSON.stringify(opts.table)}).`);
+        }
+        if (!isSafeIdentifier(String(opts.owner))) {
+          throw new CLIError(`--owner must be a bare column name (got ${JSON.stringify(opts.owner)}).`);
+        }
+        if (!isLikelyEmail(String(opts.userA)) || !isLikelyEmail(String(opts.userB))) {
+          throw new CLIError('--user-a and --user-b must be valid email addresses.');
+        }
+
+        const aToken = await login(baseUrl, opts.userA, opts.password);
+        const bToken = await login(baseUrl, opts.userB, opts.password);
+        const anon = await getAnonKey();
+        if (!aToken || !bToken || !anon) {
+          throw new CLIError(
+            'Login or anon-key fetch returned empty — seed BOTH users first. An empty token turns every probe into an anonymous request that silently "passes" isolation.',
+          );
+        }
+
+        const { rows } = await runRawSql(
+          `select id from auth.users where email='${String(opts.userA).replace(/'/g, "''")}'`,
+        );
+        const aId = (rows[0] as { id?: string })?.id;
+        if (!aId) throw new CLIError(`Could not find user A (${opts.userA}) — seed it first.`);
+
+        // All three probes use the SAME owner-scoped filter so we measure "can X read A's
+        // rows", not "can X read any row". Checking the whole table for the anon control would
+        // false-positive a leak on any table that intentionally exposes some public rows.
+        const filter = `${opts.owner}=eq.${encodeURIComponent(aId)}`;
+        const bReadRowsOfA = await recordsCount(baseUrl, opts.table, filter, bToken, anon);
+        const aReadOwnRows = await recordsCount(baseUrl, opts.table, filter, aToken, anon);
+        const anonReadRows = await recordsCount(baseUrl, opts.table, filter, undefined, anon);
+
+        const { type, evidence } = classifyRls({ bReadRowsOfA, aReadOwnRows, anonReadRows });
+        const finding = { type, table: opts.table as string, evidence };
+        trackVerifyFinding(finding, config);
+        await shutdownAnalytics(); // flush the PostHog event before exit
+
+        if (json) {
+          outputJson({ passed: type === 'none', finding });
+        } else if (type === 'rls_leak') {
+          outputInfo(`❌ rls_leak on ${opts.table}: B read ${bReadRowsOfA} of A's rows (anon read ${anonReadRows}).`);
+        } else if (type === 'rls_overrestrict') {
+          outputInfo(`❌ rls_overrestrict on ${opts.table}: A could not read its own rows (positive control empty).`);
+        } else {
+          outputInfo(`✅ isolation holds on ${opts.table}: B=0, anon=0, A=${aReadOwnRows}.`);
+        }
+        process.exitCode = type === 'none' ? 0 : 1;
+      } catch (e) {
+        handleError(e, json);
+      }
+    });
+}

src/commands/verify/truth.test.ts

@@ -0,0 +1,81 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { Command } from 'commander';
+import { registerVerifyTruthCommand } from './truth.js';
+
+vi.mock('../../lib/config.js', () => ({
+  getProjectConfig: vi.fn(() => ({
+    project_id: 'p1', project_name: 'n', org_id: 'o1', region: 'us-east',
+    api_key: 'key', oss_host: 'https://h',
+  })),
+}));
+vi.mock('../../lib/api/oss.js', () => ({ runRawSql: vi.fn() }));
+vi.mock('../../lib/analytics.js', () => ({
+  trackVerifyFinding: vi.fn(),
+  shutdownAnalytics: vi.fn(async () => {}),
+}));
+
+function makeProgram() {
+  const program = new Command().exitOverride();
+  program.option('--json');
+  registerVerifyTruthCommand(program.command('verify'));
+  return program;
+}
+
+describe('verify truth (command)', () => {
+  let exitSpy: ReturnType<typeof vi.spyOn>;
+  beforeEach(async () => {
+    vi.clearAllMocks();
+    process.exitCode = undefined;
+    const { runRawSql } = await import('../../lib/api/oss.js');
+    (runRawSql as ReturnType<typeof vi.fn>).mockResolvedValue({ rows: [] });
+    exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => {
+      throw new Error(`exit:${code}`);
+    }) as never);
+    vi.spyOn(console, 'error').mockImplementation(() => {});
+    vi.spyOn(console, 'log').mockImplementation(() => {});
+  });
+  afterEach(() => {
+    exitSpy.mockRestore();
+    vi.restoreAllMocks();
+    process.exitCode = undefined;
+  });
+
+  it('rejects a non-read query before touching the DB', async () => {
+    const { runRawSql } = await import('../../lib/api/oss.js');
+    await expect(
+      makeProgram().parseAsync(['verify', 'truth', '--query', 'delete from t', '--expect', '1', '--json'], { from: 'user' }),
+    ).rejects.toThrow(/exit:/);
+    expect(runRawSql).not.toHaveBeenCalled();
+  });
+
+  it('rejects when both --expect and --expect-count are given', async () => {
+    const { runRawSql } = await import('../../lib/api/oss.js');
+    await expect(
+      makeProgram().parseAsync(['verify', 'truth', '--query', 'select 1', '--expect', '1', '--expect-count', '1', '--json'], { from: 'user' }),
+    ).rejects.toThrow(/exit:/);
+    expect(runRawSql).not.toHaveBeenCalled();
+  });
+
+  it('rejects a non-integer --expect-count', async () => {
+    await expect(
+      makeProgram().parseAsync(['verify', 'truth', '--query', 'select count(*) from t', '--expect-count', 'abc', '--json'], { from: 'user' }),
+    ).rejects.toThrow(/exit:/);
+  });
+
+  it('passes (exit 0) + records & flushes a finding when DB matches the claim', async () => {
+    const oss = await import('../../lib/api/oss.js');
+    (oss.runRawSql as ReturnType<typeof vi.fn>).mockResolvedValue({ rows: [{ n: 3 }] });
+    await makeProgram().parseAsync(['verify', 'truth', '--query', 'select n', '--expect', '3', '--json'], { from: 'user' });
+    expect(process.exitCode).toBe(0);
+    const { trackVerifyFinding, shutdownAnalytics } = await import('../../lib/analytics.js');
+    expect(trackVerifyFinding).toHaveBeenCalledTimes(1);
+    expect(shutdownAnalytics).toHaveBeenCalled();
+  });
+
+  it('flags false_pass (exit 1) when DB differs from the claim', async () => {
+    const oss = await import('../../lib/api/oss.js');
+    (oss.runRawSql as ReturnType<typeof vi.fn>).mockResolvedValue({ rows: [{ n: 1 }] });
+    await makeProgram().parseAsync(['verify', 'truth', '--query', 'select n', '--expect', '3', '--json'], { from: 'user' });
+    expect(process.exitCode).toBe(1);
+  });
+});

src/commands/verify/truth.ts

@@ -0,0 +1,68 @@
+import type { Command } from 'commander';
+import { CLIError, getRootOpts, handleError } from '../../lib/errors.js';
+import { getProjectConfig } from '../../lib/config.js';
+import { outputJson, outputInfo } from '../../lib/output.js';
+import { shutdownAnalytics, trackVerifyFinding } from '../../lib/analytics.js';
+import { classifyTruth, isReadOnlyQuery } from '../../lib/verify-probe.js';
+import { runRawSql } from '../../lib/api/oss.js';
+
+export function registerVerifyTruthCommand(verify: Command): void {
+  verify
+    .command('truth')
+    .description('Backend-truth cross-check — compare a DB read to what the UI claimed (experimental)')
+    .requiredOption('--query <sql>', 'a read proving what the UI showed; compares the first column of the first row')
+    .option('--expect <value>', 'the value the UI displayed (compared as a scalar)')
+    .option('--expect-count <n>', 'expect this many rows instead of a scalar value')
+    .option('--table <name>', 'table name, for the finding label')
+    .action(async (opts, cmd) => {
+      const { json } = getRootOpts(cmd);
+      try {
+        const config = getProjectConfig();
+        if (!config) throw new CLIError('No linked project found — run `insforge link` first.');
+        if (!isReadOnlyQuery(opts.query)) {
+          throw new CLIError(
+            'verify truth expects a single read query — it must start with SELECT or WITH and not chain statements. (This guard blocks common destructive forms, not a hard read-only guarantee — pass a plain read.)',
+          );
+        }
+        if (opts.expect !== undefined && opts.expectCount !== undefined) {
+          throw new CLIError('Provide either --expect <value> or --expect-count <n>, not both.');
+        }
+
+        const { rows } = await runRawSql(opts.query);
+
+        let result: { type: 'false_pass' | 'none'; evidence: Record<string, unknown> };
+        if (opts.expectCount !== undefined) {
+          // Compare as a number so `--expect-count 03` matches 3 rows (string compare wouldn't).
+          const expected = Number(opts.expectCount);
+          if (!Number.isInteger(expected) || expected < 0) {
+            throw new CLIError(`--expect-count must be a non-negative integer (got ${JSON.stringify(opts.expectCount)}).`);
+          }
+          result = classifyTruth(rows.length, String(expected));
+        } else if (opts.expect !== undefined) {
+          const first = rows[0];
+          const dbValue =
+            first && typeof first === 'object' ? Object.values(first as Record<string, unknown>)[0] : first;
+          result = classifyTruth(dbValue, String(opts.expect));
+        } else {
+          throw new CLIError('Provide --expect <value> (scalar) or --expect-count <n> (row count).');
+        }
+
+        const finding = { type: result.type, table: opts.table as string | undefined, evidence: result.evidence };
+        trackVerifyFinding(finding, config);
+        await shutdownAnalytics(); // flush the PostHog event before exit
+
+        if (json) {
+          outputJson({ passed: result.type === 'none', finding });
+        } else if (result.type === 'false_pass') {
+          outputInfo(
+            `❌ false_pass${opts.table ? ` on ${opts.table}` : ''}: UI claimed ${JSON.stringify(result.evidence.ui_claimed)} but DB has ${JSON.stringify(result.evidence.db_actual)}.`,
+          );
+        } else {
+          outputInfo(`✅ backend truth matches: ${JSON.stringify(result.evidence.db_actual)}.`);
+        }
+        process.exitCode = result.type === 'none' ? 0 : 1;
+      } catch (e) {
+        handleError(e, json);
+      }
+    });
+}