bun test: --timings for duration-balanced --shard/--parallel, docs for --parallel & --isolate

bench/test/.gitignore

@@ -1,2 +1,6 @@
 suite/
 node_modules/
+app/src/
+app/tests/
+app/preload.ts
+app/*.json

bench/test/README.md

@@ -1,36 +1,47 @@
 # `bun test --parallel` and `--isolate` benchmarks
 
-Two synthetic suites that show where each flag pays off. The fixture files use
-global `test`/`expect` (no `bun:test` import) so the same suite runs under
-vitest for comparison.
+## `app/`: bun vs jest vs vitest on an ordinary suite
+
+`app/setup.ts` generates a small app — `src/` modules built on `zod`,
+`date-fns` and `lodash` — and N TypeScript test files that import it and run
+8 tests each, plus a `preload.ts` setup file (custom matcher + hooks) wired
+into each runner (`bunfig.toml`, `setupFilesAfterEnv`, `setupFiles`). The test
+files use global `describe`/`test`/`expect`, so the identical suite runs under
+all three runners with their stock configs (`jest.config.cjs` uses `@swc/jest`;
+`vitest.config.ts` sets `globals: true`).
 
 ```sh
 cd bench/test
-bun install                       # for vitest
-bun parallel/setup.ts
-bun isolate-cache/setup.ts
+bun install
+bun app/setup.ts 2000 20         # files, items-per-test
+cd app                           # bunfig.toml / jest.config.cjs / vitest.config.ts here wire in preload.ts
+hyperfine --warmup 1 --runs 3 -N \
+  -n 'bun test'                         'bun test tests' \
+  -n 'bun test --parallel'              'bun test --parallel tests' \
+  -n 'bun test --parallel --no-isolate' 'bun test --parallel --no-isolate tests' \
+  -n 'jest'                             '../node_modules/.bin/jest' \
+  -n 'vitest run'                       '../node_modules/.bin/vitest run' \
+  -n 'vitest run --no-isolate'          '../node_modules/.bin/vitest run --no-isolate'
 ```
 
-## `--parallel`: many independent files
+## `parallel/`: many independent slow files
 
 ```sh
-hyperfine --warmup 1 \
-  'bun test ./parallel/suite' \
-  'bun test --parallel ./parallel/suite' \
-  'npx vitest run parallel/suite'
+bun parallel/setup.ts
+hyperfine --warmup 1 'bun test ./parallel/suite' 'bun test --parallel ./parallel/suite' 'npx vitest run parallel/suite'
 ```
 
 32 files × 4 tests each, ~100ms per file. Serial walks them one at a time;
-`--parallel` spreads them across CPU-count workers (lazily — the first slow
-file triggers scale-up). Vitest runs files in worker threads by default.
+`--parallel` spreads them across CPU-count workers.
 
-## `--isolate`: parse-heavy shared dependency
+## `isolate/`: parse-heavy shared dependency
 
 ```sh
+bun isolate/setup.ts
 hyperfine --warmup 1 \
-  -n 'bun --isolate (cache off)' 'BUN_FEATURE_FLAG_DISABLE_ISOLATION_SOURCE_CACHE=1 bun test --isolate ./isolate-cache/suite' \
-  -n 'bun --isolate (cache on)'  'bun test --isolate ./isolate-cache/suite' \
-  -n 'vitest'                    'npx vitest run isolate-cache/suite'
+  -n 'bun --isolate (cache off)' 'BUN_FEATURE_FLAG_DISABLE_ISOLATION_SOURCE_CACHE=1 bun test --isolate ./isolate/suite' \
+  -n 'bun --isolate (cache on)'  'bun test --isolate ./isolate/suite' \
+  -n 'vitest'                    'npx vitest run isolate/suite'
 ```
 
 30 test files all import a single ~2MB function. Under `--isolate` each file

bench/test/app/bunfig.toml

@@ -0,0 +1,2 @@
+[test]
+preload = ["./preload.ts"]

bench/test/app/jest.config.cjs

@@ -0,0 +1,7 @@
+/** @type {import('jest').Config} */
+module.exports = {
+  testEnvironment: "node",
+  roots: ["<rootDir>/tests"],
+  transform: { "^.+\\.tsx?$": ["@swc/jest"] },
+  setupFilesAfterEnv: ["<rootDir>/preload.ts"],
+};

bench/test/app/setup.ts

@@ -0,0 +1,114 @@
+// Generates a small "app": src/ modules that pull in zod, date-fns and lodash,
+// plus N test files that each import the app barrel and run a handful of
+// cheap assertions. Per-file cost is dominated by loading the module graph,
+// which is the common shape for unit-test suites. The test files use global
+// describe/test/expect so the identical suite runs under bun, vitest and jest.
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+
+const FILES = Number(process.argv[2] ?? 64);
+const TESTS_PER_FILE = 8;
+const ITEMS = Number(process.argv[3] ?? 200);
+for (const [name, v, lo, hi] of [
+  ["files", FILES, 1, 10_000],
+  ["items", ITEMS, 2, 1_000_000],
+] as const) {
+  if (!Number.isSafeInteger(v) || v < lo || v > hi)
+    throw new Error(`${name} must be an integer in [${lo}, ${hi}], got ${v}`);
+}
+const MODULES = 24;
+const root = import.meta.dir;
+
+rmSync(root + "/src", { recursive: true, force: true });
+rmSync(root + "/tests", { recursive: true, force: true });
+rmSync(root + "/preload.ts", { force: true });
+mkdirSync(root + "/src", { recursive: true });
+mkdirSync(root + "/tests", { recursive: true });
+
+for (let m = 0; m < MODULES; m++) {
+  writeFileSync(
+    `${root}/src/inventory${m}.ts`,
+    `import { z } from "zod";
+import { addDays, differenceInCalendarDays, parseISO } from "date-fns";
+import groupBy from "lodash/groupBy";
+import sortBy from "lodash/sortBy";
+
+export const Item${m} = z.object({
+  id: z.string().uuid(),
+  name: z.string().min(1).max(64),
+  tags: z.array(z.string()).default([]),
+  price: z.number().nonnegative(),
+  createdAt: z.string().datetime(),
+});
+export type Item${m} = z.infer<typeof Item${m}>;
+
+export function make${m}(i: number): Item${m} {
+  return Item${m}.parse({
+    id: "00000000-0000-4000-8000-" + String(i).padStart(12, "0"),
+    name: "item-" + i,
+    tags: i % 2 ? ["odd", "m${m}"] : ["even"],
+    price: i * 1.5,
+    createdAt: addDays(parseISO("2024-01-01T00:00:00Z"), i).toISOString(),
+  });
+}
+
+export function summarize${m}(items: Item${m}[]) {
+  const byTag = groupBy(items, it => it.tags[0]);
+  const sorted = sortBy(items, it => -it.price);
+  const spanDays = items.length
+    ? differenceInCalendarDays(parseISO(items[items.length - 1].createdAt), parseISO(items[0].createdAt))
+    : 0;
+  return { tags: Object.keys(byTag).sort(), top: sorted[0]?.name ?? null, spanDays };
+}
+`,
+  );
+}
+
+let barrel = "";
+for (let m = 0; m < MODULES; m++) barrel += `export * from "./inventory${m}";\n`;
+writeFileSync(`${root}/src/index.ts`, barrel);
+
+for (let f = 0; f < FILES; f++) {
+  const m = f % MODULES;
+  let body = `import { Item${m}, make${m}, summarize${m} } from "../src";\n\n`;
+  body += `describe("inventory${m} (file ${f})", () => {\n`;
+  for (let t = 0; t < TESTS_PER_FILE; t++) {
+    body += `  test("case ${t}", () => {
+    const items = Array.from({ length: ${ITEMS + t} }, (_, i) => make${m}(i));
+    const s = summarize${m}(items);
+    expect(s.top).toBe("item-${ITEMS - 1 + t}");
+    expect(s.spanDays).toBe(${ITEMS - 1 + t});
+    expect(s.tags).toEqual(["even", "odd"]);
+    expect(Item${m}.safeParse({ id: "nope" }).success).toBe(false);
+    (expect(items[0]) as any).toBeItemNamed("item-0");
+    expect((globalThis as any).__fixture).toBeDefined();
+  });\n`;
+  }
+  body += `});\n`;
+  writeFileSync(`${root}/tests/inventory${String(f).padStart(4, "0")}.test.ts`, body);
+}
+
+// A setup file every runner loads before each test file (bun --preload,
+// jest setupFilesAfterEnv, vitest setupFiles): a custom matcher + a global hook.
+writeFileSync(
+  `${root}/preload.ts`,
+  `import { make0 } from "./src";
+
+expect.extend({
+  toBeItemNamed(received: unknown, name: string) {
+    const pass = typeof received === "object" && received !== null && (received as any).name === name;
+    return { pass, message: () => \`expected \${JSON.stringify(received)} to be an item named \${name}\` };
+  },
+});
+
+let made = 0;
+beforeEach(() => {
+  made++;
+  (globalThis as any).__fixture = make0(made);
+});
+afterEach(() => {
+  (globalThis as any).__fixture = undefined;
+});
+`,
+);
+
+console.log(`wrote ${MODULES} src modules, preload.ts and ${FILES} test files × ${TESTS_PER_FILE} tests to ${root}`);

bench/test/app/vitest.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: "node",
+    include: ["tests/**/*.test.ts"],
+    setupFiles: ["./preload.ts"],
+  },
+});