ci: measure async GC cost on Linux (diagnostic, do not merge)

.github/workflows/openclaw-async-gc-proof.yml

@@ -0,0 +1,62 @@
+name: Async GC Linux proof (diagnostic only)
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/openclaw-async-gc-proof.yml
+      - bench/gc-async-ci-proof.py
+      - bench/gc-async-stack.mjs
+      - bench/gc-async-calibrated.mjs
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  linux-proof:
+    if: github.repository == 'openclaw/bun'
+    runs-on: blacksmith-16vcpu-ubuntu-2404
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: 3f65cff0d2c2fbdae7921634fa0724c633c8b6bd
+          path: source
+          persist-credentials: false
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '26.12.1'
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: build-linux-x64
+          run-id: 40153838318
+          github-token: ${{ github.token }}
+          path: artifacts/baseline
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: build-linux-x64
+          run-id: 40175945777
+          github-token: ${{ github.token }}
+          path: artifacts/candidate
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: build-linux-x64
+          run-id: 40171805209
+          github-token: ${{ github.token }}
+          path: artifacts/control
+      - name: Measure pinned portable binaries and run unchanged tests
+        run: python3 bench/gc-async-ci-proof.py
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: always()
+        with:
+          name: async-gc-linux-proof
+          path: |
+            proof/results/**
+            proof/surrounding/**
+            proof/*.json
+            proof/*.log
+          if-no-files-found: error
+          retention-days: 14

bench/gc-async-calibrated.mjs

@@ -0,0 +1,146 @@
+// node calibrated-benchmark.mjs <baseline-bun> <candidate-bun> <output-directory> <noop-bun>
+// Same frozen workloads; all three binaries share each interleaved round.
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { arch, cpus, platform } from "node:os";
+import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { Worker } from "node:worker_threads";
+
+const script = fileURLToPath(import.meta.url);
+const cpuMs = usage => (usage.user + usage.system) / 1000;
+
+async function measure(name) {
+  assert(process.versions.bun, "Run benchmark cases with the selected Bun executable");
+  if (name === "gc-loop") {
+    for (let i = 0; i < 2000; i++) Bun.gc(false);
+    Bun.gc(true);
+    const cpu = process.cpuUsage();
+    const start = performance.now();
+    for (let i = 0; i < 200000; i++) Bun.gc(false);
+    return { iterations: 200000, elapsedMs: performance.now() - start, cpuMs: cpuMs(process.cpuUsage(cpu)) };
+  }
+
+  assert.equal(name, "idle-worker");
+  const worker = new Worker(
+    `const { parentPort } = require("node:worker_threads");
+     async function turn(i) {
+       globalThis.payload = new Array(524288).fill(i);
+       // The controller can throttle after quiet ticks; request collection before every park.
+       Bun.gc(false);
+       await new Promise(resolve => setTimeout(resolve, 0));
+     }
+     (async () => {
+       for (let i = 0; i < 25; i++) await turn(i);
+       Bun.gc(true);
+       parentPort.postMessage("ready");
+       parentPort.once("message", async () => {
+         const start = performance.now();
+         for (let i = 0; i < 500; i++) await turn(i);
+         parentPort.postMessage({ iterations: 500, elapsedMs: performance.now() - start,
+           checksum: payload[0] + payload.length });
+         parentPort.close();
+       });
+     })().catch(error => { throw error; });`,
+    { eval: true },
+  );
+  let measured;
+  await new Promise((resolve, reject) => {
+    let cpu;
+    worker.on("error", reject);
+    worker.on("message", message => {
+      if (message === "ready") {
+        cpu = process.cpuUsage();
+        worker.postMessage("go");
+      } else {
+        measured = { ...message, cpuMs: cpuMs(process.cpuUsage(cpu)) };
+      }
+    });
+    worker.on("exit", code => {
+      if (code !== 0 || !measured) reject(new Error(`Worker exited with ${code} before completing the workload`));
+      else resolve();
+    });
+  });
+  assert.equal(measured.checksum, 524288 + 499);
+  return measured;
+}
+
+if (process.argv[2] === "--case") {
+  console.log(JSON.stringify(await measure(process.argv[3])));
+} else {
+  const [baselineArg, candidateArg, outputArg, noopArg] = process.argv.slice(2);
+  assert(baselineArg && candidateArg && outputArg && noopArg, "Expected baseline, candidate, output directory, and no-op control");
+  const output = resolve(outputArg);
+  assert(!existsSync(output), "Use a fresh output directory to preserve every comparison");
+  mkdirSync(output, { recursive: true });
+  const baseEnv = { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: "0", NO_COLOR: "1" };
+  for (const key of ["NODE_OPTIONS", "BUN_OPTIONS", "BUN_DIAG_ASYNC_GC_CLEAR", "BUN_GC_TIMER_DISABLE",
+    "BUN_GC_TIMER_INTERVAL", "BUN_IDLE_GC_SECONDS", "BUN_GARBAGE_COLLECTOR_LEVEL", "BUN_TEST_DRAIN_EVENT_LOOP"])
+    delete baseEnv[key];
+  const binaries = { baseline: resolve(baselineArg), candidate: resolve(candidateArg), noop: resolve(noopArg) };
+  const identities = Object.fromEntries(Object.entries(binaries).map(([label, exe]) => {
+    const child = spawnSync(exe, ["-p", "JSON.stringify({revision:Bun.revision,webkit:process.versions.webkit,arch:process.arch})"],
+      { env: baseEnv, encoding: "utf8" });
+    assert.equal(child.status, 0, child.stderr);
+    return [label, { ...JSON.parse(child.stdout), sha256: createHash("sha256").update(readFileSync(exe)).digest("hex") }];
+  }));
+  assert.notEqual(identities.baseline.sha256, identities.candidate.sha256, "Select two distinct executables");
+  assert.equal(identities.baseline.arch, arch());
+  assert.equal(identities.candidate.arch, arch());
+  assert.equal(identities.noop.arch, arch());
+  assert.equal(new Set(Object.values(identities).map(i => i.sha256)).size, 3);
+  const result = { platform: platform(), arch: arch(), cpu: cpus()[0].model, identities, trials: [], summaries: {} };
+  const save = () => writeFileSync(join(output, "results.json"), JSON.stringify(result, null, 2) + "\n");
+  save();
+  const median = values => {
+    const sorted = [...values].sort((a, b) => a - b);
+    return (sorted[4] + sorted[5]) / 2;
+  };
+  for (const name of ["gc-loop", "idle-worker"]) {
+    for (let round = 1; round <= 10; round++) {
+      const orders = [["baseline", "candidate", "noop"], ["noop", "candidate", "baseline"],
+        ["candidate", "noop", "baseline"], ["baseline", "noop", "candidate"],
+        ["noop", "baseline", "candidate"], ["candidate", "baseline", "noop"]];
+      for (const label of orders[(round - 1) % orders.length]) {
+        const dir = join(output, `${name}-${round}-${label}`);
+        mkdirSync(join(dir, "home"), { recursive: true, mode: 0o700 });
+        mkdirSync(join(dir, "tmp"), { mode: 0o700 });
+        const env = { ...baseEnv, HOME: join(dir, "home"), TMPDIR: join(dir, "tmp"),
+          BUN_GC_TIMER_DISABLE: name === "gc-loop" ? "1" : "0", BUN_GC_TIMER_INTERVAL: "1",
+          BUN_IDLE_GC_SECONDS: "1,2", BUN_GC_RUNS_UNTIL_SKIP_RELEASE_ACCESS: "0" };
+        if (name === "idle-worker") env.BUN_JSC_numberOfGCMarkers = "2";
+        const child = spawnSync(binaries[label], [script, "--case", name],
+          { env, encoding: "utf8", timeout: 120000, maxBuffer: 1024 * 1024 });
+        writeFileSync(join(dir, "stdout.log"), child.stdout ?? "");
+        writeFileSync(join(dir, "stderr.log"), child.stderr ?? "");
+        assert.equal(child.status, 0, child.error?.message ?? child.stderr);
+        const measurement = JSON.parse(child.stdout);
+        assert(Number.isFinite(measurement.elapsedMs) && measurement.elapsedMs > 0);
+        assert(Number.isFinite(measurement.cpuMs) && measurement.cpuMs > 0);
+        result.trials.push({ name, round, label, ...measurement });
+        save();
+        console.log(JSON.stringify(result.trials.at(-1)));
+      }
+    }
+    const rows = label => result.trials.filter(t => t.name === name && t.label === label);
+    const summary = {};
+    for (const metric of ["elapsedMs", "cpuMs"]) {
+      const baseline = rows("baseline"), candidate = rows("candidate"), noop = rows("noop");
+      summary[metric] = { baselineMedian: median(baseline.map(t => t[metric])),
+        candidateMedian: median(candidate.map(t => t[metric])),
+        noopMedian: median(noop.map(t => t[metric])),
+        noopPairedMedianRatio: median(noop.map((t, i) => t[metric] / baseline[i][metric])),
+        pairedMedianRatio: median(candidate.map((t, i) => t[metric] / baseline[i][metric])) };
+    }
+    // Idle wall time includes intentional parking; CPU time exposes the collection cost.
+    summary.primaryMetric = name === "idle-worker" ? "cpuMs" : "elapsedMs";
+    summary.limit = Math.max(1.02, summary[summary.primaryMetric].noopPairedMedianRatio + 0.01);
+    summary.passed = summary[summary.primaryMetric].pairedMedianRatio <= summary.limit;
+    result.summaries[name] = summary;
+    save();
+  }
+  console.log(JSON.stringify(result.summaries, null, 2));
+  assert(Object.values(result.summaries).every(summary => summary.passed), "Async GC cost exceeded the predeclared calibrated limit");
+}

bench/gc-async-ci-proof.py

@@ -0,0 +1,87 @@
+"""Temporary Linux-only proof; this diagnostic branch must never be merged."""
+import hashlib
+import json
+import os
+from pathlib import Path
+import signal
+import subprocess
+import time
+import zipfile
+
+checkout = Path.cwd()
+source = checkout / "source"
+root = checkout / "proof"
+root.mkdir()
+(root / "home").mkdir(mode=0o700)
+(root / "tmp").mkdir(mode=0o700)
+head = "3f65cff0d2c2fbdae7921634fa0724c633c8b6bd"
+assert subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=source, text=True).strip() == head
+hashes = {
+    "artifacts/baseline/bun-linux-x64.zip": "5b17f7c554d7669c0a9253f5aa2e56c205a1a84c5ef136433b1a1717083f39ab",
+    "artifacts/candidate/bun-linux-x64.zip": "7215831627f3eb225fa1a1757a23456f686d5a0aa94100b5740812dec7dd3393",
+    "artifacts/control/bun-linux-x64.zip": "85a265a2533d3c960caf1f3ccd73f1a269aeb88e3831ec0b4b04c9c3f924d82f",
+    "source/bench/gc-async-stack.mjs": "09db8a7bee4be019544abc9af6159f739e0866148860dbd59784680ffe77101b",
+    "source/test/js/node/worker_threads/worker_threads.test.ts": "2c246a36b6f09dd1c593450b0d160af991d666b9b4f76ff02dd22fe96e8d73fc",
+}
+for name, expected in hashes.items():
+    assert hashlib.sha256((checkout / name).read_bytes()).hexdigest() == expected, name
+(root / "input-hashes.json").write_text(json.dumps(hashes, indent=2) + "\n")
+env = {**os.environ, "HOME": str(root / "home"), "TMPDIR": str(root / "tmp"), "CI": "true", "NO_COLOR": "1"}
+for key in ["NODE_OPTIONS", "NODE_PATH", "BUN_OPTIONS", "BUN_WEBKIT_PATH", "LD_PRELOAD", "LD_LIBRARY_PATH"]:
+    env.pop(key, None)
+binaries = {}
+identities = {}
+for label, revision, engine in [
+    ("baseline", "6aba42d75d4bcee905087fa518cb2df6ff49bb0d", "419b5a0470fc059a610e98697691bf6f7cedd841"),
+    ("candidate", head, "b82bc917e99110b61c3a21202d6def7457aa194c"),
+    ("control", "6b7c19f0db6b3cadfa02391c553919ad6c1409da", "b82bc917e99110b61c3a21202d6def7457aa194c"),
+]:
+    with zipfile.ZipFile(checkout / "artifacts" / label / "bun-linux-x64.zip") as archive:
+        archive.extractall(root / label)
+    exe = root / label / "bun-linux-x64" / "bun"
+    exe.chmod(0o755)
+    binaries[label] = str(exe)
+    identity = json.loads(subprocess.check_output(
+        [str(exe), "-p", "JSON.stringify({revision:Bun.revision,webkit:process.versions.webkit,arch:process.arch})"], env=env, text=True))
+    assert identity == {"revision": revision, "webkit": engine, "arch": "x64"}
+    identities[label] = {**identity, "sha256": hashlib.sha256(exe.read_bytes()).hexdigest()}
+(root / "identities.json").write_text(json.dumps(identities, indent=2) + "\n")
+with (root / "cost.log").open("w") as log:
+    cost = subprocess.run(["node", str(checkout / "bench/gc-async-calibrated.mjs"), binaries["baseline"], binaries["candidate"], str(root / "results"), binaries["control"]], env=env, stdout=log, stderr=log)
+(root / "cost-exit.json").write_text(json.dumps({"exitCode": cost.returncode}) + "\n")
+print((root / "cost.log").read_text(), flush=True)
+out = root / "surrounding"
+out.mkdir()
+target = "test/js/node/worker_threads/worker_threads.test.ts"
+commands = [("immediate-gc-" + str(i), [target, "-t", "wakes a parked worker for an immediate GC timer"]) for i in range(1, 11)]
+files = []
+for directory in ["test/js/node/worker_threads", "test/js/node/timers", "test/js/node/timers.promises", "test/js/web/timers"]:
+    files += [str(p.relative_to(source)) for p in (source / directory).iterdir() if ".test." in p.name and p.is_file()]
+assert len(files) == 22
+commands += [(name, [name]) for name in sorted(files)]
+rows = []
+for i, (label, args) in enumerate(commands):
+    dest = out / str(i)
+    dest.mkdir()
+    (dest / "home").mkdir(mode=0o700)
+    (dest / "tmp").mkdir(mode=0o700)
+    test_env = {**env, "HOME": str(dest / "home"), "TMPDIR": str(dest / "tmp"),
+                "BUN_RUNTIME_TRANSPILER_CACHE_PATH": "0", "BUN_ENABLE_CRASH_REPORTING": "0",
+                "BUN_FEATURE_FLAG_INTERNAL_FOR_TESTING": "1", "BUN_DEBUG_QUIET_LOGS": "1",
+                "BUN_DISABLE_SLOW_FILESYSTEM_WARNING": "1", "BUN_GARBAGE_COLLECTOR_LEVEL": "1",
+                "BUN_JSC_randomIntegrityAuditRate": "1.0", "GITHUB_ACTIONS": "true"}
+    start = time.monotonic()
+    with (dest / "test.log").open("w") as log:
+        proc = subprocess.Popen([binaries["candidate"], "test", *args], cwd=source, env=test_env, stdout=log, stderr=log, start_new_session=True)
+        try:
+            code = proc.wait(timeout=180)
+        except subprocess.TimeoutExpired:
+            code = "harness_timeout"
+            os.killpg(proc.pid, signal.SIGKILL)
+            proc.wait()
+    row = {"label": label, "args": args, "exitCode": code, "seconds": time.monotonic() - start, "log": str(dest / "test.log")}
+    rows.append(row)
+    (out / "results.json").write_text(json.dumps(rows, indent=2) + "\n")
+    print(json.dumps(row), flush=True)
+assert cost.returncode == 0, "Cost gate failed; retain results without retries"
+assert all(row["exitCode"] == 0 for row in rows), "Surrounding failure; retain logs without retries"