ci: qualify test files for the parallel bucket per file, not per directory

scripts/update-parallel-allowlist.mjs

@@ -21,7 +21,6 @@ const { values: opts } = parseArgs({
 });
 const FAST_MS = parseInt(opts["fast-ms"], 10);
 const WANT_BUILDS = parseInt(opts.builds, 10);
-const DIR_MIN_FRACTION = 2 / 3;
 
 const token = process.env.BUILDKITE_API_TOKEN || process.env.BUILDKITE_TOKEN;
 if (!token) {
@@ -126,13 +125,31 @@ console.error(`scanning annotations from ${builds.length} builds (#${Math.min(..
 const flakeCounts = await collectFlakes(builds);
 console.error(`${flakeCounts.size} distinct files flaked or failed in that window`);
 
+// prestart-map.mjs is a hint list for the shard-level service prestart, not a
+// registry of every test that talks to a container, so also treat any file that
+// calls describeWithContainer( as a docker-service test.
 const dockerPrefixes = Object.keys(dockerPrestartMap);
+const usesContainer = file => /describeWithContainer\s*\(/.test(readFileSync(join(testDir, file), "utf8"));
 const files = listBunTestFiles();
+// Fast on every lane we have timings for: a file that is quick on Linux but
+// slow elsewhere would otherwise hold a bucket slot for its whole run.
+const slowest = file => {
+  const entry = durations[file];
+  if (!entry) return undefined;
+  const known = Object.values(entry).filter(ms => typeof ms === "number");
+  return known.length ? Math.max(...known) : undefined;
+};
+// bun install / link / global tests share the user-level bin and cache dirs;
+// run together they race on linking (EEXIST) even though each passes alone.
+const sharedStatePrefixes = ["cli/install/"];
+const sharedStateExempt = ["cli/install/hosted-git-info/", "cli/install/migration/"];
 const isGood = file => {
-  if (dockerPrefixes.some(prefix => file.startsWith(prefix))) return false;
+  if (dockerPrefixes.some(prefix => file.startsWith(prefix)) || usesContainer(file)) return false;
   if (/stress/i.test(file)) return false;
+  if (sharedStatePrefixes.some(p => file.startsWith(p)) && !sharedStateExempt.some(p => file.startsWith(p)))
+    return false;
   if (flakeCounts.has(file)) return false;
-  const ms = durations[file]?.default;
+  const ms = slowest(file);
   return ms === undefined || ms <= FAST_MS;
 };
 
@@ -144,12 +161,15 @@ for (const file of files) {
   byDir.get(dir).push(file);
 }
 
+// Per file: a directory is listed as soon as one of its files qualifies, and
+// the rest of that directory goes to excludeFiles. (The runner reads dirs +
+// excludeFiles, so the shape is unchanged.)
 const dirs = [];
 const excludeFiles = [];
 let eligibleFiles = 0;
 for (const [dir, dirFiles] of [...byDir.entries()].sort()) {
   const good = dirFiles.filter(isGood);
-  if (good.length < Math.max(1, Math.ceil(dirFiles.length * DIR_MIN_FRACTION))) continue;
+  if (good.length === 0) continue;
   dirs.push(dir);
   eligibleFiles += good.length;
   for (const f of dirFiles) if (!isGood(f)) excludeFiles.push(f);
@@ -162,7 +182,7 @@ const out = {
     builds_scanned: builds.length,
     build_range: [Math.min(...builds), Math.max(...builds)],
     fast_ms: FAST_MS,
-    rule: `a directory is listed when >= ${Math.round(DIR_MIN_FRACTION * 100)}% of its bun test files are <= ${FAST_MS}ms (median, default lane of expected-durations.json) with zero flaky/failed annotations in the scanned builds; its other files are listed in excludeFiles; docker-service and stress-named tests never qualify`,
+    rule: `a bun test file qualifies when its slowest lane median in expected-durations.json is <= ${FAST_MS}ms (or it has no entry) and it had zero flaky/failed annotations in the scanned builds; docker-service (prestart-map prefixes or describeWithContainer callers), stress-named and cli/install tests (shared bin/cache dirs; hosted-git-info and migration exempt) never qualify. Every directory with a qualifying file is listed and its other files go in excludeFiles`,
     stats: { dirs: dirs.length, files: eligibleFiles, excluded: excludeFiles.length },
   },
   dirs,

test/napi/node-napi-tests/harness.ts

@@ -149,6 +149,22 @@ async function tryBuildFast(dir: string): Promise<boolean> {
   return results.every(Boolean);
 }
 
+// `bun x node-gyp@11` installs node-gyp into the shared bunx cache on first use and
+// links its bin dependencies (nopt, which, glob, semver) into the shared bin dir.
+// Several addon builds doing that at once race on those links (EEXIST) and fail,
+// so callers that build concurrently warm the cache once, serially, first.
+export async function warmNodeGyp() {
+  const child = spawn({
+    cmd: [bunExe(), "--bun", "x", "node-gyp@11", "--version"],
+    stderr: "pipe",
+    stdout: "ignore",
+    stdin: "ignore",
+    env: bunEnv,
+  });
+  const [stderr, exitCode] = await Promise.all([new Response(child.stderr).text(), child.exited]);
+  if (exitCode !== 0) console.warn(`warming node-gyp failed (builds will install it themselves):\n${stderr}`);
+}
+
 export async function buildWithNodeGyp(dir: string) {
   const child = spawn({
     // `configure build` instead of `rebuild`: `clean` is pure waste (CI checkouts are always

test/napi/node-napi-tests/prebuild.ts

@@ -3,7 +3,7 @@ import { availableParallelism } from "node:os";
 import { existsSync } from "node:fs";
 import { copyFile, mkdir, writeFile } from "node:fs/promises";
 import { basename, dirname, join, relative, resolve } from "node:path";
-import { build, buildWithNodeGyp, isBuilt, parseBindingGyp } from "./harness";
+import { build, buildWithNodeGyp, isBuilt, parseBindingGyp, warmNodeGyp } from "./harness";
 
 const dirs = [...new Set(process.argv.slice(2).map(dir => resolve(dir)))];
 if (dirs.length === 0) {
@@ -22,6 +22,7 @@ const failed: string[] = [];
 
 if (pending.length) {
   if (process.platform !== "win32") {
+    await warmNodeGyp();
     const width = Math.max(2, Math.min(8, availableParallelism()));
     let next = 0;
     const worker = async () => {