Skip to content

ci(parallel-allowlist): carry forward previous excludeFiles on regen #36868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
robobun wants to merge 6 commits into from
Closed

ci(parallel-allowlist): carry forward previous excludeFiles on regen #36868

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
5e3c4ad
ci(parallel-allowlist): carry forward previous excludeFiles on regen
robobun Aug 4, 2026
1515e8a
review: fail loud when previous allowlist is unreadable; tighten dock…
robobun Aug 4, 2026
25d2902
review: realpath argv[1] so the main-module guard survives a symlinke…
robobun Aug 4, 2026
c269d13
review: make the no-CLI-on-import test assert via a spawned subprocess
robobun Aug 4, 2026
68c20b1
review: drain stdout in the no-CLI-on-import subprocess test
robobun Aug 4, 2026
7911386
review: compute dir threshold over fresh files so carried excludes do…
robobun Aug 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
307 changes: 176 additions & 131 deletions scripts/update-parallel-allowlist.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,49 +1,63 @@
#!/usr/bin/env node

import { readdirSync, readFileSync, writeFileSync } from "node:fs";
import { readdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { parseArgs } from "node:util";
import { prestartMap as dockerPrestartMap } from "../test/docker/prestart-map.mjs";

const __dirname = dirname(fileURLToPath(import.meta.url));
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const testDir = join(__dirname, "..", "test");
const outputPath = join(testDir, "parallel-allowlist.json");
const durationsPath = join(testDir, "expected-durations.json");

const { values: opts } = parseArgs({
options: {
builds: { type: "string", default: "300" },
"fast-ms": { type: "string", default: "15000" },
org: { type: "string", default: "bun" },
pipeline: { type: "string", default: "bun" },
},
});
const FAST_MS = parseInt(opts["fast-ms"], 10);
const WANT_BUILDS = parseInt(opts.builds, 10);
const DIR_MIN_FRACTION = 2 / 3;

const token = process.env.BUILDKITE_API_TOKEN || process.env.BUILDKITE_TOKEN;
if (!token) {
console.error("BUILDKITE_API_TOKEN is required");
process.exit(1);
}
// Pure: given the scan inputs, compute the {dirs, excludeFiles} table.
// Exported for test/internal/parallel-allowlist.test.ts so the carry-forward
// rule below is covered without hitting the Buildkite API.
export function computeAllowlist({ files, flaky, durations, previousExcludes, dockerPrefixes, fastMs }) {
const isGood = file => {
if (dockerPrefixes.some(prefix => file.startsWith(prefix))) return false;
if (/stress/i.test(file)) return false;
// A file already in excludeFiles never runs inside the parallel batch, so
// the annotation scan cannot observe it flaking there; "zero flakes" is
// not evidence it became batch-safe. Carry the exclusion forward. To drop
// one, remove it from parallel-allowlist.json by hand; the next regen then
// has real batch data for it and re-excludes only if it flakes again.
if (previousExcludes.has(file)) return false;
if (flaky.has(file)) return false;
const ms = durations[file]?.default;
return ms === undefined || ms <= fastMs;
};

const api = async path => {
for (let attempt = 0; ; attempt++) {
const r = await fetch(`https://api.buildkite.com/v2/organizations/${opts.org}/pipelines/${opts.pipeline}/${path}`, {
headers: { Authorization: `Bearer ${token}` },
signal: AbortSignal.timeout(60_000),
});
if (r.ok) return r;
if ((r.status === 429 || r.status >= 500) && attempt < 8) {
const backoff = Number(r.headers.get("retry-after")) * 1000 || 1000 * 2 ** attempt;
await new Promise(resolve => setTimeout(resolve, backoff));
continue;
}
throw new Error(`${path}: ${r.status} ${r.statusText}`);
const byDir = new Map();
for (const file of files) {
const slash = file.lastIndexOf("/");
const dir = slash === -1 ? "" : file.slice(0, slash);
if (!byDir.has(dir)) byDir.set(dir, []);
byDir.get(dir).push(file);
}

const dirs = [];
const excludeFiles = [];
let eligibleFiles = 0;
for (const [dir, dirFiles] of [...byDir.entries()].sort()) {
// Threshold is computed over files with fresh evidence only: a carried
// exclude has no batch data in this window, so letting it count against
// dir eligibility can drop the dir, which drops its carried excludes and
// recreates the ping-pong at the dir level.
const fresh = dirFiles.filter(f => !previousExcludes.has(f));
const good = fresh.filter(isGood);
if (good.length < Math.max(1, Math.ceil(fresh.length * DIR_MIN_FRACTION))) continue;
dirs.push(dir);
eligibleFiles += good.length;
for (const f of dirFiles) if (!isGood(f)) excludeFiles.push(f);
Comment thread
robobun marked this conversation as resolved.
Resolved
}
};

return { dirs, excludeFiles, eligibleFiles, totalDirs: byDir.size };
}

const isNodeStyle = p =>
p.includes("js/node/test/parallel/") || p.includes("js/node/test/sequential/") || p.includes("js/bun/test/parallel/");
Expand All @@ -63,113 +77,144 @@ function listBunTestFiles() {
return out.sort();
}

async function findBuilds(want) {
const picked = [];
for (let page = 1; picked.length < want && page <= 25; page++) {
const batch = await (await api(`builds?state[]=passed&state[]=failed&per_page=100&page=${page}`)).json();
if (!batch.length) break;
for (const b of batch) {
if (!b.pull_request) continue;
const testJobs = (b.jobs || []).filter(j => j.step_key?.endsWith("test-bun") && j.finished_at);
if (testJobs.length >= 50) picked.push(b.number);
if (picked.length >= want) break;
}
console.error(` build page ${page}: ${picked.length}/${want}`);
if (process.argv[1] && realpathSync(process.argv[1]) === __filename) {
const { values: opts } = parseArgs({
options: {
builds: { type: "string", default: "300" },
"fast-ms": { type: "string", default: "15000" },
org: { type: "string", default: "bun" },
pipeline: { type: "string", default: "bun" },
},
});
const FAST_MS = parseInt(opts["fast-ms"], 10);
const WANT_BUILDS = parseInt(opts.builds, 10);

const token = process.env.BUILDKITE_API_TOKEN || process.env.BUILDKITE_TOKEN;
if (!token) {
console.error("BUILDKITE_API_TOKEN is required");
process.exit(1);
}
return picked;
}

async function collectFlakes(builds) {
const counts = new Map();
let done = 0;
const queue = [...builds];
const worker = async () => {
for (;;) {
const build = queue.shift();
if (!build) return;
const annotations = [];
for (let page = 1; ; page++) {
const batch = await (await api(`builds/${build}/annotations?per_page=100&page=${page}`)).json();
annotations.push(...batch);
if (batch.length < 100) break;
const api = async path => {
for (let attempt = 0; ; attempt++) {
const r = await fetch(
`https://api.buildkite.com/v2/organizations/${opts.org}/pipelines/${opts.pipeline}/${path}`,
{
headers: { Authorization: `Bearer ${token}` },
signal: AbortSignal.timeout(60_000),
},
);
if (r.ok) return r;
if ((r.status === 429 || r.status >= 500) && attempt < 8) {
const backoff = Number(r.headers.get("retry-after")) * 1000 || 1000 * 2 ** attempt;
await new Promise(resolve => setTimeout(resolve, backoff));
continue;
}
throw new Error(`${path}: ${r.status} ${r.statusText}`);
}
};

async function findBuilds(want) {
const picked = [];
for (let page = 1; picked.length < want && page <= 25; page++) {
const batch = await (await api(`builds?state[]=passed&state[]=failed&per_page=100&page=${page}`)).json();
if (!batch.length) break;
for (const b of batch) {
if (!b.pull_request) continue;
const testJobs = (b.jobs || []).filter(j => j.step_key?.endsWith("test-bun") && j.finished_at);
if (testJobs.length >= 50) picked.push(b.number);
if (picked.length >= want) break;
}
const seen = new Set();
for (const a of annotations) {
if (a.context !== "flaky" && a.style !== "error") continue;
for (const m of (a.body_html || "").matchAll(/<code>test\/([^<]+)<\/code><\/a> - /g)) {
seen.add(m[1]);
console.error(` build page ${page}: ${picked.length}/${want}`);
}
return picked;
}

async function collectFlakes(builds) {
const counts = new Map();
let done = 0;
const queue = [...builds];
const worker = async () => {
for (;;) {
const build = queue.shift();
if (!build) return;
const annotations = [];
for (let page = 1; ; page++) {
const batch = await (await api(`builds/${build}/annotations?per_page=100&page=${page}`)).json();
annotations.push(...batch);
if (batch.length < 100) break;
}
const seen = new Set();
for (const a of annotations) {
if (a.context !== "flaky" && a.style !== "error") continue;
for (const m of (a.body_html || "").matchAll(/<code>test\/([^<]+)<\/code><\/a> - /g)) {
seen.add(m[1]);
}
}
for (const file of seen) counts.set(file, (counts.get(file) || 0) + 1);
if (++done % 50 === 0) console.error(` annotations ${done}/${builds.length}`);
}
for (const file of seen) counts.set(file, (counts.get(file) || 0) + 1);
if (++done % 50 === 0) console.error(` annotations ${done}/${builds.length}`);
};
await Promise.all(Array.from({ length: 4 }, worker));
return counts;
}

let durations = {};
try {
durations = JSON.parse(readFileSync(durationsPath, "utf8"));
} catch (e) {
console.error(`could not read ${durationsPath}: ${e?.message || e}`);
process.exit(1);
}

let previousExcludes = new Set();
try {
const prev = JSON.parse(readFileSync(outputPath, "utf8")).excludeFiles;
if (!Array.isArray(prev)) throw new Error("excludeFiles is not an array");
previousExcludes = new Set(prev);
} catch (e) {
if (e?.code !== "ENOENT") {
console.error(`could not read previous ${outputPath}: ${e?.message || e}`);
process.exit(1);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
};
await Promise.all(Array.from({ length: 4 }, worker));
return counts;
}
}

let durations = {};
try {
durations = JSON.parse(readFileSync(durationsPath, "utf8"));
} catch (e) {
console.error(`could not read ${durationsPath}: ${e?.message || e}`);
process.exit(1);
}
console.error(`finding ${WANT_BUILDS} recent finished PR builds...`);
const builds = await findBuilds(WANT_BUILDS);
if (builds.length < 50) {
console.error(`only found ${builds.length} builds; refusing to generate an allowlist from that little history`);
process.exit(1);
}
console.error(`scanning annotations from ${builds.length} builds (#${Math.min(...builds)}..#${Math.max(...builds)})`);
const flakeCounts = await collectFlakes(builds);
console.error(`${flakeCounts.size} distinct files flaked or failed in that window`);

console.error(`finding ${WANT_BUILDS} recent finished PR builds...`);
const builds = await findBuilds(WANT_BUILDS);
if (builds.length < 50) {
console.error(`only found ${builds.length} builds; refusing to generate an allowlist from that little history`);
process.exit(1);
}
console.error(`scanning annotations from ${builds.length} builds (#${Math.min(...builds)}..#${Math.max(...builds)})`);
const flakeCounts = await collectFlakes(builds);
console.error(`${flakeCounts.size} distinct files flaked or failed in that window`);

const dockerPrefixes = Object.keys(dockerPrestartMap);
const files = listBunTestFiles();
const isGood = file => {
if (dockerPrefixes.some(prefix => file.startsWith(prefix))) return false;
if (/stress/i.test(file)) return false;
if (flakeCounts.has(file)) return false;
const ms = durations[file]?.default;
return ms === undefined || ms <= FAST_MS;
};

const byDir = new Map();
for (const file of files) {
const slash = file.lastIndexOf("/");
const dir = slash === -1 ? "" : file.slice(0, slash);
if (!byDir.has(dir)) byDir.set(dir, []);
byDir.get(dir).push(file);
}
const files = listBunTestFiles();
const { dirs, excludeFiles, eligibleFiles, totalDirs } = computeAllowlist({
files,
flaky: flakeCounts,
durations,
previousExcludes,
dockerPrefixes: Object.keys(dockerPrestartMap),
fastMs: FAST_MS,
});

const dirs = [];
const excludeFiles = [];
let eligibleFiles = 0;
for (const [dir, dirFiles] of [...byDir.entries()].sort()) {
const good = dirFiles.filter(isGood);
if (good.length < Math.max(1, Math.ceil(dirFiles.length * DIR_MIN_FRACTION))) continue;
dirs.push(dir);
eligibleFiles += good.length;
for (const f of dirFiles) if (!isGood(f)) excludeFiles.push(f);
}
const out = {
_meta: {
generated_at: new Date().toISOString(),
generated_by: "scripts/update-parallel-allowlist.mjs",
builds_scanned: builds.length,
build_range: [Math.min(...builds), Math.max(...builds)],
fast_ms: FAST_MS,
rule: `a directory is listed when >= ${Math.round(DIR_MIN_FRACTION * 100)}% of its bun test files are <= ${FAST_MS}ms (median, default lane of expected-durations.json) with zero flaky/failed annotations in the scanned builds; its other files are listed in excludeFiles; docker-service and stress-named tests never qualify; previous excludeFiles are carried forward (no batch data while excluded)`,
stats: { dirs: dirs.length, files: eligibleFiles, excluded: excludeFiles.length },
},
dirs,
excludeFiles,
};

const out = {
_meta: {
generated_at: new Date().toISOString(),
generated_by: "scripts/update-parallel-allowlist.mjs",
builds_scanned: builds.length,
build_range: [Math.min(...builds), Math.max(...builds)],
fast_ms: FAST_MS,
rule: `a directory is listed when >= ${Math.round(DIR_MIN_FRACTION * 100)}% of its bun test files are <= ${FAST_MS}ms (median, default lane of expected-durations.json) with zero flaky/failed annotations in the scanned builds; its other files are listed in excludeFiles; docker-service and stress-named tests never qualify`,
stats: { dirs: dirs.length, files: eligibleFiles, excluded: excludeFiles.length },
},
dirs,
excludeFiles,
};

writeFileSync(outputPath, JSON.stringify(out, null, 2) + "\n");
console.error(
`wrote ${outputPath}: ${dirs.length}/${byDir.size} dirs, ${eligibleFiles}/${files.length} files eligible, ${excludeFiles.length} excluded`,
);
writeFileSync(outputPath, JSON.stringify(out, null, 2) + "\n");
console.error(
`wrote ${outputPath}: ${dirs.length}/${totalDirs} dirs, ${eligibleFiles}/${files.length} files eligible, ${excludeFiles.length} excluded`,
);
}
Loading