diff --git a/benchmarks/perf/format-pr-comment.mjs b/benchmarks/perf/format-pr-comment.mjs index b8c317322..805a6c079 100644 --- a/benchmarks/perf/format-pr-comment.mjs +++ b/benchmarks/perf/format-pr-comment.mjs @@ -13,26 +13,28 @@ if (results.run.kind !== "pull_request") { process.exit(0); } -const response = JSON.parse(await readFile(responsePath, "utf8")); -const uploadedComparison = response.comparison; -if (!uploadedComparison) - throw new Error("Performance upload response did not include a comparison"); - const resultBenchmarks = Array.isArray(results.benchmarks) ? results.benchmarks : []; const resultsByBenchmark = new Map( resultBenchmarks.map((benchmark) => [benchmark.benchmarkId, benchmark]), ); +const response = await readUploadResponse(responsePath); +const uploadedComparison = response.comparison; +if (!uploadedComparison && resultBenchmarks.length === 0) { + await writeFile(outputPath, ""); + process.exit(0); +} const hasPairedBaseline = resultBenchmarks.some((benchmark) => benchmark.baselineSamples); +const comparisonSource = uploadedComparison ?? localComparison(resultBenchmarks); const comparison = { - ...uploadedComparison, + ...comparisonSource, baseline: hasPairedBaseline ? { sha: results.run.baseSha, - shortSha: results.run.baseSha.slice(0, 7), - measuredAt: results.run.measuredAt, + shortSha: shortSha(results.run.baseSha), + measuredAt: comparisonSource.baseline?.measuredAt ?? null, } - : uploadedComparison.baseline, - measurements: uploadedComparison.measurements.map((measurement) => { + : comparisonSource.baseline, + measurements: comparisonSource.measurements.map((measurement) => { const benchmark = resultsByBenchmark.get(measurement.benchmarkId); return benchmark?.baselineSamples ? { @@ -64,6 +66,51 @@ function formatValue(value, unit) { return `${Number(value.toFixed(2))} ${unit}`; } +function shortSha(sha) { + return typeof sha === "string" && sha.length > 0 ? sha.slice(0, 7) : "unknown"; +} + +async function readUploadResponse(path) { + try { + return JSON.parse(await readFile(path, "utf8")); + } catch (error) { + // The skipped-upload fields are diagnostic metadata; comment formatting is + // gated by the presence of a dashboard comparison. + if (error?.code === "ENOENT") return { uploaded: false, reason: "missing_response" }; + throw error; + } +} + +function localComparison(benchmarks) { + const hasLocalBaseline = + results.run.baseSha && benchmarks.some((benchmark) => benchmark.baselineSamples); + + return { + uploaded: false, + head: { + sha: results.run.commitSha, + shortSha: shortSha(results.run.commitSha), + measuredAt: results.run.measuredAt, + }, + baseline: hasLocalBaseline + ? { + sha: results.run.baseSha, + shortSha: shortSha(results.run.baseSha), + measuredAt: null, + } + : null, + measurements: benchmarks.map((benchmark) => ({ + benchmarkId: benchmark.benchmarkId, + label: benchmark.label, + implementationLabel: benchmark.implementationLabel, + unit: benchmark.unit, + lowerIsBetter: benchmark.lowerIsBetter, + baseline: benchmark.baselineSamples, + current: benchmark.samples, + })), + }; +} + function measurementChange(measurement) { if (!measurement.baseline) return null; return ( @@ -115,7 +162,9 @@ const hasCurrentOnlyMeasurement = comparison.measurements.some( (measurement) => !measurement.baseline && !resultsByBenchmark.get(measurement.benchmarkId)?.baselineSamples, ); -const dashboardUrl = `https://vinext.dev/benchmarks/pull/${results.run.pullRequest}`; +const dashboardUrl = uploadedComparison + ? `https://vinext.dev/benchmarks/pull/${results.run.pullRequest}` + : null; const rows = measurements.map((measurement) => [ escapeCell(measurement.label), @@ -140,7 +189,7 @@ const body = [ : `Compared \`${comparison.head.shortSha}\` against base \`${comparison.baseline.shortSha}\`. Paired benchmarks use alternating same-runner rounds; unpaired benchmarks have no baseline.${skippedNextjs ? " Next.js was unchanged and skipped." : ""}` : `Compared \`${comparison.head.shortSha}\` against base \`${comparison.baseline.shortSha}\` using alternating same-runner rounds.${skippedNextjs ? " Next.js was unchanged and skipped." : ""}` : `Compared \`${comparison.head.shortSha}\` against base \`${comparison.baseline.shortSha}\`.` - : `Measured \`${comparison.head.shortSha}\`. No benchmark run is available for base \`${results.run.baseSha.slice(0, 7)}\`.`, + : `Measured \`${comparison.head.shortSha}\`. No benchmark run is available for base \`${shortSha(results.run.baseSha)}\`.`, "", comparison.baseline ? `**${improvements} improved · ${regressions} regressed · ${neutral} within ±1.5%**` @@ -150,7 +199,9 @@ const body = [ "|---|---|---:|---:|---:|", ...rows.map((row) => `| ${row} |`), "", - `[View detailed results and traces](${dashboardUrl})`, + dashboardUrl + ? `[View detailed results and traces](${dashboardUrl})` + : "Dashboard upload was unavailable for this run.", "", `🟢 improvement · 🔴 regression · ⚫ change below 1.5%${ hasPairedBaseline diff --git a/benchmarks/perf/upload-results.mjs b/benchmarks/perf/upload-results.mjs index 50611e191..ff99966c4 100644 --- a/benchmarks/perf/upload-results.mjs +++ b/benchmarks/perf/upload-results.mjs @@ -11,8 +11,17 @@ const secret = process.env.COMPAT_INGEST_SECRET; const artifactRoot = process.env.VINEXT_PERF_ARTIFACT_ROOT ? resolve(process.env.VINEXT_PERF_ARTIFACT_ROOT) : null; +const uploadResponsePath = process.env.VINEXT_PERF_UPLOAD_RESPONSE_PATH + ? resolve(process.env.VINEXT_PERF_UPLOAD_RESPONSE_PATH) + : null; if (!secret) { + if (uploadResponsePath) { + await writeFile( + uploadResponsePath, + `${JSON.stringify({ uploaded: false, reason: "missing_secret" })}\n`, + ); + } console.log("COMPAT_INGEST_SECRET is not configured; skipping performance upload."); process.exit(0); } @@ -116,8 +125,8 @@ try { const responseBody = await uploadMetadata(); metadataCommitted = true; - if (process.env.VINEXT_PERF_UPLOAD_RESPONSE_PATH) { - await writeFile(resolve(process.env.VINEXT_PERF_UPLOAD_RESPONSE_PATH), `${responseBody}\n`); + if (uploadResponsePath) { + await writeFile(uploadResponsePath, `${responseBody}\n`); } console.log(responseBody); } catch (error) { diff --git a/tests/performance-benchmarks.test.ts b/tests/performance-benchmarks.test.ts index 2ee7df5db..032cb75e6 100644 --- a/tests/performance-benchmarks.test.ts +++ b/tests/performance-benchmarks.test.ts @@ -628,6 +628,113 @@ describe("paired performance benchmarks", () => { expect(comment).not.toContain("Next.js |"); }); + it("writes a skipped upload response when the dashboard secret is unavailable", () => { + const directory = mkdtempSync(join(tmpdir(), "vinext-perf-upload-skip-")); + const resultsPath = join(directory, "results.json"); + const responsePath = join(directory, "response.json"); + writeFileSync(resultsPath, JSON.stringify({ benchmarks: [] })); + + execFileSync(process.execPath, ["benchmarks/perf/upload-results.mjs", resultsPath], { + cwd: join(import.meta.dirname, ".."), + env: { + ...process.env, + COMPAT_INGEST_SECRET: "", + VINEXT_PERF_UPLOAD_RESPONSE_PATH: responsePath, + }, + }); + + expect(JSON.parse(readFileSync(responsePath, "utf8"))).toEqual({ + uploaded: false, + reason: "missing_secret", + }); + }); + + it("renders paired PR comments when the dashboard upload response is missing", () => { + const directory = mkdtempSync(join(tmpdir(), "vinext-perf-local-comment-")); + const resultsPath = join(directory, "results.json"); + const responsePath = join(directory, "response.json"); + const outputPath = join(directory, "comment.md"); + writeFileSync( + resultsPath, + JSON.stringify({ + run: { + kind: "pull_request", + pullRequest: 42, + commitSha: "a".repeat(40), + baseSha: "b".repeat(40), + measuredAt: "2026-01-01T00:00:00.000Z", + }, + benchmarks: [ + { + benchmarkId: "vinext-production-build", + label: "Production build time", + implementationLabel: "vinext", + unit: "ms", + lowerIsBetter: true, + samples: { median: 90 }, + baselineSamples: { median: 100 }, + }, + ], + }), + ); + + execFileSync( + process.execPath, + ["benchmarks/perf/format-pr-comment.mjs", resultsPath, responsePath, outputPath], + { cwd: join(import.meta.dirname, "..") }, + ); + + const comment = readFileSync(outputPath, "utf8"); + expect(comment).toContain("Compared `aaaaaaa` against base `bbbbbbb`"); + expect(comment).toContain("1 improved · 0 regressed · 0 within ±1.5%"); + expect(comment).toContain("| Production build time | vinext | 100 ms | 90 ms |"); + expect(comment).toContain("Dashboard upload was unavailable for this run."); + expect(comment).not.toContain("View detailed results and traces"); + }); + + it("does not summarize unpaired local PR comments as baseline comparisons", () => { + const directory = mkdtempSync(join(tmpdir(), "vinext-perf-unpaired-local-comment-")); + const resultsPath = join(directory, "results.json"); + const responsePath = join(directory, "response.json"); + const outputPath = join(directory, "comment.md"); + writeFileSync( + resultsPath, + JSON.stringify({ + run: { + kind: "pull_request", + pullRequest: 42, + commitSha: "a".repeat(40), + baseSha: "b".repeat(40), + measuredAt: "2026-01-01T00:00:00.000Z", + }, + benchmarks: [ + { + benchmarkId: "vinext-production-build", + label: "Production build time", + implementationLabel: "vinext", + unit: "ms", + lowerIsBetter: true, + samples: { median: 90 }, + }, + ], + }), + ); + + execFileSync( + process.execPath, + ["benchmarks/perf/format-pr-comment.mjs", resultsPath, responsePath, outputPath], + { cwd: join(import.meta.dirname, "..") }, + ); + + const comment = readFileSync(outputPath, "utf8"); + expect(comment).toContain( + "Measured `aaaaaaa`. No benchmark run is available for base `bbbbbbb`.", + ); + expect(comment).toContain("1 measurements recorded · baseline unavailable"); + expect(comment).toContain("| Production build time | vinext | — | 90 ms | New |"); + expect(comment).not.toContain("0 improved · 0 regressed · 0 within ±1.5%"); + }); + it("labels mixed paired and historical PR comment baselines", () => { const directory = mkdtempSync(join(tmpdir(), "vinext-perf-mixed-comment-")); const resultsPath = join(directory, "results.json");