Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 142 additions & 0 deletions .github/workflows/experimental-windows-timing.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
name: Experimental Windows visual timing

on:
pull_request:
branches: [master]
paths:
- ".github/workflows/experimental-windows-timing.yml"
- "Apps/scripts/windowsSharedTiming/**"
workflow_dispatch:

permissions:
contents: read

jobs:
timing:
runs-on: windows-2022
timeout-minutes: 180
env:
npm_config_fetch_retries: "5"
steps:
- uses: actions/checkout@v5

- name: Checkout exact native prototype
uses: actions/checkout@v5
with:
repository: bghgary/BabylonNative
ref: 8be7cacaca510d8b16bc9c44500516e82abb86ec
path: Build/TimingSource
persist-credentials: false

- name: Checkout exact shared suite and references
uses: actions/checkout@v5
with:
repository: bghgary/Babylon.js
ref: 345ce53604cb74d27f164a9b3b6604e26fb6adb4
path: Build/TimingSuite
persist-credentials: false

- uses: actions/setup-node@v4
with:
node-version: "24"

- name: Install the shared-suite package manager
shell: pwsh
run: |
npm install --prefix "$env:RUNNER_TEMP\shared-npm" npm@11.18.0 --no-audit --no-fund
if ($LASTEXITCODE -ne 0) { throw 'Package-manager setup failed.' }
"$env:RUNNER_TEMP\shared-npm\node_modules\.bin" >> $env:GITHUB_PATH

- name: Prepare the real shared suite
working-directory: Build/TimingSuite
run: npm ci --no-audit --no-fund

- name: Check timing report validation
run: node --test Apps/scripts/windowsSharedTiming/run.test.cjs

- name: Configure and build pinned Release V8 D3D11 Playground
shell: pwsh
run: |
$ErrorActionPreference = 'Stop'
$watch = [Diagnostics.Stopwatch]::StartNew()
cmake -S Build\TimingSource -B Build\WindowsTiming -G "Visual Studio 17 2022" -A x64 `
-DNAPI_JAVASCRIPT_ENGINE=V8 -DGRAPHICS_API=D3D11 `
-DBABYLON_NATIVE_PLUGIN_NATIVEDRACO=ON -DBABYLON_NATIVE_PLUGIN_NATIVEMESHOPT=ON `
-DBX_CONFIG_DEBUG=OFF -DBABYLON_DEBUG_TRACE=OFF
if ($LASTEXITCODE -ne 0) { throw 'Configure failed.' }
$configureMs = $watch.Elapsed.TotalMilliseconds
$watch.Restart()
cmake --build Build\WindowsTiming --config Release --target Playground -- /m
if ($LASTEXITCODE -ne 0) { throw 'Build failed.' }
New-Item -ItemType Directory -Force Build\timing-artifacts | Out-Null
@{
configureMs = $configureMs
buildMs = $watch.Elapsed.TotalMilliseconds
nativeHead = (git -C Build\TimingSource rev-parse HEAD)
suiteHead = (git -C Build\TimingSuite rev-parse HEAD)
v8Packages = @(Get-ChildItem Build\WindowsTiming\NuGet\packages -Directory |
Where-Object Name -Like 'v8-*' | Select-Object -ExpandProperty Name)
} | ConvertTo-Json | Set-Content Build\timing-artifacts\build.json

- name: Verify runtime package pins
shell: pwsh
run: |
$versions = @{}
foreach ($name in @('babylonjs', 'babylonjs-addons', 'babylonjs-loaders')) {
$package = Get-Content "Build\TimingSource\Apps\node_modules\$name\package.json" -Raw | ConvertFrom-Json
$versions[$name] = $package.version
if ($package.version -ne '9.29.0') { throw "Unexpected $name version." }
}
$versions | ConvertTo-Json | Set-Content Build\timing-artifacts\packages.json
$files = @{
'babylonjs' = 'babylon.max.js'
'babylonjs-addons' = 'babylonjs.addons.js'
'babylonjs-loaders' = 'babylonjs.loaders.js'
}
foreach ($name in $files.Keys) {
$source = Get-FileHash "Build\TimingSource\Apps\node_modules\$name\$($files[$name])"
$runtime = Get-FileHash "Build\WindowsTiming\Apps\Playground\Release\Scripts\$($files[$name])"
if ($source.Hash -ne $runtime.Hash) { throw "Runtime bundle differs: $name" }
}

- name: Run exact mobile workloads through per-scene Playwright
timeout-minutes: 120
shell: pwsh
env:
TIMING_SUITE_ROOT: ${{ github.workspace }}\Build\TimingSuite
BABYLON_NATIVE_PLAYGROUND: ${{ github.workspace }}\Build\WindowsTiming\Apps\Playground\Release\Playground.exe
TIMING_ARTIFACTS: ${{ github.workspace }}\Build\timing-artifacts
run: |
@{
processor = @(Get-CimInstance Win32_Processor | Select-Object Name, NumberOfCores, NumberOfLogicalProcessors)
graphics = @(Get-CimInstance Win32_VideoController | Select-Object Name, DriverVersion, AdapterRAM)
os = Get-CimInstance Win32_OperatingSystem | Select-Object Caption, Version, BuildNumber
runnerImage = $env:ImageVersion
configuration = 'Release'
engine = 'V8'
graphicsApi = 'D3D11'
} | ConvertTo-Json -Depth 10 | Set-Content Build\timing-artifacts\machine.json
node Apps\scripts\windowsSharedTiming\run.cjs

- name: Summarize actual timing and coverage
if: ${{ !cancelled() }}
shell: pwsh
run: |
"## Experimental Windows timing (not a migration gate)" >> $env:GITHUB_STEP_SUMMARY
$file = 'Build\timing-artifacts\summary.json'
if (Test-Path $file) {
$summary = Get-Content $file -Raw | ConvertFrom-Json
foreach ($phase in $summary.phases) {
"$($phase.mode): selected=$($phase.selected), executed=$($phase.executed), passed=$($phase.passed), failed=$($phase.failed), completed=$($phase.completed), process=$([math]::Round($phase.processWallMs / 1000, 3))s." >> $env:GITHUB_STEP_SUMMARY
}
} else {
"No complete timing report. This run establishes no Windows execution timing." >> $env:GITHUB_STEP_SUMMARY
}

- name: Upload timing reports and image differences
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v6
with:
name: experimental-windows-shared-scenes
path: Build/timing-artifacts/
if-no-files-found: warn
9 changes: 9 additions & 0 deletions Apps/scripts/windowsSharedTiming/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Current shared-suite Windows timing

This experiment runs Babylon.js's actual `visualization.native.test.ts`, once for the 417-scene Android workload and then for the 397-scene iOS workload. Each phase uses one worker, zero retries, the original reference images and unchanged scene render counts and tolerances. Every scene remains a separate Playwright test.

The fixed workload records the measured mobile suite revision and hashes the shared metadata and original references. Metadata hashes normalize Windows line endings; image hashes use the original bytes. Set `TIMING_SUITE_ROOT`, `BABYLON_NATIVE_PLAYGROUND` and a fresh `TIMING_ARTIFACTS` directory, then run `node run.cjs`. The suite checkout needs its normal dependencies and prepared test tools. The executable must contain the native Win32 Playwright bridge and the pinned Babylon.js 9.29.0 runtime.

Reports retain failures and image differences. A nonzero exit remains a failed experiment; timings do not qualify failing scenes as passing. The second phase follows the first on the same machine, so report the order rather than interpreting both as cold-machine measurements. Build and dependency setup are outside the measured Playwright processes.

This is not a production migration. The earlier 305-scene batch experiment remains in `../windowsTiming`; its timings are not current per-scene suite measurements.
31 changes: 31 additions & 0 deletions Apps/scripts/windowsSharedTiming/playwright.config.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
const path = require("node:path");
const { createRequire } = require("node:module");

for (const name of ["TIMING_SUITE_ROOT", "TIMING_ARTIFACTS", "TIMING_PHASE", "BABYLON_NATIVE_PLAYGROUND"]) {
if (!process.env[name]) {
throw new Error(`Set ${name} before running the experiment`);
}
}
const root = path.resolve(process.env.TIMING_SUITE_ROOT);
const { defineConfig } = createRequire(path.join(root, "package.json"))("@playwright/test");
delete process.env.BABYLON_NATIVE_TRANSPORT;
process.env.BABYLON_NATIVE_PLATFORM = "win32";
process.env.BABYLON_NATIVE_GRAPHICS_API = "d3d11";
process.env.TIMEOUT = "300000";
const output = path.join(process.env.TIMING_ARTIFACTS, process.env.TIMING_PHASE);

module.exports = defineConfig({
testDir: path.join(root, "packages", "tools", "tests", "test", "playwright"),
testMatch: "visualization.native.test.ts",
projects: [{ name: "native", use: {} }],
workers: 1,
fullyParallel: true,
retries: 0,
timeout: 300_000,
globalTimeout: 95 * 60_000,
updateSnapshots: "none",
snapshotPathTemplate: path.join(root, "packages", "tools", "tests", "test", "visualization", "ReferenceImages", "{arg}{ext}"),
outputDir: path.join(output, "test-results"),
use: { trace: "off", video: "off", screenshot: "off" },
reporter: [["line"], ["json", { outputFile: path.join(output, "playwright.json") }]],
});
139 changes: 139 additions & 0 deletions Apps/scripts/windowsSharedTiming/run.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
const fs = require("node:fs");
const path = require("node:path");
const crypto = require("node:crypto");
const { spawn, execFileSync } = require("node:child_process");

function hash(file) {
const bytes = fs.readFileSync(file);
const content = file.endsWith(".json") ? bytes.toString("utf8").replace(/\r\n/g, "\n") : bytes;
return crypto.createHash("sha256").update(content).digest("hex");
}

function summarizePhase(report, titles) {
const specs = [];
function visit(suite) {
specs.push(...(suite.specs || []));
for (const child of suite.suites || []) {
visit(child);
}
}
visit(report);
if (new Set(titles).size !== titles.length || specs.length !== titles.length ||
new Set(specs.map(spec => spec.title)).size !== titles.length ||
specs.some(spec => !titles.includes(spec.title))) {
throw new Error("Executed scene names differ from the fixed workload");
}
const stagesMs = {};
const failures = [];
let testsMs = 0;
let passed = 0;
let capturedFrames = 0;
for (const spec of specs) {
if (spec.tests.length !== 1 || spec.tests[0].results.length !== 1) {
throw new Error(`Expected exactly one attempt for ${spec.title}`);
}
const result = spec.tests[0].results[0];
if (!["passed", "failed", "timedOut"].includes(result.status)) {
throw new Error(`Scene did not complete: ${spec.title} (${result.status})`);
}
testsMs += result.duration;
for (const step of result.steps || []) {
stagesMs[step.title] = (stagesMs[step.title] || 0) + step.duration;
if (step.title === "Compare screenshot") {
capturedFrames++;
}
}
if (result.status === "passed") {
passed++;
} else {
failures.push({ title: spec.title, status: result.status, errors: result.errors });
}
}
return {
selected: titles.length, executed: specs.length, passed, failed: failures.length,
completed: true, playwrightMs: report.stats.duration, testsMs, capturedFrames,
stagesMs, failures,
};
}

async function main() {
for (const name of ["TIMING_SUITE_ROOT", "TIMING_ARTIFACTS", "BABYLON_NATIVE_PLAYGROUND"]) {
if (!process.env[name]) {
throw new Error(`Set ${name} before running the experiment`);
}
}
const root = path.resolve(process.env.TIMING_SUITE_ROOT);
const artifacts = path.resolve(process.env.TIMING_ARTIFACTS);
const executable = path.resolve(process.env.BABYLON_NATIVE_PLAYGROUND);
const workload = JSON.parse(fs.readFileSync(path.join(__dirname, "workload.json"), "utf8"));
const suiteHead = execFileSync("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
if (suiteHead !== workload.suiteHead) {
throw new Error("Shared suite revision differs from the measured mobile revision");
}
for (const file of workload.files) {
if (hash(path.join(root, file.path)) !== file.sha256) {
throw new Error(`Shared scene metadata or original reference changed: ${file.path}`);
}
}
fs.mkdirSync(artifacts, { recursive: true });
if (fs.existsSync(path.join(artifacts, "summary.json"))) {
throw new Error(`Refusing to overwrite an earlier experiment: ${artifacts}`);
}
const summary = { suiteHead, executableSha256: hash(executable), workers: 1, retries: 0, phases: [] };
const persist = () => fs.writeFileSync(path.join(artifacts, "summary.json"), JSON.stringify(summary, null, 2));
const cli = require.resolve("@playwright/test/cli", { paths: [root] });
for (const phase of workload.phases) {
const output = path.join(artifacts, phase.name);
if (fs.existsSync(output)) {
throw new Error(`Refusing to overwrite an earlier phase: ${output}`);
}
fs.mkdirSync(output);
const escape = title => title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const grep = `visualization\\.native\\.test\\.ts (?:${phase.titles.map(escape).join("|")})$`;
const result = { mode: phase.name, selected: phase.titles.length, completed: false, startedAtMs: Date.now() };
summary.phases.push(result);
persist();
const started = performance.now();
const log = fs.openSync(path.join(output, "runner.log"), "wx");
try {
result.exitCode = await new Promise((resolve, reject) => {
const child = spawn(process.execPath, [cli, "test", "--config", path.join(__dirname, "playwright.config.cjs"),
"--grep", grep, "--workers", "1", "--retries", "0"], {
cwd: root,
env: { ...process.env, TIMING_SUITE_ROOT: root, TIMING_ARTIFACTS: artifacts,
TIMING_PHASE: phase.name, BABYLON_NATIVE_PLAYGROUND: executable },
stdio: ["ignore", log, log],
});
child.once("error", reject);
child.once("exit", (code, signal) => {
if (signal) {
reject(new Error(`Playwright terminated by ${signal}`));
} else {
resolve(code ?? 1);
}
});
});
result.processWallMs = performance.now() - started;
const report = JSON.parse(fs.readFileSync(path.join(output, "playwright.json"), "utf8"));
Object.assign(result, summarizePhase(report, phase.titles));
console.log(JSON.stringify({ ...result, failures: undefined }));
} catch (error) {
result.error = error.stack || String(error);
throw error;
} finally {
fs.closeSync(log);
persist();
}
}
if (summary.phases.some(phase => phase.exitCode !== 0 || phase.failed !== 0 || !phase.completed)) {
throw new Error("The visual experiment retained failures; inspect the per-scene reports and image differences");
}
}

module.exports = { summarizePhase };
if (require.main === module) {
main().catch(error => {
console.error(error);
process.exitCode = 1;
});
}
49 changes: 49 additions & 0 deletions Apps/scripts/windowsSharedTiming/run.test.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
const { test } = require("node:test");
const assert = require("node:assert/strict");
const { summarizePhase } = require("./run.cjs");

function report(status = "passed") {
return {
stats: { duration: 20 },
suites: [{ specs: [{
title: "Scene",
tests: [{ results: [{ status, duration: 12, errors: status === "passed" ? [] : [{ message: "Mismatch" }],
steps: [{ title: "Compare screenshot", duration: 3, steps: [{ title: "Nested assertion", duration: 2 }] }] }] }],
}] }],
};
}

test("reports one completed test and counts only top-level steps", () => {
const summary = summarizePhase(report(), ["Scene"]);
assert.equal(summary.executed, 1);
assert.equal(summary.passed, 1);
assert.equal(summary.failed, 0);
assert.equal(summary.testsMs, 12);
assert.equal(summary.capturedFrames, 1);
assert.deepEqual(summary.stagesMs, { "Compare screenshot": 3 });
});

test("retains failed comparisons and error details", () => {
const summary = summarizePhase(report("failed"), ["Scene"]);
assert.equal(summary.passed, 0);
assert.equal(summary.failed, 1);
assert.equal(summary.failures[0].errors[0].message, "Mismatch");
});

test("rejects absent, substituted and duplicated scenes", () => {
assert.throws(() => summarizePhase(report(), ["Missing"]), /scene names/);
assert.throws(() => summarizePhase(report(), ["Scene", "Missing"]), /scene names/);
assert.throws(() => summarizePhase(report(), ["Scene", "Scene"]), /scene names/);
const duplicate = report();
duplicate.suites[0].specs.push(duplicate.suites[0].specs[0]);
assert.throws(() => summarizePhase(duplicate, ["Scene", "Other"]), /scene names/);
});

test("rejects retries and skipped or interrupted tests", () => {
const retried = report();
retried.suites[0].specs[0].tests[0].results.push(retried.suites[0].specs[0].tests[0].results[0]);
assert.throws(() => summarizePhase(retried, ["Scene"]), /one attempt/);
for (const status of ["skipped", "interrupted"]) {
assert.throws(() => summarizePhase(report(status), ["Scene"]), /did not complete/);
}
});
Loading
Loading