[JSC] Module loader: a load that a withdrawn termination cut leaves nothing that a later load waits for

JSTests/BUN-TEST-DIFFERENCES.md

@@ -240,6 +240,7 @@ Not upstream's: a sync neither brings nor changes them. Most are named `stress/b
 | `stress/module-loader-load-sync.js` | `$vm.moduleLoaderLoadSync({ loader }, specifier)` is `JSModuleLoader::loadModuleSync()`, the load of `require()` of an ES module in Bun: the graph has run when the call returns, a load that `import()` began is taken over, what the module throws is thrown, and a top-level await is a `TypeError` |
 | `stress/shell-module-loader-fetch-hook.js` | `setModuleLoaderFetchHook(fn)` of the shell: `fn(key)` runs before the file of a module is read, a string that it returns is the text of the module, an object makes a module without text whose exports are read when the graph is about to be linked (a deferred synthetic module, `SyntheticSourceProvider::createDeferred()`), the hook is asked for each fetch of a JavaScript module, and what it throws is what the load fails with. Tests use it for modules without files and for script inside the fetch step of a load |
 | `stress/module-link-cut-short-by-termination.js` | a termination that begins in the link of a module, in `CyclicModuleRecord::initializeEnvironment()` or in `SyntheticModuleRecord::initializeExports()` of a module whose exports are made at the link, and that is withdrawn leaves no environment and no export entries on a module that is linked again: the next `import()` links it (an assertion build stopped at `ASSERT(!m_moduleEnvironment)` or at `Duplicate export entry name`) |
+| `stress/module-load-cut-short-by-termination.js`, `-graphs.js`, `-sync.js` and `-nested.js` (with `stress/resources/module-load-cut-short-by-termination/walk.js`) | a termination at each trap check of an `import()` or of a synchronous load, withdrawn when the call ends, leaves nothing in the module loader that a later load waits for: a later `import()` of the same modules has them, each body runs once, and an `import()` that was in flight is answered with its module or not at all. The same for a load that is cut inside the fetch hook of another load: the outer load asks again and gets its modules. A later load rejects only where a module was stopped in its evaluation, and waits only for a module that waits in its evaluation for something that the termination stopped |
 
 `jsc` cannot compile a program ahead of time by itself (the compiler's input comes from the front end, in Bun's repository), so
 the programs that test ahead-of-time compilation are in Bun's `test/aot`.

JSTests/stress/module-link-cut-short-by-termination.js

@@ -6,7 +6,7 @@
 // sets an environment again. So the cut link has to leave none: ASSERT(!m_moduleEnvironment) in an assertion build.
 //
 // Each walk takes a termination at each trap check of an import() in turn ($vm.callWithTerminationAtTrapCheck()) and
-// imports the module again. A cut inside the load is not the subject here: there the later import() can stay pending.
+// imports the module again. A cut inside the load is the subject of module-load-cut-short-by-termination.js.
 
 function shouldBe(actual, expected, what) {
     if (actual !== expected)

JSTests/stress/module-load-cut-short-by-termination-graphs.js

@@ -0,0 +1,12 @@
+//@ runDefault("--useDollarVM=1", "--useJIT=0")
+
+// As module-load-cut-short-by-termination.js, for more graphs: a chain whose middle module is loaded again first, a
+// diamond, a cycle, a graph with a top-level await, and a graph whose root awaits its own import().
+
+load("./resources/module-load-cut-short-by-termination/walk.js");
+
+walk("chain", { begin: beginImport, check: checkImports, leaf: "mid.js" });
+walk("diamond", { begin: beginImport, check: checkImports });
+walk("cycle", { begin: beginImport, check: checkImports });
+walk("await", { begin: beginImport, check: checkImports });
+walk("aimp", { begin: beginImport, check: checkImports });

JSTests/stress/module-load-cut-short-by-termination-nested.js

@@ -0,0 +1,172 @@
+//@ runDefault("--useDollarVM=1", "--useJIT=0")
+
+// As module-load-cut-short-by-termination.js, but the load that is cut runs inside the fetch hook of another load,
+// and the termination is withdrawn there. The outer load goes on behind the hook: it has begun to load the same
+// module, so some of its steps wait for promises of the load that was cut. Each of them asks again
+// (JSModuleLoader::askAgainForAbandonedLoad()), and the outer load gets its modules.
+//
+// The modules are made by the fetch hook of the shell (setModuleLoaderFetchHook()). In each shape the hook for one
+// module loads another module synchronously, with a termination at the n-th trap check, and the hook for a module that
+// this load needs throws while the load runs, so the load never ends by itself inside the hook.
+
+const directory = "./module-load-cut-short-by-termination-nested/";
+const terminationValue = "JavaScript execution terminated.";
+globalThis.bodies = new Map;
+
+function shouldBe(actual, expected, what) {
+    if (actual !== expected)
+        throw new Error(`${what}: expected ${expected}, got ${actual}`);
+}
+
+const module = (name, imports, value) => imports.map(([file, binding]) => `import { value as ${binding} } from "./${file}";\n`).join("")
+    + `globalThis.bodies.set("${name}", (globalThis.bodies.get("${name}") ?? 0) + 1);\nexport const value = ${value};\n`;
+const shapes = [
+    {
+        // root.js imports d.js and x.js, and d.js imports spin.js. The hook for x.js loads d.js, which the outer load
+        // has asked for already.
+        name: "siblings",
+        sources: {
+            "root.js": module("root.js", [["d.js", "d"], ["x.js", "x"]], "d + x"),
+            "d.js": module("d.js", [["spin.js", "spin"]], "spin + 1"),
+            "x.js": module("x.js", [], "1"),
+            "spin.js": module("spin.js", [], "39"),
+        },
+        hookFile: "x.js",
+        loadedInHook: "d.js",
+        endsLoadInHook: "spin.js",
+        values: { "root.js": 41, "d.js": 40 },
+    },
+    {
+        // root.js and m.js import each other. m.js imports w.js and k.js, k.js imports z.js, and w.js imports w2.js.
+        // The hook for z.js loads w.js. A module of a cycle is in the list of loaded modules of its importer before
+        // its own load has ended.
+        name: "cycle",
+        sources: {
+            "root.js": module("root.js", [["m.js", "m"]], "m"),
+            "m.js": `import "./root.js";\n` + module("m.js", [["w.js", "w"], ["k.js", "k"]], "w + k"),
+            "k.js": module("k.js", [["z.js", "z"]], "z"),
+            "w.js": module("w.js", [["w2.js", "w2"]], "w2 + 1"),
+            "z.js": module("z.js", [], "1"),
+            "w2.js": module("w2.js", [], "39"),
+        },
+        hookFile: "z.js",
+        loadedInHook: "w.js",
+        endsLoadInHook: "w2.js",
+        values: { "root.js": 41, "w.js": 40 },
+    },
+];
+const endOfNestedLoad = new Error("the load inside the hook ends here");
+
+let shape = shapes[0];
+let round = null;
+setModuleLoaderFetchHook(key => {
+    const name = key.slice(key.lastIndexOf("/") + 1);
+    if (!round || !(name in shape.sources))
+        return shape.sources[name];
+    if (name == shape.endsLoadInHook && round.inHook)
+        throw endOfNestedLoad;
+    if (name == shape.hookFile && !round.hookRan) {
+        round.hookRan = true;
+        round.inHook = true;
+        try {
+            round.wasCut = $vm.callWithTerminationAtTrapCheck(() => {
+                $vm.moduleLoaderLoadSync(round.loader, directory + shape.loadedInHook);
+            }, round.n);
+        } catch (error) {
+            round.threw = error;
+        }
+        round.inHook = false;
+    }
+    return shape.sources[name];
+});
+
+function importFile(loader, file) {
+    const load = { file, state: "pending", value: undefined };
+    $vm.moduleLoaderImport(loader, directory + file).then(namespace => {
+        load.value = namespace.value;
+        load.state = "fulfilled";
+    }, error => {
+        load.value = error;
+        load.state = "rejected";
+    });
+    return load;
+}
+
+// A load has its module, or the error with which the load inside the hook ended by itself before the termination came.
+// It does not wait, and it is not answered with anything else.
+function check(what, load) {
+    if (load.state == "pending")
+        throw new Error(`${what}: ${load.file} is still pending`);
+    if (load.state == "rejected") {
+        shouldBe(load.value, endOfNestedLoad, `${what}: what ${load.file} rejected with`);
+        return false;
+    }
+    shouldBe(load.value, shape.values[load.file], `${what}: the value of ${load.file}`);
+    return true;
+}
+
+const outerLoads = {
+    // The outer load is an import() and its checkpoint.
+    "import()": loader => {
+        const root = importFile(loader, "root.js");
+        drainMicrotasks();
+        return root;
+    },
+    // The outer load is a synchronous one.
+    "synchronous load": loader => {
+        const root = { file: "root.js", state: "fulfilled", value: undefined };
+        try {
+            root.value = $vm.moduleLoaderLoadSync(loader, directory + "root.js").value;
+        } catch (error) {
+            root.state = "rejected";
+            root.value = error;
+        }
+        return root;
+    },
+};
+
+for (shape of shapes) {
+for (const [outerName, outerLoad] of Object.entries(outerLoads)) {
+    // A function has more trap checks while its inline caches are empty, so the walk loads the graph a few times first.
+    for (let i = 0; i < 10; ++i)
+        outerLoad($vm.createModuleLoader());
+
+    let roundsWithModule = 0;
+    for (let n = 1; ; ++n) {
+        if (n > 100000)
+            throw new Error(`${shape.name}, ${outerName}: the load inside the hook has no end`);
+        const what = `${shape.name}, ${outerName}, trap check ${n} of the load inside the hook`;
+        const loader = $vm.createModuleLoader();
+        bodies.clear();
+        round = { loader, n, hookRan: false, inHook: false, wasCut: undefined, threw: undefined };
+        const outer = outerLoad(loader);
+        const { wasCut, threw } = round;
+        round = null;
+        shouldBe($vm.hasPendingTermination(), false, `${what}: a termination is still pending`);
+
+        const laterRoot = importFile(loader, "root.js");
+        drainMicrotasks();
+        const laterLoadedInHook = importFile(loader, shape.loadedInHook);
+        drainMicrotasks();
+        const outerHasModule = check(`${what}: the outer load`, outer);
+        shouldBe(check(`${what}: a later import()`, laterRoot), outerHasModule, `${what}: a later import() of root.js has the module`);
+        check(`${what}: a later import()`, laterLoadedInHook);
+        for (const [file, runs] of bodies) {
+            if (runs > 1)
+                throw new Error(`${what}: the body of ${file} ran ${runs} times`);
+        }
+        if (outerHasModule)
+            shouldBe(bodies.size, Object.keys(shape.sources).length, `${what}: the modules that have run`);
+
+        if (!wasCut) {
+            // The load inside the hook ended with the error of the hook for the module that it needs.
+            shouldBe(threw, endOfNestedLoad, `${what}: what the load inside the hook threw`);
+            break;
+        }
+        if (outerHasModule)
+            ++roundsWithModule;
+    }
+    if (!roundsWithModule)
+        throw new Error(`${shape.name}, ${outerName}: no outer load got its module behind a cut`);
+}
+}

JSTests/stress/module-load-cut-short-by-termination-sync.js

@@ -0,0 +1,47 @@
+//@ runDefault("--useDollarVM=1", "--useJIT=0")
+
+// As module-load-cut-short-by-termination.js, but the call that is cut is a synchronous load
+// (JSModuleLoader::loadModuleSync(), what `require()` of an ES module is in Bun). Its jobs run in the synchronous queue
+// of the loader, and a termination ends the call with jobs left in that queue.
+
+load("./resources/module-load-cut-short-by-termination/walk.js");
+
+function beginSynchronousLoad(loader, name, round) {
+    round.value = loadSynchronously(loader, name).value;
+}
+
+// The same while an import() of the root is in flight: the synchronous load takes over the steps of that load, and
+// runs jobs of the import() too. The import() gets its module or, where the termination cut one of its own jobs,
+// no answer.
+function beginSynchronousLoadBehindImport(loader, name, round) {
+    round.imports = [importFile(loader, name, "root.js")];
+    round.value = loadSynchronously(loader, name).value;
+}
+
+function checkSynchronousLoad(what, graph, round, wasCut, settled) {
+    if (round.imports) {
+        // Behind the call the jobs of the import() wait in the queue of the VM.
+        round.mayStayPending = !settled;
+        checkImports(what, graph, round, wasCut, settled);
+    }
+    if (wasCut) {
+        shouldBe(round.value, undefined, `${what}: the call returned`);
+        return;
+    }
+    if (graph.waitsInEvaluation) {
+        // The load cannot finish inside the call.
+        if (!(round.threw instanceof TypeError))
+            throw new Error(`${what}: the synchronous load of a module that waits threw ${round.threw}`);
+        return;
+    }
+    shouldBe(round.threw, undefined, `${what}: the synchronous load threw`);
+    shouldBe(round.value, graph.values["root.js"], `${what}: the value of the synchronous load`);
+}
+
+walk("two", { begin: beginSynchronousLoad, check: checkSynchronousLoad, synchronousLaterLoad: true });
+walk("two", { begin: beginSynchronousLoad, check: checkSynchronousLoad, secondCuts: n => [n, n + 5, 2 * n] });
+walk("two", { begin: beginSynchronousLoadBehindImport, check: checkSynchronousLoad });
+walk("chain", { begin: beginSynchronousLoad, check: checkSynchronousLoad });
+walk("cycle", { begin: beginSynchronousLoad, check: checkSynchronousLoad });
+walk("await", { begin: beginSynchronousLoad, check: checkSynchronousLoad });
+walk("aimp", { begin: beginSynchronousLoad, check: checkSynchronousLoad });

JSTests/stress/module-load-cut-short-by-termination.js

@@ -0,0 +1,87 @@
+//@ runDefault("--useDollarVM=1", "--useJIT=0")
+
+// A termination can end a module load at any exception check, and an embedder can then withdraw it
+// (VM::cancelTermination(): node:vm's `timeout` in Bun). The loader goes on after that, so it must hold nothing that
+// makes a later load of the same modules wait for ever:
+// - a registry entry whose module promise or load promise nothing settles, because the step that settles it was cut,
+// - a graph loading state that waits for a request which was never made,
+// - a load job that was only queued when the checkpoint ended, and that the queue dropped.
+//
+// Here the call that is cut is an import() and the checkpoint that runs its jobs.
+// module-load-cut-short-by-termination-graphs.js does the same for more graphs,
+// module-load-cut-short-by-termination-sync.js for a synchronous load, and
+// module-load-cut-short-by-termination-nested.js for a load inside the fetch hook of another load.
+
+load("./resources/module-load-cut-short-by-termination/walk.js");
+
+// The jobs of the load share the queue with jobs of script, so a job of script is cut with jobs of the load behind it.
+// The jobs of script that the cut checkpoint drops stay dropped: only the jobs of the loader go on.
+let roundsWithDroppedJobs = 0;
+function beginImportAmongJobsOfScript(loader, name, round) {
+    round.ticks = 0;
+    round.imports = [importFile(loader, name, "root.js")];
+    let job = Promise.resolve();
+    for (let i = 0; i < 6; ++i)
+        job = job.then(() => { ++round.ticks; });
+    drainMicrotasks();
+}
+function checkImportsAmongJobsOfScript(what, graph, round, wasCut, settled) {
+    checkImports(what, graph, round, wasCut, settled);
+    if (!settled || !round.imports)
+        return;
+    if (!wasCut)
+        shouldBe(round.ticks, 6, `${what}: the jobs of script that ran`);
+    else if (round.ticks < 6)
+        ++roundsWithDroppedJobs;
+}
+
+// Two import() of the root are in flight when the termination comes.
+function beginTwoImports(loader, name, round) {
+    round.imports = [importFile(loader, name, "root.js"), importFile(loader, name, "root.js")];
+    drainMicrotasks();
+}
+
+walk("two", { begin: beginImport, check: checkImports, synchronousLaterLoad: true });
+walk("two", { begin: beginImport, check: checkImports, secondCuts: n => [n, n + 5, 2 * n] });
+walk("two", { begin: beginTwoImports, check: checkImports });
+walk("two", { begin: beginImportAmongJobsOfScript, check: checkImportsAmongJobsOfScript });
+if (!roundsWithDroppedJobs)
+    throw new Error("no cut checkpoint dropped a job of script");
+walk("chain", { begin: beginImport, check: checkImports });
+
+// A module of the graph cannot be fetched, so the load of the root fails. A later load of the root is answered with
+// that error, and is not left to wait. The error of one load is not kept in a registry entry that got no fetch: two
+// later loads of the missing module each fail with an error of their own.
+{
+    const failed = load => load.state == "rejected" && load.value instanceof Error && load.value.message.includes("Could not open file");
+    const round = n => {
+        const loader = $vm.createModuleLoader();
+        const what = `failing, trap check ${n}`;
+        const begin = () => {
+            importFile(loader, "failing", "root.js");
+            drainMicrotasks();
+        };
+        const wasCut = n ? $vm.callWithTerminationAtTrapCheck(begin, n) : (begin(), false);
+        shouldBe($vm.hasPendingTermination(), false, `${what}: a termination is still pending`);
+        // The missing module first: a cut behind the fetch hook has left its entry without a fetch.
+        const missing = importFile(loader, "failing", "missing.js");
+        drainMicrotasks();
+        const missingAgain = importFile(loader, "failing", "missing.js");
+        drainMicrotasks();
+        const root = importFile(loader, "failing", "root.js");
+        drainMicrotasks();
+        for (const load of [root, missing, missingAgain]) {
+            if (!failed(load))
+                throw new Error(`${what}: the later import() of ${load.file} is ${load.state} (${load.value})`);
+        }
+        if (missing.value === missingAgain.value)
+            throw new Error(`${what}: two loads of the missing module failed with one error object`);
+        return wasCut;
+    };
+    for (let i = 0; i < 10; ++i)
+        round(0);
+    for (let n = 1; round(n); ++n) {
+        if (n > 100000)
+            throw new Error("failing: the call has no end");
+    }
+}

JSTests/stress/resources/module-load-cut-short-by-termination/aimp/dep.js

@@ -0,0 +1,2 @@
+(globalThis.bodies ??= new Map).set(import.meta.filename, (globalThis.bodies.get(import.meta.filename) ?? 0) + 1);
+export const value = 40;

JSTests/stress/resources/module-load-cut-short-by-termination/aimp/root.js

@@ -0,0 +1,3 @@
+(globalThis.bodies ??= new Map).set(import.meta.filename, (globalThis.bodies.get(import.meta.filename) ?? 0) + 1);
+const dep = await import("./dep.js");
+export const value = dep.value + 1;

JSTests/stress/resources/module-load-cut-short-by-termination/await/dep.js

@@ -0,0 +1,3 @@
+(globalThis.bodies ??= new Map).set(import.meta.filename, (globalThis.bodies.get(import.meta.filename) ?? 0) + 1);
+await null;
+export const value = 40;

JSTests/stress/resources/module-load-cut-short-by-termination/await/root.js

@@ -0,0 +1,3 @@
+import { value as dep } from "./dep.js";
+(globalThis.bodies ??= new Map).set(import.meta.filename, (globalThis.bodies.get(import.meta.filename) ?? 0) + 1);
+export const value = dep + 1;

JSTests/stress/resources/module-load-cut-short-by-termination/chain/dep.js

@@ -0,0 +1,2 @@
+(globalThis.bodies ??= new Map).set(import.meta.filename, (globalThis.bodies.get(import.meta.filename) ?? 0) + 1);
+export const value = 40;

JSTests/stress/resources/module-load-cut-short-by-termination/chain/mid.js

@@ -0,0 +1,3 @@
+import { value as dep } from "./dep.js";
+(globalThis.bodies ??= new Map).set(import.meta.filename, (globalThis.bodies.get(import.meta.filename) ?? 0) + 1);
+export const value = dep + 1;

JSTests/stress/resources/module-load-cut-short-by-termination/chain/root.js

@@ -0,0 +1,3 @@
+import { value as mid } from "./mid.js";
+(globalThis.bodies ??= new Map).set(import.meta.filename, (globalThis.bodies.get(import.meta.filename) ?? 0) + 1);
+export const value = mid + 1;

JSTests/stress/resources/module-load-cut-short-by-termination/cycle/dep.js

@@ -0,0 +1,4 @@
+import { forty } from "./root.js";
+(globalThis.bodies ??= new Map).set(import.meta.filename, (globalThis.bodies.get(import.meta.filename) ?? 0) + 1);
+export function one() { return 1; }
+export const value = forty();

JSTests/stress/resources/module-load-cut-short-by-termination/cycle/root.js

@@ -0,0 +1,4 @@
+import { one } from "./dep.js";
+(globalThis.bodies ??= new Map).set(import.meta.filename, (globalThis.bodies.get(import.meta.filename) ?? 0) + 1);
+export function forty() { return 40; }
+export const value = forty() + one();

JSTests/stress/resources/module-load-cut-short-by-termination/diamond/dep.js

@@ -0,0 +1,2 @@
+(globalThis.bodies ??= new Map).set(import.meta.filename, (globalThis.bodies.get(import.meta.filename) ?? 0) + 1);
+export const value = 40;

JSTests/stress/resources/module-load-cut-short-by-termination/diamond/left.js

@@ -0,0 +1,3 @@
+import { value as dep } from "./dep.js";
+(globalThis.bodies ??= new Map).set(import.meta.filename, (globalThis.bodies.get(import.meta.filename) ?? 0) + 1);
+export const value = dep + 1;