From 0e72f219f1548ba8c6c8365a01676d10bad6f305 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:31:25 +0000 Subject: [PATCH 1/2] test(jsonl): generate the cases of json-from-utf8-bytes.test.ts as bytes, nest 100,000 deep The generator of the seeded cases writes UTF-8 bytes. Before, it put each text together from strings, one character at a time, and encoded the text at the end. The bytes of every case stay the same. The two tests of nesting beyond the native stack nest 100,000 levels, and 20,000 in a debug or ASAN build, in place of 600,000. The native stack holds about 12,000 levels in a release build, 2,000 in a debug build and 900 under AddressSanitizer. The tests of nesting, of `read` and of shared bytes compare `values`, `read`, `done` and `error` with exact values. The test of three-byte sequences also compares with JSON.parse. --- .../js/bun/jsonl/json-from-utf8-bytes.test.ts | 282 +++++++++++------- 1 file changed, 177 insertions(+), 105 deletions(-) diff --git a/test/js/bun/jsonl/json-from-utf8-bytes.test.ts b/test/js/bun/jsonl/json-from-utf8-bytes.test.ts index eebae6d18638..c19588b778c2 100644 --- a/test/js/bun/jsonl/json-from-utf8-bytes.test.ts +++ b/test/js/bun/jsonl/json-from-utf8-bytes.test.ts @@ -24,10 +24,14 @@ function generator(seed: number) { type Random = ReturnType; const encoder = new TextEncoder(); -const decode = (bytes: Uint8Array) => new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes); +const decoder = new TextDecoder("utf-8", { ignoreBOM: true }); +const decode = (bytes: Uint8Array) => decoder.decode(bytes); +const utf8 = (text: string) => encoder.encode(text); const lengths = [0, 1, 2, 3, 7, 8, 14, 15, 16, 17, 18, 26, 27, 28, 29, 30, 31, 32, 33, 63, 64, 65, 85, 86, 127, 128]; const longLengths = [129, 254, 255, 256, 257, 258, 300, 511, 513, 1000, 4097]; +// The pieces of a case are UTF-8 already, and a case is written from them byte by byte. Strings put together and encoded +// at the end give the same bytes, and take 40 times as long in a debug build. const alphabets = [ "abcdefghijklmnopqrstuvwxyz0123456789 _-./:", "abc \u00e9\u00fc\u00ff\u00a0\u0080", @@ -35,7 +39,7 @@ const alphabets = [ "a\u{10000}\u{1f600}\u{10ffff}", "\u4e2d\u6587\u5b57\u7b26\u3042\u3044\uac00", "a\u00e9\u4e2d\u{1f600}", -]; +].map(alphabet => Array.from(alphabet, utf8)); const escapes = [ '\\"', "\\\\", @@ -53,7 +57,8 @@ const escapes = [ "\\udc00", "\\u0000", "\\uFFFF", -]; +].map(utf8); +const escapesOneIn = [0, 0, 3, 20, 100]; const names = [ "a", "id", @@ -75,7 +80,12 @@ const names = [ "x".repeat(300), "\u4e2d".repeat(90), "\u00e9".repeat(130), -]; +].map(name => ({ + plain: utf8(JSON.stringify(name)), + escaped: utf8( + '"' + Array.from(name, unit => "\\u" + unit.charCodeAt(0).toString(16).padStart(4, "0")).join("") + '"', + ), +})); const numbers = [ "0", "-0", @@ -102,42 +112,64 @@ const numbers = [ "0.30000000000000004", "1".repeat(400), "0." + "3".repeat(400), -]; -const spaces = ["", "", "", "", " ", "\t", "\r", " ", " \t "]; +].map(utf8); +const keywords = ["true", "false", "null"].map(utf8); +const counts = [0, 1, 2, 3, 5, 9]; +const spaces = ["", "", "", "", " ", "\t", "\r", " ", " \t "].map(utf8); +const lineEnds = ["\n", "\n", "\n", "\r\n", "\n\n", " \n", ""].map(utf8); +const [quote, colon, comma, openBrace, closeBrace, openBracket, closeBracket] = Array.from('":,{}[]', utf8); -function stringLiteral(random: Random): string { - const alphabet = Array.from(random.pick(alphabets)); +// The bytes of the case in the making. +let caseBytes = new Uint8Array(1 << 16); +let caseLength = 0; +function write(piece: Uint8Array) { + if (caseLength + piece.length > caseBytes.length) { + const larger = new Uint8Array(2 * (caseLength + piece.length)); + larger.set(caseBytes); + caseBytes = larger; + } + for (let i = 0; i < piece.length; i++) caseBytes[caseLength++] = piece[i]; +} + +function writeString(random: Random) { + const alphabet = random.pick(alphabets); const length = random.chance(8) ? random.pick(longLengths) : random.pick(lengths); - const escapesOneIn = random.pick([0, 0, 3, 20, 100]); - let text = '"'; - for (let i = 0; i < length; i++) - text += escapesOneIn && random.chance(escapesOneIn) ? random.pick(escapes) : random.pick(alphabet); - return text + '"'; + // `| 0`: pick() returns values of every type. Without it the optimizing compiler does not know `oneIn` for a number, + // and `%` in chance(oneIn) becomes a call into the runtime, which records a stack trace on the ASAN lane of CI. + const oneIn = random.pick(escapesOneIn) | 0; + write(quote); + for (let i = 0; i < length; i++) write(oneIn && random.chance(oneIn) ? random.pick(escapes) : random.pick(alphabet)); + write(quote); } -function nameLiteral(random: Random): string { - if (random.chance(12)) return stringLiteral(random); +function writeName(random: Random) { + if (random.chance(12)) return writeString(random); const name = random.pick(names); - if (random.chance(15)) - return '"' + Array.from(name, unit => "\\u" + unit.charCodeAt(0).toString(16).padStart(4, "0")).join("") + '"'; - return JSON.stringify(name); + write(random.chance(15) ? name.escaped : name.plain); } -function valueText(random: Random, depth: number): string { +function writeValue(random: Random, depth: number) { const kind = random.below(depth > 4 ? 6 : 10); - const space = () => random.pick(spaces); - if (kind < 3) return stringLiteral(random); - if (kind < 5) return random.pick(numbers); - if (kind < 6) return random.pick(["true", "false", "null"]); - const count = random.pick([0, 1, 2, 3, 5, 9]); - const parts: string[] = []; - if (kind < 8) { - for (let i = 0; i < count; i++) - parts.push(space() + nameLiteral(random) + space() + ":" + space() + valueText(random, depth + 1) + space()); - return "{" + (count ? parts.join(",") : space()) + "}"; + if (kind < 3) return writeString(random); + if (kind < 5) return write(random.pick(numbers)); + if (kind < 6) return write(random.pick(keywords)); + const count = random.pick(counts); + const isObject = kind < 8; + write(isObject ? openBrace : openBracket); + if (!count) write(random.pick(spaces)); + for (let i = 0; i < count; i++) { + if (i) write(comma); + write(random.pick(spaces)); + if (isObject) { + writeName(random); + write(random.pick(spaces)); + write(colon); + write(random.pick(spaces)); + } + writeValue(random, depth + 1); + write(random.pick(spaces)); } - for (let i = 0; i < count; i++) parts.push(space() + valueText(random, depth + 1) + space()); - return "[" + (count ? parts.join(",") : space()) + "]"; + write(isObject ? closeBrace : closeBracket); } const brokenPieces: number[][] = [ @@ -195,8 +227,18 @@ const brokenPieces: number[][] = [ [0x6e, 0x75, 0x6c], ]; +// `bytes` with `removed` of them from `at` on replaced by `inserted`, as Array.prototype.splice() does it. +function spliced(bytes: Uint8Array, at: number, removed: number, inserted: readonly number[]): Uint8Array { + const rest = bytes.subarray(Math.min(at + removed, bytes.length)); + const result = new Uint8Array(at + inserted.length + rest.length); + result.set(bytes.subarray(0, at)); + result.set(inserted, at); + result.set(rest, at + inserted.length); + return result; +} + function damage(random: Random, bytes: Uint8Array): Uint8Array { - let list = Array.from(bytes); + let list: Uint8Array = bytes.slice(); for (let times = random.pick([1, 1, 1, 2, 4]); times > 0; times--) { const at = random.below(list.length + 1); switch (random.below(5)) { @@ -204,19 +246,19 @@ function damage(random: Random, bytes: Uint8Array): Uint8Array { list = list.slice(0, at); break; case 1: - list.splice(at, 0, ...random.pick(brokenPieces)); + list = spliced(list, at, 0, random.pick(brokenPieces)); break; case 2: - list.splice(at, random.pick([1, 1, 2, 3])); + list = spliced(list, at, random.pick([1, 1, 2, 3]), []); break; case 3: if (at < list.length) list[at] = random.below(256); break; default: - list.splice(at, random.pick([1, 2, 3]), ...random.pick(brokenPieces)); + list = spliced(list, at, random.pick([1, 2, 3]), random.pick(brokenPieces)); } } - return new Uint8Array(list); + return list; } type Outcome = { value: unknown } | { error: string }; @@ -245,10 +287,23 @@ function expectSame(actual: Outcome, expected: Outcome, what: string, seed: numb } function bytesOfCase(random: Random): Uint8Array { - const bytes = encoder.encode(random.pick(spaces) + valueText(random, 0) + random.pick(spaces)); + caseLength = 0; + write(random.pick(spaces)); + writeValue(random, 0); + write(random.pick(spaces)); + const bytes = caseBytes.slice(0, caseLength); return random.chance(2) ? damage(random, bytes) : bytes; } +// One value for a line of several: with a space where a carriage return would be. +function lineOfCase(random: Random): Uint8Array { + caseLength = 0; + writeValue(random, 1); + const line = caseBytes.slice(0, caseLength); + for (let at = line.indexOf(0x0d); at !== -1; at = line.indexOf(0x0d, at + 1)) line[at] = 0x20; + return line; +} + // What json() of a body does in front of the parser. A Response or a Request looks for an empty body before it drops the mark, a Blob behind that. function expectedOfBody(bytes: Uint8Array, kind: "blob" | "response"): Outcome { const whole = bytes; @@ -258,6 +313,11 @@ function expectedOfBody(bytes: Uint8Array, kind: "blob" | "response"): Outcome { } const beginsWithUTF16Mark = (bytes: Uint8Array) => bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe; +// The parser takes a native frame for each level of an array or an object until it has used the 5 MB of stack that +// JavaScriptCore allows (maxPerThreadStackUsage), and keeps a stack of its own from there on. On x64 that is after about +// 12,000 levels in a release build, 2,000 in a debug build and 900 under AddressSanitizer. +const beyondTheNativeStack = isDebug || isASAN ? 20_000 : 100_000; + describe("JSON from UTF-8 bytes against JSON.parse of the decoded text", () => { test("Bun.JSONL.parse and parseChunk of one value", () => { for (let seed = firstSeed; seed < firstSeed + cases; seed++) { @@ -286,18 +346,15 @@ describe("JSON from UTF-8 bytes against JSON.parse of the decoded text", () => { bytes, ); } - }, 120_000); + }, 30_000); test("Bun.JSONL.parseChunk of many lines, whole and in slices", () => { for (let seed = firstSeed; seed < firstSeed + cases / 4; seed++) { const random = generator(seed ^ 0x5bd1e995); const parts: Uint8Array[] = []; for (let lines = random.pick([1, 2, 3, 8, 30]); lines > 0; lines--) { - const line = encoder.encode(valueText(random, 1).replaceAll("\r", " ")); - parts.push( - random.chance(10) ? damage(random, line) : line, - encoder.encode(random.pick(["\n", "\n", "\n", "\r\n", "\n\n", " \n", ""])), - ); + const line = lineOfCase(random); + parts.push(random.chance(10) ? damage(random, line) : line, random.pick(lineEnds)); } const bytes = new Uint8Array(Buffer.concat(parts)); if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) continue; @@ -319,7 +376,7 @@ describe("JSON from UTF-8 bytes against JSON.parse of the decoded text", () => { ); } } - }, 120_000); + }, 30_000); test("short texts of three-byte sequences, well formed and not, at every place", () => { const sequences = [ @@ -351,6 +408,7 @@ describe("JSON from UTF-8 bytes against JSON.parse of the decoded text", () => { // The runner of CI has a collection follow every expect(), and this test compares 1,449 times. const fromBytes: unknown[] = []; const fromText: unknown[] = []; + const expected: unknown[] = []; for (const tail of ['"\n', '",1] \n', '" \n']) { for (let count = 1; count <= 6; count++) { for (let place = 0; place < count; place++) { @@ -359,24 +417,37 @@ describe("JSON from UTF-8 bytes against JSON.parse of the decoded text", () => { for (let i = 0; i < count; i++) inner.push(...(i === place ? sequence : plain)); const open = tail.startsWith('",') ? [0x5b, 0x22] : [0x22]; const bytes = new Uint8Array([...open, ...inner, ...encoder.encode(tail)]); - fromBytes.push([hex(bytes), Bun.JSONL.parse(bytes)]); - fromText.push([hex(bytes), Bun.JSONL.parse(decode(bytes))]); + const text = decode(bytes); + const what = hex(bytes); + fromBytes.push([what, Bun.JSONL.parse(bytes)]); + fromText.push([what, Bun.JSONL.parse(text)]); + expected.push([what, [JSON.parse(text)]]); } } } } - expect(fromBytes).toEqual(fromText); + expect(fromText).toEqual(expected); + expect(fromBytes).toEqual(expected); }); test("`read` counts three bytes for each U+FFFD that stands for an invalid sequence", () => { + const results: unknown[] = []; + const expected: unknown[] = []; for (const broken of brokenPieces.filter(piece => piece[0] >= 0x80)) { const first = new Uint8Array([0x22, 0x61, ...broken, 0x62, 0x22]); const bytes = new Uint8Array([...first, 0x0a, 0x5b, 0x31, 0x2c]); - const result = Bun.JSONL.parseChunk(bytes); - expect(result.values).toEqual([decode(first.subarray(1, first.length - 1))]); - expect(result.read).toBe(encoder.encode(decode(first)).length); - expect(result.done).toBe(false); + results.push([hex(bytes), Bun.JSONL.parseChunk(bytes)]); + expected.push([ + hex(bytes), + { + values: [decode(first.subarray(1, first.length - 1))], + read: encoder.encode(decode(first)).length, + done: false, + error: null, + }, + ]); } + expect(results).toStrictEqual(expected); }); test("json() of a Response, a Request and a Blob", async () => { @@ -408,7 +479,7 @@ describe("JSON from UTF-8 bytes against JSON.parse of the decoded text", () => { bytes, ); } - }, 120_000); + }, 30_000); test("json() of a file", async () => { using dir = tempDir("json-from-utf8-bytes", {}); @@ -426,27 +497,33 @@ describe("JSON from UTF-8 bytes against JSON.parse of the decoded text", () => { bytes, ); } - }, 120_000); + }, 30_000); test("arrays and objects nested beyond the native stack", () => { - for (const depth of [1000, 100000, 600000]) { - for (const [open, close] of [ - ["[", "]"], - ['{"\u00e9":', "}"], - ]) { + for (const depth of [1000, beyondTheNativeStack]) { + for (const [open, close, key] of [ + ["[", "]", 0], + ['{"\u00e9":', "}", "\u00e9"], + ] as const) { for (const leaf of ['"\u4e2d"', "1", ""]) { const text = open.repeat(depth) + leaf + close.repeat(leaf ? depth : 0); - const fromText = Bun.JSONL.parseChunk(text); - const fromBytes = Bun.JSONL.parseChunk(encoder.encode(text)); - expect([fromBytes.values.length, fromBytes.done, fromBytes.error?.message]).toEqual([ - fromText.values.length, - fromText.done, - fromText.error?.message, - ]); + for (const input of [text, encoder.encode(text)]) { + const { values, ...rest } = Bun.JSONL.parseChunk(input); + // A comparison would go down one native frame for each level as well, so the levels are counted. + let levels = 0; + let value: any = values[0]; + for (; typeof value === "object" && value !== null; value = value[key]) levels++; + const what = `${depth} of ${open} around ${leaf || "nothing"}, ${typeof input === "string" ? "text" : "bytes"}`; + expect([what, values.length, levels, value, rest]).toStrictEqual( + leaf + ? [what, 1, depth, JSON.parse(leaf), { read: input.length, done: true, error: null }] + : [what, 0, 0, undefined, { read: 0, done: false, error: null }], + ); + } } } } - }, 120_000); + }, 30_000); test("a line nested beyond the native stack among other lines", () => { const depthOf = (value: unknown) => { @@ -457,48 +534,38 @@ describe("JSON from UTF-8 bytes against JSON.parse of the decoded text", () => { const summaryOf = (values: unknown[]) => values.map(value => (Array.isArray(value) ? depthOf(value) : JSON.stringify(value))); const [first, second] = ['{"a":"\u00e9\u4e2d"}', '{"b":2}']; - for (const depth of [1000, 600000]) { + for (const depth of [1000, beyondTheNativeStack]) { const deep = "[".repeat(depth) + "]".repeat(depth); - for (const [lines, values] of [ - [ - [first, second, deep], - [first, second, depth], - ], - [ - [first, deep, second], - [first, depth, second], - ], - [ - [deep, first, second], - [depth, first, second], - ], - [ - [first, deep, deep], - [first, depth, depth], - ], - [ - [first, second, deep, '{"c":'], - [first, second, depth], - ], - [ - [first, second, deep, "}"], - [first, second, depth], - ], - ] as [string[], (string | number)[]][]) { + // The lines, what they give, and `done` and `error` of parseChunk(). The first three lines are values. + for (const [lines, values, done, error] of [ + [[first, second, deep], [first, second, depth], true, null], + [[first, deep, second], [first, depth, second], true, null], + [[deep, first, second], [depth, first, second], true, null], + [[first, deep, deep], [first, depth, depth], true, null], + [[first, second, deep, '{"c":'], [first, second, depth], false, null], + [[first, second, deep, "}"], [first, second, depth], false, "SyntaxError: Failed to parse JSONL"], + ] as [string[], (string | number)[], boolean, string | null][]) { const text = lines.join("\n") + "\n"; - const bytes = encoder.encode(text); - expect(summaryOf(Bun.JSONL.parse(bytes))).toEqual(values); - expect(summaryOf(Bun.JSONL.parse(text))).toEqual(values); - const [fromText, fromBytes] = [Bun.JSONL.parseChunk(text), Bun.JSONL.parseChunk(bytes)]; - expect(summaryOf(fromBytes.values)).toEqual(values); - expect([fromBytes.read, fromBytes.done, fromBytes.error?.message]).toEqual([ - encoder.encode(text.slice(0, fromText.read)).length, - fromText.done, - fromText.error?.message, - ]); + const valueLines = lines.slice(0, 3).join("\n"); + const shape = lines.map(line => (line === deep ? "deep" : line)).join(" "); + for (const [input, read] of [ + [text, valueLines.length], + [encoder.encode(text), encoder.encode(valueLines).length], + ] as const) { + const chunk = Bun.JSONL.parseChunk(input); + const what = `${depth}: ${shape}, ${typeof input === "string" ? "text" : "bytes"}`; + expect([ + what, + summaryOf(Bun.JSONL.parse(input)), + summaryOf(chunk.values), + chunk.read, + chunk.done, + chunk.error && chunk.error.name + ": " + chunk.error.message, + ]).toStrictEqual([what, values, values, read, done, error]); + } } } - }, 120_000); + }, 30_000); test("a name or a text seen as Latin-1 is not taken for the same bytes of UTF-8, nor the other way around", () => { for (const name of ["\u00e9", "caf\u00e9", "\u00c3\u00a9", "\u00ff\u00fe"]) { @@ -521,7 +588,12 @@ describe("JSON from UTF-8 bytes against JSON.parse of the decoded text", () => { const shared = new Uint8Array(new SharedArrayBuffer(1024 * 4)); const { written } = encoder.encodeInto(text + "\n", new Uint8Array(4096)); shared.set(encoder.encode(text + "\n")); - expect(Bun.JSONL.parseChunk(shared, 0, written).values).toEqual([JSON.parse(text)]); + expect(Bun.JSONL.parseChunk(shared, 0, written)).toStrictEqual({ + values: [JSON.parse(text)], + read: written - 1, + done: true, + error: null, + }); expect(Bun.JSONL.parse(shared.subarray(0, written))).toEqual([JSON.parse(text)]); }); }); From 7b9cbf8bfaddba82a314d5df306ad14192ebdbcb Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Fri, 30 Oct 2026 13:04:01 +0000 Subject: [PATCH 2/2] test(jsonl): build the nested texts with Buffer.alloc, drop the limit of the file test The two tests of nesting build their texts with Buffer.alloc() in place of String.prototype.repeat(), as test/CLAUDE.md asks. The Latin-1 encoding keeps each text a string of 8-bit characters, as repeat() made it, so the text goes to the same parser as before. The test of json() of a file takes 0.3 s in a debug build. It has no limit of its own any more. --- test/js/bun/jsonl/json-from-utf8-bytes.test.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/js/bun/jsonl/json-from-utf8-bytes.test.ts b/test/js/bun/jsonl/json-from-utf8-bytes.test.ts index c19588b778c2..37dde119c402 100644 --- a/test/js/bun/jsonl/json-from-utf8-bytes.test.ts +++ b/test/js/bun/jsonl/json-from-utf8-bytes.test.ts @@ -317,6 +317,8 @@ const beginsWithUTF16Mark = (bytes: Uint8Array) => bytes.length >= 2 && bytes[0] // JavaScriptCore allows (maxPerThreadStackUsage), and keeps a stack of its own from there on. On x64 that is after about // 12,000 levels in a release build, 2,000 in a debug build and 900 under AddressSanitizer. const beyondTheNativeStack = isDebug || isASAN ? 20_000 : 100_000; +// `text` of Latin-1 characters `count` times over, in a string of 8-bit characters as text.repeat(count) is one. +const repeated = (text: string, count: number) => Buffer.alloc(count * text.length, text, "latin1").toString("latin1"); describe("JSON from UTF-8 bytes against JSON.parse of the decoded text", () => { test("Bun.JSONL.parse and parseChunk of one value", () => { @@ -497,7 +499,7 @@ describe("JSON from UTF-8 bytes against JSON.parse of the decoded text", () => { bytes, ); } - }, 30_000); + }); test("arrays and objects nested beyond the native stack", () => { for (const depth of [1000, beyondTheNativeStack]) { @@ -506,7 +508,7 @@ describe("JSON from UTF-8 bytes against JSON.parse of the decoded text", () => { ['{"\u00e9":', "}", "\u00e9"], ] as const) { for (const leaf of ['"\u4e2d"', "1", ""]) { - const text = open.repeat(depth) + leaf + close.repeat(leaf ? depth : 0); + const text = repeated(open, depth) + leaf + repeated(close, leaf ? depth : 0); for (const input of [text, encoder.encode(text)]) { const { values, ...rest } = Bun.JSONL.parseChunk(input); // A comparison would go down one native frame for each level as well, so the levels are counted. @@ -535,7 +537,7 @@ describe("JSON from UTF-8 bytes against JSON.parse of the decoded text", () => { values.map(value => (Array.isArray(value) ? depthOf(value) : JSON.stringify(value))); const [first, second] = ['{"a":"\u00e9\u4e2d"}', '{"b":2}']; for (const depth of [1000, beyondTheNativeStack]) { - const deep = "[".repeat(depth) + "]".repeat(depth); + const deep = repeated("[", depth) + repeated("]", depth); // The lines, what they give, and `done` and `error` of parseChunk(). The first three lines are values. for (const [lines, values, done, error] of [ [[first, second, deep], [first, second, depth], true, null],