From 9240348c71b023d541bc23c95548b9b8dc312e6b Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Mon, 2 Nov 2026 04:54:26 +0000 Subject: [PATCH 1/2] json: copy the text between two escapes in bulk The parser decoded a string that has an escape one code point at a time. `parse_string_slow` now finds the next backslash with a search and copies the text before it in one step. It does so for UTF-8 text, in a document where no string so far has a control character: stage 1 sets a flag for that. From the first byte that is not UTF-8, or from the first bad escape, `decode_string_escapes` decodes the rest of the string into the same buffer. If the flag is set, it decodes the whole string. That loop does not change, so it still makes every error and every repair of text that is not UTF-8. A string of escapes only, such as "\n", needs no check for UTF-8. A string of 64 bytes or less is checked without a call to simdutf. A printed error through a 4.26 MB source map costs 21.6 M instructions, and 244.6 M before. --- bench/json-corpus/escaped-strings.mjs | 107 ++++ src/parsers/json_stage2.rs | 553 ++++++++++++++++++ test/bundler/bundler_loader.test.ts | 27 + .../bun/jsonc/json-differential-fuzz.test.ts | 6 +- test/js/bun/jsonc/jsonc.test.ts | 113 ++++ test/js/bun/resolve/jsonc.test.ts | 64 +- 6 files changed, 868 insertions(+), 2 deletions(-) create mode 100644 bench/json-corpus/escaped-strings.mjs diff --git a/bench/json-corpus/escaped-strings.mjs b/bench/json-corpus/escaped-strings.mjs new file mode 100644 index 000000000000..812d20d10247 --- /dev/null +++ b/bench/json-corpus/escaped-strings.mjs @@ -0,0 +1,107 @@ +// Writes documents whose strings have escapes into this directory, one per kind of text: +// `bun bench/json-corpus/escaped-strings.mjs`. Two benches read every document of the directory: +// `bun bench/json-corpus/jsonc-parse.mjs strings-` and `scripts/bench-json-rust.sh strings-`. +// A string with an escape is decoded into a copy, so the cost of these documents is the cost of that decoder. +import { writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; + +const dir = dirname(Bun.fileURLToPath(import.meta.url)); +const MB = 1024 * 1024; +const repeat = (unit, bytes) => Buffer.alloc(bytes - (bytes % Buffer.byteLength(unit)), unit).toString(); + +// A string with no escape next to one that has an escape: both are in one 64-byte block of the index. +const neighbours = (count, clean) => Array.from({ length: count }, (_, i) => (i % 2 ? "a\nb" : clean(i))); +const ascii = length => i => ("t" + i).padEnd(length, "x").slice(0, length); + +// The signature of a package version in a registry manifest: lines of 60 base64 characters that end with \r\n. +const base64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +function signature(seed) { + let text = "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v4.10.10\r\nComment: https://openpgpjs.org\r\n\r\n"; + for (let line = 0; line < 8; line++) { + for (let i = 0; i < 60; i++) text += base64[(seed = (seed * 1103515245 + 12345) >>> 0) >>> 26]; + text += "\r\n"; + } + return text + "=AbCd\r\n-----END PGP SIGNATURE-----\r\n"; +} + +const documents = { + // Source text as a source map holds it: a line break every 64 bytes. + "source-lines": { sourcesContent: [repeat("// " + Buffer.alloc(60, "p").toString() + "\n", MB)] }, + // One megabyte between two escapes. + "long-run": ["\n" + repeat("p", MB) + "\n"], + // Source text with tabs, CRLF line ends and quoted strings: two to four escapes next to each other. + "tabs-crlf": { + sourcesContent: [repeat('function f(a, b) {\r\n\tif (a === "x") {\r\n\t\treturn b + "y";\r\n\t}\r\n}\r\n', MB)], + }, + // JSON in a string: a quote every one to ten bytes. + "json-in-json": { + payload: JSON.stringify( + Array.from({ length: 12000 }, (_, i) => ({ id: i, name: "item" + i, ok: i % 2 === 0, tags: ["a", "b"] })), + ), + }, + "pgp-signatures": { + versions: Object.fromEntries( + Array.from({ length: 1200 }, (_, i) => [`1.0.${i}`, { dist: { "npm-signature": signature(i + 1) } }]), + ), + }, + // One character between two escapes, of one to four bytes. + "one-byte-runs": [repeat("a\n", MB / 2)], + "one-accent-runs": [repeat("é\n", MB / 2)], + "one-cjk-runs": [repeat("日\n", MB / 2)], + "one-emoji-runs": [repeat("😀\n", MB / 2)], + // Short strings, each with two or four escapes. + "short-strings": Object.fromEntries( + Array.from({ length: 16000 }, (_, i) => ["script:" + i, `tsc -p "./tsconfig.${i}.json" && echo "done ${i}"`]), + ), + // Strings of about 20 bytes with one escape each, in ASCII and not. + "short-one-escape": Array.from({ length: 40000 }, (_, i) => `line ${i} of the text\n`), + "short-accents": Array.from({ length: 40000 }, (_, i) => `café numéro ${i}\n`), + // Keys with an escape. A key is decoded like a value. + "escaped-keys": Object.fromEntries(Array.from({ length: 16000 }, (_, i) => [`key\t${i}`, i])), + "neighbours-8": neighbours(40000, ascii(8)), + "neighbours-16": neighbours(40000, ascii(16)), + "neighbours-40": neighbours(24000, ascii(40)), + "neighbours-72": neighbours(16000, ascii(72)), + // The same with text of 150 bytes that changes between ASCII and other characters. + "neighbours-mixed": neighbours( + 8000, + i => `${i} Tiếng Việt là ngôn ngữ chính thức của Việt Nam, hơn 90 triệu người dùng hằng ngày.`, + ), + // Nothing but escapes. + "escape-only": [repeat('\n\t"\\', MB / 2)], + // Text that is not ASCII: CJK lines, and accented Latin words between ASCII ones. + cjk: [repeat("日本語のテキスト中文文档安装运行测试代码注释\n", MB)], + accents: [repeat("// naïve café über " + Buffer.alloc(40, "p").toString() + "\n", MB)], + // No escape at all. A string of this document is not copied. + none: Object.fromEntries( + Array.from({ length: 16000 }, (_, i) => ["key" + i, "value number " + i + " in plain text"]), + ), +}; + +for (const [name, value] of Object.entries(documents)) { + writeFileSync(join(dir, `strings-${name}.json`), JSON.stringify(value)); +} + +// Every character as a \u escape, which JSON.stringify does not write. +const escaped = Array.from(repeat('日本語のテキスト\n\t"é"', (3 * MB) / 8), unit => + Array.from({ length: unit.length }, (_, i) => "\\u" + unit.charCodeAt(i).toString(16).padStart(4, "0")).join(""), +).join(""); +writeFileSync(join(dir, "strings-all-escaped.json"), `["${escaped}"]`); + +// Lines of bytes that are not UTF-8, each with a \n escape. Only a bench that reads bytes sees them as they are: +// `scripts/bench-json-rust.sh strings-invalid`, or an import of the file with `with { type: "jsonc" }`. +const invalid = { + // One byte that starts no sequence in a line of ASCII. + sparse: Buffer.concat([Buffer.from("// " + Buffer.alloc(57, "p").toString()), Buffer.from([0xff])]), + // The same in a line of CJK text. + cjk: Buffer.concat([Buffer.from("日本語のテキスト中文文档安装运行测试代码"), Buffer.from([0xff])]), + // Russian words in Windows-1251, where each letter is one byte that is not ASCII. + cp1251: Buffer.from( + Array.from("Привет мир, это текст в кодировке Windows-1251 ", c => c.charCodeAt(0) - (c >= "А" ? 0x350 : 0)), + ), +}; +for (const [name, unit] of Object.entries(invalid)) { + const line = Buffer.concat([unit, Buffer.from("\\n")]); + const body = Buffer.alloc(MB - (MB % line.length), line); + writeFileSync(join(dir, `strings-invalid-${name}.json`), Buffer.concat([Buffer.from('["'), body, Buffer.from('"]')])); +} diff --git a/src/parsers/json_stage2.rs b/src/parsers/json_stage2.rs index d33efe2ba627..ac40855bff8a 100644 --- a/src/parsers/json_stage2.rs +++ b/src/parsers/json_stage2.rs @@ -7,6 +7,7 @@ use bun_ast::{E, Expr, Loc, Log, Range, Source, usize2loc}; use bun_core::StackCheck; use bun_core::strings; use bun_core::strings::CodePoint; +use bun_simdutf_sys::simdutf; use crate::json::JSONOptions; use crate::json_index::{self as jidx, StructuralIndex}; @@ -513,6 +514,36 @@ impl<'a, 's, 'i> Parser<'a, 's, 'i> { #[cold] fn parse_string_slow(&mut self, body: &'s [u8]) -> PResult { + if self.idx.flags & jidx::FLAG_HAS_CTRL_IN_STRING != 0 { + return self.parse_string_by_code_point(body); + } + // No string so far has a control character, so a string with no backslash is its text. + let first = text_len(body); + if first == body.len() { + return Ok(E::EString::init(body)); + } + self.decode_string(body, first) + } + + /// `parse_string_slow` for a string with a backslash: `first` is the place of the first one. + #[inline(never)] + fn decode_string(&mut self, body: &'s [u8], first: usize) -> PResult { + let mut buf = core::mem::take(&mut self.scratch_str); + buf.clear(); + let plain = decode_plain_text(body, first, &mut buf); + if plain < body.len() { + // The rest starts with a byte that is not UTF-8, or with a bad escape. + self.decode_escapes(&body[plain..], &mut buf)?; + } + let owned = self.alloc_owned_str(&buf); + self.scratch_str = buf; + Ok(E::EString::init(owned.slice())) + } + + /// What `parse_string_slow` does if a string of the document has a control character. + #[cold] + #[inline(never)] + fn parse_string_by_code_point(&mut self, body: &'s [u8]) -> PResult { let mut first_special = None; for (k, &b) in body.iter().enumerate() { if b == b'\\' || b < 0x20 { @@ -1290,6 +1321,158 @@ fn push_codepoint(buf: &mut Vec, cp: CodePoint) { buf.extend_from_slice(&tmp[..n]); } +/// The length of the text at the start of `text`, up to a backslash. +#[inline(always)] +fn text_len(text: &[u8]) -> usize { + const ONES: u64 = 0x0101_0101_0101_0101; + // Below this many bytes the kernel reads no whole vector on some processors. + const KERNEL_MIN: usize = 64; + // A word read first makes one byte between two escapes slower than the loop this replaces. + let mut len = 0; + while len < 4 { + match text.get(len) { + Some(&byte) if byte != b'\\' => len += 1, + _ => return len, + } + } + while let Some(head) = text[len..].first_chunk::<8>() { + if len > 4 && text.len() - len >= KERNEL_MIN { + let rest = &text[len..]; + return len + strings::index_of_char_usize(rest, b'\\').unwrap_or(rest.len()); + } + let word = u64::from_le_bytes(*head) ^ (ONES * 0x5C); + // Sets the high bit of each backslash. Above the first one a borrow can set more. + let backslash = word.wrapping_sub(ONES) & !word & (ONES * 0x80); + if backslash != 0 { + return len + (backslash.trailing_zeros() / 8) as usize; + } + len += 8; + } + while text.get(len).is_some_and(|&byte| byte != b'\\') { + len += 1; + } + len +} + +/// The length of the UTF-8 text at the start of `body`: it ends before a byte of no character. +#[inline] +fn utf8_len(body: &[u8]) -> usize { + // The call to simdutf costs more than the check of a short body. + if body.len() <= 64 { + if body.is_ascii() { + return body.len(); + } + return match core::str::from_utf8(body) { + Ok(_) => body.len(), + Err(error) => error.valid_up_to(), + }; + } + let result = simdutf::validate::with_errors::utf8(body); + if result.is_successful() { + body.len() + } else { + result.count + } +} + +/// The value of the hex digits `digits`, if every byte is one. +#[inline] +fn hex_value(digits: Option<&[u8]>) -> Option { + let mut value: u32 = 0; + for &digit in digits? { + value = value * 16 + u32::from(bun_core::fmt::hex_digit_value(digit)?); + } + Some(value) +} + +/// Decodes the row of escapes at the backslash `body[pos]`. `Ok`: its end. `Err`: a bad escape. +#[inline(always)] +fn decode_escapes_at(body: &[u8], mut pos: usize, out: &mut Vec) -> Result { + loop { + debug_assert_eq!(body.get(pos), Some(&b'\\')); + let start = pos; + // `decode_string_escapes` drops a backslash that ends the body. + let Some(&escaped) = body.get(pos + 1) else { + return Ok(body.len()); + }; + pos += 2; + match escaped { + b'b' => out.push(0x08), + b'f' => out.push(0x0c), + b'n' => out.push(0x0a), + b'r' => out.push(0x0d), + b't' => out.push(0x09), + b'v' => out.push(0x0b), + b'8' | b'9' | b'"' | b'\\' | b'/' => out.push(escaped), + b'x' => { + let value = hex_value(body.get(pos..pos + 2)).ok_or(start)?; + pos += 2; + push_codepoint(out, value as CodePoint); + } + b'u' => { + let mut value = hex_value(body.get(pos..pos + 4)).ok_or(start)?; + pos += 4; + if strings::u16_is_lead(value as u16) + && body.get(pos..pos + 2) == Some(b"\\u".as_slice()) + && let Some(trail) = hex_value(body.get(pos + 2..pos + 6)) + && strings::u16_is_trail(trail as u16) + { + value = strings::u16_get_supplementary(value as u16, trail as u16); + pos += 6; + } + push_codepoint(out, value as CodePoint); + } + _ => return Err(start), + } + if body.get(pos) != Some(&b'\\') { + return Ok(pos); + } + } +} + +/// Decodes `body` up to a byte that is not UTF-8 or a bad escape, and returns where it stops. +#[inline(always)] +fn decode_plain_text(body: &[u8], first: usize, out: &mut Vec) -> usize { + out.reserve(body.len()); + let mut copied = 0; + if body.first() == Some(&b'\\') { + // Escapes are ASCII, and a string such as "\n" has no text to check. + match decode_escapes_at(body, 0, out) { + Ok(after) => copied = after, + Err(bad) => return bad, + } + if copied == body.len() { + return copied; + } + } + let end = utf8_len(body); + let mut pos = if copied == 0 { + first.min(end) + } else { + copied + text_len(&body[copied..end]) + }; + loop { + // `body[copied..pos]` is text, and `body[pos]` is a backslash if `pos` is not `end`. + match &body[copied..pos] { + [] => {} + [byte] => out.push(*byte), + text => out.extend_from_slice(text), + } + if pos == end { + return end; + } + match decode_escapes_at(body, pos, out) { + Ok(after) => copied = after, + Err(bad) => return bad, + } + // Many strings end with an escape. + if copied == end { + return end; + } + pos = copied + text_len(&body[copied..end]); + } +} + fn read_trail_surrogate_escape( iterator: &strings::CodepointIterator<'_>, iter: &mut strings::Cursor, @@ -1431,3 +1614,373 @@ pub(crate) fn decode_auto_quoted( decode_string_escapes::(&mut l, body, &mut buf)?; Ok(E::String::init(bump.alloc_slice_copy(&buf))) } + +#[cfg(test)] +mod tests { + use super::*; + + /// The decoded bytes, or the error and what it logged. + type Outcome = Result, String>; + + fn logged(err: crate::Error, log: &Log) -> String { + let messages: Vec = log + .msgs + .iter() + .map(|m| String::from_utf8_lossy(&m.data.text).into_owned()) + .collect(); + format!("{err:?}: {messages:?}") + } + + /// `decode_string_escapes` of `body`, after the text that `buf` has. + fn decode_by_code_point(allow_raw_control: bool, body: &[u8], mut buf: Vec) -> Outcome { + let source = Source::init_path_string("string.json", body); + let mut log = Log::init(); + let mut l = MiniLog { + log: &mut log, + source: &source, + prev_error_loc: Loc::EMPTY, + }; + let result = if allow_raw_control { + decode_string_escapes::(&mut l, body, &mut buf) + } else { + decode_string_escapes::(&mut l, body, &mut buf) + }; + result.map(|()| buf).map_err(|err| logged(err, &log)) + } + + /// What the parser makes of the one string of the document `["body"]`. + fn parsed(body: &[u8]) -> Outcome { + let doc = [b"[\"".as_slice(), body, b"\"]"].concat(); + bun_ast::initialize_store_or_reset(); + let _scope = bun_ast::StoreResetGuard::new(); + let source = Source::init_path_string("doc.json", &doc); + let mut log = Log::init(); + let parsed = crate::json::ParsedJson::parse_json(&source, &mut log) + .map_err(|err| logged(err, &log))?; + let Data::EArrayJSON(root) = parsed.root.data else { + panic!("the document is an array"); + }; + match root.get().items() { + [E::JsonValue::String(string)] => Ok(string.slice().to_vec()), + items => panic!("{} items in {:?}", items.len(), bstr::BStr::new(&doc)), + } + } + + /// What main makes of a string: it decodes one that has a backslash or a control character. + fn expected(body: &[u8]) -> Outcome { + if body.iter().any(|b| *b == b'\\' || *b < 0x20) { + decode_by_code_point(false, body, Vec::new()) + } else { + Ok(body.to_vec()) + } + } + + /// Whether `"body"` is one string token: each quote of the body is escaped. + fn is_one_string(body: &[u8]) -> bool { + let mut i = 0; + while i < body.len() { + match body[i] { + b'"' => return false, + b'\\' if i + 1 == body.len() => return false, + b'\\' => i += 2, + _ => i += 1, + } + } + true + } + + /// Every byte that takes its own path through a decoder, and its neighbours. + const BYTES: &[u8] = &[ + // Printable ASCII, the escape letters, hex digits of each case, and `g`. + b' ', b'a', b'Z', b'~', 0x7F, b'\\', b'"', b'\'', b'/', b'b', b'f', b'n', b'r', b't', b'v', + b'u', b'x', b'0', b'7', b'8', b'9', b'A', b'D', b'F', b'c', b'd', b'g', + // Control characters: the two that end a line, and three that do not. + 0x00, 0x09, 0x0A, 0x0D, 0x1F, + // Continuation bytes, then lead bytes: overlong (C0, C1, E0, F0), surrogate (ED), invalid. + 0x80, 0x8F, 0x90, 0x9F, 0xA0, 0xBF, 0xC0, 0xC1, 0xC2, 0xDF, 0xE0, 0xE1, 0xED, 0xEF, 0xF0, + 0xF1, 0xF4, 0xF5, 0xF7, 0xF8, 0xFF, + ]; + + /// Pieces that a body is built from, so that long and well-formed sequences occur often. + const PIECES: &[&[u8]] = &[ + b"plain text", + b"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + b"\\n", + b"\\t", + b"\\\"", + b"\\\\", + b"\\/", + b"\\b\\f\\r\\v", + b"\\8\\9", + b"\\0", + b"\\'", + b"\\x41", + b"\\xff", + b"\\x4", + b"\\u0041", + b"\\u00e9", + b"\\u65e5", + b"\\uD83D\\uDE00", + b"\\uD83D", + b"\\uDE00", + b"\\uD83D\\u0041", + b"\\uD83D\\uD83D\\uDE00", + b"\\uD83D\\uDE0", + b"\\uD83D\\n", + b"\\u12", + b"\\u12G4", + b"\\", + "é".as_bytes(), + "日本語".as_bytes(), + "😀".as_bytes(), + "\u{7f}\u{80}\u{7ff}\u{800}\u{ffff}\u{10000}\u{10ffff}".as_bytes(), + // A lead and a trail surrogate as WTF-8, alone and as a pair. + &[0xED, 0xA0, 0xBD], + &[0xED, 0xB8, 0x80], + &[0xED, 0xA0, 0xBD, 0xED, 0xB8, 0x80], + // Sequences cut short, overlong, or past U+10FFFF. + &[0xC3], + &[0xE6, 0x97], + &[0xF0, 0x9F, 0x98], + &[0xC0, 0x80], + &[0xE0, 0x80, 0x80], + &[0xF0, 0x80, 0x80, 0x80], + &[0xF4, 0x90, 0x80, 0x80], + &[0x80], + &[0xFF], + &[0x00], + &[0x0A], + &[0x1F], + ]; + + struct Rng(u64); + + impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } + + /// A body of up to ten parts: a byte of `BYTES`, a run of ASCII, or one of `PIECES`. + fn body(&mut self, run: usize, body: &mut Vec) { + body.clear(); + for _ in 0..self.below(10) { + match self.below(8) { + 0 => body.push(BYTES[self.below(BYTES.len())]), + 1 => { + let len = self.below(run); + body.extend(core::iter::repeat_n(b'a' + (len % 26) as u8, len)); + } + _ => body.extend_from_slice(PIECES[self.below(PIECES.len())]), + } + } + } + } + + /// Calls `check` with every body of one or two bytes, many of three, and random ones. + fn for_each_body(mut check: impl FnMut(&[u8])) { + check(b""); + for a in 0..=255u8 { + check(&[a]); + for b in 0..=255u8 { + check(&[a, b]); + } + } + for &a in BYTES { + for &b in BYTES { + for &c in BYTES { + check(&[a, b, c]); + } + } + } + let mut rng = Rng(0x9E3779B97F4A7C15); + let mut body = Vec::new(); + for _ in 0..200_000 { + body.clear(); + for _ in 0..rng.below(14) { + body.push(BYTES[rng.below(BYTES.len())]); + } + check(&body); + } + for _ in 0..300_000 { + // The runs cross the widths that the search for a backslash reads in one step. + rng.body(150, &mut body); + check(&body); + } + } + + #[test] + fn text_len_matches_a_byte_loop() { + let check = |text: &[u8]| { + let expected = text.iter().take_while(|b| **b != b'\\').count(); + assert_eq!(text_len(text), expected, "{:?}", bstr::BStr::new(text)); + }; + // Every byte value at each place that is read by a branch or in a word, alone or in a pair. + for filler in [b'a', 0xE6] { + let mut text = [filler; 30]; + for first in 0..28 { + for value in 0..=255u8 { + text.fill(filler); + text[first] = value; + check(&text); + for second in first + 1..28 { + for other in [0x00, 0x1F, 0x5B, 0x5C, 0x5D, 0x7F, 0x80, 0xDC, 0xFF] { + text[second] = other; + check(&text); + } + text[second] = filler; + } + } + } + } + // A backslash at each place of a text of each length up to 200 bytes, and no backslash. + let text = [b'a'; 200]; + for len in 0..=text.len() { + check(&text[..len]); + for at in 0..len { + let mut text = text; + text[at] = b'\\'; + check(&text[..len]); + } + } + } + + #[test] + fn utf8_len_is_the_same_for_a_short_and_a_long_body() { + // A short body is checked without simdutf. 70 bytes of ASCII after it make it long. + let mut long = Vec::new(); + for_each_body(|body| { + long.clear(); + long.extend_from_slice(body); + long.resize(body.len() + 70, b'a'); + let len = utf8_len(body); + let expected = if len == body.len() { long.len() } else { len }; + assert_eq!( + utf8_len(&long), + expected, + "body {:?}", + bstr::BStr::new(body) + ); + }); + } + + #[test] + fn bulk_decoder_matches_the_code_point_decoder() { + let (mut whole, mut part, mut rejected) = (0, 0, 0); + for_each_body(|body| { + let mut in_bulk = Vec::new(); + let plain = decode_plain_text(body, text_len(body), &mut in_bulk); + // The bulk decoder takes a control character as text, so the other decoder does too. + let actual = decode_by_code_point(true, &body[plain..], in_bulk); + let expected = decode_by_code_point(true, body, Vec::new()); + assert_eq!( + actual, + expected, + "body {:?}, in bulk up to {plain}", + bstr::BStr::new(body) + ); + match actual { + Ok(_) if plain == body.len() => whole += 1, + Ok(_) => part += 1, + Err(_) => rejected += 1, + } + }); + assert!( + whole > 50_000 && part > 50_000 && rejected > 50_000, + "{whole} decoded in bulk, {part} in part, {rejected} rejected" + ); + } + + #[test] + fn parser_decodes_like_the_code_point_decoder() { + let (mut decoded, mut rejected) = (0, 0); + for_each_body(|body| { + if !is_one_string(body) { + return; + } + let (expected, actual) = (expected(body), parsed(body)); + assert!( + expected == actual, + "body {:?}\n by code point: {expected:?}\n parser: {actual:?}", + bstr::BStr::new(body) + ); + match expected { + Ok(_) => decoded += 1, + Err(_) => rejected += 1, + } + }); + assert!( + decoded > 50_000 && rejected > 50_000, + "{decoded} decoded, {rejected} rejected" + ); + } + + #[test] + fn parser_decodes_long_strings_like_the_code_point_decoder() { + let mut rng = Rng(0xD1B54A32D192ED03); + let (mut part, mut body) = (Vec::new(), Vec::new()); + let (mut decoded, mut rejected) = (0, 0); + // Stage 1 reads 8192 bytes at a time, and the tape holds strings in buffers of 4096 bytes. + for target in [4095, 4096, 4097, 8191, 8192, 8193, 20_000] { + for _ in 0..200 { + // One body in eight has parts that are not a string, at any place in it. + let only_what_decodes = rng.below(8) != 0; + body.clear(); + while body.len() < target { + rng.body(52, &mut part); + if is_one_string(&part) && (!only_what_decodes || expected(&part).is_ok()) { + body.extend_from_slice(&part); + } + } + let (expected, actual) = (expected(&body), parsed(&body)); + assert!( + expected == actual, + "body {:?}\n by code point: {:?}\n parser: {:?}", + bstr::BStr::new(&body), + expected.as_ref().map(Vec::len), + actual.as_ref().map(Vec::len), + ); + match expected { + Ok(_) => decoded += 1, + Err(_) => rejected += 1, + } + } + } + assert!( + decoded > 1_000 && rejected > 100, + "{decoded} decoded, {rejected} rejected" + ); + } + + #[test] + fn decoded_text_of_known_bodies() { + assert_eq!(parsed(b"a\\nb"), Ok(b"a\nb".to_vec())); + assert_eq!( + parsed(b"\\b\\f\\n\\r\\t\\v\\8\\9\\\"\\\\\\/"), + Ok(b"\x08\x0c\n\r\t\x0b89\"\\/".to_vec()) + ); + assert_eq!(parsed(b"\\x41\\xe9\\u65e5"), Ok("Aé日".as_bytes().to_vec())); + assert_eq!(parsed(b"\\uD83D\\uDE00"), Ok("😀".as_bytes().to_vec())); + assert_eq!(parsed(b"\\uD83D"), Ok(vec![0xED, 0xA0, 0xBD])); + // A byte that starts no sequence is its code point. A lead byte without its tail is U+FFFD. + assert_eq!( + parsed(b"\\n\x80\xE6\x97"), + Ok("\n\u{80}\u{FFFD}\u{97}".as_bytes().to_vec()) + ); + assert!( + parsed(b"a\nb") + .unwrap_err() + .contains("Unterminated string literal") + ); + assert!(parsed(b"a\x01b").unwrap_err().contains("Syntax Error")); + assert!(parsed(b"\\n\x01").unwrap_err().contains("Syntax Error")); + assert!(parsed(b"\\q").unwrap_err().contains("Syntax Error")); + assert!(parsed(b"\\u12G4").unwrap_err().contains("Syntax Error")); + } +} diff --git a/test/bundler/bundler_loader.test.ts b/test/bundler/bundler_loader.test.ts index 289cbb6eae2b..908c001b148e 100644 --- a/test/bundler/bundler_loader.test.ts +++ b/test/bundler/bundler_loader.test.ts @@ -170,6 +170,33 @@ describe("bundler", async () => { }, }); + { + // A string of a JSON file that has an escape is decoded. If its bytes are not UTF-8, a lead byte without its tail + // becomes U+FFFD, and a byte that starts no sequence becomes its code point. + const file: Record = {}; + const read: Record = {}; + // The parser finds the next escape by a branch, by a word of 8 bytes, and by a vector. The bytes are behind each. + for (const distance of [0, 1, 3, 4, 7, 11, 12, 19, 35, 67, 131, 300]) { + file["d" + distance] = "\\n" + "a".repeat(distance) + "\xE9z \x80\xA0 \xE2\x82A\\t"; + read["d" + distance] = "\n" + "a".repeat(distance) + "\uFFFDz \x80\xA0 \uFFFD\x82A\t"; + } + // The text of this string is longer than its bytes in the file: U+FFFD has three bytes. + file.long = "\\n" + "caf\xE9 au lait \xA9 ".repeat(400); + read.long = "\n" + "caf\uFFFD au lait \xA9 ".repeat(400); + const entries = Object.entries(file).map(([key, value]) => `"${key}":"${value}"`); + itBundled("bun/loader-json-escaped-string-not-utf8", { + target: "bun", + files: { + "/entry.ts": /* js */ ` + import json from "./a.json"; + process.stdout.write(JSON.stringify(json)); + `, + "/a.json": Buffer.from(`{${entries.join(",")}}`, "latin1"), + }, + run: { stdout: JSON.stringify(read) }, + }); + } + itBundled("bun/loader-json-proto-key-is-own-property", { target: "bun", files: { diff --git a/test/js/bun/jsonc/json-differential-fuzz.test.ts b/test/js/bun/jsonc/json-differential-fuzz.test.ts index d62b34e9ac7f..6035dce82b43 100644 --- a/test/js/bun/jsonc/json-differential-fuzz.test.ts +++ b/test/js/bun/jsonc/json-differential-fuzz.test.ts @@ -52,7 +52,7 @@ function genStringToken(rng: Rng): string { let body = ""; const chunks = rng.int(7); for (let i = 0; i < chunks; i++) { - switch (rng.int(8)) { + switch (rng.int(9)) { case 0: case 1: case 2: { @@ -75,6 +75,10 @@ function genStringToken(rng: Rng): string { case 7: body += rng.pick(["", "__proto__", "constructor", "0", "1e3", "true", "null", "//", "/*", "*/"]); break; + case 8: + // Text between two escapes that is longer than the 4, 8 and 16 to 64 bytes the parser reads in one step. + body += rng.pick(["a", "~", "é", "日", "🚀"]).repeat(rng.range(7, 150)); + break; } } return '"' + body + '"'; diff --git a/test/js/bun/jsonc/jsonc.test.ts b/test/js/bun/jsonc/jsonc.test.ts index 38a9b48a9c10..e664621c3bf0 100644 --- a/test/js/bun/jsonc/jsonc.test.ts +++ b/test/js/bun/jsonc/jsonc.test.ts @@ -349,6 +349,119 @@ test("Bun.JSONC.parse matches JSON.parse on escape-heavy strings", () => { } }); +test("Bun.JSONC.parse matches JSON.parse for text of any length around an escape", () => { + // The parser copies the text between two escapes in one step. To find the next escape it reads 4 bytes, then words + // of 8 bytes, then vectors once 64 bytes are left. It checks a string of 64 bytes or less for UTF-8 in another way. + // So the text has each length around those, and the escape is at each end and in the middle of it. + const escapes = [ + "\\n", + '\\"', + "\\\\", + "\\/", + "\\u00e9", + "\\u65E5", + "\\ud83d\\ude00", + "\\ud83d", + "\\ude00", + "\\ud83d\\u0041", + "\\b\\f\\r\\t", + ]; + const texts = [ + (n: number) => "a".repeat(n), + (n: number) => "é".repeat(n), + (n: number) => "\x7f~ ".repeat(n).slice(0, n), + (n: number) => "日本語".repeat(n).slice(0, n), + (n: number) => "a🚀".repeat(n).slice(0, n * 3), + (n: number) => "aé".repeat(n).slice(0, n), + ]; + const docsOf = (run: string, escape: string) => [ + `"${escape}${run}"`, + `"${run}${escape}"`, + `"${run}${escape}${run}"`, + `"${escape}${run}${escape}${escape}${run}${escape}"`, + `{"${run}${escape}":["${run}","${escape}${run}"]}`, + ]; + // One document for each length holds the strings of every text and escape: a call costs more than its bytes. + const wrong: string[] = []; + const check = (docs: string[]) => { + const doc = `[${docs.join(",")}]`; + const [actual, expected] = [Bun.JSONC.parse(doc) as unknown[], JSON.parse(doc) as unknown[]]; + // Compare as JSON text, which spells a lone surrogate as an escape. + if (JSON.stringify(actual) === JSON.stringify(expected)) return; + // Name each string that differs. + for (const [i, value] of expected.entries()) { + const [got, want] = [JSON.stringify(actual[i]), JSON.stringify(value)]; + if (got !== want) wrong.push(`${docs[i].slice(0, 80)} (${docs[i].length} chars): ${got?.slice(0, 80)}`); + } + if (actual.length !== expected.length) wrong.push(`${actual.length} values, not ${expected.length}`); + }; + for (const length of [0, 1, 3, 4, 5, 11, 12, 13, 20, 27, 28, 29, 59, 60, 61, 62, 63, 64, 65, 66, 75, 76, 77, 300]) { + check(texts.flatMap(text => escapes.flatMap(escape => docsOf(text(length), escape)))); + } + // 4096 bytes are one buffer of strings in the parser, and it reads the document 8192 bytes at a time. That + // depends on the number of bytes and not on the characters, so these lengths have two of the texts. + for (const length of [4095, 4096, 9000]) { + for (const text of texts.slice(0, 2)) for (const doc of docsOf(text(length), "\\n")) check([doc]); + } + expect(wrong).toEqual([]); + // JSON.parse does not take these escapes, and the parser of bun does. + expect(Bun.JSONC.parse('"\\x41\\xe9"')).toBe("Aé"); + expect(Bun.JSONC.parse('"\\8\\9\\v"')).toBe("89\v"); + expect(Bun.JSONC.parse(`"${"é".repeat(40)}\\x41\\8\\v${"a".repeat(40)}"`)).toBe( + `${"é".repeat(40)}A8\v${"a".repeat(40)}`, + ); +}); + +test("Bun.JSONC.parse rejects a bad escape or a control character after text of any length", () => { + const SYNTAX = "JSONC Parse error: Syntax Error"; + const LINE_BREAK = "JSONC Parse error: Unterminated string literal"; + const bads: [bad: string, message: string][] = [ + // One of each kind first: a string of thousands of bytes has only these five. + ["\\q", SYNTAX], + ["\n", LINE_BREAK], + ["\t", SYNTAX], + // The first of two that are bad makes the message. + ["\n\\q", LINE_BREAK], + ["\\q\n", SYNTAX], + ["\\u12", SYNTAX], + ["\\u12G4", SYNTAX], + ["\\x4", SYNTAX], + ["\\0", SYNTAX], + ["\\'", SYNTAX], + ["\r", LINE_BREAK], + ["\x00", SYNTAX], + ["\x1f", SYNTAX], + ]; + // Each document is also not JSON: it has a control character or a bad escape in a string. + const wrong: string[] = []; + for (const length of [0, 1, 4, 5, 12, 13, 28, 29, 60, 61, 62, 63, 64, 65, 76, 77, 4095, 4096, 9000]) { + // The text has no hex digit, so it does not complete an escape that is cut short. + for (const run of ["z".repeat(length), "é".repeat(length)]) { + for (const [bad, message] of length < 4095 ? bads : bads.slice(0, 5)) { + for (const doc of [ + `"${run}${bad}"`, + `"\\n${run}${bad}${run}"`, + `{"\\n${run}${bad}":1}`, + // Another string has the escape. The parser then looks for a control character in no string by itself: + // it takes the word of the index for it. A comment or a single quote makes the index read byte by byte. + `["${run}\\n","${run}${bad}"]`, + `['${run}\\n',"${run}${bad}"]`, + `/* c */["${run}\\n","${run}${bad}"]`, + ]) { + let got = "no error"; + try { + Bun.JSONC.parse(doc); + } catch (error) { + got = (error as Error).message; + } + if (got !== message) wrong.push(`${JSON.stringify(doc.slice(0, 60))} (${doc.length} chars): ${got}`); + } + } + } + } + expect(wrong).toEqual([]); +}); + test("Bun.JSONC.parse matches JSON.parse on lone and paired surrogate escapes", () => { for (const doc of [ '"\\ud800"', diff --git a/test/js/bun/resolve/jsonc.test.ts b/test/js/bun/resolve/jsonc.test.ts index fd73f3b7714a..88adb8b44306 100644 --- a/test/js/bun/resolve/jsonc.test.ts +++ b/test/js/bun/resolve/jsonc.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test"; -import { bunRun, tempDir } from "harness"; +import { bunEnv, bunExe, bunRun, tempDir } from "harness"; import { join } from "path"; test.concurrent("empty jsonc - package.json", async () => { await using dir = tempDir("jsonc", { @@ -57,3 +57,65 @@ test.concurrent("imported JSON strings match JSON.parse exactly (escapes, lone s }); expect(await bunRun(join(dir, "index.ts"))).toSpawn(); }); + +test.concurrent("a .jsonc import repairs bytes that are not UTF-8 in a string with an escape", async () => { + // A string with an escape is decoded. One that is not UTF-8 is decoded one code point at a time, and repaired. + const escaped = (bytes: number[]) => Buffer.concat([Buffer.from("a\\n"), Buffer.from(bytes), Buffer.from("z\\t")]); + const decoded = (codePoints: number[]) => [0x61, 0x0a, ...codePoints, 0x7a, 0x09]; + const cases: Record = { + // A byte that starts no sequence is taken as its code point. + continuation: [escaped([0x80]), decoded([0x80])], + lastContinuation: [escaped([0xbf]), decoded([0xbf])], + notALead: [escaped([0xff]), decoded([0xff])], + // A sequence that is cut short, or overlong, is U+FFFD for its first byte. + leadThenAscii: [escaped([0xc3]), decoded([0xfffd])], + cutSequence: [escaped([0xe6, 0x97]), decoded([0xfffd, 0x97])], + overlong: [escaped([0xc0, 0x80]), decoded([0xfffd, 0x80])], + // Half of a surrogate pair stays as it is. + surrogateHalf: [escaped([0xed, 0xa0, 0xbd]), decoded([0xd83d])], + // One byte that is not UTF-8 changes nothing in the characters around it. + twoStretches: [escaped([0xc3, 0xa9, 0x20, 0xff, 0xc3, 0xa9]), decoded([0xe9, 0x20, 0xff, 0xe9])], + wellFormed: [escaped([0xc3, 0xa9, 0xf0, 0x9f, 0x9a, 0x80]), decoded([0xe9, 0x1f680])], + // A string that is longer than a buffer of the parser, with the byte between two runs of text. + long: [ + Buffer.concat([ + Buffer.alloc(5000, "p"), + Buffer.from([0xff, 0xc3, 0xa9]), + Buffer.alloc(5000, "q"), + Buffer.from("\\n"), + ]), + [...Array(5000).fill(0x70), 0xff, 0xe9, ...Array(5000).fill(0x71), 0x0a], + ], + // Text that is longer after the repair: each byte becomes a character of two bytes. + grows: [Buffer.concat([Buffer.alloc(5000, 0xff), Buffer.from("\\n")]), [...Array(5000).fill(0xff), 0x0a]], + }; + // The parser finds the next escape by a branch, by a word of 8 bytes, and by a vector. The byte is behind each. + for (const distance of [1, 3, 4, 7, 11, 12, 19, 35, 67, 131]) { + const text = Array(distance).fill(0x62); + cases["after" + distance] = [escaped([...text, 0xe9, ...text]), decoded([...text, 0xfffd, ...text])]; + } + const parts = [Buffer.from("{")]; + for (const [name, [body]] of Object.entries(cases)) { + parts.push(Buffer.from(`"${name}":"`), body, Buffer.from(`",`)); + } + parts.push(Buffer.from(`}`)); + using dir = tempDir("jsonc-not-utf8", { + "weird.jsonc": Buffer.concat(parts), + "index.ts": ` + import file from "./weird.jsonc"; + const codePoints = (text: string) => Array.from(text, c => c.codePointAt(0)); + console.log(JSON.stringify(Object.entries(file).map(([name, text]) => [name, codePoints(text as string)]))); + `, + }); + await using proc = Bun.spawn({ + cmd: [bunExe(), "index.ts"], + env: bunEnv, + cwd: String(dir), + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toBe(""); + expect(JSON.parse(stdout)).toEqual(Object.entries(cases).map(([name, [, codePoints]]) => [name, codePoints])); + expect(exitCode).toBe(0); +}); From c19c36ddd4dc0309e3d72f2afa571ca8d71c23f0 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Mon, 2 Nov 2026 06:41:16 +0000 Subject: [PATCH 2/2] json: test the escape table of both decoders, build test strings with Buffer.alloc The parser has two decoders for a string with an escape, and each one has its own table of escapes. A new test in jsonc.test.ts reads every character after a backslash through both: alone, where the bulk decoder runs, and before a string with a line break, where the code point decoder runs. Both must take the same escapes. The new tests build a run of equal characters with Buffer.alloc, as test/CLAUDE.md asks. The strings do not change. The doc comment of text_len says why it reads short text itself. --- src/parsers/json_stage2.rs | 2 +- test/bundler/bundler_loader.test.ts | 10 ++- .../bun/jsonc/json-differential-fuzz.test.ts | 3 +- test/js/bun/jsonc/jsonc.test.ts | 80 ++++++++++++++++--- 4 files changed, 80 insertions(+), 15 deletions(-) diff --git a/src/parsers/json_stage2.rs b/src/parsers/json_stage2.rs index ac40855bff8a..e89e676f7525 100644 --- a/src/parsers/json_stage2.rs +++ b/src/parsers/json_stage2.rs @@ -1321,7 +1321,7 @@ fn push_codepoint(buf: &mut Vec, cp: CodePoint) { buf.extend_from_slice(&tmp[..n]); } -/// The length of the text at the start of `text`, up to a backslash. +/// The length of `text` up to a backslash. It reads short text itself: `index_of_char` was slower. #[inline(always)] fn text_len(text: &[u8]) -> usize { const ONES: u64 = 0x0101_0101_0101_0101; diff --git a/test/bundler/bundler_loader.test.ts b/test/bundler/bundler_loader.test.ts index 908c001b148e..9b8d5c4313b3 100644 --- a/test/bundler/bundler_loader.test.ts +++ b/test/bundler/bundler_loader.test.ts @@ -177,12 +177,14 @@ describe("bundler", async () => { const read: Record = {}; // The parser finds the next escape by a branch, by a word of 8 bytes, and by a vector. The bytes are behind each. for (const distance of [0, 1, 3, 4, 7, 11, 12, 19, 35, 67, 131, 300]) { - file["d" + distance] = "\\n" + "a".repeat(distance) + "\xE9z \x80\xA0 \xE2\x82A\\t"; - read["d" + distance] = "\n" + "a".repeat(distance) + "\uFFFDz \x80\xA0 \uFFFD\x82A\t"; + const text = Buffer.alloc(distance, "a").toString(); + file["d" + distance] = "\\n" + text + "\xE9z \x80\xA0 \xE2\x82A\\t"; + read["d" + distance] = "\n" + text + "\uFFFDz \x80\xA0 \uFFFD\x82A\t"; } // The text of this string is longer than its bytes in the file: U+FFFD has three bytes. - file.long = "\\n" + "caf\xE9 au lait \xA9 ".repeat(400); - read.long = "\n" + "caf\uFFFD au lait \xA9 ".repeat(400); + const repeated = (text: string) => Buffer.alloc(400 * Buffer.byteLength(text), text).toString(); + file.long = "\\n" + repeated("caf\xE9 au lait \xA9 "); + read.long = "\n" + repeated("caf\uFFFD au lait \xA9 "); const entries = Object.entries(file).map(([key, value]) => `"${key}":"${value}"`); itBundled("bun/loader-json-escaped-string-not-utf8", { target: "bun", diff --git a/test/js/bun/jsonc/json-differential-fuzz.test.ts b/test/js/bun/jsonc/json-differential-fuzz.test.ts index 6035dce82b43..62b1e0012b86 100644 --- a/test/js/bun/jsonc/json-differential-fuzz.test.ts +++ b/test/js/bun/jsonc/json-differential-fuzz.test.ts @@ -77,7 +77,8 @@ function genStringToken(rng: Rng): string { break; case 8: // Text between two escapes that is longer than the 4, 8 and 16 to 64 bytes the parser reads in one step. - body += rng.pick(["a", "~", "é", "日", "🚀"]).repeat(rng.range(7, 150)); + const text = rng.pick(["a", "~", "é", "日", "🚀"]); + body += Buffer.alloc(rng.range(7, 150) * Buffer.byteLength(text), text).toString(); break; } } diff --git a/test/js/bun/jsonc/jsonc.test.ts b/test/js/bun/jsonc/jsonc.test.ts index e664621c3bf0..3256578a38bf 100644 --- a/test/js/bun/jsonc/jsonc.test.ts +++ b/test/js/bun/jsonc/jsonc.test.ts @@ -349,6 +349,9 @@ test("Bun.JSONC.parse matches JSON.parse on escape-heavy strings", () => { } }); +// `count` copies of `text`. "x".repeat(count) is slow in a debug build. +const repeated = (text: string, count: number) => Buffer.alloc(count * Buffer.byteLength(text), text).toString(); + test("Bun.JSONC.parse matches JSON.parse for text of any length around an escape", () => { // The parser copies the text between two escapes in one step. To find the next escape it reads 4 bytes, then words // of 8 bytes, then vectors once 64 bytes are left. It checks a string of 64 bytes or less for UTF-8 in another way. @@ -367,12 +370,12 @@ test("Bun.JSONC.parse matches JSON.parse for text of any length around an escape "\\b\\f\\r\\t", ]; const texts = [ - (n: number) => "a".repeat(n), - (n: number) => "é".repeat(n), - (n: number) => "\x7f~ ".repeat(n).slice(0, n), - (n: number) => "日本語".repeat(n).slice(0, n), - (n: number) => "a🚀".repeat(n).slice(0, n * 3), - (n: number) => "aé".repeat(n).slice(0, n), + (n: number) => repeated("a", n), + (n: number) => repeated("é", n), + (n: number) => repeated("\x7f~ ", n).slice(0, n), + (n: number) => repeated("日本語", n).slice(0, n), + (n: number) => repeated("a🚀", n).slice(0, n * 3), + (n: number) => repeated("aé", n).slice(0, n), ]; const docsOf = (run: string, escape: string) => [ `"${escape}${run}"`, @@ -407,8 +410,8 @@ test("Bun.JSONC.parse matches JSON.parse for text of any length around an escape // JSON.parse does not take these escapes, and the parser of bun does. expect(Bun.JSONC.parse('"\\x41\\xe9"')).toBe("Aé"); expect(Bun.JSONC.parse('"\\8\\9\\v"')).toBe("89\v"); - expect(Bun.JSONC.parse(`"${"é".repeat(40)}\\x41\\8\\v${"a".repeat(40)}"`)).toBe( - `${"é".repeat(40)}A8\v${"a".repeat(40)}`, + expect(Bun.JSONC.parse(`"${repeated("é", 40)}\\x41\\8\\v${repeated("a", 40)}"`)).toBe( + `${repeated("é", 40)}A8\v${repeated("a", 40)}`, ); }); @@ -436,7 +439,7 @@ test("Bun.JSONC.parse rejects a bad escape or a control character after text of const wrong: string[] = []; for (const length of [0, 1, 4, 5, 12, 13, 28, 29, 60, 61, 62, 63, 64, 65, 76, 77, 4095, 4096, 9000]) { // The text has no hex digit, so it does not complete an escape that is cut short. - for (const run of ["z".repeat(length), "é".repeat(length)]) { + for (const run of [repeated("z", length), repeated("é", length)]) { for (const [bad, message] of length < 4095 ? bads : bads.slice(0, 5)) { for (const doc of [ `"${run}${bad}"`, @@ -462,6 +465,65 @@ test("Bun.JSONC.parse rejects a bad escape or a control character after text of expect(wrong).toEqual([]); }); +test("Bun.JSONC.parse takes the same escapes whether or not a string of the document has a control character", () => { + // The parser has two decoders with an escape table each. A string with no control character in the document so far + // goes to the first. A later string with a line break in it sends every string to the second. Then the first of + // the two errors tells if the second decoder took the escape. + const SYNTAX = "JSONC Parse error: Syntax Error"; + const LINE_BREAK = "JSONC Parse error: Unterminated string literal"; + const taken: Record = { + b: "\b", + f: "\f", + n: "\n", + r: "\r", + t: "\t", + v: "\v", + "8": "8", + "9": "9", + '"': '"', + "\\": "\\", + "/": "/", + }; + // The value as JSON text, or the message of the error. + const outcome = (doc: string) => { + try { + return JSON.stringify(Bun.JSONC.parse(doc)); + } catch (error) { + return (error as Error).message; + } + }; + const rows: [escape: string, value: string | undefined][] = [ + ["\\x41", "A"], + ["\\xe9", "é"], + ["\\x4", undefined], + ["\\xZZ", undefined], + ["\\x", undefined], + ["\\u0041", "A"], + ["\\u65e5", "日"], + ["\\u12", undefined], + ["\\uZZZZ", undefined], + ["\\u", undefined], + ]; + // Every other character after a backslash, but a control character: that is an error of its own. + for (let code = 0x20; code <= 0xff; code++) { + const letter = String.fromCharCode(code); + if (letter !== "x" && letter !== "u" && !(code >= 0x7f && code < 0xa0)) rows.push(["\\" + letter, taken[letter]]); + } + const wrong: string[] = []; + for (const [escape, value] of rows) { + const alone = outcome(`"${escape}"`); + if (alone !== (value === undefined ? SYNTAX : JSON.stringify(value))) { + wrong.push(`${JSON.stringify(escape)} alone: ${alone}`); + } + const beforeLineBreak = outcome(`["${escape}","a\nb"]`); + if (beforeLineBreak !== (value === undefined ? SYNTAX : LINE_BREAK)) { + wrong.push(`${JSON.stringify(escape)} before a line break: ${beforeLineBreak}`); + } + } + expect(rows.filter(([, value]) => value !== undefined).length).toBe(15); + expect(wrong).toEqual([]); +}); + test("Bun.JSONC.parse matches JSON.parse on lone and paired surrogate escapes", () => { for (const doc of [ '"\\ud800"',