diff --git a/bench/json-corpus/escaped-strings.mjs b/bench/json-corpus/escaped-strings.mjs new file mode 100644 index 000000000000..812d20d10247 --- /dev/null +++ b/bench/json-corpus/escaped-strings.mjs @@ -0,0 +1,107 @@ +// Writes documents whose strings have escapes into this directory, one per kind of text: +// `bun bench/json-corpus/escaped-strings.mjs`. Two benches read every document of the directory: +// `bun bench/json-corpus/jsonc-parse.mjs strings-` and `scripts/bench-json-rust.sh strings-`. +// A string with an escape is decoded into a copy, so the cost of these documents is the cost of that decoder. +import { writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; + +const dir = dirname(Bun.fileURLToPath(import.meta.url)); +const MB = 1024 * 1024; +const repeat = (unit, bytes) => Buffer.alloc(bytes - (bytes % Buffer.byteLength(unit)), unit).toString(); + +// A string with no escape next to one that has an escape: both are in one 64-byte block of the index. +const neighbours = (count, clean) => Array.from({ length: count }, (_, i) => (i % 2 ? "a\nb" : clean(i))); +const ascii = length => i => ("t" + i).padEnd(length, "x").slice(0, length); + +// The signature of a package version in a registry manifest: lines of 60 base64 characters that end with \r\n. +const base64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +function signature(seed) { + let text = "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v4.10.10\r\nComment: https://openpgpjs.org\r\n\r\n"; + for (let line = 0; line < 8; line++) { + for (let i = 0; i < 60; i++) text += base64[(seed = (seed * 1103515245 + 12345) >>> 0) >>> 26]; + text += "\r\n"; + } + return text + "=AbCd\r\n-----END PGP SIGNATURE-----\r\n"; +} + +const documents = { + // Source text as a source map holds it: a line break every 64 bytes. + "source-lines": { sourcesContent: [repeat("// " + Buffer.alloc(60, "p").toString() + "\n", MB)] }, + // One megabyte between two escapes. + "long-run": ["\n" + repeat("p", MB) + "\n"], + // Source text with tabs, CRLF line ends and quoted strings: two to four escapes next to each other. + "tabs-crlf": { + sourcesContent: [repeat('function f(a, b) {\r\n\tif (a === "x") {\r\n\t\treturn b + "y";\r\n\t}\r\n}\r\n', MB)], + }, + // JSON in a string: a quote every one to ten bytes. + "json-in-json": { + payload: JSON.stringify( + Array.from({ length: 12000 }, (_, i) => ({ id: i, name: "item" + i, ok: i % 2 === 0, tags: ["a", "b"] })), + ), + }, + "pgp-signatures": { + versions: Object.fromEntries( + Array.from({ length: 1200 }, (_, i) => [`1.0.${i}`, { dist: { "npm-signature": signature(i + 1) } }]), + ), + }, + // One character between two escapes, of one to four bytes. + "one-byte-runs": [repeat("a\n", MB / 2)], + "one-accent-runs": [repeat("é\n", MB / 2)], + "one-cjk-runs": [repeat("日\n", MB / 2)], + "one-emoji-runs": [repeat("😀\n", MB / 2)], + // Short strings, each with two or four escapes. + "short-strings": Object.fromEntries( + Array.from({ length: 16000 }, (_, i) => ["script:" + i, `tsc -p "./tsconfig.${i}.json" && echo "done ${i}"`]), + ), + // Strings of about 20 bytes with one escape each, in ASCII and not. + "short-one-escape": Array.from({ length: 40000 }, (_, i) => `line ${i} of the text\n`), + "short-accents": Array.from({ length: 40000 }, (_, i) => `café numéro ${i}\n`), + // Keys with an escape. A key is decoded like a value. + "escaped-keys": Object.fromEntries(Array.from({ length: 16000 }, (_, i) => [`key\t${i}`, i])), + "neighbours-8": neighbours(40000, ascii(8)), + "neighbours-16": neighbours(40000, ascii(16)), + "neighbours-40": neighbours(24000, ascii(40)), + "neighbours-72": neighbours(16000, ascii(72)), + // The same with text of 150 bytes that changes between ASCII and other characters. + "neighbours-mixed": neighbours( + 8000, + i => `${i} Tiếng Việt là ngôn ngữ chính thức của Việt Nam, hơn 90 triệu người dùng hằng ngày.`, + ), + // Nothing but escapes. + "escape-only": [repeat('\n\t"\\', MB / 2)], + // Text that is not ASCII: CJK lines, and accented Latin words between ASCII ones. + cjk: [repeat("日本語のテキスト中文文档安装运行测试代码注释\n", MB)], + accents: [repeat("// naïve café über " + Buffer.alloc(40, "p").toString() + "\n", MB)], + // No escape at all. A string of this document is not copied. + none: Object.fromEntries( + Array.from({ length: 16000 }, (_, i) => ["key" + i, "value number " + i + " in plain text"]), + ), +}; + +for (const [name, value] of Object.entries(documents)) { + writeFileSync(join(dir, `strings-${name}.json`), JSON.stringify(value)); +} + +// Every character as a \u escape, which JSON.stringify does not write. +const escaped = Array.from(repeat('日本語のテキスト\n\t"é"', (3 * MB) / 8), unit => + Array.from({ length: unit.length }, (_, i) => "\\u" + unit.charCodeAt(i).toString(16).padStart(4, "0")).join(""), +).join(""); +writeFileSync(join(dir, "strings-all-escaped.json"), `["${escaped}"]`); + +// Lines of bytes that are not UTF-8, each with a \n escape. Only a bench that reads bytes sees them as they are: +// `scripts/bench-json-rust.sh strings-invalid`, or an import of the file with `with { type: "jsonc" }`. +const invalid = { + // One byte that starts no sequence in a line of ASCII. + sparse: Buffer.concat([Buffer.from("// " + Buffer.alloc(57, "p").toString()), Buffer.from([0xff])]), + // The same in a line of CJK text. + cjk: Buffer.concat([Buffer.from("日本語のテキスト中文文档安装运行测试代码"), Buffer.from([0xff])]), + // Russian words in Windows-1251, where each letter is one byte that is not ASCII. + cp1251: Buffer.from( + Array.from("Привет мир, это текст в кодировке Windows-1251 ", c => c.charCodeAt(0) - (c >= "А" ? 0x350 : 0)), + ), +}; +for (const [name, unit] of Object.entries(invalid)) { + const line = Buffer.concat([unit, Buffer.from("\\n")]); + const body = Buffer.alloc(MB - (MB % line.length), line); + writeFileSync(join(dir, `strings-invalid-${name}.json`), Buffer.concat([Buffer.from('["'), body, Buffer.from('"]')])); +} diff --git a/src/parsers/json_stage2.rs b/src/parsers/json_stage2.rs index d33efe2ba627..e89e676f7525 100644 --- a/src/parsers/json_stage2.rs +++ b/src/parsers/json_stage2.rs @@ -7,6 +7,7 @@ use bun_ast::{E, Expr, Loc, Log, Range, Source, usize2loc}; use bun_core::StackCheck; use bun_core::strings; use bun_core::strings::CodePoint; +use bun_simdutf_sys::simdutf; use crate::json::JSONOptions; use crate::json_index::{self as jidx, StructuralIndex}; @@ -513,6 +514,36 @@ impl<'a, 's, 'i> Parser<'a, 's, 'i> { #[cold] fn parse_string_slow(&mut self, body: &'s [u8]) -> PResult { + if self.idx.flags & jidx::FLAG_HAS_CTRL_IN_STRING != 0 { + return self.parse_string_by_code_point(body); + } + // No string so far has a control character, so a string with no backslash is its text. + let first = text_len(body); + if first == body.len() { + return Ok(E::EString::init(body)); + } + self.decode_string(body, first) + } + + /// `parse_string_slow` for a string with a backslash: `first` is the place of the first one. + #[inline(never)] + fn decode_string(&mut self, body: &'s [u8], first: usize) -> PResult { + let mut buf = core::mem::take(&mut self.scratch_str); + buf.clear(); + let plain = decode_plain_text(body, first, &mut buf); + if plain < body.len() { + // The rest starts with a byte that is not UTF-8, or with a bad escape. + self.decode_escapes(&body[plain..], &mut buf)?; + } + let owned = self.alloc_owned_str(&buf); + self.scratch_str = buf; + Ok(E::EString::init(owned.slice())) + } + + /// What `parse_string_slow` does if a string of the document has a control character. + #[cold] + #[inline(never)] + fn parse_string_by_code_point(&mut self, body: &'s [u8]) -> PResult { let mut first_special = None; for (k, &b) in body.iter().enumerate() { if b == b'\\' || b < 0x20 { @@ -1290,6 +1321,158 @@ fn push_codepoint(buf: &mut Vec, cp: CodePoint) { buf.extend_from_slice(&tmp[..n]); } +/// The length of `text` up to a backslash. It reads short text itself: `index_of_char` was slower. +#[inline(always)] +fn text_len(text: &[u8]) -> usize { + const ONES: u64 = 0x0101_0101_0101_0101; + // Below this many bytes the kernel reads no whole vector on some processors. + const KERNEL_MIN: usize = 64; + // A word read first makes one byte between two escapes slower than the loop this replaces. + let mut len = 0; + while len < 4 { + match text.get(len) { + Some(&byte) if byte != b'\\' => len += 1, + _ => return len, + } + } + while let Some(head) = text[len..].first_chunk::<8>() { + if len > 4 && text.len() - len >= KERNEL_MIN { + let rest = &text[len..]; + return len + strings::index_of_char_usize(rest, b'\\').unwrap_or(rest.len()); + } + let word = u64::from_le_bytes(*head) ^ (ONES * 0x5C); + // Sets the high bit of each backslash. Above the first one a borrow can set more. + let backslash = word.wrapping_sub(ONES) & !word & (ONES * 0x80); + if backslash != 0 { + return len + (backslash.trailing_zeros() / 8) as usize; + } + len += 8; + } + while text.get(len).is_some_and(|&byte| byte != b'\\') { + len += 1; + } + len +} + +/// The length of the UTF-8 text at the start of `body`: it ends before a byte of no character. +#[inline] +fn utf8_len(body: &[u8]) -> usize { + // The call to simdutf costs more than the check of a short body. + if body.len() <= 64 { + if body.is_ascii() { + return body.len(); + } + return match core::str::from_utf8(body) { + Ok(_) => body.len(), + Err(error) => error.valid_up_to(), + }; + } + let result = simdutf::validate::with_errors::utf8(body); + if result.is_successful() { + body.len() + } else { + result.count + } +} + +/// The value of the hex digits `digits`, if every byte is one. +#[inline] +fn hex_value(digits: Option<&[u8]>) -> Option { + let mut value: u32 = 0; + for &digit in digits? { + value = value * 16 + u32::from(bun_core::fmt::hex_digit_value(digit)?); + } + Some(value) +} + +/// Decodes the row of escapes at the backslash `body[pos]`. `Ok`: its end. `Err`: a bad escape. +#[inline(always)] +fn decode_escapes_at(body: &[u8], mut pos: usize, out: &mut Vec) -> Result { + loop { + debug_assert_eq!(body.get(pos), Some(&b'\\')); + let start = pos; + // `decode_string_escapes` drops a backslash that ends the body. + let Some(&escaped) = body.get(pos + 1) else { + return Ok(body.len()); + }; + pos += 2; + match escaped { + b'b' => out.push(0x08), + b'f' => out.push(0x0c), + b'n' => out.push(0x0a), + b'r' => out.push(0x0d), + b't' => out.push(0x09), + b'v' => out.push(0x0b), + b'8' | b'9' | b'"' | b'\\' | b'/' => out.push(escaped), + b'x' => { + let value = hex_value(body.get(pos..pos + 2)).ok_or(start)?; + pos += 2; + push_codepoint(out, value as CodePoint); + } + b'u' => { + let mut value = hex_value(body.get(pos..pos + 4)).ok_or(start)?; + pos += 4; + if strings::u16_is_lead(value as u16) + && body.get(pos..pos + 2) == Some(b"\\u".as_slice()) + && let Some(trail) = hex_value(body.get(pos + 2..pos + 6)) + && strings::u16_is_trail(trail as u16) + { + value = strings::u16_get_supplementary(value as u16, trail as u16); + pos += 6; + } + push_codepoint(out, value as CodePoint); + } + _ => return Err(start), + } + if body.get(pos) != Some(&b'\\') { + return Ok(pos); + } + } +} + +/// Decodes `body` up to a byte that is not UTF-8 or a bad escape, and returns where it stops. +#[inline(always)] +fn decode_plain_text(body: &[u8], first: usize, out: &mut Vec) -> usize { + out.reserve(body.len()); + let mut copied = 0; + if body.first() == Some(&b'\\') { + // Escapes are ASCII, and a string such as "\n" has no text to check. + match decode_escapes_at(body, 0, out) { + Ok(after) => copied = after, + Err(bad) => return bad, + } + if copied == body.len() { + return copied; + } + } + let end = utf8_len(body); + let mut pos = if copied == 0 { + first.min(end) + } else { + copied + text_len(&body[copied..end]) + }; + loop { + // `body[copied..pos]` is text, and `body[pos]` is a backslash if `pos` is not `end`. + match &body[copied..pos] { + [] => {} + [byte] => out.push(*byte), + text => out.extend_from_slice(text), + } + if pos == end { + return end; + } + match decode_escapes_at(body, pos, out) { + Ok(after) => copied = after, + Err(bad) => return bad, + } + // Many strings end with an escape. + if copied == end { + return end; + } + pos = copied + text_len(&body[copied..end]); + } +} + fn read_trail_surrogate_escape( iterator: &strings::CodepointIterator<'_>, iter: &mut strings::Cursor, @@ -1431,3 +1614,373 @@ pub(crate) fn decode_auto_quoted( decode_string_escapes::(&mut l, body, &mut buf)?; Ok(E::String::init(bump.alloc_slice_copy(&buf))) } + +#[cfg(test)] +mod tests { + use super::*; + + /// The decoded bytes, or the error and what it logged. + type Outcome = Result, String>; + + fn logged(err: crate::Error, log: &Log) -> String { + let messages: Vec = log + .msgs + .iter() + .map(|m| String::from_utf8_lossy(&m.data.text).into_owned()) + .collect(); + format!("{err:?}: {messages:?}") + } + + /// `decode_string_escapes` of `body`, after the text that `buf` has. + fn decode_by_code_point(allow_raw_control: bool, body: &[u8], mut buf: Vec) -> Outcome { + let source = Source::init_path_string("string.json", body); + let mut log = Log::init(); + let mut l = MiniLog { + log: &mut log, + source: &source, + prev_error_loc: Loc::EMPTY, + }; + let result = if allow_raw_control { + decode_string_escapes::(&mut l, body, &mut buf) + } else { + decode_string_escapes::(&mut l, body, &mut buf) + }; + result.map(|()| buf).map_err(|err| logged(err, &log)) + } + + /// What the parser makes of the one string of the document `["body"]`. + fn parsed(body: &[u8]) -> Outcome { + let doc = [b"[\"".as_slice(), body, b"\"]"].concat(); + bun_ast::initialize_store_or_reset(); + let _scope = bun_ast::StoreResetGuard::new(); + let source = Source::init_path_string("doc.json", &doc); + let mut log = Log::init(); + let parsed = crate::json::ParsedJson::parse_json(&source, &mut log) + .map_err(|err| logged(err, &log))?; + let Data::EArrayJSON(root) = parsed.root.data else { + panic!("the document is an array"); + }; + match root.get().items() { + [E::JsonValue::String(string)] => Ok(string.slice().to_vec()), + items => panic!("{} items in {:?}", items.len(), bstr::BStr::new(&doc)), + } + } + + /// What main makes of a string: it decodes one that has a backslash or a control character. + fn expected(body: &[u8]) -> Outcome { + if body.iter().any(|b| *b == b'\\' || *b < 0x20) { + decode_by_code_point(false, body, Vec::new()) + } else { + Ok(body.to_vec()) + } + } + + /// Whether `"body"` is one string token: each quote of the body is escaped. + fn is_one_string(body: &[u8]) -> bool { + let mut i = 0; + while i < body.len() { + match body[i] { + b'"' => return false, + b'\\' if i + 1 == body.len() => return false, + b'\\' => i += 2, + _ => i += 1, + } + } + true + } + + /// Every byte that takes its own path through a decoder, and its neighbours. + const BYTES: &[u8] = &[ + // Printable ASCII, the escape letters, hex digits of each case, and `g`. + b' ', b'a', b'Z', b'~', 0x7F, b'\\', b'"', b'\'', b'/', b'b', b'f', b'n', b'r', b't', b'v', + b'u', b'x', b'0', b'7', b'8', b'9', b'A', b'D', b'F', b'c', b'd', b'g', + // Control characters: the two that end a line, and three that do not. + 0x00, 0x09, 0x0A, 0x0D, 0x1F, + // Continuation bytes, then lead bytes: overlong (C0, C1, E0, F0), surrogate (ED), invalid. + 0x80, 0x8F, 0x90, 0x9F, 0xA0, 0xBF, 0xC0, 0xC1, 0xC2, 0xDF, 0xE0, 0xE1, 0xED, 0xEF, 0xF0, + 0xF1, 0xF4, 0xF5, 0xF7, 0xF8, 0xFF, + ]; + + /// Pieces that a body is built from, so that long and well-formed sequences occur often. + const PIECES: &[&[u8]] = &[ + b"plain text", + b"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + b"\\n", + b"\\t", + b"\\\"", + b"\\\\", + b"\\/", + b"\\b\\f\\r\\v", + b"\\8\\9", + b"\\0", + b"\\'", + b"\\x41", + b"\\xff", + b"\\x4", + b"\\u0041", + b"\\u00e9", + b"\\u65e5", + b"\\uD83D\\uDE00", + b"\\uD83D", + b"\\uDE00", + b"\\uD83D\\u0041", + b"\\uD83D\\uD83D\\uDE00", + b"\\uD83D\\uDE0", + b"\\uD83D\\n", + b"\\u12", + b"\\u12G4", + b"\\", + "é".as_bytes(), + "日本語".as_bytes(), + "😀".as_bytes(), + "\u{7f}\u{80}\u{7ff}\u{800}\u{ffff}\u{10000}\u{10ffff}".as_bytes(), + // A lead and a trail surrogate as WTF-8, alone and as a pair. + &[0xED, 0xA0, 0xBD], + &[0xED, 0xB8, 0x80], + &[0xED, 0xA0, 0xBD, 0xED, 0xB8, 0x80], + // Sequences cut short, overlong, or past U+10FFFF. + &[0xC3], + &[0xE6, 0x97], + &[0xF0, 0x9F, 0x98], + &[0xC0, 0x80], + &[0xE0, 0x80, 0x80], + &[0xF0, 0x80, 0x80, 0x80], + &[0xF4, 0x90, 0x80, 0x80], + &[0x80], + &[0xFF], + &[0x00], + &[0x0A], + &[0x1F], + ]; + + struct Rng(u64); + + impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } + + /// A body of up to ten parts: a byte of `BYTES`, a run of ASCII, or one of `PIECES`. + fn body(&mut self, run: usize, body: &mut Vec) { + body.clear(); + for _ in 0..self.below(10) { + match self.below(8) { + 0 => body.push(BYTES[self.below(BYTES.len())]), + 1 => { + let len = self.below(run); + body.extend(core::iter::repeat_n(b'a' + (len % 26) as u8, len)); + } + _ => body.extend_from_slice(PIECES[self.below(PIECES.len())]), + } + } + } + } + + /// Calls `check` with every body of one or two bytes, many of three, and random ones. + fn for_each_body(mut check: impl FnMut(&[u8])) { + check(b""); + for a in 0..=255u8 { + check(&[a]); + for b in 0..=255u8 { + check(&[a, b]); + } + } + for &a in BYTES { + for &b in BYTES { + for &c in BYTES { + check(&[a, b, c]); + } + } + } + let mut rng = Rng(0x9E3779B97F4A7C15); + let mut body = Vec::new(); + for _ in 0..200_000 { + body.clear(); + for _ in 0..rng.below(14) { + body.push(BYTES[rng.below(BYTES.len())]); + } + check(&body); + } + for _ in 0..300_000 { + // The runs cross the widths that the search for a backslash reads in one step. + rng.body(150, &mut body); + check(&body); + } + } + + #[test] + fn text_len_matches_a_byte_loop() { + let check = |text: &[u8]| { + let expected = text.iter().take_while(|b| **b != b'\\').count(); + assert_eq!(text_len(text), expected, "{:?}", bstr::BStr::new(text)); + }; + // Every byte value at each place that is read by a branch or in a word, alone or in a pair. + for filler in [b'a', 0xE6] { + let mut text = [filler; 30]; + for first in 0..28 { + for value in 0..=255u8 { + text.fill(filler); + text[first] = value; + check(&text); + for second in first + 1..28 { + for other in [0x00, 0x1F, 0x5B, 0x5C, 0x5D, 0x7F, 0x80, 0xDC, 0xFF] { + text[second] = other; + check(&text); + } + text[second] = filler; + } + } + } + } + // A backslash at each place of a text of each length up to 200 bytes, and no backslash. + let text = [b'a'; 200]; + for len in 0..=text.len() { + check(&text[..len]); + for at in 0..len { + let mut text = text; + text[at] = b'\\'; + check(&text[..len]); + } + } + } + + #[test] + fn utf8_len_is_the_same_for_a_short_and_a_long_body() { + // A short body is checked without simdutf. 70 bytes of ASCII after it make it long. + let mut long = Vec::new(); + for_each_body(|body| { + long.clear(); + long.extend_from_slice(body); + long.resize(body.len() + 70, b'a'); + let len = utf8_len(body); + let expected = if len == body.len() { long.len() } else { len }; + assert_eq!( + utf8_len(&long), + expected, + "body {:?}", + bstr::BStr::new(body) + ); + }); + } + + #[test] + fn bulk_decoder_matches_the_code_point_decoder() { + let (mut whole, mut part, mut rejected) = (0, 0, 0); + for_each_body(|body| { + let mut in_bulk = Vec::new(); + let plain = decode_plain_text(body, text_len(body), &mut in_bulk); + // The bulk decoder takes a control character as text, so the other decoder does too. + let actual = decode_by_code_point(true, &body[plain..], in_bulk); + let expected = decode_by_code_point(true, body, Vec::new()); + assert_eq!( + actual, + expected, + "body {:?}, in bulk up to {plain}", + bstr::BStr::new(body) + ); + match actual { + Ok(_) if plain == body.len() => whole += 1, + Ok(_) => part += 1, + Err(_) => rejected += 1, + } + }); + assert!( + whole > 50_000 && part > 50_000 && rejected > 50_000, + "{whole} decoded in bulk, {part} in part, {rejected} rejected" + ); + } + + #[test] + fn parser_decodes_like_the_code_point_decoder() { + let (mut decoded, mut rejected) = (0, 0); + for_each_body(|body| { + if !is_one_string(body) { + return; + } + let (expected, actual) = (expected(body), parsed(body)); + assert!( + expected == actual, + "body {:?}\n by code point: {expected:?}\n parser: {actual:?}", + bstr::BStr::new(body) + ); + match expected { + Ok(_) => decoded += 1, + Err(_) => rejected += 1, + } + }); + assert!( + decoded > 50_000 && rejected > 50_000, + "{decoded} decoded, {rejected} rejected" + ); + } + + #[test] + fn parser_decodes_long_strings_like_the_code_point_decoder() { + let mut rng = Rng(0xD1B54A32D192ED03); + let (mut part, mut body) = (Vec::new(), Vec::new()); + let (mut decoded, mut rejected) = (0, 0); + // Stage 1 reads 8192 bytes at a time, and the tape holds strings in buffers of 4096 bytes. + for target in [4095, 4096, 4097, 8191, 8192, 8193, 20_000] { + for _ in 0..200 { + // One body in eight has parts that are not a string, at any place in it. + let only_what_decodes = rng.below(8) != 0; + body.clear(); + while body.len() < target { + rng.body(52, &mut part); + if is_one_string(&part) && (!only_what_decodes || expected(&part).is_ok()) { + body.extend_from_slice(&part); + } + } + let (expected, actual) = (expected(&body), parsed(&body)); + assert!( + expected == actual, + "body {:?}\n by code point: {:?}\n parser: {:?}", + bstr::BStr::new(&body), + expected.as_ref().map(Vec::len), + actual.as_ref().map(Vec::len), + ); + match expected { + Ok(_) => decoded += 1, + Err(_) => rejected += 1, + } + } + } + assert!( + decoded > 1_000 && rejected > 100, + "{decoded} decoded, {rejected} rejected" + ); + } + + #[test] + fn decoded_text_of_known_bodies() { + assert_eq!(parsed(b"a\\nb"), Ok(b"a\nb".to_vec())); + assert_eq!( + parsed(b"\\b\\f\\n\\r\\t\\v\\8\\9\\\"\\\\\\/"), + Ok(b"\x08\x0c\n\r\t\x0b89\"\\/".to_vec()) + ); + assert_eq!(parsed(b"\\x41\\xe9\\u65e5"), Ok("Aé日".as_bytes().to_vec())); + assert_eq!(parsed(b"\\uD83D\\uDE00"), Ok("😀".as_bytes().to_vec())); + assert_eq!(parsed(b"\\uD83D"), Ok(vec![0xED, 0xA0, 0xBD])); + // A byte that starts no sequence is its code point. A lead byte without its tail is U+FFFD. + assert_eq!( + parsed(b"\\n\x80\xE6\x97"), + Ok("\n\u{80}\u{FFFD}\u{97}".as_bytes().to_vec()) + ); + assert!( + parsed(b"a\nb") + .unwrap_err() + .contains("Unterminated string literal") + ); + assert!(parsed(b"a\x01b").unwrap_err().contains("Syntax Error")); + assert!(parsed(b"\\n\x01").unwrap_err().contains("Syntax Error")); + assert!(parsed(b"\\q").unwrap_err().contains("Syntax Error")); + assert!(parsed(b"\\u12G4").unwrap_err().contains("Syntax Error")); + } +} diff --git a/test/bundler/bundler_loader.test.ts b/test/bundler/bundler_loader.test.ts index 289cbb6eae2b..9b8d5c4313b3 100644 --- a/test/bundler/bundler_loader.test.ts +++ b/test/bundler/bundler_loader.test.ts @@ -170,6 +170,35 @@ describe("bundler", async () => { }, }); + { + // A string of a JSON file that has an escape is decoded. If its bytes are not UTF-8, a lead byte without its tail + // becomes U+FFFD, and a byte that starts no sequence becomes its code point. + const file: Record = {}; + const read: Record = {}; + // The parser finds the next escape by a branch, by a word of 8 bytes, and by a vector. The bytes are behind each. + for (const distance of [0, 1, 3, 4, 7, 11, 12, 19, 35, 67, 131, 300]) { + const text = Buffer.alloc(distance, "a").toString(); + file["d" + distance] = "\\n" + text + "\xE9z \x80\xA0 \xE2\x82A\\t"; + read["d" + distance] = "\n" + text + "\uFFFDz \x80\xA0 \uFFFD\x82A\t"; + } + // The text of this string is longer than its bytes in the file: U+FFFD has three bytes. + const repeated = (text: string) => Buffer.alloc(400 * Buffer.byteLength(text), text).toString(); + file.long = "\\n" + repeated("caf\xE9 au lait \xA9 "); + read.long = "\n" + repeated("caf\uFFFD au lait \xA9 "); + const entries = Object.entries(file).map(([key, value]) => `"${key}":"${value}"`); + itBundled("bun/loader-json-escaped-string-not-utf8", { + target: "bun", + files: { + "/entry.ts": /* js */ ` + import json from "./a.json"; + process.stdout.write(JSON.stringify(json)); + `, + "/a.json": Buffer.from(`{${entries.join(",")}}`, "latin1"), + }, + run: { stdout: JSON.stringify(read) }, + }); + } + itBundled("bun/loader-json-proto-key-is-own-property", { target: "bun", files: { diff --git a/test/js/bun/jsonc/json-differential-fuzz.test.ts b/test/js/bun/jsonc/json-differential-fuzz.test.ts index d62b34e9ac7f..62b1e0012b86 100644 --- a/test/js/bun/jsonc/json-differential-fuzz.test.ts +++ b/test/js/bun/jsonc/json-differential-fuzz.test.ts @@ -52,7 +52,7 @@ function genStringToken(rng: Rng): string { let body = ""; const chunks = rng.int(7); for (let i = 0; i < chunks; i++) { - switch (rng.int(8)) { + switch (rng.int(9)) { case 0: case 1: case 2: { @@ -75,6 +75,11 @@ function genStringToken(rng: Rng): string { case 7: body += rng.pick(["", "__proto__", "constructor", "0", "1e3", "true", "null", "//", "/*", "*/"]); break; + case 8: + // Text between two escapes that is longer than the 4, 8 and 16 to 64 bytes the parser reads in one step. + const text = rng.pick(["a", "~", "é", "日", "🚀"]); + body += Buffer.alloc(rng.range(7, 150) * Buffer.byteLength(text), text).toString(); + break; } } return '"' + body + '"'; diff --git a/test/js/bun/jsonc/jsonc.test.ts b/test/js/bun/jsonc/jsonc.test.ts index 38a9b48a9c10..3256578a38bf 100644 --- a/test/js/bun/jsonc/jsonc.test.ts +++ b/test/js/bun/jsonc/jsonc.test.ts @@ -349,6 +349,181 @@ test("Bun.JSONC.parse matches JSON.parse on escape-heavy strings", () => { } }); +// `count` copies of `text`. "x".repeat(count) is slow in a debug build. +const repeated = (text: string, count: number) => Buffer.alloc(count * Buffer.byteLength(text), text).toString(); + +test("Bun.JSONC.parse matches JSON.parse for text of any length around an escape", () => { + // The parser copies the text between two escapes in one step. To find the next escape it reads 4 bytes, then words + // of 8 bytes, then vectors once 64 bytes are left. It checks a string of 64 bytes or less for UTF-8 in another way. + // So the text has each length around those, and the escape is at each end and in the middle of it. + const escapes = [ + "\\n", + '\\"', + "\\\\", + "\\/", + "\\u00e9", + "\\u65E5", + "\\ud83d\\ude00", + "\\ud83d", + "\\ude00", + "\\ud83d\\u0041", + "\\b\\f\\r\\t", + ]; + const texts = [ + (n: number) => repeated("a", n), + (n: number) => repeated("é", n), + (n: number) => repeated("\x7f~ ", n).slice(0, n), + (n: number) => repeated("日本語", n).slice(0, n), + (n: number) => repeated("a🚀", n).slice(0, n * 3), + (n: number) => repeated("aé", n).slice(0, n), + ]; + const docsOf = (run: string, escape: string) => [ + `"${escape}${run}"`, + `"${run}${escape}"`, + `"${run}${escape}${run}"`, + `"${escape}${run}${escape}${escape}${run}${escape}"`, + `{"${run}${escape}":["${run}","${escape}${run}"]}`, + ]; + // One document for each length holds the strings of every text and escape: a call costs more than its bytes. + const wrong: string[] = []; + const check = (docs: string[]) => { + const doc = `[${docs.join(",")}]`; + const [actual, expected] = [Bun.JSONC.parse(doc) as unknown[], JSON.parse(doc) as unknown[]]; + // Compare as JSON text, which spells a lone surrogate as an escape. + if (JSON.stringify(actual) === JSON.stringify(expected)) return; + // Name each string that differs. + for (const [i, value] of expected.entries()) { + const [got, want] = [JSON.stringify(actual[i]), JSON.stringify(value)]; + if (got !== want) wrong.push(`${docs[i].slice(0, 80)} (${docs[i].length} chars): ${got?.slice(0, 80)}`); + } + if (actual.length !== expected.length) wrong.push(`${actual.length} values, not ${expected.length}`); + }; + for (const length of [0, 1, 3, 4, 5, 11, 12, 13, 20, 27, 28, 29, 59, 60, 61, 62, 63, 64, 65, 66, 75, 76, 77, 300]) { + check(texts.flatMap(text => escapes.flatMap(escape => docsOf(text(length), escape)))); + } + // 4096 bytes are one buffer of strings in the parser, and it reads the document 8192 bytes at a time. That + // depends on the number of bytes and not on the characters, so these lengths have two of the texts. + for (const length of [4095, 4096, 9000]) { + for (const text of texts.slice(0, 2)) for (const doc of docsOf(text(length), "\\n")) check([doc]); + } + expect(wrong).toEqual([]); + // JSON.parse does not take these escapes, and the parser of bun does. + expect(Bun.JSONC.parse('"\\x41\\xe9"')).toBe("Aé"); + expect(Bun.JSONC.parse('"\\8\\9\\v"')).toBe("89\v"); + expect(Bun.JSONC.parse(`"${repeated("é", 40)}\\x41\\8\\v${repeated("a", 40)}"`)).toBe( + `${repeated("é", 40)}A8\v${repeated("a", 40)}`, + ); +}); + +test("Bun.JSONC.parse rejects a bad escape or a control character after text of any length", () => { + const SYNTAX = "JSONC Parse error: Syntax Error"; + const LINE_BREAK = "JSONC Parse error: Unterminated string literal"; + const bads: [bad: string, message: string][] = [ + // One of each kind first: a string of thousands of bytes has only these five. + ["\\q", SYNTAX], + ["\n", LINE_BREAK], + ["\t", SYNTAX], + // The first of two that are bad makes the message. + ["\n\\q", LINE_BREAK], + ["\\q\n", SYNTAX], + ["\\u12", SYNTAX], + ["\\u12G4", SYNTAX], + ["\\x4", SYNTAX], + ["\\0", SYNTAX], + ["\\'", SYNTAX], + ["\r", LINE_BREAK], + ["\x00", SYNTAX], + ["\x1f", SYNTAX], + ]; + // Each document is also not JSON: it has a control character or a bad escape in a string. + const wrong: string[] = []; + for (const length of [0, 1, 4, 5, 12, 13, 28, 29, 60, 61, 62, 63, 64, 65, 76, 77, 4095, 4096, 9000]) { + // The text has no hex digit, so it does not complete an escape that is cut short. + for (const run of [repeated("z", length), repeated("é", length)]) { + for (const [bad, message] of length < 4095 ? bads : bads.slice(0, 5)) { + for (const doc of [ + `"${run}${bad}"`, + `"\\n${run}${bad}${run}"`, + `{"\\n${run}${bad}":1}`, + // Another string has the escape. The parser then looks for a control character in no string by itself: + // it takes the word of the index for it. A comment or a single quote makes the index read byte by byte. + `["${run}\\n","${run}${bad}"]`, + `['${run}\\n',"${run}${bad}"]`, + `/* c */["${run}\\n","${run}${bad}"]`, + ]) { + let got = "no error"; + try { + Bun.JSONC.parse(doc); + } catch (error) { + got = (error as Error).message; + } + if (got !== message) wrong.push(`${JSON.stringify(doc.slice(0, 60))} (${doc.length} chars): ${got}`); + } + } + } + } + expect(wrong).toEqual([]); +}); + +test("Bun.JSONC.parse takes the same escapes whether or not a string of the document has a control character", () => { + // The parser has two decoders with an escape table each. A string with no control character in the document so far + // goes to the first. A later string with a line break in it sends every string to the second. Then the first of + // the two errors tells if the second decoder took the escape. + const SYNTAX = "JSONC Parse error: Syntax Error"; + const LINE_BREAK = "JSONC Parse error: Unterminated string literal"; + const taken: Record = { + b: "\b", + f: "\f", + n: "\n", + r: "\r", + t: "\t", + v: "\v", + "8": "8", + "9": "9", + '"': '"', + "\\": "\\", + "/": "/", + }; + // The value as JSON text, or the message of the error. + const outcome = (doc: string) => { + try { + return JSON.stringify(Bun.JSONC.parse(doc)); + } catch (error) { + return (error as Error).message; + } + }; + const rows: [escape: string, value: string | undefined][] = [ + ["\\x41", "A"], + ["\\xe9", "é"], + ["\\x4", undefined], + ["\\xZZ", undefined], + ["\\x", undefined], + ["\\u0041", "A"], + ["\\u65e5", "日"], + ["\\u12", undefined], + ["\\uZZZZ", undefined], + ["\\u", undefined], + ]; + // Every other character after a backslash, but a control character: that is an error of its own. + for (let code = 0x20; code <= 0xff; code++) { + const letter = String.fromCharCode(code); + if (letter !== "x" && letter !== "u" && !(code >= 0x7f && code < 0xa0)) rows.push(["\\" + letter, taken[letter]]); + } + const wrong: string[] = []; + for (const [escape, value] of rows) { + const alone = outcome(`"${escape}"`); + if (alone !== (value === undefined ? SYNTAX : JSON.stringify(value))) { + wrong.push(`${JSON.stringify(escape)} alone: ${alone}`); + } + const beforeLineBreak = outcome(`["${escape}","a\nb"]`); + if (beforeLineBreak !== (value === undefined ? SYNTAX : LINE_BREAK)) { + wrong.push(`${JSON.stringify(escape)} before a line break: ${beforeLineBreak}`); + } + } + expect(rows.filter(([, value]) => value !== undefined).length).toBe(15); + expect(wrong).toEqual([]); +}); + test("Bun.JSONC.parse matches JSON.parse on lone and paired surrogate escapes", () => { for (const doc of [ '"\\ud800"', diff --git a/test/js/bun/resolve/jsonc.test.ts b/test/js/bun/resolve/jsonc.test.ts index fd73f3b7714a..88adb8b44306 100644 --- a/test/js/bun/resolve/jsonc.test.ts +++ b/test/js/bun/resolve/jsonc.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test"; -import { bunRun, tempDir } from "harness"; +import { bunEnv, bunExe, bunRun, tempDir } from "harness"; import { join } from "path"; test.concurrent("empty jsonc - package.json", async () => { await using dir = tempDir("jsonc", { @@ -57,3 +57,65 @@ test.concurrent("imported JSON strings match JSON.parse exactly (escapes, lone s }); expect(await bunRun(join(dir, "index.ts"))).toSpawn(); }); + +test.concurrent("a .jsonc import repairs bytes that are not UTF-8 in a string with an escape", async () => { + // A string with an escape is decoded. One that is not UTF-8 is decoded one code point at a time, and repaired. + const escaped = (bytes: number[]) => Buffer.concat([Buffer.from("a\\n"), Buffer.from(bytes), Buffer.from("z\\t")]); + const decoded = (codePoints: number[]) => [0x61, 0x0a, ...codePoints, 0x7a, 0x09]; + const cases: Record = { + // A byte that starts no sequence is taken as its code point. + continuation: [escaped([0x80]), decoded([0x80])], + lastContinuation: [escaped([0xbf]), decoded([0xbf])], + notALead: [escaped([0xff]), decoded([0xff])], + // A sequence that is cut short, or overlong, is U+FFFD for its first byte. + leadThenAscii: [escaped([0xc3]), decoded([0xfffd])], + cutSequence: [escaped([0xe6, 0x97]), decoded([0xfffd, 0x97])], + overlong: [escaped([0xc0, 0x80]), decoded([0xfffd, 0x80])], + // Half of a surrogate pair stays as it is. + surrogateHalf: [escaped([0xed, 0xa0, 0xbd]), decoded([0xd83d])], + // One byte that is not UTF-8 changes nothing in the characters around it. + twoStretches: [escaped([0xc3, 0xa9, 0x20, 0xff, 0xc3, 0xa9]), decoded([0xe9, 0x20, 0xff, 0xe9])], + wellFormed: [escaped([0xc3, 0xa9, 0xf0, 0x9f, 0x9a, 0x80]), decoded([0xe9, 0x1f680])], + // A string that is longer than a buffer of the parser, with the byte between two runs of text. + long: [ + Buffer.concat([ + Buffer.alloc(5000, "p"), + Buffer.from([0xff, 0xc3, 0xa9]), + Buffer.alloc(5000, "q"), + Buffer.from("\\n"), + ]), + [...Array(5000).fill(0x70), 0xff, 0xe9, ...Array(5000).fill(0x71), 0x0a], + ], + // Text that is longer after the repair: each byte becomes a character of two bytes. + grows: [Buffer.concat([Buffer.alloc(5000, 0xff), Buffer.from("\\n")]), [...Array(5000).fill(0xff), 0x0a]], + }; + // The parser finds the next escape by a branch, by a word of 8 bytes, and by a vector. The byte is behind each. + for (const distance of [1, 3, 4, 7, 11, 12, 19, 35, 67, 131]) { + const text = Array(distance).fill(0x62); + cases["after" + distance] = [escaped([...text, 0xe9, ...text]), decoded([...text, 0xfffd, ...text])]; + } + const parts = [Buffer.from("{")]; + for (const [name, [body]] of Object.entries(cases)) { + parts.push(Buffer.from(`"${name}":"`), body, Buffer.from(`",`)); + } + parts.push(Buffer.from(`}`)); + using dir = tempDir("jsonc-not-utf8", { + "weird.jsonc": Buffer.concat(parts), + "index.ts": ` + import file from "./weird.jsonc"; + const codePoints = (text: string) => Array.from(text, c => c.codePointAt(0)); + console.log(JSON.stringify(Object.entries(file).map(([name, text]) => [name, codePoints(text as string)]))); + `, + }); + await using proc = Bun.spawn({ + cmd: [bunExe(), "index.ts"], + env: bunEnv, + cwd: String(dir), + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toBe(""); + expect(JSON.parse(stdout)).toEqual(Object.entries(cases).map(([name, [, codePoints]]) => [name, codePoints])); + expect(exitCode).toBe(0); +});